From fd9a4fa65e0a5189957032c0046eb286311fda62 Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Thu, 17 Sep 2026 23:33:43 -0400 Subject: [PATCH 01/48] ci: Fix Release Version Resolution for Untagged and Resumed Runs (#233) * ci: Fix Release Version Resolution for Untagged and Resumed Runs The release workflow resolved its version in one inline shell block under `set -euo pipefail`, where two paths could not succeed. Filtering tags through `grep` made a no-match fatal. On the ordinary untagged tip of `main`, `git tag --points-at HEAD | grep -E '^v[0-9]+...'` exits 1, and the step died before reaching its skip handling or `next-release-version.sh`, so a deployable commit could not obtain a release version (#228). Selecting stable tags now reads exit 1 as an empty answer while exit 2 and above still fail the release, which also lets the missing-previous-tag case report its own error. The rerun-resume path then rejected the tag it had itself chosen. With a stable tag already pointing at `HEAD` and no release published, the version comes from that tag, and the following existence check failed merely because the ref existed (#229). It now compares the tag's commit against the release commit, so only a tag on some other commit is a collision; `Create tag` already tolerates a tag that exists. The block moved into `.github/scripts/resolve-release-version.sh`, beside the `next-release-version.sh` it calls, so `tests/release-version-resolution.sh` can cover every path: automatic, resumed, skipped, dispatched, pushed-tag, and the runs that must be refused, each against a throwaway repository with a stubbed `gh`. * fix: harden release resolver execution --------- Co-authored-by: Lia Co-authored-by: Danny Avila --- .github/scripts/resolve-release-version.sh | 195 +++++++++++++++ .github/workflows/ci.yml | 3 + .github/workflows/release.yml | 153 ++---------- tests/release-version-resolution.sh | 265 +++++++++++++++++++++ 4 files changed, 483 insertions(+), 133 deletions(-) create mode 100755 .github/scripts/resolve-release-version.sh create mode 100755 tests/release-version-resolution.sh diff --git a/.github/scripts/resolve-release-version.sh b/.github/scripts/resolve-release-version.sh new file mode 100755 index 00000000..ef5f8667 --- /dev/null +++ b/.github/scripts/resolve-release-version.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash + +# Resolves which version a release run publishes, or decides that it publishes +# nothing, and records the decision in $GITHUB_OUTPUT. Extracted from +# .github/workflows/release.yml so the four entry paths — an automatic release +# after CI, a rerun resuming a release whose tag was already cut, a dispatch, +# and a pushed tag — are covered by tests/release-version-resolution.sh. +# +# Inputs arrive as environment variables, mirroring the workflow's env block: +# +# EVENT_NAME github.event_name +# HEAD_SHA github.event.workflow_run.head_sha +# INPUT_VERSION github.event.inputs.version +# INPUT_DRAFT github.event.inputs.draft +# REF_NAME github.ref_name +# REF_TYPE github.ref_type +# GH_TOKEN a token `gh release view` can read releases with +# +# Outputs: skip, and for a real release version, app_version, chart_version, +# prerelease, latest, draft. Run from the repository root; the chart fields are +# read from helm/codeapi/Chart.yaml relative to it. + +set -euo pipefail + +EVENT_NAME="${EVENT_NAME:-}" +HEAD_SHA="${HEAD_SHA:-}" +INPUT_VERSION="${INPUT_VERSION:-}" +INPUT_DRAFT="${INPUT_DRAFT:-}" +REF_NAME="${REF_NAME:-}" +REF_TYPE="${REF_TYPE:-}" +GITHUB_OUTPUT="${GITHUB_OUTPUT:?GITHUB_OUTPUT must name the step output file}" + +STABLE_TAG_PATTERN='^v[0-9]+[.][0-9]+[.][0-9]+$' + +# `grep` exits 1 when nothing matches, and under `pipefail` that would abort the +# step. A commit with no stable tag is the ordinary state of `main`, so a +# no-match reads as an empty answer while a genuine grep failure — exit 2 and +# above — still fails the release. +select_stable_tags() { + local status=0 + grep -E "$STABLE_TAG_PATTERN" || status=$? + [ "$status" -le 1 ] +} + +# Highest stable tag among those `git tag` selects, empty when there are none. +newest_stable_tag() { + git tag "$@" | select_stable_tags | sort -V | tail -n 1 +} + +# The commit a tag resolves to. The caller first verifies that the ref exists, +# so a failure here means the tag ultimately names a non-commit object. +tag_commit() { + git rev-parse -q --verify "refs/tags/$1^{commit}" +} + +SKIP=false +VERSION="" +HEAD_COMMIT="" + +if [ "$EVENT_NAME" = "workflow_run" ]; then + HEAD_COMMIT="$(git rev-parse HEAD)" + if [ "$HEAD_COMMIT" != "$HEAD_SHA" ]; then + echo "::error::Checked out SHA does not match the successful CI run" + exit 1 + fi + + REMOTE_MAIN_SHA="$(git ls-remote origin refs/heads/main | awk '{print $1}')" + if [ -z "$REMOTE_MAIN_SHA" ]; then + echo "::error::Could not resolve the current main branch tip" + exit 1 + fi + if [ "$REMOTE_MAIN_SHA" != "$HEAD_SHA" ]; then + echo "main advanced after this CI run; the newer successful run will release the combined changes" + SKIP=true + fi + + # A rerun after tag creation but before release publication resumes the + # missing release rather than incrementing the version again. + EXACT_TAG="$(newest_stable_tag --points-at HEAD)" + if [ "$SKIP" = "false" ] && [ -n "$EXACT_TAG" ]; then + if gh release view "$EXACT_TAG" >/dev/null 2>&1; then + echo "$EXACT_TAG already publishes this commit; nothing to do" + SKIP=true + else + VERSION="$EXACT_TAG" + fi + elif [ "$SKIP" = "false" ]; then + PREVIOUS_TAG="$(newest_stable_tag --merged HEAD)" + if [ -z "$PREVIOUS_TAG" ]; then + echo "::error::Automatic releases require an existing stable vMAJOR.MINOR.PATCH tag" + exit 1 + fi + VERSION="$(.github/scripts/next-release-version.sh "$PREVIOUS_TAG" "$PREVIOUS_TAG..HEAD")" + if [ -z "$VERSION" ]; then + echo "Only documentation, workflow, or test files changed since $PREVIOUS_TAG; no release needed" + SKIP=true + fi + fi +elif [ "$EVENT_NAME" = "workflow_dispatch" ]; then + # Releases describe what shipped to main. Dispatching from a topic branch + # would tag a commit that is not on the release line. + if [ "$REF_TYPE" != "branch" ] || [ "$REF_NAME" != "main" ]; then + echo "::error::Releases must be cut from main; this run is on '$REF_NAME'" + exit 1 + fi + VERSION="$INPUT_VERSION" +else + VERSION="$REF_NAME" +fi + +if [ "$SKIP" = "true" ]; then + echo "skip=true" >> "$GITHUB_OUTPUT" + exit 0 +fi + +# A bare "2.0.0" typed into the dispatch box is accepted; everything downstream +# works with the v-prefixed form the tag actually uses. +case "$VERSION" in + v*) ;; + *) VERSION="v$VERSION" ;; +esac + +if [[ ! "$VERSION" =~ ^v[0-9]+[.][0-9]+[.][0-9]+(-rc[0-9]+)?$ ]]; then + echo "::error::Release tags must be v.. or v..-rcN, for example v1.0.0 or v1.1.0-rc1 (got '$VERSION')" + exit 1 +fi + +if [ "$EVENT_NAME" = "workflow_run" ]; then + # A tag already pointing at this commit is the resumed release above, and the + # publish steps tolerate it. Only a tag on some other commit is a collision. + if git show-ref --verify --quiet "refs/tags/$VERSION"; then + if ! EXISTING_TAG_COMMIT="$(tag_commit "$VERSION")"; then + echo "::error::Calculated tag $VERSION already exists but does not point to a commit" + exit 1 + fi + if [ "$EXISTING_TAG_COMMIT" != "$HEAD_COMMIT" ]; then + echo "::error::Calculated tag $VERSION already exists on a different commit" + exit 1 + fi + fi +fi + +read_chart_field() { + grep -m1 "^$1:" helm/codeapi/Chart.yaml \ + | sed -E "s/^$1:[[:space:]]*//; s/[[:space:]]*#.*//; s/^[\"']//; s/[\"']\$//" +} +APP_VERSION="$(read_chart_field appVersion)" +CHART_VERSION="$(read_chart_field version)" + +if [ "$EVENT_NAME" = "workflow_dispatch" ] \ + && git rev-parse -q --verify "refs/tags/$VERSION" >/dev/null; then + echo "::error::Tag $VERSION already exists. Pick a new version, or delete the tag if it was cut in error." + exit 1 +fi + +case "$VERSION" in + *-rc*) PRERELEASE=true ;; + *) PRERELEASE=false ;; +esac + +# `latest` moves only when this is the highest stable version, so re-cutting an +# older patch cannot drag it backwards. The tag under dispatch does not exist +# yet, hence adding it to the comparison. +LATEST=false +if [ "$PRERELEASE" = "false" ]; then + HIGHEST_STABLE="$( + { + git tag --list 'v[0-9]*' + printf '%s\n' "$VERSION" + } \ + | select_stable_tags \ + | sort -V \ + | tail -n 1 + )" + if [ "$HIGHEST_STABLE" = "$VERSION" ]; then + LATEST=true + fi +fi + +DRAFT=false +if [ "$INPUT_DRAFT" = "true" ]; then + DRAFT=true +fi + +{ + echo "skip=false" + echo "version=$VERSION" + echo "app_version=$APP_VERSION" + echo "chart_version=$CHART_VERSION" + echo "prerelease=$PRERELEASE" + echo "latest=$LATEST" + echo "draft=$DRAFT" +} >> "$GITHUB_OUTPUT" + +echo "Releasing $VERSION (chart $CHART_VERSION, appVersion $APP_VERSION, prerelease=$PRERELEASE, latest=$LATEST, draft=$DRAFT)" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7259ec87..b645f4b8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,6 +46,9 @@ jobs: - name: Release versioning run: tests/release-versioning.sh + - name: Release version resolution + run: tests/release-version-resolution.sh + - name: Validate sandbox Dockerfiles run: | docker buildx build --check -f api/Dockerfile . diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ea8a5364..684f1654 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -61,6 +61,20 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 30 steps: + # `workflow_run` can be rerun for a commit older than this resolver. Save + # the helper from the revision that supplied this workflow before the + # release checkout replaces the working tree with that historical SHA. + - name: Checkout release workflow + if: github.event_name == 'workflow_run' + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + fetch-depth: 1 + ref: ${{ github.workflow_sha }} + + - name: Preserve release resolver + if: github.event_name == 'workflow_run' + run: install -m 755 .github/scripts/resolve-release-version.sh "$RUNNER_TEMP/resolve-release-version.sh" + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 with: # Full history and tags: resolving whether this release is the newest @@ -78,139 +92,12 @@ jobs: REF_NAME: ${{ github.ref_name }} REF_TYPE: ${{ github.ref_type }} GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - - SKIP=false - if [ "$EVENT_NAME" = "workflow_run" ]; then - if [ "$(git rev-parse HEAD)" != "$HEAD_SHA" ]; then - echo "::error::Checked out SHA does not match the successful CI run" - exit 1 - fi - - REMOTE_MAIN_SHA="$(git ls-remote origin refs/heads/main | awk '{print $1}')" - if [ -z "$REMOTE_MAIN_SHA" ]; then - echo "::error::Could not resolve the current main branch tip" - exit 1 - fi - if [ "$REMOTE_MAIN_SHA" != "$HEAD_SHA" ]; then - echo "main advanced after this CI run; the newer successful run will release the combined changes" - SKIP=true - fi - - # A rerun after tag creation but before release publication resumes - # the missing release rather than incrementing the version again. - EXACT_TAG="$({ git tag --points-at HEAD || true; } | grep -E '^v[0-9]+[.][0-9]+[.][0-9]+$' | sort -V | tail -n 1)" - if [ "$SKIP" = "false" ] && [ -n "$EXACT_TAG" ]; then - if gh release view "$EXACT_TAG" >/dev/null 2>&1; then - echo "$EXACT_TAG already publishes this commit; nothing to do" - SKIP=true - else - VERSION="$EXACT_TAG" - fi - elif [ "$SKIP" = "false" ]; then - PREVIOUS_TAG="$(git tag --merged HEAD \ - | grep -E '^v[0-9]+[.][0-9]+[.][0-9]+$' \ - | sort -V \ - | tail -n 1)" - if [ -z "$PREVIOUS_TAG" ]; then - echo "::error::Automatic releases require an existing stable vMAJOR.MINOR.PATCH tag" - exit 1 - fi - VERSION="$(.github/scripts/next-release-version.sh "$PREVIOUS_TAG" "$PREVIOUS_TAG..HEAD")" - if [ -z "$VERSION" ]; then - echo "Only documentation, workflow, or test files changed since $PREVIOUS_TAG; no release needed" - SKIP=true - fi - fi - elif [ "$EVENT_NAME" = "workflow_dispatch" ]; then - # Releases describe what shipped to main. Dispatching from a topic - # branch would tag a commit that is not on the release line. - if [ "$REF_TYPE" != "branch" ] || [ "$REF_NAME" != "main" ]; then - echo "::error::Releases must be cut from main; this run is on '$REF_NAME'" - exit 1 - fi - VERSION="$INPUT_VERSION" - else - VERSION="$REF_NAME" - fi - - if [ "$SKIP" = "true" ]; then - echo "skip=true" >> "$GITHUB_OUTPUT" - exit 0 - fi - - # A bare "2.0.0" typed into the dispatch box is accepted; everything - # downstream works with the v-prefixed form the tag actually uses. - case "$VERSION" in - v*) ;; - *) VERSION="v$VERSION" ;; - esac - - if [[ ! "$VERSION" =~ ^v[0-9]+[.][0-9]+[.][0-9]+(-rc[0-9]+)?$ ]]; then - echo "::error::Release tags must be v.. or v..-rcN, for example v1.0.0 or v1.1.0-rc1 (got '$VERSION')" - exit 1 - fi - - if [ "$EVENT_NAME" = "workflow_run" ] \ - && git rev-parse -q --verify "refs/tags/$VERSION" >/dev/null; then - echo "::error::Calculated tag $VERSION already exists on a different commit" - exit 1 - fi - - read_chart_field() { - grep -m1 "^$1:" helm/codeapi/Chart.yaml \ - | sed -E "s/^$1:[[:space:]]*//; s/[[:space:]]*#.*//; s/^[\"']//; s/[\"']\$//" - } - APP_VERSION="$(read_chart_field appVersion)" - CHART_VERSION="$(read_chart_field version)" - - if [ "$EVENT_NAME" = "workflow_dispatch" ] \ - && git rev-parse -q --verify "refs/tags/$VERSION" >/dev/null; then - echo "::error::Tag $VERSION already exists. Pick a new version, or delete the tag if it was cut in error." - exit 1 - fi - - case "$VERSION" in - *-rc*) PRERELEASE=true ;; - *) PRERELEASE=false ;; - esac - - # `latest` moves only when this is the highest stable version, so - # re-cutting an older patch cannot drag it backwards. The tag under - # dispatch does not exist yet, hence adding it to the comparison. - LATEST=false - if [ "$PRERELEASE" = "false" ]; then - HIGHEST_STABLE="$( - { - git tag --list 'v[0-9]*' - printf '%s\n' "$VERSION" - } \ - | grep -E '^v[0-9]+[.][0-9]+[.][0-9]+$' \ - | sort -V \ - | tail -n 1 - )" - if [ "$HIGHEST_STABLE" = "$VERSION" ]; then - LATEST=true - fi - fi - - DRAFT=false - if [ "$INPUT_DRAFT" = "true" ]; then - DRAFT=true - fi - - { - echo "skip=false" - echo "version=$VERSION" - echo "app_version=$APP_VERSION" - echo "chart_version=$CHART_VERSION" - echo "prerelease=$PRERELEASE" - echo "latest=$LATEST" - echo "draft=$DRAFT" - } >> "$GITHUB_OUTPUT" - - echo "Releasing $VERSION (chart $CHART_VERSION, appVersion $APP_VERSION, prerelease=$PRERELEASE, latest=$LATEST, draft=$DRAFT)" + RESOLVER_PATH: ${{ github.event_name == 'workflow_run' && format('{0}/resolve-release-version.sh', runner.temp) || '.github/scripts/resolve-release-version.sh' }} + # The resolution itself lives in a script so that every path through it + # — automatic release, resumed release, dispatch, pushed tag, and the + # runs that must skip or fail — is covered by + # tests/release-version-resolution.sh in CI. + run: "$RESOLVER_PATH" # helm is preinstalled on ubuntu-latest, the same way the chart tests in # ci.yml depend on it. diff --git a/tests/release-version-resolution.sh b/tests/release-version-resolution.sh new file mode 100755 index 00000000..c70d50d3 --- /dev/null +++ b/tests/release-version-resolution.sh @@ -0,0 +1,265 @@ +#!/usr/bin/env bash + +# Covers .github/scripts/resolve-release-version.sh: the version a release run +# publishes, and the runs that have to skip or fail instead. Every case builds a +# throwaway repository with an `origin` the resolver can query and a stubbed +# `gh`, so nothing here reaches the network or the real repository. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# The resolver is deliberately kept outside each throwaway checkout. That +# mirrors release.yml preserving the workflow revision in RUNNER_TEMP before a +# workflow_run checks out the possibly historical release commit. +RESOLVER="$ROOT/.github/scripts/resolve-release-version.sh" +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +REPO="$WORK/repo" +ORIGIN="$WORK/origin.git" +OUTPUT="$WORK/github_output" +LOG="$WORK/log" +CASE='' +STATUS=0 +FAILURES=0 + +# `gh release view` is the only gh call the resolver makes. PUBLISHED lists the +# releases that already exist; anything else must not be invoked at all. +mkdir -p "$WORK/bin" +cat > "$WORK/bin/gh" <<'STUB' +#!/usr/bin/env bash +if [ "$1" = 'release' ] && [ "$2" = 'view' ]; then + for published in ${PUBLISHED:-}; do + if [ "$published" = "$3" ]; then + exit 0 + fi + done + exit 1 +fi +echo "unexpected gh invocation: $*" >&2 +exit 2 +STUB +chmod +x "$WORK/bin/gh" +PATH="$WORK/bin:$PATH" + +git_repo() { + git -C "$REPO" "$@" +} + +# A fresh repository whose layout matches what the resolver reads from the +# checkout: the version bump script it shells out to, and the chart it takes the +# app and chart versions from, trailing comments and quotes included. +new_case() { + CASE="$1" + rm -rf "$REPO" "$ORIGIN" + git init -q --bare "$ORIGIN" + git init -q -b main "$REPO" + git_repo config user.name test + git_repo config user.email test@example.com + git_repo remote add origin "$ORIGIN" + mkdir -p "$REPO/.github/scripts" "$REPO/helm/codeapi" + cp "$ROOT/.github/scripts/next-release-version.sh" "$REPO/.github/scripts/" + cat > "$REPO/helm/codeapi/Chart.yaml" <<'CHART' +apiVersion: v2 +name: codeapi +version: 0.3.1 # Chart version +appVersion: "2.0.0" # App version +CHART + commit api/runtime.ts initial 'chore: initial import' + publish_main +} + +commit() { + local path="$1" content="$2" message="$3" + mkdir -p "$REPO/$(dirname "$path")" + printf '%s\n' "$content" > "$REPO/$path" + git_repo add "$path" + git_repo commit -q -m "$message" +} + +publish_main() { + git_repo push -q origin main +} + +head_sha() { + git_repo rev-parse HEAD +} + +# Runs the resolver in the throwaway repository. Arguments are KEY=VALUE pairs +# standing in for the workflow's env block. +resolve() { + : > "$OUTPUT" + set +e + (cd "$REPO" && env GITHUB_OUTPUT="$OUTPUT" "$@" bash "$RESOLVER") > "$LOG" 2>&1 + STATUS=$? + set -e +} + +fail() { + echo "FAIL [$CASE] $1" >&2 + sed 's/^/ | /' "$LOG" >&2 + FAILURES=$((FAILURES + 1)) +} + +expect_status() { + if [ "$STATUS" != "$1" ]; then + fail "exit status: expected $1, got $STATUS" + fi +} + +expect_output() { + local actual + actual="$(sed -n "s/^$1=//p" "$OUTPUT" | tail -n 1)" + if [ "$actual" != "$2" ]; then + fail "output $1: expected '$2', got '$actual'" + fi +} + +expect_log() { + if ! grep -qF "$1" "$LOG"; then + fail "expected log to mention: $1" + fi +} + +# An untagged tip of main is the ordinary automatic-release path: the version +# comes from Conventional Commit intent since the last stable tag. Filtering +# tags with `grep` used to abort the step here, because no match under +# `pipefail` looks like a command failure. +new_case 'automatic release from an untagged commit' +git_repo tag v1.2.3 +commit api/runtime.ts repaired 'fix: repair execution' +publish_main +resolve EVENT_NAME=workflow_run HEAD_SHA="$(head_sha)" +expect_status 0 +expect_output skip false +expect_output version v1.2.4 +expect_output app_version 2.0.0 +expect_output chart_version 0.3.1 +expect_output prerelease false +expect_output latest true +expect_output draft false + +new_case 'documentation-only range releases nothing' +git_repo tag v1.2.3 +commit docs/guide.md docs 'docs: clarify deployment' +publish_main +resolve EVENT_NAME=workflow_run HEAD_SHA="$(head_sha)" +expect_status 0 +expect_output skip true +expect_output version '' +expect_log 'no release needed' + +new_case 'a repository without a stable tag reports why' +commit api/runtime.ts repaired 'fix: repair execution' +publish_main +resolve EVENT_NAME=workflow_run HEAD_SHA="$(head_sha)" +expect_status 1 +expect_log 'Automatic releases require an existing stable' + +# The rerun-to-publish recovery path: a previous run created the tag and then +# failed before the release existed. +new_case 'a rerun resumes the tag that already points at HEAD' +git_repo tag v1.2.3 +commit api/runtime.ts repaired 'fix: repair execution' +git_repo tag v1.2.4 +publish_main +resolve EVENT_NAME=workflow_run HEAD_SHA="$(head_sha)" PUBLISHED='' +expect_status 0 +expect_output skip false +expect_output version v1.2.4 +expect_output latest true + +new_case 'a published tag at HEAD releases nothing twice' +git_repo tag v1.2.3 +commit api/runtime.ts repaired 'fix: repair execution' +git_repo tag v1.2.4 +publish_main +resolve EVENT_NAME=workflow_run HEAD_SHA="$(head_sha)" PUBLISHED='v1.2.4' +expect_status 0 +expect_output skip true +expect_log 'already publishes this commit' + +new_case 'a calculated tag held by another commit is a collision' +git_repo tag v1.2.3 +git_repo checkout -q -b elsewhere +commit api/runtime.ts diverged 'fix: unrelated work' +git_repo tag v1.2.4 +git_repo checkout -q main +commit api/runtime.ts repaired 'fix: repair execution' +publish_main +resolve EVENT_NAME=workflow_run HEAD_SHA="$(head_sha)" +expect_status 1 +expect_log 'already exists on a different commit' + +new_case 'a calculated tag held by a non-commit object is a collision' +git_repo tag v1.2.3 +blob="$(printf 'not a commit\n' | git_repo hash-object -w --stdin)" +git_repo update-ref refs/tags/v1.2.4 "$blob" +commit api/runtime.ts repaired 'fix: repair execution' +publish_main +resolve EVENT_NAME=workflow_run HEAD_SHA="$(head_sha)" +expect_status 1 +expect_log 'already exists but does not point to a commit' + +new_case 'a stale CI run defers to the newer tip' +git_repo tag v1.2.3 +commit api/runtime.ts repaired 'fix: repair execution' +publish_main +commit api/runtime.ts advanced 'fix: land more work' +resolve EVENT_NAME=workflow_run HEAD_SHA="$(head_sha)" +expect_status 0 +expect_output skip true +expect_log 'main advanced after this CI run' + +new_case 'a dispatched version may omit the v prefix' +git_repo tag v1.2.3 +resolve EVENT_NAME=workflow_dispatch REF_TYPE=branch REF_NAME=main \ + INPUT_VERSION=2.0.0 INPUT_DRAFT=true +expect_status 0 +expect_output version v2.0.0 +expect_output prerelease false +expect_output latest true +expect_output draft true + +new_case 'a release candidate is a prerelease and never latest' +git_repo tag v1.2.3 +resolve EVENT_NAME=workflow_dispatch REF_TYPE=branch REF_NAME=main \ + INPUT_VERSION=v1.3.0-rc1 +expect_status 0 +expect_output version v1.3.0-rc1 +expect_output prerelease true +expect_output latest false + +new_case 'dispatching from a topic branch is refused' +resolve EVENT_NAME=workflow_dispatch REF_TYPE=branch REF_NAME=feature/x \ + INPUT_VERSION=v1.3.0 +expect_status 1 +expect_log 'Releases must be cut from main' + +new_case 'dispatching an existing version is refused' +git_repo tag v1.2.3 +resolve EVENT_NAME=workflow_dispatch REF_TYPE=branch REF_NAME=main \ + INPUT_VERSION=v1.2.3 +expect_status 1 +expect_log 'already exists' + +new_case 'a malformed version is refused' +resolve EVENT_NAME=workflow_dispatch REF_TYPE=branch REF_NAME=main \ + INPUT_VERSION=1.2 +expect_status 1 +expect_log 'Release tags must be' + +new_case 'a pushed older patch tag does not become latest' +git_repo tag v9.9.9 +resolve EVENT_NAME=push REF_TYPE=tag REF_NAME=v1.0.1 +expect_status 0 +expect_output version v1.0.1 +expect_output prerelease false +expect_output latest false + +if [ "$FAILURES" -ne 0 ]; then + echo "$FAILURES release version resolution assertion(s) failed" >&2 + exit 1 +fi + +echo 'release version resolution tests passed' From 54aca9262ecc1eca6ad3746e6f5b72169897afcf Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 19 Sep 2026 09:50:44 -0400 Subject: [PATCH 02/48] feat: Route GitHub App credentials per repository (#236) * feat: Route GitHub App credentials per repository * test: Make repository routing assertion deterministic * fix: Harden repository credential routing * fix: Bound shared GitHub credential refreshes * fix: Bind GitHub credentials to admitted workspaces --- docs/remote-bridge/worker-runbook.md | 19 +- packages/code/README.md | 14 +- packages/code/src/cli.test.ts | 59 ++- packages/code/src/cli.ts | 52 ++- packages/code/src/github.test.ts | 442 +++++++++++++++++++++- packages/code/src/github.ts | 445 ++++++++++++++++++++--- packages/code/src/native-process.test.ts | 5 +- packages/code/src/native-process.ts | 12 +- packages/code/src/native-sandbox.test.ts | 45 +++ packages/code/src/native-sandbox.ts | 26 +- 10 files changed, 1037 insertions(+), 82 deletions(-) diff --git a/docs/remote-bridge/worker-runbook.md b/docs/remote-bridge/worker-runbook.md index 7511a265..16c24742 100644 --- a/docs/remote-bridge/worker-runbook.md +++ b/docs/remote-bridge/worker-runbook.md @@ -323,13 +323,21 @@ Configure the worker, preferably in a separate service drop-in: ```ini [Service] Environment=LIBRECHAT_CODE_GITHUB_APP_ID=12345 -Environment=LIBRECHAT_CODE_GITHUB_INSTALLATION_ID=67890 Environment=LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE=/home/librechat-code/.config/librechat-code/github-app.pem ``` -The trusted worker mints short-lived installation tokens. Sandboxed commands -receive masked Git/`gh` credentials only for the configured GitHub hosts; the -token is not written to the repository, remote URL, or Git configuration. +Install the same App separately on every personal account or organization the +worker is allowed to use. The trusted worker resolves the correct installation +from the repository containing each command's working directory, then mints and +caches a repository-scoped token. Cross-repository work therefore does not +require changing an installation ID or restarting the worker. Set +`LIBRECHAT_CODE_GITHUB_INSTALLATION_ID` only as a legacy fixed-installation +fallback. + +Sandboxed commands receive masked Git/`gh` credentials only for the configured +GitHub hosts; the token is not written to the repository, remote URL, or Git +configuration. Git commits receive the App bot's canonical no-reply identity so +GitHub renders the bot profile and avatar. ## 10. Run under systemd @@ -518,7 +526,8 @@ command, cancellation, or settlement whose effects may be incomplete. - [ ] Pairing is principal-bound and the identity file is private. - [ ] Definitions are outside roots and immutable to sandboxed tools. - [ ] Workspace ancestors are not group/other writable. -- [ ] GitHub App is optional, least-privilege, and installed only where needed. +- [ ] GitHub App is optional, least-privilege, and installed on every account + the worker is expected to use. - [ ] Approval policy remains enforced independently of worker capability. - [ ] Service manager uses the intended executable and configuration. - [ ] Worker is online, ready, and advertises the expected workspace. diff --git a/packages/code/README.md b/packages/code/README.md index 08dc54a7..69d85b16 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -283,14 +283,24 @@ repositories the agent may access: ```bash LIBRECHAT_CODE_GITHUB_APP_ID=12345 \ -LIBRECHAT_CODE_GITHUB_INSTALLATION_ID=67890 \ LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE=/secure/librechat-agent.pem \ librechat-code run --worker-dir /path/to/project --allow-workspace-commands ``` The private key must be an owner-only regular file outside the workspace. It is read only by the trusted worker, which mints and refreshes short-lived -installation tokens. A personal access token is supported as a fallback with +installation tokens. At startup, the worker binds each explicitly admitted +workspace root to its Git repository. Commands in those independent roots can +use simultaneous installations on personal accounts and organizations without +being restarted or reconfigured, while a command cannot gain access by changing +its workspace's remote URL. Tokens are scoped and cached per repository. For +compatibility with deployments +that intentionally bind a worker to one installation, set the optional legacy +`LIBRECHAT_CODE_GITHUB_INSTALLATION_ID` fallback. + +App-authenticated commits use the GitHub App bot's canonical no-reply identity, +so GitHub links them to the bot profile and avatar. A personal access token is +supported as a fallback with `LIBRECHAT_CODE_GITHUB_TOKEN`, but the GitHub App is the safer default because its repository access and permissions can be narrowly installed and revoked. Native Windows credential storage is unavailable until native DACL removal and diff --git a/packages/code/src/cli.test.ts b/packages/code/src/cli.test.ts index 18456072..f39b28d9 100644 --- a/packages/code/src/cli.test.ts +++ b/packages/code/src/cli.test.ts @@ -1,5 +1,9 @@ import assert from 'node:assert/strict'; import { spawnSync } from 'node:child_process'; +import { generateKeyPairSync } from 'node:crypto'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import test from 'node:test'; @@ -233,6 +237,59 @@ test('CLI validates GitHub App credentials before worker registration', () => { assert.doesNotMatch(result.stderr, /fetch failed/); }); +test('CLI accepts repository-routed GitHub App authentication without a fixed installation', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'cli-github-routing-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const preload = join(directory, 'fetch.mjs'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile( + privateKeyPath, + privateKey.export({ type: 'pkcs8', format: 'pem' }), + { mode: 0o600 }, + ); + await writeFile( + preload, + ` + globalThis.fetch = async (input) => { + const url = String(input); + if (url.endsWith('/app')) return Response.json({ slug: 'lia-by-librechat' }); + if (url.endsWith('/users/lia-by-librechat%5Bbot%5D')) { + return Response.json({ id: 328778573, login: 'lia-by-librechat[bot]', type: 'Bot' }); + } + throw new Error('test stopped after GitHub App validation'); + }; + `, + ); + const result = spawnSync( + process.execPath, + [ + '--import', + preload, + fileURLToPath(new URL('./cli.js', import.meta.url)), + ], + { + encoding: 'utf8', + timeout: 10_000, + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_WORKER_DIR: directory, + LIBRECHAT_CODE_ALLOW_WORKSPACE_COMMANDS: 'true', + LIBRECHAT_CODE_GITHUB_TOKEN: undefined, + LIBRECHAT_CODE_GITHUB_APP_ID: '123', + LIBRECHAT_CODE_GITHUB_INSTALLATION_ID: undefined, + LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE: privateKeyPath, + }, + }, + ); + assert.notEqual(result.status, 0); + assert.doesNotMatch(result.stderr, /GitHub App authentication requires/); + assert.doesNotMatch(result.stderr, /installation ID/i); +}); + test('CLI requires a runtime image for Docker supervision', () => { const result = spawnSync( process.execPath, @@ -449,6 +506,6 @@ test('CLI host-only enterprise configuration sends App JWTs to GHES, never GitHu }, }); assert.equal(result.status, 1, result.stderr); - assert.match(result.stderr, /GITHUB_REQUEST:https:\/\/github\.example\.test\/api\/v3\/app\/installations\/456\/access_tokens/); + assert.match(result.stderr, /GITHUB_REQUEST:https:\/\/github\.example\.test\/api\/v3\/app/); assert.doesNotMatch(result.stderr, /GITHUB_REQUEST:https:\/\/api\.github\.com/); }); diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 524c5853..96757d57 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -46,6 +46,8 @@ import type { LocalWorkspaceConfig } from './workspace.js'; import { GITHUB_ALLOWED_DOMAINS, GitHubAppCredentialProvider, + gitHubRepositoryForAdmittedDirectory, + gitHubRepositoryForDirectory, gitHubCommandCredentialEnvironment, gitHubMaskedCredentialVariables, StaticGitHubCredentialProvider, @@ -149,6 +151,7 @@ function githubCredentials(): { host: string; privateKeyPath?: string; mode?: 'app' | 'token'; + repositoryRouting?: boolean; policyIdentity: string; } { const token = nonEmpty(process.env.LIBRECHAT_CODE_GITHUB_TOKEN); @@ -161,9 +164,9 @@ function githubCredentials(): { ); const appValues = [appId, installationId, privateKeyPath]; const hasApp = appValues.some(Boolean); - if (hasApp && !appValues.every(Boolean)) { + if (hasApp && (!appId || !privateKeyPath)) { throw new Error( - 'GitHub App authentication requires LIBRECHAT_CODE_GITHUB_APP_ID, LIBRECHAT_CODE_GITHUB_INSTALLATION_ID, and LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE', + 'GitHub App authentication requires LIBRECHAT_CODE_GITHUB_APP_ID and LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE; LIBRECHAT_CODE_GITHUB_INSTALLATION_ID is an optional legacy fallback', ); } if (hasApp && token) { @@ -203,6 +206,7 @@ function githubCredentials(): { return { host, mode: 'app', + repositoryRouting: !installationId, policyIdentity: gitHubAuthenticationPolicyIdentity({ mode: 'app', host, @@ -212,7 +216,7 @@ function githubCredentials(): { privateKeyPath, provider: new GitHubAppCredentialProvider({ appId: appId!, - installationId: installationId!, + installationId, privateKeyPath: privateKeyPath!, host, apiUrl, @@ -715,6 +719,19 @@ async function run( }), ]), ); + // Bind credentials to immutable, explicitly admitted roots. The repository + // remote is operator input at startup, never an authorization input that a + // sandboxed command may change for its next invocation. + const admittedGitHubRepositories = github.provider && github.repositoryRouting + ? new Map( + await Promise.all( + roots.map(async root => [ + root.root, + await gitHubRepositoryForDirectory(root.root, github.host), + ] as const), + ), + ) + : undefined; const localWorkspaceTools = workerDirectory ? await LocalWorkspaceTools.create({ workspaces: roots, @@ -928,18 +945,34 @@ async function run( ...(github.provider ? { maskedEnvironment: { - variables: gitHubMaskedCredentialVariables(github.host), - async resolve(signal?: AbortSignal) { + variables: gitHubMaskedCredentialVariables( + github.host, + ), + async resolve(signal?: AbortSignal, cwd?: string) { + const repository = cwd && admittedGitHubRepositories + ? gitHubRepositoryForAdmittedDirectory( + cwd, + admittedGitHubRepositories, + ) + : undefined; + if (!repository && github.repositoryRouting) { + return {}; + } return gitHubCommandCredentialEnvironment( - await github.provider!.getCredential(signal), + await github.provider!.getCredential(signal, repository), github.host, ); }, - wrapCommand(command: string, platform: NodeJS.Platform) { + wrapCommand( + command: string, + platform: NodeJS.Platform, + environment: Readonly>, + ) { return wrapGitHubCredentialCommand( command, github.host, platform, + environment, ); }, }, @@ -1022,7 +1055,10 @@ async function run( ); } try { - await github.provider?.getCredential(controller.signal); + await github.provider?.validate?.(controller.signal); + if (github.provider && !github.provider.validate) { + await github.provider.getCredential(controller.signal); + } await nativeCommandSandbox?.prepare(); for (const environment of option(args, '--reset-workspace-quarantine') == null ? environments : []) { const setup = environment.definition.setup; diff --git a/packages/code/src/github.test.ts b/packages/code/src/github.test.ts index 4b028239..73e02927 100644 --- a/packages/code/src/github.test.ts +++ b/packages/code/src/github.test.ts @@ -1,4 +1,5 @@ import { generateKeyPairSync } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; import { chmod, mkdtemp, @@ -8,12 +9,14 @@ import { writeFile, } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { dirname, join } from 'node:path'; import test from 'node:test'; import assert from 'node:assert/strict'; import { GITHUB_ALLOWED_DOMAINS, + GITHUB_AUTHOR_EMAIL_ENV_NAME, + GITHUB_AUTHOR_NAME_ENV_NAME, GitHubAppCredentialProvider, StaticGitHubCredentialProvider, gitHubAuthenticationPolicyIdentity, @@ -22,6 +25,8 @@ import { gitHubMaskedCredentialVariables, GITHUB_CREDENTIAL_ENV_NAME, gitHubCredentialEnvironment, + gitHubRepositoryForAdmittedDirectory, + gitHubRepositoryForDirectory, normalizeGitHubHost, wrapGitHubCredentialCommand, } from './github.js'; @@ -136,6 +141,358 @@ test('mints and caches a short-lived GitHub App installation token', async (t) = assert.equal(calls, 1); }); +test('routes and scopes GitHub App tokens per repository installation', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-routing-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile( + privateKeyPath, + privateKey.export({ type: 'pkcs8', format: 'pem' }), + { mode: 0o600 }, + ); + const calls: Array<{ url: string; body?: string }> = []; + const provider = new GitHubAppCredentialProvider({ + appId: '123', + privateKeyPath, + now: () => new Date('2030-01-01T00:00:00Z'), + fetch: (async (input, init) => { + const url = String(input); + calls.push({ url, body: typeof init?.body === 'string' ? init.body : undefined }); + if (url.endsWith('/app')) { + return Response.json({ slug: 'lia-by-librechat' }); + } + if (url.endsWith('/users/lia-by-librechat%5Bbot%5D')) { + return Response.json({ + id: 328778573, + login: 'lia-by-librechat[bot]', + type: 'Bot', + }); + } + if (url.endsWith('/repos/danny-avila/LibreChat/installation')) { + return Response.json({ id: 111 }); + } + if (url.endsWith('/repos/LibreChat-AI/code-interpreter/installation')) { + return Response.json({ id: 222 }); + } + const installation = /\/app\/installations\/(\d+)\/access_tokens$/.exec(url)?.[1]; + if (installation) { + return Response.json( + { + token: `ghs_${installation}_abcdefghijklmnopqrstuvwxyz`, + expires_at: '2030-01-01T01:00:00Z', + }, + { status: 201 }, + ); + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + + await provider.validate(); + const [personal, organization] = await Promise.all([ + provider.getCredential(undefined, 'danny-avila/LibreChat'), + provider.getCredential(undefined, 'LibreChat-AI/code-interpreter'), + ]); + assert.equal( + (await provider.getCredential(undefined, 'danny-avila/LibreChat')).value, + personal.value, + ); + assert.equal(personal.value, 'ghs_111_abcdefghijklmnopqrstuvwxyz'); + assert.equal(organization.value, 'ghs_222_abcdefghijklmnopqrstuvwxyz'); + assert.deepEqual(personal.actor, { + name: 'lia-by-librechat[bot]', + email: + '328778573+lia-by-librechat[bot]@users.noreply.github.com', + }); + assert.equal( + calls.filter(call => call.url.includes('/repos/danny-avila/')).length, + 1, + ); + assert.deepEqual( + calls + .filter(call => call.url.endsWith('/access_tokens')) + .map(call => JSON.parse(call.body ?? '{}')) + .map(body => body.repositories[0]) + .sort(), + [ + 'LibreChat', + 'code-interpreter', + ], + ); +}); + +test('keeps a shared token refresh alive when one waiter is cancelled', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-cancel-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile( + privateKeyPath, + privateKey.export({ type: 'pkcs8', format: 'pem' }), + { mode: 0o600 }, + ); + let releaseToken!: (response: Response) => void; + const tokenResponse = new Promise(resolve => { + releaseToken = resolve; + }); + let mintCount = 0; + const provider = new GitHubAppCredentialProvider({ + appId: '123', + privateKeyPath, + now: () => new Date('2030-01-01T00:00:00Z'), + fetch: (async (input) => { + const url = String(input); + if (url.endsWith('/app')) return Response.json({ slug: 'lia' }); + if (url.endsWith('/users/lia%5Bbot%5D')) { + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); + } + if (url.endsWith('/repos/acme/project/installation')) { + return Response.json({ id: 111 }); + } + if (url.endsWith('/app/installations/111/access_tokens')) { + mintCount += 1; + return tokenResponse; + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + await provider.validate(); + const firstController = new AbortController(); + const first = provider.getCredential( + firstController.signal, + 'acme/project', + ); + const second = provider.getCredential(undefined, 'acme/project'); + firstController.abort(new Error('first command cancelled')); + await assert.rejects(first, /first command cancelled/); + const third = provider.getCredential(undefined, 'acme/project'); + releaseToken( + Response.json({ + token: 'ghs_shared_abcdefghijklmnopqrstuvwxyz', + expires_at: '2030-01-01T01:00:00Z', + }), + ); + assert.equal( + (await second).value, + 'ghs_shared_abcdefghijklmnopqrstuvwxyz', + ); + assert.equal((await third).value, 'ghs_shared_abcdefghijklmnopqrstuvwxyz'); + assert.equal(mintCount, 1); +}); + +test('refreshes a cached repository installation after App reinstallation', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-reinstall-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile( + privateKeyPath, + privateKey.export({ type: 'pkcs8', format: 'pem' }), + { mode: 0o600 }, + ); + let now = new Date('2030-01-01T00:00:00Z'); + let lookupCount = 0; + let oldMintCount = 0; + const provider = new GitHubAppCredentialProvider({ + appId: '123', + privateKeyPath, + now: () => now, + fetch: (async (input) => { + const url = String(input); + if (url.endsWith('/app')) return Response.json({ slug: 'lia' }); + if (url.endsWith('/users/lia%5Bbot%5D')) { + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); + } + if (url.endsWith('/repos/acme/project/installation')) { + lookupCount += 1; + return Response.json({ id: lookupCount === 1 ? 111 : 222 }); + } + if (url.endsWith('/app/installations/111/access_tokens')) { + oldMintCount += 1; + return oldMintCount === 1 + ? Response.json({ + token: 'ghs_old_abcdefghijklmnopqrstuvwxyz', + expires_at: '2030-01-01T01:00:00Z', + }) + : Response.json({}, { status: 404 }); + } + if (url.endsWith('/app/installations/222/access_tokens')) { + return Response.json({ + token: 'ghs_new_abcdefghijklmnopqrstuvwxyz', + expires_at: '2030-01-01T02:00:00Z', + }); + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + await provider.validate(); + assert.equal( + (await provider.getCredential(undefined, 'acme/project')).value, + 'ghs_old_abcdefghijklmnopqrstuvwxyz', + ); + now = new Date('2030-01-01T00:56:00Z'); + assert.equal( + (await provider.getCredential(undefined, 'acme/project')).value, + 'ghs_new_abcdefghijklmnopqrstuvwxyz', + ); + assert.equal(lookupCount, 2); +}); + +test('validates a configured fixed installation by minting its token', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-fixed-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile( + privateKeyPath, + privateKey.export({ type: 'pkcs8', format: 'pem' }), + { mode: 0o600 }, + ); + let minted = false; + const provider = new GitHubAppCredentialProvider({ + appId: '123', + installationId: '456', + privateKeyPath, + now: () => new Date('2030-01-01T00:00:00Z'), + fetch: (async (input) => { + const url = String(input); + if (url.endsWith('/app')) return Response.json({ slug: 'lia' }); + if (url.endsWith('/users/lia%5Bbot%5D')) { + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); + } + if (url.endsWith('/app/installations/456/access_tokens')) { + minted = true; + return Response.json({ + token: 'ghs_fixed_abcdefghijklmnopqrstuvwxyz', + expires_at: '2030-01-01T01:00:00Z', + }); + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + await provider.validate(); + assert.equal(minted, true); +}); + +test('discovers the GitHub repository from a command working directory', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-repo-')); + t.after(() => rm(directory, { recursive: true, force: true })); + execFileSync('git', ['init', directory]); + execFileSync('git', [ + '-C', + directory, + 'remote', + 'add', + 'origin', + 'git@github.com:LibreChat-AI/code-interpreter.git', + ]); + assert.equal( + await gitHubRepositoryForDirectory(directory), + 'LibreChat-AI/code-interpreter', + ); + assert.equal( + await gitHubRepositoryForDirectory(directory, 'github.example.test'), + undefined, + ); + execFileSync('git', [ + '-C', + directory, + 'remote', + 'set-url', + 'origin', + 'https://github.example.test:8443/acme/project.git', + ]); + assert.equal( + await gitHubRepositoryForDirectory(directory, 'github.example.test'), + 'acme/project', + ); +}); + +test('keeps repository authorization bound to the admitted workspace root', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-binding-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const nested = join(directory, 'packages', 'app'); + await mkdir(nested, { recursive: true }); + execFileSync('git', ['init', directory]); + execFileSync('git', [ + '-C', + directory, + 'remote', + 'add', + 'origin', + 'git@github.com:acme/allowed.git', + ]); + const admitted = new Map([ + [directory, await gitHubRepositoryForDirectory(directory)], + ]); + execFileSync('git', [ + '-C', + directory, + 'remote', + 'set-url', + 'origin', + 'git@github.com:acme/not-authorized.git', + ]); + assert.equal( + gitHubRepositoryForAdmittedDirectory(nested, admitted), + 'acme/allowed', + ); + assert.equal( + gitHubRepositoryForAdmittedDirectory(dirname(directory), admitted), + undefined, + ); +}); + +test('uses the configured GHES host for the App bot no-reply identity', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-ghes-identity-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile( + privateKeyPath, + privateKey.export({ type: 'pkcs8', format: 'pem' }), + { mode: 0o600 }, + ); + const provider = new GitHubAppCredentialProvider({ + appId: '123', + installationId: '456', + privateKeyPath, + host: 'github.example.test', + now: () => new Date('2030-01-01T00:00:00Z'), + fetch: (async (input) => { + const url = String(input); + if (url.endsWith('/app')) return Response.json({ slug: 'lia' }); + if (url.endsWith('/users/lia%5Bbot%5D')) { + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); + } + if (url.endsWith('/app/installations/456/access_tokens')) { + return Response.json({ + token: 'ghs_enterprise_abcdefghijklmnopqrstuvwxyz', + expires_at: '2030-01-01T01:00:00Z', + }); + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + await provider.validate(); + const credential = await provider.getCredential(); + assert.deepEqual(credential.actor, { + name: 'lia[bot]', + email: '1234+lia[bot]@users.noreply.github.example.test', + }); + const wrapped = wrapGitHubCredentialCommand( + 'git commit -m test', + 'github.example.test', + 'linux', + gitHubCommandCredentialEnvironment(credential, 'github.example.test'), + ); + assert.match( + wrapped, + /user\.email=1234\+lia\[bot\]@users\.noreply\.github\.example\.test/, + ); +}); + test('builds process-scoped Git HTTPS authorization without embedding credentials in URLs', async () => { const provider = new StaticGitHubCredentialProvider( 'github_pat_abcdefghijklmnopqrstuvwxyz', @@ -177,6 +534,76 @@ test('adds a GitHub CLI token only to the command-sandbox credential bundle', as ); }); +test('binds Git commits to the GitHub App bot identity', () => { + const environment = gitHubCommandCredentialEnvironment({ + value: 'ghs_abcdefghijklmnopqrstuvwxyz', + actor: { + name: 'lia-by-librechat[bot]', + email: + '328778573+lia-by-librechat[bot]@users.noreply.github.com', + }, + }); + assert.equal(environment[GITHUB_AUTHOR_NAME_ENV_NAME], 'lia-by-librechat[bot]'); + assert.equal( + environment[GITHUB_AUTHOR_EMAIL_ENV_NAME], + '328778573+lia-by-librechat[bot]@users.noreply.github.com', + ); + const variables = gitHubMaskedCredentialVariables('github.com'); + assert.ok(!variables.some(variable => variable.name === GITHUB_AUTHOR_NAME_ENV_NAME)); + assert.ok(!variables.some(variable => variable.name === GITHUB_AUTHOR_EMAIL_ENV_NAME)); + const wrapped = wrapGitHubCredentialCommand( + 'git commit -m test', + 'github.com', + 'linux', + environment, + ); + assert.match(wrapped, /user\.name=/); + assert.match(wrapped, /user\.email=/); +}); + +test('records the canonical App bot as Git author and committer', async (t) => { + if (process.platform === 'win32') { + t.skip('POSIX command wrapper integration is unavailable on Windows'); + return; + } + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-author-')); + t.after(() => rm(directory, { recursive: true, force: true })); + execFileSync('git', ['init', directory]); + const environment = gitHubCommandCredentialEnvironment({ + value: 'ghs_abcdefghijklmnopqrstuvwxyz', + actor: { + name: 'lia-by-librechat[bot]', + email: + '328778573+lia-by-librechat[bot]@users.noreply.github.com', + }, + }); + const wrapped = wrapGitHubCredentialCommand( + 'git commit --allow-empty -m test', + 'github.com', + process.platform, + environment, + ); + execFileSync('/bin/bash', ['-lc', wrapped], { + cwd: directory, + env: { PATH: process.env.PATH, ...environment }, + }); + assert.equal( + execFileSync( + 'git', + [ + '-C', + directory, + 'show', + '-s', + '--format=%an|%ae|%cn|%ce', + 'HEAD', + ], + { encoding: 'utf8' }, + ).trim(), + 'lia-by-librechat[bot]|328778573+lia-by-librechat[bot]@users.noreply.github.com|lia-by-librechat[bot]|328778573+lia-by-librechat[bot]@users.noreply.github.com', + ); +}); + test('selects the GitHub CLI token variable for public and enterprise hosts', () => { assert.equal(gitHubCliTokenEnvironmentName('github.com'), 'GH_TOKEN'); assert.equal( @@ -217,6 +644,9 @@ test('composes the masked credential with SRT Git configuration inside the sandb 'git push', 'github.com', 'darwin', + { + [GITHUB_CREDENTIAL_ENV_NAME]: 'masked-authorization', + }, ); assert.match(wrapped, /http\.proxyAuthMethod=basic/); assert.match(wrapped, /http\.https:\/\/github\.com\/\.extraheader/); @@ -227,6 +657,16 @@ test('composes the masked credential with SRT Git configuration inside the sandb assert.ok(!wrapped.includes('github_pat_')); }); +test('omits Git authorization when repository routing resolves no credential', () => { + const wrapped = wrapGitHubCredentialCommand( + 'git clone https://github.com/LibreChat-AI/LibreChat.git', + 'github.com', + 'linux', + {}, + ); + assert.doesNotMatch(wrapped, /Authorization: Basic/); +}); + test('targets GitHub CLI at an enterprise host without exposing its token', () => { const wrapped = wrapGitHubCredentialCommand( 'gh pr create', diff --git a/packages/code/src/github.ts b/packages/code/src/github.ts index c727e70d..4c79fa37 100644 --- a/packages/code/src/github.ts +++ b/packages/code/src/github.ts @@ -1,10 +1,15 @@ import { constants } from 'node:fs'; +import { execFile } from 'node:child_process'; import { createHash, createPrivateKey, sign } from 'node:crypto'; import { open } from 'node:fs/promises'; -import { dirname } from 'node:path'; +import { dirname, isAbsolute, relative, sep } from 'node:path'; +import { promisify } from 'node:util'; +import { projectRemote } from './projects.js'; import { assertPrivateStorageAcl, assertPrivateStorageAncestors, assertPrivateStorageSupported } from './private-storage.js'; export const GITHUB_CREDENTIAL_ENV_NAME = 'LIBRECHAT_CODE_GITHUB_AUTHORIZATION'; +export const GITHUB_AUTHOR_NAME_ENV_NAME = 'LIBRECHAT_CODE_GITHUB_AUTHOR_NAME'; +export const GITHUB_AUTHOR_EMAIL_ENV_NAME = 'LIBRECHAT_CODE_GITHUB_AUTHOR_EMAIL'; export const GITHUB_ALLOWED_DOMAINS = [ 'github.com', '*.github.com', @@ -18,15 +23,24 @@ export const GITHUB_ALLOWED_DOMAINS = [ export interface GitHubCredential { value: string; expiresAt?: Date; + actor?: { + name: string; + email: string; + }; } export interface GitHubCredentialProvider { - getCredential(signal?: AbortSignal): Promise; + getCredential( + signal?: AbortSignal, + repository?: string, + ): Promise; + validate?(signal?: AbortSignal): Promise; } export interface GitHubAppCredentialProviderOptions { appId: string; - installationId: string; + /** Legacy fixed installation. Omit to resolve the installation per repository. */ + installationId?: string; privateKeyPath: string; apiUrl?: string; /** Git HTTPS hostname; non-public hosts default to the GHES /api/v3 base. */ @@ -36,6 +50,110 @@ export interface GitHubAppCredentialProviderOptions { platform?: NodeJS.Platform; } +const execFileAsync = promisify(execFile); +const GITHUB_SHARED_REQUEST_TIMEOUT_MS = 30_000; + +async function waitForShared( + promise: Promise, + signal?: AbortSignal, +): Promise { + if (!signal) return promise; + signal.throwIfAborted(); + return new Promise((resolve, reject) => { + const aborted = () => { + try { + signal.throwIfAborted(); + } catch (error) { + reject(error); + } + }; + signal.addEventListener('abort', aborted, { once: true }); + void promise.then(resolve, reject).finally(() => { + signal.removeEventListener('abort', aborted); + }); + }); +} + +function repositoryName(value: string): { owner: string; name: string } { + const match = /^([A-Za-z0-9_.-]+)\/([A-Za-z0-9_.-]+)$/.exec(value); + if (!match) throw new Error('GitHub repository must be owner/name'); + return { owner: match[1], name: match[2] }; +} + +/** Resolve only the repository containing the admitted command cwd. */ +export async function gitHubRepositoryForDirectory( + cwd: string, + host = 'github.com', + signal?: AbortSignal, +): Promise { + let remote: string; + try { + const result = await execFileAsync( + 'git', + [ + '--no-optional-locks', + '-C', + cwd, + '-c', + 'core.fsmonitor=false', + 'config', + '--local', + '--no-includes', + '--get', + 'remote.origin.url', + ], + { + env: { + PATH: process.env.PATH, + SYSTEMROOT: process.env.SYSTEMROOT, + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_TERMINAL_PROMPT: '0', + GIT_OPTIONAL_LOCKS: '0', + LC_ALL: 'C', + }, + encoding: 'utf8', + maxBuffer: 4096, + timeout: 1500, + signal, + }, + ); + remote = result.stdout.trim(); + } catch { + signal?.throwIfAborted(); + return undefined; + } + const normalized = projectRemote(remote); + if (!normalized) return undefined; + const separator = normalized.indexOf('/'); + const remoteHost = normalized + .slice(0, separator) + .replace(/:[1-9][0-9]*$/, ''); + if (remoteHost !== normalizeGitHubHost(host)) { + return undefined; + } + const repository = normalized.slice(separator + 1); + repositoryName(repository); + return repository; +} + +/** Return the startup-bound repository for the admitted root containing cwd. */ +export function gitHubRepositoryForAdmittedDirectory( + cwd: string, + repositories: ReadonlyMap, +): string | undefined { + for (const [root, repository] of repositories) { + const path = relative(root, cwd); + if ( + path === '' || + (path !== '..' && !path.startsWith(`..${sep}`) && !isAbsolute(path)) + ) { + return repository; + } + } + return undefined; +} + function base64UrlJson(value: unknown): string { return Buffer.from(JSON.stringify(value)).toString('base64url'); } @@ -96,8 +214,13 @@ function createAppJwt(appId: string, privateKey: string, now: Date): string { } export class GitHubAppCredentialProvider implements GitHubCredentialProvider { - private cached?: GitHubCredential; + private readonly cached = new Map(); + private readonly inFlight = new Map>(); + private readonly installationIds = new Map(); + private actor?: GitHubCredential['actor']; + private actorInFlight?: Promise>; private readonly apiUrl: string; + private readonly host: string; constructor(private readonly options: GitHubAppCredentialProviderOptions) { if ((options.platform ?? process.platform) === 'win32') { @@ -106,10 +229,12 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { ); } assertPositiveIdentifier('GitHub App ID', options.appId); - assertPositiveIdentifier( - 'GitHub App installation ID', - options.installationId, - ); + if (options.installationId != null) { + assertPositiveIdentifier( + 'GitHub App installation ID', + options.installationId, + ); + } const host = options.host == null ? undefined : normalizeGitHubHost(options.host); const apiUrl = new URL(options.apiUrl ?? ( host != null && host !== 'github.com' @@ -126,58 +251,246 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { if (host != null && host !== apiHost) { throw new Error('LIBRECHAT_CODE_GITHUB_HOST must match the GitHub App API hostname'); } + this.host = host ?? apiHost; this.apiUrl = apiUrl.href.replace(/\/+$/, ''); } - async getCredential(signal?: AbortSignal): Promise { - const now = (this.options.now ?? (() => new Date()))(); - if ( - this.cached?.expiresAt != null && - this.cached.expiresAt.getTime() - now.getTime() > 5 * 60_000 - ) { - return this.cached; - } + private async appJwt(now: Date): Promise { const privateKey = await readPrivateKey(this.options.privateKeyPath); - const jwt = createAppJwt(this.options.appId, privateKey, now); + return createAppJwt(this.options.appId, privateKey, now); + } + + private async request( + path: string, + jwt: string, + signal?: AbortSignal, + init?: RequestInit, + ): Promise { const request = this.options.fetch ?? globalThis.fetch; - const response = await request( - `${this.apiUrl}/app/installations/${this.options.installationId}/access_tokens`, - { - method: 'POST', - redirect: 'error', - headers: { - Accept: 'application/vnd.github+json', - Authorization: `Bearer ${jwt}`, - 'X-GitHub-Api-Version': '2022-11-28', - }, - signal, + return request(`${this.apiUrl}${path}`, { + redirect: 'error', + ...init, + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${jwt}`, + 'X-GitHub-Api-Version': '2022-11-28', + ...init?.headers, }, + signal, + }); + } + + private async resolveActor( + jwt: string, + signal?: AbortSignal, + ): Promise> { + if (this.actor) return this.actor; + if (!this.actorInFlight) { + const pending = (async () => { + const sharedSignal = AbortSignal.timeout( + GITHUB_SHARED_REQUEST_TIMEOUT_MS, + ); + const appResponse = await this.request('/app', jwt, sharedSignal); + if (!appResponse.ok) { + throw new Error( + `GitHub App identity request failed with status ${appResponse.status}`, + ); + } + const app = (await appResponse.json()) as { slug?: unknown }; + if ( + typeof app.slug !== 'string' || + !/^[A-Za-z0-9-]+$/.test(app.slug) + ) { + throw new Error('GitHub App identity response is invalid'); + } + const login = `${app.slug}[bot]`; + const userResponse = await this.request( + `/users/${encodeURIComponent(login)}`, + jwt, + sharedSignal, + ); + if (!userResponse.ok) { + throw new Error( + `GitHub App bot identity request failed with status ${userResponse.status}`, + ); + } + const user = (await userResponse.json()) as { + id?: unknown; + login?: unknown; + type?: unknown; + }; + if ( + !Number.isSafeInteger(user.id) || + Number(user.id) <= 0 || + user.login !== login || + user.type !== 'Bot' + ) { + throw new Error('GitHub App bot identity response is invalid'); + } + return { + name: login, + email: `${user.id}+${login}@users.noreply.${this.host}`, + }; + })(); + this.actorInFlight = pending; + void pending.then( + actor => { + this.actor = actor; + if (this.actorInFlight === pending) this.actorInFlight = undefined; + }, + () => { + if (this.actorInFlight === pending) this.actorInFlight = undefined; + }, + ); + } + return waitForShared(this.actorInFlight, signal); + } + + async validate(signal?: AbortSignal): Promise { + const now = (this.options.now ?? (() => new Date()))(); + const jwt = await this.appJwt(now); + await this.resolveActor(jwt, signal); + if (this.options.installationId) { + await this.getCredential(signal); + } + } + + private async resolveInstallationId( + repository: string, + jwt: string, + signal?: AbortSignal, + ): Promise { + if (this.options.installationId) return this.options.installationId; + const cached = this.installationIds.get(repository); + if (cached) return cached; + const { owner, name } = repositoryName(repository); + const response = await this.request( + `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/installation`, + jwt, + signal, ); if (!response.ok) { throw new Error( - `GitHub App token request failed with status ${response.status}`, + response.status === 404 + ? `GitHub App is not installed for ${repository}` + : `GitHub App installation lookup failed with status ${response.status}`, ); } - const body = (await response.json()) as { - token?: unknown; - expires_at?: unknown; - }; - if ( - typeof body.token !== 'string' || - body.token.length < 20 || - typeof body.expires_at !== 'string' - ) { - throw new Error('GitHub App token response is invalid'); + const body = (await response.json()) as { id?: unknown }; + if (!Number.isSafeInteger(body.id) || Number(body.id) <= 0) { + throw new Error('GitHub App installation response is invalid'); + } + const installationId = String(body.id); + this.installationIds.set(repository, installationId); + return installationId; + } + + async getCredential( + signal?: AbortSignal, + repository?: string, + ): Promise { + signal?.throwIfAborted(); + if (!this.options.installationId && !repository) { + throw new Error( + 'GitHub App authentication requires a GitHub repository for this command', + ); } - const expiresAt = new Date(body.expires_at); + if (repository) repositoryName(repository); + const now = (this.options.now ?? (() => new Date()))(); + const key = this.options.installationId ?? repository!; + const cached = this.cached.get(key); if ( - !Number.isFinite(expiresAt.getTime()) || - expiresAt.getTime() <= now.getTime() + cached?.expiresAt != null && + cached.expiresAt.getTime() - now.getTime() > 5 * 60_000 ) { - throw new Error('GitHub App token expiry is invalid'); + return cached; } - this.cached = { value: body.token, expiresAt }; - return this.cached; + const existing = this.inFlight.get(key); + if (existing) return waitForShared(existing, signal); + const pending = (async () => { + const sharedSignal = AbortSignal.timeout( + GITHUB_SHARED_REQUEST_TIMEOUT_MS, + ); + const jwt = await this.appJwt(now); + const scopedRepository = repository + ? repositoryName(repository).name + : undefined; + const installationId = await this.resolveInstallationId( + repository ?? '', + jwt, + sharedSignal, + ); + let response = await this.request( + `/app/installations/${installationId}/access_tokens`, + jwt, + sharedSignal, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + ...(this.options.installationId + ? {} + : { body: JSON.stringify({ repositories: [scopedRepository] }) }), + }, + ); + if (!this.options.installationId && response.status === 404) { + this.installationIds.delete(repository!); + const refreshedInstallationId = await this.resolveInstallationId( + repository!, + jwt, + sharedSignal, + ); + response = await this.request( + `/app/installations/${refreshedInstallationId}/access_tokens`, + jwt, + sharedSignal, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ repositories: [scopedRepository] }), + }, + ); + } + if (!response.ok) { + throw new Error( + `GitHub App token request failed with status ${response.status}`, + ); + } + const body = (await response.json()) as { + token?: unknown; + expires_at?: unknown; + }; + if ( + typeof body.token !== 'string' || + body.token.length < 20 || + typeof body.expires_at !== 'string' + ) { + throw new Error('GitHub App token response is invalid'); + } + const expiresAt = new Date(body.expires_at); + if ( + !Number.isFinite(expiresAt.getTime()) || + expiresAt.getTime() <= now.getTime() + ) { + throw new Error('GitHub App token expiry is invalid'); + } + const credential = { + value: body.token, + expiresAt, + ...(this.actor ? { actor: this.actor } : {}), + }; + this.cached.set(key, credential); + return credential; + })(); + this.inFlight.set(key, pending); + const clearPending = () => { + if (this.inFlight.get(key) === pending) this.inFlight.delete(key); + }; + void pending.then(clearPending, clearPending); + return waitForShared(pending, signal); } } @@ -211,6 +524,12 @@ export function gitHubCommandCredentialEnvironment( return { ...gitHubCredentialEnvironment(credential), [gitHubCliTokenEnvironmentName(host)]: credential.value, + ...(credential.actor + ? { + [GITHUB_AUTHOR_NAME_ENV_NAME]: credential.actor.name, + [GITHUB_AUTHOR_EMAIL_ENV_NAME]: credential.actor.email, + } + : {}), }; } @@ -260,12 +579,10 @@ export function gitHubAuthenticationPolicyIdentity(options: { return `${identity}:fingerprint:${fingerprint}`; } if (options.mode !== 'app') return identity; - if (!options.appId || !options.installationId) { - throw new Error( - 'GitHub App policy identity requires an App and installation ID', - ); + if (!options.appId) { + throw new Error('GitHub App policy identity requires an App ID'); } - return `${identity}:app:${options.appId}:installation:${options.installationId}`; + return `${identity}:app:${options.appId}:installation:${options.installationId ?? 'repository'}`; } export function normalizeGitHubHost(value: string): string { @@ -284,24 +601,48 @@ export function wrapGitHubCredentialCommand( command: string, host = 'github.com', platform: NodeJS.Platform = process.platform, + environment: Readonly> = {}, ): string { const key = `http.https://${host}/.extraheader`; const cliHost = host === 'github.com' ? undefined : host; + const hasCredential = Boolean(environment[GITHUB_CREDENTIAL_ENV_NAME]); + const actorName = environment[GITHUB_AUTHOR_NAME_ENV_NAME]; + const actorEmail = environment[GITHUB_AUTHOR_EMAIL_ENV_NAME]; + const noReplyHost = `users.noreply.${normalizeGitHubHost(host)}` + .replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const hasActor = + /^[A-Za-z0-9_.-]+\[bot\]$/.test(actorName ?? '') && + new RegExp( + `^[1-9][0-9]+\\+[A-Za-z0-9_.-]+\\[bot\\]@${noReplyHost}$`, + ).test(actorEmail ?? ''); if (platform === 'win32') { return [ 'set "GIT_CONFIG_GLOBAL=NUL"', 'set "GIT_CONFIG_NOSYSTEM=1"', ...(cliHost ? [`set "GH_HOST=${cliHost}"`] : []), - `set "GIT_CONFIG_PARAMETERS='http.proxyAuthMethod=basic' '${key}=Authorization: Basic %${GITHUB_CREDENTIAL_ENV_NAME}%'"`, + ...(hasCredential + ? [`set "GIT_CONFIG_PARAMETERS='http.proxyAuthMethod=basic' '${key}=Authorization: Basic %${GITHUB_CREDENTIAL_ENV_NAME}%'"`] + : ['set "GIT_CONFIG_PARAMETERS="']), + ...(hasActor + ? [`set "GIT_CONFIG_PARAMETERS=%GIT_CONFIG_PARAMETERS% 'user.name=${actorName}' 'user.email=${actorEmail}'"`] + : []), `set "${GITHUB_CREDENTIAL_ENV_NAME}="`, + `set "${GITHUB_AUTHOR_NAME_ENV_NAME}="`, + `set "${GITHUB_AUTHOR_EMAIL_ENV_NAME}="`, command, ].join(' && '); } return [ 'export GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_NOSYSTEM=1', ...(cliHost ? [`export GH_HOST=${cliHost}`] : []), - `export GIT_CONFIG_PARAMETERS="'http.proxyAuthMethod=basic' '${key}=Authorization: Basic \${${GITHUB_CREDENTIAL_ENV_NAME}}'"`, + ...(hasCredential + ? [`export GIT_CONFIG_PARAMETERS="'http.proxyAuthMethod=basic' '${key}=Authorization: Basic \${${GITHUB_CREDENTIAL_ENV_NAME}}'"`] + : ['export GIT_CONFIG_PARAMETERS=']), + ...(hasActor + ? [`export GIT_CONFIG_PARAMETERS="\${GIT_CONFIG_PARAMETERS} 'user.name=${actorName}' 'user.email=${actorEmail}'"`] + : []), `unset ${GITHUB_CREDENTIAL_ENV_NAME}`, + `unset ${GITHUB_AUTHOR_NAME_ENV_NAME} ${GITHUB_AUTHOR_EMAIL_ENV_NAME}`, command, ].join(';\n'); } diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts index 89651845..b3016937 100644 --- a/packages/code/src/native-process.test.ts +++ b/packages/code/src/native-process.test.ts @@ -181,12 +181,14 @@ test('executor forwards the resolved command policy without worker credentials', test('executor hands credentials over IPC only for the current command', async () => { const fake = fixture(); + let credentialCwd: string | undefined; const sandbox = new NativeProcessWorkspaceCommandSandbox( { workspaceRoot: '/workspace', maskedEnvironment: { variables: [{ name: 'TOKEN', injectHosts: ['github.com'] }], - async resolve() { + async resolve(_signal, cwd) { + credentialCwd = cwd; return { TOKEN: 'per-command-secret' }; }, wrapCommand(command) { @@ -208,6 +210,7 @@ test('executor hands credentials over IPC only for the current command', async ( assert.deepEqual(fake.messages[1].credentials, { TOKEN: 'per-command-secret', }); + assert.equal(credentialCwd, '/workspace'); assert.equal(fake.messages[1].wrappedCommand, 'wrapped printf ok'); await sandbox.close(); }); diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index 70cb2bd1..ba411042 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -2,7 +2,7 @@ import { execFile, fork } from 'node:child_process'; import { randomUUID } from 'node:crypto'; import { constants as fsConstants } from 'node:fs'; import { access, realpath } from 'node:fs/promises'; -import { isAbsolute, join, relative, sep } from 'node:path'; +import { isAbsolute, join, relative, resolve, sep } from 'node:path'; import { promisify } from 'node:util'; import { WorkspaceToolError } from './workspace.js'; import { NATIVE_PROGRAMMATIC_COMMAND } from './native-programmatic.js'; @@ -436,11 +436,15 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan if (signal?.aborted) throw new Error('aborted'); programmaticExecutables = await this.resolveProgrammaticExecutables(); if (signal?.aborted) throw new Error('aborted'); - credentials = await this.options.maskedEnvironment?.resolve(signal); + credentials = await this.options.maskedEnvironment?.resolve( + signal, + this.options.workspaceRoot, + ); if (signal?.aborted) throw new Error('aborted'); wrappedCommand = this.options.maskedEnvironment?.wrapCommand?.( NATIVE_PROGRAMMATIC_COMMAND, process.platform, + credentials ?? {}, ); if (signal?.aborted) throw new Error('aborted'); } catch (error) { @@ -529,11 +533,13 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan try { await this.prepare(); if (signal?.aborted) throw new Error('aborted'); - credentials = await this.options.maskedEnvironment?.resolve(signal); + const cwd = resolve(this.options.workspaceRoot, request.cwd ?? '.'); + credentials = await this.options.maskedEnvironment?.resolve(signal, cwd); if (signal?.aborted) throw new Error('aborted'); wrappedCommand = this.options.maskedEnvironment?.wrapCommand?.( request.command, process.platform, + credentials ?? {}, ); if (signal?.aborted) throw new Error('aborted'); } catch (error) { diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index e610a0fc..b1e4c5df 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -1031,6 +1031,51 @@ test('masks a host credential for only its injection host and restores the paren }); }); +test('keeps trusted public command context out of credential masking', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + allowedDomains: ['github.com'], + maskedEnvironment: { + variables: [ + { + name: 'LIBRECHAT_CODE_TEST_CREDENTIAL', + injectHosts: ['github.com'], + }, + ], + async resolve() { + return { + LIBRECHAT_CODE_TEST_CREDENTIAL: 'real-secret', + LIBRECHAT_CODE_TEST_PUBLIC_IDENTITY: 'lia[bot]', + }; + }, + wrapCommand(command, _platform, environment) { + assert.equal( + environment.LIBRECHAT_CODE_TEST_PUBLIC_IDENTITY, + 'lia[bot]', + ); + return `export LIBRECHAT_CODE_TEST_PUBLIC_IDENTITY="${environment.LIBRECHAT_CODE_TEST_PUBLIC_IDENTITY}"; ${command}`; + }, + }, + manager: fake.manager, + }); + + const result = await sandbox.execute({ + ...request, + command: + 'printf "%s|%s" "$LIBRECHAT_CODE_TEST_CREDENTIAL" "$LIBRECHAT_CODE_TEST_PUBLIC_IDENTITY"', + }); + + assert.equal(result.stdout, 'Authorization: Bearer srt-sentinel|lia[bot]'); + assert.ok( + !fake.config?.credentials?.envVars?.some( + variable => variable.name === 'LIBRECHAT_CODE_TEST_PUBLIC_IDENTITY', + ), + ); +}); + test('serializes credential handoff across concurrent sandbox instances', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 84602a35..5bf61c50 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -182,8 +182,15 @@ export interface NativeSrtWorkspaceCommandSandboxOptions { injectHosts: string[]; extract?: string; }>; - resolve(signal?: AbortSignal): Promise>; - wrapCommand?(command: string, platform: NodeJS.Platform): string; + resolve( + signal?: AbortSignal, + cwd?: string, + ): Promise>; + wrapCommand?( + command: string, + platform: NodeJS.Platform, + environment: Readonly>, + ): string; }; } @@ -872,18 +879,19 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); } const commandId = `librechat-code-${randomUUID()}`; - const sandboxedCommand = this.options.maskedEnvironment?.wrapCommand - ? this.options.maskedEnvironment.wrapCommand( - request.command, - this.platform, - ) - : request.command; let wrapped: Awaited< ReturnType >; try { const credentialEnvironment = - await this.options.maskedEnvironment?.resolve(signal); + await this.options.maskedEnvironment?.resolve(signal, cwd); + const sandboxedCommand = this.options.maskedEnvironment?.wrapCommand + ? this.options.maskedEnvironment.wrapCommand( + request.command, + this.platform, + credentialEnvironment ?? {}, + ) + : request.command; wrapped = await this.withTemporaryHostEnvironment( { ...TRUSTED_GIT_ENVIRONMENT, From dc48249741e9d9bd010d19a3d5fcf56e2c159b79 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 19 Sep 2026 11:12:23 -0400 Subject: [PATCH 03/48] fix: Authenticate GitHub App bot identity lookup (#237) --- packages/code/src/github.test.ts | 61 ++++++++++++++++++++++---------- packages/code/src/github.ts | 54 +++++++++++++++++++++++----- 2 files changed, 89 insertions(+), 26 deletions(-) diff --git a/packages/code/src/github.test.ts b/packages/code/src/github.test.ts index 73e02927..352bf1c3 100644 --- a/packages/code/src/github.test.ts +++ b/packages/code/src/github.test.ts @@ -104,23 +104,30 @@ test('mints and caches a short-lived GitHub App installation token', async (t) = { mode: 0o600 }, ); await chmod(privateKeyPath, 0o600); - let calls = 0; + const calls: Array<{ url: string; authorization: string | null }> = []; const request = async ( - _input: string | URL | Request, + input: string | URL | Request, init?: RequestInit, ) => { - calls += 1; - assert.match( - String(new Headers(init?.headers).get('authorization')), - /^Bearer eyJ/, - ); - return new Response( - JSON.stringify({ + const url = String(input); + const authorization = new Headers(init?.headers).get('authorization'); + calls.push({ url, authorization }); + if (url.endsWith('/app')) { + assert.match(String(authorization), /^Bearer eyJ/); + return Response.json({ slug: 'lia' }); + } + if (url.endsWith('/app/installations/456/access_tokens')) { + assert.match(String(authorization), /^Bearer eyJ/); + return Response.json({ token: 'ghs_abcdefghijklmnopqrstuvwxyz', expires_at: '2030-01-01T01:00:00Z', - }), - { status: 201 }, - ); + }, { status: 201 }); + } + if (url.endsWith('/users/lia%5Bbot%5D')) { + assert.equal(authorization, 'Bearer ghs_abcdefghijklmnopqrstuvwxyz'); + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); + } + return Response.json({}, { status: 404 }); }; const provider = new GitHubAppCredentialProvider({ appId: '123', @@ -138,7 +145,15 @@ test('mints and caches a short-lived GitHub App installation token', async (t) = (await provider.getCredential()).value, 'ghs_abcdefghijklmnopqrstuvwxyz', ); - assert.equal(calls, 1); + assert.equal(calls.filter(call => call.url.endsWith('/app')).length, 1); + assert.equal( + calls.filter(call => call.url.endsWith('/access_tokens')).length, + 1, + ); + assert.equal( + calls.filter(call => call.url.endsWith('/users/lia%5Bbot%5D')).length, + 1, + ); }); test('routes and scopes GitHub App tokens per repository installation', async (t) => { @@ -767,16 +782,26 @@ test('App JWT requests use the resolved public or enterprise endpoint', async (t now: () => new Date('2030-01-01T00:00:00Z'), fetch: async (input, init) => { calls++; - assert.equal(String(input), `${expected}/app/installations/456/access_tokens`); - assert.equal(init?.method, 'POST'); + const url = String(input); assert.equal(init?.redirect, 'error'); - assert.match(new Headers(init?.headers).get('authorization')!, /^Bearer eyJ/); - return new Response(JSON.stringify({ token: 'ghs_abcdefghijklmnopqrstuvwxyz', expires_at: '2030-01-01T01:00:00Z' }), { status: 201 }); + const authorization = new Headers(init?.headers).get('authorization'); + if (url === `${expected}/app`) { + assert.match(authorization!, /^Bearer eyJ/); + return Response.json({ slug: 'lia' }); + } + if (url === `${expected}/app/installations/456/access_tokens`) { + assert.equal(init?.method, 'POST'); + assert.match(authorization!, /^Bearer eyJ/); + return new Response(JSON.stringify({ token: 'ghs_abcdefghijklmnopqrstuvwxyz', expires_at: '2030-01-01T01:00:00Z' }), { status: 201 }); + } + assert.equal(url, `${expected}/users/lia%5Bbot%5D`); + assert.equal(authorization, 'Bearer ghs_abcdefghijklmnopqrstuvwxyz'); + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); }, }); await provider.getCredential(); await provider.getCredential(); - assert.equal(calls, 1); + assert.equal(calls, 3); } }); diff --git a/packages/code/src/github.ts b/packages/code/src/github.ts index 4c79fa37..fb6b6fdc 100644 --- a/packages/code/src/github.ts +++ b/packages/code/src/github.ts @@ -217,6 +217,8 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { private readonly cached = new Map(); private readonly inFlight = new Map>(); private readonly installationIds = new Map(); + private appLogin?: string; + private appLoginInFlight?: Promise; private actor?: GitHubCredential['actor']; private actorInFlight?: Promise>; private readonly apiUrl: string; @@ -280,12 +282,12 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { }); } - private async resolveActor( + private async resolveAppLogin( jwt: string, signal?: AbortSignal, - ): Promise> { - if (this.actor) return this.actor; - if (!this.actorInFlight) { + ): Promise { + if (this.appLogin) return this.appLogin; + if (!this.appLoginInFlight) { const pending = (async () => { const sharedSignal = AbortSignal.timeout( GITHUB_SHARED_REQUEST_TIMEOUT_MS, @@ -303,10 +305,41 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { ) { throw new Error('GitHub App identity response is invalid'); } - const login = `${app.slug}[bot]`; + return `${app.slug}[bot]`; + })(); + this.appLoginInFlight = pending; + void pending.then( + login => { + this.appLogin = login; + if (this.appLoginInFlight === pending) { + this.appLoginInFlight = undefined; + } + }, + () => { + if (this.appLoginInFlight === pending) { + this.appLoginInFlight = undefined; + } + }, + ); + } + return waitForShared(this.appLoginInFlight, signal); + } + + private async resolveActor( + jwt: string, + installationToken: string, + signal?: AbortSignal, + ): Promise> { + if (this.actor) return this.actor; + if (!this.actorInFlight) { + const pending = (async () => { + const sharedSignal = AbortSignal.timeout( + GITHUB_SHARED_REQUEST_TIMEOUT_MS, + ); + const login = await this.resolveAppLogin(jwt, sharedSignal); const userResponse = await this.request( `/users/${encodeURIComponent(login)}`, - jwt, + installationToken, sharedSignal, ); if (!userResponse.ok) { @@ -349,7 +382,7 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { async validate(signal?: AbortSignal): Promise { const now = (this.options.now ?? (() => new Date()))(); const jwt = await this.appJwt(now); - await this.resolveActor(jwt, signal); + await this.resolveAppLogin(jwt, signal); if (this.options.installationId) { await this.getCredential(signal); } @@ -477,10 +510,15 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { ) { throw new Error('GitHub App token expiry is invalid'); } + const actor = await this.resolveActor( + jwt, + body.token, + sharedSignal, + ); const credential = { value: body.token, expiresAt, - ...(this.actor ? { actor: this.actor } : {}), + actor, }; this.cached.set(key, credential); return credential; From 277fa7742d383eb1b6606ec228cdafe36af043a4 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 19 Sep 2026 14:43:11 -0400 Subject: [PATCH 04/48] feat: Advertise Workspace Command Timeout Ceiling (#238) --- packages/code/src/protocol.ts | 2 ++ service/src/bridge/index.ts | 1 + service/src/bridge/router.test.ts | 54 ++++++++++++++++++++++++++++++- service/src/bridge/router.ts | 15 +++++++++ 4 files changed, 71 insertions(+), 1 deletion(-) diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index c5353ae2..27220735 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -712,6 +712,8 @@ export interface BridgeWorkerStatusResponse { online: boolean; ready: boolean; leaseExpiresInMs?: number; + /** Server-owned execution ceiling for workspace commands. Omitted by legacy servers. */ + maxCommandTimeoutMs?: number; capabilities?: BridgeWorkerCapabilities; } diff --git a/service/src/bridge/index.ts b/service/src/bridge/index.ts index a1ec5d78..16857fad 100644 --- a/service/src/bridge/index.ts +++ b/service/src/bridge/index.ts @@ -21,4 +21,5 @@ export default createBridgeRouter({ adminToken: env.BRIDGE_TOKEN, configuredWorkerId: env.BRIDGE_WORKER_ID, allowDynamicWorkers: env.BRIDGE_DYNAMIC_WORKERS, + maxCommandTimeoutMs: env.JOB_TIMEOUT, }); diff --git a/service/src/bridge/router.test.ts b/service/src/bridge/router.test.ts index af0e765c..63640111 100644 --- a/service/src/bridge/router.test.ts +++ b/service/src/bridge/router.test.ts @@ -10,7 +10,10 @@ import { createBridgeIdentity, signBridgeRequest, } from '../../../packages/code/src/identity'; -import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; +import { + BRIDGE_PROTOCOL_VERSION, + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, +} from '../../../packages/code/src/protocol'; import { RedisBridgePairingStore } from './pairing'; import { createBridgeRouter } from './router'; import { RedisBridgeStore } from './store'; @@ -115,6 +118,55 @@ describe('paired bridge HTTP API', () => { expect(unauthorized.status).toBe(401); }); + test('advertises the effective server command timeout for command-capable workers', async () => { + const store = new RedisBridgeStore(redis); + const app = express(); + app.use(json()); + app.use( + '/v1/bridge', + createBridgeRouter({ + store, + pairings: new RedisBridgePairingStore(redis), + authMode: 'static', + adminToken: 'strong-administrator-bootstrap-token', + configuredWorkerId: 'command-worker', + maxCommandTimeoutMs: 900_000, + }), + ); + server = createServer(app); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') { + throw new Error('Expected TCP listener'); + } + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'command-worker', + incarnationId: 'incarnation-00000001', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['execute_command'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + + const response = await fetch( + `http://127.0.0.1:${address.port}/v1/bridge/workers/command-worker/status`, + { headers: { Authorization: 'Bearer strong-administrator-bootstrap-token' } }, + ); + + expect(response.status).toBe(200); + await expect(response.json()).resolves.toMatchObject({ + workerId: 'command-worker', + maxCommandTimeoutMs: BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, + }); + }); + test('rejects a malformed optional binding for a configured worker', async () => { const app = express(); app.use(json()); diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 369b306c..73870772 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -7,6 +7,7 @@ import type { BridgePrincipalType, BridgeWorkerBinding } from './pairing'; import type { CodeBridgeAssignment, CodeBridgeSettlement } from './store'; import { + BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, BRIDGE_PROTOCOL_VERSION, isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, @@ -36,6 +37,7 @@ export interface BridgeRouterOptions { adminToken: string; configuredWorkerId?: string; allowDynamicWorkers?: boolean; + maxCommandTimeoutMs?: number; } function sameToken(left: string, right: string): boolean { @@ -132,6 +134,16 @@ function isSettlement(value: unknown): value is CodeBridgeSettlement { export function createBridgeRouter(options: BridgeRouterOptions): Router { const router = Router(); if (options.enabled === false) return router; + if ( + options.maxCommandTimeoutMs !== undefined && + (!Number.isSafeInteger(options.maxCommandTimeoutMs) || options.maxCommandTimeoutMs < 1) + ) { + throw new RangeError('Workspace command timeout must be a positive safe integer'); + } + const maxCommandTimeoutMs = + options.maxCommandTimeoutMs == null + ? undefined + : Math.min(options.maxCommandTimeoutMs, BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS); const configuredWorker = (workerId: string): boolean => options.allowDynamicWorkers === true || @@ -324,10 +336,13 @@ export function createBridgeRouter(options: BridgeRouterOptions): Router { return; } const status = await options.store.workerStatus(workerId); + const supportsCommands = + status.capabilities?.workspaceTools?.operations.includes('execute_command') === true; res.json({ protocolVersion: BRIDGE_PROTOCOL_VERSION, workerId, ...status, + ...(supportsCommands && maxCommandTimeoutMs != null ? { maxCommandTimeoutMs } : {}), }); }), ); From 95ebbd3cca39948c42e92d6c8228ace6a2f44298 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sat, 19 Sep 2026 21:42:37 -0400 Subject: [PATCH 05/48] =?UTF-8?q?=F0=9F=8C=B3=20feat:=20Provision=20Conver?= =?UTF-8?q?sation-Scoped=20Code=20Worktrees=20(#239)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: provision conversation-scoped code worktrees * fix: isolate conversation checkout metadata * docs: clarify isolated conversation checkouts * fix: revalidate conversation checkout sources * fix: preserve synchronous legacy execution startup * fix: harden conversation worktree lifecycle * test: use canonical workspace isolation keys * fix: secure conversation worktree provisioning * fix: preserve isolated workspace lifecycle * fix: harden conversation worktree provisioning * fix: fence worktree setup and credential routing * fix: use kernel-backed provisioning locks * fix: load worktree locking only when provisioned * fix: retain conversation provisioning ownership through recovery * fix: reserve provisioning before launching checkout writers * fix: pin Git provisioning inputs and close instance admission gaps --- packages/code/README.md | 59 +- packages/code/src/cli.ts | 190 +++- packages/code/src/git-snapshot.ts | 343 ++++++ packages/code/src/native-pool.test.ts | 120 ++- packages/code/src/native-pool.ts | 106 +- packages/code/src/native-process.test.ts | 1 + packages/code/src/native-sandbox.test.ts | 31 + packages/code/src/process-lock.test.ts | 61 ++ packages/code/src/process-lock.ts | 59 ++ packages/code/src/protocol.test.ts | 42 +- packages/code/src/protocol.ts | 42 +- packages/code/src/root-identity.ts | 16 + packages/code/src/worker-slots.test.ts | 35 +- packages/code/src/worker.ts | 163 ++- packages/code/src/workspace-cli.test.ts | 32 + packages/code/src/workspace-instances.test.ts | 234 +++++ packages/code/src/workspace-instances.ts | 232 ++++ packages/code/src/workspace-worker.test.ts | 100 ++ packages/code/src/worktrees.test.ts | 988 ++++++++++++++++++ packages/code/src/worktrees.ts | 718 +++++++++++++ service/src/bridge/concurrent-store.test.ts | 69 +- service/src/bridge/router.ts | 1 + service/src/bridge/store.test.ts | 12 +- service/src/bridge/store.ts | 55 +- service/src/bridge/workspace-instance.test.ts | 40 + service/src/bridge/workspace-instance.ts | 17 + service/src/service/programmatic-router.ts | 34 +- .../src/service/programmatic-state.test.ts | 16 + service/src/service/programmatic-state.ts | 12 + service/src/service/replay-state.ts | 2 + service/src/types/service.ts | 4 + service/src/workspace-tools/router.test.ts | 31 + service/src/workspace-tools/router.ts | 19 +- 33 files changed, 3803 insertions(+), 81 deletions(-) create mode 100644 packages/code/src/git-snapshot.ts create mode 100644 packages/code/src/process-lock.test.ts create mode 100644 packages/code/src/process-lock.ts create mode 100644 packages/code/src/workspace-instances.test.ts create mode 100644 packages/code/src/workspace-instances.ts create mode 100644 packages/code/src/worktrees.test.ts create mode 100644 packages/code/src/worktrees.ts create mode 100644 service/src/bridge/workspace-instance.test.ts create mode 100644 service/src/bridge/workspace-instance.ts diff --git a/packages/code/README.md b/packages/code/README.md index 69d85b16..e30b80cb 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -706,8 +706,63 @@ Slots are per machine, not a fleet-wide execution limit. A busy machine does not consume another machine's slots. Requests for the same root remain serialized, including commands started through background tools. Independent checkouts can use different slots; selecting subdirectories beneath one registered parent root -does not create separate scheduling boundaries. Linked Git worktrees share Git -metadata and are not supported by selected-project registration. +does not create separate scheduling boundaries. + +To bind each conversation to an isolated checkout of the selected Git +repository, configure worker-owned conversation worktrees: + +```sh +librechat-code run \ + --worker-dir /projects/LibreChat \ + --workspace-lease-slots 4 \ + --conversation-worktree-root /var/lib/librechat-code/worktrees \ + --conversation-worktree-max 64 \ + --conversation-worktree-clone-timeout-ms 300000 \ + --allow-workspace-writes \ + --allow-workspace-commands +``` + +`LIBRECHAT_CODE_CONVERSATION_WORKTREE_ROOT` and +`LIBRECHAT_CODE_CONVERSATION_WORKTREE_MAX` are the environment equivalents; +`LIBRECHAT_CODE_CONVERSATION_WORKTREE_CLONE_TIMEOUT_MS` controls the bounded +clone budget (five minutes by default, from 30 seconds through 30 minutes). +The storage root must be owner-controlled, must not overlap a registered +workspace, and every registered source must be a Git repository. The worker +creates a deterministic branch in an isolated local checkout for the opaque +conversation identity supplied by LibreChat. Each checkout owns its writable +Git metadata and object storage, without alternates or hardlinks to the source. +Provisioning pins the source Git-directory and object-store identities. It copies +Git data through no-follow, descriptor-relative reads into private staging before +running Git; source hooks and config includes are not used. The clone budget +also bounds this snapshot. Local hardlinks only connect private staging to its +new checkout, never to the source; staging is removed before setup. Source +alternates admitted at worker startup are materialized into independent objects. +Git metadata replacement requires operator recovery, not automatic re-admission. +Host paths remain private. The configured count +is a hard per-machine quota, provisioning is serialized, and operations for one +conversation remain serialized while different conversations may occupy +different lease slots. Recognizable abandoned checkouts without a lifecycle +record are discarded before admission. New provisioning reserves its record +before starting Git or setup; a worker crash leaves that checkout reserved for +operator recovery because child processes might still be running. +Reservations count even when a crash happens before a checkout directory exists. + +Cancellation also covers waiting for the provisioning lock, cloning, and setup. +The worker waits for setup cleanup before releasing the assignment. If cleanup +cannot be confirmed, the checkout stays reserved and fails closed on restart. +A completed checkout with a changed source identity or invalid completion record +is preserved for operator recovery, including any uncommitted work. After stopping +the worker and confirming no executor still uses the checkout, an operator can +archive the affected checkout and its adjacent `.complete` record before retrying. +Also archive any adjacent `.source` staging directory. Pre-release version-1 +completion records are deliberately preserved but not admitted by this version; +they do not contain the required source Git identity binding. + +GitHub App routing is inherited from the operator-admitted source repository; +commands cannot select a different installation by rewriting a worktree remote. +Legacy requests without a conversation identity continue to use the selected +source root. Older Code API deployments do not negotiate the capability, so the +worker omits it until every request path understands the isolation boundary. Admission waits at most 30 seconds. A `WORKSPACE_QUEUE_TIMEOUT` response (HTTP 503, `Retry-After: 1`) means the operation was not assigned or started; wait for diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 96757d57..26c60e9a 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -36,6 +36,9 @@ import { import { RuntimeWorkspaceCommandSandbox } from './workspace-runtime.js'; import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; import { NativeWorkspaceCommandPool } from './native-pool.js'; +import { GitWorktreeWorkspaceTools, internalWorkspaceId } from './workspace-instances.js'; +import { GitWorktreeManager } from './worktrees.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; import { resolveNativeSrtCommandPolicy, serializeNativeSrtCommandPolicy, @@ -61,8 +64,9 @@ import type { WorkspaceToolExecutor } from './workspace.js'; import { BRIDGE_WORKSPACE_NAME_MAX_LENGTH, BridgeProtocolError, - isValidBridgeWorkerCapabilities, - isValidBridgeWorkerId, + isValidBridgeWorkerCapabilities, + isValidBridgeWorkerId, + workspaceIsolationKey, } from './protocol.js'; function workspaceSecurityIdentity( @@ -600,6 +604,32 @@ async function run( ); if (workspaceLeaseSlots > 8) throw new Error('Workspace lease slots cannot exceed 8'); + const conversationWorktreeRoot = + option(args, '--conversation-worktree-root') ?? + process.env.LIBRECHAT_CODE_CONVERSATION_WORKTREE_ROOT?.trim(); + const conversationWorktreeMax = positiveInteger( + 'LIBRECHAT_CODE_CONVERSATION_WORKTREE_MAX', + option(args, '--conversation-worktree-max') ?? + process.env.LIBRECHAT_CODE_CONVERSATION_WORKTREE_MAX, + 64, + ); + if (conversationWorktreeMax > 1024) { + throw new Error('Conversation worktree capacity cannot exceed 1024'); + } + const conversationWorktreeCloneTimeoutMs = positiveInteger( + 'LIBRECHAT_CODE_CONVERSATION_WORKTREE_CLONE_TIMEOUT_MS', + option(args, '--conversation-worktree-clone-timeout-ms') ?? + process.env.LIBRECHAT_CODE_CONVERSATION_WORKTREE_CLONE_TIMEOUT_MS, + 5 * 60_000, + ); + if ( + conversationWorktreeCloneTimeoutMs < 30_000 || + conversationWorktreeCloneTimeoutMs > 30 * 60_000 + ) { + throw new Error( + 'Conversation worktree clone timeout must be between 30000 and 1800000 milliseconds', + ); + } const roots: LocalWorkspaceConfig[] = canonicalWorkerDirectory ? [ { @@ -701,6 +731,16 @@ async function run( 'Concurrent workspace leases require native-srt commands', ); } + if ( + conversationWorktreeRoot && + (!allowWorkspaceCommands || + commandSandboxMode !== 'native-srt' || + workspaceLeaseSlots < 2) + ) { + throw new Error( + 'Conversation worktrees require native-srt commands and at least two workspace lease slots', + ); + } if ( roots.length > 1 && process.env.LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE?.trim() @@ -732,6 +772,14 @@ async function run( ), ) : undefined; + const repositoriesByWorkspace = admittedGitHubRepositories + ? new Map( + roots.map((root) => [ + root.id, + admittedGitHubRepositories.get(root.root), + ]), + ) + : undefined; const localWorkspaceTools = workerDirectory ? await LocalWorkspaceTools.create({ workspaces: roots, @@ -981,7 +1029,7 @@ async function run( }; const nativeCommandSandbox = allowWorkspaceCommands && commandSandboxMode === 'native-srt' - ? roots.length > 1 || workspaceLeaseSlots > 1 + ? roots.length > 1 || workspaceLeaseSlots > 1 || conversationWorktreeRoot ? new NativeWorkspaceCommandPool( new Map( roots.map(root => [ @@ -1008,6 +1056,105 @@ async function run( incarnationId, }), }); + } + const conversationWorktrees = conversationWorktreeRoot + ? new GitWorktreeManager({ + cloneTimeoutMs: conversationWorktreeCloneTimeoutMs, + maxCount: conversationWorktreeMax, + root: conversationWorktreeRoot, + ...(nativeCommandSandbox instanceof NativeWorkspaceCommandPool + ? { + prepareInstance: async (instance, signal) => { + const setup = environments.find( + (environment) => + environment.definition.name === instance.sourceWorkspaceId, + )?.definition.setup; + if (!setup) return; + if (admittedGitHubRepositories) { + admittedGitHubRepositories.set( + instance.root, + repositoriesByWorkspace?.get(instance.sourceWorkspaceId), + ); + } + const id = internalWorkspaceId(instance.sourceWorkspaceId, instance.id); + await nativeCommandSandbox.registerRoot(id, { + ...nativeOptions, + workspaceIdentity: instance.identity, + workspaceRoot: instance.root, + }); + const result = await nativeCommandSandbox.execute( + { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: id, + command: setup.command, + timeoutMs: setup.timeoutMs, + maxOutputBytes: 8192, + }, + signal, + ); + if (result.exitCode !== 0 || result.timedOut) { + throw new Error( + `Environment ${instance.sourceWorkspaceId} setup failed for its conversation worktree`, + ); + } + }, + discardInstance: async (instance) => { + await nativeCommandSandbox.unregisterRoot( + internalWorkspaceId(instance.sourceWorkspaceId, instance.id), + ); + admittedGitHubRepositories?.delete(instance.root); + }, + } + : {}), + sources: new Map( + await Promise.all( + roots.map(async (root) => [ + root.id, + { + root: root.root, + identity: + root.identity ?? + (await captureWorkspaceRootIdentity(root.root)), + }, + ] as const), + ), + ), + }) + : undefined; + let conversationWorkspaceTools: GitWorktreeWorkspaceTools | undefined; + if (conversationWorktrees && workspaceTools) { + if (!(nativeCommandSandbox instanceof NativeWorkspaceCommandPool)) { + throw new Error('Conversation worktrees require a native command pool'); + } + conversationWorkspaceTools = new GitWorktreeWorkspaceTools({ + commandPool: nativeCommandSandbox, + delegate: workspaceTools, + manager: conversationWorktrees, + onResolve(workspaceId, root) { + if (admittedGitHubRepositories) { + admittedGitHubRepositories.set( + root, + repositoriesByWorkspace?.get(workspaceId), + ); + } + }, + sources: new Map( + roots.map((root) => [ + root.id, + { + command: { + ...nativeOptions, + workspaceIdentity: root.identity, + workspaceRoot: root.root, + }, + repositoryInstructions: args.includes('--repository-instructions'), + writable: root.writable ?? false, + }, + ]), + ), + }); + workspaceTools = conversationWorkspaceTools; } if (workspaceTools && environments.length) { workspaceTools = new EnvironmentWorkspaceTools( @@ -1059,6 +1206,7 @@ async function run( if (github.provider && !github.provider.validate) { await github.provider.getCredential(controller.signal); } + await conversationWorktrees?.prepare(); await nativeCommandSandbox?.prepare(); for (const environment of option(args, '--reset-workspace-quarantine') == null ? environments : []) { const setup = environment.definition.setup; @@ -1110,7 +1258,34 @@ async function run( capabilities, workspaceTools, ...(nativeProgrammaticEnabled && nativeCommandSandbox - ? { workspaceProgrammatic: nativeCommandSandbox } + ? { + workspaceProgrammatic: + conversationWorkspaceTools ?? nativeCommandSandbox, + } + : {}), + ...(conversationWorktrees + ? { + workspaceQuarantineResolver: async ( + selectedWorkspaceId: string, + workspaceInstanceId: string, + ) => + workspaceMutationGuard( + defaultWorkspaceQuarantinePath({ + codeApiUrl, + workerId, + workspaceRoot: await conversationWorktrees.plannedRoot( + selectedWorkspaceId, + workspaceInstanceId, + ), + }), + workerId, + workspaceIsolationKey( + selectedWorkspaceId, + workspaceInstanceId, + ), + incarnationId, + ), + } : {}), ...(workspaceLeaseSlots > 1 || roots.length > 1 ? { @@ -1222,14 +1397,19 @@ async function run( } const resetNativeRoot = option(args, '--reset-workspace-quarantine'); if (resetNativeRoot != null) { + const resetWorkspaceInstance = option( + args, + '--reset-workspace-instance', + ); await worker.refreshCredential(controller.signal); await worker.registerForMaintenance(controller.signal); await worker.resetNativeWorkspace( resetNativeRoot, controller.signal, + resetWorkspaceInstance, ); process.stdout.write( - `librechat-code: reset acknowledged for native workspace ${resetNativeRoot}\n`, + `librechat-code: reset acknowledged for native workspace ${resetNativeRoot}${resetWorkspaceInstance ? ` instance ${resetWorkspaceInstance}` : ''}\n`, ); return; } diff --git a/packages/code/src/git-snapshot.ts b/packages/code/src/git-snapshot.ts new file mode 100644 index 00000000..7c57beda --- /dev/null +++ b/packages/code/src/git-snapshot.ts @@ -0,0 +1,343 @@ +import { constants, close, fstat, read } from 'node:fs'; +import { mkdir, open, opendir, writeFile } from 'node:fs/promises'; +import { createHash } from 'node:crypto'; +import { join, resolve } from 'node:path'; +import { promisify } from 'node:util'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; + +const closeFd = promisify(close); +const statFd = promisify(fstat); +const readFd = promisify(read); +let binding: + | Promise<{ + openat: (fd: number, name: string, flags: number) => number; + errno: () => number; + }> + | undefined; + +async function childFd( + parent: number, + name: string +): Promise { + if (!name || name === '.' || name === '..' || name.includes('/')) + throw new Error('Invalid Git metadata entry'); + binding ??= import('koffi').then(({ default: koffi }) => ({ + openat: koffi + .load(null) + .func('int openat(int dirfd, const char *path, int flags)'), + errno: () => koffi.errno(), + })); + const native = await binding; + // Node does not expose O_CLOEXEC. Set it atomically with openat so unrelated + // concurrent executor spawns cannot inherit privileged source descriptors. + if (process.platform !== 'darwin' && process.platform !== 'linux') + throw new Error('Git snapshots require a POSIX host'); + const closeOnExec = process.platform === 'darwin' ? 0x1000000 : 0x80000; + const fd = native.openat( + parent, + name, + constants.O_RDONLY | + constants.O_NOFOLLOW | + constants.O_NONBLOCK | + closeOnExec + ); + if (fd >= 0) return fd; + if (native.errno() === 2) return undefined; // ENOENT on supported POSIX hosts + throw new Error('Git metadata entry is unavailable or is a symbolic link'); +} + +async function withDirectory( + identity: WorkspaceRootIdentity, + operation: (fd: number) => Promise +): Promise { + const handle = await open( + identity.path, + constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW + ); + try { + const current = await handle.stat({ bigint: true }); + if ( + current.dev.toString() !== identity.dev || + current.ino.toString() !== identity.ino + ) { + throw new Error('Source Git metadata changed after admission'); + } + return await operation(handle.fd); + } finally { + await handle.close(); + } +} + +async function textAt( + parent: number, + name: string, + limit = 16 * 1024 +): Promise { + const fd = await childFd(parent, name); + if (fd == null) return undefined; + try { + const metadata = await statFd(fd); + if (!metadata.isFile() || metadata.size > limit) + throw new Error('Invalid Git metadata file'); + const buffer = Buffer.alloc(metadata.size); + let offset = 0; + while (offset < buffer.length) { + const { bytesRead } = await readFd( + fd, + buffer, + offset, + buffer.length - offset, + offset + ); + if (!bytesRead) throw new Error('Git metadata changed during snapshot'); + offset += bytesRead; + } + return buffer.toString('utf8'); + } finally { + await closeFd(fd); + } +} + +async function alternatesAt(parent: number): Promise { + const fd = await childFd(parent, 'info'); + if (fd == null) return undefined; + try { + if (!(await statFd(fd)).isDirectory()) + throw new Error('Invalid Git objects info directory'); + return await textAt(fd, 'alternates'); + } finally { + await closeFd(fd); + } +} + +/** Copies only regular files/directories through descriptor-relative, no-follow opens. + * Renaming a parent or replacing a child with a symlink never expands the read grant. + * No source config, hooks, object alternates, or executable helpers reach Git. + */ +async function copyEntry( + parent: number, + name: string, + destination: string, + signal: AbortSignal | undefined, + depth = 0 +): Promise { + signal?.throwIfAborted(); + if (depth > 64) + throw new Error('Git metadata nesting exceeds snapshot limit'); + const fd = await childFd(parent, name); + if (fd == null) return; + try { + const metadata = await statFd(fd); + if (metadata.isDirectory()) { + await mkdir(destination, { recursive: true, mode: 0o700 }); + const directory = await opendir(`/dev/fd/${fd}`); + for await (const entry of directory) { + await copyEntry( + fd, + entry.name, + join(destination, entry.name), + signal, + depth + 1 + ); + } + } else if (metadata.isFile()) { + const target = await open(destination, 'w', 0o600); + try { + const buffer = Buffer.alloc(128 * 1024); + let offset = 0; + while (offset < metadata.size) { + signal?.throwIfAborted(); + const { bytesRead } = await readFd( + fd, + buffer, + 0, + Math.min(buffer.length, metadata.size - offset), + offset + ); + if (!bytesRead) + throw new Error('Git metadata changed during snapshot'); + await target.writeFile(buffer.subarray(0, bytesRead)); + offset += bytesRead; + } + } finally { + await target.close(); + } + } else { + throw new Error('Git snapshot requires regular files and directories'); + } + } finally { + await closeFd(fd); + } +} + +export class GitSourceSnapshot { + private constructor( + private readonly source: WorkspaceRootIdentity, + private readonly gitDirectory: WorkspaceRootIdentity, + private readonly common: WorkspaceRootIdentity, + private readonly gitfile: string | undefined, + private readonly commondir: string | undefined, + private readonly objects: Array<{ + identity: WorkspaceRootIdentity; + alternates: string | undefined; + }> + ) {} + + get fingerprint(): string { + return createHash('sha256') + .update( + JSON.stringify([ + this.gitDirectory, + this.common, + this.gitfile, + this.commondir, + this.objects, + ]) + ) + .digest('hex'); + } + + static async admit( + source: WorkspaceRootIdentity + ): Promise { + let gitfile: string | undefined; + await withDirectory(source, async (fd) => { + const git = await childFd(fd, '.git'); + if (git == null) + throw new Error('Source workspace is not a Git repository'); + try { + if (!(await statFd(git)).isDirectory()) + gitfile = await textAt(fd, '.git'); + } finally { + await closeFd(git); + } + }); + if (gitfile != null && !/^gitdir: .+\n?$/.test(gitfile)) + throw new Error('Invalid source Git directory pointer'); + const gitDirectory = await captureWorkspaceRootIdentity( + gitfile == null + ? join(source.path, '.git') + : resolve(source.path, gitfile.slice(8).trim()) + ); + const commondir = await withDirectory(gitDirectory, (fd) => + textAt(fd, 'commondir') + ); + const common = + commondir == null + ? gitDirectory + : await captureWorkspaceRootIdentity( + resolve(gitDirectory.path, commondir.trim()) + ); + const objects: Array<{ + identity: WorkspaceRootIdentity; + alternates: string | undefined; + }> = []; + const visit = async (path: string): Promise => { + const identity = await captureWorkspaceRootIdentity(path); + if (objects.some((entry) => entry.identity.path === identity.path)) + return; + if (objects.length >= 32) + throw new Error('Too many source Git object stores'); + const alternates = await withDirectory(identity, alternatesAt); + objects.push({ identity, alternates }); + for (const alternate of alternates?.split('\n').filter(Boolean) ?? []) { + if (alternate.startsWith('"')) + throw new Error('Quoted Git alternate paths are unsupported'); + await visit(resolve(identity.path, alternate)); + } + }; + await visit(join(common.path, 'objects')); + const admitted = new GitSourceSnapshot( + source, + gitDirectory, + common, + gitfile, + commondir, + objects + ); + await admitted.validate(); + return admitted; + } + + async validate(): Promise { + await withDirectory(this.source, async (fd) => { + if (this.gitfile != null) { + if ((await textAt(fd, '.git')) !== this.gitfile) + throw new Error('Source Git metadata changed after admission'); + } else { + // Compare the directory reached from the admitted root, not just its name. + const child = await childFd(fd, '.git'); + if (child == null) + throw new Error('Source Git metadata changed after admission'); + try { + const current = await promisify(fstat)(child, { + bigint: true, + }); + if ( + current.dev.toString() !== this.gitDirectory.dev || + current.ino.toString() !== this.gitDirectory.ino + ) { + throw new Error('Source Git metadata changed after admission'); + } + } finally { + await closeFd(child); + } + } + }); + await withDirectory(this.gitDirectory, async (fd) => { + if ((await textAt(fd, 'commondir')) !== this.commondir) + throw new Error('Source Git metadata changed after admission'); + }); + await withDirectory(this.common, async () => {}); + for (const entry of this.objects) { + await withDirectory(entry.identity, async (fd) => { + if ((await alternatesAt(fd)) !== entry.alternates) + throw new Error('Source Git alternates changed after admission'); + }); + } + } + + async copyTo(destination: string, signal?: AbortSignal): Promise { + await this.validate(); + await mkdir(destination, { mode: 0o700 }); + await mkdir(join(destination, 'objects'), { mode: 0o700 }); + await mkdir(join(destination, 'refs'), { mode: 0o700 }); + await withDirectory(this.gitDirectory, (fd) => + copyEntry(fd, 'HEAD', join(destination, 'HEAD'), signal) + ); + await withDirectory(this.common, async (fd) => { + for (const name of ['refs', 'packed-refs', 'shallow']) + await copyEntry(fd, name, join(destination, name), signal); + // Kept outside Git's config name; caller may query origin with --no-includes. + const config = await textAt(fd, 'config', 1024 * 1024); + if (config != null) + await writeFile(join(destination, 'source-config'), config, { + mode: 0o600, + }); + }); + for (const entry of [...this.objects].reverse()) { + await withDirectory(entry.identity, async (fd) => { + const directory = await opendir(`/dev/fd/${fd}`); + for await (const child of directory) { + // Object info (notably alternates) never crosses into private staging. + if (child.name === 'pack' || /^[a-f0-9]{2}$/.test(child.name)) { + await copyEntry( + fd, + child.name, + join(destination, 'objects', child.name), + signal + ); + } + } + }); + } + await writeFile( + join(destination, 'config'), + '[core]\nrepositoryformatversion = 0\nbare = true\n', + { mode: 0o600 } + ); + await this.validate(); + signal?.throwIfAborted(); + } +} diff --git a/packages/code/src/native-pool.test.ts b/packages/code/src/native-pool.test.ts index cad30a52..ce25caa4 100644 --- a/packages/code/src/native-pool.test.ts +++ b/packages/code/src/native-pool.test.ts @@ -5,7 +5,7 @@ import { WorkspaceToolError } from './workspace.js'; import type { WorkspaceExecuteCommandRequest } from './protocol.js'; const roots = new Map( - ['a', 'b', 'c'].map((id) => [id, { workspaceRoot: `/fixture/${id}` }]), + ['a', 'b', 'c'].map((id) => [id, { workspaceRoot: `/fixture/${id}` }]) ); const request = (workspaceId: string): WorkspaceExecuteCommandRequest => ({ protocolVersion: 1, @@ -14,6 +14,36 @@ const request = (workspaceId: string): WorkspaceExecuteCommandRequest => ({ command: 'fixture', }); +test('failed provisioning roots are removed only after confirmed executor cleanup', async () => { + let failClose = true; + const pool = new NativeWorkspaceCommandPool(roots, 2, () => ({ + async prepare() {}, + async execute() { + throw new Error('not used'); + }, + async close() { + if (failClose) throw new Error('cleanup unconfirmed'); + }, + })); + await pool.registerRoot('instance', { workspaceRoot: '/fixture/instance' }); + // Allocate this executor without dispatching a command. + const controller = new AbortController(); + controller.abort(); + await assert.rejects( + pool.execute(request('instance'), controller.signal), + /cancelled/ + ); + await assert.rejects(pool.unregisterRoot('instance'), /cleanup unconfirmed/); + failClose = false; + await pool.unregisterRoot('instance'); + await assert.rejects(pool.execute(request('instance')), /unavailable/); + await pool.close(); + await assert.rejects( + pool.registerRoot('new', { workspaceRoot: '/fixture/new' }), + /unavailable/ + ); +}); + test('native pool preflights every registered root with bounded concurrency', async () => { const prepared: string[] = []; let active = 0; @@ -22,7 +52,7 @@ test('native pool preflights every registered root with bounded concurrency', as async prepare() { active += 1; peak = Math.max(peak, active); - await new Promise(resolve => setTimeout(resolve, 5)); + await new Promise((resolve) => setTimeout(resolve, 5)); prepared.push(options.workspaceRoot); active -= 1; }, @@ -38,6 +68,90 @@ test('native pool preflights every registered root with bounded concurrency', as await pool.close(); }); +test('native pool admits worker-owned roots after startup', async () => { + const created: string[] = []; + const pool = new NativeWorkspaceCommandPool( + new Map([['primary', { workspaceRoot: '/fixture/primary' }]]), + 2, + (options) => ({ + async prepare() {}, + async close() {}, + async execute(req) { + created.push(options.workspaceRoot); + return { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: req.workspaceId, + stdout: '', + stderr: '', + exitCode: 0, + truncated: false, + timedOut: false, + }; + }, + }) + ); + await pool.registerRoot('conversation', { + workspaceRoot: '/fixture/conversation', + }); + await pool.execute(request('conversation')); + assert.deepEqual(created, ['/fixture/conversation']); + await assert.rejects( + async () => + pool.registerRoot('conversation', { + workspaceRoot: '/fixture/replaced', + }), + { code: 'REGISTRATION_INVALID' } + ); + await pool.close(); +}); + +test('native pool retires a cached executor when a root inode changes', async () => { + let created = 0; + let closed = 0; + const pool = new NativeWorkspaceCommandPool( + new Map([['primary', { workspaceRoot: '/fixture/primary' }]]), + 2, + () => { + created++; + return { + async prepare() {}, + async close() { + closed++; + }, + async execute(req) { + return { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: req.workspaceId, + stdout: '', + stderr: '', + exitCode: 0, + truncated: false, + timedOut: false, + }; + }, + }; + } + ); + const options = (ino: string) => ({ + workspaceRoot: '/fixture/conversation', + workspaceIdentity: { + path: '/fixture/conversation', + dev: '1', + ino, + }, + }); + await pool.registerRoot('conversation', options('1')); + await pool.execute(request('conversation')); + await pool.registerRoot('conversation', options('2')); + await pool.execute(request('conversation')); + + assert.equal(created, 2); + assert.equal(closed, 1); + await pool.close(); +}); + test('a known-clean executor failure is retired without replaying the command', async () => { let created = 0; let executed = 0; @@ -55,7 +169,7 @@ test('a known-clean executor failure is retired without replaying the command', throw new WorkspaceToolError( 'prepare failed', 'COMMAND_UNAVAILABLE', - false, + false ); return { protocolVersion: 1, diff --git a/packages/code/src/native-pool.ts b/packages/code/src/native-pool.ts index 766409b1..59db3e0a 100644 --- a/packages/code/src/native-pool.ts +++ b/packages/code/src/native-pool.ts @@ -12,9 +12,7 @@ interface Entry { NativeProcessWorkspaceCommandSandbox, 'prepare' | 'execute' | 'close' > & - Partial< - Pick - >; + Partial>; busy: boolean; } @@ -27,12 +25,12 @@ export class NativeWorkspaceCommandPool { private allocation: Promise = Promise.resolve(); private closing = false; constructor( - private readonly roots: ReadonlyMap, + roots: ReadonlyMap, private readonly capacity: number, private readonly createSandbox: ( - options: NativeProcessSandboxOptions, + options: NativeProcessSandboxOptions ) => Entry['sandbox'] = (options) => - new NativeProcessWorkspaceCommandSandbox(options), + new NativeProcessWorkspaceCommandSandbox(options) ) { if ( !Number.isSafeInteger(capacity) || @@ -42,6 +40,74 @@ export class NativeWorkspaceCommandPool { ) { throw new Error('Native executor capacity must be between 1 and 8'); } + this.roots = new Map(roots); + } + + private readonly roots: Map; + + /** Confirm executor cleanup before a failed provisioning attempt removes its root. */ + async unregisterRoot(id: string): Promise { + const pending = this.allocation.then(async () => { + const entry = this.entries.get(id); + if (entry?.busy) { + throw new WorkspaceToolError( + 'Native workspace still executing', + 'COMMAND_UNAVAILABLE' + ); + } + if (entry) await entry.sandbox.close(); + this.entries.delete(id); + this.roots.delete(id); + }); + this.allocation = pending.catch(() => undefined); + await pending; + } + + /** Add or safely replace a worker-owned isolated root. */ + async registerRoot( + id: string, + options: NativeProcessSandboxOptions + ): Promise { + const pending = this.allocation.then(async () => { + if (this.closing) { + throw new WorkspaceToolError( + 'Native workspace unavailable', + 'REGISTRATION_INVALID' + ); + } + const existing = this.roots.get(id); + if (!existing) { + this.roots.set(id, options); + return; + } + if (existing.workspaceRoot !== options.workspaceRoot) { + throw new WorkspaceToolError( + 'Native workspace identity changed', + 'REGISTRATION_INVALID' + ); + } + if ( + existing.workspaceIdentity?.dev === options.workspaceIdentity?.dev && + existing.workspaceIdentity?.ino === options.workspaceIdentity?.ino && + existing.workspaceIdentity?.path === options.workspaceIdentity?.path + ) { + return; + } + const entry = this.entries.get(id); + if (entry?.busy) { + throw new WorkspaceToolError( + 'Native workspace changed during execution', + 'REGISTRATION_INVALID' + ); + } + if (entry) { + await entry.sandbox.close(); + this.entries.delete(id); + } + this.roots.set(id, options); + }); + this.allocation = pending.catch(() => undefined); + await pending; } private allocate(root: string): Promise { @@ -50,23 +116,23 @@ export class NativeWorkspaceCommandPool { if (this.closing || !options) throw new WorkspaceToolError( 'Native workspace unavailable', - 'REGISTRATION_INVALID', + 'REGISTRATION_INVALID' ); let entry = this.entries.get(root); if (entry?.busy) throw new WorkspaceToolError( 'Native workspace already executing', - 'COMMAND_UNAVAILABLE', + 'COMMAND_UNAVAILABLE' ); if (!entry) { if (this.entries.size >= this.capacity) { const idle = [...this.entries].find( - ([, candidate]) => !candidate.busy, + ([, candidate]) => !candidate.busy ); if (!idle) throw new WorkspaceToolError( 'Native executor capacity reached', - 'COMMAND_UNAVAILABLE', + 'COMMAND_UNAVAILABLE' ); await idle[1].sandbox.close(); this.entries.delete(idle[0]); @@ -88,7 +154,7 @@ export class NativeWorkspaceCommandPool { if (error instanceof WorkspaceToolError) throw error; throw new WorkspaceToolError( 'Native executor allocation failed', - 'COMMAND_UNAVAILABLE', + 'COMMAND_UNAVAILABLE' ); }); this.allocation = checked.catch(() => undefined); @@ -112,14 +178,14 @@ export class NativeWorkspaceCommandPool { entry.busy = false; } } - }, - ), + } + ) ); } async execute( request: WorkspaceExecuteCommandRequest, - signal?: AbortSignal, + signal?: AbortSignal ): Promise { const entry = await this.allocate(request.workspaceId); let enteredExecutor = false; @@ -127,7 +193,7 @@ export class NativeWorkspaceCommandPool { if (signal?.aborted) throw new WorkspaceToolError( 'Command cancelled before dispatch', - 'EXECUTION_ABORTED', + 'EXECUTION_ABORTED' ); enteredExecutor = true; return await entry.sandbox.execute(request, signal); @@ -156,7 +222,7 @@ export class NativeWorkspaceCommandPool { async executeProgrammatic( workspaceId: string, request: BridgeWorkspaceProgrammaticRequest, - signal?: AbortSignal, + signal?: AbortSignal ): Promise { const entry = await this.allocate(workspaceId); let enteredExecutor = false; @@ -164,19 +230,19 @@ export class NativeWorkspaceCommandPool { if (signal?.aborted) throw new WorkspaceToolError( 'Programmatic execution cancelled before dispatch', - 'EXECUTION_ABORTED', + 'EXECUTION_ABORTED' ); enteredExecutor = true; if (!entry.sandbox.executeProgrammatic) { throw new WorkspaceToolError( 'Native programmatic executor is unavailable', - 'COMMAND_UNAVAILABLE', + 'COMMAND_UNAVAILABLE' ); } return await entry.sandbox.executeProgrammatic( workspaceId, request, - signal, + signal ); } catch (error) { if ( @@ -202,7 +268,7 @@ export class NativeWorkspaceCommandPool { this.closing = true; await this.allocation; const results = await Promise.allSettled( - [...this.entries.values()].map((entry) => entry.sandbox.close()), + [...this.entries.values()].map((entry) => entry.sandbox.close()) ); this.entries.clear(); const errors = results diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts index b3016937..5f7eedbb 100644 --- a/packages/code/src/native-process.test.ts +++ b/packages/code/src/native-process.test.ts @@ -146,6 +146,7 @@ test('executor bootstrap excludes bridge credentials and Node injection variable assert.deepEqual(fake.options?.execArgv, []); assert.deepEqual(fake.options?.env, { PATH: '/bin' }); assert.equal(JSON.stringify(fake.messages).includes('secret'), false); + assert.equal('gitSharedObjectDirectory' in fake.messages[0].options, false); await sandbox.close(); }); diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index b1e4c5df..eab72966 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -590,6 +590,37 @@ test('trusted-vm permits unmatched egress and local development sockets', async ]); }); +test('recreated executors never grant reads through a replaced Git object directory', async t => { + const parent = await mkdtemp(join(tmpdir(), 'librechat-code-worktree-')); + const root = join(parent, 'worktree'); + const gitSharedObjectDirectory = join(parent, 'source.git', 'objects'); + await mkdir(join(root, '.git'), { recursive: true }); + await mkdir(gitSharedObjectDirectory, { recursive: true }); + await symlink(gitSharedObjectDirectory, join(root, '.git', 'objects')); + t.after(() => rm(parent, { recursive: true, force: true })); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fake.manager, + }); + t.after(() => sandbox.close()); + + await sandbox.prepare(); + + assert.equal( + fake.config?.filesystem.allowRead?.includes( + await realpath(gitSharedObjectDirectory), + ), + false, + ); + assert.equal( + fake.config?.filesystem.allowWrite?.includes( + await realpath(gitSharedObjectDirectory), + ), + false, + ); +}); + test('provides an isolated scratch directory to commands and restores the host environment', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); diff --git a/packages/code/src/process-lock.test.ts b/packages/code/src/process-lock.test.ts new file mode 100644 index 00000000..3594a1a9 --- /dev/null +++ b/packages/code/src/process-lock.test.ts @@ -0,0 +1,61 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { withProcessLock } from './process-lock.js'; + +test( + 'kernel lock survives contention and is released when the owning process crashes', + { timeout: 10_000 }, + async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-lock-')); + const path = join(directory, '.provision.lock'); + const moduleUrl = new URL('./process-lock.js', import.meta.url).href; + const child = spawn( + process.execPath, + [ + '--input-type=module', + '-e', + ` + import { withProcessLock } from ${JSON.stringify(moduleUrl)}; + await withProcessLock(${JSON.stringify(path)}, async () => { + process.stdout.write('locked'); + await new Promise(() => { setInterval(() => {}, 1000); }); + }); + `, + ], + { stdio: ['ignore', 'pipe', 'pipe'] } + ); + const closed = once(child, 'close'); + t.after(async () => { + child.kill('SIGKILL'); + await closed; + await rm(directory, { recursive: true, force: true }); + }); + await once(child.stdout!, 'data'); + let entered = false; + await assert.rejects( + withProcessLock( + path, + async () => { + entered = true; + }, + AbortSignal.timeout(100) + ) + ); + assert.equal(entered, false); + child.kill('SIGKILL'); + await closed; + await withProcessLock( + path, + async () => { + entered = true; + }, + AbortSignal.timeout(1000) + ); + assert.equal(entered, true); + } +); diff --git a/packages/code/src/process-lock.ts b/packages/code/src/process-lock.ts new file mode 100644 index 00000000..aeca561a --- /dev/null +++ b/packages/code/src/process-lock.ts @@ -0,0 +1,59 @@ +import { constants } from 'node:fs'; +import { open } from 'node:fs/promises'; +import { setTimeout as delay } from 'node:timers/promises'; + +const LOCK_EX = 2; +const LOCK_NB = 4; +const LOCK_UN = 8; +let binding: + | Promise<{ + flock: (fd: number, operation: number) => number; + eagain: number; + errno: () => number; + }> + | undefined; + +async function lockBinding() { + binding ??= import('koffi').then(({ default: koffi }) => ({ + flock: koffi.load(null).func('int flock(int fd, int operation)'), + eagain: koffi.os.errno.EAGAIN, + errno: () => koffi.errno(), + })); + return await binding; +} + +/** Process-lifetime advisory lock; the kernel releases it on crash or restart. */ +export async function withProcessLock( + path: string, + operation: () => Promise, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted(); + if (process.platform !== 'darwin' && process.platform !== 'linux') { + throw new Error('Conversation worktree locking requires a POSIX host'); + } + const native = await lockBinding(); + const handle = await open( + path, + constants.O_CREAT | constants.O_RDWR | constants.O_NOFOLLOW, + 0o600 + ); + try { + for (;;) { + signal?.throwIfAborted(); + if (native.flock(handle.fd, LOCK_EX | LOCK_NB) === 0) break; + const errno = native.errno(); + if (errno !== native.eagain) { + throw new Error( + `Conversation worktree lock failed with errno ${errno}` + ); + } + await delay(50, undefined, { signal }); + } + signal?.throwIfAborted(); + return await operation(); + } finally { + native.flock(handle.fd, LOCK_UN); + await handle.close(); + } +} diff --git a/packages/code/src/protocol.test.ts b/packages/code/src/protocol.test.ts index 08cba1c7..5c26c462 100644 --- a/packages/code/src/protocol.test.ts +++ b/packages/code/src/protocol.test.ts @@ -10,6 +10,7 @@ import { isValidBridgeWorkerId, isWorkspaceToolRequest, isWorkspaceToolResult, + workspaceIsolationKey, } from './protocol.js'; import type { WorkspaceEditFileRequest, @@ -90,6 +91,19 @@ test('bridgeWorkerPath encodes worker-controlled path segments', () => { ); }); +test('workspace isolation keys keep roots and instances in disjoint namespaces', () => { + const instanceId = 'a'.repeat(64); + assert.notEqual( + workspaceIsolationKey(`foo:git-worktree:${instanceId}`), + workspaceIsolationKey('foo', instanceId), + ); + assert.equal(workspaceIsolationKey('foo'), 'foo'); + assert.notEqual( + workspaceIsolationKey('foo'), + workspaceIsolationKey('workspace:foo'), + ); +}); + test('bridge worker IDs reject path, whitespace, and oversized values', () => { assert.equal(isValidBridgeWorkerId('engineering-vm:1'), true); assert.equal(isValidBridgeWorkerId('engineering/vm'), false); @@ -262,6 +276,20 @@ test('workspace file listing accepts only bounded portable requests and results' afterPath: 'src/app.ts', }; assert.equal(isWorkspaceToolRequest(request), true); + assert.equal( + isWorkspaceToolRequest({ + ...request, + workspaceInstanceId: 'a'.repeat(64), + }), + true, + ); + assert.equal( + isWorkspaceToolRequest({ + ...request, + workspaceInstanceId: 'conversation-1', + }), + false, + ); assert.equal( isWorkspaceToolRequest({ ...request, path: '../outside' }), false, @@ -596,7 +624,11 @@ test('workspace capabilities allow per-workspace operation restrictions', () => operations: ['read_file', 'write_file'], workspaces: [ { id: 'readonly', operations: ['read_file'] }, - { id: 'writable', operations: ['read_file', 'write_file'] }, + { + id: 'writable', + operations: ['read_file', 'write_file'], + workspaceInstances: ['git_worktree'], + }, ], }, }; @@ -660,6 +692,7 @@ test('workspace programmatic requests accept only stable input cache identities' max_output_files: 50, max_output_file_bytes: 10_000_000, session_id: 'session-1', + workspace_instance_id: 'a'.repeat(64), files: [ { name: 'main.sh', content: 'echo ready' }, { @@ -672,6 +705,13 @@ test('workspace programmatic requests accept only stable input cache identities' }, }; assert.equal(isBridgeWorkspaceProgrammaticRequest(request), true); + assert.equal( + isBridgeWorkspaceProgrammaticRequest({ + ...request, + body: { ...request.body, workspace_instance_id: '../escape' }, + }), + false, + ); assert.equal( isBridgeWorkspaceProgrammaticRequest({ ...request, diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index 27220735..c678a1af 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -259,6 +259,16 @@ export function bridgeArtifactMediaType(name: string): string { export type BridgeProtocolVersion = typeof BRIDGE_PROTOCOL_VERSION; +/** Collision-free identity shared by scheduling and worker quarantine state. */ +export function workspaceIsolationKey( + workspaceId: string, + instanceId?: string, +): string { + return instanceId === undefined + ? workspaceId + : `\0git-worktree\0${workspaceId}\0${instanceId}`; +} + export type BridgeWorkspaceToolOperation = | 'read_file' | 'search_text' @@ -280,6 +290,8 @@ export interface BridgeWorkspaceDescriptor { instructions?: RepositoryInstructionDescriptor[]; /** Optional per-workspace restriction. Omitted by protocol-v1 readers. */ operations?: BridgeWorkspaceToolOperation[]; + /** Worker-owned isolation schemes available beneath this selected root. */ + workspaceInstances?: ['git_worktree']; environment?: { fingerprint: string; repo?: string; @@ -308,6 +320,7 @@ export interface WorkspaceReadFileRequest { protocolVersion: BridgeProtocolVersion; operation: 'read_file'; workspaceId: string; + workspaceInstanceId?: string; path: string; startLine?: number; maxLines?: number; @@ -350,6 +363,7 @@ export interface WorkspaceSearchTextRequest { protocolVersion: BridgeProtocolVersion; operation: 'search_text'; workspaceId: string; + workspaceInstanceId?: string; query: string; path?: string; maxResults?: number; @@ -374,6 +388,7 @@ export interface WorkspaceListFilesRequest { protocolVersion: BridgeProtocolVersion; operation: 'list_files'; workspaceId: string; + workspaceInstanceId?: string; path?: string; maxResults?: number; /** Continue strictly after this canonical path from a previous page. */ @@ -394,6 +409,7 @@ export interface WorkspaceWriteFileRequest { protocolVersion: BridgeProtocolVersion; operation: 'write_file'; workspaceId: string; + workspaceInstanceId?: string; path: string; content: string; /** False requires an atomic create and refuses to replace an existing file. */ @@ -413,6 +429,7 @@ interface WorkspaceEditFileRequestBase { protocolVersion: BridgeProtocolVersion; operation: 'edit_file'; workspaceId: string; + workspaceInstanceId?: string; path: string; /** Refuses the mutation unless current file bytes match this preview revision. */ expectedBaseSha256?: string; @@ -457,6 +474,7 @@ interface WorkspacePreviewEditRequestBase { protocolVersion: BridgeProtocolVersion; operation: 'preview_edit'; workspaceId: string; + workspaceInstanceId?: string; path: string; } @@ -494,6 +512,7 @@ export interface WorkspaceExecuteCommandRequest { protocolVersion: BridgeProtocolVersion; operation: 'execute_command'; workspaceId: string; + workspaceInstanceId?: string; /** Shell source evaluated only inside the selected sandbox runtime. */ command: string; /** Portable path relative to the workspace root; defaults to '.'. */ @@ -538,6 +557,7 @@ const WORKSPACE_READ_REQUEST_KEYS = new Set([ 'protocolVersion', 'operation', 'workspaceId', + 'workspaceInstanceId', 'path', 'startLine', 'maxLines', @@ -546,6 +566,7 @@ const WORKSPACE_SEARCH_REQUEST_KEYS = new Set([ 'protocolVersion', 'operation', 'workspaceId', + 'workspaceInstanceId', 'query', 'path', 'maxResults', @@ -554,6 +575,7 @@ const WORKSPACE_LIST_REQUEST_KEYS = new Set([ 'protocolVersion', 'operation', 'workspaceId', + 'workspaceInstanceId', 'path', 'maxResults', 'afterPath', @@ -562,6 +584,7 @@ const WORKSPACE_WRITE_REQUEST_KEYS = new Set([ 'protocolVersion', 'operation', 'workspaceId', + 'workspaceInstanceId', 'path', 'content', 'overwrite', @@ -570,6 +593,7 @@ const WORKSPACE_EDIT_REQUEST_KEYS = new Set([ 'protocolVersion', 'operation', 'workspaceId', + 'workspaceInstanceId', 'path', 'oldText', 'newText', @@ -580,6 +604,7 @@ const WORKSPACE_PREVIEW_EDIT_REQUEST_KEYS = new Set([ 'protocolVersion', 'operation', 'workspaceId', + 'workspaceInstanceId', 'path', 'oldText', 'newText', @@ -591,6 +616,7 @@ const WORKSPACE_COMMAND_REQUEST_KEYS = new Set([ 'protocolVersion', 'operation', 'workspaceId', + 'workspaceInstanceId', 'command', 'cwd', 'timeoutMs', @@ -702,6 +728,8 @@ export interface BridgeWorkerRegistrationResponse { supportedWorkspaceListFileFeatures?: WorkspaceListFileFeature[]; /** PTC languages this Code API can safely route into a selected workspace. */ supportedWorkspaceProgrammaticLanguages?: WorkspaceProgrammaticLanguage[]; + /** Workspace isolation schemes this Code API understands and can route. */ + supportedWorkspaceInstanceTypes?: ['git_worktree']; } /** Administrator-visible liveness for a configured worker. Credentials, @@ -748,6 +776,7 @@ export type BridgeProgrammaticPayloadFile = export interface BridgeWorkspaceProgrammaticBody { language: 'bash'; version: string; + workspace_instance_id?: string; /** Stable identity shared by every replay iteration of one execution. */ execution_id?: string; /** Declared replay tools; zero allows the worker to skip the probe pass. */ @@ -912,6 +941,9 @@ export function isBridgeWorkspaceProgrammaticRequest( typeof body.version !== 'string' || body.version.length === 0 || body.version.length > BRIDGE_RUNTIME_MAX_LENGTH || + (body.workspace_instance_id !== undefined && + (typeof body.workspace_instance_id !== 'string' || + !/^[a-f0-9]{64}$/.test(body.workspace_instance_id))) || (body.execution_id !== undefined && (typeof body.execution_id !== 'string' || !/^[A-Za-z0-9_-]{1,128}$/.test(body.execution_id))) || @@ -1161,7 +1193,10 @@ export function isWorkspaceToolRequest( if ( request.protocolVersion !== BRIDGE_PROTOCOL_VERSION || typeof request.workspaceId !== 'string' || - !isValidBridgeWorkerId(request.workspaceId) + !isValidBridgeWorkerId(request.workspaceId) || + (request.workspaceInstanceId !== undefined && + (typeof request.workspaceInstanceId !== 'string' || + !/^[a-f0-9]{64}$/.test(request.workspaceInstanceId))) ) { return false; } @@ -1614,12 +1649,17 @@ export function isValidBridgeWorkspaceToolCapabilities( key !== 'id' && key !== 'name' && key !== 'operations' && + key !== 'workspaceInstances' && key !== 'instructions' && key !== 'environment', ) || typeof descriptor.id !== 'string' || !isValidBridgeWorkerId(descriptor.id) || workspaceIds.has(descriptor.id) || + (descriptor.workspaceInstances !== undefined && + (!Array.isArray(descriptor.workspaceInstances) || + descriptor.workspaceInstances.length !== 1 || + descriptor.workspaceInstances[0] !== 'git_worktree')) || (descriptor.instructions !== undefined && (!Array.isArray(descriptor.instructions) || descriptor.instructions.length > 1 || !descriptor.instructions.every(isRepositoryInstructionDescriptor))) || (descriptor.environment !== undefined && !isValidCodeEnvironmentDescriptor(descriptor.environment)) || diff --git a/packages/code/src/root-identity.ts b/packages/code/src/root-identity.ts index 3ddfe453..dd3bb2f5 100644 --- a/packages/code/src/root-identity.ts +++ b/packages/code/src/root-identity.ts @@ -6,6 +6,22 @@ export interface WorkspaceRootIdentity { ino: string; } +/** Capture the inode-bound identity of a canonical workspace grant. */ +export async function captureWorkspaceRootIdentity( + root: string, +): Promise { + const canonical = await realpath(root); + const current = await lstat(canonical, { bigint: true }); + if (!current.isDirectory() || current.isSymbolicLink()) { + throw new Error('Workspace root must be a real directory'); + } + return { + path: canonical, + dev: current.dev.toString(), + ino: current.ino.toString(), + }; +} + /** Revalidation of a trusted snapshot, never a fresh grant to a replacement. */ export async function matchesWorkspaceRoot( root: string, diff --git a/packages/code/src/worker-slots.test.ts b/packages/code/src/worker-slots.test.ts index 6e8cc8b1..fda6c659 100644 --- a/packages/code/src/worker-slots.test.ts +++ b/packages/code/src/worker-slots.test.ts @@ -5,6 +5,7 @@ import type { BridgeAssignment, BridgeWorkspaceToolCapabilities, } from './protocol.js'; +import { workspaceIsolationKey } from './protocol.js'; const capabilities: BridgeWorkspaceToolCapabilities = { protocolVersion: 1, @@ -229,7 +230,8 @@ for (const cancelled of [false, true]) { rejectUnexecutedAssignment: () => Promise; executeOwned: () => Promise; }; - internals.activeWorkspaceAssignments.set('a', { + const workspaceKey = workspaceIsolationKey('a'); + internals.activeWorkspaceAssignments.set(workspaceKey, { id: 'previous', done: new Promise(() => {}), }); @@ -257,7 +259,10 @@ for (const cancelled of [false, true]) { controller.signal, ); assert.equal(rejected, true); - assert.equal(internals.activeWorkspaceAssignments.get('a')?.id, 'previous'); + assert.equal( + internals.activeWorkspaceAssignments.get(workspaceKey)?.id, + 'previous', + ); }); } @@ -281,7 +286,8 @@ test('a local cleanup handoff preserves the new assignment owner and remaining b executeOwned: (assignment: BridgeAssignment) => Promise; }; let release!: () => void; - internals.activeWorkspaceAssignments.set('a', { + const workspaceKey = workspaceIsolationKey('a'); + internals.activeWorkspaceAssignments.set(workspaceKey, { id: 'previous', done: new Promise((resolve) => { release = resolve; @@ -290,7 +296,10 @@ test('a local cleanup handoff preserves the new assignment owner and remaining b let executed = false; internals.executeOwned = async (assignment) => { executed = true; - assert.equal(internals.activeWorkspaceAssignments.get('a')?.id, 'next'); + assert.equal( + internals.activeWorkspaceAssignments.get(workspaceKey)?.id, + 'next', + ); assert.ok(assignment.remainingMs! < 1000 && assignment.remainingMs! > 0); }; const pending = worker.executeAndSettle({ @@ -306,7 +315,7 @@ test('a local cleanup handoff preserves the new assignment owner and remaining b } as BridgeAssignment); await new Promise((resolve) => setTimeout(resolve, 5)); assert.equal(executed, false); - internals.activeWorkspaceAssignments.delete('a'); + internals.activeWorkspaceAssignments.delete(workspaceKey); release(); await pending; assert.equal(executed, true); @@ -332,14 +341,19 @@ test('programmatic work on an independent workspace bypasses another root cleanu >; executeOwned: (assignment: BridgeAssignment) => Promise; }; - internals.activeWorkspaceAssignments.set('a', { + const workspaceAKey = workspaceIsolationKey('a'); + const workspaceBKey = workspaceIsolationKey('b'); + internals.activeWorkspaceAssignments.set(workspaceAKey, { id: 'previous', done: new Promise(() => {}), }); let executed = false; internals.executeOwned = async () => { executed = true; - assert.equal(internals.activeWorkspaceAssignments.get('b')?.id, 'next'); + assert.equal( + internals.activeWorkspaceAssignments.get(workspaceBKey)?.id, + 'next', + ); }; await worker.executeAndSettle({ assignmentId: 'next', @@ -357,6 +371,9 @@ test('programmatic work on an independent workspace bypasses another root cleanu }, } as BridgeAssignment); assert.equal(executed, true); - assert.equal(internals.activeWorkspaceAssignments.has('b'), false); - assert.equal(internals.activeWorkspaceAssignments.get('a')?.id, 'previous'); + assert.equal(internals.activeWorkspaceAssignments.has(workspaceBKey), false); + assert.equal( + internals.activeWorkspaceAssignments.get(workspaceAKey)?.id, + 'previous', + ); }); diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 697a3c55..2a5896fd 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -7,6 +7,7 @@ import { bridgeWorkerPath, isBridgeWorkspaceProgrammaticRequest, isWorkspaceToolResult, + workspaceIsolationKey, } from './protocol.js'; import { EndpointRuntimeSupervisor } from './runtime.js'; import { signBridgeRequest } from './identity.js'; @@ -54,6 +55,13 @@ export interface BridgeWorkerOptions { workspaceMutationQuarantine?: WorkspaceMutationQuarantine; /** Required per-root durable guards when opting into concurrent workspace leases. */ workspaceQuarantines?: ReadonlyMap; + /** Resolve a durable guard for a worker-owned dynamic workspace instance. */ + workspaceQuarantineResolver?: ( + workspaceId: string, + workspaceInstanceId: string, + ) => + | WorkspaceMutationQuarantine + | Promise; leaseWaitMs?: number; leaseTransportGraceMs?: number; registrationTransportTimeoutMs?: number; @@ -208,7 +216,14 @@ function workspaceCapabilitiesMatch( operation === executor.workspaces[index]?.operations?.[operationIndex], ) ?? - executor.workspaces[index]?.operations == null), + executor.workspaces[index]?.operations == null) && + workspace.workspaceInstances?.length === + executor.workspaces[index]?.workspaceInstances?.length && + (workspace.workspaceInstances?.every( + (instanceType, instanceIndex) => + instanceType === + executor.workspaces[index]?.workspaceInstances?.[instanceIndex], + ) ?? executor.workspaces[index]?.workspaceInstances == null), ) ); } @@ -223,7 +238,8 @@ function registrationCompatibleCapabilities( (operation) => operation === 'read_file' || operation === 'search_text', ) && workspaceTools.workspaces.every( - (workspace) => workspace.operations == null, + (workspace) => + workspace.operations == null && workspace.workspaceInstances == null, )) ) { return capabilities; @@ -244,7 +260,11 @@ function registrationCompatibleCapabilities( ) { return []; } - const { operations: _operations, ...compatibleWorkspace } = workspace; + const { + operations: _operations, + workspaceInstances: _workspaceInstances, + ...compatibleWorkspace + } = workspace; return [{ ...compatibleWorkspace, ...(workspace.environment ? { environment: { ...workspace.environment, actions: [] }, } : {}) }]; @@ -329,6 +349,12 @@ function supportedWorkspaceCapabilities( return workspaceOperations.length === 0 ? [] : [{ ...workspace, + ...(workspace.workspaceInstances != null && + registration.supportedWorkspaceInstanceTypes?.includes( + 'git_worktree', + ) + ? { workspaceInstances: workspace.workspaceInstances } + : { workspaceInstances: undefined }), ...(workspace.operations ? { operations: workspaceOperations } : {}), ...(workspace.environment && !workspaceOperations.includes('execute_command') ? { environment: { ...workspace.environment, actions: [] }, @@ -479,12 +505,23 @@ export class BridgeWorker { operation === 'execute_command', ) === true && options.workspaceMutationQuarantine == null && - options.workspaceQuarantines == null + options.workspaceQuarantines == null && + options.workspaceQuarantineResolver == null ) { throw new BridgeProtocolError( 'Workspace mutation capabilities require durable quarantine storage', ); } + if ( + options.capabilities.workspaceTools?.workspaces.some( + (root) => (root.workspaceInstances?.length ?? 0) > 0, + ) && + options.workspaceQuarantineResolver == null + ) { + throw new BridgeProtocolError( + 'Workspace instance capabilities require a durable quarantine resolver', + ); + } if ((options.capabilities.workspaceLeaseSlots ?? 1) > 1) { if ( options.capabilities.requiresReadyConfirmation !== true || @@ -785,14 +822,25 @@ export class BridgeWorker { async resetNativeWorkspace( workspaceId: string, signal?: AbortSignal, + workspaceInstanceId?: string, ): Promise { - const guard = this.options.workspaceQuarantines?.get(workspaceId); + const workspace = this.options.capabilities.workspaceTools?.workspaces.find( + (root) => root.id === workspaceId, + ); + const key = workspaceIsolationKey(workspaceId, workspaceInstanceId); + const guard = + workspaceInstanceId == null + ? this.options.workspaceQuarantines?.get(workspaceId) + : await this.options.workspaceQuarantineResolver?.( + workspaceId, + workspaceInstanceId, + ); if ( !guard || this.activeWorkspaceAssignments.size > 0 || - !this.options.capabilities.workspaceTools?.workspaces.some( - (root) => root.id === workspaceId, - ) + workspace == null || + (workspaceInstanceId != null && + workspace.workspaceInstances?.includes('git_worktree') !== true) ) { throw new BridgeProtocolError( 'Native workspace reset requires an idle registered root', @@ -808,14 +856,14 @@ export class BridgeWorker { { protocolVersion: BRIDGE_PROTOCOL_VERSION, incarnationId: this.incarnationId, - runtimeSessionId: `native-workspace:${workspaceId}`, + runtimeSessionId: `native-workspace:${key}`, confirmDiscarded: true, }, this.options.resetTransportTimeoutMs ?? DEFAULT_CONTROL_TRANSPORT_TIMEOUT_MS, signal, ); - this.quarantinedWorkspaces.delete(workspaceId); + this.quarantinedWorkspaces.delete(key); } async lease( @@ -1340,8 +1388,18 @@ export class BridgeWorker { private workspaceGuard( assignment: BridgeAssignment, - ): WorkspaceMutationQuarantine | undefined { - const workspaceId = this.assignmentWorkspaceId(assignment); + ): + | WorkspaceMutationQuarantine + | Promise + | undefined { + const workspaceId = this.assignmentBaseWorkspaceId(assignment); + const instanceId = this.assignmentWorkspaceInstanceId(assignment); + if (workspaceId != null && instanceId != null) { + return this.options.workspaceQuarantineResolver?.( + workspaceId, + instanceId, + ); + } return workspaceId != null ? (this.options.workspaceQuarantines?.get(workspaceId) ?? this.options.workspaceMutationQuarantine) @@ -1350,6 +1408,15 @@ export class BridgeWorker { private assignmentWorkspaceId( assignment: BridgeAssignment, + ): string | undefined { + const workspaceId = this.assignmentBaseWorkspaceId(assignment); + if (workspaceId == null) return undefined; + const instanceId = this.assignmentWorkspaceInstanceId(assignment); + return workspaceIsolationKey(workspaceId, instanceId); + } + + private assignmentBaseWorkspaceId( + assignment: BridgeAssignment, ): string | undefined { if ( assignment.executionKind === 'workspace_tool' && @@ -1366,11 +1433,33 @@ export class BridgeWorker { return undefined; } + private assignmentWorkspaceInstanceId( + assignment: BridgeAssignment, + ): string | undefined { + if ( + assignment.executionKind === 'workspace_tool' && + isWorkspaceToolRequest(assignment.request) + ) { + return assignment.request.workspaceInstanceId; + } + if ( + assignment.executionKind === 'workspace_programmatic' && + isBridgeWorkspaceProgrammaticRequest(assignment.request) + ) { + return assignment.request.body.workspace_instance_id; + } + return undefined; + } + private async executeOwned( assignment: BridgeAssignment, signal?: AbortSignal, ): Promise { - const guard = this.workspaceGuard(assignment); + const unresolvedGuard = this.workspaceGuard(assignment); + const guard = + unresolvedGuard instanceof Promise + ? await unresolvedGuard + : unresolvedGuard; if (signal?.aborted === true) { throw signal.reason instanceof Error ? signal.reason @@ -1484,12 +1573,17 @@ export class BridgeWorker { throw new BridgeProtocolError('Invalid workspace tool request'); } const workspaceRequest = assignment.request; + const workspaceKey = this.assignmentWorkspaceId(assignment)!; try { - if (this.quarantinedWorkspaces.has(workspaceRequest.workspaceId)) { + if (this.quarantinedWorkspaces.has(workspaceKey)) { throw new Error('Workspace requires an explicit quarantine reset'); } - if (this.options.workspaceQuarantines != null) + if ( + this.options.workspaceQuarantines != null || + this.options.workspaceQuarantineResolver != null + ) { await guard?.assertAvailable(); + } } catch (error) { throw new BridgeWorkspaceQuarantinedError( 'Workspace is quarantined', @@ -1513,6 +1607,14 @@ export class BridgeWorker { if (workspace == null) { throw new BridgeProtocolError('Workspace is not advertised'); } + if ( + workspaceRequest.workspaceInstanceId != null && + workspace.workspaceInstances?.includes('git_worktree') !== true + ) { + throw new BridgeProtocolError( + 'Workspace instance type is not advertised', + ); + } if ( workspace.operations != null && !workspace.operations.includes(workspaceRequest.operation) @@ -1575,7 +1677,7 @@ export class BridgeWorker { if (isMutation) { this.mutationGuardArmed = true; try { - this.armedWorkspaces.add(workspaceRequest.workspaceId); + this.armedWorkspaces.add(workspaceKey); await guard!.arm( `Workspace mutation ${workspaceRequest.operation} is pending settlement`, assignment.assignmentId, @@ -1630,12 +1732,17 @@ export class BridgeWorker { 'Worker does not provide valid selected-workspace programmatic execution', ); } + const workspaceKey = this.assignmentWorkspaceId(assignment)!; try { - if (this.quarantinedWorkspaces.has(workspaceId)) { + if (this.quarantinedWorkspaces.has(workspaceKey)) { throw new Error('Workspace requires an explicit quarantine reset'); } - if (this.options.workspaceQuarantines != null) + if ( + this.options.workspaceQuarantines != null || + this.options.workspaceQuarantineResolver != null + ) { await guard?.assertAvailable(); + } } catch (error) { throw new BridgeWorkspaceQuarantinedError( 'Workspace is quarantined', @@ -1657,9 +1764,17 @@ export class BridgeWorker { 'Selected-workspace programmatic execution is not advertised', ); } + if ( + assignment.request.body.workspace_instance_id != null && + workspace.workspaceInstances?.includes('git_worktree') !== true + ) { + throw new BridgeProtocolError( + 'Workspace instance type is not advertised', + ); + } this.mutationGuardArmed = true; try { - this.armedWorkspaces.add(workspaceId); + this.armedWorkspaces.add(workspaceKey); await guard!.arm( 'Workspace programmatic execution is pending settlement', assignment.assignmentId, @@ -2079,11 +2194,15 @@ export class BridgeWorker { ): Promise { if (runtimeSessionId == null) { try { - await ( + const unresolvedGuard = assignment == null ? this.options.workspaceMutationQuarantine - : this.workspaceGuard(assignment) - )?.quarantine(message, cause, assignment?.assignmentId); + : this.workspaceGuard(assignment); + const guard = + unresolvedGuard instanceof Promise + ? await unresolvedGuard + : unresolvedGuard; + await guard?.quarantine(message, cause, assignment?.assignmentId); return new BridgeWorkspaceQuarantinedError(message, cause); } catch (error) { return new BridgeWorkspaceQuarantinedError( diff --git a/packages/code/src/workspace-cli.test.ts b/packages/code/src/workspace-cli.test.ts index c1bc7fce..1de68818 100644 --- a/packages/code/src/workspace-cli.test.ts +++ b/packages/code/src/workspace-cli.test.ts @@ -115,6 +115,38 @@ test('CLI supports native SRT by default and validates explicit runtime mode', a assert.match(noWorkspace.stderr, /require.*registered directory/i); }); +test('CLI requires concurrent native slots for conversation worktrees', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-conversation-')); + const workspaceRoot = join(root, 'workspace'); + const worktreeRoot = join(root, 'worktrees'); + await mkdir(workspaceRoot); + t.after(() => rm(root, { recursive: true, force: true })); + const result = spawnSync( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'run', + '--worker-dir', + workspaceRoot, + '--allow-workspace-writes', + '--allow-workspace-commands', + '--conversation-worktree-root', + worktreeRoot, + ], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + }, + }, + ); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /at least two workspace lease slots/i); +}); + test('CLI advertises explicitly enabled writes without exposing the workspace root', async (t) => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-cli-')); const workspaceRoot = join(root, ' '); diff --git a/packages/code/src/workspace-instances.test.ts b/packages/code/src/workspace-instances.test.ts new file mode 100644 index 00000000..a1cd6859 --- /dev/null +++ b/packages/code/src/workspace-instances.test.ts @@ -0,0 +1,234 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { mkdtemp, readFile, realpath, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import test from 'node:test'; + +import { GitWorktreeWorkspaceTools } from './workspace-instances.js'; +import { LocalWorkspaceTools } from './workspace.js'; +import { GitWorktreeManager } from './worktrees.js'; +import { readRepositoryInstructions } from './instructions.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; + +const execFileAsync = promisify(execFile); + +async function repository(): Promise<{ parent: string; root: string }> { + const parent = await mkdtemp(join(tmpdir(), 'librechat-instance-tools-')); + const root = join(parent, 'source'); + await execFileAsync('git', ['init', root]); + await writeFile(join(root, 'README.md'), 'source\n'); + await writeFile(join(root, 'AGENTS.md'), 'follow repository rules\n'); + await execFileAsync('git', ['-C', root, 'add', 'README.md', 'AGENTS.md']); + await execFileAsync('git', [ + '-C', + root, + '-c', + 'user.name=Test', + '-c', + 'user.email=test@example.com', + 'commit', + '-m', + 'initial', + ]); + return { parent, root: await realpath(root) }; +} + +async function source(root: string) { + return { root, identity: await captureWorkspaceRootIdentity(root) }; +} + +test('routes each conversation to its own writable Git worktree', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const delegate = await LocalWorkspaceTools.create({ + repositoryInstructions: true, + workspaces: [{ id: 'primary', root: fixture.root, writable: true }], + }); + const manager = new GitWorktreeManager({ + maxCount: 4, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + const tools = new GitWorktreeWorkspaceTools({ + delegate, + manager, + sources: new Map([ + ['primary', { repositoryInstructions: true, writable: true }], + ]), + }); + const firstId = 'a'.repeat(64); + const secondId = 'b'.repeat(64); + + assert.deepEqual(tools.capabilities.workspaces[0]?.workspaceInstances, [ + 'git_worktree', + ]); + await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + workspaceInstanceId: firstId, + path: 'conversation.txt', + content: 'first', + }); + await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + workspaceInstanceId: secondId, + path: 'conversation.txt', + content: 'second', + }); + + const first = await manager.resolve('primary', firstId); + const second = await manager.resolve('primary', secondId); + assert.equal( + await readFile(join(first.root, 'conversation.txt'), 'utf8'), + 'first', + ); + assert.equal( + await readFile(join(second.root, 'conversation.txt'), 'utf8'), + 'second', + ); + await assert.rejects(readFile(join(fixture.root, 'conversation.txt')), { + code: 'ENOENT', + }); + + const result = await tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + workspaceInstanceId: firstId, + path: 'conversation.txt', + }); + assert.equal(result.workspaceId, 'primary'); + assert.equal(result.operation, 'read_file'); + assert.equal(result.content, 'first'); + + await writeFile(join(fixture.root, 'AGENTS.md'), 'local repository rules\n'); + const instructions = await readRepositoryInstructions(fixture.root); + assert.ok(instructions); + const instructionResult = await tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + workspaceInstanceId: firstId, + path: instructions.descriptor.path, + instructionSha256: instructions.descriptor.sha256, + }); + assert.equal(instructionResult.operation, 'read_file'); + assert.equal(instructionResult.content, 'local repository rules\n'); +}); + +test('reports provisioning rejection as an atomic workspace error', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const delegate = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root: fixture.root, writable: true }], + }); + const tools = new GitWorktreeWorkspaceTools({ + delegate, + manager: new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }), + sources: new Map([ + ['primary', { repositoryInstructions: false, writable: true }], + ]), + }); + await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + workspaceInstanceId: 'a'.repeat(64), + path: 'first.txt', + content: 'first', + }); + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + workspaceInstanceId: 'b'.repeat(64), + path: 'second.txt', + content: 'second', + }), + { + code: 'WRITE_UNAVAILABLE', + mutationMayHaveCommitted: false, + requiresQuarantine: false, + }, + ); +}); + +test('rebuilds file executors only after operator recovery releases the reservation', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + const delegate = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root: fixture.root, writable: true }], + }); + const tools = new GitWorktreeWorkspaceTools({ + delegate, + manager, + sources: new Map([ + ['primary', { repositoryInstructions: false, writable: true }], + ]), + }); + const instanceId = 'c'.repeat(64); + const request = { + protocolVersion: 1 as const, + operation: 'read_file' as const, + workspaceId: 'primary', + workspaceInstanceId: instanceId, + path: 'README.md', + }; + await tools.execute(request); + const initial = await manager.resolve('primary', instanceId); + await rm(initial.root, { recursive: true, force: true }); + + await assert.rejects(tools.execute(request), { + code: 'WRITE_UNAVAILABLE', + }); + await assert.rejects(tools.execute(request), /capacity is exhausted/); + // Missing checkout directories do not prove an interrupted writer is gone. + // Simulate operator recovery after confirming there is no active executor. + await rm(`${initial.root}.complete`); + const recovered = await tools.execute(request); + assert.equal(recovered.operation, 'read_file'); + assert.equal(recovered.content, 'source'); +}); + +test('leaves legacy requests on the selected source workspace', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const delegate = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root: fixture.root, writable: false }], + }); + const tools = new GitWorktreeWorkspaceTools({ + delegate, + manager: new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }), + sources: new Map([ + ['primary', { repositoryInstructions: false, writable: false }], + ]), + }); + + const result = await tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'README.md', + }); + assert.equal(result.operation, 'read_file'); + assert.equal(result.content, 'source'); +}); diff --git a/packages/code/src/workspace-instances.ts b/packages/code/src/workspace-instances.ts new file mode 100644 index 00000000..68e76a10 --- /dev/null +++ b/packages/code/src/workspace-instances.ts @@ -0,0 +1,232 @@ +import { createHash } from 'node:crypto'; + +import { NativeWorkspaceCommandPool } from './native-pool.js'; +import { GitWorktreeManager } from './worktrees.js'; +import { LocalWorkspaceTools, WorkspaceToolError } from './workspace.js'; + +import type { NativeProcessSandboxOptions } from './native-process.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; +import type { + BridgeWorkspaceProgrammaticRequest, + WorkspaceExecuteCommandRequest, + WorkspaceToolRequest, + WorkspaceToolResult, +} from './protocol.js'; +import type { WorkspaceToolExecutor } from './workspace.js'; + +interface WorkspaceInstanceSource { + command?: NativeProcessSandboxOptions; + repositoryInstructions: boolean; + writable: boolean; +} + +export interface GitWorktreeWorkspaceToolsOptions { + commandPool?: NativeWorkspaceCommandPool; + delegate: WorkspaceToolExecutor; + manager: GitWorktreeManager; + onResolve?: (workspaceId: string, root: string) => void; + sources: ReadonlyMap; +} + +export function internalWorkspaceId(workspaceId: string, instanceId: string): string { + return `instance-${createHash('sha256') + .update(`${workspaceId}\0${instanceId}`) + .digest('hex')}`; +} + +function publicResult( + result: WorkspaceToolResult, + workspaceId: string, +): WorkspaceToolResult { + return { ...result, workspaceId }; +} + +/** Resolve an opaque conversation binding into an isolated Git worktree. */ +export class GitWorktreeWorkspaceTools implements WorkspaceToolExecutor { + readonly mutationFailuresAreAtomic?: true; + readonly capabilities: WorkspaceToolExecutor['capabilities']; + private readonly executors = new Map< + string, + { identity: WorkspaceRootIdentity; value: Promise } + >(); + + constructor(private readonly options: GitWorktreeWorkspaceToolsOptions) { + this.mutationFailuresAreAtomic = options.delegate.mutationFailuresAreAtomic; + this.capabilities = { + ...options.delegate.capabilities, + workspaces: options.delegate.capabilities.workspaces.map((workspace) => ({ + ...workspace, + ...(options.sources.has(workspace.id) + ? { workspaceInstances: ['git_worktree' as const] } + : {}), + })), + }; + } + + private async executor( + workspaceId: string, + instanceId: string, + signal?: AbortSignal, + ): Promise<{ + executor: LocalWorkspaceTools; + identity: WorkspaceRootIdentity; + internalId: string; + root: string; + }> { + const source = this.options.sources.get(workspaceId); + if (!source) { + throw new WorkspaceToolError( + 'Workspace does not allow conversation worktrees', + 'INVALID_REQUEST', + ); + } + let instance; + try { + instance = await this.options.manager.resolve( + workspaceId, + instanceId, + signal, + ); + } catch (error) { + if (error instanceof WorkspaceToolError) throw error; + if ( + signal?.aborted || + (error instanceof Error && error.name === 'AbortError') + ) { + throw new WorkspaceToolError( + 'Conversation worktree provisioning aborted', + 'EXECUTION_ABORTED', + ); + } + throw new WorkspaceToolError( + error instanceof Error + ? error.message + : 'Conversation worktree provisioning failed', + 'WRITE_UNAVAILABLE', + ); + } + this.options.onResolve?.(workspaceId, instance.root); + const internalId = internalWorkspaceId(workspaceId, instanceId); + const key = `${workspaceId}\0${instanceId}`; + let cached = this.executors.get(key); + if ( + cached == null || + cached.identity.dev !== instance.identity.dev || + cached.identity.ino !== instance.identity.ino || + cached.identity.path !== instance.identity.path + ) { + cached = { + identity: instance.identity, + value: LocalWorkspaceTools.create({ + repositoryInstructions: source.repositoryInstructions, + workspaces: [ + { + id: internalId, + identity: instance.identity, + root: instance.root, + writable: source.writable, + }, + ], + }), + }; + this.executors.set(key, cached); + } + return { + executor: await cached.value, + identity: instance.identity, + internalId, + root: instance.root, + }; + } + + async execute( + request: WorkspaceToolRequest, + signal?: AbortSignal, + ): Promise { + if (!request.workspaceInstanceId) { + return await this.options.delegate.execute(request, signal); + } + if ( + request.operation === 'read_file' && + request.instructionSha256 !== undefined + ) { + const { workspaceInstanceId: _workspaceInstanceId, ...sourceRequest } = + request; + return await this.options.delegate.execute(sourceRequest, signal); + } + const { workspaceInstanceId, ...baseRequest } = request; + const source = this.options.sources.get(request.workspaceId); + const resolved = await this.executor( + request.workspaceId, + workspaceInstanceId, + signal, + ); + const isolatedRequest = { + ...baseRequest, + workspaceId: resolved.internalId, + } as WorkspaceToolRequest; + if (request.operation === 'execute_command') { + if (!source?.command || !this.options.commandPool) { + throw new WorkspaceToolError( + 'Conversation worktree commands are unavailable', + 'COMMAND_DISABLED', + ); + } + await this.options.commandPool.registerRoot(resolved.internalId, { + ...source.command, + workspaceIdentity: resolved.identity, + workspaceRoot: resolved.root, + }); + return publicResult( + await this.options.commandPool.execute( + isolatedRequest as WorkspaceExecuteCommandRequest, + signal, + ), + request.workspaceId, + ); + } + return publicResult( + await resolved.executor.execute(isolatedRequest, signal), + request.workspaceId, + ); + } + + async executeProgrammatic( + workspaceId: string, + request: BridgeWorkspaceProgrammaticRequest, + signal?: AbortSignal, + ): Promise { + const instanceId = request.body.workspace_instance_id; + if (!instanceId) { + if (!this.options.commandPool) { + throw new WorkspaceToolError( + 'Workspace programmatic execution is unavailable', + 'COMMAND_DISABLED', + ); + } + return await this.options.commandPool.executeProgrammatic( + workspaceId, + request, + signal, + ); + } + const source = this.options.sources.get(workspaceId); + if (!source?.command || !this.options.commandPool) { + throw new WorkspaceToolError( + 'Conversation worktree programmatic execution is unavailable', + 'COMMAND_DISABLED', + ); + } + const resolved = await this.executor(workspaceId, instanceId, signal); + await this.options.commandPool.registerRoot(resolved.internalId, { + ...source.command, + workspaceIdentity: resolved.identity, + workspaceRoot: resolved.root, + }); + return await this.options.commandPool.executeProgrammatic( + resolved.internalId, + request, + signal, + ); + } +} diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index d97735dc..96a5d1f0 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -7,6 +7,19 @@ import { SandboxWorkspaceTools, WorkspaceToolError } from './workspace.js'; const incarnationId = 'incarnation-00000001'; +test('instance advertisement requires a guard resolver even for reads and with base guards', () => { + for (const operation of ['read_file', 'write_file'] as const) { + const workspaceTools = { protocolVersion: 1 as const, operations: [operation], workspaces: [{ id: 'primary', workspaceInstances: ['git_worktree'] as ['git_worktree'] }] }; + assert.throws(() => new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: false, sandboxProfile: 'anthropic-srt', runtimes: [], workspaceTools }, + workspaceQuarantines: new Map([['primary', mutationQuarantine()]]), + workspaceTools: { capabilities: workspaceTools, async execute() { throw new Error('must not execute'); } }, + }), /instance capabilities require a durable quarantine resolver/); + } +}); + test('worker clears named actions when command execution is not negotiated', async () => { const workspaceTools = { protocolVersion: 1 as const, @@ -974,6 +987,93 @@ test('worker executes a workspace tool assignment locally without acquiring a sa }); }); +test('worker isolates dynamic worktree guards from collision-shaped root IDs', async () => { + const instanceId = 'a'.repeat(64); + const collisionRoot = `foo:git-worktree:${instanceId}`; + const lifecycle: string[] = []; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['write_file' as const], + workspaces: [ + { id: 'foo', workspaceInstances: ['git_worktree'] as ['git_worktree'] }, + { id: collisionRoot }, + ], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + mutationFailuresAreAtomic: true, + async execute(request) { + return { + protocolVersion: 1, + operation: 'write_file', + workspaceId: request.workspaceId, + path: 'result.txt', + created: true, + bytesWritten: 2, + }; + }, + }, + workspaceQuarantines: new Map([ + [ + collisionRoot, + mutationQuarantine( + undefined, + () => lifecycle.push('root:arm'), + () => lifecycle.push('root:clear'), + ), + ], + ]), + workspaceQuarantineResolver: async () => + mutationQuarantine( + undefined, + () => lifecycle.push('instance:arm'), + () => lifecycle.push('instance:clear'), + ), + fetchImpl: async () => + Response.json({ protocolVersion: 1, accepted: true }), + }); + const assignment = (workspaceId: string, suffix: string) => ({ + protocolVersion: 1 as const, + assignmentId: `assignment-${suffix}`, + workerId: 'vm-1', + incarnationId, + generation: 4, + leaseToken: `lease-token-that-is-long-enough-${suffix}`, + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool' as const, + request: { + protocolVersion: 1 as const, + operation: 'write_file' as const, + workspaceId, + path: 'result.txt', + content: 'ok', + ...(workspaceId === 'foo' ? { workspaceInstanceId: instanceId } : {}), + }, + }); + + await worker.executeAndSettle(assignment('foo', 'instance')); + await worker.executeAndSettle(assignment(collisionRoot, 'root')); + + assert.deepEqual(lifecycle, [ + 'instance:arm', + 'instance:clear', + 'root:arm', + 'root:clear', + ]); +}); + test('worker executes programmatic Bash in the selected workspace and preserves its fence', async () => { const programmaticRequests: object[] = []; const quarantineEvents: string[] = []; diff --git a/packages/code/src/worktrees.test.ts b/packages/code/src/worktrees.test.ts new file mode 100644 index 00000000..c353c24e --- /dev/null +++ b/packages/code/src/worktrees.test.ts @@ -0,0 +1,988 @@ +import assert from 'node:assert/strict'; +import { execFile, spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { + mkdir, + mkdtemp, + readFile, + realpath, + rename, + rm, + stat, + symlink, + writeFile, +} from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { promisify } from 'node:util'; +import test from 'node:test'; +import { setTimeout as delay } from 'node:timers/promises'; + +import { GitWorktreeManager } from './worktrees.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; +import { GitSourceSnapshot } from './git-snapshot.js'; + +const execFileAsync = promisify(execFile); + +test('aborting private staging releases its reservation only after cleanup', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + const controller = new AbortController(); + const copy = GitSourceSnapshot.prototype.copyTo; + const mocked = t.mock.method( + GitSourceSnapshot.prototype, + 'copyTo', + async function ( + this: GitSourceSnapshot, + destination: string, + signal?: AbortSignal + ) { + await copy.call(this, destination, signal); + controller.abort(); + } + ); + const id = 'b'.repeat(64); + const path = await manager.plannedRoot('primary', id); + await assert.rejects(manager.resolve('primary', id, controller.signal), { + name: 'AbortError', + }); + for (const name of [path, `${path}.source`, `${path}.complete`]) + await assert.rejects(stat(name), { code: 'ENOENT' }); + mocked.mock.restore(); + assert.ok(await manager.resolve('primary', 'c'.repeat(64))); +}); + +test('setup cannot publish a checkout that redirects its Git metadata', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + prepareInstance: async (instance) => { + await writeFile( + join(instance.root, '.git', 'commondir'), + join(fixture.root, '.git') + ); + }, + }); + await assert.rejects( + manager.resolve('primary', 'd'.repeat(64)), + /must not redirect/ + ); +}); + +test('private snapshots preserve SHA-256 repositories and the configured origin', async (t) => { + const fixture = await repository('sha256'); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + await git( + fixture.root, + 'remote', + 'add', + 'origin', + 'https://github.com/example/repo.git' + ); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + const instance = await manager.resolve('primary', 'a'.repeat(64)); + assert.equal( + await git(instance.root, 'rev-parse', '--show-object-format'), + 'sha256' + ); + assert.equal( + await git(instance.root, 'remote', 'get-url', 'origin'), + 'https://github.com/example/repo.git' + ); + assert.equal( + await readFile(join(instance.root, 'README.md'), 'utf8'), + 'source\n' + ); +}); + +test('sidecar-only crash reservations consume quota after restart', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const admitted = await source(fixture.root); + const options = { + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', admitted]]), + }; + const manager = new GitWorktreeManager(options); + const path = await manager.plannedRoot('primary', 'a'.repeat(64)); + await mkdir(join(path, '..'), { recursive: true }); + await writeFile( + `${path}.complete`, + JSON.stringify({ + version: 2, + source: admitted.identity, + sourceGit: (await GitSourceSnapshot.admit(admitted.identity)).fingerprint, + provisioningFailed: true, + }) + ); + const restarted = new GitWorktreeManager(options); + await assert.rejects( + restarted.resolve('primary', 'b'.repeat(64)), + /capacity is exhausted/ + ); + await assert.rejects( + restarted.resolve('primary', 'a'.repeat(64)), + /operator recovery required/ + ); + await assert.rejects(stat(path), { code: 'ENOENT' }); +}); + +test('linked-worktree sources retain their own HEAD and admitted common objects', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const linked = join(fixture.parent, 'linked'); + await git(fixture.root, 'worktree', 'add', '-b', 'linked', linked); + await writeFile(join(linked, 'linked.txt'), 'linked\n'); + await git(linked, 'add', 'linked.txt'); + await git( + linked, + '-c', + 'user.name=Test', + '-c', + 'user.email=test@example.com', + 'commit', + '-m', + 'linked' + ); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(linked)]]), + }); + const instance = await manager.resolve('primary', 'e'.repeat(64)); + assert.equal( + await readFile(join(instance.root, 'linked.txt'), 'utf8'), + 'linked\n' + ); +}); + +test('restart cannot rebind a completed checkout to replacement source Git metadata', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const options = { + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }; + const id = 'f'.repeat(64); + const instance = await new GitWorktreeManager(options).resolve('primary', id); + await writeFile(join(instance.root, 'uncommitted.txt'), 'keep'); + await rename(join(fixture.root, '.git'), join(fixture.root, '.git-original')); + await git(fixture.root, 'init'); + await assert.rejects( + new GitWorktreeManager(options).resolve('primary', id), + /source identity changed/ + ); + assert.equal( + await readFile(join(instance.root, 'uncommitted.txt'), 'utf8'), + 'keep' + ); +}); + +test('rejects source Git redirection without changing the admitted working-directory inode', async (t) => { + const fixture = await repository(); + const other = await repository(); + t.after(() => + Promise.all( + [fixture, other].map(({ parent }) => + rm(parent, { recursive: true, force: true }) + ) + ) + ); + const manager = new GitWorktreeManager({ + maxCount: 2, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + await manager.prepare(); + await rename(join(fixture.root, '.git'), join(fixture.root, '.git-original')); + await writeFile( + join(fixture.root, '.git'), + `gitdir: ${join(other.root, '.git')}\n` + ); + await assert.rejects( + manager.resolve('primary', 'c'.repeat(64)), + /Git metadata changed/ + ); +}); + +test('source replacement after validation cannot redirect the private snapshot', async (t) => { + const fixture = await repository(); + const other = await repository(); + t.after(() => + Promise.all( + [fixture, other].map(({ parent }) => + rm(parent, { recursive: true, force: true }) + ) + ) + ); + const snapshot = await GitSourceSnapshot.admit( + ( + await source(fixture.root) + ).identity + ); + const validate = snapshot.validate.bind(snapshot); + t.mock.method(snapshot, 'validate', async () => { + await validate(); + await rename( + join(fixture.root, '.git'), + join(fixture.root, '.git-original') + ); + await symlink(join(other.root, '.git'), join(fixture.root, '.git')); + }); + await assert.rejects(snapshot.copyTo(join(fixture.parent, 'snapshot'))); + await assert.rejects( + stat(join(fixture.parent, 'snapshot', 'objects', 'pack')), + { code: 'ENOENT' } + ); +}); + +test('cached instances reject object-directory redirection before executor recreation', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + const id = 'd'.repeat(64); + const instance = await manager.resolve('primary', id); + await rename( + join(instance.root, '.git', 'objects'), + join(instance.root, '.git', 'objects-original') + ); + await symlink( + join(fixture.root, '.git', 'objects'), + join(instance.root, '.git', 'objects') + ); + await assert.rejects( + manager.resolve('primary', id), + /does not own its Git objects/ + ); +}); + +test('snapshot rejects symlinked refs and never copies source hooks or config includes', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + await git(fixture.root, 'config', 'include.path', '/not-readable/config'); + const snapshot = await GitSourceSnapshot.admit( + ( + await source(fixture.root) + ).identity + ); + await snapshot.copyTo(join(fixture.parent, 'snapshot')); + assert.equal( + await readFile(join(fixture.parent, 'snapshot', 'config'), 'utf8'), + '[core]\nrepositoryformatversion = 0\nbare = true\n' + ); + await assert.rejects(stat(join(fixture.parent, 'snapshot', 'hooks')), { + code: 'ENOENT', + }); + await symlink( + join(fixture.root, 'README.md'), + join(fixture.root, '.git', 'refs', 'bad') + ); + await assert.rejects( + snapshot.copyTo(join(fixture.parent, 'snapshot-bad')), + /symbolic link/ + ); +}); + +test( + 'restart preserves a checkout reserved by a crashed provisioning process', + { timeout: 10_000 }, + async (t) => { + const fixture = await repository(); + const admitted = await source(fixture.root); + const storage = join(fixture.parent, 'instances'); + const id = '9'.repeat(64); + const child = spawn( + process.execPath, + [ + '--input-type=module', + '-e', + ` + import { GitWorktreeManager } from ${JSON.stringify( + new URL('./worktrees.js', import.meta.url).href + )}; + const manager = new GitWorktreeManager({ + maxCount: 1, root: ${JSON.stringify(storage)}, + sources: new Map([['primary', ${JSON.stringify(admitted)}]]), + prepareInstance: async () => { + process.stdout.write('setup-started'); + await new Promise(() => { setInterval(() => {}, 1000); }); + }, + }); + await manager.resolve('primary', ${JSON.stringify(id)}); + `, + ], + { stdio: ['ignore', 'pipe', 'pipe'] } + ); + const closed = once(child, 'close'); + t.after(async () => { + child.kill('SIGKILL'); + await closed; + await rm(fixture.parent, { recursive: true, force: true }); + }); + await once(child.stdout!, 'data'); + child.kill('SIGKILL'); + await closed; + const restarted = new GitWorktreeManager({ + maxCount: 1, + root: storage, + sources: new Map([['primary', admitted]]), + }); + await assert.rejects( + restarted.resolve('primary', id), + /operator recovery required/ + ); + await assert.rejects( + restarted.resolve('primary', '8'.repeat(64)), + /capacity is exhausted/ + ); + assert.equal( + await readFile( + join(await restarted.plannedRoot('primary', id), 'README.md'), + 'utf8' + ), + 'source\n' + ); + } +); + +test('cached checkouts revalidate their admitted source without deleting user work', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + const id = 'f'.repeat(64); + const instance = await manager.resolve('primary', id); + await writeFile(join(instance.root, 'pending.txt'), 'user work'); + await rename(fixture.root, `${fixture.root}.original`); + await mkdir(fixture.root); + await assert.rejects( + manager.resolve('primary', id), + /source changed after admission/ + ); + assert.equal( + await readFile(join(instance.root, 'pending.txt'), 'utf8'), + 'user work' + ); +}); + +test('preserves a failed setup checkout until executor cleanup is confirmed', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const options = { + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + prepareInstance: async () => { + throw new Error('setup failed'); + }, + discardInstance: async () => { + throw new Error('child cleanup unconfirmed'); + }, + }; + const manager = new GitWorktreeManager(options); + const id = 'a'.repeat(64); + await assert.rejects(manager.resolve('primary', id), /cleanup unconfirmed/); + const root = await manager.plannedRoot('primary', id); + assert.equal(await readFile(join(root, 'README.md'), 'utf8'), 'source\n'); + await assert.rejects( + new GitWorktreeManager(options).resolve('primary', id), + /operator recovery required/ + ); + await assert.rejects( + new GitWorktreeManager(options).resolve('primary', 'b'.repeat(64)), + /capacity is exhausted/ + ); + assert.equal(await readFile(join(root, 'README.md'), 'utf8'), 'source\n'); +}); + +test('cancellation waits for setup cleanup before releasing provisioning ownership', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + let started!: () => void; + const setupStarted = new Promise((resolve) => { + started = resolve; + }); + let cleanupFinished = false; + let setupRoot = ''; + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + prepareInstance: async (instance, signal) => { + const reservation = JSON.parse( + await readFile(`${instance.root}.complete`, 'utf8') + ); + assert.equal(reservation.provisioningFailed, true); + setupRoot = instance.root; + started(); + try { + await delay(60_000, undefined, { signal }); + await writeFile(join(instance.root, 'LATE'), 'should never happen'); + } finally { + await delay(20); + cleanupFinished = true; + } + }, + }); + const controller = new AbortController(); + const pending = manager.resolve('primary', 'a'.repeat(64), controller.signal); + const rejected = assert.rejects(pending, { name: 'AbortError' }); + await setupStarted; + controller.abort(); + await rejected; + assert.equal(cleanupFinished, true); + await assert.rejects(stat(setupRoot), { code: 'ENOENT' }); + await assert.rejects(stat(`${setupRoot}.complete`), { code: 'ENOENT' }); +}); + +test('cancels lock wait without provisioning while another caller continues', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + let started!: () => void; + const setupStarted = new Promise((resolve) => { + started = resolve; + }); + let release!: () => void; + const released = new Promise((resolve) => { + release = resolve; + }); + t.after(release); + let setups = 0; + const options = { + maxCount: 2, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + prepareInstance: async () => { + setups++; + started(); + await released; + }, + }; + const active = new GitWorktreeManager(options).resolve( + 'primary', + 'a'.repeat(64) + ); + await setupStarted; + const controller = new AbortController(); + const manager = new GitWorktreeManager(options); + const waiting = manager.resolve('primary', 'b'.repeat(64), controller.signal); + const rejected = assert.rejects(waiting, { name: 'AbortError' }); + await delay(75); + controller.abort(); + await rejected; + assert.equal(setups, 1); + release(); + await active; + await assert.rejects( + stat(await manager.plannedRoot('primary', 'b'.repeat(64))), + { code: 'ENOENT' } + ); +}); + +test('recovery preserves unknown directories and malformed completion records', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const options = { + maxCount: 4, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }; + const manager = new GitWorktreeManager(options); + const first = await manager.resolve('primary', 'a'.repeat(64)); + const unrelated = join(options.root, 'operator-backups', 'important'); + await mkdir(unrelated, { recursive: true }); + await writeFile(join(unrelated, 'notes'), 'keep'); + await manager.resolve('primary', 'b'.repeat(64)); + assert.equal(await readFile(join(unrelated, 'notes'), 'utf8'), 'keep'); + await writeFile(`${first.root}.complete`, '{"version":0}'); + await assert.rejects( + new GitWorktreeManager(options).resolve('primary', 'a'.repeat(64)), + /completion record is invalid/ + ); + await assert.rejects( + new GitWorktreeManager(options).resolve('primary', 'c'.repeat(64)), + /completion record is invalid/ + ); + assert.equal( + await readFile(join(first.root, 'README.md'), 'utf8'), + 'source\n' + ); +}); + +async function git(root: string, ...args: string[]): Promise { + const result = await execFileAsync('git', ['-C', root, ...args], { + encoding: 'utf8', + env: { + PATH: process.env.PATH, + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_TERMINAL_PROMPT: '0', + LC_ALL: 'C', + }, + }); + return result.stdout.trim(); +} + +async function repository( + objectFormat = 'sha1' +): Promise<{ parent: string; root: string }> { + const parent = await mkdtemp(join(tmpdir(), 'librechat-worktrees-')); + const root = join(parent, 'source'); + await execFileAsync('git', ['init', `--object-format=${objectFormat}`, root]); + await writeFile(join(root, 'README.md'), 'source\n'); + await git(root, 'add', 'README.md'); + await git( + root, + '-c', + 'user.name=Test', + '-c', + 'user.email=test@example.com', + 'commit', + '-m', + 'initial' + ); + return { parent, root: await realpath(root) }; +} + +async function source(root: string) { + return { root, identity: await captureWorkspaceRootIdentity(root) }; +} + +test('creates and reuses an isolated worktree for one conversation identity', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const manager = new GitWorktreeManager({ + maxCount: 4, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + const id = 'a'.repeat(64); + + const [first, concurrent] = await Promise.all([ + manager.resolve('primary', id), + manager.resolve('primary', id), + ]); + assert.deepEqual(concurrent, first); + assert.notEqual(first.root, fixture.root); + assert.equal( + (await realpath(join(first.root, '.git', 'objects'))).startsWith( + first.root + ), + true + ); + const instanceCommon = await realpath( + await git( + first.root, + 'rev-parse', + '--path-format=absolute', + '--git-common-dir' + ) + ); + assert.equal(instanceCommon.startsWith(first.root), true); + assert.equal( + await readFile(join(first.root, 'README.md'), 'utf8'), + 'source\n' + ); + + await writeFile(join(first.root, 'README.md'), 'conversation\n'); + assert.equal( + await readFile(join(fixture.root, 'README.md'), 'utf8'), + 'source\n' + ); + + const restarted = new GitWorktreeManager({ + maxCount: 4, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + assert.equal((await restarted.resolve('primary', id)).root, first.root); +}); + +test('replaces an incomplete checkout before admitting it after restart', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const storage = join(fixture.parent, 'instances'); + const id = 'c'.repeat(64); + const manager = new GitWorktreeManager({ + maxCount: 4, + root: storage, + sources: new Map([['primary', await source(fixture.root)]]), + }); + const first = await manager.resolve('primary', id); + await writeFile(join(first.root, 'README.md'), 'partial mutation\n'); + await rm(`${first.root}.complete`); + + const restarted = new GitWorktreeManager({ + maxCount: 4, + root: storage, + sources: new Map([['primary', await source(fixture.root)]]), + }); + const recovered = await restarted.resolve('primary', id); + assert.equal( + await readFile(join(recovered.root, 'README.md'), 'utf8'), + 'source\n' + ); +}); + +test('does not count an incomplete checkout against capacity after restart', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const storage = join(fixture.parent, 'instances'); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: storage, + sources: new Map([['primary', await source(fixture.root)]]), + }); + const abandoned = await manager.resolve('primary', 'c'.repeat(64)); + await rm(`${abandoned.root}.complete`); + const staleMarker = `${abandoned.root}.complete.1.tmp`; + await writeFile(staleMarker, '1\n'); + + const restarted = new GitWorktreeManager({ + maxCount: 1, + root: storage, + sources: new Map([['primary', await source(fixture.root)]]), + }); + const replacement = await restarted.resolve('primary', 'd'.repeat(64)); + assert.equal((await stat(replacement.root)).isDirectory(), true); + await assert.rejects(stat(abandoned.root), { code: 'ENOENT' }); + await assert.rejects(stat(staleMarker), { code: 'ENOENT' }); +}); + +test('keeps a conversation checkout independent of source object pruning', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + await writeFile(join(fixture.root, 'SECOND.md'), 'second\n'); + await git(fixture.root, 'add', 'SECOND.md'); + await git( + fixture.root, + '-c', + 'user.name=Test', + '-c', + 'user.email=test@example.com', + 'commit', + '-m', + 'second' + ); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + const instance = await manager.resolve('primary', 'e'.repeat(64)); + const retainedHead = await git(instance.root, 'rev-parse', 'HEAD'); + + await git(fixture.root, 'reset', '--hard', 'HEAD~1'); + await git(fixture.root, 'reflog', 'expire', '--expire=now', '--all'); + await git(fixture.root, 'gc', '--prune=now'); + + assert.equal(await git(instance.root, 'rev-parse', 'HEAD'), retainedHead); + assert.equal( + await readFile(join(instance.root, 'SECOND.md'), 'utf8'), + 'second\n' + ); + await assert.rejects( + readFile(join(instance.root, '.git', 'objects', 'info', 'alternates')), + { code: 'ENOENT' } + ); +}); + +test('dissociates a checkout from inherited source alternates', async (t) => { + const upstream = await repository(); + const sharedParent = await mkdtemp( + join(tmpdir(), 'librechat-shared-source-') + ); + const sharedRoot = join(sharedParent, 'source'); + t.after(() => + Promise.all([ + rm(upstream.parent, { recursive: true, force: true }), + rm(sharedParent, { recursive: true, force: true }), + ]) + ); + await execFileAsync('git', ['clone', '--shared', upstream.root, sharedRoot]); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(sharedParent, 'instances'), + sources: new Map([['primary', await source(await realpath(sharedRoot))]]), + }); + const instance = await manager.resolve('primary', 'f'.repeat(64)); + await rm(upstream.root, { recursive: true, force: true }); + + assert.equal( + await git(instance.root, 'rev-parse', 'HEAD^{commit}'), + await git(instance.root, 'rev-parse', 'HEAD') + ); + await assert.rejects( + readFile(join(instance.root, '.git', 'objects', 'info', 'alternates')), + { code: 'ENOENT' } + ); +}); + +test('provisions an orphan branch for a repository with an unborn HEAD', async (t) => { + const parent = await mkdtemp(join(tmpdir(), 'librechat-empty-source-')); + const root = join(parent, 'source'); + await execFileAsync('git', ['init', root]); + t.after(() => rm(parent, { recursive: true, force: true })); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(parent, 'instances'), + sources: new Map([['primary', await source(await realpath(root))]]), + }); + + const instance = await manager.resolve('primary', '0'.repeat(64)); + assert.match( + await git(instance.root, 'branch', '--show-current'), + /^librechat\/conversation-/ + ); + await assert.rejects(git(instance.root, 'rev-parse', '--verify', 'HEAD')); +}); + +test('rejects replacement of the admitted worktree storage root', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const storage = join(fixture.parent, 'instances'); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: storage, + sources: new Map([['primary', await source(fixture.root)]]), + }); + await manager.resolve('primary', '1'.repeat(64)); + await rename(storage, `${storage}.original`); + await mkdir(storage, { mode: 0o700 }); + + await assert.rejects( + manager.resolve('primary', '1'.repeat(64)), + /storage changed after admission/ + ); +}); + +test('keeps conversations and source repositories isolated', async (t) => { + const first = await repository(); + const second = await repository(); + t.after(() => + Promise.all([ + rm(first.parent, { recursive: true, force: true }), + rm(second.parent, { recursive: true, force: true }), + ]) + ); + const storage = await mkdtemp(join(tmpdir(), 'librechat-worktree-storage-')); + t.after(() => rm(storage, { recursive: true, force: true })); + const manager = new GitWorktreeManager({ + maxCount: 4, + root: storage, + sources: new Map([ + ['first', await source(first.root)], + ['second', await source(second.root)], + ]), + }); + + const firstConversation = await manager.resolve('first', '1'.repeat(64)); + const secondConversation = await manager.resolve('first', '2'.repeat(64)); + const otherRepository = await manager.resolve('second', '1'.repeat(64)); + assert.equal( + new Set([ + firstConversation.root, + secondConversation.root, + otherRepository.root, + ]).size, + 3 + ); +}); + +test('rejects invalid identities, overlapping storage and exhausted capacity', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + assert.throws( + () => + new GitWorktreeManager({ + cloneTimeoutMs: 29_999, + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([ + [ + 'primary', + { + root: fixture.root, + identity: { + path: fixture.root, + dev: '1', + ino: '1', + }, + }, + ], + ]), + }), + /clone timeout/ + ); + const overlapping = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.root, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + await assert.rejects( + overlapping.resolve('primary', 'a'.repeat(64)), + /must not overlap/ + ); + + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }); + await assert.rejects(manager.resolve('primary', '../escape'), /SHA-256/); + const first = await manager.resolve('primary', 'a'.repeat(64)); + assert.equal((await stat(first.root)).isDirectory(), true); + await assert.rejects( + manager.resolve('primary', 'b'.repeat(64)), + /capacity is exhausted/ + ); +}); + +test('serializes provisioning across manager instances sharing storage', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const options = { + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + }; + const results = await Promise.allSettled([ + new GitWorktreeManager(options).resolve('primary', 'a'.repeat(64)), + new GitWorktreeManager(options).resolve('primary', 'b'.repeat(64)), + ]); + assert.equal( + results.filter((result) => result.status === 'fulfilled').length, + 1 + ); + assert.equal( + results.filter((result) => result.status === 'rejected').length, + 1 + ); + assert.match( + ( + results.find( + (result) => result.status === 'rejected' + ) as PromiseRejectedResult + ).reason.message, + /capacity is exhausted/ + ); +}); + +test('prepares a new checkout before publishing its completion marker', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + let attempts = 0; + const options = { + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([['primary', await source(fixture.root)]]), + prepareInstance: async (instance: { root: string }) => { + attempts += 1; + if (attempts === 1) throw new Error('setup failed'); + await writeFile(join(instance.root, 'prepared'), 'yes\n'); + }, + }; + const id = 'c'.repeat(64); + await assert.rejects( + new GitWorktreeManager(options).resolve('primary', id), + /setup failed/ + ); + const instance = await new GitWorktreeManager(options).resolve('primary', id); + assert.equal( + await readFile(join(instance.root, 'prepared'), 'utf8'), + 'yes\n' + ); + assert.equal(attempts, 2); +}); + +test('preserves a completed checkout when its admitted source changes', async (t) => { + const first = await repository(); + const second = await repository(); + t.after(() => rm(first.parent, { recursive: true, force: true })); + t.after(() => rm(second.parent, { recursive: true, force: true })); + await writeFile(join(second.root, 'README.md'), 'replacement\n'); + await git(second.root, 'add', 'README.md'); + await git( + second.root, + '-c', + 'user.name=Test', + '-c', + 'user.email=test@example.com', + 'commit', + '-m', + 'replacement' + ); + const storage = join(first.parent, 'instances'); + const id = 'e'.repeat(64); + const original = await new GitWorktreeManager({ + maxCount: 1, + root: storage, + sources: new Map([['primary', await source(first.root)]]), + }).resolve('primary', id); + await writeFile(join(original.root, 'UNCOMMITTED.md'), 'user work\n'); + await assert.rejects( + new GitWorktreeManager({ + maxCount: 1, + root: storage, + sources: new Map([['primary', await source(second.root)]]), + }).resolve('primary', id), + /source identity changed/ + ); + assert.equal( + await readFile(join(original.root, 'UNCOMMITTED.md'), 'utf8'), + 'user work\n' + ); + assert.equal( + await readFile(join(original.root, 'README.md'), 'utf8'), + 'source\n' + ); +}); + +test('rejects a source whose admitted filesystem identity changed', async (t) => { + const fixture = await repository(); + t.after(() => rm(fixture.parent, { recursive: true, force: true })); + const metadata = await stat(fixture.root, { bigint: true }); + const manager = new GitWorktreeManager({ + maxCount: 1, + root: join(fixture.parent, 'instances'), + sources: new Map([ + [ + 'primary', + { + root: fixture.root, + identity: { + path: fixture.root, + dev: metadata.dev.toString(), + ino: (metadata.ino + 1n).toString(), + }, + }, + ], + ]), + }); + + await assert.rejects( + manager.resolve('primary', 'd'.repeat(64)), + /source changed after admission/ + ); +}); diff --git a/packages/code/src/worktrees.ts b/packages/code/src/worktrees.ts new file mode 100644 index 00000000..ab19aa38 --- /dev/null +++ b/packages/code/src/worktrees.ts @@ -0,0 +1,718 @@ +import { execFile } from 'node:child_process'; +import { createHash, randomUUID } from 'node:crypto'; +import { + lstat, + mkdir, + readFile, + readdir, + realpath, + rename, + rm, + stat, + writeFile, +} from 'node:fs/promises'; +import { isAbsolute, join, relative, resolve, sep } from 'node:path'; +import { promisify } from 'node:util'; + +import { matchesWorkspaceRoot } from './root-identity.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; +import { assertPrivateStorageAncestors } from './private-storage.js'; +import { withProcessLock } from './process-lock.js'; +import { GitSourceSnapshot } from './git-snapshot.js'; + +const execFileAsync = promisify(execFile); +const WORKTREE_INSTANCE_PATTERN = /^[a-f0-9]{64}$/; +const COMPLETION_TEMP_PATTERN = /^[a-f0-9]{64}\.complete\.[a-f0-9-]+\.tmp$/; +const GIT_TIMEOUT_MS = 30_000; +const DEFAULT_CLONE_TIMEOUT_MS = 5 * 60_000; + +export interface GitWorktreeSource { + identity: WorkspaceRootIdentity; + root: string; +} + +export interface GitWorktreeInstance { + id: string; + identity: WorkspaceRootIdentity; + root: string; + sourceWorkspaceId: string; +} + +export interface GitWorktreeManagerOptions { + cloneTimeoutMs?: number; + maxCount: number; + root: string; + sources: ReadonlyMap; + prepareInstance?: ( + instance: GitWorktreeInstance, + signal?: AbortSignal + ) => Promise; + discardInstance?: (instance: GitWorktreeInstance) => Promise | void; +} + +const PROVISIONING_LOCK = '.provision.lock'; + +function isInside(parent: string, candidate: string): boolean { + const path = relative(parent, candidate); + return ( + path === '' || + (path !== '..' && !path.startsWith(`..${sep}`) && !isAbsolute(path)) + ); +} + +function gitEnvironment(): NodeJS.ProcessEnv { + return { + PATH: process.env.PATH, + SYSTEMROOT: process.env.SYSTEMROOT, + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_TERMINAL_PROMPT: '0', + GIT_OPTIONAL_LOCKS: '0', + LC_ALL: 'C', + }; +} + +async function git( + root: string, + args: string[], + signal?: AbortSignal, + timeout = GIT_TIMEOUT_MS +): Promise { + const execution = execFileAsync( + 'git', + ['--no-optional-locks', '-C', root, ...args], + { + encoding: 'utf8', + env: gitEnvironment(), + maxBuffer: 16 * 1024, + signal, + timeout, + } + ); + const closed = new Promise((resolve) => + execution.child.once('close', () => resolve()) + ); + try { + return (await execution).stdout.trim(); + } finally { + // execFile's AbortError callback can run before its child exits. Retain the + // provisioning lock and directory until the writer is actually gone. + const killTimer = setTimeout(() => execution.child.kill('SIGKILL'), 1000); + killTimer.unref(); + try { + await closed; + } finally { + clearTimeout(killTimer); + } + } +} + +async function sourceConfig( + root: string, + key: string, + signal?: AbortSignal +): Promise { + try { + const remote = await git( + root, + [ + 'config', + '--no-includes', + '--file', + join(root, 'source-config'), + '--get', + key, + ], + signal + ); + return remote || undefined; + } catch { + signal?.throwIfAborted(); + return undefined; + } +} + +async function hasCommittedHead( + root: string, + signal?: AbortSignal +): Promise { + try { + await git(root, ['rev-parse', '--verify', 'HEAD'], signal); + return true; + } catch (error) { + signal?.throwIfAborted(); + if (error instanceof Error && 'code' in error && error.code === 128) { + return false; + } + throw error; + } +} + +async function directoryIdentity(path: string): Promise { + const metadata = await lstat(path, { bigint: true }); + if (!metadata.isDirectory() || metadata.isSymbolicLink()) { + throw new Error('Conversation worktree must be a real directory'); + } + return { + path, + dev: metadata.dev.toString(), + ino: metadata.ino.toString(), + }; +} + +export class GitWorktreeManager { + private readonly instances = new Map(); + private readonly sourceSnapshots = new Map< + string, + Promise + >(); + private canonicalRoot?: Promise<{ + identity: WorkspaceRootIdentity; + path: string; + }>; + + constructor(private readonly options: GitWorktreeManagerOptions) { + if ( + !Number.isSafeInteger(options.maxCount) || + options.maxCount < 1 || + options.maxCount > 1024 || + options.sources.size === 0 + ) { + throw new Error( + 'Conversation worktree capacity must be between 1 and 1024' + ); + } + if ( + options.cloneTimeoutMs !== undefined && + (!Number.isSafeInteger(options.cloneTimeoutMs) || + options.cloneTimeoutMs < GIT_TIMEOUT_MS || + options.cloneTimeoutMs > 30 * 60_000) + ) { + throw new Error( + 'Conversation worktree clone timeout must be between 30000 and 1800000 milliseconds' + ); + } + } + + private async root(): Promise { + this.canonicalRoot ??= (async () => { + const configuredRoot = resolve(this.options.root); + await assertPrivateStorageAncestors(configuredRoot, true); + await mkdir(configuredRoot, { + mode: 0o700, + recursive: true, + }); + await assertPrivateStorageAncestors(configuredRoot); + const root = await realpath(this.options.root); + await assertPrivateStorageAncestors(root); + const metadata = await stat(root); + if ( + !metadata.isDirectory() || + (process.platform !== 'win32' && (metadata.mode & 0o022) !== 0) + ) { + throw new Error( + 'Conversation worktree root must not be group or world writable' + ); + } + for (const source of this.options.sources.values()) { + const sourceRoot = await realpath(source.root); + if (isInside(sourceRoot, root) || isInside(root, sourceRoot)) { + throw new Error( + 'Conversation worktree storage must not overlap a source workspace' + ); + } + } + return { + identity: await directoryIdentity(root), + path: root, + }; + })(); + const root = await this.canonicalRoot; + if (!(await matchesWorkspaceRoot(root.path, root.identity))) { + throw new Error('Conversation worktree storage changed after admission'); + } + return root.path; + } + + private key(sourceWorkspaceId: string, instanceId: string): string { + return `${sourceWorkspaceId}\0${instanceId}`; + } + + private branch(sourceWorkspaceId: string, instanceId: string): string { + const source = createHash('sha256') + .update(sourceWorkspaceId) + .digest('hex') + .slice(0, 8); + return `librechat/conversation-${source}-${instanceId.slice(0, 31)}`; + } + + private async instancePath( + sourceWorkspaceId: string, + instanceId: string + ): Promise { + const sourceDirectory = createHash('sha256') + .update(sourceWorkspaceId) + .digest('hex') + .slice(0, 24); + return join(await this.root(), sourceDirectory, instanceId); + } + + async plannedRoot( + sourceWorkspaceId: string, + instanceId: string + ): Promise { + if (!WORKTREE_INSTANCE_PATTERN.test(instanceId)) { + throw new Error( + 'Conversation worktree identity must be a SHA-256 digest' + ); + } + if (!this.options.sources.has(sourceWorkspaceId)) { + throw new Error('Conversation worktree source is unavailable'); + } + return await this.instancePath(sourceWorkspaceId, instanceId); + } + + async prepare(): Promise { + await this.root(); + await Promise.all( + [...this.options.sources].map(async ([_workspaceId, source]) => { + const sourceRoot = await this.admittedSourceRoot(source); + await this.sourceSnapshot(sourceRoot, source); + }) + ); + } + + private async admittedSourceRoot(source: GitWorktreeSource): Promise { + const sourceRoot = await realpath(source.root); + if (!(await matchesWorkspaceRoot(sourceRoot, source.identity))) { + throw new Error('Conversation worktree source changed after admission'); + } + return sourceRoot; + } + + private async sourceSnapshot( + root: string, + source: GitWorktreeSource + ): Promise { + let snapshot = this.sourceSnapshots.get(root); + if (!snapshot) { + snapshot = GitSourceSnapshot.admit(source.identity); + this.sourceSnapshots.set(root, snapshot); + } + const admitted = await snapshot; + await admitted.validate(); + return admitted; + } + + private async countInstances(): Promise { + const root = await this.root(); + const sourceDirectories = await readdir(root, { withFileTypes: true }); + let count = 0; + for (const sourceDirectory of sourceDirectories) { + if (sourceDirectory.name.startsWith(PROVISIONING_LOCK)) continue; + if ( + !/^[a-f0-9]{24}$/.test(sourceDirectory.name) || + !sourceDirectory.isDirectory() || + sourceDirectory.isSymbolicLink() + ) + continue; + const entries = await readdir(join(root, sourceDirectory.name), { + withFileTypes: true, + }); + const reserved = new Set(); + for (const entry of entries) { + const id = entry.name.endsWith('.complete') + ? entry.name.slice(0, -9) + : ''; + if (!WORKTREE_INSTANCE_PATTERN.test(id)) continue; + if (!entry.isFile() || entry.isSymbolicLink()) + throw new Error('Invalid worktree reservation'); + if ( + await this.hasCompletionMarker(join(root, sourceDirectory.name, id)) + ) { + reserved.add(id); + count += 1; + } + } + for (const entry of entries) { + if (entry.isFile() && COMPLETION_TEMP_PATTERN.test(entry.name)) { + await rm(join(root, sourceDirectory.name, entry.name), { + force: true, + }); + continue; + } + if ( + !WORKTREE_INSTANCE_PATTERN.test(entry.name) || + !entry.isDirectory() || + entry.isSymbolicLink() + ) + continue; + const path = join(root, sourceDirectory.name, entry.name); + if (!reserved.has(entry.name)) { + await rm(path, { recursive: true, force: true }); + await rm(this.completionMarker(path), { force: true }); + } + } + } + return count; + } + + private async withProvisioningLock( + operation: () => Promise, + signal?: AbortSignal + ): Promise { + return await withProcessLock( + join(await this.root(), PROVISIONING_LOCK), + operation, + signal + ); + } + + private completionMarker(path: string): string { + return `${path}.complete`; + } + + private async hasCompletionMarker( + path: string, + source?: WorkspaceRootIdentity, + sourceGit?: string + ): Promise { + try { + const record = JSON.parse( + await readFile(this.completionMarker(path), 'utf8') + ) as { + version?: unknown; + source?: Partial; + provisioningFailed?: boolean; + sourceGit?: string; + }; + const valid = + record.version === 2 && + typeof record.sourceGit === 'string' && + WORKTREE_INSTANCE_PATTERN.test(record.sourceGit) && + typeof record.source?.path === 'string' && + typeof record.source.dev === 'string' && + typeof record.source.ino === 'string'; + if (!valid) + throw new Error( + 'Conversation worktree completion record is invalid; existing checkout preserved' + ); + if ( + source != null && + (record.source!.path !== source.path || + record.source!.dev !== source.dev || + record.source!.ino !== source.ino || + record.sourceGit !== sourceGit) + ) { + throw new Error( + 'Conversation worktree source identity changed; existing checkout preserved' + ); + } + if (source != null && record.provisioningFailed) { + throw new Error( + 'Conversation worktree setup cleanup is unconfirmed; operator recovery required' + ); + } + return true; + } catch (error) { + if ( + error instanceof Error && + 'code' in error && + error.code === 'ENOENT' + ) { + return false; + } + throw error; + } + } + + private async writeCompletionMarker( + path: string, + source: WorkspaceRootIdentity, + sourceGit: string, + provisioningFailed = false + ): Promise { + const marker = this.completionMarker(path); + const temporary = `${marker}.${randomUUID()}.tmp`; + try { + await writeFile( + temporary, + `${JSON.stringify({ + version: 2, + source, + sourceGit, + ...(provisioningFailed ? { provisioningFailed: true } : {}), + })}\n`, + { mode: 0o600, flag: 'wx' } + ); + await rename(temporary, marker); + } finally { + await rm(temporary, { force: true }); + } + } + + private async validateRepository( + sourceWorkspaceId: string, + instanceId: string, + path: string, + signal?: AbortSignal + ): Promise { + const canonicalPath = await realpath(path); + if (canonicalPath !== path || !isInside(await this.root(), canonicalPath)) { + throw new Error( + 'Conversation worktree escaped its configured storage root' + ); + } + signal?.throwIfAborted(); + const instanceCommon = join(canonicalPath, '.git'); + const gitMetadata = await lstat(instanceCommon); + if ( + !gitMetadata.isDirectory() || + gitMetadata.isSymbolicLink() || + (await realpath(instanceCommon)) !== instanceCommon + ) { + throw new Error('Conversation worktree does not own its Git metadata'); + } + try { + await lstat(join(instanceCommon, 'commondir')); + throw new Error( + 'Conversation worktree must not redirect its Git metadata' + ); + } catch (error) { + if ( + !(error instanceof Error) || + !('code' in error) || + error.code !== 'ENOENT' + ) + throw error; + } + const instanceObjects = await realpath(join(instanceCommon, 'objects')); + if (!isInside(canonicalPath, instanceObjects)) { + throw new Error('Conversation worktree does not own its Git objects'); + } + try { + await lstat(join(instanceObjects, 'info', 'alternates')); + throw new Error( + 'Conversation worktree must not use external Git objects' + ); + } catch (error) { + if ( + !(error instanceof Error) || + !('code' in error) || + error.code !== 'ENOENT' + ) { + throw error; + } + } + return { + id: instanceId, + identity: await directoryIdentity(canonicalPath), + root: canonicalPath, + sourceWorkspaceId, + }; + } + + private async validateExisting( + sourceWorkspaceId: string, + instanceId: string, + path: string, + source: WorkspaceRootIdentity, + sourceGit: string, + signal?: AbortSignal + ): Promise { + if (!(await this.hasCompletionMarker(path, source, sourceGit))) { + const error = new Error('Conversation worktree is incomplete'); + Object.assign(error, { code: 'EINCOMPLETE' }); + throw error; + } + return await this.validateRepository( + sourceWorkspaceId, + instanceId, + path, + signal + ); + } + + private async createLocked( + sourceWorkspaceId: string, + instanceId: string, + signal?: AbortSignal + ): Promise { + if (!WORKTREE_INSTANCE_PATTERN.test(instanceId)) { + throw new Error( + 'Conversation worktree identity must be a SHA-256 digest' + ); + } + const source = this.options.sources.get(sourceWorkspaceId); + if (!source) throw new Error('Conversation worktree source is unavailable'); + const sourceRoot = await this.admittedSourceRoot(source); + const snapshot = await this.sourceSnapshot(sourceRoot, source); + const path = await this.instancePath(sourceWorkspaceId, instanceId); + try { + return await this.validateExisting( + sourceWorkspaceId, + instanceId, + path, + source.identity, + snapshot.fingerprint, + signal + ); + } catch (error) { + if (!(error instanceof Error) || !('code' in error)) { + throw error; + } + if (error.code === 'EINCOMPLETE') { + await rm(path, { recursive: true, force: true }); + await rm(this.completionMarker(path), { force: true }); + } else if (error.code !== 'ENOENT') { + throw error; + } + } + if ((await this.countInstances()) >= this.options.maxCount) { + throw new Error('Conversation worktree capacity is exhausted'); + } + await mkdir(resolve(path, '..'), { mode: 0o700, recursive: true }); + const branch = this.branch(sourceWorkspaceId, instanceId); + let instance: GitWorktreeInstance | undefined; + const staging = `${path}.source`; + try { + // Reserve before launching any writer. A worker crash may leave a Git + // child or setup executor alive after the parent's kernel lock releases. + // Recovery must not sweep or reuse that uncertain directory. + await this.writeCompletionMarker( + path, + source.identity, + snapshot.fingerprint, + true + ); + const cloneSignal = AbortSignal.any([ + ...(signal ? [signal] : []), + AbortSignal.timeout( + this.options.cloneTimeoutMs ?? DEFAULT_CLONE_TIMEOUT_MS + ), + ]); + await snapshot.copyTo(staging, cloneSignal); + const remote = await sourceConfig( + staging, + 'remote.origin.url', + cloneSignal + ); + const objectFormat = await sourceConfig( + staging, + 'extensions.objectformat', + cloneSignal + ); + if ( + objectFormat && + objectFormat !== 'sha1' && + objectFormat !== 'sha256' + ) { + throw new Error('Unsupported source Git object format'); + } + if (objectFormat === 'sha256') { + await writeFile( + join(staging, 'config'), + '[core]\nrepositoryformatversion = 1\nbare = true\n[extensions]\nobjectformat = sha256\n', + { mode: 0o600 } + ); + } + await git( + resolve(path, '..'), + ['clone', '--local', '--no-checkout', '--no-tags', staging, path], + cloneSignal, + this.options.cloneTimeoutMs ?? DEFAULT_CLONE_TIMEOUT_MS + ); + await rm(staging, { recursive: true, force: true }); + const sourceHasHead = await hasCommittedHead(path, signal); + if (remote) { + await git(path, ['remote', 'set-url', 'origin', remote], signal); + } else { + await git(path, ['remote', 'remove', 'origin'], signal); + } + await git( + path, + sourceHasHead + ? ['checkout', '--force', '-b', branch, 'HEAD'] + : ['checkout', '--orphan', branch], + signal + ); + instance = await this.validateRepository( + sourceWorkspaceId, + instanceId, + path, + signal + ); + await this.options.prepareInstance?.(instance, signal); + signal?.throwIfAborted(); + if (!(await matchesWorkspaceRoot(instance.root, instance.identity))) { + throw new Error('Conversation worktree changed during setup'); + } + await this.validateRepository( + sourceWorkspaceId, + instanceId, + path, + signal + ); + await this.admittedSourceRoot(source); + await snapshot.validate(); + await this.writeCompletionMarker( + path, + source.identity, + snapshot.fingerprint + ); + return instance; + } catch (error) { + if (instance) { + // The reservation remains until executor cleanup is confirmed. + await this.options.discardInstance?.(instance); + } + await rm(path, { recursive: true, force: true }); + await rm(staging, { recursive: true, force: true }); + await rm(this.completionMarker(path), { force: true }); + throw error; + } + } + + private async create( + sourceWorkspaceId: string, + instanceId: string, + signal?: AbortSignal + ): Promise { + return await this.withProvisioningLock( + () => this.createLocked(sourceWorkspaceId, instanceId, signal), + signal + ); + } + + async resolve( + sourceWorkspaceId: string, + instanceId: string, + signal?: AbortSignal + ): Promise { + signal?.throwIfAborted(); + const key = this.key(sourceWorkspaceId, instanceId); + const cached = this.instances.get(key); + if (cached) { + await this.root(); + const source = this.options.sources.get(sourceWorkspaceId)!; + await this.sourceSnapshot(await this.admittedSourceRoot(source), source); + if (!(await matchesWorkspaceRoot(cached.root, cached.identity))) { + this.instances.delete(key); + throw new Error('Conversation worktree changed after admission'); + } + await this.validateRepository( + sourceWorkspaceId, + instanceId, + cached.root, + signal + ); + return cached; + } + // The same kernel lock coordinates callers and processes. Keep cancellation + // attached through setup and cleanup; never release a lease while detached + // provisioning is still mutating the checkout. + const instance = await this.create(sourceWorkspaceId, instanceId, signal); + this.instances.set(key, instance); + return instance; + } +} diff --git a/service/src/bridge/concurrent-store.test.ts b/service/src/bridge/concurrent-store.test.ts index 736b7216..33f3d082 100644 --- a/service/src/bridge/concurrent-store.test.ts +++ b/service/src/bridge/concurrent-store.test.ts @@ -2,7 +2,10 @@ import { afterEach, expect, test } from 'bun:test'; import RedisMock from 'ioredis-mock'; import type Redis from 'ioredis'; import { RedisBridgeStore } from './store'; -import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; +import { + BRIDGE_PROTOCOL_VERSION, + workspaceIsolationKey, +} from '../../../packages/code/src/protocol'; import type { CodeBridgeAssignment } from './store'; const redis = new RedisMock() as unknown as Redis; @@ -26,13 +29,20 @@ async function register(workspaceLeaseSlots = 2) { workspaceTools: { protocolVersion: BRIDGE_PROTOCOL_VERSION, operations: ['read_file'], - workspaces: [{ id: 'a' }, { id: 'b' }], + workspaces: [ + { id: 'a', workspaceInstances: ['git_worktree'] }, + { id: 'b' }, + ], }, }, }); await store.confirmReady(workerId, incarnationId, generation); } -function dispatch(workspaceId: string, signal = new AbortController().signal) { +function dispatch( + workspaceId: string, + signal = new AbortController().signal, + workspaceInstanceId?: string, +) { const promise = store.dispatchWorkspaceTool({ workerId, signal, @@ -41,6 +51,7 @@ function dispatch(workspaceId: string, signal = new AbortController().signal) { protocolVersion: BRIDGE_PROTOCOL_VERSION, operation: 'read_file', workspaceId, + ...(workspaceInstanceId == null ? {} : { workspaceInstanceId }), path: 'file.txt', }, }); @@ -346,6 +357,52 @@ test('same-root work waits while another root progresses', async () => { await Promise.all([nextA, b]); }); +test('conversation worktrees on one source use independent capacity lanes', async () => { + await register(); + const firstId = 'a'.repeat(64); + const secondId = 'b'.repeat(64); + const firstPending = dispatch('a', undefined, firstId); + const first = (await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 0, + ))!; + const samePending = dispatch('a', undefined, firstId); + const secondPending = dispatch('a', undefined, secondId); + const second = (await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 1, + ))!; + expect(second.request).toMatchObject({ + workspaceId: 'a', + workspaceInstanceId: secondId, + }); + await settle(first); + await firstPending; + const same = (await store.lease( + workerId, + incarnationId, + 1000, + undefined, + undefined, + 0, + ))!; + expect(same.request).toMatchObject({ + workspaceId: 'a', + workspaceInstanceId: firstId, + }); + await settle(second); + await settle(same); + await Promise.all([samePending, secondPending]); +}); + test('queued cancellation never leases and does not block another root', async () => { await register(); const a = dispatch('a'); @@ -548,7 +605,11 @@ test('post-settlement fences are authenticated, idempotent, and invalidated by r await expect(dispatch('a')).rejects.toMatchObject({ code: 'WORKSPACE_QUARANTINED', }); - await store.resetWorkspace(workerId, incarnationId, 'native-workspace:a'); + await store.resetWorkspace( + workerId, + incarnationId, + `native-workspace:${workspaceIsolationKey('a')}`, + ); await expect( store.settle( workerId, diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 73870772..5b319733 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -430,6 +430,7 @@ router.post( supportedWorkspaceEditFileFeatures: ['expected_base_sha256'], supportedWorkspaceListFileFeatures: ['after_path'], supportedWorkspaceProgrammaticLanguages: ['bash'], + supportedWorkspaceInstanceTypes: ['git_worktree'], }); } catch (error) { if (error instanceof BridgeStoreError) { diff --git a/service/src/bridge/store.test.ts b/service/src/bridge/store.test.ts index 9d271551..354108af 100644 --- a/service/src/bridge/store.test.ts +++ b/service/src/bridge/store.test.ts @@ -4,7 +4,7 @@ import RedisMock from 'ioredis-mock'; import type Redis from 'ioredis'; import type * as t from '../types'; import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; -import { RedisBridgeStore } from './store'; +import { RedisBridgeStore, workspaceAdmissionId } from './store'; import type { RegisteredBridgeWorker } from './store'; @@ -27,6 +27,16 @@ afterEach(async () => { }); describe('RedisBridgeStore', () => { + test('uses disjoint admission identities for roots and worktree instances', () => { + const instanceId = 'a'.repeat(64); + expect( + workspaceAdmissionId(`foo:git-worktree:${instanceId}`), + ).not.toBe(workspaceAdmissionId('foo', instanceId)); + expect(workspaceAdmissionId('foo')).not.toBe( + workspaceAdmissionId('workspace:foo'), + ); + }); + test('reports an atomic, capability-limited worker status snapshot', async () => { const store = new RedisBridgeStore(redis); const capabilities = { diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 4eaabd7f..2ffeec66 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -15,8 +15,9 @@ import { BRIDGE_PROTOCOL_VERSION, isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, - isWorkspaceToolRequest, - isWorkspaceToolResult, + isWorkspaceToolRequest, + isWorkspaceToolResult, + workspaceIsolationKey, } from '../../../packages/code/src/protocol'; import type { BridgeWorkerBinding } from './pairing'; import { BridgeAdmissionQueue } from './admission'; @@ -124,6 +125,12 @@ function supportsWorkspaceTool( if (!supportsOperation) { return supportsOperation; } + if ( + request.workspaceInstanceId !== undefined && + workspace.workspaceInstances?.includes('git_worktree') !== true + ) { + return false; + } if (request.operation === 'list_files' && request.afterPath !== undefined) { return capabilities?.listFileFeatures?.includes('after_path') === true; } @@ -156,6 +163,7 @@ function supportsWorkspaceProgrammatic( registration: RegisteredBridgeWorker, workspaceId: string, language: string, + workspaceInstanceId?: string, ): boolean { const capabilities = registration.capabilities.workspaceTools; const workspace = capabilities?.workspaces.find( @@ -163,6 +171,8 @@ function supportsWorkspaceProgrammatic( ); return ( workspace != null && + (workspaceInstanceId === undefined || + workspace.workspaceInstances?.includes('git_worktree') === true) && capabilities?.operations.includes('execute_command') === true && (workspace.operations == null || workspace.operations.includes('execute_command')) && @@ -172,6 +182,25 @@ function supportsWorkspaceProgrammatic( ); } +function workspaceInstanceId(body: t.PayloadBody): string | undefined { + if ( + typeof body === 'object' && + body != null && + 'workspace_instance_id' in body && + typeof body.workspace_instance_id === 'string' + ) { + return body.workspace_instance_id; + } + return undefined; +} + +export function workspaceAdmissionId( + workspaceId: string, + instanceId?: string, +): string { + return workspaceIsolationKey(workspaceId, instanceId); +} + function workerKey(workerId: string): string { return `${PREFIX}:worker:${encodeURIComponent(workerId)}`; } @@ -807,6 +836,7 @@ export class RedisBridgeStore { registration, args.workspaceId, args.body.language, + workspaceInstanceId(args.body), ) ) { throw new BridgeStoreError( @@ -843,6 +873,15 @@ export class RedisBridgeStore { let workspaceLeaseSlot: number | undefined; const selectedWorkspaceId = args.workspaceRequest?.workspaceId ?? args.workspaceId; + const selectedWorkspaceInstanceId = + args.workspaceRequest?.workspaceInstanceId ?? workspaceInstanceId(args.body); + const selectedWorkspaceAdmissionId = + selectedWorkspaceId == null + ? undefined + : workspaceAdmissionId( + selectedWorkspaceId, + selectedWorkspaceInstanceId, + ); const workspaceSlots = selectedWorkspaceId != null && (registration.capabilities.workspaceLeaseSlots ?? 1) > 1 @@ -864,7 +903,7 @@ export class RedisBridgeStore { args.deadlineAtMs, workspaceSlots == null ? undefined - : selectedWorkspaceId, + : selectedWorkspaceAdmissionId, ), args, 'Bridge admission enqueue', @@ -899,7 +938,7 @@ export class RedisBridgeStore { workerId: args.workerId, incarnationId: lockIncarnationId, assignmentId, - workspaceId: selectedWorkspaceId!, + workspaceId: selectedWorkspaceAdmissionId!, capacity: registration.capabilities.workspaceLeaseSlots!, expiresAtMs: Date.now() + ttlSeconds * 1000, }), @@ -961,6 +1000,7 @@ export class RedisBridgeStore { current.registration, args.workspaceId, args.body.language, + workspaceInstanceId(args.body), )) ) { throw new BridgeStoreError( @@ -986,14 +1026,14 @@ export class RedisBridgeStore { generation, leaseToken, leaseTokenHash: tokenHash(leaseToken), - ...(selectedWorkspaceId == null ? {} : { - workspaceFence: `native-workspace:${selectedWorkspaceId}`, + ...(selectedWorkspaceAdmissionId == null ? {} : { + workspaceFence: `native-workspace:${selectedWorkspaceAdmissionId}`, }), ...(workspaceLeaseSlot === undefined ? {} : { workspaceLeaseSlot, - workspaceFence: `native-workspace:${selectedWorkspaceId!}`, + workspaceFence: `native-workspace:${selectedWorkspaceAdmissionId!}`, }), ...(registration.identityId != null ? { workerIdentityId: registration.identityId } @@ -1082,6 +1122,7 @@ export class RedisBridgeStore { replacement.registration, args.workspaceId, args.body.language, + workspaceInstanceId(args.body), ) ) { throw new BridgeStoreError( diff --git a/service/src/bridge/workspace-instance.test.ts b/service/src/bridge/workspace-instance.test.ts new file mode 100644 index 00000000..44e93300 --- /dev/null +++ b/service/src/bridge/workspace-instance.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from 'bun:test'; +import { principalWorkspaceInstanceId } from './workspace-instance'; + +describe('principalWorkspaceInstanceId', () => { + it('keeps principal components distinct even when identifiers contain delimiters', () => { + const instanceId = 'a'.repeat(64); + expect(principalWorkspaceInstanceId({ instanceId, tenantId: 'tenant\0user', principalId: 'a' })) + .not.toBe(principalWorkspaceInstanceId({ instanceId, tenantId: 'tenant', principalId: 'user\0a' })); + }); + it('is stable only within the same authenticated principal', () => { + const instanceId = 'a'.repeat(64); + const first = principalWorkspaceInstanceId({ + instanceId, + tenantId: 'tenant', + principalId: 'user-a', + }); + expect(first).toMatch(/^[a-f0-9]{64}$/); + expect( + principalWorkspaceInstanceId({ + instanceId, + tenantId: 'tenant', + principalId: 'user-a', + }) + ).toBe(first); + expect( + principalWorkspaceInstanceId({ + instanceId, + tenantId: 'tenant', + principalId: 'user-b', + }) + ).not.toBe(first); + expect( + principalWorkspaceInstanceId({ + instanceId, + tenantId: 'other', + principalId: 'user-a', + }) + ).not.toBe(first); + }); +}); diff --git a/service/src/bridge/workspace-instance.ts b/service/src/bridge/workspace-instance.ts new file mode 100644 index 00000000..4c49afbb --- /dev/null +++ b/service/src/bridge/workspace-instance.ts @@ -0,0 +1,17 @@ +import { createHash } from 'node:crypto'; + +/** Bind a caller-selected conversation identity to the authenticated principal. */ +export function principalWorkspaceInstanceId(args: { + instanceId: string; + tenantId: string; + principalId: string; +}): string { + return createHash('sha256') + .update(JSON.stringify([ + 'codeapi-workspace-instance-v1', + args.tenantId, + args.principalId, + args.instanceId, + ])) + .digest('hex'); +} diff --git a/service/src/service/programmatic-router.ts b/service/src/service/programmatic-router.ts index 1063fbfe..a646b9ad 100644 --- a/service/src/service/programmatic-router.ts +++ b/service/src/service/programmatic-router.ts @@ -44,6 +44,7 @@ import { SessionKeyResolutionError, } from '../session-key'; import { getCredentialId, getPrincipalOrReject } from '../auth/principal'; +import { principalWorkspaceInstanceId } from '../bridge/workspace-instance'; import { getExecutionIdentity } from '../execution-identity'; import { PROGRAMMATIC_RUNTIME_SESSION_EXEMPTION } from '../runtime-session/job-policy'; import { @@ -81,6 +82,7 @@ import { authorizeRequestedFiles, } from './file-authorization'; import { + bindReplayWorkspaceInstance, buildReplayExecutionState, resolveReplayStateSandboxBackend, } from './programmatic-state'; @@ -335,7 +337,7 @@ function buildReplayPayload( state: ExecutionState, history: Record, ): t.PayloadBody { - return createProgrammaticPayload({ + const payload = createProgrammaticPayload({ req, session_id: state.session_id, execution_id: state.execution_id, @@ -347,6 +349,7 @@ function buildReplayPayload( filesOverride: state.files, language: state.language ?? 'python', }); + return bindReplayWorkspaceInstance(payload, state); } async function runReplayIteration( @@ -501,10 +504,17 @@ async function handleReplayInitial( userId: string; bridgeWorkerId?: string; workspaceId?: string; + workspaceInstanceId?: string; }, cancellation: ReplayRequestCancellation, ): Promise { - const { apiKeyId, userId, bridgeWorkerId, workspaceId } = params; + const { + apiKeyId, + userId, + bridgeWorkerId, + workspaceId, + workspaceInstanceId, + } = params; const { code, tools, user_id, files } = req.body as t.ProgrammaticRequestBody; let timeout: number; @@ -660,6 +670,7 @@ async function handleReplayInitial( language, bridgeWorkerId, workspaceId, + workspaceInstanceId, executionProfile: env.EXECUTION_PROFILE, executionProfileSource: env.EXECUTION_PROFILE_SOURCE, sandboxBackend: resolveReplayStateSandboxBackend({ @@ -1279,6 +1290,7 @@ router.post( const requestedLanguage: unknown = rawBody.language ?? rawBody.lang; let bridgeWorkerId: string | undefined; let workspaceId: string | undefined; + let workspaceInstanceId: string | undefined; if (continuation_token == null || continuation_token === '') { try { const bridgeSelection = resolveBridgeWorkerSelection({ @@ -1312,6 +1324,23 @@ router.post( } workspaceId = requestedWorkspaceId; } + const requestedWorkspaceInstanceId = rawBody.workspace_instance_id; + if (requestedWorkspaceInstanceId !== undefined) { + if ( + workspaceId == null || + typeof requestedWorkspaceInstanceId !== 'string' || + !/^[a-f0-9]{64}$/.test(requestedWorkspaceInstanceId) + ) { + return res.status(400).json({ + error: 'Invalid code workspace instance ID', + }); + } + workspaceInstanceId = principalWorkspaceInstanceId({ + instanceId: requestedWorkspaceInstanceId, + tenantId: principal.tenantId, + principalId: principal.userId, + }); + } } catch (error) { if (error instanceof BridgeWorkerSelectionError) { return res @@ -1428,6 +1457,7 @@ router.post( userId, bridgeWorkerId, workspaceId, + workspaceInstanceId, }, cancellation); } if (workspaceId != null) { diff --git a/service/src/service/programmatic-state.test.ts b/service/src/service/programmatic-state.test.ts index 81405021..b0bff413 100644 --- a/service/src/service/programmatic-state.test.ts +++ b/service/src/service/programmatic-state.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from 'bun:test'; import type { CodeApiAuthContext, RequestFile } from '../types'; import type { LCTool } from '../preamble'; import { + bindReplayWorkspaceInstance, buildReplayExecutionState, resolveReplayStateSandboxBackend, } from './programmatic-state'; @@ -84,6 +85,7 @@ describe('buildReplayExecutionState', () => { authContext, bridgeWorkerId: 'code-user_123', workspaceId: 'project-a', + workspaceInstanceId: 'a'.repeat(64), sandboxBackend: 'remote-bridge', executionProfile: 'stateful', executionProfileSource: 'explicit', @@ -104,6 +106,7 @@ describe('buildReplayExecutionState', () => { apiKeyId: 'key_legacy', bridgeWorkerId: 'code-user_123', workspaceId: 'project-a', + workspaceInstanceId: 'a'.repeat(64), sandboxBackend: 'remote-bridge', executionProfile: 'stateful', executionProfileSource: 'explicit', @@ -119,6 +122,19 @@ describe('buildReplayExecutionState', () => { }); }); + test('binds a selected conversation checkout into every replay payload', () => { + const payload = { language: 'bash', version: '5.2', files: [] }; + expect( + bindReplayWorkspaceInstance(payload, { + workspaceInstanceId: 'b'.repeat(64), + }), + ).toEqual({ + ...payload, + workspace_instance_id: 'b'.repeat(64), + }); + expect(bindReplayWorkspaceInstance(payload, {})).toBe(payload); + }); + test('falls back to JWT identity only when no managed auth context exists', () => { const state = build({ authContext: undefined, userId: 'user_api_key' }); diff --git a/service/src/service/programmatic-state.ts b/service/src/service/programmatic-state.ts index e6bda59a..38c72486 100644 --- a/service/src/service/programmatic-state.ts +++ b/service/src/service/programmatic-state.ts @@ -39,6 +39,7 @@ export interface BuildReplayExecutionStateParams { language: 'python' | 'bash'; bridgeWorkerId?: string; workspaceId?: string; + workspaceInstanceId?: string; sandboxBackend?: SandboxBackendName; executionProfile: ExecutionProfile; executionProfileSource: ExecutionProfileSource; @@ -68,6 +69,7 @@ export function buildReplayExecutionState( apiKeyId: params.apiKeyId, bridgeWorkerId: params.bridgeWorkerId, workspaceId: params.workspaceId, + workspaceInstanceId: params.workspaceInstanceId, sandboxBackend: params.sandboxBackend, executionProfile: params.executionProfile, executionProfileSource: params.executionProfileSource, @@ -83,3 +85,13 @@ export function buildReplayExecutionState( language: params.language, }; } + +/** Bind the authenticated conversation checkout to every replay iteration. */ +export function bindReplayWorkspaceInstance( + payload: t.PayloadBody, + state: Pick, +): t.PayloadBody { + return state.workspaceInstanceId == null + ? payload + : { ...payload, workspace_instance_id: state.workspaceInstanceId }; +} diff --git a/service/src/service/replay-state.ts b/service/src/service/replay-state.ts index 3b65cedf..fd4e90e3 100644 --- a/service/src/service/replay-state.ts +++ b/service/src/service/replay-state.ts @@ -118,6 +118,8 @@ export interface ExecutionState { bridgeWorkerId?: string; /** Selected workspace retained and bound across every replay iteration. */ workspaceId?: string; + /** Selected conversation checkout retained across every replay iteration. */ + workspaceInstanceId?: string; /** Original queue/backend target retained across replay continuations. */ sandboxBackend?: SandboxBackendName; /** Original producer profile retained so continuations use the same queue. */ diff --git a/service/src/types/service.ts b/service/src/types/service.ts index 0404ad16..91c3ed89 100644 --- a/service/src/types/service.ts +++ b/service/src/types/service.ts @@ -204,6 +204,8 @@ export type PayloadFileRef = { export interface PayloadBody { language: string; version: string; + /** Opaque conversation checkout selected and authenticated by the API. */ + workspace_instance_id?: string; /** Stable identity shared by all replay iterations of one execution. */ execution_id?: string; replay_tool_count?: number; @@ -393,6 +395,8 @@ export interface ProgrammaticRequestBody { * legacy `/exec` sandbox body), so the router accepts either key and * normalizes to `language`. If both are present, `language` wins. */ lang?: 'python' | 'bash'; + /** Opaque conversation checkout binding for a selected native workspace. */ + workspace_instance_id?: string; } export interface ProgrammaticToolCall { diff --git a/service/src/workspace-tools/router.test.ts b/service/src/workspace-tools/router.test.ts index 04738b8a..a52b3f0b 100644 --- a/service/src/workspace-tools/router.test.ts +++ b/service/src/workspace-tools/router.test.ts @@ -13,6 +13,7 @@ import { executionProfileMiddleware } from '../middleware/execution-profile'; import { hostedAppPreviewGateway } from '../hosted-app/preview-gateway'; import { applyPrincipal } from '../auth/principal'; import { BridgeStoreError } from '../bridge/store'; +import { principalWorkspaceInstanceId } from '../bridge/workspace-instance'; import { bridgeStoreStatus, createWorkspaceToolsRouter } from './router'; import type { WorkspaceToolRequest } from '../../../packages/code/src/protocol'; @@ -36,6 +37,36 @@ test('maps invalid worker results to an upstream failure', () => { expect(bridgeStoreStatus(new BridgeStoreError('WORKER_QUEUE_FULL', 'queue full'))).toBe(429); }); +test('binds instance admission to the authenticated tenant and user while preserving legacy requests', async () => { + const app = express(); + app.use(json()); + app.use((req, _res, next) => { + applyPrincipal(req, { userId: 'user-1', tenantId: 'tenant-1', principalSource: 'librechat_jwt', codeWorkerId: 'user-worker' }); + next(); + }); + const dispatched: WorkspaceToolRequest[] = []; + app.use(createWorkspaceToolsRouter({ + backend: 'remote-bridge', configuredWorkerId: 'user-worker', dynamicWorkers: false, + store: { async dispatchWorkspaceTool(args) { + dispatched.push(args.request); + return { protocolVersion: 1, generation: 1, leaseToken: 'lease', incarnationId: 'incarnation', status: 'rejected', error: 'fixture' }; + } }, + })); + server = createServer(app); + await new Promise(resolve => server!.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Missing listener'); + for (const workspaceInstanceId of ['a'.repeat(64), undefined]) { + const response = await fetch(`http://127.0.0.1:${address.port}/workspace-tools/execute`, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', workspaceInstanceId, path: 'README.md' }), + }); + await response.json(); + } + expect(dispatched[0]?.workspaceInstanceId).toBe(principalWorkspaceInstanceId({ instanceId: 'a'.repeat(64), tenantId: 'tenant-1', principalId: 'user-1' })); + expect(dispatched[1]?.workspaceInstanceId).toBeUndefined(); +}); + test.each<[WorkspaceToolRequest, number, number?]>([ [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, undefined], [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 35_000, undefined], diff --git a/service/src/workspace-tools/router.ts b/service/src/workspace-tools/router.ts index eb89370e..3e1f9fee 100644 --- a/service/src/workspace-tools/router.ts +++ b/service/src/workspace-tools/router.ts @@ -18,6 +18,7 @@ import { BridgeWorkerSelectionError, resolveBridgeWorkerSelection, } from '../bridge/selection'; +import { principalWorkspaceInstanceId } from '../bridge/workspace-instance'; interface WorkspaceToolsRouterOptions { store: Pick; @@ -82,12 +83,22 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) return; } outcome.operation = req.body.operation; - const request: WorkspaceToolRequest = req.body.operation === 'execute_command' - ? { ...req.body, timeoutMs: Math.min( - req.body.timeoutMs ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, + const principalRequest: WorkspaceToolRequest = req.body.workspaceInstanceId == null + ? req.body + : { + ...req.body, + workspaceInstanceId: principalWorkspaceInstanceId({ + instanceId: req.body.workspaceInstanceId, + tenantId: principal.tenantId, + principalId: principal.userId, + }), + }; + const request: WorkspaceToolRequest = principalRequest.operation === 'execute_command' + ? { ...principalRequest, timeoutMs: Math.min( + principalRequest.timeoutMs ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, options.timeoutMs ?? Number.MAX_SAFE_INTEGER, ) } - : req.body; + : principalRequest; const executionBudgetMs = request.operation === 'execute_command' ? request.timeoutMs! + 5_000 : Math.min(options.timeoutMs ?? 30_000, 30_000); From c8b3e1490416753b6f633e4cdbf9100e93aeb849 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sun, 20 Sep 2026 05:44:11 -0400 Subject: [PATCH 06/48] fix: Close native scratch directory streams (#240) --- docs/remote-bridge/README.md | 5 ++++- packages/code/src/native-scratch.ts | 30 +++++++++++++++++------------ 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index f66499b2..efe84b84 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -235,7 +235,10 @@ execution. - Remote bridge deployments use backend-specific BullMQ queues and serialize the expected backend on every new job, preventing Lambda or HTTP consumers from accepting attached-worker executions. -- Code API permits one active assignment per worker. +- Code API negotiates a bounded number of active workspace assignments per + worker. The lower API or worker slot ceiling wins, and assignments sharing + the same workspace isolation key remain serialized while independent + conversation worktrees may run concurrently. - Dynamic workers are fenced to their server-issued tenant before assignment. - Each assignment has an absolute deadline, generation, and random lease token. - Settlements with the wrong worker, generation, token, or expired deadline are diff --git a/packages/code/src/native-scratch.ts b/packages/code/src/native-scratch.ts index 0ecd330e..8394c1ae 100644 --- a/packages/code/src/native-scratch.ts +++ b/packages/code/src/native-scratch.ts @@ -151,19 +151,25 @@ export async function restoreScratchTraversal( if (directoryFd === undefined) continue; try { const directory = await opendir(descriptorPath(directoryFd)); - for await (const entry of directory) { - entriesInspected += 1; - if (entriesInspected > MAX_SCRATCH_ENTRIES) { - throw new Error('Native sandbox scratch cleanup exceeded its entry limit'); + try { + while (true) { + const entry = await directory.read(); + if (entry === null) break; + entriesInspected += 1; + if (entriesInspected > MAX_SCRATCH_ENTRIES) { + throw new Error('Native sandbox scratch cleanup exceeded its entry limit'); + } + if (!entry.isDirectory()) continue; + if (components.length >= MAX_SCRATCH_DEPTH) { + throw new Error('Native sandbox scratch cleanup exceeded its depth limit'); + } + if (pending.length >= MAX_SCRATCH_DIRECTORIES) { + throw new Error('Native sandbox scratch cleanup exceeded its directory limit'); + } + pending.push([...components, entry.name]); } - if (!entry.isDirectory()) continue; - if (components.length >= MAX_SCRATCH_DEPTH) { - throw new Error('Native sandbox scratch cleanup exceeded its depth limit'); - } - if (pending.length >= MAX_SCRATCH_DIRECTORIES) { - throw new Error('Native sandbox scratch cleanup exceeded its directory limit'); - } - pending.push([...components, entry.name]); + } finally { + await directory.close(); } } finally { if (directoryFd !== root.fd) closeDirectory(directoryFd); From 67d75d859aee891923c40cde1db073489d74a424 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 23 Sep 2026 19:56:56 -0400 Subject: [PATCH 07/48] feat(code): route trusted VM GitHub App tokens by checkout (#248) * feat(code): route trusted VM GitHub App tokens by checkout * fix(code): resolve checkout credentials within canonical root * fix(code): reject cross-root credential aliases --- docs/remote-bridge/worker-runbook.md | 19 ++++-- packages/code/README.md | 26 +++++++-- packages/code/src/cli.test.ts | 86 +++++++++++++++++++++++++--- packages/code/src/cli.ts | 39 ++++++++++--- packages/code/src/github.test.ts | 79 +++++++++++++++++++++++++ packages/code/src/github.ts | 47 ++++++++++++--- 6 files changed, 261 insertions(+), 35 deletions(-) diff --git a/docs/remote-bridge/worker-runbook.md b/docs/remote-bridge/worker-runbook.md index 16c24742..3043c541 100644 --- a/docs/remote-bridge/worker-runbook.md +++ b/docs/remote-bridge/worker-runbook.md @@ -327,12 +327,19 @@ Environment=LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE=/home/librechat-code/.config/ ``` Install the same App separately on every personal account or organization the -worker is allowed to use. The trusted worker resolves the correct installation -from the repository containing each command's working directory, then mints and -caches a repository-scoped token. Cross-repository work therefore does not -require changing an installation ID or restarting the worker. Set -`LIBRECHAT_CODE_GITHUB_INSTALLATION_ID` only as a legacy fixed-installation -fallback. +worker is allowed to use. By default, the worker binds each admitted workspace +root to its repository at startup, then mints and caches repository-scoped +tokens. Different admitted roots can use different installations without +restarting the worker. For a trusted VM with multiple checkouts under one root, +set `LIBRECHAT_CODE_GITHUB_REPOSITORY_ROUTING=checkout` and use the `trusted-vm` +command policy. This opt-in resolves the local `origin` URL of each command's +current checkout, including linked worktrees. It remains inside the admitted +filesystem root, but anyone able to alter a checkout's remote can select any +repository where the App is installed; keep the App's installation scope narrow. +Pass the checkout as the command working directory; changing directories only +inside the shell cannot change the token chosen before command launch. +Set `LIBRECHAT_CODE_GITHUB_INSTALLATION_ID` only as a legacy +fixed-installation fallback; it cannot be combined with checkout routing. Sandboxed commands receive masked Git/`gh` credentials only for the configured GitHub hosts; the token is not written to the repository, remote URL, or Git diff --git a/packages/code/README.md b/packages/code/README.md index e30b80cb..5a06b92a 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -293,9 +293,23 @@ installation tokens. At startup, the worker binds each explicitly admitted workspace root to its Git repository. Commands in those independent roots can use simultaneous installations on personal accounts and organizations without being restarted or reconfigured, while a command cannot gain access by changing -its workspace's remote URL. Tokens are scoped and cached per repository. For -compatibility with deployments -that intentionally bind a worker to one installation, set the optional legacy +its workspace's remote URL. Tokens are scoped and cached per repository. + +For trusted VMs that intentionally work in multiple Git checkouts beneath one +admitted root, opt in to `--github-repository-routing checkout` (or +`LIBRECHAT_CODE_GITHUB_REPOSITORY_ROUTING=checkout`) together with the +`trusted-vm` command policy. Each command then uses the repository identified +by its current checkout's local `origin` URL, including linked worktrees. +The command must set its working directory to that checkout; a shell `cd` +inside a command does not change which credential was selected before launch. +This does not widen the admitted filesystem roots, but a command able to alter +a checkout's remote can obtain a token for **any repository where the App is +installed**. Use this mode only where the machine operator trusts the VM and +the App's installation scope; the default `admitted` mode keeps the startup +binding. Checkout routing requires an App without a fixed installation ID. + +For compatibility with deployments that intentionally bind a worker to one +installation, set the optional legacy `LIBRECHAT_CODE_GITHUB_INSTALLATION_ID` fallback. App-authenticated commits use the GitHub App bot's canonical no-reply identity, @@ -758,8 +772,10 @@ Also archive any adjacent `.source` staging directory. Pre-release version-1 completion records are deliberately preserved but not admitted by this version; they do not contain the required source Git identity binding. -GitHub App routing is inherited from the operator-admitted source repository; -commands cannot select a different installation by rewriting a worktree remote. +By default, GitHub App routing is inherited from the operator-admitted source +repository; commands cannot select a different installation by rewriting a +worktree remote. On trusted VMs, the opt-in checkout routing mode above instead +uses the current worktree's local `origin` URL, within the admitted root. Legacy requests without a conversation identity continue to use the selected source root. Older Code API deployments do not negotiate the capability, so the worker omits it until every request path understands the isolation boundary. diff --git a/packages/code/src/cli.test.ts b/packages/code/src/cli.test.ts index f39b28d9..f95d7eee 100644 --- a/packages/code/src/cli.test.ts +++ b/packages/code/src/cli.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { spawnSync } from 'node:child_process'; import { generateKeyPairSync } from 'node:crypto'; -import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -242,6 +242,8 @@ test('CLI accepts repository-routed GitHub App authentication without a fixed in t.after(() => rm(directory, { recursive: true, force: true })); const privateKeyPath = join(directory, 'app.pem'); const preload = join(directory, 'fetch.mjs'); + const workspace = join(directory, 'workspace'); + await mkdir(workspace); const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); await writeFile( privateKeyPath, @@ -251,13 +253,8 @@ test('CLI accepts repository-routed GitHub App authentication without a fixed in await writeFile( preload, ` - globalThis.fetch = async (input) => { - const url = String(input); - if (url.endsWith('/app')) return Response.json({ slug: 'lia-by-librechat' }); - if (url.endsWith('/users/lia-by-librechat%5Bbot%5D')) { - return Response.json({ id: 328778573, login: 'lia-by-librechat[bot]', type: 'Bot' }); - } - throw new Error('test stopped after GitHub App validation'); + globalThis.fetch = async () => { + throw new Error('test reached GitHub App validation'); }; `, ); @@ -276,7 +273,7 @@ test('CLI accepts repository-routed GitHub App authentication without a fixed in LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', - LIBRECHAT_CODE_WORKER_DIR: directory, + LIBRECHAT_CODE_WORKER_DIR: workspace, LIBRECHAT_CODE_ALLOW_WORKSPACE_COMMANDS: 'true', LIBRECHAT_CODE_GITHUB_TOKEN: undefined, LIBRECHAT_CODE_GITHUB_APP_ID: '123', @@ -288,6 +285,77 @@ test('CLI accepts repository-routed GitHub App authentication without a fixed in assert.notEqual(result.status, 0); assert.doesNotMatch(result.stderr, /GitHub App authentication requires/); assert.doesNotMatch(result.stderr, /installation ID/i); + assert.match(result.stderr, /test reached GitHub App validation/); + + const checkout = spawnSync( + process.execPath, + ['--import', preload, fileURLToPath(new URL('./cli.js', import.meta.url))], + { + encoding: 'utf8', + timeout: 10_000, + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_WORKER_DIR: workspace, + LIBRECHAT_CODE_ALLOW_WORKSPACE_COMMANDS: 'true', + LIBRECHAT_CODE_COMMAND_POLICY_PRESET: 'trusted-vm', + LIBRECHAT_CODE_GITHUB_TOKEN: undefined, + LIBRECHAT_CODE_GITHUB_APP_ID: '123', + LIBRECHAT_CODE_GITHUB_INSTALLATION_ID: undefined, + LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE: privateKeyPath, + LIBRECHAT_CODE_GITHUB_REPOSITORY_ROUTING: 'checkout', + }, + }, + ); + assert.notEqual(checkout.status, 0); + assert.doesNotMatch( + checkout.stderr, + /Checkout GitHub repository routing requires/, + ); + assert.match(checkout.stderr, /test reached GitHub App validation/); +}); + +test('CLI rejects checkout routing outside a trusted VM or without repository-scoped App auth', () => { + const base = { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_WORKER_DIR: process.cwd(), + LIBRECHAT_CODE_ALLOW_WORKSPACE_COMMANDS: 'true', + LIBRECHAT_CODE_GITHUB_APP_ID: '123', + LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE: '/does/not/matter', + LIBRECHAT_CODE_GITHUB_INSTALLATION_ID: undefined, + LIBRECHAT_CODE_GITHUB_TOKEN: undefined, + LIBRECHAT_CODE_GITHUB_REPOSITORY_ROUTING: 'checkout', + }; + const cli = fileURLToPath(new URL('./cli.js', import.meta.url)); + const restricted = spawnSync(process.execPath, [cli], { + encoding: 'utf8', + env: base, + }); + assert.notEqual(restricted.status, 0); + assert.match(restricted.stderr, /requires the trusted-vm command policy/); + + const fixed = spawnSync(process.execPath, [cli], { + encoding: 'utf8', + env: { + ...base, + LIBRECHAT_CODE_GITHUB_INSTALLATION_ID: '456', + LIBRECHAT_CODE_COMMAND_POLICY_PRESET: 'trusted-vm', + }, + }); + assert.notEqual(fixed.status, 0); + assert.match(fixed.stderr, /without a fixed installation ID/); + + const invalid = spawnSync(process.execPath, [cli], { + encoding: 'utf8', + env: { ...base, LIBRECHAT_CODE_GITHUB_REPOSITORY_ROUTING: 'unknown' }, + }); + assert.notEqual(invalid.status, 0); + assert.match(invalid.stderr, /must be admitted or checkout/); }); test('CLI requires a runtime image for Docker supervision', () => { diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 26c60e9a..841164fb 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -49,7 +49,7 @@ import type { LocalWorkspaceConfig } from './workspace.js'; import { GITHUB_ALLOWED_DOMAINS, GitHubAppCredentialProvider, - gitHubRepositoryForAdmittedDirectory, + gitHubRepositoryForCommand, gitHubRepositoryForDirectory, gitHubCommandCredentialEnvironment, gitHubMaskedCredentialVariables, @@ -150,12 +150,13 @@ function nonEmpty(value: string | undefined): string | undefined { return value?.trim().length ? value : undefined; } -function githubCredentials(): { +function githubCredentials(args: string[]): { provider?: GitHubCredentialProvider; host: string; privateKeyPath?: string; mode?: 'app' | 'token'; repositoryRouting?: boolean; + checkoutRouting?: boolean; policyIdentity: string; } { const token = nonEmpty(process.env.LIBRECHAT_CODE_GITHUB_TOKEN); @@ -163,6 +164,13 @@ function githubCredentials(): { const installationId = nonEmpty( process.env.LIBRECHAT_CODE_GITHUB_INSTALLATION_ID, ); + const routing = + option(args, '--github-repository-routing')?.trim().toLowerCase() ?? + process.env.LIBRECHAT_CODE_GITHUB_REPOSITORY_ROUTING?.trim().toLowerCase() ?? + 'admitted'; + if (routing !== 'admitted' && routing !== 'checkout') { + throw new Error('GitHub repository routing must be admitted or checkout'); + } const privateKeyPath = nonEmpty( process.env.LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE, ); @@ -178,6 +186,11 @@ function githubCredentials(): { 'Configure either GitHub App authentication or a GitHub token, not both', ); } + if (routing === 'checkout' && (!hasApp || installationId)) { + throw new Error( + 'Checkout GitHub repository routing requires a GitHub App without a fixed installation ID', + ); + } const configuredHostValue = nonEmpty( process.env.LIBRECHAT_CODE_GITHUB_HOST, ); @@ -211,12 +224,13 @@ function githubCredentials(): { host, mode: 'app', repositoryRouting: !installationId, + checkoutRouting: routing === 'checkout', policyIdentity: gitHubAuthenticationPolicyIdentity({ mode: 'app', host, appId, installationId, - }), + }) + (routing === 'checkout' ? ':routing:checkout' : ''), privateKeyPath, provider: new GitHubAppCredentialProvider({ appId: appId!, @@ -530,9 +544,11 @@ async function run( } const github = runtimeSessionId == null - ? githubCredentials() + ? githubCredentials(args) : { host: 'github.com', + repositoryRouting: false, + checkoutRouting: false, policyIdentity: gitHubAuthenticationPolicyIdentity({ host: 'github.com', }), @@ -547,6 +563,11 @@ async function run( 'GitHub authentication currently requires the native-srt command sandbox', ); } + if (github.checkoutRouting && commandPolicy.preset !== 'trusted-vm') { + throw new Error( + 'Checkout GitHub repository routing requires the trusted-vm command policy', + ); + } const githubDomains = github.provider ? github.host === 'github.com' ? [...GITHUB_ALLOWED_DOMAINS] @@ -759,9 +780,8 @@ async function run( }), ]), ); - // Bind credentials to immutable, explicitly admitted roots. The repository - // remote is operator input at startup, never an authorization input that a - // sandboxed command may change for its next invocation. + // Keep an admission boundary even when trusted-VM checkout routing uses a + // nested repository's remote for the current command. const admittedGitHubRepositories = github.provider && github.repositoryRouting ? new Map( await Promise.all( @@ -998,9 +1018,12 @@ async function run( ), async resolve(signal?: AbortSignal, cwd?: string) { const repository = cwd && admittedGitHubRepositories - ? gitHubRepositoryForAdmittedDirectory( + ? await gitHubRepositoryForCommand( cwd, admittedGitHubRepositories, + github.checkoutRouting ? 'checkout' : 'admitted', + github.host, + signal, ) : undefined; if (!repository && github.repositoryRouting) { diff --git a/packages/code/src/github.test.ts b/packages/code/src/github.test.ts index 352bf1c3..b13d70f7 100644 --- a/packages/code/src/github.test.ts +++ b/packages/code/src/github.test.ts @@ -4,6 +4,7 @@ import { chmod, mkdtemp, mkdir, + realpath, rm, symlink, writeFile, @@ -26,6 +27,7 @@ import { GITHUB_CREDENTIAL_ENV_NAME, gitHubCredentialEnvironment, gitHubRepositoryForAdmittedDirectory, + gitHubRepositoryForCommand, gitHubRepositoryForDirectory, normalizeGitHubHost, wrapGitHubCredentialCommand, @@ -459,6 +461,83 @@ test('keeps repository authorization bound to the admitted workspace root', asyn ); }); +test('checkout routing follows nested repositories only inside an admitted root', async (t) => { + const directory = await realpath( + await mkdtemp(join(tmpdir(), 'librechat-code-github-checkout-')), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const nested = join(directory, 'worktrees', 'other'); + await mkdir(nested, { recursive: true }); + execFileSync('git', ['init', directory]); + execFileSync('git', [ + '-C', directory, 'remote', 'add', 'origin', 'git@github.com:acme/outer.git', + ]); + execFileSync('git', ['init', nested]); + execFileSync('git', [ + '-C', nested, 'remote', 'add', 'origin', 'git@github.com:acme/inner.git', + ]); + const admitted = new Map([[directory, 'acme/outer']]); + + assert.equal( + await gitHubRepositoryForCommand(nested, admitted, 'admitted'), + 'acme/outer', + ); + assert.equal( + await gitHubRepositoryForCommand(nested, admitted, 'checkout'), + 'acme/inner', + ); + execFileSync('git', [ + '-C', nested, 'remote', 'set-url', 'origin', 'git@github.com:acme/changed.git', + ]); + assert.equal( + await gitHubRepositoryForCommand(nested, admitted, 'checkout'), + 'acme/changed', + ); + assert.equal( + await gitHubRepositoryForCommand(dirname(directory), admitted, 'checkout'), + undefined, + ); + const outside = await realpath( + await mkdtemp(join(tmpdir(), 'librechat-code-github-outside-')), + ); + t.after(() => rm(outside, { recursive: true, force: true })); + execFileSync('git', ['init', outside]); + execFileSync('git', [ + '-C', outside, 'remote', 'add', 'origin', 'git@github.com:acme/outside.git', + ]); + const escaped = join(directory, 'worktrees', 'escaped'); + await symlink(outside, escaped); + assert.equal( + await gitHubRepositoryForCommand(escaped, admitted, 'checkout'), + undefined, + ); + admitted.set(outside, 'acme/outside'); + assert.equal( + await gitHubRepositoryForCommand(escaped, admitted, 'checkout'), + undefined, + ); + assert.equal( + await gitHubRepositoryForCommand(nested, admitted, 'checkout', 'github.example.test'), + undefined, + ); + + const linked = join(directory, 'worktrees', 'linked'); + execFileSync('git', [ + '-C', nested, '-c', 'user.name=Test', '-c', 'user.email=test@example.com', + 'commit', '--allow-empty', '-m', 'initial', + ]); + execFileSync('git', ['-C', nested, 'worktree', 'add', '--detach', linked]); + assert.equal( + await gitHubRepositoryForCommand(linked, admitted, 'checkout'), + 'acme/changed', + ); + admitted.set(linked, 'acme/linked'); + assert.equal( + await gitHubRepositoryForCommand(linked, admitted, 'admitted'), + 'acme/linked', + ); +}); + test('uses the configured GHES host for the App bot no-reply identity', async (t) => { const directory = await mkdtemp(join(tmpdir(), 'librechat-code-ghes-identity-')); t.after(() => rm(directory, { recursive: true, force: true })); diff --git a/packages/code/src/github.ts b/packages/code/src/github.ts index fb6b6fdc..19b8e639 100644 --- a/packages/code/src/github.ts +++ b/packages/code/src/github.ts @@ -1,7 +1,7 @@ import { constants } from 'node:fs'; import { execFile } from 'node:child_process'; import { createHash, createPrivateKey, sign } from 'node:crypto'; -import { open } from 'node:fs/promises'; +import { open, realpath } from 'node:fs/promises'; import { dirname, isAbsolute, relative, sep } from 'node:path'; import { promisify } from 'node:util'; import { projectRemote } from './projects.js'; @@ -137,21 +137,54 @@ export async function gitHubRepositoryForDirectory( return repository; } -/** Return the startup-bound repository for the admitted root containing cwd. */ -export function gitHubRepositoryForAdmittedDirectory( +function admittedRepositoryEntry( cwd: string, repositories: ReadonlyMap, -): string | undefined { - for (const [root, repository] of repositories) { +): readonly [string, string | undefined] | undefined { + let closest: readonly [string, string | undefined] | undefined; + for (const entry of repositories) { + const [root] = entry; const path = relative(root, cwd); if ( path === '' || (path !== '..' && !path.startsWith(`..${sep}`) && !isAbsolute(path)) ) { - return repository; + if (!closest || root.length > closest[0].length) closest = entry; } } - return undefined; + return closest; +} + +/** Return the startup-bound repository for the admitted root containing cwd. */ +export function gitHubRepositoryForAdmittedDirectory( + cwd: string, + repositories: ReadonlyMap, +): string | undefined { + return admittedRepositoryEntry(cwd, repositories)?.[1]; +} + +/** Resolve a checkout repository only when its cwd remains inside an admitted root. */ +export async function gitHubRepositoryForCommand( + cwd: string, + repositories: ReadonlyMap, + routing: 'admitted' | 'checkout', + host = 'github.com', + signal?: AbortSignal, +): Promise { + const admitted = admittedRepositoryEntry(cwd, repositories); + if (!admitted) return undefined; + if (routing === 'admitted') return admitted[1]; + let canonicalCwd: string; + try { + canonicalCwd = await realpath(cwd); + } catch { + signal?.throwIfAborted(); + return undefined; + } + if (admittedRepositoryEntry(canonicalCwd, repositories)?.[0] !== admitted[0]) { + return undefined; + } + return gitHubRepositoryForDirectory(canonicalCwd, host, signal); } function base64UrlJson(value: unknown): string { From cf0e6683ccad320c7467a10d55cf7090a1bbfb88 Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 23:13:38 -0400 Subject: [PATCH 08/48] =?UTF-8?q?=F0=9F=A7=B9=20fix:=20Await=20Job=20Clean?= =?UTF-8?q?up=20and=20Measure=20Retained=20Resources=20(#249)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🧹 fix: Await Job Cleanup and Measure Retained Resources * 📡 fix: Scrape Sandbox Runner Metrics in Helm --------- Co-authored-by: Lia --- .github/workflows/ci.yml | 14 ++ api/README.md | 67 ++++++++ api/src/api/v2-cleanup.test.ts | 189 +++++++++++++++++++++ api/src/api/v2.ts | 4 + api/src/cleanup-metrics.test.ts | 110 ++++++++++++ api/src/cleanup-metrics.ts | 185 ++++++++++++++++++++ api/src/cleanup.integration.test.ts | 184 ++++++++++++++++++++ api/src/job-cleanup-metrics.test.ts | 118 +++++++++++++ api/src/job.ts | 83 +++++---- api/src/nsjail.ts | 4 +- helm/codeapi/Chart.yaml | 2 +- helm/codeapi/README.md | 37 ++++ helm/codeapi/templates/network-policy.yaml | 10 ++ helm/codeapi/templates/podmonitor.yaml | 22 +++ helm/codeapi/values.yaml | 5 + tests/sandbox_runner_metrics.sh | 147 ++++++++++++++++ 16 files changed, 1143 insertions(+), 38 deletions(-) create mode 100644 api/src/api/v2-cleanup.test.ts create mode 100644 api/src/cleanup-metrics.test.ts create mode 100644 api/src/cleanup-metrics.ts create mode 100644 api/src/cleanup.integration.test.ts create mode 100644 api/src/job-cleanup-metrics.test.ts create mode 100755 tests/sandbox_runner_metrics.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b645f4b8..4076d8f6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,9 @@ jobs: - name: Sandbox-runner liveness checks run: tests/sandbox_runner_healthcheck.sh + - name: Sandbox-runner metrics discovery and network policy + run: tests/sandbox_runner_metrics.sh + - name: Bridge pairing rollout safety run: tests/bridge_pairing_rollout.sh @@ -121,6 +124,17 @@ jobs: - name: Bun tests run: bun run test + - name: File-heavy workspace cleanup on tmpfs + run: | + docker run --rm --user 0 \ + --tmpfs /tmp:rw,size=1g \ + --mount "type=bind,source=$GITHUB_WORKSPACE,target=/work,readonly" \ + --workdir /work/api \ + --env SANDBOX_CLEANUP_TMPFS_TEST=1 \ + --env SANDBOX_LOG_LEVEL=error \ + oven/bun:1.3.14-debian \ + bun test src/cleanup.integration.test.ts + service-unit-tests: name: Service Unit Tests runs-on: ubuntu-latest diff --git a/api/README.md b/api/README.md index 38e4cd98..34b32145 100644 --- a/api/README.md +++ b/api/README.md @@ -139,3 +139,70 @@ validation behavior. Roll out this sandbox behavior before enabling timeout forwarding in the service's plain `/exec` handler. Older sandboxes reject caps above their local runtime limit; older services remain compatible with updated sandboxes. + +### Runner memory after cleanup + +`Job.cleanup()` emits `Post-cleanup resources` after workspace removal and UID +release (or quarantine). `/execute` now waits for cleanup before sending success +or execution-failure responses. Artifact upload still finishes before cleanup. +Persistent session workspaces and their pinned UIDs are intentionally preserved; +failed disposable cleanup still quarantines the directory and retains its UID +for the existing retry path. This change does not alter those policies. + +The following metrics use only fixed, low-cardinality labels: + +| Metric suffix (prefix `codeapi_sandbox_`) | Meaning | +| --- | --- | +| `post_cleanup_memory_bytes{kind}` | `current`, `anon`, `file`, `shmem` at the visible cgroup v2 mount root | +| `post_cleanup_tmp_used{resource}` | `/tmp` allocated `bytes`, allocated `inodes`, and `is_tmpfs` (0 or 1) | +| `post_cleanup_workspaces{kind}` | Remaining `disposable`, `session`, and `other` entries in `/tmp/sandbox` | +| `cleanup_total{mode,outcome}` | Disposable/session cleanup attempts: `removed`, `preserved`, `retained`, `error` | +| `post_cleanup_sample_success{source}` | Whether `memory`, `tmp`, or `workspaces` was readable on the last sample | +| `post_cleanup_timestamp_seconds` | When the last cleanup sample was taken | + +These are runner-wide **last-cleanup samples**, not live gauges or per-job +memory attribution. Other jobs can still be running. Reaper-only changes are +visible on the next job cleanup, not immediately. Interpret the workspace counts +alongside active executions and the sample timestamp. Unavailable sources are +reported as unavailable and their old gauge values are removed, never replaced +with a healthy-looking zero. The structured log also includes active UID slots +and the number of retained cleanup retries. Synthetic jobs update metrics while +suppressing successful per-job logs as before. + +The visible cgroup root covers the API and sibling NsJail cgroups. The old +`Post-execution memory` log instead samples `/proc/self/cgroup` **before** job +cleanup, so it can have a different scope. `file` includes `shmem`; do not add +them or assume all `file` memory is reclaimable page cache. + +`statfs` measures allocation, including deleted-but-open files on the sampled +mount, without walking user files. The runner's 1 GiB `/tmp` mount is distinct +from each jail's 20 MiB `/tmp` mount. Its ceiling does not cap all container +memory. If workspace counts fall but shmem does not, investigate descendant +processes and retained mounts as well as files. A stable API-process FD count +alone cannot exclude those cases. Memory requests and HPA settings are unchanged. + +#### Cleanup stress tests + +Ordinary unit tests cover metrics, unavailable sources, session/disposable +classification and response ordering. The opt-in stress tests must run in an +**isolated test container**, never on an active runner: + +```bash +# From api/, with /tmp mounted as tmpfs and per-job chown available: +SANDBOX_CLEANUP_TMPFS_TEST=1 bun test src/cleanup.integration.test.ts + +# Additionally requires the runner's NsJail binary, spec-guard, config, +# and normal namespace/cgroup permissions. Run this file alone. +NSJAIL_CONFIG=/sandbox_api/config/sandbox.cfg \ +SANDBOX_CLEANUP_NSJAIL_TEST=1 bun test src/cleanup.integration.test.ts +``` + +The first test repeats real Job priming and cleanup with large payloads and many +small files, checking tmpfs bytes/inodes and UID slots after every iteration. +CI runs it in a dedicated Bun container with a 1 GiB `/tmp` tmpfs. It does not +execute NsJail. The second test uses real NsJail jobs covering normal completion, +timeout and output overflow, including a detached child holding an unlinked file. +It asserts no processes remain under the job UID before removing its workspace, +then checks workspace removal and post-cleanup tmpfs allocation. This test is +skipped unless explicitly enabled; a unit-test pass is not proof of namespace +teardown on the production kernel. diff --git a/api/src/api/v2-cleanup.test.ts b/api/src/api/v2-cleanup.test.ts new file mode 100644 index 00000000..d1efee5d --- /dev/null +++ b/api/src/api/v2-cleanup.test.ts @@ -0,0 +1,189 @@ +import { afterAll, afterEach, beforeAll, expect, test } from 'bun:test'; +import express from 'express'; +import { mkdtemp, rm, writeFile } from 'fs/promises'; +import type { Server } from 'http'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { config } from '../config'; +import { Job } from '../job'; +import { loadPackage } from '../runtime'; +import { ValidationError } from '../validation'; +import router from './v2'; + +let server: Server; +let url: string; +let directory: string; +const language = 'cleanup-order-test'; +const originalPrime = Job.prototype.prime; +const originalExecute = Job.prototype.execute; +const originalUpload = Job.prototype.uploadGeneratedFiles; +const originalCleanup = Job.prototype.cleanup; +const requireManifest = config.require_execution_manifest; + +beforeAll(async () => { + directory = await mkdtemp(join(tmpdir(), 'cleanup-order-')); + await writeFile( + join(directory, 'pkg-info.json'), + JSON.stringify({ language, version: '1.0.0', aliases: [] }) + ); + loadPackage(directory); + const app = express(); + app.use(router); + await new Promise(resolve => { + server = app.listen(0, '127.0.0.1', () => resolve()); + }); + const address = server.address(); + url = `http://127.0.0.1:${ + typeof address === 'object' && address ? address.port : 0 + }/execute`; +}); +afterAll(async () => { + await new Promise(resolve => server.close(() => resolve())); + await rm(directory, { recursive: true, force: true }); +}); +afterEach(() => { + Job.prototype.prime = originalPrime; + Job.prototype.execute = originalExecute; + Job.prototype.uploadGeneratedFiles = originalUpload; + Job.prototype.cleanup = originalCleanup; + config.require_execution_manifest = requireManifest; +}); + +for (const outcome of [ + 'success', + 'prime_failure', + 'execution_failure', + 'validation_failure', +] as const) { + test(`${outcome} waits for cleanup before sending a response and cleans exactly once`, async () => { + config.require_execution_manifest = false; + const events: string[] = []; + let releaseCleanup!: () => void; + const cleanupGate = new Promise(resolve => { + releaseCleanup = resolve; + }); + let markCleanupStarted!: () => void; + const cleanupStarted = new Promise(resolve => { + markCleanupStarted = resolve; + }); + Job.prototype.prime = async function () { + events.push('prime'); + if (outcome === 'prime_failure') + throw new Error('scripted prime failure'); + }; + Job.prototype.execute = async function () { + events.push('execute'); + if (outcome === 'execution_failure') + throw new Error('scripted execution failure'); + if (outcome === 'validation_failure') + throw new ValidationError('scripted validation failure'); + return { files: [{ id: 'artifact', name: 'result.txt' }] } as Awaited< + ReturnType + >; + }; + Job.prototype.uploadGeneratedFiles = async function () { + events.push('upload'); + return new Set(['artifact']); + }; + Job.prototype.cleanup = async function () { + events.push('cleanup-start'); + markCleanupStarted(); + await cleanupGate; + events.push('cleanup-end'); + }; + let responseArrived = false; + const pending = fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + language, + version: '1.0.0', + files: [{ name: 'main.txt', content: 'test' }], + }), + }).then(response => { + responseArrived = true; + return response; + }); + try { + await Promise.race([ + cleanupStarted, + pending.then(response => { + throw new Error(`Response preceded cleanup: ${response.status}`); + }), + ]); + // Give an incorrectly early res.json() time to reach the client. + await new Promise(resolve => setTimeout(resolve, 30)); + expect(responseArrived).toBe(false); + } finally { + releaseCleanup(); + const response = await pending; + expect(response.status).toBe( + outcome === 'success' + ? 200 + : outcome === 'validation_failure' + ? 400 + : 500 + ); + await response.text(); + } + expect(events.filter(event => event === 'cleanup-start')).toHaveLength(1); + expect(events[events.length - 1]).toBe('cleanup-end'); + if (outcome === 'success') + expect(events).toEqual([ + 'prime', + 'execute', + 'upload', + 'cleanup-start', + 'cleanup-end', + ]); + }); +} + +test('client disconnect does not clean a workspace while execution is still running', async () => { + config.require_execution_manifest = false; + let releaseExecution!: () => void; + const executionGate = new Promise(resolve => { + releaseExecution = resolve; + }); + let markExecuting!: () => void; + const executing = new Promise(resolve => { + markExecuting = resolve; + }); + let markCleaned!: () => void; + const cleaned = new Promise(resolve => { + markCleaned = resolve; + }); + let cleanupCount = 0; + Job.prototype.prime = async function () {}; + Job.prototype.execute = async function () { + markExecuting(); + await executionGate; + return {} as Awaited>; + }; + Job.prototype.cleanup = async function () { + cleanupCount++; + markCleaned(); + }; + const controller = new AbortController(); + const pending = fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + signal: controller.signal, + body: JSON.stringify({ + language, + version: '1.0.0', + files: [{ name: 'main.txt', content: 'test' }], + }), + }).catch(() => undefined); + try { + await executing; + controller.abort(); + await pending; + await new Promise(resolve => setTimeout(resolve, 30)); + expect(cleanupCount).toBe(0); + } finally { + releaseExecution(); + await cleaned; + } + expect(cleanupCount).toBe(1); +}); diff --git a/api/src/api/v2.ts b/api/src/api/v2.ts index e2f252ce..121cf2e3 100644 --- a/api/src/api/v2.ts +++ b/api/src/api/v2.ts @@ -588,9 +588,13 @@ router.post('/execute', express.json({ limit: config.execute_body_limit }), asyn } } + /* Upload must finish before cleanup, and cleanup must settle before + * acknowledging completion. Failed removals retain a quarantined UID. */ + await cleanupHandler(); metricsOutcome = 'success'; return res.status(200).json(result); } catch (error) { + await cleanupHandler(); /* Deliberately BEFORE the ValidationError branch below: once priming has * completed, the workspace has been written to, so any later failure — * including a validation one — leaves state the next execute must not diff --git a/api/src/cleanup-metrics.test.ts b/api/src/cleanup-metrics.test.ts new file mode 100644 index 00000000..e1f61b22 --- /dev/null +++ b/api/src/cleanup-metrics.test.ts @@ -0,0 +1,110 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { + postCleanupMemory, + postCleanupTmp, + postCleanupWorkspaces, + readCleanupResources, + recordCleanupResources, +} from './cleanup-metrics'; + +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) + fs.rmSync(root, { recursive: true, force: true }); +}); + +function fixture() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cleanup-metrics-')); + roots.push(root); + const cgroupRoot = path.join(root, 'cgroup'); + const workspaceRoot = path.join(root, 'workspaces'); + fs.mkdirSync(cgroupRoot); + fs.mkdirSync(workspaceRoot); + fs.writeFileSync(path.join(cgroupRoot, 'memory.current'), '1048576\n'); + fs.writeFileSync( + path.join(cgroupRoot, 'memory.stat'), + 'anon 262144\nfile 786432\nshmem 524288\n' + ); + return { cgroupRoot, workspaceRoot, tmpRoot: root }; +} + +describe('post-cleanup resources', () => { + test('reads byte-accurate cgroup counters without adding shmem to file', () => { + const paths = fixture(); + // The API child cgroup excludes sibling jails. Sample the visible root. + fs.mkdirSync(path.join(paths.cgroupRoot, 'sandbox_api')); + fs.writeFileSync( + path.join(paths.cgroupRoot, 'sandbox_api/memory.current'), + '1' + ); + const snapshot = readCleanupResources(paths); + expect(snapshot.memory).toEqual({ + current: 1048576, + anon: 262144, + file: 786432, + shmem: 524288, + }); + expect(snapshot.tmp?.bytes).toBeGreaterThanOrEqual(0); + expect(snapshot.tmp?.inodes).toBeGreaterThanOrEqual(0); + expect(snapshot.errors).toEqual({}); + }); + + test('separates disposable, persistent and unexpected entries without following symlinks', () => { + const paths = fixture(); + fs.mkdirSync(path.join(paths.workspaceRoot, 'ws_active')); + fs.mkdirSync(path.join(paths.workspaceRoot, 'session')); + fs.mkdirSync(path.join(paths.workspaceRoot, 'unknown')); + fs.symlinkSync(paths.tmpRoot, path.join(paths.workspaceRoot, 'ws_symlink')); + expect(readCleanupResources(paths).workspaces).toEqual({ + disposable: 1, + session: 1, + other: 2, + }); + }); + + test('does not turn missing or malformed statistics into a healthy zero', () => { + const paths = fixture(); + fs.writeFileSync( + path.join(paths.cgroupRoot, 'memory.stat'), + 'anon 1\nfile 2\n' + ); + expect(readCleanupResources(paths).memory).toBeNull(); + fs.writeFileSync( + path.join(paths.cgroupRoot, 'memory.current'), + 'not-a-number' + ); + const snapshot = readCleanupResources(paths); + expect(snapshot.memory).toBeNull(); + expect(snapshot.tmp).not.toBeNull(); + expect(snapshot.workspaces).not.toBeNull(); + expect(snapshot.errors.memory).toBe('unavailable'); + }); + + test('unreadable sources remove stale gauges without failing cleanup', async () => { + const paths = fixture(); + const context = { + job: 'test', + mode: 'disposable', + outcome: 'removed', + suppressSuccessLogs: true, + } as const; + recordCleanupResources(context, readCleanupResources(paths)); + expect((await postCleanupMemory.get()).values).toHaveLength(4); + expect((await postCleanupTmp.get()).values).toHaveLength(3); + expect((await postCleanupWorkspaces.get()).values).toHaveLength(3); + fs.rmSync(paths.tmpRoot, { recursive: true }); + const snapshot = readCleanupResources(paths); + expect(snapshot.errors).toEqual({ + memory: 'ENOENT', + tmp: 'ENOENT', + workspaces: 'ENOENT', + }); + expect(() => recordCleanupResources(context, snapshot)).not.toThrow(); + expect((await postCleanupMemory.get()).values).toHaveLength(0); + expect((await postCleanupTmp.get()).values).toHaveLength(0); + expect((await postCleanupWorkspaces.get()).values).toHaveLength(0); + }); +}); diff --git a/api/src/cleanup-metrics.ts b/api/src/cleanup-metrics.ts new file mode 100644 index 00000000..7444960d --- /dev/null +++ b/api/src/cleanup-metrics.ts @@ -0,0 +1,185 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import { Counter, Gauge } from 'prom-client'; +import { logger } from './logger'; +import { + SANDBOX_WORKSPACE_ROOT, + SESSION_WORKSPACE_ID, + WORKSPACE_ID_PREFIX, + retainedWorkspaceCleanupCount, + sandboxJobUidPool, +} from './workspace-isolation'; + +const memoryKinds = ['current', 'anon', 'file', 'shmem'] as const; +const workspaceKinds = ['disposable', 'session', 'other'] as const; +type MemoryKind = typeof memoryKinds[number]; +type WorkspaceKind = typeof workspaceKinds[number]; + +export const postCleanupMemory = new Gauge({ + name: 'codeapi_sandbox_post_cleanup_memory_bytes', + help: 'Last post-cleanup cgroup-root memory sample; file includes shmem, not additive', + labelNames: ['kind'] as const, +}); +export const postCleanupTmp = new Gauge({ + name: 'codeapi_sandbox_post_cleanup_tmp_used', + help: 'Last post-cleanup /tmp filesystem allocation, including unlinked open files', + labelNames: ['resource'] as const, +}); +export const postCleanupWorkspaces = new Gauge({ + name: 'codeapi_sandbox_post_cleanup_workspaces', + help: 'Remaining workspace-root entries after cleanup; includes other active jobs', + labelNames: ['kind'] as const, +}); +const cleanupAttempts = new Counter({ + name: 'codeapi_sandbox_cleanup_total', + help: 'Job cleanup attempts; session workspaces are intentionally preserved', + labelNames: ['mode', 'outcome'] as const, +}); +const sampleSuccess = new Gauge({ + name: 'codeapi_sandbox_post_cleanup_sample_success', + help: 'Whether each source was readable on the last post-cleanup sample', + labelNames: ['source'] as const, +}); +const sampleTimestamp = new Gauge({ + name: 'codeapi_sandbox_post_cleanup_timestamp_seconds', + help: 'Timestamp of the last job cleanup resource sample, not a live usage gauge', +}); + +export interface CleanupResourceSnapshot { + memory: Record | null; + tmp: { bytes: number; inodes: number; isTmpfs: boolean } | null; + workspaces: Record | null; + errors: Partial>; +} + +/** Read only fixed kernel files and the workspace root, never recurse into + * untrusted workspace trees or follow their symlinks. Synchronous sampling + * avoids older, slower samples overwriting newer ones during parallel cleanup. + * The visible cgroup mount root includes the API AND sibling NsJail cgroups; + * /proc/self/cgroup may point only at the delegated sandbox_api child instead. + * These are runner-wide observations, not memory attributable to one job. */ +export function readCleanupResources( + paths: { + cgroupRoot?: string; + tmpRoot?: string; + workspaceRoot?: string; + } = {} +): CleanupResourceSnapshot { + const snapshot: CleanupResourceSnapshot = { + memory: null, + tmp: null, + workspaces: null, + errors: {}, + }; + function sample( + source: keyof CleanupResourceSnapshot['errors'], + read: () => T + ): T | null { + try { + return read(); + } catch (error) { + snapshot.errors[source] = + (error as NodeJS.ErrnoException).code ?? 'unavailable'; + return null; + } + } + snapshot.memory = sample('memory', () => { + const root = paths.cgroupRoot ?? '/sys/fs/cgroup'; + const current = fs + .readFileSync(path.join(root, 'memory.current'), 'utf8') + .trim(); + const stat = fs.readFileSync(path.join(root, 'memory.stat'), 'utf8'); + const values = Object.fromEntries( + stat + .trim() + .split('\n') + .map(line => line.trim().split(/\s+/)) + ); + const result = {} as Record; + for (const kind of memoryKinds) { + const raw = kind === 'current' ? current : values[kind]; + if (!raw || !/^\d+$/.test(raw) || !Number.isSafeInteger(Number(raw))) { + throw new Error('Invalid memory statistic'); + } + result[kind] = Number(raw); + } + return result; + }); + snapshot.tmp = sample('tmp', () => { + const stat = fs.statfsSync(paths.tmpRoot ?? '/tmp'); + return { + bytes: (stat.blocks - stat.bfree) * stat.bsize, + inodes: stat.files - stat.ffree, + isTmpfs: stat.type === 0x01021994, + }; + }); + snapshot.workspaces = sample('workspaces', () => { + const counts = { disposable: 0, session: 0, other: 0 }; + for (const entry of fs.readdirSync( + paths.workspaceRoot ?? SANDBOX_WORKSPACE_ROOT, + { withFileTypes: true } + )) { + const kind = + entry.isDirectory() && entry.name === SESSION_WORKSPACE_ID + ? 'session' + : entry.isDirectory() && entry.name.startsWith(WORKSPACE_ID_PREFIX) + ? 'disposable' + : 'other'; + counts[kind]++; + } + return counts; + }); + return snapshot; +} + +export function recordCleanupResources( + context: { + job: string; + mode: 'disposable' | 'session'; + outcome: 'removed' | 'preserved' | 'retained' | 'error'; + suppressSuccessLogs?: boolean; + }, + snapshot = readCleanupResources() +): void { + cleanupAttempts.inc({ mode: context.mode, outcome: context.outcome }); + sampleTimestamp.set(Date.now() / 1000); + for (const source of ['memory', 'tmp', 'workspaces'] as const) { + sampleSuccess.set({ source }, snapshot[source] === null ? 0 : 1); + } + for (const kind of memoryKinds) { + if (snapshot.memory) postCleanupMemory.set({ kind }, snapshot.memory[kind]); + else postCleanupMemory.remove({ kind }); + } + for (const resource of ['bytes', 'inodes', 'is_tmpfs'] as const) { + if (snapshot.tmp) { + postCleanupTmp.set( + { resource }, + resource === 'is_tmpfs' + ? Number(snapshot.tmp.isTmpfs) + : snapshot.tmp[resource] + ); + } else postCleanupTmp.remove({ resource }); + } + for (const kind of workspaceKinds) { + if (snapshot.workspaces) + postCleanupWorkspaces.set({ kind }, snapshot.workspaces[kind]); + else postCleanupWorkspaces.remove({ kind }); + } + if ( + !context.suppressSuccessLogs || + context.outcome === 'retained' || + context.outcome === 'error' + ) { + logger.info( + { + job: context.job, + mode: context.mode, + outcome: context.outcome, + ...snapshot, + activeUidSlots: sandboxJobUidPool.activeCount(), + retainedCleanups: retainedWorkspaceCleanupCount(), + }, + 'Post-cleanup resources' + ); + } +} diff --git a/api/src/cleanup.integration.test.ts b/api/src/cleanup.integration.test.ts new file mode 100644 index 00000000..0d08e804 --- /dev/null +++ b/api/src/cleanup.integration.test.ts @@ -0,0 +1,184 @@ +import { expect, test } from 'bun:test'; +import * as fs from 'fs'; +import * as fsp from 'fs/promises'; +import * as os from 'os'; +import * as path from 'path'; +import * as semver from 'semver'; +import { config } from './config'; +import { Job } from './job'; +import type { Runtime } from './runtime'; +import { readCleanupResources } from './cleanup-metrics'; +import { prepareWorkspaceRoot, sandboxJobUidPool } from './workspace-isolation'; + +// Run only in a dedicated Linux test container with /tmp mounted as tmpfs. +// The real-jail variant additionally needs the runner image's namespace/cgroup +// permissions, nsjail, spec-guard and NSJAIL_CONFIG. Never target a live runner. +const tmpfsEnabled = process.env.SANDBOX_CLEANUP_TMPFS_TEST === '1'; +const nsjailEnabled = process.env.SANDBOX_CLEANUP_NSJAIL_TEST === '1'; + +function runtime(pkgdir: string): Runtime { + return { + language: 'bash', + version: new semver.SemVer('5.2.0'), + aliases: [], + pkgdir, + compiled: false, + env_vars: {}, + timeouts: { compile: 1000, run: 1000 }, + cpu_times: { compile: 1000, run: 1000 }, + memory_limits: { compile: 128 * 1048576, run: 128 * 1048576 }, + max_process_count: 64, + max_open_files: 64, + max_file_size: 16 * 1048576, + output_max_size: 4096, + }; +} + +function jobFor(rt: Runtime, content: string): Job { + return new Job({ + session_id: 'cleanup-stress', + runtime: rt, + files: [{ name: 'main.sh', content }], + args: [], + stdin: '', + timeouts: rt.timeouts, + cpu_times: rt.cpu_times, + memory_limits: rt.memory_limits, + is_synthetic: true, + }); +} + +function workspace(job: Job): string { + return (job as unknown as { submissionDir: string }).submissionDir; +} + +function assertTmpfsBaseline( + baseline: ReturnType +): void { + const after = readCleanupResources(); + expect(after.tmp?.isTmpfs).toBe(true); + // Allow a few metadata pages, not one retained 8 MiB payload per execution. + expect(after.tmp!.bytes).toBeLessThanOrEqual(baseline.tmp!.bytes + 64 * 1024); + expect(after.tmp!.inodes).toBeLessThanOrEqual(baseline.tmp!.inodes + 4); + expect(after.workspaces).toEqual(baseline.workspaces); +} + +test.skipIf(!tmpfsEnabled)( + 'repeated file-heavy Job cleanup returns tmpfs bytes and inodes near baseline', + async () => { + expect(process.platform).toBe('linux'); + await prepareWorkspaceRoot(); + const baseline = readCleanupResources(); + expect(baseline.tmp?.isTmpfs).toBe(true); + const activeBefore = sandboxJobUidPool.activeCount(); + const payload = Buffer.alloc(8 * 1048576, 0x61); + for (let iteration = 0; iteration < 24; iteration++) { + const job = jobFor(runtime('/tmp'), 'echo test'); + let dir = ''; + try { + await job.prime(); + dir = workspace(job); + await fsp.writeFile(path.join(dir, 'payload'), payload); + for (let i = 0; i < 64; i++) + await fsp.writeFile(path.join(dir, `small-${i}`), 'x'); + expect(readCleanupResources().tmp!.bytes).toBeGreaterThan( + baseline.tmp!.bytes + payload.length + ); + } finally { + await job.cleanup(); + } + expect(fs.existsSync(dir)).toBe(false); + expect(sandboxJobUidPool.activeCount()).toBe(activeBefore); + assertTmpfsBaseline(baseline); + } + }, + 30_000 +); + +function processesForUid(uid: number): string[] { + return fs.readdirSync('/proc').filter(pid => { + if (!/^\d+$/.test(pid)) return false; + try { + return ( + fs + .readFileSync(`/proc/${pid}/status`, 'utf8') + .match(/^Uid:\s+(\d+)/m)?.[1] === String(uid) + ); + } catch (error) { + if ( + (error as NodeJS.ErrnoException).code === 'ENOENT' || + (error as NodeJS.ErrnoException).code === 'ESRCH' + ) + return false; + throw error; + } + }); +} + +test.skipIf(!nsjailEnabled)( + 'real NsJail releases detached descendants and file-heavy workspaces on success, timeout and overflow', + async () => { + expect(process.platform).toBe('linux'); + expect(config.per_job_uids).toBe(true); + expect(process.getuid?.()).toBe(0); + await prepareWorkspaceRoot(); + const pkgdir = await fsp.mkdtemp( + path.join(os.tmpdir(), 'cleanup-runtime-') + ); + await fsp.chmod(pkgdir, 0o755); + await fsp.writeFile(path.join(pkgdir, 'run'), 'exec /bin/bash "$@"\n', { + mode: 0o644, + }); + const baseline = readCleanupResources(); + expect(baseline.tmp?.isTmpfs).toBe(true); + const activeBefore = sandboxJobUidPool.activeCount(); + try { + for (let iteration = 0; iteration < 12; iteration++) { + const ending = iteration % 3; + const job = jobFor( + runtime(pkgdir), + `set -eu +# Both linked workspace data and an unlinked open file held by a detached child. +dd if=/dev/zero of=payload bs=1048576 count=8 status=none +# Per-jail /tmp is a different mount from the runner's /tmp. +dd if=/dev/zero of=/tmp/held bs=1048576 count=8 status=none +exec 3 /mnt/data/child-ready; exec sleep 30' >/dev/null 2>&1 & +while [ ! -f child-ready ]; do sleep 0.01; done +${ + ending === 0 + ? 'echo complete' + : ending === 1 + ? 'sleep 30' + : "while :; do printf '%1024s' x; done" +} +` + ); + let dir = ''; + let uid = -1; + try { + await job.prime(); + dir = workspace(job); + uid = (job as unknown as { jobIdentity: { uid: number } }).jobIdentity + .uid; + const result = await job.execute(); + expect(result.run?.status).toBe( + ending === 0 ? null : ending === 1 ? 'TO' : 'OL' + ); + if (ending === 0) expect(result.run?.stdout).toContain('complete'); + // Check before workspace removal can conceal a surviving writer. + expect(processesForUid(uid)).toEqual([]); + } finally { + await job.cleanup(); + } + expect(fs.existsSync(dir)).toBe(false); + expect(sandboxJobUidPool.activeCount()).toBe(activeBefore); + assertTmpfsBaseline(baseline); + } + } finally { + await fsp.rm(pkgdir, { recursive: true, force: true }); + } + }, + 60_000 +); diff --git a/api/src/job-cleanup-metrics.test.ts b/api/src/job-cleanup-metrics.test.ts new file mode 100644 index 00000000..6806f637 --- /dev/null +++ b/api/src/job-cleanup-metrics.test.ts @@ -0,0 +1,118 @@ +import { afterEach, expect, spyOn, test } from 'bun:test'; +import * as fsp from 'fs/promises'; +import * as os from 'os'; +import * as path from 'path'; +import * as semver from 'semver'; +import { register } from 'prom-client'; +import { Job } from './job'; +import type { Runtime } from './runtime'; +import type { SessionWorkspace } from './session-workspace'; +import { + clearRetainedWorkspaceCleanupsForTest, + retainedWorkspaceCleanupCount, + retryRetainedWorkspaceCleanups, + sandboxJobUidPool, +} from './workspace-isolation'; + +import type { + SandboxJobIdentity, + SandboxWorkspaceLease, +} from './workspace-isolation'; + +interface Internals { + submissionDir: string; + workspaceLease?: SandboxWorkspaceLease; + jobIdentity?: SandboxJobIdentity; +} +const roots: string[] = []; +afterEach(async () => { + await retryRetainedWorkspaceCleanups(); + clearRetainedWorkspaceCleanupsForTest(); + for (const root of roots.splice(0)) + await fsp.rm(root, { recursive: true, force: true }); +}); + +async function fixture(persistent = false) { + const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'job-cleanup-metric-')); + roots.push(dir); + const identity = sandboxJobUidPool.acquire()!; + expect(identity).not.toBeNull(); + const lease = { dir, workspaceId: path.basename(dir), identity }; + const rt: Runtime = { + language: 'bash', + version: new semver.SemVer('5.2.0'), + aliases: [], + pkgdir: '/tmp', + compiled: false, + env_vars: {}, + timeouts: { compile: 1000, run: 1000 }, + cpu_times: { compile: 1000, run: 1000 }, + memory_limits: { compile: -1, run: -1 }, + max_process_count: 64, + max_open_files: 64, + max_file_size: 1000, + output_max_size: 1000, + }; + const job = new Job({ + runtime: rt, + files: [], + args: [], + stdin: '', + timeouts: rt.timeouts, + cpu_times: rt.cpu_times, + memory_limits: rt.memory_limits, + is_synthetic: true, + session: persistent ? ({} as SessionWorkspace) : undefined, + }); + Object.assign(job as unknown as Internals, { + submissionDir: dir, + workspaceLease: lease, + jobIdentity: identity, + }); + return { job, dir, identity }; +} + +async function counter(mode: string, outcome: string): Promise { + const metric = register.getSingleMetric('codeapi_sandbox_cleanup_total')!; + return ( + (await metric.get()).values.find( + value => value.labels.mode === mode && value.labels.outcome === outcome + )?.value ?? 0 + ); +} + +test('failed removal is measured as retained and does not release its UID before retry', async () => { + const activeBefore = sandboxJobUidPool.activeCount(); + const retainedBefore = await counter('disposable', 'retained'); + const { job, dir } = await fixture(); + const rm = spyOn(fsp, 'rm').mockRejectedValueOnce( + Object.assign(new Error('test busy workspace'), { code: 'EBUSY' }) + ); + try { + await job.cleanup(); + expect(await counter('disposable', 'retained')).toBe(retainedBefore + 1); + expect(sandboxJobUidPool.activeCount()).toBe(activeBefore + 1); + expect(retainedWorkspaceCleanupCount()).toBe(1); + expect((await fsp.lstat(dir)).isDirectory()).toBe(true); + } finally { + rm.mockRestore(); + await retryRetainedWorkspaceCleanups(); + } + expect(sandboxJobUidPool.activeCount()).toBe(activeBefore); + expect(await fsp.lstat(dir).catch(() => null)).toBeNull(); +}); + +test('session cleanup is measured as preserved and retains both files and pinned UID', async () => { + const activeBefore = sandboxJobUidPool.activeCount(); + const preservedBefore = await counter('session', 'preserved'); + const { job, dir, identity } = await fixture(true); + try { + await fsp.writeFile(path.join(dir, 'state'), 'keep'); + await job.cleanup(); + expect(await counter('session', 'preserved')).toBe(preservedBefore + 1); + expect(await fsp.readFile(path.join(dir, 'state'), 'utf8')).toBe('keep'); + expect(sandboxJobUidPool.activeCount()).toBe(activeBefore + 1); + } finally { + sandboxJobUidPool.release(identity); + } +}); diff --git a/api/src/job.ts b/api/src/job.ts index 0939de08..5c2fddef 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -14,6 +14,7 @@ import { logger as rootLogger } from './logger'; import { getRuntimes } from './runtime'; import { execute } from './nsjail'; import { config } from './config'; +import { recordCleanupResources } from './cleanup-metrics'; import { internalServiceHeaders } from './internal-service-auth'; import { EGRESS_GRANT_HEADER, EGRESS_ERROR_CODE_HEADER } from './egress'; import { injectTraceHeaders } from './telemetry'; @@ -2716,49 +2717,61 @@ export class Job { this.log.info('Cleaning up'); } - /* Session mode: the workspace and pinned UID belong to the long-lived - * session, not this job. Keep both so the next call sees prior files; - * teardown happens on the /terminate hook (or explicit session reset). */ - if (this.session) { - this.workspaceLease = undefined; - this.submissionDir = ''; - this.jobIdentity = undefined; - return; - } - - let workspaceRemoved = true; - const workspaceLease = this.workspaceLease; - const jobIdentity = this.jobIdentity; - - if (workspaceLease) { - try { - workspaceRemoved = await cleanupSandboxWorkspace(workspaceLease); - } catch (error) { - workspaceRemoved = false; - this.log.error({ submissionDir: this.submissionDir, err: error }, 'Failed to clean up'); - } finally { + let outcome: 'removed' | 'preserved' | 'retained' | 'error' = 'error'; + try { + /* Session mode: the workspace and pinned UID belong to the long-lived + * session, not this job. Keep both so the next call sees prior files; + * teardown happens on the /terminate hook (or explicit session reset). */ + if (this.session) { this.workspaceLease = undefined; this.submissionDir = ''; + this.jobIdentity = undefined; + outcome = 'preserved'; + return; } - } - if (jobIdentity) { - if (!workspaceLease || workspaceRemoved) { - releaseJobIdentity(jobIdentity); - } else { - retainWorkspaceCleanupUntilRemoved(workspaceLease, () => { + let workspaceRemoved = true; + const workspaceLease = this.workspaceLease; + const jobIdentity = this.jobIdentity; + + if (workspaceLease) { + try { + workspaceRemoved = await cleanupSandboxWorkspace(workspaceLease); + } catch (error) { + workspaceRemoved = false; + this.log.error({ submissionDir: this.submissionDir, err: error }, 'Failed to clean up'); + } finally { + this.workspaceLease = undefined; + this.submissionDir = ''; + } + } + + if (jobIdentity) { + if (!workspaceLease || workspaceRemoved) { releaseJobIdentity(jobIdentity); - this.log.info( + } else { + retainWorkspaceCleanupUntilRemoved(workspaceLease, () => { + releaseJobIdentity(jobIdentity); + this.log.info( + { uid: jobIdentity.uid, gid: jobIdentity.gid, slot: jobIdentity.slot }, + 'Released retained sandbox job UID slot after workspace cleanup', + ); + }); + this.log.error( { uid: jobIdentity.uid, gid: jobIdentity.gid, slot: jobIdentity.slot }, - 'Released retained sandbox job UID slot after workspace cleanup', + 'Retaining sandbox job UID slot after failed workspace cleanup', ); - }); - this.log.error( - { uid: jobIdentity.uid, gid: jobIdentity.gid, slot: jobIdentity.slot }, - 'Retaining sandbox job UID slot after failed workspace cleanup', - ); + } + this.jobIdentity = undefined; } - this.jobIdentity = undefined; + outcome = workspaceRemoved ? 'removed' : 'retained'; + } finally { + recordCleanupResources({ + job: this.uuid, + mode: this.session ? 'session' : 'disposable', + outcome, + suppressSuccessLogs: this.isSynthetic, + }); } } } diff --git a/api/src/nsjail.ts b/api/src/nsjail.ts index 87a2b791..a9f02931 100644 --- a/api/src/nsjail.ts +++ b/api/src/nsjail.ts @@ -523,10 +523,10 @@ export async function execute(opts: ExecuteOptions, setupGate: NsJailSetupGate = }); const wallTime = Date.now() - startTime; - // Log memory metrics after each execution to track potential leaks. + // Pre-cleanup diagnostic only: Job.cleanup records post-cleanup resources. // Reads from /proc/self/cgroup to find the actual cgroup path, then reads // memory.current and memory.stat from that cgroup. - // Distinguishes real usage (anon) from reclaimable kernel page cache (file). + // `file` includes shmem/tmpfs and is not all reclaimable page cache. try { const cgroupLine = fs.readFileSync('/proc/self/cgroup', 'utf8').trim(); // cgroup v2 format: "0::" diff --git a/helm/codeapi/Chart.yaml b/helm/codeapi/Chart.yaml index 4e122bbf..dfcdd45b 100644 --- a/helm/codeapi/Chart.yaml +++ b/helm/codeapi/Chart.yaml @@ -3,7 +3,7 @@ apiVersion: v2 name: codeapi description: A Helm chart for Code Interpreter API - scalable code execution service type: application -version: 0.3.1 # Chart version (bump this when you change the chart) +version: 0.3.2 # Chart version (bump this when you change the chart) appVersion: "2.0.0" # App version (bump this when you change the app) # Keywords for searching diff --git a/helm/codeapi/README.md b/helm/codeapi/README.md index a063e597..f63950ae 100644 --- a/helm/codeapi/README.md +++ b/helm/codeapi/README.md @@ -55,6 +55,43 @@ platform rather than templated here: external ingress/service mesh, KEDA-style queue-depth autoscaling, and cloud-IAM secret delivery (the env hooks below cover all of them). +**Prometheus scraping.** With `metrics.enabled=true` and +`workerSandbox.enabled=true`, separate PodMonitors scrape the service-worker's `health` port and the sandbox-runner's +`sandbox` port at `/metrics`. Only the latter exports +`codeapi_sandbox_post_cleanup_*` metrics. Both inherit `metrics.interval` and +`metrics.scrapeTimeout`. Install the Prometheus Operator PodMonitor CRD and +configure Prometheus to discover this release's PodMonitors and namespace. + +The default NetworkPolicy allows runner ingress only from service-worker pods. +To permit scraping with `networkPolicy.enabled=true`, explicitly match your +trusted Prometheus namespace **and** pod labels in one peer, for example: + +```yaml +metrics: + enabled: true + sandboxRunner: + ingressFrom: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: observability + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus +``` + +Replace the example labels with those of your scraper. The default empty list +adds no access, even when metrics are enabled. This rule is rendered only while +metrics and the worker/sandbox tier are enabled. It uses the configured runner +port and does not change sandbox egress. Any policies on the Prometheus side +must also allow the scrape connection. When chart network policies are disabled, +allow scraping through your platform's policies instead. + +NetworkPolicy cannot restrict access to an HTTP path. These peers can reach the +runner's execution endpoint as well, so do not use unrestricted peers such as +`{}` or disable execution-manifest verification to enable scraping. No extra +Service or externally exposed port is created. Chart upgrades do not modify +runner images or require changes to stored session state for this scrape wiring. + **Pairing-fence rollbacks.** Do not use a direct `helm rollback` from a chart revision containing the bridge pairing fence to an older revision. Helm runs rollback hooks from the target revision, so a pre-fence target cannot stop its diff --git a/helm/codeapi/templates/network-policy.yaml b/helm/codeapi/templates/network-policy.yaml index 2184e9f9..fe24123f 100644 --- a/helm/codeapi/templates/network-policy.yaml +++ b/helm/codeapi/templates/network-policy.yaml @@ -3,6 +3,7 @@ Network Policies for CodeAPI. The important boundary is sandbox-runner: - service-worker may call sandbox-runner. + - explicitly configured metrics peers may reach the same runner port. - sandbox-runner may call only egress-gateway (plus DNS if required). - egress-gateway delegates only to file-server and tool-call-server. */}} @@ -70,6 +71,15 @@ spec: ports: - protocol: TCP port: {{ .Values.workerSandbox.sandbox.port }} + {{- if and .Values.metrics.enabled .Values.workerSandbox.enabled }} + {{- with (.Values.metrics.sandboxRunner | default dict).ingressFrom }} + - from: + {{- toYaml . | nindent 6 }} + ports: + - protocol: TCP + port: {{ $.Values.workerSandbox.sandbox.port }} + {{- end }} + {{- end }} egress: - to: - namespaceSelector: {} diff --git a/helm/codeapi/templates/podmonitor.yaml b/helm/codeapi/templates/podmonitor.yaml index 405e3bc3..aee33f44 100644 --- a/helm/codeapi/templates/podmonitor.yaml +++ b/helm/codeapi/templates/podmonitor.yaml @@ -43,6 +43,28 @@ spec: matchLabels: {{- include "codeapi.serviceWorker.selectorLabels" . | nindent 6 }} {{- end }} +{{- if .Values.workerSandbox.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: {{ include "codeapi.fullname" . }}-sandbox-runner + labels: + {{- include "codeapi.sandboxRunner.labels" . | nindent 4 }} +spec: + podMetricsEndpoints: + - path: /metrics + port: sandbox + {{- with .Values.metrics.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.metrics.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + selector: + matchLabels: + {{- include "codeapi.sandboxRunner.selectorLabels" . | nindent 6 }} +{{- end }} {{- if .Values.egressGateway.enabled }} --- apiVersion: monitoring.coreos.com/v1 diff --git a/helm/codeapi/values.yaml b/helm/codeapi/values.yaml index bafed305..0a242639 100644 --- a/helm/codeapi/values.yaml +++ b/helm/codeapi/values.yaml @@ -512,6 +512,11 @@ metrics: enabled: false # Set to true to create PodMonitors for Prometheus scraping interval: "30s" scrapeTimeout: "10s" + sandboxRunner: + # Explicit NetworkPolicy peers allowed to scrape the runner when metrics and + # networkPolicy are enabled. Empty preserves worker-only ingress. This port + # also serves /execute, so select only trusted Prometheus pods (see README). + ingressFrom: [] # ============================================================================= # NETWORK POLICIES diff --git a/tests/sandbox_runner_metrics.sh b/tests/sandbox_runner_metrics.sh new file mode 100755 index 00000000..df7c9ff3 --- /dev/null +++ b/tests/sandbox_runner_metrics.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT +command -v helm >/dev/null + +# Like block_root_package_delivery.sh, render only this chart's templates so +# tests do not download Redis or MinIO dependencies or contact a cluster. +mkdir "$TMP_DIR/chart" +cp "$ROOT/helm/codeapi/values.yaml" "$TMP_DIR/chart/values.yaml" +cp -R "$ROOT/helm/codeapi/templates" "$TMP_DIR/chart/templates" +awk '/^dependencies:/{exit} {print}' "$ROOT/helm/codeapi/Chart.yaml" > "$TMP_DIR/chart/Chart.yaml" + +render() { + helm template telemetry "$TMP_DIR/chart" \ + --set executionManifest.privateKey=test \ + --set executionManifest.publicKey=test "$@" +} + +resource() { + local file="$1" kind="$2" name="$3" + awk -v kind="$kind" -v name="$name" ' + function emit() { if (foundKind && foundName) printf "%s", document } + /^---$/ { emit(); document=""; foundKind=0; foundName=0; next } + { document=document $0 "\n" } + $0 == "kind: " kind { foundKind=1 } + $0 == " name: " name { foundName=1 } + END { emit() } + ' "$file" +} + +contains() { + if ! grep -Fq -- "$2" "$1"; then + echo "Missing '$2' in $(basename "$1")" >&2 + exit 1 + fi +} + +check_monitor() { + local rendered="$1" component="$2" port="$3" + local monitor="$TMP_DIR/$component-monitor.yaml" + resource "$rendered" PodMonitor "telemetry-codeapi-$component" > "$monitor" + contains "$monitor" ' - path: /metrics' + contains "$monitor" " port: $port" + contains "$monitor" ' interval: 17s' + contains "$monitor" ' scrapeTimeout: 7s' + # Exact selectors must match the corresponding Deployment's pod labels, + # including this release, and not accidentally select the service worker. + sed -n '/^ selector:/,$p' "$monitor" > "$TMP_DIR/selector.yaml" + contains "$TMP_DIR/selector.yaml" ' app.kubernetes.io/name: codeapi' + contains "$TMP_DIR/selector.yaml" ' app.kubernetes.io/instance: telemetry' + contains "$TMP_DIR/selector.yaml" " app.kubernetes.io/component: $component" + resource "$rendered" Deployment "telemetry-codeapi-$component" > "$TMP_DIR/deployment.yaml" + contains "$TMP_DIR/deployment.yaml" " app.kubernetes.io/component: $component" + contains "$TMP_DIR/deployment.yaml" " - name: $port" +} + +render > "$TMP_DIR/default.yaml" +if grep -q '^kind: PodMonitor$' "$TMP_DIR/default.yaml"; then + echo 'metrics.enabled=false must not create PodMonitors' >&2 + exit 1 +fi + +render --set metrics.enabled=true --set metrics.interval=17s --set metrics.scrapeTimeout=7s \ + --set workerSandbox.sandbox.port=2345 > "$TMP_DIR/enabled.yaml" +check_monitor "$TMP_DIR/enabled.yaml" sandbox-runner sandbox +check_monitor "$TMP_DIR/enabled.yaml" service-worker health +contains "$TMP_DIR/deployment.yaml" ' app.kubernetes.io/instance: telemetry' +resource "$TMP_DIR/enabled.yaml" Deployment telemetry-codeapi-sandbox-runner > "$TMP_DIR/runner.yaml" +contains "$TMP_DIR/runner.yaml" ' containerPort: 2345' + +# Enabling metrics must not silently weaken sandbox isolation. +resource "$TMP_DIR/default.yaml" NetworkPolicy telemetry-codeapi-sandbox-runner > "$TMP_DIR/default-policy.yaml" +resource "$TMP_DIR/enabled.yaml" NetworkPolicy telemetry-codeapi-sandbox-runner > "$TMP_DIR/enabled-policy.yaml" +# Use the same configured runner port when comparing policies. +sed 's/port: 2345/port: 2000/' "$TMP_DIR/enabled-policy.yaml" > "$TMP_DIR/normalized-policy.yaml" +cmp "$TMP_DIR/default-policy.yaml" "$TMP_DIR/normalized-policy.yaml" + +cat > "$TMP_DIR/scraper.yaml" <<'YAML' +metrics: + sandboxRunner: + ingressFrom: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: metrics-test + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus-test +YAML +render --set metrics.enabled=true --set workerSandbox.sandbox.port=2345 \ + -f "$TMP_DIR/scraper.yaml" > "$TMP_DIR/scraper-render.yaml" +resource "$TMP_DIR/scraper-render.yaml" NetworkPolicy telemetry-codeapi-sandbox-runner > "$TMP_DIR/scraper-policy.yaml" +contains "$TMP_DIR/scraper-policy.yaml" 'kubernetes.io/metadata.name: metrics-test' +contains "$TMP_DIR/scraper-policy.yaml" 'app.kubernetes.io/name: prometheus-test' +contains "$TMP_DIR/scraper-policy.yaml" 'app.kubernetes.io/component: service-worker' +# The namespace and pod selectors must be ANDed in ONE peer, not two ORed peers. +contains "$TMP_DIR/scraper-policy.yaml" ' - namespaceSelector:' +contains "$TMP_DIR/scraper-policy.yaml" ' podSelector:' +[[ "$(grep -Fc ' port: 2345' "$TMP_DIR/scraper-policy.yaml")" == 2 ]] +# Scraping changes only ingress, never the sandbox egress boundary. +sed -n '/^ egress:/,$p' "$TMP_DIR/scraper-policy.yaml" > "$TMP_DIR/scraper-egress.yaml" +sed -n '/^ egress:/,$p' "$TMP_DIR/default-policy.yaml" > "$TMP_DIR/default-egress.yaml" +cmp "$TMP_DIR/default-egress.yaml" "$TMP_DIR/scraper-egress.yaml" + +render -f "$TMP_DIR/scraper.yaml" > "$TMP_DIR/disabled-scraper.yaml" +resource "$TMP_DIR/disabled-scraper.yaml" NetworkPolicy telemetry-codeapi-sandbox-runner > "$TMP_DIR/disabled-scraper-policy.yaml" +cmp "$TMP_DIR/default-policy.yaml" "$TMP_DIR/disabled-scraper-policy.yaml" + +render --set metrics.enabled=true --set workerSandbox.enabled=false \ + -f "$TMP_DIR/scraper.yaml" > "$TMP_DIR/no-workers.yaml" +[[ -z "$(resource "$TMP_DIR/no-workers.yaml" PodMonitor telemetry-codeapi-sandbox-runner)" ]] +[[ -z "$(resource "$TMP_DIR/no-workers.yaml" PodMonitor telemetry-codeapi-service-worker)" ]] +resource "$TMP_DIR/no-workers.yaml" NetworkPolicy telemetry-codeapi-sandbox-runner > "$TMP_DIR/no-workers-policy.yaml" +if grep -q prometheus-test "$TMP_DIR/no-workers-policy.yaml"; then + echo 'disabled workerSandbox must not enable scrape ingress' >&2 + exit 1 +fi + +# Older values may omit the new subtree; this must remain worker-only ingress. +render --set metrics.enabled=true --set metrics.sandboxRunner=null > "$TMP_DIR/old-values.yaml" +resource "$TMP_DIR/old-values.yaml" NetworkPolicy telemetry-codeapi-sandbox-runner > "$TMP_DIR/old-values-policy.yaml" +cmp "$TMP_DIR/default-policy.yaml" "$TMP_DIR/old-values-policy.yaml" + +render --set metrics.enabled=true --set api.enabled=false > "$TMP_DIR/no-api.yaml" +[[ -n "$(resource "$TMP_DIR/no-api.yaml" PodMonitor telemetry-codeapi-sandbox-runner)" ]] +[[ -z "$(resource "$TMP_DIR/no-api.yaml" PodMonitor telemetry-codeapi-api)" ]] + +render --set metrics.enabled=true --set metrics.interval= --set metrics.scrapeTimeout= > "$TMP_DIR/no-timing.yaml" +resource "$TMP_DIR/no-timing.yaml" PodMonitor telemetry-codeapi-sandbox-runner > "$TMP_DIR/no-timing-monitor.yaml" +if grep -Eq 'interval:|scrapeTimeout:' "$TMP_DIR/no-timing-monitor.yaml"; then + echo 'empty scrape timing overrides must defer to Prometheus defaults' >&2 + exit 1 +fi + +render --set metrics.enabled=true --set networkPolicy.enabled=false \ + -f "$TMP_DIR/scraper.yaml" > "$TMP_DIR/no-policy.yaml" +[[ -n "$(resource "$TMP_DIR/no-policy.yaml" PodMonitor telemetry-codeapi-sandbox-runner)" ]] +if grep -q '^kind: NetworkPolicy$' "$TMP_DIR/no-policy.yaml"; then + echo 'networkPolicy.enabled=false must not create NetworkPolicies' >&2 + exit 1 +fi + +helm lint "$TMP_DIR/chart" --set executionManifest.privateKey=test --set executionManifest.publicKey=test \ + --set metrics.enabled=true -f "$TMP_DIR/scraper.yaml" +echo 'Sandbox runner metrics chart regressions passed' From b03582ae8740e6b56d12945e95f57cd658048fd8 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sun, 27 Sep 2026 11:41:22 -0400 Subject: [PATCH 09/48] feat: Allow Installation-Scoped GitHub Tokens for Trusted Workers (#263) * feat(code): allow installation-scoped GitHub tokens on trusted workers * fix(code): revalidate GitHub installation token grants --------- Co-authored-by: Lia --- docs/remote-bridge/worker-runbook.md | 9 + packages/code/README.md | 11 ++ packages/code/src/cli.test.ts | 29 +++ packages/code/src/cli.ts | 23 ++- packages/code/src/github.test.ts | 266 +++++++++++++++++++++++++++ packages/code/src/github.ts | 159 ++++++++++++---- 6 files changed, 459 insertions(+), 38 deletions(-) diff --git a/docs/remote-bridge/worker-runbook.md b/docs/remote-bridge/worker-runbook.md index 3043c541..493eb000 100644 --- a/docs/remote-bridge/worker-runbook.md +++ b/docs/remote-bridge/worker-runbook.md @@ -338,6 +338,15 @@ filesystem root, but anyone able to alter a checkout's remote can select any repository where the App is installed; keep the App's installation scope narrow. Pass the checkout as the command working directory; changing directories only inside the shell cannot change the token chosen before command launch. +For a trusted VM that needs to switch among repositories in the same installed +account or organization inside one command, set +`LIBRECHAT_CODE_GITHUB_TOKEN_SCOPE=installation`. The resolved installation +token covers only repositories and permissions GitHub granted to that App +installation. It refreshes after two minutes so newly approved permissions +become available without a worker restart. The default is `repository`. +Commands spanning different accounts or organizations must start in a checkout +from the target account or organization; a shell `cd` cannot switch the +installation chosen at command launch. Set `LIBRECHAT_CODE_GITHUB_INSTALLATION_ID` only as a legacy fixed-installation fallback; it cannot be combined with checkout routing. diff --git a/packages/code/README.md b/packages/code/README.md index 5a06b92a..a87c87df 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -308,6 +308,17 @@ installed**. Use this mode only where the machine operator trusts the VM and the App's installation scope; the default `admitted` mode keeps the startup binding. Checkout routing requires an App without a fixed installation ID. +On a trusted VM, `--github-token-scope installation` (or +`LIBRECHAT_CODE_GITHUB_TOKEN_SCOPE=installation`) mints one token for all +repositories GitHub grants to the resolved App installation. This lets a +command started in one checkout push to another repository in the same account +or organization, including through `cd` or `git -C`, and use organization +Projects. GitHub still enforces the installation's selected repositories and +permissions. Tokens are shared by installation, refreshed after two minutes, +and kept out of the sandbox's readable environment. The default remains +`repository`. A command crossing to another account or organization still +needs to start in a checkout belonging to that account or organization. + For compatibility with deployments that intentionally bind a worker to one installation, set the optional legacy `LIBRECHAT_CODE_GITHUB_INSTALLATION_ID` fallback. diff --git a/packages/code/src/cli.test.ts b/packages/code/src/cli.test.ts index f95d7eee..f6737d5f 100644 --- a/packages/code/src/cli.test.ts +++ b/packages/code/src/cli.test.ts @@ -358,6 +358,35 @@ test('CLI rejects checkout routing outside a trusted VM or without repository-sc assert.match(invalid.stderr, /must be admitted or checkout/); }); +test('CLI permits installation-scoped GitHub tokens only for a trusted VM with routed App auth', () => { + const cli = fileURLToPath(new URL('./cli.js', import.meta.url)); + const base = { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_WORKER_DIR: process.cwd(), + LIBRECHAT_CODE_ALLOW_WORKSPACE_COMMANDS: 'true', + LIBRECHAT_CODE_GITHUB_TOKEN: undefined, + LIBRECHAT_CODE_GITHUB_APP_ID: '123', + LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE: '/does/not/matter', + LIBRECHAT_CODE_GITHUB_INSTALLATION_ID: undefined, + LIBRECHAT_CODE_GITHUB_TOKEN_SCOPE: 'installation', + }; + const restricted = spawnSync(process.execPath, [cli], { encoding: 'utf8', env: base }); + assert.match(restricted.stderr, /Installation-scoped GitHub tokens require the trusted-vm/); + const trusted = spawnSync(process.execPath, [cli], { + encoding: 'utf8', + env: { ...base, LIBRECHAT_CODE_COMMAND_POLICY_PRESET: 'trusted-vm' }, + }); + assert.doesNotMatch(trusted.stderr, /Installation-scoped GitHub tokens require/); + const fixed = spawnSync(process.execPath, [cli], { + encoding: 'utf8', + env: { ...base, LIBRECHAT_CODE_GITHUB_INSTALLATION_ID: '456' }, + }); + assert.match(fixed.stderr, /without a fixed installation ID/); +}); + test('CLI requires a runtime image for Docker supervision', () => { const result = spawnSync( process.execPath, diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 841164fb..6e3a0062 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -157,6 +157,7 @@ function githubCredentials(args: string[]): { mode?: 'app' | 'token'; repositoryRouting?: boolean; checkoutRouting?: boolean; + installationTokenScope?: boolean; policyIdentity: string; } { const token = nonEmpty(process.env.LIBRECHAT_CODE_GITHUB_TOKEN); @@ -191,6 +192,18 @@ function githubCredentials(args: string[]): { 'Checkout GitHub repository routing requires a GitHub App without a fixed installation ID', ); } + const tokenScope = + option(args, '--github-token-scope')?.trim().toLowerCase() ?? + process.env.LIBRECHAT_CODE_GITHUB_TOKEN_SCOPE?.trim().toLowerCase() ?? + 'repository'; + if (tokenScope !== 'repository' && tokenScope !== 'installation') { + throw new Error('GitHub token scope must be repository or installation'); + } + if (tokenScope === 'installation' && (!hasApp || installationId)) { + throw new Error( + 'Installation-scoped GitHub tokens require a GitHub App without a fixed installation ID', + ); + } const configuredHostValue = nonEmpty( process.env.LIBRECHAT_CODE_GITHUB_HOST, ); @@ -225,16 +238,19 @@ function githubCredentials(args: string[]): { mode: 'app', repositoryRouting: !installationId, checkoutRouting: routing === 'checkout', + installationTokenScope: tokenScope === 'installation', policyIdentity: gitHubAuthenticationPolicyIdentity({ mode: 'app', host, appId, installationId, - }) + (routing === 'checkout' ? ':routing:checkout' : ''), + }) + (routing === 'checkout' ? ':routing:checkout' : '') + + (tokenScope === 'installation' ? ':scope:installation' : ''), privateKeyPath, provider: new GitHubAppCredentialProvider({ appId: appId!, installationId, + tokenScope, privateKeyPath: privateKeyPath!, host, apiUrl, @@ -568,6 +584,11 @@ async function run( 'Checkout GitHub repository routing requires the trusted-vm command policy', ); } + if (github.installationTokenScope && commandPolicy.preset !== 'trusted-vm') { + throw new Error( + 'Installation-scoped GitHub tokens require the trusted-vm command policy', + ); + } const githubDomains = github.provider ? github.host === 'github.com' ? [...GITHUB_ALLOWED_DOMAINS] diff --git a/packages/code/src/github.test.ts b/packages/code/src/github.test.ts index b13d70f7..5d401515 100644 --- a/packages/code/src/github.test.ts +++ b/packages/code/src/github.test.ts @@ -239,6 +239,272 @@ test('routes and scopes GitHub App tokens per repository installation', async (t ); }); +test('installation scope shares one token across repositories in an organization and refreshes grants', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-installation-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile( + privateKeyPath, + privateKey.export({ type: 'pkcs8', format: 'pem' }), + { mode: 0o600 }, + ); + let now = new Date('2030-01-01T00:00:00Z'); + const minted: Array<{ installation: string; body?: string }> = []; + const provider = new GitHubAppCredentialProvider({ + appId: '123', + tokenScope: 'installation', + privateKeyPath, + now: () => now, + fetch: (async (input, init) => { + const url = String(input); + if (url.endsWith('/app')) return Response.json({ slug: 'lia' }); + if (url.endsWith('/users/lia%5Bbot%5D')) { + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); + } + if (/\/repos\/LibreChat-AI\/(agents|LibreChat)\/installation$/.test(url)) { + return Response.json({ id: 111 }); + } + if (url.endsWith('/repos/ClickHouse/Agents/installation')) { + return Response.json({ id: 222 }); + } + const installation = /\/app\/installations\/(\d+)\/access_tokens$/.exec(url)?.[1]; + if (installation) { + minted.push({ + installation, + body: typeof init?.body === 'string' ? init.body : undefined, + }); + return Response.json({ + token: `ghs_${installation}_${minted.length}_abcdefghijklmnopqrstuvwxyz`, + expires_at: '2030-01-01T01:00:00Z', + }, { status: 201 }); + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + const [agents, librechat] = await Promise.all([ + provider.getCredential(undefined, 'LibreChat-AI/agents'), + provider.getCredential(undefined, 'LibreChat-AI/LibreChat'), + ]); + assert.equal(agents.value, librechat.value); + assert.deepEqual(minted, [{ installation: '111', body: undefined }]); + const clickhouse = await provider.getCredential(undefined, 'ClickHouse/Agents'); + assert.notEqual(clickhouse.value, librechat.value); + assert.deepEqual(minted.map(entry => entry.installation), ['111', '222']); + now = new Date('2030-01-01T00:02:01Z'); + const refreshed = await provider.getCredential(undefined, 'LibreChat-AI/LibreChat'); + assert.notEqual(refreshed.value, librechat.value); + assert.deepEqual(minted.map(entry => entry.installation), ['111', '222', '111']); +}); + +test('installation scope shares a bounded lookup when a waiter cancels', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-shared-lookup-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile(privateKeyPath, privateKey.export({ type: 'pkcs8', format: 'pem' }), { + mode: 0o600, + }); + let releaseLookup!: (response: Response) => void; + const lookupResponse = new Promise(resolve => { releaseLookup = resolve; }); + let lookupStarted!: () => void; + const started = new Promise(resolve => { lookupStarted = resolve; }); + const cancelled = new AbortController(); + let lookups = 0; + let mints = 0; + const provider = new GitHubAppCredentialProvider({ + appId: '123', + tokenScope: 'installation', + privateKeyPath, + now: () => new Date('2030-01-01T00:00:00Z'), + fetch: (async (input, init) => { + const url = String(input); + if (url.endsWith('/repos/acme/project/installation')) { + lookups += 1; + assert.ok(init?.signal); + assert.notEqual(init.signal, cancelled.signal); + lookupStarted(); + return lookupResponse; + } + if (url.endsWith('/app/installations/111/access_tokens')) { + mints += 1; + return Response.json({ + token: 'ghs_shared_abcdefghijklmnopqrstuvwxyz', + expires_at: '2030-01-01T01:00:00Z', + }, { status: 201 }); + } + if (url.endsWith('/app')) return Response.json({ slug: 'lia' }); + if (url.endsWith('/users/lia%5Bbot%5D')) { + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + const first = provider.getCredential(cancelled.signal, 'acme/project'); + const others = Array.from({ length: 12 }, () => + provider.getCredential(undefined, 'acme/project')); + await started; + cancelled.abort(new Error('first command cancelled')); + await assert.rejects(first, /first command cancelled/); + releaseLookup(Response.json({ id: 111 })); + const credentials = await Promise.all(others); + assert.equal(lookups, 1); + assert.equal(mints, 1); + assert.equal(new Set(credentials.map(credential => credential.value)).size, 1); +}); + +test('installation scope times out a stalled lookup without a caller signal', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-lookup-timeout-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile(privateKeyPath, privateKey.export({ type: 'pkcs8', format: 'pem' }), { + mode: 0o600, + }); + const timeout = new AbortController(); + t.mock.method(AbortSignal, 'timeout', (ms: number) => { + assert.equal(ms, 30_000); + return timeout.signal; + }); + let lookupStarted!: () => void; + const started = new Promise(resolve => { lookupStarted = resolve; }); + const provider = new GitHubAppCredentialProvider({ + appId: '123', + tokenScope: 'installation', + privateKeyPath, + fetch: (async (input, init) => { + if (String(input).endsWith('/repos/acme/project/installation')) { + assert.equal(init?.signal, timeout.signal); + lookupStarted(); + return new Promise((_resolve, reject) => { + timeout.signal.addEventListener('abort', () => reject(timeout.signal.reason), { once: true }); + }); + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + const pending = provider.getCredential(undefined, 'acme/project'); + await started; + timeout.abort(new DOMException('Installation lookup timed out', 'TimeoutError')); + await assert.rejects(pending, { name: 'TimeoutError' }); +}); + +test('installation replacement checks every shared waiter’s repository grant', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-replaced-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile(privateKeyPath, privateKey.export({ type: 'pkcs8', format: 'pem' }), { + mode: 0o600, + }); + let now = new Date('2030-01-01T00:00:00Z'); + let replaced = false; + let bLookups = 0; + let oldMints = 0; + let releaseOldMint!: () => void; + const oldMintResponse = new Promise(resolve => { releaseOldMint = resolve; }); + let oldMintStarted!: () => void; + const oldMintPending = new Promise(resolve => { oldMintStarted = resolve; }); + const provider = new GitHubAppCredentialProvider({ + appId: '123', + tokenScope: 'installation', + privateKeyPath, + now: () => now, + fetch: (async input => { + const url = String(input); + if (url.endsWith('/repos/acme/a/installation')) { + return Response.json({ id: replaced ? 222 : 111 }); + } + if (url.endsWith('/repos/acme/b/installation')) { + bLookups += 1; + return replaced ? Response.json({}, { status: 404 }) : Response.json({ id: 111 }); + } + if (url.endsWith('/app/installations/111/access_tokens')) { + oldMints += 1; + if (replaced) { + oldMintStarted(); + await oldMintResponse; + return Response.json({}, { status: 404 }); + } + return Response.json({ + token: 'ghs_old_abcdefghijklmnopqrstuvwxyz', + expires_at: '2030-01-01T00:06:00Z', + }, { status: 201 }); + } + if (url.endsWith('/app/installations/222/access_tokens')) { + return Response.json({ + token: 'ghs_new_abcdefghijklmnopqrstuvwxyz', + expires_at: '2030-01-01T01:00:00Z', + }, { status: 201 }); + } + if (url.endsWith('/app')) return Response.json({ slug: 'lia' }); + if (url.endsWith('/users/lia%5Bbot%5D')) { + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + const old = await provider.getCredential(undefined, 'acme/a'); + assert.equal((await provider.getCredential(undefined, 'acme/b')).value, old.value); + replaced = true; + now = new Date('2030-01-01T00:01:01Z'); // Token needs refresh; both mappings are still live. + const forA = provider.getCredential(undefined, 'acme/a'); + await oldMintPending; + const forB = provider.getCredential(undefined, 'acme/b'); + await new Promise(resolve => setImmediate(resolve)); // Join A's stale installation mint. + releaseOldMint(); + const [a, b] = await Promise.allSettled([forA, forB]); + assert.equal(a.status, 'fulfilled'); + if (a.status === 'fulfilled') assert.equal(a.value.value, 'ghs_new_abcdefghijklmnopqrstuvwxyz'); + assert.equal(b.status, 'rejected'); + if (b.status === 'rejected') assert.match(String(b.reason), /not installed for acme\/b/); + assert.equal(oldMints, 2); // One initial mint and just one shared failed refresh. + assert.equal(bLookups, 2); // B must revalidate instead of receiving A's new token. +}); + +test('installation scope follows a repository transfer after the lookup expires', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-transfer-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile(privateKeyPath, privateKey.export({ type: 'pkcs8', format: 'pem' }), { + mode: 0o600, + }); + let now = new Date('2030-01-01T00:00:00Z'); + let lookupCount = 0; + const provider = new GitHubAppCredentialProvider({ + appId: '123', + tokenScope: 'installation', + privateKeyPath, + now: () => now, + fetch: (async (input) => { + const url = String(input); + if (url.endsWith('/app')) return Response.json({ slug: 'lia' }); + if (url.endsWith('/users/lia%5Bbot%5D')) { + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); + } + if (url.endsWith('/repos/acme/project/installation')) { + lookupCount += 1; + return Response.json({ id: lookupCount === 1 ? 111 : 222 }); + } + const installation = /\/app\/installations\/(\d+)\/access_tokens$/.exec(url)?.[1]; + if (installation) { + return Response.json({ + token: `ghs_${installation}_abcdefghijklmnopqrstuvwxyz`, + expires_at: '2030-01-01T01:00:00Z', + }); + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + assert.equal((await provider.getCredential(undefined, 'acme/project')).value, + 'ghs_111_abcdefghijklmnopqrstuvwxyz'); + now = new Date('2030-01-01T00:02:01Z'); + assert.equal((await provider.getCredential(undefined, 'acme/project')).value, + 'ghs_222_abcdefghijklmnopqrstuvwxyz'); + assert.equal(lookupCount, 2); +}); + test('keeps a shared token refresh alive when one waiter is cancelled', async (t) => { const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-cancel-')); t.after(() => rm(directory, { recursive: true, force: true })); diff --git a/packages/code/src/github.ts b/packages/code/src/github.ts index 19b8e639..8519938f 100644 --- a/packages/code/src/github.ts +++ b/packages/code/src/github.ts @@ -41,6 +41,8 @@ export interface GitHubAppCredentialProviderOptions { appId: string; /** Legacy fixed installation. Omit to resolve the installation per repository. */ installationId?: string; + /** Opt in to all repositories granted to the resolved installation. */ + tokenScope?: 'repository' | 'installation'; privateKeyPath: string; apiUrl?: string; /** Git HTTPS hostname; non-public hosts default to the GHES /api/v3 base. */ @@ -52,6 +54,13 @@ export interface GitHubAppCredentialProviderOptions { const execFileAsync = promisify(execFile); const GITHUB_SHARED_REQUEST_TIMEOUT_MS = 30_000; +const GITHUB_INSTALLATION_CACHE_MS = 2 * 60_000; + +class InstallationChangedError extends Error { + constructor(readonly installationId: string) { + super('GitHub App installation changed while issuing a token'); + } +} async function waitForShared( promise: Promise, @@ -248,8 +257,14 @@ function createAppJwt(appId: string, privateKey: string, now: Date): string { export class GitHubAppCredentialProvider implements GitHubCredentialProvider { private readonly cached = new Map(); + private readonly cachedAt = new Map(); private readonly inFlight = new Map>(); private readonly installationIds = new Map(); + private readonly installationIdCachedAt = new Map(); + private readonly installationIdInFlight = new Map< + string, + Promise<{ id: string; jwt: string }> + >(); private appLogin?: string; private appLoginInFlight?: Promise; private actor?: GitHubCredential['actor']; @@ -423,32 +438,80 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { private async resolveInstallationId( repository: string, - jwt: string, signal?: AbortSignal, - ): Promise { - if (this.options.installationId) return this.options.installationId; + jwt?: string, + ): Promise<{ id: string; jwt?: string }> { + if (this.options.installationId) { + return { id: this.options.installationId, jwt }; + } const cached = this.installationIds.get(repository); - if (cached) return cached; - const { owner, name } = repositoryName(repository); - const response = await this.request( - `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/installation`, - jwt, - signal, - ); - if (!response.ok) { - throw new Error( - response.status === 404 - ? `GitHub App is not installed for ${repository}` - : `GitHub App installation lookup failed with status ${response.status}`, + const now = (this.options.now ?? (() => new Date()))().getTime(); + if ( + cached && + now - (this.installationIdCachedAt.get(repository) ?? 0) < GITHUB_INSTALLATION_CACHE_MS + ) return { id: cached, jwt }; + const existing = this.installationIdInFlight.get(repository); + if (existing) return waitForShared(existing, signal); + const pending = (async () => { + const sharedSignal = AbortSignal.timeout(GITHUB_SHARED_REQUEST_TIMEOUT_MS); + const lookupJwt = jwt ?? await this.appJwt( + (this.options.now ?? (() => new Date()))(), ); + const { owner, name } = repositoryName(repository); + const response = await this.request( + `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/installation`, + lookupJwt, + sharedSignal, + ); + if (!response.ok) { + throw new Error( + response.status === 404 + ? `GitHub App is not installed for ${repository}` + : `GitHub App installation lookup failed with status ${response.status}`, + ); + } + const body = (await response.json()) as { id?: unknown }; + if (!Number.isSafeInteger(body.id) || Number(body.id) <= 0) { + throw new Error('GitHub App installation response is invalid'); + } + const id = String(body.id); + this.installationIds.set(repository, id); + this.installationIdCachedAt.set( + repository, + (this.options.now ?? (() => new Date()))().getTime(), + ); + return { id, jwt: lookupJwt }; + })(); + this.installationIdInFlight.set(repository, pending); + const clearPending = () => { + if (this.installationIdInFlight.get(repository) === pending) { + this.installationIdInFlight.delete(repository); + } + }; + void pending.then(clearPending, clearPending); + return waitForShared(pending, signal); + } + + private async waitForCredential( + pending: Promise, + signal: AbortSignal | undefined, + repository: string | undefined, + retried: boolean, + ): Promise { + try { + return await waitForShared(pending, signal); + } catch (error) { + if (!(error instanceof InstallationChangedError) || retried || !repository) { + throw error; + } + // A shared mint can serve several repositories. Never pass its replacement + // installation's token to a repository whose grant has not been checked. + if (this.installationIds.get(repository) === error.installationId) { + this.installationIds.delete(repository); + this.installationIdCachedAt.delete(repository); + } + return this.getCredentialAttempt(signal, repository, true); } - const body = (await response.json()) as { id?: unknown }; - if (!Number.isSafeInteger(body.id) || Number(body.id) <= 0) { - throw new Error('GitHub App installation response is invalid'); - } - const installationId = String(body.id); - this.installationIds.set(repository, installationId); - return installationId; } async getCredential( @@ -462,32 +525,46 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { ); } if (repository) repositoryName(repository); + return this.getCredentialAttempt(signal, repository, false); + } + + private async getCredentialAttempt( + signal: AbortSignal | undefined, + repository: string | undefined, + retried: boolean, + ): Promise { + signal?.throwIfAborted(); + const installationScope = this.options.tokenScope === 'installation'; + const installation = installationScope + ? await this.resolveInstallationId(repository ?? '', signal) + : undefined; const now = (this.options.now ?? (() => new Date()))(); - const key = this.options.installationId ?? repository!; + const key = installation?.id ?? this.options.installationId ?? repository!; const cached = this.cached.get(key); if ( cached?.expiresAt != null && - cached.expiresAt.getTime() - now.getTime() > 5 * 60_000 + cached.expiresAt.getTime() - now.getTime() > 5 * 60_000 && + (!installationScope || now.getTime() - (this.cachedAt.get(key) ?? 0) < GITHUB_INSTALLATION_CACHE_MS) ) { return cached; } const existing = this.inFlight.get(key); - if (existing) return waitForShared(existing, signal); + if (existing) return this.waitForCredential(existing, signal, repository, retried); const pending = (async () => { const sharedSignal = AbortSignal.timeout( GITHUB_SHARED_REQUEST_TIMEOUT_MS, ); - const jwt = await this.appJwt(now); - const scopedRepository = repository + const jwt = installation?.jwt ?? await this.appJwt(now); + const scopedRepository = !installationScope && repository ? repositoryName(repository).name : undefined; - const installationId = await this.resolveInstallationId( + const resolvedInstallationId = installation?.id ?? (await this.resolveInstallationId( repository ?? '', - jwt, sharedSignal, - ); + jwt, + )).id; let response = await this.request( - `/app/installations/${installationId}/access_tokens`, + `/app/installations/${resolvedInstallationId}/access_tokens`, jwt, sharedSignal, { @@ -495,18 +572,20 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { headers: { 'Content-Type': 'application/json', }, - ...(this.options.installationId + ...(this.options.installationId || installationScope ? {} : { body: JSON.stringify({ repositories: [scopedRepository] }) }), }, ); if (!this.options.installationId && response.status === 404) { + if (installationScope) throw new InstallationChangedError(resolvedInstallationId); this.installationIds.delete(repository!); - const refreshedInstallationId = await this.resolveInstallationId( + this.installationIdCachedAt.delete(repository!); + const refreshedInstallationId = (await this.resolveInstallationId( repository!, - jwt, sharedSignal, - ); + jwt, + )).id; response = await this.request( `/app/installations/${refreshedInstallationId}/access_tokens`, jwt, @@ -516,7 +595,9 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { headers: { 'Content-Type': 'application/json', }, - body: JSON.stringify({ repositories: [scopedRepository] }), + ...(installationScope + ? {} + : { body: JSON.stringify({ repositories: [scopedRepository] }) }), }, ); } @@ -554,6 +635,10 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { actor, }; this.cached.set(key, credential); + this.cachedAt.set( + key, + (this.options.now ?? (() => new Date()))().getTime(), + ); return credential; })(); this.inFlight.set(key, pending); @@ -561,7 +646,7 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { if (this.inFlight.get(key) === pending) this.inFlight.delete(key); }; void pending.then(clearPending, clearPending); - return waitForShared(pending, signal); + return this.waitForCredential(pending, signal, repository, retried); } } From 67329888d66060b7e21416f539f6f0ddf56fff28 Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Sun, 27 Sep 2026 15:19:05 -0400 Subject: [PATCH 10/48] =?UTF-8?q?=E2=8F=B3=20fix:=20Let=20BYOM=20Workspace?= =?UTF-8?q?=20Calls=20Queue=20Past=20Thirty=20Seconds=20(#264)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ⏳ fix: Let BYOM Workspace Calls Queue Past Thirty Seconds * fix: Recompute BYOM Reservation TTLs After Admission --------- Co-authored-by: Lia --- docs/byom-worker-admission.md | 48 +++++++++------ docs/remote-bridge/README.md | 6 ++ packages/code/README.md | 10 +++- service/src/bridge/concurrent-store.test.ts | 45 ++++++++++++++ service/src/bridge/store.ts | 14 +++-- service/src/bridge/worker-admission.test.ts | 66 +++++++++++++++++++++ service/src/workspace-tools/router.test.ts | 41 +++++++++---- service/src/workspace-tools/router.ts | 28 +++++---- 8 files changed, 212 insertions(+), 46 deletions(-) diff --git a/docs/byom-worker-admission.md b/docs/byom-worker-admission.md index 2c2b354b..2fcd9f9f 100644 --- a/docs/byom-worker-admission.md +++ b/docs/byom-worker-admission.md @@ -5,14 +5,17 @@ The limit is 32 admitted requests per worker, including the active request. When the limit is reached, the workspace endpoint returns HTTP 429 with `WORKER_QUEUE_FULL`. A different worker has an independent admission queue. -The workspace HTTP endpoint allows at most 30 seconds for admission. After -admission and worker validation, a separate execution deadline starts. Commands -receive their requested timeout (30 seconds by default, up to five minutes), -capped by the operator's `JOB_TIMEOUT`, plus five seconds to settle the result. -Read/search/list operations receive up to 30 seconds, also capped by `JOB_TIMEOUT`. -Disconnecting or cancelling removes the waiting -request without cancelling the active assignment. Expired entries are pruned; -Redis key expiry also bounds state left by a crashed API process. +The Code API workspace HTTP endpoint allows up to the smaller of `JOB_TIMEOUT` +and five minutes for admission while the caller stays connected. After admission +and worker validation, a separate execution deadline starts. Commands receive +their requested timeout (30 seconds by default, up to five minutes), capped by +the operator's `JOB_TIMEOUT`, plus five seconds to settle the result. Other +operations receive up to 30 seconds, also capped by `JOB_TIMEOUT`. +Disconnecting or cancelling removes a waiting request without cancelling the +active assignment. Expired entries are pruned; Redis key expiry also bounds +state left by a crashed API process. Reservations derive their TTL at acquisition +from the remaining absolute deadline or a fresh execution budget; enqueued +assignment records use the final execution deadline, not the elapsed queue budget. After admission, the API revalidates the worker incarnation, identity, tenant binding and workspace operation. A waiting request cannot migrate to a replacement @@ -26,13 +29,24 @@ Existing workers still execute one assignment at a time. Parallel execution acro workspaces requires separate lease claims and isolated native sandbox contexts; this admission change does not advertise that capability. -LibreChat must allow queue time plus execution/settlement time and five seconds -for HTTP delivery: 65 seconds for reads, 70 seconds for default commands, and -340 seconds for five-minute commands. Either side can be upgraded first. Older -clients still cancel at their earlier deadline; newer clients preserve errors from -older servers without retrying mutations. Both updates are needed for the full -waiting budget. Any reverse proxy request timeout must accommodate these totals. -The worker package does not need an update for the deadline change. +Clients and reverse proxies must allow queue time plus execution/settlement time +and five seconds for HTTP delivery. With the default five-minute `JOB_TIMEOUT`, +that is at least 335 seconds for non-command tools, 340 seconds for default +commands, and 610 seconds for five-minute commands. With a smaller `JOB_TIMEOUT`, +use `min(JOB_TIMEOUT, 300s)` for the queue, plus `min(JOB_TIMEOUT, 30s)` for other +operations or `min(JOB_TIMEOUT, requested command timeout) + 5s` for commands, +plus five seconds for delivery. -Focused regression coverage lives in `service/src/bridge/admission.test.ts` and -`service/src/bridge/worker-admission.test.ts`. +At the time of this change, LibreChat's `getWorkspaceToolTimeoutMs` still budgets +only 30 seconds for a single admission attempt (65/70/340 seconds in total). +Its `maxQueueWaitMs` is a retry horizon after a typed capacity rejection, **not** +a per-attempt HTTP timeout. Updating Code API alone therefore does not guarantee +the full wait. An earlier client, tool, or proxy timeout disconnects the request; +if work was already admitted, a mutation may have run and must not be blindly +retried. Match LibreChat's per-attempt timeout and each intermediary to the new +budget before relying on it. Existing workers do not need an update. + +Focused regression coverage lives in `service/src/bridge/admission.test.ts`, +`service/src/bridge/worker-admission.test.ts`, +`service/src/bridge/concurrent-store.test.ts`, and +`service/src/workspace-tools/router.test.ts`. diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index efe84b84..c5bc573c 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -239,6 +239,12 @@ execution. worker. The lower API or worker slot ceiling wins, and assignments sharing the same workspace isolation key remain serialized while independent conversation worktrees may run concurrently. +- Workspace tool admission waits for capacity up to the smaller of `JOB_TIMEOUT` + and five minutes while the HTTP caller remains connected. Disconnects cancel + waiting, and admitted work receives a separate execution budget. A shorter + client or proxy timeout can end the wait sooner; Code API does not receive an + absolute caller deadline. See [BYOM worker admission](../byom-worker-admission.md) + for the caller and proxy timeout requirements. - Dynamic workers are fenced to their server-issued tenant before assignment. - Each assignment has an absolute deadline, generation, and random lease token. - Settlements with the wrong worker, generation, token, or expired deadline are diff --git a/packages/code/README.md b/packages/code/README.md index a87c87df..87a6fe61 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -791,11 +791,17 @@ Legacy requests without a conversation identity continue to use the selected source root. Older Code API deployments do not negotiate the capability, so the worker omits it until every request path understands the isolation boundary. -Admission waits at most 30 seconds. A `WORKSPACE_QUEUE_TIMEOUT` response (HTTP -503, `Retry-After: 1`) means the operation was not assigned or started; wait for +On an updated Code API, admission waits up to the smaller of `JOB_TIMEOUT` and +five minutes while the HTTP caller remains connected; older Code API versions +waited at most 30 seconds. A `WORKSPACE_QUEUE_TIMEOUT` response (HTTP 503, +`Retry-After: 1`) means the operation was not assigned or started; wait for capacity before submitting it again. This is distinct from `ASSIGNMENT_EXPIRED` or a transport timeout after dispatch, where execution may have occurred and mutations must not be blindly retried. No automatic retry is added by this policy. +Align the client's per-attempt timeout and any proxy with the queue **plus** +execution budget before relying on the longer wait. See the +[BYOM worker admission guide](../../docs/byom-worker-admission.md) for the +current client limitation and the timeout calculations. Keep the existing URL, pairing/identity, and network policy configuration. The primary root keeps its configured workspace ID (default `primary`). Repeat diff --git a/service/src/bridge/concurrent-store.test.ts b/service/src/bridge/concurrent-store.test.ts index 33f3d082..6bdfde88 100644 --- a/service/src/bridge/concurrent-store.test.ts +++ b/service/src/bridge/concurrent-store.test.ts @@ -357,6 +357,51 @@ test('same-root work waits while another root progresses', async () => { await Promise.all([nextA, b]); }); +test('a long same-root queue allowance does not extend slot or assignment TTLs', async () => { + await register(); + const active = dispatch('a'); + const activeAssignment = await store.lease( + workerId, incarnationId, 1000, undefined, undefined, 0, + ); + const controller = new AbortController(); + const queued = store.dispatchWorkspaceTool({ + workerId, + signal: controller.signal, + deadlineAtMs: Date.now() + 300_000, + executionTimeoutMs: 305_000, + request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'a', path: 'second.txt' }, + }); + void queued.catch(() => undefined); + await new Promise((resolve) => setTimeout(resolve, 150)); + await settle(activeAssignment!); + await active; + + const assignment = await store.lease( + workerId, incarnationId, 1000, undefined, undefined, 0, + ); + if (assignment == null) { + controller.abort(); + await queued.catch(() => undefined); + throw new Error('Queued request was not leased'); + } + expect(assignment.request).toMatchObject({ path: 'second.txt' }); + try { + const expiresAtMs = Date.parse(assignment.expiresAt); + const slotExpiresAtMs = Number(await redis.hget( + `codeapi:bridge:v1:worker:${workerId}:workspace-slots`, 'e:0', + )); + expect(slotExpiresAtMs).toBeGreaterThan(expiresAtMs + 28_000); + expect(slotExpiresAtMs).toBeLessThan(expiresAtMs + 32_000); + const assignmentTtlMs = await redis.pttl(`codeapi:bridge:v1:assignment:${assignment.assignmentId}`); + const remainingMs = expiresAtMs - Date.now(); + expect(assignmentTtlMs).toBeGreaterThan(remainingMs + 28_000); + expect(assignmentTtlMs).toBeLessThan(remainingMs + 32_000); + } finally { + await settle(assignment); + await queued; + } +}); + test('conversation worktrees on one source use independent capacity lanes', async () => { await register(); const firstId = 'a'.repeat(64); diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 2ffeec66..3d9d7e1c 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -863,10 +863,6 @@ export class RedisBridgeStore { const assignmentId = randomBytes(18).toString('base64url'); const leaseToken = randomBytes(32).toString('base64url'); - // The lock is acquired before admission finishes; it must outlive the later execution deadline. - const ttlSeconds = assignmentTtlSeconds( - args.deadlineAtMs + (args.executionTimeoutMs ?? 0), - ); const lockIncarnationId = registration.incarnationId; let assignment: StoredAssignment | undefined; let enqueueAttempted = false; @@ -931,6 +927,14 @@ export class RedisBridgeStore { ); continue; } + // A queued request may have waited nearly its full admission budget. + // Reserve for the *remaining* absolute deadline or a fresh execution + // budget, not the original queue window plus execution again. + const ttlSeconds = assignmentTtlSeconds( + args.executionTimeoutMs === undefined + ? args.deadlineAtMs + : Date.now() + args.executionTimeoutMs, + ); if (workspaceSlots != null) { workspaceLeaseSlot = await this.dispatchCommand( () => @@ -1070,7 +1074,7 @@ export class RedisBridgeStore { enqueueAttempted = true; return this.enqueueForActiveIncarnation( assignment!, - ttlSeconds, + assignmentTtlSeconds(Date.parse(assignment!.expiresAt)), readyToken, ); }, diff --git a/service/src/bridge/worker-admission.test.ts b/service/src/bridge/worker-admission.test.ts index 655748b1..98363c34 100644 --- a/service/src/bridge/worker-admission.test.ts +++ b/service/src/bridge/worker-admission.test.ts @@ -161,6 +161,72 @@ test('execution receives a fresh budget after waiting and the lock covers long c await second; }); +test('a long queue allowance does not extend serial lock or assignment TTLs after admission', async () => { + await register(); + const active = dispatch('first'); + const activeAssignment = await store.lease(workerId, incarnationId, 1000); + const controller = new AbortController(); + const queued = dispatch('second', controller, 300_000, 305_000); + void queued.catch(() => undefined); + await new Promise((resolve) => setTimeout(resolve, 150)); + await settle(activeAssignment); + await active; + + const assignment = await store.lease(workerId, incarnationId, 1000); + if (assignment == null) { + controller.abort(); + await queued.catch(() => undefined); + throw new Error('Queued request was not leased'); + } + expect(assignment.request).toMatchObject({ path: 'second' }); + try { + const expiresAtMs = Date.parse(assignment.expiresAt); + for (const key of [ + `codeapi:bridge:v1:worker:${workerId}:lock`, + `codeapi:bridge:v1:worker:${workerId}:lock:incarnation`, + `codeapi:bridge:v1:assignment:${assignment.assignmentId}`, + ]) { + const ttlMs = await redis.pttl(key); + const remainingMs = expiresAtMs - Date.now(); + expect(ttlMs).toBeGreaterThan(remainingMs + 28_000); + expect(ttlMs).toBeLessThan(remainingMs + 32_000); + } + } finally { + await settle(assignment); + await queued; + } +}); + +test('absolute-deadline callers keep only their remaining deadline in the serial lock TTL', async () => { + await register(); + const active = dispatch('first'); + const activeAssignment = await store.lease(workerId, incarnationId, 1000); + const controller = new AbortController(); + const queued = dispatch('absolute', controller, 2_500); + void queued.catch(() => undefined); + await new Promise((resolve) => setTimeout(resolve, 1450)); + await settle(activeAssignment); + await active; + + const assignment = await store.lease(workerId, incarnationId, 1000); + if (assignment == null) { + controller.abort(); + await queued.catch(() => undefined); + throw new Error('Queued request was not leased'); + } + expect(assignment.request).toMatchObject({ path: 'absolute' }); + try { + const remainingMs = Date.parse(assignment.expiresAt) - Date.now(); + expect(remainingMs).toBeGreaterThan(0); + const ttlMs = await redis.pttl(`codeapi:bridge:v1:worker:${workerId}:lock`); + expect(ttlMs).toBeGreaterThan(remainingMs + 28_000); + expect(ttlMs).toBeLessThan(remainingMs + 31_000); + } finally { + await settle(assignment); + await queued; + } +}); + test('execution expires independently of an unused queue allowance', async () => { await register(); const completion = dispatch('short', new AbortController(), 5000, 150); diff --git a/service/src/workspace-tools/router.test.ts b/service/src/workspace-tools/router.test.ts index a52b3f0b..18d24a76 100644 --- a/service/src/workspace-tools/router.test.ts +++ b/service/src/workspace-tools/router.test.ts @@ -67,13 +67,16 @@ test('binds instance admission to the authenticated tenant and user while preser expect(dispatched[1]?.workspaceInstanceId).toBeUndefined(); }); -test.each<[WorkspaceToolRequest, number, number?]>([ - [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, undefined], - [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 35_000, undefined], - [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready', timeoutMs: 300_000 }, 305_000, undefined], - [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 6000, 1000], - [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 35_000, 600_000], -])('separates the admission deadline from execution budget for %j', async (request, expectedExecution, ceiling) => { +test.each<[WorkspaceToolRequest, number, number?, number?]>([ + [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, undefined, undefined], + [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, 125_000, undefined], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 35_000, undefined, undefined], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready', timeoutMs: 90_000 }, 95_000, 125_000, undefined], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready', timeoutMs: 300_000 }, 305_000, undefined, undefined], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 6000, 1000, undefined], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 35_000, 600_000, undefined], + [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, 125_000, 5000], +])('separates the admission deadline from execution budget for %j', async (request, expectedExecution, ceiling, queueTimeoutMs) => { const app = express(); app.use(json()); app.use((req, _res, next) => { @@ -86,6 +89,7 @@ test.each<[WorkspaceToolRequest, number, number?]>([ app.use(createWorkspaceToolsRouter({ backend: 'remote-bridge', configuredWorkerId: 'user-worker', dynamicWorkers: false, timeoutMs: ceiling, + queueTimeoutMs, store: { async dispatchWorkspaceTool(args) { executionBudget = args.executionTimeoutMs; if (args.request.operation === 'execute_command') commandTimeout = args.request.timeoutMs; @@ -103,8 +107,19 @@ test.each<[WorkspaceToolRequest, number, number?]>([ await response.json(); expect(executionBudget).toBe(expectedExecution); if (request.operation === 'execute_command') expect(commandTimeout).toBe(expectedExecution - 5000); - expect(queueRemaining).toBeGreaterThan(29_000); - expect(queueRemaining).toBeLessThanOrEqual(30_000); + const expectedQueueBudget = queueTimeoutMs ?? Math.min(ceiling ?? 30_000, 300_000); + expect(queueRemaining).toBeGreaterThan(expectedQueueBudget - 1000); + expect(queueRemaining).toBeLessThanOrEqual(expectedQueueBudget); +}); + +test.each([0, 300_001, Number.POSITIVE_INFINITY])('rejects an unbounded queue override (%s)', (queueTimeoutMs) => { + expect(() => createWorkspaceToolsRouter({ + backend: 'remote-bridge', + configuredWorkerId: 'user-worker', + dynamicWorkers: false, + queueTimeoutMs, + store: { async dispatchWorkspaceTool() { throw new Error('must not dispatch'); } }, + })).toThrow('Workspace queue timeout must be between 1 and 300000 milliseconds'); }); test('rejects new workspace dispatches while the service is shutting down', async () => { @@ -404,7 +419,7 @@ test.each([ status: expectedStatus, errorCode, outcome: 'completed', - deadlineBudgetMs: 60_000, + deadlineBudgetMs: 330_000, dispatchDurationMs: expect.any(Number), }), ); @@ -486,6 +501,7 @@ test('logs a disconnected dispatch once without inventing HTTP 200', async () => const closed = Promise.withResolvers(); const settlementGate = Promise.withResolvers(); let dispatchAborted = false; + let queueRemaining: number | undefined; let closeConnection = (): void => { throw new Error('connection not ready'); }; app.use(json()); app.use((req, res, next) => { @@ -499,8 +515,10 @@ test('logs a disconnected dispatch once without inventing HTTP 200', async () => backend: 'remote-bridge', configuredWorkerId: 'user-worker', dynamicWorkers: true, + timeoutMs: 125_000, store: { - async dispatchWorkspaceTool({ signal }) { + async dispatchWorkspaceTool({ deadlineAtMs, signal }) { + queueRemaining = deadlineAtMs - Date.now(); started.resolve(); return await new Promise((_resolve, reject) => { signal.addEventListener( @@ -531,6 +549,7 @@ test('logs a disconnected dispatch once without inventing HTTP 200', async () => closeConnection(); await expect(response).rejects.toThrow(); await closed.promise; + expect(queueRemaining).toBeGreaterThan(120_000); expect(dispatchAborted).toBe(true); expect(logSpy).not.toHaveBeenCalled(); settlementGate.resolve(); diff --git a/service/src/workspace-tools/router.ts b/service/src/workspace-tools/router.ts index 3e1f9fee..51e47f26 100644 --- a/service/src/workspace-tools/router.ts +++ b/service/src/workspace-tools/router.ts @@ -20,6 +20,8 @@ import { } from '../bridge/selection'; import { principalWorkspaceInstanceId } from '../bridge/workspace-instance'; +const MAX_WORKSPACE_QUEUE_WAIT_MS = 5 * 60_000; + interface WorkspaceToolsRouterOptions { store: Pick; backend: 'http' | 'lambda-microvm' | 'remote-bridge'; @@ -50,15 +52,19 @@ export function bridgeStoreStatus(error: BridgeStoreError): number { } export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions): Router { - const queueBudgetMs = options.queueTimeoutMs ?? 30_000; - if (!Number.isSafeInteger(queueBudgetMs) || queueBudgetMs < 1 || queueBudgetMs > 30_000) { - throw new RangeError('Workspace queue timeout must be between 1 and 30000 milliseconds'); - } if (options.timeoutMs !== undefined && ( !Number.isSafeInteger(options.timeoutMs) || options.timeoutMs < 1 )) { throw new RangeError('Workspace execution timeout must be a positive safe integer'); } + // The HTTP disconnect cancels waiting; this bounds admission while the caller remains connected. + const queueBudgetMs = options.queueTimeoutMs ?? Math.min( + options.timeoutMs ?? 30_000, + MAX_WORKSPACE_QUEUE_WAIT_MS, + ); + if (!Number.isSafeInteger(queueBudgetMs) || queueBudgetMs < 1 || queueBudgetMs > MAX_WORKSPACE_QUEUE_WAIT_MS) { + throw new RangeError('Workspace queue timeout must be between 1 and 300000 milliseconds'); + } const router = Router(); router.post( @@ -86,13 +92,13 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) const principalRequest: WorkspaceToolRequest = req.body.workspaceInstanceId == null ? req.body : { - ...req.body, - workspaceInstanceId: principalWorkspaceInstanceId({ - instanceId: req.body.workspaceInstanceId, - tenantId: principal.tenantId, - principalId: principal.userId, - }), - }; + ...req.body, + workspaceInstanceId: principalWorkspaceInstanceId({ + instanceId: req.body.workspaceInstanceId, + tenantId: principal.tenantId, + principalId: principal.userId, + }), + }; const request: WorkspaceToolRequest = principalRequest.operation === 'execute_command' ? { ...principalRequest, timeoutMs: Math.min( principalRequest.timeoutMs ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, From 2a310027800a6a4936ae277ddf6cb0179d2b74b2 Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 07:54:34 -0400 Subject: [PATCH 11/48] =?UTF-8?q?=F0=9F=94=90=20feat:=20Recover=20Code=20A?= =?UTF-8?q?PI=20Machine=20Credentials=20After=20Outages=20(#260)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🔐 feat: Recover Code API Machine Credentials After Outages * 🔐 fix: Protect Recovery Budgets and Persist Compose Enrollment * 🛡️ fix: Limit Untrusted Code API Recovery Traffic --------- Co-authored-by: Lia --- docker-compose.yaml | 17 +- docs/adr/001-stateful-code-environments.md | 7 +- docs/remote-bridge/README.md | 71 ++- docs/remote-bridge/worker-runbook.md | 15 +- packages/code/package.json | 4 + packages/code/src/identity.test.ts | 43 ++ packages/code/src/identity.ts | 94 ++++ packages/code/src/protocol.ts | 26 ++ service/src/bridge/index.ts | 18 +- service/src/bridge/pairing.ts | 480 ++++++++++++++++++- service/src/bridge/recovery-router.test.ts | 213 +++++++++ service/src/bridge/recovery.test.ts | 518 +++++++++++++++++++++ service/src/bridge/router.ts | 109 ++++- service/src/config.ts | 16 + tests/compose-bridge-config.cjs | 24 +- 15 files changed, 1634 insertions(+), 21 deletions(-) create mode 100644 service/src/bridge/recovery-router.test.ts create mode 100644 service/src/bridge/recovery.test.ts diff --git a/docker-compose.yaml b/docker-compose.yaml index ab37fd24..1d7d28cf 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -15,6 +15,12 @@ services: - CODEAPI_BRIDGE_DYNAMIC_WORKERS=${CODEAPI_BRIDGE_DYNAMIC_WORKERS:-true} - CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS=${CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS:-1} - CODEAPI_BRIDGE_WORKER_ID=${CODEAPI_BRIDGE_WORKER_ID:-} + - CODEAPI_BRIDGE_RECOVERY_SERVER_ID=${CODEAPI_BRIDGE_RECOVERY_SERVER_ID:-} + - CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS=${CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS:-0} + - CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS=${CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS:-60} + - CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE:-12} + - CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE:-30} + - CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE:-240} - CODEAPI_AUTH_PROVIDER=${CODEAPI_AUTH_PROVIDER:-} - CODEAPI_ALLOW_AUTH_PROVIDER_NONE=${CODEAPI_ALLOW_AUTH_PROVIDER_NONE:-} - CODEAPI_JWT_ISSUER=${CODEAPI_JWT_ISSUER:-} @@ -64,6 +70,12 @@ services: - CODEAPI_BRIDGE_DYNAMIC_WORKERS=${CODEAPI_BRIDGE_DYNAMIC_WORKERS:-true} - CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS=${CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS:-1} - CODEAPI_BRIDGE_WORKER_ID=${CODEAPI_BRIDGE_WORKER_ID:-} + - CODEAPI_BRIDGE_RECOVERY_SERVER_ID=${CODEAPI_BRIDGE_RECOVERY_SERVER_ID:-} + - CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS=${CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS:-0} + - CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS=${CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS:-60} + - CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE:-12} + - CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE:-30} + - CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE:-240} - CODEAPI_AUTH_PROVIDER=${CODEAPI_AUTH_PROVIDER:-} - CODEAPI_JWT_SINGLE_TENANT_ID=${CODEAPI_JWT_SINGLE_TENANT_ID:-} - CODEAPI_TENANT_ISOLATION_STRICT=${CODEAPI_TENANT_ISOLATION_STRICT:-} @@ -213,9 +225,11 @@ services: redis: image: redis:7-alpine container_name: redis - command: redis-server --requirepass localdev + command: redis-server --requirepass localdev --appendonly yes ports: - ${CODEAPI_REDIS_PORT:-16379}:6379 + volumes: + - redis_data:/data minio: image: quay.io/minio/minio @@ -232,3 +246,4 @@ services: volumes: minio_data: + redis_data: diff --git a/docs/adr/001-stateful-code-environments.md b/docs/adr/001-stateful-code-environments.md index cfa44bf4..7992ded8 100644 --- a/docs/adr/001-stateful-code-environments.md +++ b/docs/adr/001-stateful-code-environments.md @@ -52,8 +52,11 @@ worker replacement; the UI and operator documentation must not imply otherwise. - The VM requires no inbound internet listener. - Code API, not the worker, authenticates LibreChat users and normalizes work. - A stolen short-lived credential is insufficient without the worker private - key; a stolen private key is insufficient after credential expiry or - revocation. + key. In the original pairing-only model, a stolen private key is insufficient + after credential expiry or revocation. With optional durable machine + enrollment and signed credential recovery, the private key itself remains + a revocable long-lived credential: access-credential expiry alone does not + protect against theft of that key. Revocation invalidates both. - Pairing codes and credentials are stored by digest where lookup permits. - One configured worker has at most one active fenced assignment. - Sandbox isolation and default-deny egress remain the mandatory default; diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index c5bc573c..96241ad7 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -31,6 +31,42 @@ CODEAPI_BRIDGE_TOKEN= CODEAPI_BRIDGE_AUTH_MODE=paired ``` +To opt in to durable machine authorization on every Code API replica, set a +single stable public **Code API** origin (not the LibreChat URL): + +```dotenv +CODEAPI_BRIDGE_RECOVERY_SERVER_ID=https://code.example.com +# 0 (default): enrolled machine keys remain authorized until revoked. +# CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS=0 +# CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS=60 +# CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE=12 +# CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE=30 +# CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE=240 +``` + +Omitting the server ID retains the existing pairing and refresh behavior and +hides the recovery routes. Deploy the compatible Code API version to **all** +replicas before setting this value and enrolling workers again. Older Code API +replicas can still pair or refresh a worker but do not write durable enrollment; +they must not serve device login or recovery requests. Only a pairing redeemed +after this option is enabled has a recoverable key. Updating Code API alone +does not make old workers reconnect automatically: the CLI must also implement +this recovery protocol in the later worker release. + +Store Redis state durably across restarts. The primary `docker-compose.yaml` +now uses Redis AOF and a named `/data` volume; preserve that volume when +recreating the stack. If upgrading a running stack with an in-memory Redis, +migrate its state before recreating the container: mounting an empty volume +does **not** preserve active assignments, fences, or earlier revocations. Other +deployments must provide equivalent durable Redis (for example, a managed +persistent Redis service and backups). Revocation and +machine enrollment share that state across replicas; do not configure eviction +of authorization keys. If enrollment state is missing, credentials minted under +that enrollment fail closed, and the worker must be explicitly enrolled again. +Restoring a backup from *before* a revocation can revive trust; reconcile +revocations after recovery from backup. Use a distinct server ID for each Code +API deployment and keep it stable when the endpoint changes behind a proxy. + Use `strict` instead of `affinity` if every request must include a runtime session hint. In hardened mode, startup requires the bridge token to be at least 32 bytes. `PTC_MODE=blocking` is rejected; replay mode is required because a @@ -226,7 +262,40 @@ execution. atomically on their first redemption attempt. - Worker credentials expire after fifteen minutes and are bound to an Ed25519 public key. Exact-request signatures include the HTTP method, path, body - digest, timestamp, nonce, and credential. + digest, timestamp, nonce, and credential. With recovery enabled, redeeming a + pairing also persists a separate machine authorization and its public key in + Redis without a TTL by default; an operator can instead set a bounded + enrollment lifetime. +- `POST /v1/bridge/workers/:workerId/credentials/challenge` does not require + an administrator token or an existing access credential, but **does** require + the enrolled key. Its JSON body contains `protocolVersion: 1`, + `operation: "credential.challenge"`, the configured `serverId`, the matching + `workerId`, a fresh UTC ISO `timestamp`, a random 32-byte base64url `nonce`, + and `signature` computed with `signBridgeRecoveryStart(privateKey, fields)` + from `@librechat/code/identity`. Code API verifies the signed fields and + consumes the nonce once before charging the machine's shared challenge + budget; a fabricated request cannot exhaust another worker's budget. +- The response is a short-lived, single-use challenge with the server ID, + worker ID, enrollment generation, operation and expiry. Sign those fields + with `signBridgeRecovery(privateKey, challenge)` and send the fields plus + `signature` to `POST .../credentials/recover` to obtain a new short-lived + credential. Invalid proofs are limited per high-entropy challenge; only + successfully verified, unused proofs consume the machine's shared recovery + budget. Separately, both recovery endpoints limit all incoming requests per + connection peer *before* key verification, including well-formed JSON with + malformed or forged proofs; forged headers and worker IDs cannot bypass + that limit or consume the signed machine budget. All limits live in shared + Redis; HTTP 429 means back off. When a reverse proxy connects to Code API, + its clients share that peer's limit. Restrict direct backend access and apply + client-IP and global + abuse limits at the trusted ingress to keep one proxy peer from becoming a + shared bottleneck; do not trust an arbitrary `X-Forwarded-For` on Code API. +- Recovery and revocation are atomic Redis transitions across API replicas. + A missing, revoked, expired or superseded enrollment never creates new + credentials. Recovery only restores transport authentication. It does not + clear assignment fences, worker or workspace quarantine, or uncertain + execution state. The worker private key is a durable, revocable credential; + expiry of an access credential alone does **not** protect against key theft. - Accepted proof nonces cannot be replayed, credentials rotate before expiry, and an administrator can revoke the active worker identity immediately. - Assignment leases bind to a stable paired identity rather than an individual diff --git a/docs/remote-bridge/worker-runbook.md b/docs/remote-bridge/worker-runbook.md index 493eb000..aeeef4c4 100644 --- a/docs/remote-bridge/worker-runbook.md +++ b/docs/remote-bridge/worker-runbook.md @@ -488,11 +488,16 @@ it still advertises named environments. ### Expired bridge credential -A running worker refreshes its short-lived credential automatically. If a -machine is offline long enough that refresh can no longer authenticate, issue -a fresh one-time pairing for the same worker ID and redeem it with a newly -generated keypair. Reusing the worker ID preserves the LibreChat environment -record and its agent assignments; creating a new ID creates a new environment. +A running worker refreshes its short-lived credential automatically. With +Code API durable enrollment enabled, a worker that still has its enrolled +private key can request a short-lived challenge and recover a new access +credential without manual re-pairing. The current CLI does **not** yet invoke +that endpoint automatically; update it when worker reconnect support ships. +Until then, or if enrollment is missing or revoked, use the one-time operator +pairing fallback. A new pairing replaces the Code API worker identity and may +require LibreChat environment reauthorization; reusing a worker ID alone does +not guarantee preservation of its LibreChat environment or agent assignments. +Never clear quarantine or workspace fences as part of credential recovery. ### Failed environment setup or uncertain mutation diff --git a/packages/code/package.json b/packages/code/package.json index b00ca807..d3cf714b 100644 --- a/packages/code/package.json +++ b/packages/code/package.json @@ -15,6 +15,10 @@ "types": "./dist/protocol.d.ts", "import": "./dist/protocol.js" }, + "./identity": { + "types": "./dist/identity.d.ts", + "import": "./dist/identity.js" + }, "./worker": { "types": "./dist/worker.d.ts", "import": "./dist/worker.js" diff --git a/packages/code/src/identity.test.ts b/packages/code/src/identity.test.ts index 4e92b91f..bef3b673 100644 --- a/packages/code/src/identity.test.ts +++ b/packages/code/src/identity.test.ts @@ -1,9 +1,14 @@ import assert from 'node:assert/strict'; +import { randomBytes } from 'node:crypto'; import test from 'node:test'; import { createBridgeIdentity, + signBridgeRecovery, + signBridgeRecoveryStart, signBridgeRequest, + verifyBridgeRecovery, + verifyBridgeRecoveryStart, verifyBridgeRequest, } from './identity.js'; @@ -33,3 +38,41 @@ test('worker identity proves possession for the exact HTTP request', () => { false, ); }); + +test('signed recovery starts bind operation, server, worker, time and nonce', () => { + const identity = createBridgeIdentity(); + const start = { + operation: 'credential.challenge' as const, + serverId: 'https://code.example.test', + workerId: 'vm-1', + timestamp: new Date().toISOString(), + nonce: randomBytes(32).toString('base64url'), + }; + const signature = signBridgeRecoveryStart(identity.privateKey, start); + assert.equal(verifyBridgeRecoveryStart(identity.publicKey, start, signature), true); + for (const modified of [ + { ...start, operation: 'credential.recover' as 'credential.challenge' }, + { ...start, serverId: 'https://other.example.test' }, + { ...start, workerId: 'vm-2' }, + { ...start, timestamp: new Date(Date.now() + 60_000).toISOString() }, + { ...start, nonce: randomBytes(32).toString('base64url') }, + ]) { + assert.equal(verifyBridgeRecoveryStart(identity.publicKey, modified, signature), false); + } + + const challenge = { + operation: 'credential.recover' as const, + serverId: start.serverId, + workerId: start.workerId, + enrollmentGeneration: randomBytes(18).toString('base64url'), + challenge: randomBytes(32).toString('base64url'), + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }; + assert.equal(verifyBridgeRecovery(identity.publicKey, challenge, signature), false); + assert.equal( + verifyBridgeRecoveryStart( + identity.publicKey, start, signBridgeRecovery(identity.privateKey, challenge), + ), + false, + ); +}); diff --git a/packages/code/src/identity.ts b/packages/code/src/identity.ts index ab11c865..3556520c 100644 --- a/packages/code/src/identity.ts +++ b/packages/code/src/identity.ts @@ -19,6 +19,25 @@ export interface BridgeRequestProofInput { body: string; } +/** Signed without an access credential before requesting a server recovery challenge. */ +export interface BridgeRecoveryStartProofInput { + operation: 'credential.challenge'; + serverId: string; + workerId: string; + timestamp: string; + nonce: string; +} + +/** Signed separately from access-credential requests, so neither proof can be reused for the other. */ +export interface BridgeRecoveryProofInput { + operation: 'credential.recover'; + serverId: string; + workerId: string; + enrollmentGeneration: string; + challenge: string; + expiresAt: string; +} + export function createBridgeIdentity(): BridgeIdentity { const { publicKey, privateKey } = generateKeyPairSync('ed25519', { publicKeyEncoding: { type: 'spki', format: 'pem' }, @@ -64,3 +83,78 @@ export function verifyBridgeRequest( return false; } } + +function canonicalBridgeRecoveryStart(input: BridgeRecoveryStartProofInput): string { + return [ + 'librechat-code:bridge-recovery-start:v1', + input.operation, + input.serverId, + input.workerId, + input.timestamp, + input.nonce, + ].join('\n'); +} + +export function signBridgeRecoveryStart( + privateKey: string, + input: BridgeRecoveryStartProofInput, +): string { + return sign(null, Buffer.from(canonicalBridgeRecoveryStart(input)), privateKey).toString( + 'base64url', + ); +} + +export function verifyBridgeRecoveryStart( + publicKey: string, + input: BridgeRecoveryStartProofInput, + signature: string, +): boolean { + try { + return verify( + null, + Buffer.from(canonicalBridgeRecoveryStart(input)), + publicKey, + Buffer.from(signature, 'base64url'), + ); + } catch { + return false; + } +} + +function canonicalBridgeRecovery(input: BridgeRecoveryProofInput): string { + return [ + 'librechat-code:bridge-recovery:v1', + input.operation, + input.serverId, + input.workerId, + input.enrollmentGeneration, + input.challenge, + input.expiresAt, + ].join('\n'); +} + +export function signBridgeRecovery( + privateKey: string, + input: BridgeRecoveryProofInput, +): string { + return sign(null, Buffer.from(canonicalBridgeRecovery(input)), privateKey).toString( + 'base64url', + ); +} + +export function verifyBridgeRecovery( + publicKey: string, + input: BridgeRecoveryProofInput, + signature: string, +): boolean { + try { + return verify( + null, + Buffer.from(canonicalBridgeRecovery(input)), + publicKey, + Buffer.from(signature, 'base64url'), + ); + } catch { + return false; + } +} diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index c678a1af..7657fc9c 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -759,6 +759,32 @@ export interface BridgeWorkerCredentialResponse { expiresAt: string; } +/** The enrolled machine signs this request before Code API issues a challenge. */ +export interface BridgeRecoveryChallengeRequest { + protocolVersion: BridgeProtocolVersion; + operation: 'credential.challenge'; + serverId: string; + workerId: string; + timestamp: string; + nonce: string; + signature: string; +} + +/** A short-lived, single-use challenge for an already enrolled machine key. */ +export interface BridgeRecoveryChallengeResponse { + protocolVersion: BridgeProtocolVersion; + operation: 'credential.recover'; + serverId: string; + workerId: string; + enrollmentGeneration: string; + challenge: string; + expiresAt: string; +} + +export interface BridgeRecoveryRequest extends BridgeRecoveryChallengeResponse { + signature: string; +} + export interface BridgeSandboxRequest { body: TBody; headers: Record; diff --git a/service/src/bridge/index.ts b/service/src/bridge/index.ts index 16857fad..b04de55f 100644 --- a/service/src/bridge/index.ts +++ b/service/src/bridge/index.ts @@ -11,7 +11,23 @@ export const bridgeStore = new RedisBridgeStore( undefined, env.BRIDGE_MAX_WORKSPACE_LEASE_SLOTS, ); -export const bridgePairings = new RedisBridgePairingStore(connection); +export const bridgePairings = new RedisBridgePairingStore( + connection, + undefined, + undefined, + undefined, + undefined, + env.BRIDGE_RECOVERY_SERVER_ID + ? { + serverId: env.BRIDGE_RECOVERY_SERVER_ID, + enrollmentTtlSeconds: env.BRIDGE_ENROLLMENT_TTL_SECONDS, + challengeTtlSeconds: env.BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS, + maxChallengesPerMinute: env.BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE, + maxAttemptsPerMinute: env.BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE, + maxUntrustedRequestsPerMinute: env.BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE, + } + : undefined, +); export default createBridgeRouter({ enabled: isBridgeEnabled(), diff --git a/service/src/bridge/pairing.ts b/service/src/bridge/pairing.ts index 9eb6d76c..b8a32162 100644 --- a/service/src/bridge/pairing.ts +++ b/service/src/bridge/pairing.ts @@ -6,13 +6,26 @@ import { import type Redis from 'ioredis'; -import { verifyBridgeRequest } from '../../../packages/code/src/identity'; +import { + verifyBridgeRecovery, + verifyBridgeRecoveryStart, + verifyBridgeRequest, +} from '../../../packages/code/src/identity'; +import type { + BridgeRecoveryProofInput, + BridgeRecoveryStartProofInput, +} from '../../../packages/code/src/identity'; const PREFIX = 'codeapi:bridge:v1'; const DEFAULT_PAIRING_TTL_SECONDS = 10 * 60; const DEFAULT_CREDENTIAL_TTL_SECONDS = 15 * 60; const PROOF_NONCE_TTL_SECONDS = 2 * 60; const PROOF_CLOCK_SKEW_MS = 60_000; +const DEFAULT_CHALLENGE_TTL_SECONDS = 60; +const DEFAULT_CHALLENGES_PER_MINUTE = 12; +const DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE = 30; +const DEFAULT_UNTRUSTED_RECOVERY_REQUESTS_PER_MINUTE = 240; +const RECOVERY_START_NONCE_TTL_SECONDS = 3 * 60; const LEGACY_SCAN_CLAIM_TTL_MS = 5_000; const LEGACY_SCAN_POLL_INTERVAL_MS = 25; const LEGACY_SCAN_PENDING = 'pending'; @@ -41,6 +54,7 @@ elseif (generation or '0') ~= ARGV[2] then redis.call('DEL', KEYS[1]) return 0 end +if ARGV[7] ~= '' and (generation or '0') ~= ARGV[9] then return 0 end redis.call('DEL', KEYS[1]) if redis.call('GET', KEYS[5]) == KEYS[1] then redis.call('DEL', KEYS[5]) @@ -48,9 +62,26 @@ end redis.call('SET', KEYS[3], ARGV[3], 'EX', ARGV[4]) redis.call('SET', KEYS[4], ARGV[5], 'EX', ARGV[4]) redis.call('SET', KEYS[6], ARGV[6], 'EX', ARGV[4]) +if ARGV[7] ~= '' then + if tonumber(ARGV[8]) > 0 then + redis.call('SET', KEYS[7], ARGV[7], 'EX', ARGV[8]) + else + redis.call('SET', KEYS[7], ARGV[7]) + end + redis.call('SET', KEYS[8], ARGV[10]) +else + redis.call('DEL', KEYS[7], KEYS[8]) +end return 1 `; const ROTATE_CREDENTIAL_SCRIPT = ` +if ARGV[6] ~= '' then + if redis.call('GET', KEYS[5]) ~= ARGV[6] or (redis.call('GET', KEYS[6]) or '0') ~= ARGV[7] or redis.call('GET', KEYS[7]) ~= ARGV[8] then + return 0 + end +elseif redis.call('EXISTS', KEYS[7]) == 1 then + return 0 +end local activeDigest = redis.call('GET', KEYS[1]) local previous = redis.call('GET', KEYS[2]) if not activeDigest or not previous then @@ -82,12 +113,45 @@ if credential then redis.call('DEL', KEYS[3]) redis.call('DEL', ARGV[1] .. credential) end -redis.call('DEL', KEYS[1], KEYS[3], KEYS[4], KEYS[5], KEYS[6], KEYS[7]) +redis.call('DEL', KEYS[1], KEYS[3], KEYS[4], KEYS[5], KEYS[6], KEYS[7], KEYS[8]) if activeIncarnation then redis.call('SET', ARGV[2] .. activeIncarnation .. ':fenced', '1') end return 1 `; +const CREATE_RECOVERY_CHALLENGE_SCRIPT = ` +if redis.call('GET', KEYS[1]) ~= ARGV[1] or (redis.call('GET', KEYS[4]) or '0') ~= ARGV[4] or redis.call('GET', KEYS[5]) ~= ARGV[6] then + return 0 +end +if redis.call('EXISTS', KEYS[6]) == 1 then return -2 end +local count = redis.call('INCR', KEYS[2]) +if count == 1 then redis.call('EXPIRE', KEYS[2], 60) end +if count > tonumber(ARGV[2]) then return -1 end +if redis.call('SET', KEYS[3], ARGV[3], 'EX', ARGV[5], 'NX') ~= 'OK' then return 0 end +redis.call('SET', KEYS[6], '1', 'EX', ARGV[7]) +return 1 +`; +const LIMIT_RECOVERY_ATTEMPTS_SCRIPT = ` +local count = redis.call('INCR', KEYS[1]) +if count == 1 then redis.call('EXPIRE', KEYS[1], 60) end +if count > tonumber(ARGV[1]) then return 0 end +return 1 +`; +const COMPLETE_RECOVERY_CHALLENGE_SCRIPT = ` +if redis.call('GET', KEYS[1]) ~= ARGV[1] or redis.call('GET', KEYS[2]) ~= ARGV[2] or (redis.call('GET', KEYS[6]) or '0') ~= ARGV[3] or redis.call('GET', KEYS[7]) ~= ARGV[8] then + return 0 +end +local stableIdentity = redis.call('GET', KEYS[5]) +if stableIdentity and stableIdentity ~= ARGV[4] then return 0 end +local count = redis.call('INCR', KEYS[8]) +if count == 1 then redis.call('EXPIRE', KEYS[8], 60) end +if count > tonumber(ARGV[9]) then return -1 end +redis.call('DEL', KEYS[2]) +redis.call('SET', KEYS[3], ARGV[5], 'EX', ARGV[6]) +redis.call('SET', KEYS[4], ARGV[7], 'EX', ARGV[6]) +redis.call('SET', KEYS[5], ARGV[4], 'EX', ARGV[6]) +return 1 +`; const RELEASE_LEGACY_SCAN_CLAIM_SCRIPT = ` if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) @@ -147,8 +211,34 @@ interface StoredCredential { publicKey: string; expiresAt: string; binding?: BridgeWorkerBinding; + /** Present only for credentials backed by durable machine enrollment. */ + enrollmentGeneration?: string; +} + +interface StoredEnrollment { + workerId: string; + serverId: string; + identityId: string; + generation: string; + pairingGeneration: number; + publicKey: string; + binding?: BridgeWorkerBinding; +} + +export interface BridgeRecoveryOptions { + /** Stable HTTPS origin of this Code API deployment, identical on every replica. */ + serverId: string; + /** Zero (the default) keeps enrollment until explicit revocation. */ + enrollmentTtlSeconds?: number; + challengeTtlSeconds?: number; + maxChallengesPerMinute?: number; + maxAttemptsPerMinute?: number; + /** Shared per-connection-peer cap, separate from machine-signed budgets. */ + maxUntrustedRequestsPerMinute?: number; } +export type BridgeRecoveryChallenge = BridgeRecoveryProofInput; + export interface BridgePairing { workerId: string; code: string; @@ -168,7 +258,10 @@ export class BridgePairingError extends Error { | 'PUBLIC_KEY_INVALID' | 'CREDENTIAL_INVALID' | 'PROOF_INVALID' - | 'PROOF_REPLAYED', + | 'PROOF_REPLAYED' + | 'ENROLLMENT_INVALID' + | 'CHALLENGE_INVALID' + | 'RECOVERY_RATE_LIMITED', message: string, ) { super(message); @@ -196,6 +289,39 @@ function workerStableIdentityKey(workerId: string): string { return `${PREFIX}:stable-identity:${workerId}`; } +function workerEnrollmentKey(workerId: string): string { + return `${PREFIX}:enrollment:${workerId}`; +} + +function workerEnrollmentRequiredKey(workerId: string): string { + return `${PREFIX}:enrollment-required:${workerId}`; +} + +function recoveryChallengeKey(challenge: string): string { + return `${PREFIX}:recovery:challenge:${digest(challenge)}`; +} + +function recoveryStartNonceKey(workerId: string, nonce: string): string { + return `${PREFIX}:recovery:nonce:${workerId}:${digest(nonce)}`; +} + +function recoveryChallengeAttemptKey(challenge: string): string { + return `${PREFIX}:recovery:attempt:${digest(challenge)}`; +} + +function untrustedRecoveryRateKey(peer: string, operation: 'challenge' | 'recover'): string { + // Never partition by a caller-supplied worker ID or an untrusted forwarded IP. + return `${PREFIX}:recovery:untrusted:${operation}:${digest(peer)}`; +} + +function recoveryRateKey( + workerId: string, + enrollmentGeneration: string, + operation: 'start' | 'complete', +): string { + return `${PREFIX}:recovery:rate:${operation}:${workerId}:${enrollmentGeneration}`; +} + function workerPairingGenerationKey(workerId: string): string { return `${PREFIX}:pairing-generation:${workerId}`; } @@ -236,8 +362,120 @@ export class RedisBridgePairingStore { private readonly credentialTtlSeconds = DEFAULT_CREDENTIAL_TTL_SECONDS, private readonly legacyScanClaimTtlMs = LEGACY_SCAN_CLAIM_TTL_MS, private readonly rollbackEpoch = - process.env.CODEAPI_BRIDGE_PAIRING_ROLLBACK_EPOCH?.trim() ?? '', - ) {} + process.env.CODEAPI_BRIDGE_PAIRING_ROLLBACK_EPOCH?.trim() ?? '', + private readonly recovery?: BridgeRecoveryOptions, + ) { + if (recovery == null) return; + let server: URL; + try { + server = new URL(recovery.serverId); + } catch { + throw new Error('Bridge recovery requires a stable HTTPS server origin'); + } + if ( + server.protocol !== 'https:' || + server.username !== '' || + server.password !== '' || + server.pathname !== '/' || + server.search !== '' || + server.hash !== '' || + server.origin !== recovery.serverId + ) { + throw new Error('Bridge recovery requires a stable HTTPS server origin'); + } + for (const [value, max] of [ + [recovery.enrollmentTtlSeconds ?? 0, 10 * 365 * 24 * 3600], + [recovery.challengeTtlSeconds ?? DEFAULT_CHALLENGE_TTL_SECONDS, 300], + [recovery.maxChallengesPerMinute ?? DEFAULT_CHALLENGES_PER_MINUTE, 120], + [recovery.maxAttemptsPerMinute ?? DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE, 120], + [recovery.maxUntrustedRequestsPerMinute ?? DEFAULT_UNTRUSTED_RECOVERY_REQUESTS_PER_MINUTE, 1200], + ]) { + if (!Number.isSafeInteger(value) || value < 0 || value > max) { + throw new RangeError('Invalid bridge recovery lifetime or rate limit'); + } + } + if ( + (recovery.challengeTtlSeconds ?? DEFAULT_CHALLENGE_TTL_SECONDS) === 0 || + (recovery.maxChallengesPerMinute ?? DEFAULT_CHALLENGES_PER_MINUTE) === 0 || + (recovery.maxAttemptsPerMinute ?? DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE) === 0 || + (recovery.maxUntrustedRequestsPerMinute ?? DEFAULT_UNTRUSTED_RECOVERY_REQUESTS_PER_MINUTE) === 0 + ) { + throw new RangeError('Bridge recovery challenge lifetime and rate limits must be positive'); + } + } + + get recoveryEnabled(): boolean { + return this.recovery != null; + } + + async limitUntrustedRecovery( + peer: string, + operation: 'challenge' | 'recover', + ): Promise { + if (this.recovery == null) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Machine recovery is disabled'); + } + const allowed = await this.redis.eval( + LIMIT_RECOVERY_ATTEMPTS_SCRIPT, + 1, + untrustedRecoveryRateKey(peer, operation), + String(this.recovery.maxUntrustedRequestsPerMinute ?? DEFAULT_UNTRUSTED_RECOVERY_REQUESTS_PER_MINUTE), + ); + if (allowed !== 1) { + throw new BridgePairingError('RECOVERY_RATE_LIMITED', 'Too many recovery requests from this peer'); + } + } + + private checkedEnrollment( + workerId: string, + raw: string | null, + pairingGeneration: string | null, + requiredGeneration: string | null, + ): StoredEnrollment { + let parsed: unknown; + try { + parsed = raw == null ? undefined : JSON.parse(raw); + } catch { + parsed = undefined; + } + const enrollment = ( + typeof parsed === 'object' && parsed != null && !Array.isArray(parsed) + ? parsed + : {} + ) as Partial; + if ( + this.recovery == null || + enrollment.workerId !== workerId || + enrollment.serverId !== this.recovery.serverId || + typeof enrollment.identityId !== 'string' || + !/^[A-Za-z0-9_-]{24}$/.test(enrollment.identityId) || + typeof enrollment.generation !== 'string' || + !/^[A-Za-z0-9_-]{24}$/.test(enrollment.generation) || + enrollment.generation !== requiredGeneration || + typeof enrollment.publicKey !== 'string' || + !validEd25519PublicKey(enrollment.publicKey) || + !Number.isSafeInteger(enrollment.pairingGeneration) || + String(enrollment.pairingGeneration) !== (pairingGeneration ?? '0') + ) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Machine enrollment is unavailable or revoked'); + } + return enrollment as StoredEnrollment; + } + + private checkEnrolledCredential( + credential: StoredCredential, + enrollment: StoredEnrollment, + ): void { + if ( + credential.workerId !== enrollment.workerId || + credential.identityId !== enrollment.identityId || + credential.publicKey !== enrollment.publicKey || + credential.enrollmentGeneration !== enrollment.generation || + JSON.stringify(credential.binding ?? null) !== JSON.stringify(enrollment.binding ?? null) + ) { + throw new BridgePairingError('CREDENTIAL_INVALID', 'Worker credential does not match its machine enrollment'); + } + } async issue( workerId: string, @@ -314,28 +552,49 @@ export class RedisBridgePairingStore { Date.now() + this.credentialTtlSeconds * 1000, ).toISOString(); const identityId = randomBytes(18).toString('base64url'); + const pairingGeneration = pairing.generation ?? Number( + (await this.redis.get(workerPairingGenerationKey(args.workerId))) ?? '0', + ); + const enrollment: StoredEnrollment | undefined = this.recovery == null + ? undefined + : { + workerId: args.workerId, + serverId: this.recovery.serverId, + identityId, + generation: randomBytes(18).toString('base64url'), + pairingGeneration, + publicKey: args.publicKey, + binding: pairing.binding, + }; const stored: StoredCredential = { workerId: args.workerId, identityId, publicKey: args.publicKey, expiresAt, binding: pairing.binding, + ...(enrollment == null ? {} : { enrollmentGeneration: enrollment.generation }), }; const accepted = await this.redis.eval( REDEEM_PAIRING_SCRIPT, - 6, + 8, codeKey, workerPairingGenerationKey(pairing.workerId), credentialDigestKey(credentialDigest), workerIdentityKey(args.workerId), workerPairingIndexKey(args.workerId), workerStableIdentityKey(args.workerId), + workerEnrollmentKey(args.workerId), + workerEnrollmentRequiredKey(args.workerId), raw, pairing.generation == null ? '' : String(pairing.generation), JSON.stringify(stored), String(this.credentialTtlSeconds), credentialDigest, identityId, + enrollment == null ? '' : JSON.stringify(enrollment), + String(this.recovery?.enrollmentTtlSeconds ?? 0), + String(pairingGeneration), + enrollment?.generation ?? '', ); if (accepted !== 1) { throw new BridgePairingError( @@ -374,10 +633,12 @@ export class RedisBridgePairingStore { ); } const credentialDigest = digest(args.credential); - const [raw, activeDigest, pairingGeneration] = await this.redis.mget( + const [raw, activeDigest, pairingGeneration, enrollmentRaw, requiredGeneration] = await this.redis.mget( credentialDigestKey(credentialDigest), workerIdentityKey(args.workerId), workerPairingGenerationKey(args.workerId), + workerEnrollmentKey(args.workerId), + workerEnrollmentRequiredKey(args.workerId), ); if (raw == null || activeDigest == null) { throw new BridgePairingError( @@ -386,6 +647,12 @@ export class RedisBridgePairingStore { ); } const stored = JSON.parse(raw) as StoredCredential; + if (stored.enrollmentGeneration != null || requiredGeneration != null || enrollmentRaw != null) { + this.checkEnrolledCredential( + stored, + this.checkedEnrollment(args.workerId, enrollmentRaw, pairingGeneration, requiredGeneration), + ); + } if (activeDigest !== credentialDigest) { const activeRaw = await this.redis.get( credentialDigestKey(activeDigest), @@ -439,6 +706,177 @@ export class RedisBridgePairingStore { }; } + async createRecoveryChallenge( + workerId: string, + request: BridgeRecoveryStartProofInput, + signature: string, + ): Promise { + if (this.recovery == null) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Machine recovery is disabled'); + } + const [raw, pairingGeneration, requiredGeneration] = await this.redis.mget( + workerEnrollmentKey(workerId), + workerPairingGenerationKey(workerId), + workerEnrollmentRequiredKey(workerId), + ); + const enrollment = this.checkedEnrollment(workerId, raw, pairingGeneration, requiredGeneration); + const proofTime = Date.parse(request.timestamp); + if ( + String(request.operation) !== 'credential.challenge' || + request.serverId !== enrollment.serverId || + request.workerId !== workerId || + !/^[A-Za-z0-9_-]{43}$/.test(request.nonce) || + !Number.isFinite(proofTime) || + Math.abs(Date.now() - proofTime) > PROOF_CLOCK_SKEW_MS || + !verifyBridgeRecoveryStart(enrollment.publicKey, request, signature) + ) { + throw new BridgePairingError('PROOF_INVALID', 'Machine recovery challenge proof is invalid'); + } + const challenge: BridgeRecoveryChallenge = { + operation: 'credential.recover', + serverId: enrollment.serverId, + workerId, + enrollmentGeneration: enrollment.generation, + challenge: randomBytes(32).toString('base64url'), + expiresAt: new Date( + Date.now() + (this.recovery.challengeTtlSeconds ?? DEFAULT_CHALLENGE_TTL_SECONDS) * 1000, + ).toISOString(), + }; + const created = await this.redis.eval( + CREATE_RECOVERY_CHALLENGE_SCRIPT, + 6, + workerEnrollmentKey(workerId), + recoveryRateKey(workerId, enrollment.generation, 'start'), + recoveryChallengeKey(challenge.challenge), + workerPairingGenerationKey(workerId), + workerEnrollmentRequiredKey(workerId), + recoveryStartNonceKey(workerId, request.nonce), + raw!, + String(this.recovery.maxChallengesPerMinute ?? DEFAULT_CHALLENGES_PER_MINUTE), + JSON.stringify(challenge), + String(enrollment.pairingGeneration), + String(this.recovery.challengeTtlSeconds ?? DEFAULT_CHALLENGE_TTL_SECONDS), + enrollment.generation, + String(RECOVERY_START_NONCE_TTL_SECONDS), + ); + if (created === -2) { + throw new BridgePairingError('PROOF_REPLAYED', 'Machine recovery challenge proof was already used'); + } + if (created === -1) { + throw new BridgePairingError('RECOVERY_RATE_LIMITED', 'Too many machine recovery challenges'); + } + if (created !== 1) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Machine enrollment is unavailable or revoked'); + } + return challenge; + } + + async recoverCredential( + workerId: string, + proof: BridgeRecoveryChallenge, + signature: string, + ): Promise { + if (this.recovery == null) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Machine recovery is disabled'); + } + const challengeKey = recoveryChallengeKey(proof.challenge); + const [enrollmentRaw, challengeRaw, pairingGeneration, requiredGeneration] = await this.redis.mget( + workerEnrollmentKey(workerId), + challengeKey, + workerPairingGenerationKey(workerId), + workerEnrollmentRequiredKey(workerId), + ); + const enrollment = this.checkedEnrollment( + workerId, enrollmentRaw, pairingGeneration, requiredGeneration, + ); + let parsed: unknown; + try { + parsed = challengeRaw == null ? undefined : JSON.parse(challengeRaw); + } catch { + parsed = undefined; + } + const saved = ( + typeof parsed === 'object' && parsed != null && !Array.isArray(parsed) + ? parsed + : {} + ) as Partial; + if ( + saved.operation !== 'credential.recover' || + saved.serverId !== enrollment.serverId || + saved.workerId !== workerId || + saved.enrollmentGeneration !== enrollment.generation || + saved.challenge !== proof.challenge || + saved.expiresAt !== proof.expiresAt || + String(proof.operation) !== saved.operation || + saved.serverId !== proof.serverId || + saved.workerId !== proof.workerId || + saved.enrollmentGeneration !== proof.enrollmentGeneration || + typeof saved.expiresAt !== 'string' || + !Number.isFinite(Date.parse(saved.expiresAt)) || + Date.parse(saved.expiresAt) <= Date.now() + ) { + throw new BridgePairingError('CHALLENGE_INVALID', 'Machine recovery challenge is invalid or expired'); + } + // Invalid signatures can only exhaust the high-entropy challenge they know, + // never the enrolled worker's shared quota. The worker-wide counter is + // charged inside completion, after the key proof and replay checks. + const attempts = await this.redis.eval( + LIMIT_RECOVERY_ATTEMPTS_SCRIPT, + 1, + recoveryChallengeAttemptKey(proof.challenge), + String(this.recovery.maxAttemptsPerMinute ?? DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE), + ); + if (attempts !== 1) { + throw new BridgePairingError('RECOVERY_RATE_LIMITED', 'Too many machine recovery attempts'); + } + if (!verifyBridgeRecovery(enrollment.publicKey, saved as BridgeRecoveryChallenge, signature)) { + throw new BridgePairingError('PROOF_INVALID', 'Machine recovery signature is invalid'); + } + + const credential = randomBytes(32).toString('base64url'); + const credentialDigest = digest(credential); + const expiresAt = new Date(Date.now() + this.credentialTtlSeconds * 1000).toISOString(); + const stored: StoredCredential = { + workerId, + identityId: enrollment.identityId, + publicKey: enrollment.publicKey, + expiresAt, + binding: enrollment.binding, + enrollmentGeneration: enrollment.generation, + }; + // The same Redis decision consumes the proof and issues the credential. + // A revoke or replacement on another replica wins by invalidating the + // enrollment/generation comparison, with no window to recreate trust. + const issued = await this.redis.eval( + COMPLETE_RECOVERY_CHALLENGE_SCRIPT, + 8, + workerEnrollmentKey(workerId), + challengeKey, + credentialDigestKey(credentialDigest), + workerIdentityKey(workerId), + workerStableIdentityKey(workerId), + workerPairingGenerationKey(workerId), + workerEnrollmentRequiredKey(workerId), + recoveryRateKey(workerId, enrollment.generation, 'complete'), + enrollmentRaw!, + challengeRaw!, + String(enrollment.pairingGeneration), + enrollment.identityId, + JSON.stringify(stored), + String(this.credentialTtlSeconds), + credentialDigest, + enrollment.generation, + String(this.recovery.maxAttemptsPerMinute ?? DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE), + ); + if (issued === -1) { + throw new BridgePairingError('RECOVERY_RATE_LIMITED', 'Too many machine recoveries'); + } + if (issued !== 1) { + throw new BridgePairingError('CHALLENGE_INVALID', 'Machine recovery challenge is invalid or expired'); + } + return { workerId, credential, expiresAt }; + } + async revoke(workerId: string): Promise { await this.removeLegacyPairings(workerId); // Fence redemption and consume the currently indexed code atomically. An @@ -446,7 +884,7 @@ export class RedisBridgePairingStore { // that linearizes afterward installs a distinct generation and code. await this.redis.eval( REVOKE_PAIRING_SCRIPT, - 7, + 8, workerPairingIndexKey(workerId), workerPairingGenerationKey(workerId), workerIdentityKey(workerId), @@ -454,6 +892,7 @@ export class RedisBridgePairingStore { `${PREFIX}:worker:${encodeURIComponent(workerId)}`, `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation`, `${PREFIX}:worker:${encodeURIComponent(workerId)}:ready`, + workerEnrollmentKey(workerId), `${PREFIX}:credential:`, `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation:`, ); @@ -652,12 +1091,26 @@ export class RedisBridgePairingStore { ); } const previous = JSON.parse(previousRaw) as StoredCredential; + let enrollment: StoredEnrollment | undefined; + let enrollmentRaw: string | null = null; + const [raw, generation, requiredGeneration] = await this.redis.mget( + workerEnrollmentKey(workerId), + workerPairingGenerationKey(workerId), + workerEnrollmentRequiredKey(workerId), + ); + if (previous.enrollmentGeneration != null || requiredGeneration != null || raw != null) { + enrollment = this.checkedEnrollment(workerId, raw, generation, requiredGeneration); + this.checkEnrolledCredential(previous, enrollment); + enrollmentRaw = raw; + } return await this.issueCredential( workerId, previous.publicKey, previousDigest, previous.binding, previous.identityId ?? null, + enrollment, + enrollmentRaw, ); } @@ -667,6 +1120,8 @@ export class RedisBridgePairingStore { previousDigest?: string, binding?: BridgeWorkerBinding, identityId?: string | null, + enrollment?: StoredEnrollment, + enrollmentRaw?: string | null, ): Promise { const credential = randomBytes(32).toString('base64url'); const credentialDigest = digest(credential); @@ -683,20 +1138,27 @@ export class RedisBridgePairingStore { publicKey, expiresAt, binding, + ...(enrollment == null ? {} : { enrollmentGeneration: enrollment.generation }), }; if (previousDigest !== undefined) { const rotated = await this.redis.eval( ROTATE_CREDENTIAL_SCRIPT, - 4, + 7, workerIdentityKey(workerId), credentialDigestKey(previousDigest), credentialDigestKey(credentialDigest), workerStableIdentityKey(workerId), + workerEnrollmentKey(workerId), + workerPairingGenerationKey(workerId), + workerEnrollmentRequiredKey(workerId), previousDigest, credentialDigest, JSON.stringify(stored), String(this.credentialTtlSeconds), stableIdentityId ?? '', + enrollmentRaw ?? '', + String(enrollment?.pairingGeneration ?? ''), + enrollment?.generation ?? '', ); if (rotated !== 1) { throw new BridgePairingError( diff --git a/service/src/bridge/recovery-router.test.ts b/service/src/bridge/recovery-router.test.ts new file mode 100644 index 00000000..28ea4b1e --- /dev/null +++ b/service/src/bridge/recovery-router.test.ts @@ -0,0 +1,213 @@ +import { randomBytes } from 'crypto'; +import { createServer, type Server } from 'http'; + +import { afterEach, describe, expect, test } from 'bun:test'; +import express, { json } from 'express'; +import RedisMock from 'ioredis-mock'; + +import type Redis from 'ioredis'; +import type { BridgeRecoveryChallengeRequest, BridgeRecoveryChallengeResponse } from '../../../packages/code/src/protocol'; + +import { + createBridgeIdentity, + signBridgeRecovery, + signBridgeRecoveryStart, +} from '../../../packages/code/src/identity'; +import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; +import { RedisBridgePairingStore } from './pairing'; +import { createBridgeRouter } from './router'; +import { RedisBridgeStore } from './store'; + +const redis = new RedisMock() as unknown as Redis; +const workerId = 'http-recovery-worker'; +const serverId = 'https://code.example.test'; +const servers: Server[] = []; + +function signedStart(privateKey: string): BridgeRecoveryChallengeRequest { + const request = { + operation: 'credential.challenge' as const, + serverId, + workerId, + timestamp: new Date().toISOString(), + nonce: randomBytes(32).toString('base64url'), + }; + return { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + ...request, + signature: signBridgeRecoveryStart(privateKey, request), + }; +} + +async function startRouter(pairings: RedisBridgePairingStore): Promise { + const app = express(); + app.set('trust proxy', 1); + app.use(json()); + app.use('/v1/bridge', createBridgeRouter({ + store: new RedisBridgeStore(redis), + pairings, + authMode: 'paired', + adminToken: 'operator-only', + configuredWorkerId: workerId, + })); + const server = createServer(app); + servers.push(server); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Expected TCP listener'); + return `http://127.0.0.1:${address.port}/v1/bridge/workers/${workerId}/credentials`; +} + +function post( + url: string, + body: object, + headers: Record = {}, +): Promise { + return fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json', ...headers }, + body: JSON.stringify(body), + }); +} + +afterEach(async () => { + await Promise.all(servers.splice(0).map(server => new Promise((resolve, reject) => { + server.close(error => error ? reject(error) : resolve()); + }))); + await redis.flushall(); +}); + +describe('machine credential recovery HTTP API', () => { + test('does not expose recovery until the server identity is enabled', async () => { + const baseUrl = await startRouter(new RedisBridgePairingStore(redis)); + const response = await post(`${baseUrl}/challenge`, { protocolVersion: BRIDGE_PROTOCOL_VERSION }); + expect(response.status).toBe(404); + }); + + test('recovers using the stored key without administrator authentication and never leaks the binding', async () => { + const pairings = new RedisBridgePairingStore(redis, 600, 300, 5_000, '', { + serverId, + }); + const identity = createBridgeIdentity(); + const pairing = await pairings.issue(workerId, { + tenantId: 'tenant-one', principal: { type: 'user', id: 'owner-one' }, + }); + await pairings.redeem({ workerId, code: pairing.code, publicKey: identity.publicKey }); + const baseUrl = await startRouter(pairings); + const unsigned = await post(`${baseUrl}/challenge`, { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + }); + expect(unsigned.status).toBe(400); + const outsider = createBridgeIdentity(); + const forged = await post(`${baseUrl}/challenge`, signedStart(outsider.privateKey)); + expect(forged.status).toBe(401); + await expect(forged.json()).resolves.toMatchObject({ code: 'PROOF_INVALID' }); + + const challengeResponse = await post(`${baseUrl}/challenge`, signedStart(identity.privateKey)); + expect(challengeResponse.status).toBe(200); + const challenge = (await challengeResponse.json()) as BridgeRecoveryChallengeResponse; + expect(challenge).toMatchObject({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + serverId, + workerId, + operation: 'credential.recover', + }); + expect(JSON.stringify(challenge)).not.toMatch(/tenant-one|owner-one|privateKey/); + + const signature = signBridgeRecovery(identity.privateKey, challenge); + const rejected = await post(`${baseUrl}/recover`, { + ...challenge, + serverId: 'https://unrelated.example.test', + signature, + }); + expect(rejected.status).toBe(401); + await expect(rejected.json()).resolves.toMatchObject({ code: 'CHALLENGE_INVALID' }); + + const recovered = await post(`${baseUrl}/recover`, { ...challenge, signature }); + expect(recovered.status).toBe(200); + const credential = (await recovered.json()) as { workerId: string; credential: string }; + expect(credential.workerId).toBe(workerId); + expect(credential.credential.length).toBeGreaterThanOrEqual(32); + const replay = await post(`${baseUrl}/recover`, { ...challenge, signature }); + expect(replay.status).toBe(401); + await expect(replay.json()).resolves.toMatchObject({ code: 'CHALLENGE_INVALID' }); + }); + + test('limits forged starts before signature work across replicas despite spoofed proxy headers', async () => { + const first = new RedisBridgePairingStore(redis, 600, 300, 5_000, '', { + serverId, maxUntrustedRequestsPerMinute: 2, + }); + const second = new RedisBridgePairingStore(redis, 600, 300, 5_000, '', { + serverId, maxUntrustedRequestsPerMinute: 2, + }); + const identity = createBridgeIdentity(); + const attacker = createBridgeIdentity(); + const pairing = await first.issue(workerId); + await first.redeem({ workerId, code: pairing.code, publicKey: identity.publicKey }); + const firstUrl = await startRouter(first); + const secondUrl = await startRouter(second); + let signatureChecks = 0; + for (const store of [first, second]) { + const original = store.createRecoveryChallenge.bind(store); + store.createRecoveryChallenge = async ( + ...args + ): ReturnType => { + signatureChecks += 1; + return original(...args); + }; + } + + const forged = async (url: string, suffix: number): Promise => post( + `${url}/challenge`, signedStart(attacker.privateKey), + { 'X-Forwarded-For': `198.51.100.${suffix}` }, + ); + const firstResponse = await forged(firstUrl, 1); + const secondResponse = await forged(secondUrl, 2); + const blocked = await forged( + firstUrl.replace(`/workers/${workerId}/`, '/workers/attacker-selected-worker/'), + 3, + ); + expect([firstResponse.status, secondResponse.status, blocked.status]).toEqual([401, 401, 429]); + await expect(blocked.json()).resolves.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + expect(Number(blocked.headers.get('retry-after'))).toBeGreaterThan(0); + expect(signatureChecks).toBe(2); + expect(await redis.keys('codeapi:bridge:v1:recovery:rate:start:*')).toEqual([]); + + // Bogus completions use another untrusted bucket, not the worker's signed quota. + const fake = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'credential.recover', + serverId, + workerId, + enrollmentGeneration: 'a'.repeat(24), + challenge: randomBytes(32).toString('base64url'), + expiresAt: new Date(Date.now() + 60_000).toISOString(), + signature: 'a'.repeat(86), + }; + const rejected = await post(`${secondUrl}/recover`, fake); + expect(rejected.status).toBe(401); + expect(await redis.keys('codeapi:bridge:v1:recovery:rate:complete:*')).toEqual([]); + const recovered = await post(`${secondUrl}/recover`, fake); + expect(recovered.status).toBe(401); + const blockedCompletion = await post(`${secondUrl}/recover`, fake); + expect(blockedCompletion.status).toBe(429); + await expect(blockedCompletion.json()).resolves.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + }); + + test('limits recovery challenges across two API routers sharing Redis', async () => { + const first = new RedisBridgePairingStore(redis, 600, 300, 5_000, '', { + serverId: 'https://code.example.test', maxChallengesPerMinute: 1, + }); + const second = new RedisBridgePairingStore(redis, 600, 300, 5_000, '', { + serverId: 'https://code.example.test', maxChallengesPerMinute: 1, + }); + const identity = createBridgeIdentity(); + const pairing = await first.issue(workerId); + await first.redeem({ workerId, code: pairing.code, publicKey: identity.publicKey }); + const baseUrl = await startRouter(second); + const initial = signedStart(identity.privateKey); + await first.createRecoveryChallenge(workerId, initial, initial.signature); + const denied = await post(`${baseUrl}/challenge`, signedStart(identity.privateKey)); + expect(denied.status).toBe(429); + await expect(denied.json()).resolves.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + }); +}); diff --git a/service/src/bridge/recovery.test.ts b/service/src/bridge/recovery.test.ts new file mode 100644 index 00000000..443e7277 --- /dev/null +++ b/service/src/bridge/recovery.test.ts @@ -0,0 +1,518 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import { createHash, randomBytes } from 'crypto'; +import RedisMock from 'ioredis-mock'; + +import type Redis from 'ioredis'; +import type { + BridgeRecoveryProofInput, + BridgeRecoveryStartProofInput, +} from '../../../packages/code/src/identity'; +import type { + BridgeRecoveryChallenge, + BridgeRecoveryOptions, + BridgeWorkerBinding, + BridgeWorkerCredential, +} from './pairing'; + +import { + createBridgeIdentity, + signBridgeRecovery, + signBridgeRecoveryStart, + signBridgeRequest, +} from '../../../packages/code/src/identity'; +import { RedisBridgePairingStore } from './pairing'; +import { RedisBridgeStore } from './store'; + +const redis = new RedisMock() as unknown as Redis; +const serverId = 'https://code.example.test'; +const workerId = 'durable-worker'; +const binding: BridgeWorkerBinding = { + tenantId: 'tenant-one', + principal: { type: 'user', id: 'owner-one' }, +}; + +function recoverableStore(options: Partial = {}): RedisBridgePairingStore { + return new RedisBridgePairingStore(redis, 600, 300, 5_000, '', { + serverId, + ...options, + }); +} + +function authorizedRequest( + privateKey: string, + credential: string, + nonce: string, +): Parameters[0] { + const proof = { + credential, + method: 'POST', + path: '/v1/bridge/workers/register', + timestamp: new Date().toISOString(), + nonce, + body: JSON.stringify({ protocolVersion: 1, workerId }), + }; + return { + ...proof, + workerId, + signature: signBridgeRequest(privateKey, proof), + }; +} + +async function enroll( + store: RedisBridgePairingStore, + publicKey: string, +): Promise { + const pairing = await store.issue(workerId, binding); + return store.redeem({ workerId, code: pairing.code, publicKey }); +} + +function recoveryStart( + privateKey: string, + nonce = randomBytes(32).toString('base64url'), + overrides: Partial = {}, +): { proof: BridgeRecoveryStartProofInput; signature: string } { + const proof: BridgeRecoveryStartProofInput = { + operation: 'credential.challenge', + serverId, + workerId, + timestamp: new Date().toISOString(), + nonce, + ...overrides, + }; + return { proof, signature: signBridgeRecoveryStart(privateKey, proof) }; +} + +async function challengeFor( + store: RedisBridgePairingStore, + privateKey: string, +): Promise { + const { proof, signature } = recoveryStart(privateKey); + return store.createRecoveryChallenge(workerId, proof, signature); +} + +async function recover( + store: RedisBridgePairingStore, + privateKey: string, +): Promise<{ challenge: BridgeRecoveryChallenge; credential: BridgeWorkerCredential }> { + const challenge = await challengeFor(store, privateKey); + const credential = await store.recoverCredential( + workerId, + challenge, + signBridgeRecovery(privateKey, challenge), + ); + return { challenge, credential }; +} + +afterEach(async () => { + await redis.flushall(); +}); + +describe('durable bridge enrollment', () => { + test('keeps legacy pairing and refresh compatible until recovery is enabled', async () => { + const legacy = new RedisBridgePairingStore(redis); + const identity = createBridgeIdentity(); + const issued = await enroll(legacy, identity.publicKey); + expect(await redis.get(`codeapi:bridge:v1:enrollment:${workerId}`)).toBeNull(); + + const replica = recoverableStore(); + const { proof, signature } = recoveryStart(identity.privateKey); + await expect(replica.createRecoveryChallenge(workerId, proof, signature)) + .rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + const rotated = await replica.rotate(workerId); + await expect( + replica.authorize(authorizedRequest(identity.privateKey, rotated.credential, 'legacy-proof')), + ).resolves.toMatchObject({ workerId, binding }); + expect(issued.credential).not.toBe(rotated.credential); + }); + + test('recovers after access expiry and restart with the same identity and binding', async () => { + const identity = createBridgeIdentity(); + const firstReplica = recoverableStore(); + const issued = await enroll(firstReplica, identity.publicKey); + const original = await firstReplica.authorize( + authorizedRequest(identity.privateKey, issued.credential, 'before-outage'), + ); + const digest = createHash('sha256').update(issued.credential).digest('hex'); + await redis.del( + `codeapi:bridge:v1:credential:${digest}`, + `codeapi:bridge:v1:identity:${workerId}`, + `codeapi:bridge:v1:stable-identity:${workerId}`, + ); + await expect(firstReplica.rotate(workerId)).rejects.toMatchObject({ + code: 'CREDENTIAL_INVALID', + }); + + const restartedReplica = recoverableStore(); + const { challenge, credential } = await recover(restartedReplica, identity.privateKey); + expect(challenge).toMatchObject({ + operation: 'credential.recover', + serverId, + workerId, + }); + const restored = await firstReplica.authorize( + authorizedRequest(identity.privateKey, credential.credential, 'after-outage'), + ); + expect(restored).toMatchObject({ identityId: original.identityId, binding }); + expect(credential.expiresAt).toBeString(); + const rotated = await restartedReplica.rotate(workerId, restored.credentialId); + await expect(firstReplica.authorize( + authorizedRequest(identity.privateKey, rotated.credential, 'after-rotation'), + )).resolves.toMatchObject({ identityId: original.identityId, binding }); + }); + + test('requires the enrolled key and binds proofs to server, worker, generation, operation, and expiry', async () => { + const enrolled = createBridgeIdentity(); + const outsider = createBridgeIdentity(); + const store = recoverableStore(); + await enroll(store, enrolled.publicKey); + const challenge = await challengeFor(store, enrolled.privateKey); + + await expect(store.recoverCredential( + workerId, challenge, signBridgeRecovery(outsider.privateKey, challenge), + )).rejects.toMatchObject({ code: 'PROOF_INVALID' }); + for (const modified of [ + { ...challenge, serverId: 'https://other.example.test' }, + { ...challenge, workerId: 'another-worker' }, + { ...challenge, enrollmentGeneration: '0'.repeat(24) }, + { ...challenge, operation: 'credential.recover-other' as 'credential.recover' }, + { ...challenge, expiresAt: new Date(Date.now() + 120_000).toISOString() }, + ]) { + await expect(store.recoverCredential( + workerId, + modified, + signBridgeRecovery(enrolled.privateKey, modified as BridgeRecoveryProofInput), + )).rejects.toMatchObject({ code: 'CHALLENGE_INVALID' }); + } + const signature = signBridgeRecovery(enrolled.privateKey, challenge); + await store.recoverCredential(workerId, challenge, signature); + await expect(store.recoverCredential(workerId, challenge, signature)) + .rejects.toMatchObject({ code: 'CHALLENGE_INVALID' }); + }); + + test('rejects a signed challenge after its short-lived Redis window expires', async () => { + const identity = createBridgeIdentity(); + const store = recoverableStore({ challengeTtlSeconds: 1 }); + await enroll(store, identity.publicKey); + const challenge = await challengeFor(store, identity.privateKey); + await new Promise((resolve) => setTimeout(resolve, 1_100)); + await expect(store.recoverCredential( + workerId, challenge, signBridgeRecovery(identity.privateKey, challenge), + )).rejects.toMatchObject({ code: 'CHALLENGE_INVALID' }); + }); + + test('retries a lost recovery response without changing the enrolled identity', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore(); + const originalCredential = await enroll(first, identity.publicKey); + const original = await first.authorize( + authorizedRequest(identity.privateKey, originalCredential.credential, 'before-lost-response'), + ); + await recover(first, identity.privateKey); // The caller lost this credential response. + const second = recoverableStore(); + const retried = await recover(second, identity.privateKey); + const auth = await second.authorize( + authorizedRequest(identity.privateKey, retried.credential.credential, 'after-lost-response'), + ); + expect(auth.identityId).toBe(original.identityId); + expect(auth.binding).toEqual(binding); + const enrolled = JSON.parse((await redis.get(`codeapi:bridge:v1:enrollment:${workerId}`))!) as { + generation: string; + }; + expect(await redis.get(`codeapi:bridge:v1:enrollment-required:${workerId}`)) + .toBe(enrolled.generation); + }); + + test('rejects expired and missing enrollment even when an access credential remains live', async () => { + const identity = createBridgeIdentity(); + const store = recoverableStore({ enrollmentTtlSeconds: 1 }); + const issued = await enroll(store, identity.publicKey); + await new Promise((resolve) => setTimeout(resolve, 1_100)); + await expect(challengeFor(store, identity.privateKey)) + .rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(store.rotate(workerId)) + .rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(store.authorize( + authorizedRequest(identity.privateKey, issued.credential, 'expired-enrollment'), + )).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + }); + + test('never treats an unmarked access token as legacy after authorization state is lost', async () => { + const identity = createBridgeIdentity(); + const store = recoverableStore(); + const issued = await enroll(store, identity.publicKey); + const credentialKey = `codeapi:bridge:v1:credential:${createHash('sha256').update(issued.credential).digest('hex')}`; + const raw = JSON.parse((await redis.get(credentialKey))!) as { enrollmentGeneration?: string }; + delete raw.enrollmentGeneration; // Simulate a refresh by a pre-recovery replica. + await redis.set(credentialKey, JSON.stringify(raw), 'EX', 300); + await redis.del(`codeapi:bridge:v1:enrollment:${workerId}`); + + expect(await redis.get(`codeapi:bridge:v1:enrollment-required:${workerId}`)).not.toBeNull(); + await expect(store.authorize( + authorizedRequest(identity.privateKey, issued.credential, 'lost-authorization'), + )).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(store.rotate(workerId)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(challengeFor(store, identity.privateKey)) + .rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await redis.set(`codeapi:bridge:v1:enrollment:${workerId}`, 'null'); + await expect(challengeFor(store, identity.privateKey)) + .rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + }); + + test('re-pairing explicitly supersedes a previous key and binds to the new principal', async () => { + const first = createBridgeIdentity(); + const second = createBridgeIdentity(); + const store = recoverableStore(); + const initial = await enroll(store, first.publicKey); + const pending = await challengeFor(store, first.privateKey); + const replacement = await store.issue(workerId, { + tenantId: 'tenant-two', principal: { type: 'user', id: 'owner-two' }, + }); + await store.redeem({ workerId, code: replacement.code, publicKey: second.publicKey }); + + await expect(store.recoverCredential( + workerId, pending, signBridgeRecovery(first.privateKey, pending), + )).rejects.toMatchObject({ code: 'CHALLENGE_INVALID' }); + await expect(store.authorize( + authorizedRequest(first.privateKey, initial.credential, 'superseded-key'), + )).rejects.toMatchObject({ code: 'CREDENTIAL_INVALID' }); + const { credential } = await recover(store, second.privateKey); + await expect(store.authorize( + authorizedRequest(second.privateKey, credential.credential, 'new-owner'), + )).resolves.toMatchObject({ + binding: { tenantId: 'tenant-two', principal: { type: 'user', id: 'owner-two' } }, + }); + }); + + test('concurrent replicas consume a recovery challenge and count it only once', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore({ maxAttemptsPerMinute: 2 }); + const second = recoverableStore({ maxAttemptsPerMinute: 2 }); + await enroll(first, identity.publicKey); + const challenge = await challengeFor(first, identity.privateKey); + const signature = signBridgeRecovery(identity.privateKey, challenge); + const attempts = await Promise.allSettled([ + first.recoverCredential(workerId, challenge, signature), + second.recoverCredential(workerId, challenge, signature), + ]); + const issued = attempts.filter( + (result): result is PromiseFulfilledResult => + result.status === 'fulfilled', + ); + const rejected = attempts.filter( + (result): result is PromiseRejectedResult => result.status === 'rejected', + ); + expect(issued).toHaveLength(1); + expect(rejected).toHaveLength(1); + expect(rejected[0]).toMatchObject({ reason: { code: 'CHALLENGE_INVALID' } }); + const digest = createHash('sha256').update(issued[0].value.credential).digest('hex'); + expect(await redis.get(`codeapi:bridge:v1:identity:${workerId}`)).toBe(digest); + expect(await redis.get(`codeapi:bridge:v1:recovery:rate:complete:${workerId}:${challenge.enrollmentGeneration}`)) + .toBe('1'); + }); + + test('revocation beats a signed recovery pending on a different replica', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore(); + const second = recoverableStore(); + await enroll(first, identity.publicKey); + const challenge = await challengeFor(first, identity.privateKey); + const originalEval = redis.eval.bind(redis); + let release!: () => void; + let enter!: () => void; + const paused = new Promise((resolve) => { enter = resolve; }); + const resume = new Promise((resolve) => { release = resolve; }); + redis.eval = (async (script: string, ...args: unknown[]) => { + if (script.includes('local stableIdentity = redis.call')) { + enter(); + await resume; + } + return (originalEval as (...evalArgs: unknown[]) => Promise)(script, ...args); + }) as Redis['eval']; + try { + const pending = first.recoverCredential( + workerId, challenge, signBridgeRecovery(identity.privateKey, challenge), + ); + await paused; + await second.revoke(workerId); + release(); + await expect(pending).rejects.toMatchObject({ code: 'CHALLENGE_INVALID' }); + expect(await redis.get(`codeapi:bridge:v1:identity:${workerId}`)).toBeNull(); + expect(await redis.get(`codeapi:bridge:v1:enrollment:${workerId}`)).toBeNull(); + } finally { + redis.eval = originalEval as Redis['eval']; + release(); + } + }); + + test('isolates untrusted recovery limits by socket peer and endpoint from signed machine quotas', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore({ + maxUntrustedRequestsPerMinute: 1, maxChallengesPerMinute: 1, maxAttemptsPerMinute: 1, + }); + const second = recoverableStore({ + maxUntrustedRequestsPerMinute: 1, maxChallengesPerMinute: 1, maxAttemptsPerMinute: 1, + }); + await enroll(first, identity.publicKey); + await first.limitUntrustedRecovery('192.0.2.1', 'challenge'); + await expect(second.limitUntrustedRecovery('192.0.2.1', 'challenge')) + .rejects.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + await expect(second.limitUntrustedRecovery('192.0.2.2', 'challenge')) + .resolves.toBeUndefined(); + await expect(second.limitUntrustedRecovery('192.0.2.1', 'recover')) + .resolves.toBeUndefined(); + expect((await redis.keys('codeapi:bridge:v1:recovery:rate:start:*')).length).toBe(0); + const { credential } = await recover(second, identity.privateKey); + expect(credential.workerId).toBe(workerId); + }); + + test('only a signed, fresh, unused start request consumes the worker challenge budget', async () => { + const identity = createBridgeIdentity(); + const outsider = createBridgeIdentity(); + const first = recoverableStore({ maxChallengesPerMinute: 2 }); + const second = recoverableStore({ maxChallengesPerMinute: 2 }); + await enroll(first, identity.publicKey); + + for (let index = 0; index < 20; index += 1) { + const forged = recoveryStart(outsider.privateKey); + await expect(first.createRecoveryChallenge(workerId, forged.proof, forged.signature)) + .rejects.toMatchObject({ code: 'PROOF_INVALID' }); + } + const otherServer = recoveryStart(identity.privateKey, undefined, { + serverId: 'https://unrelated.example.test', + }); + await expect(first.createRecoveryChallenge(workerId, otherServer.proof, otherServer.signature)) + .rejects.toMatchObject({ code: 'PROOF_INVALID' }); + const stale = recoveryStart(identity.privateKey, undefined, { + timestamp: new Date(Date.now() - 5 * 60_000).toISOString(), + }); + await expect(first.createRecoveryChallenge(workerId, stale.proof, stale.signature)) + .rejects.toMatchObject({ code: 'PROOF_INVALID' }); + + const valid = recoveryStart(identity.privateKey); + await first.createRecoveryChallenge(workerId, valid.proof, valid.signature); + await expect(second.createRecoveryChallenge(workerId, valid.proof, valid.signature)) + .rejects.toMatchObject({ code: 'PROOF_REPLAYED' }); + await challengeFor(second, identity.privateKey); + await expect(challengeFor(first, identity.privateKey)) + .rejects.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + }); + + test('fabricated completions never spend the worker budget or block a fresh signed recovery', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore({ maxAttemptsPerMinute: 1 }); + const second = recoverableStore({ maxAttemptsPerMinute: 1 }); + await enroll(first, identity.publicKey); + const real = await challengeFor(first, identity.privateKey); + + for (let index = 0; index < 35; index += 1) { + const fake = { ...real, challenge: randomBytes(32).toString('base64url') }; + await expect(first.recoverCredential(workerId, fake, 'forged')) + .rejects.toMatchObject({ code: 'CHALLENGE_INVALID' }); + } + await first.recoverCredential(workerId, real, signBridgeRecovery(identity.privateKey, real)); + const next = await challengeFor(second, identity.privateKey); + await expect(second.recoverCredential( + workerId, next, signBridgeRecovery(identity.privateKey, next), + )).rejects.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + }); + + test('invalid signatures exhaust only their own challenge, not the enrolled worker', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore({ maxAttemptsPerMinute: 1 }); + const second = recoverableStore({ maxAttemptsPerMinute: 1 }); + await enroll(first, identity.publicKey); + const attacked = await challengeFor(first, identity.privateKey); + await expect(first.recoverCredential(workerId, attacked, 'forged')) + .rejects.toMatchObject({ code: 'PROOF_INVALID' }); + await expect(second.recoverCredential( + workerId, attacked, signBridgeRecovery(identity.privateKey, attacked), + )).rejects.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + const fresh = await challengeFor(second, identity.privateKey); + await expect(first.recoverCredential( + workerId, fresh, signBridgeRecovery(identity.privateKey, fresh), + )).resolves.toMatchObject({ workerId }); + }); + + test('key replacement starts new signed challenge and recovery budgets', async () => { + const oldKey = createBridgeIdentity(); + const newKey = createBridgeIdentity(); + const store = recoverableStore({ maxChallengesPerMinute: 1, maxAttemptsPerMinute: 1 }); + await enroll(store, oldKey.publicKey); + const oldChallenge = await challengeFor(store, oldKey.privateKey); + await store.recoverCredential( + workerId, oldChallenge, signBridgeRecovery(oldKey.privateKey, oldChallenge), + ); + const replacement = await store.issue(workerId, binding); + await store.redeem({ workerId, code: replacement.code, publicKey: newKey.publicKey }); + + await expect(challengeFor(store, oldKey.privateKey)) + .rejects.toMatchObject({ code: 'PROOF_INVALID' }); + const newChallenge = await challengeFor(store, newKey.privateKey); + await expect(store.recoverCredential( + workerId, newChallenge, signBridgeRecovery(newKey.privateKey, newChallenge), + )).resolves.toMatchObject({ workerId }); + }); + + test('revocation fences a previously verified start proof before a challenge is written', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore(); + const second = recoverableStore(); + await enroll(first, identity.publicKey); + const { proof, signature } = recoveryStart(identity.privateKey); + const originalEval = redis.eval.bind(redis); + let release!: () => void; + let enter!: () => void; + const paused = new Promise((resolve) => { enter = resolve; }); + const resume = new Promise((resolve) => { release = resolve; }); + redis.eval = (async (script: string, ...args: unknown[]) => { + if (script.includes('KEYS[6]) == 1 then return -2')) { + enter(); + await resume; + } + return (originalEval as (...evalArgs: unknown[]) => Promise)(script, ...args); + }) as Redis['eval']; + try { + const pending = first.createRecoveryChallenge(workerId, proof, signature); + await paused; + await second.revoke(workerId); + release(); + await expect(pending).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + expect(await redis.keys('codeapi:bridge:v1:recovery:challenge:*')).toEqual([]); + } finally { + redis.eval = originalEval as Redis['eval']; + release(); + } + }); + + test('recovering credentials never clears worker or workspace quarantine', async () => { + const identity = createBridgeIdentity(); + const store = recoverableStore(); + await enroll(store, identity.publicKey); + const workerQuarantine = `codeapi:bridge:v1:worker:${workerId}:incarnation:incarnation-00000001:quarantined`; + const workspaceQuarantine = `codeapi:bridge:v1:worker:${workerId}:workspace:${createHash('sha256').update('session-one').digest('hex')}:quarantined`; + await redis.set(workerQuarantine, '1'); + await redis.set(workspaceQuarantine, '1'); + const { credential } = await recover(store, identity.privateKey); + expect(await redis.get(workerQuarantine)).toBe('1'); + expect(await redis.get(workspaceQuarantine)).toBe('1'); + const auth = await store.authorize( + authorizedRequest(identity.privateKey, credential.credential, 'quarantined-machine'), + ); + await expect(new RedisBridgeStore(redis).register({ + protocolVersion: 1, + workerId, + incarnationId: 'incarnation-00000001', + capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', runtimes: ['bash'] }, + }, auth)).rejects.toMatchObject({ code: 'WORKER_QUARANTINED' }); + }); + + test('refuses non-HTTPS or non-origin deployment identity and unbounded recovery policy', () => { + for (const invalid of ['http://code.example.test', 'https://code.example.test/path', 'https://user@code.example.test']) { + expect(() => recoverableStore({ serverId: invalid })).toThrow(); + } + expect(() => recoverableStore({ maxAttemptsPerMinute: 0 })).toThrow(); + expect(() => recoverableStore({ maxUntrustedRequestsPerMinute: 0 })).toThrow(); + expect(() => recoverableStore({ maxUntrustedRequestsPerMinute: 1201 })).toThrow(); + expect(() => recoverableStore({ challengeTtlSeconds: 301 })).toThrow(); + }); +}); diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 5b319733..b3d549b1 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -310,6 +310,114 @@ export function createBridgeRouter(options: BridgeRouterOptions): Router { } })); + const untrustedRecoveryLimit = (operation: 'challenge' | 'recover'): RequestHandler => + (req, res, next) => { + if (options.authMode !== 'paired' || !options.pairings.recoveryEnabled) { + next(); + return; + } + // req.ip trusts X-Forwarded-For in our server. Use the connection peer so + // untrusted headers and arbitrary worker IDs cannot create new buckets. + void options.pairings.limitUntrustedRecovery(req.socket.remoteAddress ?? '', operation) + .then(() => next(), (error: unknown) => { + if (error instanceof BridgePairingError && error.code === 'RECOVERY_RATE_LIMITED') { + res.set('Retry-After', '60').status(429).json({ error: error.message, code: error.code }); + return; + } + next(error); + }); + }; + + router.post('/workers/:workerId/credentials/challenge', untrustedRecoveryLimit('challenge'), asyncRoute(async (req, res) => { + if (options.authMode !== 'paired' || !options.pairings.recoveryEnabled) { + res.status(404).json({ error: 'Machine recovery is disabled' }); + return; + } + const workerId = req.params.workerId; + const body = isRecord(req.body) ? req.body : {}; + if ( + !validWorkerId(workerId) || !configuredWorker(workerId) || + body.protocolVersion !== BRIDGE_PROTOCOL_VERSION || + body.operation !== 'credential.challenge' || + typeof body.serverId !== 'string' || body.serverId.length > 256 || + body.workerId !== workerId || + typeof body.timestamp !== 'string' || body.timestamp.length > 64 || + typeof body.nonce !== 'string' || !/^[A-Za-z0-9_-]{43}$/.test(body.nonce) || + typeof body.signature !== 'string' || !/^[A-Za-z0-9_-]{86}$/.test(body.signature) + ) { + res.status(400).json({ error: 'Invalid machine recovery challenge request' }); + return; + } + try { + const challenge = await options.pairings.createRecoveryChallenge( + workerId, + { + operation: 'credential.challenge', + serverId: body.serverId, + workerId, + timestamp: body.timestamp, + nonce: body.nonce, + }, + body.signature, + ); + res.json({ protocolVersion: BRIDGE_PROTOCOL_VERSION, ...challenge }); + } catch (error) { + if (error instanceof BridgePairingError) { + res.status(error.code === 'RECOVERY_RATE_LIMITED' ? 429 : 401) + .json({ error: error.message, code: error.code }); + return; + } + throw error; + } + })); + + router.post('/workers/:workerId/credentials/recover', untrustedRecoveryLimit('recover'), asyncRoute(async (req, res) => { + if (options.authMode !== 'paired' || !options.pairings.recoveryEnabled) { + res.status(404).json({ error: 'Machine recovery is disabled' }); + return; + } + const workerId = req.params.workerId; + const body = isRecord(req.body) ? req.body : {}; + if ( + !validWorkerId(workerId) || !configuredWorker(workerId) || + body.protocolVersion !== BRIDGE_PROTOCOL_VERSION || + body.operation !== 'credential.recover' || + typeof body.serverId !== 'string' || body.serverId.length > 256 || + typeof body.enrollmentGeneration !== 'string' || + !/^[A-Za-z0-9_-]{24}$/.test(body.enrollmentGeneration) || + typeof body.challenge !== 'string' || + !/^[A-Za-z0-9_-]{43}$/.test(body.challenge) || + typeof body.expiresAt !== 'string' || body.expiresAt.length > 64 || + typeof body.signature !== 'string' || + !/^[A-Za-z0-9_-]{86}$/.test(body.signature) + ) { + res.status(400).json({ error: 'Invalid machine recovery proof' }); + return; + } + try { + const credential = await options.pairings.recoverCredential( + workerId, + { + operation: 'credential.recover', + serverId: body.serverId, + workerId, + enrollmentGeneration: body.enrollmentGeneration, + challenge: body.challenge, + expiresAt: body.expiresAt, + }, + body.signature, + ); + res.json({ protocolVersion: BRIDGE_PROTOCOL_VERSION, ...credential }); + } catch (error) { + if (error instanceof BridgePairingError) { + res.status(error.code === 'RECOVERY_RATE_LIMITED' ? 429 : 401) + .json({ error: error.message, code: error.code }); + return; + } + throw error; + } + })); + router.post( '/workers/:workerId/revoke', adminAuth, @@ -768,6 +876,5 @@ router.post( }), ); - return router; } diff --git a/service/src/config.ts b/service/src/config.ts index 55570dfe..29ae854d 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -426,6 +426,22 @@ export const env = { BRIDGE_AUTH_MODE: bridgeAuthMode, /** Enrollment and lease credential shared only with the configured worker. */ BRIDGE_TOKEN: process.env.CODEAPI_BRIDGE_TOKEN ?? '', + /** Opt-in stable HTTPS origin of this Code API deployment (shared by all replicas). */ + BRIDGE_RECOVERY_SERVER_ID: process.env.CODEAPI_BRIDGE_RECOVERY_SERVER_ID ?? '', + /** Zero preserves machine authorization until explicit revocation. */ + BRIDGE_ENROLLMENT_TTL_SECONDS: Number(process.env.CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS ?? 0), + BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS: Number( + process.env.CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS ?? 60, + ), + BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE: Number( + process.env.CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE ?? 12, + ), + BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE: Number( + process.env.CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE ?? 30, + ), + BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE: Number( + process.env.CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE ?? 240, + ), /** * Runtime session affinity for stateful sandbox backends. * - `stateless` (default): no runtime sessions; `runtime_session_hint` ignored. diff --git a/tests/compose-bridge-config.cjs b/tests/compose-bridge-config.cjs index de14b553..8b3abd1d 100644 --- a/tests/compose-bridge-config.cjs +++ b/tests/compose-bridge-config.cjs @@ -15,6 +15,12 @@ function render(overrides) { CODEAPI_BRIDGE_WORKER_ID: '', CODEAPI_BRIDGE_TOKEN: '', CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS: '', + CODEAPI_BRIDGE_RECOVERY_SERVER_ID: '', + CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS: '', + CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS: '', + CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE: '', + CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE: '', + CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE: '', ...overrides, }, })); @@ -28,6 +34,12 @@ for (const overrides of [ CODEAPI_BRIDGE_DYNAMIC_WORKERS: 'false', CODEAPI_BRIDGE_WORKER_ID: 'test-worker', CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS: '4', + CODEAPI_BRIDGE_RECOVERY_SERVER_ID: 'https://code.example.test', + CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS: '86400', + CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS: '90', + CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE: '8', + CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE: '16', + CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE: '400', }, ]) { const config = render(overrides); @@ -39,10 +51,20 @@ for (const overrides of [ assert.equal(env.CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS, overrides.CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS ?? '1'); assert.equal(env.CODEAPI_BRIDGE_DYNAMIC_WORKERS, overrides.CODEAPI_BRIDGE_DYNAMIC_WORKERS ?? 'true'); assert.equal(env.CODEAPI_BRIDGE_WORKER_ID, overrides.CODEAPI_BRIDGE_WORKER_ID ?? ''); + assert.equal(env.CODEAPI_BRIDGE_RECOVERY_SERVER_ID, overrides.CODEAPI_BRIDGE_RECOVERY_SERVER_ID ?? ''); + assert.equal(env.CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS, overrides.CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS ?? '0'); + assert.equal(env.CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS, overrides.CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS ?? '60'); + assert.equal(env.CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE, overrides.CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE ?? '12'); + assert.equal(env.CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE, overrides.CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE ?? '30'); + assert.equal(env.CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE, overrides.CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE ?? '240'); } for (const name of ['egress_gateway', 'sandbox-runner']) { assert.equal(config.services[name].environment.CODEAPI_BRIDGE_TOKEN, undefined); + assert.equal(config.services[name].environment.CODEAPI_BRIDGE_RECOVERY_SERVER_ID, undefined); + assert.equal(config.services[name].environment.CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE, undefined); } + assert.match(JSON.stringify(config.services.redis.command), /--appendonly.*yes/); + assert.ok(config.services.redis.volumes.some(volume => volume.target === '/data' && volume.type === 'volume')); } assert.equal(render({}).services.api.environment.CODEAPI_BRIDGE_TOKEN, ''); -console.log('Compose bridge configuration passed (dynamic/fixed pairing, no default secret).'); +console.log('Compose bridge configuration passed (pairing, opt-in recovery, durable Redis).'); From 337ddd60a8ea2207459a52f00c33fdeb05899988 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 28 Sep 2026 08:04:06 -0400 Subject: [PATCH 12/48] =?UTF-8?q?=E2=8F=B3=20fix:=20Honor=20Workspace=20Qu?= =?UTF-8?q?eue=20Budgets=20Per=20Request=20(#265)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: bound workspace queue admission per request * fix: classify pre-enqueue admission expiry and document queue budgets --- docs/byom-worker-admission.md | 42 +++++++----- docs/remote-bridge/README.md | 14 ++-- packages/code/README.md | 17 +++-- service/src/bridge/store.ts | 72 +++++++++++++-------- service/src/bridge/worker-admission.test.ts | 26 ++++++++ service/src/workspace-tools/router.test.ts | 59 +++++++++++++---- service/src/workspace-tools/router.ts | 27 ++++++-- 7 files changed, 183 insertions(+), 74 deletions(-) diff --git a/docs/byom-worker-admission.md b/docs/byom-worker-admission.md index 2fcd9f9f..0df8f17c 100644 --- a/docs/byom-worker-admission.md +++ b/docs/byom-worker-admission.md @@ -5,8 +5,13 @@ The limit is 32 admitted requests per worker, including the active request. When the limit is reached, the workspace endpoint returns HTTP 429 with `WORKER_QUEUE_FULL`. A different worker has an independent admission queue. -The Code API workspace HTTP endpoint allows up to the smaller of `JOB_TIMEOUT` -and five minutes for admission while the caller stays connected. After admission +The Code API workspace HTTP endpoint allows 30 seconds for admission when the +request has no `X-LibreChat-Workspace-Queue-Wait-Ms` header. A caller can +advertise a positive integer allowance in milliseconds through that header, +up to five minutes and any configured server queue ceiling. An invalid or +out-of-range value is rejected before dispatch. This queue budget is +independent of `JOB_TIMEOUT`; a shorter client or proxy deadline still ends +the wait. After admission and worker validation, a separate execution deadline starts. Commands receive their requested timeout (30 seconds by default, up to five minutes), capped by the operator's `JOB_TIMEOUT`, plus five seconds to settle the result. Other @@ -29,22 +34,27 @@ Existing workers still execute one assignment at a time. Parallel execution acro workspaces requires separate lease claims and isolated native sandbox contexts; this admission change does not advertise that capability. -Clients and reverse proxies must allow queue time plus execution/settlement time -and five seconds for HTTP delivery. With the default five-minute `JOB_TIMEOUT`, -that is at least 335 seconds for non-command tools, 340 seconds for default -commands, and 610 seconds for five-minute commands. With a smaller `JOB_TIMEOUT`, -use `min(JOB_TIMEOUT, 300s)` for the queue, plus `min(JOB_TIMEOUT, 30s)` for other +Clients and reverse proxies must allow the admitted queue budget plus +execution/settlement time and five seconds for HTTP delivery. With the default +five-minute `JOB_TIMEOUT` and no queue header, that is at least 65 seconds for +non-command tools, 70 seconds for default commands, and 340 seconds for +five-minute commands. At the maximum advertised five-minute queue allowance, +those totals become 335, 340, and 610 seconds respectively. With a smaller +`JOB_TIMEOUT`, use the advertised allowance (or 30 seconds without a header), +bounded by the server queue ceiling, plus `min(JOB_TIMEOUT, 30s)` for other operations or `min(JOB_TIMEOUT, requested command timeout) + 5s` for commands, -plus five seconds for delivery. +plus five seconds for delivery. The caller should advertise only the queue time +left after reserving execution, settlement, and delivery under its own HTTP +deadline; Code API does not receive that absolute deadline. -At the time of this change, LibreChat's `getWorkspaceToolTimeoutMs` still budgets -only 30 seconds for a single admission attempt (65/70/340 seconds in total). -Its `maxQueueWaitMs` is a retry horizon after a typed capacity rejection, **not** -a per-attempt HTTP timeout. Updating Code API alone therefore does not guarantee -the full wait. An earlier client, tool, or proxy timeout disconnects the request; -if work was already admitted, a mutation may have run and must not be blindly -retried. Match LibreChat's per-attempt timeout and each intermediary to the new -budget before relying on it. Existing workers do not need an update. +LibreChat's `maxQueueWaitMs` is a retry horizon after a typed capacity +rejection, **not** a per-attempt HTTP timeout. Without its opt-in +`maxRequestTimeoutMs`, LibreChat keeps a 30-second admission allowance per +attempt. Enabling a longer client budget requires LibreChat's header support on +every API replica and a timed canary through each intermediary; changing Code +API alone does not guarantee the full wait. An earlier client, tool, or proxy +timeout disconnects the request; if work was already admitted, a mutation may +have run and must not be blindly retried. Existing workers do not need an update. Focused regression coverage lives in `service/src/bridge/admission.test.ts`, `service/src/bridge/worker-admission.test.ts`, diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index 96241ad7..464e4763 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -308,12 +308,14 @@ execution. worker. The lower API or worker slot ceiling wins, and assignments sharing the same workspace isolation key remain serialized while independent conversation worktrees may run concurrently. -- Workspace tool admission waits for capacity up to the smaller of `JOB_TIMEOUT` - and five minutes while the HTTP caller remains connected. Disconnects cancel - waiting, and admitted work receives a separate execution budget. A shorter - client or proxy timeout can end the wait sooner; Code API does not receive an - absolute caller deadline. See [BYOM worker admission](../byom-worker-admission.md) - for the caller and proxy timeout requirements. +- Workspace tool admission waits for capacity up to 30 seconds without a + `X-LibreChat-Workspace-Queue-Wait-Ms` header. A caller can advertise a longer + per-request allowance, bounded by five minutes and any server queue ceiling. + Disconnects cancel waiting, and admitted work receives a separate execution + budget capped by `JOB_TIMEOUT`. A shorter client or proxy timeout can end the + wait sooner; Code API does not receive an absolute caller deadline. See + [BYOM worker admission](../byom-worker-admission.md) for the total-request + and proxy timeout requirements. - Dynamic workers are fenced to their server-issued tenant before assignment. - Each assignment has an absolute deadline, generation, and random lease token. - Settlements with the wrong worker, generation, token, or expired deadline are diff --git a/packages/code/README.md b/packages/code/README.md index 87a6fe61..95d86485 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -791,17 +791,22 @@ Legacy requests without a conversation identity continue to use the selected source root. Older Code API deployments do not negotiate the capability, so the worker omits it until every request path understands the isolation boundary. -On an updated Code API, admission waits up to the smaller of `JOB_TIMEOUT` and -five minutes while the HTTP caller remains connected; older Code API versions -waited at most 30 seconds. A `WORKSPACE_QUEUE_TIMEOUT` response (HTTP 503, +On an updated Code API, admission waits up to 30 seconds without the +`X-LibreChat-Workspace-Queue-Wait-Ms` request header. A caller may advertise a +positive integer millisecond allowance up to five minutes, capped by any server +queue ceiling. This allowance is separate from the `JOB_TIMEOUT` execution +budget and cannot outlast a shorter client or proxy timeout. A +`WORKSPACE_QUEUE_TIMEOUT` response (HTTP 503, `Retry-After: 1`) means the operation was not assigned or started; wait for capacity before submitting it again. This is distinct from `ASSIGNMENT_EXPIRED` or a transport timeout after dispatch, where execution may have occurred and mutations must not be blindly retried. No automatic retry is added by this policy. -Align the client's per-attempt timeout and any proxy with the queue **plus** -execution budget before relying on the longer wait. See the +Align the client's per-attempt timeout and every proxy with the queue **plus** +execution, settlement, and delivery budget before relying on a longer wait. +LibreChat keeps the 30-second admission allowance unless its longer total HTTP +budget is explicitly enabled and the live ingress path is verified. See the [BYOM worker admission guide](../../docs/byom-worker-admission.md) for the -current client limitation and the timeout calculations. +timeout calculations. Keep the existing URL, pairing/identity, and network policy configuration. The primary root keeps its configured workspace ID (default `primary`). Repeat diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 3d9d7e1c..0ca75566 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -65,6 +65,26 @@ export class BridgeStoreError extends Error { } } +function classifyPreEnqueueExpiry( + error: unknown, + args: { workspaceRequest?: WorkspaceToolRequest; workspaceId?: string; signal: AbortSignal }, +): unknown { + // A queue deadline reached before the assignment enqueue attempt is a + // definite non-execution, including expiry during the initial Redis reads. + if ( + (args.workspaceRequest != null || args.workspaceId != null) && + !args.signal.aborted && + error instanceof BridgeStoreError && + error.code === 'ASSIGNMENT_EXPIRED' + ) { + return new BridgeStoreError( + 'WORKSPACE_QUEUE_TIMEOUT', + 'Workspace capacity was unavailable before the queue deadline. The operation was not started. Wait for active work to finish or select an independent workspace on a machine with available capacity.', + ); + } + return error; +} + interface StoredAssignment extends CodeBridgeAssignment { leaseTokenHash: string; workerIdentityId?: string; @@ -778,12 +798,17 @@ export class RedisBridgeStore { 'Invalid workspace execution budget', ); } - this.assertDispatchActive(args.signal, args.deadlineAtMs); - const dispatchable = await this.dispatchCommand( - () => this.dispatchableRegistration(args.workerId), - args, - 'Bridge worker registration read', - ); + let dispatchable: Awaited>; + try { + this.assertDispatchActive(args.signal, args.deadlineAtMs); + dispatchable = await this.dispatchCommand( + () => this.dispatchableRegistration(args.workerId), + args, + 'Bridge worker registration read', + ); + } catch (error) { + throw classifyPreEnqueueExpiry(error, args); + } if (dispatchable == null) { throw new BridgeStoreError( 'WORKER_OFFLINE', @@ -844,17 +869,20 @@ export class RedisBridgeStore { `Bridge worker ${args.workerId} does not advertise programmatic execution for the selected workspace`, ); } - if ( - args.runtimeSessionId !== undefined && - (await this.dispatchCommand( - () => - this.redis.exists( - workspaceQuarantineKey(args.workerId, args.runtimeSessionId ?? ''), - ), - args, - 'Bridge workspace fence read', - )) === 1 - ) { + let quarantined = false; + const runtimeSessionId = args.runtimeSessionId; + if (runtimeSessionId !== undefined) { + try { + quarantined = (await this.dispatchCommand( + () => this.redis.exists(workspaceQuarantineKey(args.workerId, runtimeSessionId)), + args, + 'Bridge workspace fence read', + )) === 1; + } catch (error) { + throw classifyPreEnqueueExpiry(error, args); + } + } + if (quarantined) { throw new BridgeStoreError( 'WORKSPACE_QUARANTINED', 'Bridge workspace is quarantined after an incomplete result commit', @@ -1181,15 +1209,7 @@ export class RedisBridgeStore { } } catch (error) { // Once enqueue starts, even a lost Redis response may hide execution. - if ( - admission != null && !enqueueAttempted && !args.signal.aborted && - error instanceof BridgeStoreError && error.code === 'ASSIGNMENT_EXPIRED' - ) { - throw new BridgeStoreError( - 'WORKSPACE_QUEUE_TIMEOUT', - 'Workspace capacity was unavailable before the queue deadline. The operation was not started. Wait for active work to finish or select an independent workspace on a machine with available capacity.', - ); - } + if (!enqueueAttempted) throw classifyPreEnqueueExpiry(error, args); throw error; } finally { if (admission != null) { diff --git a/service/src/bridge/worker-admission.test.ts b/service/src/bridge/worker-admission.test.ts index 98363c34..c77ac19e 100644 --- a/service/src/bridge/worker-admission.test.ts +++ b/service/src/bridge/worker-admission.test.ts @@ -124,6 +124,32 @@ test('an expired queued call never reaches the worker and does not strand later await third; }); +test('an already-expired workspace deadline is a definite queue timeout before registration', async () => { + await register(); + await expect(dispatch('expired-before-read', new AbortController(), -1)).rejects.toMatchObject({ + code: 'WORKSPACE_QUEUE_TIMEOUT', + }); + expect(await redis.zcard(`codeapi:bridge:v1:worker:${workerId}:admission`)).toBe(0); + expect(await store.lease(workerId, incarnationId, 20)).toBeUndefined(); +}); + +test('expiry during the registration read never becomes an ambiguous assignment error', async () => { + await register(); + const registrationRead = spyOn(redis, 'mget').mockImplementation(async () => { + await new Promise(resolve => setTimeout(resolve, 25)); + throw new Error('registration read outlived its queue budget'); + }); + try { + await expect(dispatch('expired-during-read', new AbortController(), 1)).rejects.toMatchObject({ + code: 'WORKSPACE_QUEUE_TIMEOUT', + }); + } finally { + registrationRead.mockRestore(); + } + expect(await redis.zcard(`codeapi:bridge:v1:worker:${workerId}:admission`)).toBe(0); + expect(await store.lease(workerId, incarnationId, 20)).toBeUndefined(); +}); + test('a queued request is rejected if the worker withdraws its capability', async () => { await register(); const first = dispatch('first'); diff --git a/service/src/workspace-tools/router.test.ts b/service/src/workspace-tools/router.test.ts index 18d24a76..d3cb032e 100644 --- a/service/src/workspace-tools/router.test.ts +++ b/service/src/workspace-tools/router.test.ts @@ -67,16 +67,17 @@ test('binds instance admission to the authenticated tenant and user while preser expect(dispatched[1]?.workspaceInstanceId).toBeUndefined(); }); -test.each<[WorkspaceToolRequest, number, number?, number?]>([ - [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, undefined, undefined], - [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, 125_000, undefined], - [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 35_000, undefined, undefined], - [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready', timeoutMs: 90_000 }, 95_000, 125_000, undefined], - [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready', timeoutMs: 300_000 }, 305_000, undefined, undefined], - [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 6000, 1000, undefined], - [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 35_000, 600_000, undefined], - [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, 125_000, 5000], -])('separates the admission deadline from execution budget for %j', async (request, expectedExecution, ceiling, queueTimeoutMs) => { +test.each<[WorkspaceToolRequest, number, number?, number?, number?]>([ + [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, undefined, undefined, undefined], + [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, 125_000, undefined, undefined], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 35_000, undefined, undefined, undefined], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready', timeoutMs: 90_000 }, 95_000, 125_000, undefined, 90_000], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready', timeoutMs: 300_000 }, 305_000, undefined, undefined, undefined], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 6000, 1000, undefined, undefined], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 35_000, 600_000, undefined, undefined], + [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, 125_000, 5000, 90_000], + [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, undefined, undefined, 300_000], +])('separates the admission deadline from execution budget for %j', async (request, expectedExecution, ceiling, queueTimeoutMs, advertisedQueueWaitMs) => { const app = express(); app.use(json()); app.use((req, _res, next) => { @@ -102,12 +103,15 @@ test.each<[WorkspaceToolRequest, number, number?, number?]>([ const address = server.address(); if (address == null || typeof address === 'string') throw new Error('Missing listener'); const response = await fetch(`http://127.0.0.1:${address.port}/workspace-tools/execute`, { - method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(request), + method: 'POST', headers: { + 'Content-Type': 'application/json', + ...(advertisedQueueWaitMs === undefined ? {} : { 'X-LibreChat-Workspace-Queue-Wait-Ms': String(advertisedQueueWaitMs) }), + }, body: JSON.stringify(request), }); await response.json(); expect(executionBudget).toBe(expectedExecution); if (request.operation === 'execute_command') expect(commandTimeout).toBe(expectedExecution - 5000); - const expectedQueueBudget = queueTimeoutMs ?? Math.min(ceiling ?? 30_000, 300_000); + const expectedQueueBudget = Math.min(advertisedQueueWaitMs ?? 30_000, queueTimeoutMs ?? 300_000); expect(queueRemaining).toBeGreaterThan(expectedQueueBudget - 1000); expect(queueRemaining).toBeLessThanOrEqual(expectedQueueBudget); }); @@ -122,6 +126,32 @@ test.each([0, 300_001, Number.POSITIVE_INFINITY])('rejects an unbounded queue ov })).toThrow('Workspace queue timeout must be between 1 and 300000 milliseconds'); }); +test.each(['0', '-1', '300001', '1.5', '01', '1, 2', '999999999999999999999'])('rejects invalid per-request queue allowance %j before dispatch', async (queueWait) => { + const app = express(); + app.use(json()); + app.use((req, _res, next) => { + applyPrincipal(req, { userId: 'user-1', tenantId: 'tenant-1', principalSource: 'librechat_jwt', codeWorkerId: 'user-worker' }); + next(); + }); + let dispatched = false; + app.use(createWorkspaceToolsRouter({ + backend: 'remote-bridge', configuredWorkerId: 'user-worker', dynamicWorkers: false, + store: { async dispatchWorkspaceTool() { dispatched = true; throw new Error('must not dispatch'); } }, + })); + server = createServer(app); + await new Promise(resolve => server!.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Missing listener'); + const response = await fetch(`http://127.0.0.1:${address.port}/workspace-tools/execute`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'X-LibreChat-Workspace-Queue-Wait-Ms': queueWait }, + body: JSON.stringify({ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }), + }); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ code: 'INVALID_WORKSPACE_QUEUE_WAIT' }); + expect(dispatched).toBe(false); +}); + test('rejects new workspace dispatches while the service is shutting down', async () => { let dispatched = false; const app = express(); @@ -419,7 +449,7 @@ test.each([ status: expectedStatus, errorCode, outcome: 'completed', - deadlineBudgetMs: 330_000, + deadlineBudgetMs: 60_000, dispatchDurationMs: expect.any(Number), }), ); @@ -549,7 +579,8 @@ test('logs a disconnected dispatch once without inventing HTTP 200', async () => closeConnection(); await expect(response).rejects.toThrow(); await closed.promise; - expect(queueRemaining).toBeGreaterThan(120_000); + expect(queueRemaining).toBeGreaterThan(29_000); + expect(queueRemaining).toBeLessThanOrEqual(30_000); expect(dispatchAborted).toBe(true); expect(logSpy).not.toHaveBeenCalled(); settlementGate.resolve(); diff --git a/service/src/workspace-tools/router.ts b/service/src/workspace-tools/router.ts index 51e47f26..78dbd1e7 100644 --- a/service/src/workspace-tools/router.ts +++ b/service/src/workspace-tools/router.ts @@ -21,6 +21,8 @@ import { import { principalWorkspaceInstanceId } from '../bridge/workspace-instance'; const MAX_WORKSPACE_QUEUE_WAIT_MS = 5 * 60_000; +const DEFAULT_WORKSPACE_QUEUE_WAIT_MS = 30_000; +const WORKSPACE_QUEUE_WAIT_HEADER = 'X-LibreChat-Workspace-Queue-Wait-Ms'; interface WorkspaceToolsRouterOptions { store: Pick; @@ -57,12 +59,10 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) )) { throw new RangeError('Workspace execution timeout must be a positive safe integer'); } - // The HTTP disconnect cancels waiting; this bounds admission while the caller remains connected. - const queueBudgetMs = options.queueTimeoutMs ?? Math.min( - options.timeoutMs ?? 30_000, - MAX_WORKSPACE_QUEUE_WAIT_MS, - ); - if (!Number.isSafeInteger(queueBudgetMs) || queueBudgetMs < 1 || queueBudgetMs > MAX_WORKSPACE_QUEUE_WAIT_MS) { + // A configured queue timeout is a ceiling. Legacy callers get 30 seconds even + // when their execution timeout is longer; only the per-request header opts in. + const queueCeilingMs = options.queueTimeoutMs ?? MAX_WORKSPACE_QUEUE_WAIT_MS; + if (!Number.isSafeInteger(queueCeilingMs) || queueCeilingMs < 1 || queueCeilingMs > MAX_WORKSPACE_QUEUE_WAIT_MS) { throw new RangeError('Workspace queue timeout must be between 1 and 300000 milliseconds'); } const router = Router(); @@ -88,6 +88,21 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) }); return; } + const advertisedQueueWait = req.header(WORKSPACE_QUEUE_WAIT_HEADER); + if (advertisedQueueWait !== undefined && !/^[1-9]\d*$/.test(advertisedQueueWait)) { + outcome.errorCode = 'INVALID_WORKSPACE_QUEUE_WAIT'; + res.status(400).json({ error: 'Invalid workspace queue wait', code: 'INVALID_WORKSPACE_QUEUE_WAIT' }); + return; + } + const requestedQueueWaitMs = advertisedQueueWait === undefined + ? DEFAULT_WORKSPACE_QUEUE_WAIT_MS + : Number(advertisedQueueWait); + if (!Number.isSafeInteger(requestedQueueWaitMs) || requestedQueueWaitMs > MAX_WORKSPACE_QUEUE_WAIT_MS) { + outcome.errorCode = 'INVALID_WORKSPACE_QUEUE_WAIT'; + res.status(400).json({ error: 'Invalid workspace queue wait', code: 'INVALID_WORKSPACE_QUEUE_WAIT' }); + return; + } + const queueBudgetMs = Math.min(requestedQueueWaitMs, queueCeilingMs); outcome.operation = req.body.operation; const principalRequest: WorkspaceToolRequest = req.body.workspaceInstanceId == null ? req.body From 5b71e8c14d8843e725467174b2684debdf31ab58 Mon Sep 17 00:00:00 2001 From: proseer-lars Date: Mon, 28 Sep 2026 13:15:47 -0400 Subject: [PATCH 13/48] fix: deduplicate files by destination before sandbox validation (#247) * fix: deduplicate files by destination before sandbox validation Some callers (e.g. LibreChat) send the same file more than once per exec request when a user re-uploads a file in the same conversation. The sandbox's validateExecuteFiles() rejects duplicate destinations, which causes every tool call in that conversation to fail with: Error from sandbox: [bad_request] files contains duplicate destination "filename.pdf" This surfaces to the end user as a confusing "sandbox is down" error when the sandbox itself is fine - it is just rejecting the duplicate file list. Fix: add deduplicateFilesByDestination() and call it in getJob() before validateExecuteFiles(). Keeps the first file for each destination, silently drops subsequent duplicates, and logs a warning. * fix: preserve validated destinations when deduplicating input files * fix: protect submitted code from uploaded file collisions --------- Co-authored-by: Lia --- api/src/api/v2-files.test.ts | 103 ++++++++++++++++- api/src/api/v2-session-binding.test.ts | 86 ++++++++++++++ api/src/api/v2.ts | 151 ++++++++++++++++--------- 3 files changed, 284 insertions(+), 56 deletions(-) diff --git a/api/src/api/v2-files.test.ts b/api/src/api/v2-files.test.ts index 540b529b..83918884 100644 --- a/api/src/api/v2-files.test.ts +++ b/api/src/api/v2-files.test.ts @@ -1,7 +1,8 @@ import { describe, expect, test } from 'bun:test'; import { config } from '../config'; +import { collectExecuteRequestInputFiles } from '../execution-manifest-request'; import type { TFile } from '../job'; -import { validateExecuteArguments, validateExecuteFiles } from './v2'; +import { validateExecuteArguments, validateExecuteFiles, deduplicateFilesByDestination } from './v2'; function messageOf(fn: () => void): string { try { @@ -85,4 +86,104 @@ describe('execute file validation', () => { ]; expect(() => validateExecuteFiles(files)).not.toThrow(); }); + + test('deduplicateFilesByDestination keeps the latest upload in the original destination order', () => { + const files: TFile[] = [ + { name: 'data.csv', id: 'first', storage_session_id: 'uploads' }, + { name: 'data.csv', id: 'second', storage_session_id: 'uploads' }, + { name: 'other.csv', id: 'unique', storage_session_id: 'uploads' }, + { name: 'data.csv', id: 'third', storage_session_id: 'latest-uploads' }, + ]; + const result = deduplicateFilesByDestination(files); + expect(result.map(file => file.name)).toEqual(['data.csv', 'other.csv']); + expect(result[0].id).toBe('third'); + expect(result[0].storage_session_id).toBe('latest-uploads'); + }); + + test('keeps a by-reference program in the entrypoint position when its upload is refreshed', () => { + const files: TFile[] = [ + { name: 'run.py', id: 'old-script', storage_session_id: 'uploads' }, + { name: 'data.csv', id: 'data', storage_session_id: 'uploads' }, + { name: 'run.py', id: 'new-script', storage_session_id: 'uploads' }, + ]; + const result = deduplicateFilesByDestination(files); + expect(result.map(file => file.name)).toEqual(['run.py', 'data.csv']); + expect(result[0].id).toBe('new-script'); + expect(() => validateExecuteFiles(result)).not.toThrow(); + }); + + test('rejects an upload that targets inline source, regardless of file order', () => { + const source: TFile = { name: 'main.py', content: 'print("submitted program")' }; + const upload: TFile = { name: 'main.py', id: 'uploaded-source', storage_session_id: 'uploads' }; + for (const files of [[source, upload], [upload, source]]) { + const message = messageOf(() => deduplicateFilesByDestination(files)); + expect(message).toContain('duplicate destination "main.py"'); + expect(message).toContain('inline content'); + } + expect(messageOf(() => deduplicateFilesByDestination([ + { content: 'unnamed source' } as TFile, + { name: 'file0.code', id: 'uploaded-source', storage_session_id: 'uploads' }, + ]))).toContain('duplicate destination "file0.code"'); + expect(messageOf(() => deduplicateFilesByDestination([ + source, { name: 'main.py', content: 'different source' }, + ]))).toContain('inline content'); + }); + + test('deduplicateFilesByDestination returns the same array when there are no duplicates', () => { + const files: TFile[] = [ + { name: 'a.csv', content: 'a' }, + { name: 'b.csv', content: 'b' }, + ]; + const result = deduplicateFilesByDestination(files); + expect(result).toEqual(files); + }); + + test('preserves the original destination of an unnamed file reference after deduplication', () => { + const files: TFile[] = [ + { name: 'main.py', content: 'print(1)' }, + { name: 'data.csv', id: 'old', storage_session_id: 'storage-session' }, + { name: 'data.csv', id: 'new', storage_session_id: 'storage-session' }, + { id: 'file-ref', storage_session_id: 'storage-session' } as TFile, + ]; + const deduped = deduplicateFilesByDestination(files); + expect(deduped.map(file => file.name)).toEqual(['main.py', 'data.csv', 'file3.code']); + expect(deduped[1].id).toBe('new'); + const signedDestination = collectExecuteRequestInputFiles({ files }) + .find(file => file.id === 'file-ref'); + expect(collectExecuteRequestInputFiles({ files: deduped }) + .find(file => file.id === 'file-ref')).toEqual(signedDestination); + expect(() => validateExecuteFiles(deduped)).not.toThrow(); + }); + + test('rejects malformed files even if another entry owns their destination', () => { + expect(messageOf(() => deduplicateFilesByDestination([ + { name: 'file1.code', content: 'source' }, + null as unknown as TFile, + ]))).toContain('files[1] must be an object'); + expect(messageOf(() => deduplicateFilesByDestination([ + { name: 'data.csv', content: 'old', encoding: 'invalid' as TFile['encoding'] }, + { name: 'data.csv', content: 'new' }, + ]))).toContain('files[0].encoding'); + expect(messageOf(() => deduplicateFilesByDestination([ + { name: 'data.csv', content: 'old' }, + { name: 'data.csv', id: 'file-ref' }, + ]))).toContain('files[1].storage_session_id'); + }); + + test('caps raw input count before dropping duplicates', () => { + const files = Array.from({ length: config.max_input_files + 1 }, () => ({ + name: 'data.csv', content: 'duplicate', + })); + expect(messageOf(() => deduplicateFilesByDestination(files))).toContain('cannot contain more than'); + }); + + test('deduplicateFilesByDestination allows validateExecuteFiles to accept duplicate uploads', () => { + const files: TFile[] = [ + { name: 'data.csv', id: 'first', storage_session_id: 'uploads' }, + { name: 'data.csv', id: 'second', storage_session_id: 'uploads' }, + ]; + const deduped = deduplicateFilesByDestination(files); + expect(deduped[0].id).toBe('second'); + expect(() => validateExecuteFiles(deduped)).not.toThrow(); + }); }); diff --git a/api/src/api/v2-session-binding.test.ts b/api/src/api/v2-session-binding.test.ts index 32a245f6..0987a68a 100644 --- a/api/src/api/v2-session-binding.test.ts +++ b/api/src/api/v2-session-binding.test.ts @@ -281,6 +281,92 @@ describe('per-request session binding', () => { } }); + test('uses the latest upload without renumbering later unnamed files', async () => { + config.session_workspace_enabled = false; + config.require_execution_manifest = false; + + const originalPrime = Job.prototype.prime; + const originalExecute = Job.prototype.execute; + const originalCleanup = Job.prototype.cleanup; + + let primedFiles: Array<{ name: string; id?: string; content?: string }> = []; + Job.prototype.prime = async function captureFiles(): Promise { + primedFiles = this.files; + }; + Job.prototype.execute = async function executeWithoutSandbox() { + return {} as Awaited>; + }; + Job.prototype.cleanup = async function cleanupWithoutFilesystem(): Promise {}; + + try { + const response = await fetch(`${baseUrl}/api/v2/execute`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + language: testLanguage, + version: testVersion, + files: [ + { name: 'main.txt', content: 'source' }, + { name: 'data.csv', id: 'old-upload', storage_session_id: 'uploads' }, + { name: 'data.csv', id: 'new-upload', storage_session_id: 'uploads' }, + { content: 'unnamed source' }, + ], + }), + }); + + expect(response.status).toBe(200); + expect(primedFiles.map(file => file.name)).toEqual(['main.txt', 'data.csv', 'file3.code']); + expect(primedFiles[0].content).toBe('source'); + expect(primedFiles[1].id).toBe('new-upload'); + expect(primedFiles[2].content).toBe('unnamed source'); + } finally { + Job.prototype.prime = originalPrime; + Job.prototype.execute = originalExecute; + Job.prototype.cleanup = originalCleanup; + } + }); + + test('rejects an upload targeting the submitted source before binding or priming', async () => { + config.session_workspace_enabled = true; + config.require_execution_manifest = false; + + const originalPrime = Job.prototype.prime; + const originalExecute = Job.prototype.execute; + const originalCleanup = Job.prototype.cleanup; + let primed = false; + Job.prototype.prime = async function trackPrime(): Promise { primed = true; }; + Job.prototype.execute = async function executeWithoutSandbox() { + return {} as Awaited>; + }; + Job.prototype.cleanup = async function cleanupWithoutFilesystem(): Promise {}; + + try { + const response = await fetch(`${baseUrl}/api/v2/execute`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'X-Runtime-Session-Id': 'rt_source_collision' }, + body: JSON.stringify({ + language: testLanguage, + version: testVersion, + files: [ + { name: 'main.txt', content: 'submitted program' }, + { name: 'data.csv', id: 'data', storage_session_id: 'uploads' }, + { name: 'main.txt', id: 'uploaded-program', storage_session_id: 'uploads' }, + ], + }), + }); + + expect(response.status).toBe(400); + expect((await response.json() as { message: string }).message) + .toContain('duplicate destination "main.txt"'); + expect(primed).toBe(false); + expect(getBoundSessionWorkspace()).toBeUndefined(); + } finally { + Job.prototype.prime = originalPrime; + Job.prototype.execute = originalExecute; + Job.prototype.cleanup = originalCleanup; + } + }); + test('a post-prime failure still reports the workspace as dirty', async () => { config.session_workspace_enabled = true; config.require_execution_manifest = false; diff --git a/api/src/api/v2.ts b/api/src/api/v2.ts index 121cf2e3..3ace08cf 100644 --- a/api/src/api/v2.ts +++ b/api/src/api/v2.ts @@ -46,72 +46,111 @@ import { const router = express.Router(); const SYNTHETIC_PRINCIPAL_SOURCE = 'synthetic_test'; +/** + * Deduplicate files by destination name. Some callers (e.g. LibreChat) + * send the same file more than once per request when a user re-uploads a + * file in the same conversation. The sandbox rejects duplicate destinations, + * which surfaces to the end user as a confusing "sandbox is down" error. + * Keeps the latest uploaded file at each destination without moving its + * original position (the first runnable file is the program entrypoint). + * Never replace inline source code with an uploaded file. + */ +export function deduplicateFilesByDestination(files: TFile[]): TFile[] { + if (files.length > config.max_input_files) { + throw { message: `files cannot contain more than ${config.max_input_files} destinations` }; + } + const byDestination = new Map(); + for (const [i, file] of files.entries()) { + // Validate even entries that would otherwise be replaced by a later upload. + const destination = validateExecuteFile(file, i); + const previous = byDestination.get(destination); + if (previous && (typeof previous.content === 'string' || typeof file.content === 'string')) { + throw { message: `files contains duplicate destination "${destination}" involving inline content` }; + } + // Updating a Map value preserves its first position for Job.execute's entrypoint. + // Manifest claims use original indices; Job would otherwise renumber unnamed files. + byDestination.set(destination, file.name ? file : { ...file, name: destination }); + } + if (byDestination.size < files.length) { + logger.warn( + { original: files.length, deduped: byDestination.size }, + 'Deduplicated file list before validation', + ); + } + return [...byDestination.values()]; +} + function existingDestinationConflictMessage(existing: string, destination: string): string { return existing === destination ? `files contains duplicate destination "${destination}"` : `files contains conflicting destinations "${existing}" and "${destination}"`; } +function validateExecuteFile(value: TFile, i: number): string { + if (value == null || typeof value !== 'object' || Array.isArray(value)) { + throw { message: `files[${i}] must be an object` }; + } + const file = value as TFile; + const inline = typeof file.content === 'string'; + const byRef = typeof file.id === 'string' && file.id.length > 0; + if (inline === byRef) { + throw { + message: `files[${i}] must contain exactly one of non-empty id or string content`, + }; + } + if (file.id !== undefined && !byRef) { + throw { message: `files[${i}].id must be a non-empty string if provided` }; + } + if (byRef) { + if (typeof file.storage_session_id !== 'string' || file.storage_session_id.length === 0) { + throw { message: `files[${i}].storage_session_id is required as a non-empty string for file refs` }; + } + } else if (file.storage_session_id !== undefined || file.input_cache_key !== undefined) { + throw { + message: `files[${i}] inline content cannot include storage_session_id or input_cache_key`, + }; + } + if (file.name !== undefined && typeof file.name !== 'string') { + throw { message: `files[${i}].name must be a string if provided` }; + } + if ( + file.encoding !== undefined + && !(['base64', 'hex', 'utf8'] as const).includes(file.encoding) + ) { + throw { message: `files[${i}].encoding must be base64, hex, or utf8 if provided` }; + } + if (file.entity_id !== undefined && typeof file.entity_id !== 'string') { + throw { message: `files[${i}].entity_id must be a string if provided` }; + } + if ( + file.input_cache_key !== undefined && + ( + typeof file.input_cache_key !== 'string' || + !/^[0-9a-f]{64}$/.test(file.input_cache_key) + ) + ) { + throw { message: `files[${i}].input_cache_key must be a 64-character lowercase hex digest` }; + } + const destination = file.name || `file${i}.code`; + try { + validateFilePath(destination, '/tmp/codeapi-request-validation'); + } catch (error) { + throw { + message: error instanceof Error + ? `files[${i}].name is invalid: ${error.message}` + : `files[${i}].name is invalid`, + }; + } + return destination; +} + export function validateExecuteFiles(files: TFile[]): void { if (files.length > config.max_input_files) { throw { message: `files cannot contain more than ${config.max_input_files} destinations` }; } const destinations = new Set(); for (const [i, value] of files.entries()) { - if (value == null || typeof value !== 'object' || Array.isArray(value)) { - throw { message: `files[${i}] must be an object` }; - } - const file = value as TFile; - const inline = typeof file.content === 'string'; - const byRef = typeof file.id === 'string' && file.id.length > 0; - if (inline === byRef) { - throw { - message: `files[${i}] must contain exactly one of non-empty id or string content`, - }; - } - if (file.id !== undefined && !byRef) { - throw { message: `files[${i}].id must be a non-empty string if provided` }; - } - if (byRef) { - if (typeof file.storage_session_id !== 'string' || file.storage_session_id.length === 0) { - throw { message: `files[${i}].storage_session_id is required as a non-empty string for file refs` }; - } - } else if (file.storage_session_id !== undefined || file.input_cache_key !== undefined) { - throw { - message: `files[${i}] inline content cannot include storage_session_id or input_cache_key`, - }; - } - if (file.name !== undefined && typeof file.name !== 'string') { - throw { message: `files[${i}].name must be a string if provided` }; - } - if ( - file.encoding !== undefined - && !(['base64', 'hex', 'utf8'] as const).includes(file.encoding) - ) { - throw { message: `files[${i}].encoding must be base64, hex, or utf8 if provided` }; - } - if (file.entity_id !== undefined && typeof file.entity_id !== 'string') { - throw { message: `files[${i}].entity_id must be a string if provided` }; - } - if ( - file.input_cache_key !== undefined && - ( - typeof file.input_cache_key !== 'string' || - !/^[0-9a-f]{64}$/.test(file.input_cache_key) - ) - ) { - throw { message: `files[${i}].input_cache_key must be a 64-character lowercase hex digest` }; - } - const destination = file.name || `file${i}.code`; - try { - validateFilePath(destination, '/tmp/codeapi-request-validation'); - } catch (error) { - throw { - message: error instanceof Error - ? `files[${i}].name is invalid: ${error.message}` - : `files[${i}].name is invalid`, - }; - } + const destination = validateExecuteFile(value, i); const conflict = [...destinations].find( existing => existing === destination || @@ -251,12 +290,13 @@ function getJob( runtimeSessionHeader?: string | string[], ): Job { const { - session_id, language, version, args, stdin, files, + session_id, language, version, args, stdin, files: rawFiles, compile_memory_limit, run_memory_limit, compile_timeout, run_cpu_time, compile_cpu_time, env_vars, } = body; + let files = rawFiles; if (!language || typeof language !== 'string') { throw { message: 'language is required as a string' }; @@ -271,6 +311,7 @@ function getJob( throw { message: 'tool_call_socket must be a boolean if specified' }; } validateExecuteArguments(args, stdin); + files = deduplicateFilesByDestination(files); validateExecuteFiles(files); const rt = getLatestRuntimeMatchingLanguageVersion(language, version); From 1634027e185ca2d558327c02335d8e827059a550 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 29 Sep 2026 08:13:45 -0400 Subject: [PATCH 14/48] =?UTF-8?q?=F0=9F=9B=A4=EF=B8=8F=20feat:=20Schedule?= =?UTF-8?q?=20Linked=20Worktrees=20as=20Their=20Own=20Workspace=20Lanes=20?= =?UTF-8?q?(#270)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Schedule Linked Worktrees as Their Own Workspace Lanes * fix: Refuse quarantined-parent lanes before enqueue, forward lane policy, reset lane fences - Check the parent checkout fence before the assignment is stored or queued. - Forward the linked worktree policy to the forked native sandbox. - Protect shared Git config, hooks and info even before they exist. - Reset a lane fence with --reset-workspace-worktree. * fix: Grant lanes an explicit Git write allowlist, hold checkouts on quarantined lanes, release stale lanes - A lane writes only shared objects, refs, ref logs, LFS storage and its own worktree metadata; the checkout HEAD, index, operation state, config and hooks stay read-only without per-path denies. - A checkout assignment waits on any quarantined lane beneath it. - Lane registrations are released when their worktree disappears and capped at 32, dropping command roots and credential routes. * fix: Refuse a checkout while a lane keeps a stuck fence; let lane probes read shared Git - Code API indexes the lane fences enqueued beneath each checkout. A checkout reaches enqueue only once no lane holds a slot, so an indexed fence that still exists is stuck and the checkout is refused, surviving worker restarts. - Replay probes of a lane may read its common Git directory (read-only). * fix: Guard linked-worktree Git grants and unresolved actions --------- Co-authored-by: Lia --- docs/remote-bridge/README.md | 5 +- docs/remote-bridge/projects.md | 6 +- packages/code/README.md | 56 +++ packages/code/src/cli.ts | 106 +++++- packages/code/src/linked-worktrees.test.ts | 294 ++++++++++++++++ packages/code/src/linked-worktrees.ts | 386 +++++++++++++++++++++ packages/code/src/native-process.test.ts | 16 + packages/code/src/native-process.ts | 2 + packages/code/src/native-sandbox.test.ts | 61 ++++ packages/code/src/native-sandbox.ts | 77 +++- packages/code/src/protocol.test.ts | 61 ++++ packages/code/src/protocol.ts | 79 ++++- packages/code/src/worker-slots.test.ts | 201 +++++++++++ packages/code/src/worker.ts | 175 +++++++++- packages/code/src/workspace-cli.test.ts | 41 +++ service/src/bridge/linked-worktree.test.ts | 197 +++++++++++ service/src/bridge/router.ts | 1 + service/src/bridge/slots.test.ts | 62 +++- service/src/bridge/slots.ts | 38 +- service/src/bridge/store.ts | 83 ++++- service/src/service/programmatic-router.ts | 20 +- service/src/service/programmatic-state.ts | 21 +- service/src/service/replay-state.ts | 2 + service/src/types/service.ts | 4 + 24 files changed, 1944 insertions(+), 50 deletions(-) create mode 100644 packages/code/src/linked-worktrees.test.ts create mode 100644 packages/code/src/linked-worktrees.ts create mode 100644 service/src/bridge/linked-worktree.test.ts diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index 464e4763..a37745a2 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -307,7 +307,10 @@ execution. - Code API negotiates a bounded number of active workspace assignments per worker. The lower API or worker slot ceiling wins, and assignments sharing the same workspace isolation key remain serialized while independent - conversation worktrees may run concurrently. + conversation worktrees may run concurrently. A linked-worktree lane + (`worktree: `) nests beneath its checkout's key: sibling lanes run + concurrently, while a lane and its checkout exclude each other, and a lane + cannot start while its checkout is quarantined. - Workspace tool admission waits for capacity up to 30 seconds without a `X-LibreChat-Workspace-Queue-Wait-Ms` header. A caller can advertise a longer per-request allowance, bounded by five minutes and any server queue ceiling. diff --git a/docs/remote-bridge/projects.md b/docs/remote-bridge/projects.md index 987ce112..26887ee9 100644 --- a/docs/remote-bridge/projects.md +++ b/docs/remote-bridge/projects.md @@ -41,7 +41,11 @@ workspace registration remains available for independent project directories. filesystem boundary and coordination for the common Git directory. - A worktree beneath its parent checkout overlaps that checkout. Either use disjoint execution roots under a discovery grant or explicitly exclude and - coordinate descendant worktrees before relaxing root exclusion. + coordinate descendant worktrees before relaxing root exclusion. Linked + worktree lanes (`--linked-worktree-lanes`) take the second approach for + `.worktrees/`: hierarchical admission keeps a lane and its checkout + exclusive, and each lane's sandbox can write only its worktree, shared + object and ref storage, and its own worktree metadata. - Setup and dependency links must remain within the execution policy. Sharing writable dependency directories between supposedly isolated worktrees reintroduces overlap and requires an explicit operator decision. diff --git a/packages/code/README.md b/packages/code/README.md index 95d86485..e3051930 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -791,6 +791,57 @@ Legacy requests without a conversation identity continue to use the selected source root. Older Code API deployments do not negotiate the capability, so the worker omits it until every request path understands the isolation boundary. +#### Linked worktree lanes + +Agents that keep one checkout and give each task its own linked worktree +(`git worktree add .worktrees/`) can run those tasks concurrently: + +```sh +librechat-code run \ + --worker-dir /projects/LibreChat \ + --workspace-lease-slots 4 \ + --linked-worktree-lanes \ + --allow-workspace-writes \ + --allow-workspace-commands +``` + +`LIBRECHAT_CODE_LINKED_WORKTREE_LANES=true` is the environment equivalent. The +worker then advertises `workspaceScopes: ['git_linked_worktree']` for each +registered root, and a request that names `worktree: ` runs in its own +lane at `/.worktrees/`, with `cwd` and file paths relative to that +worktree. Sibling lanes run concurrently up to the negotiated slot count. A +lane and its checkout never run at the same time: requests without a +`worktree`, including `git worktree add` or `remove` run at the root, wait for +every lane beneath the checkout, and a waiting root request holds back newer +lanes so it cannot be starved. + +Before admission the worker verifies, without running Git, that the directory +is a real linked worktree of that checkout: no symlinks on the path, a `.git` +file pointing at `/.git/worktrees/`, and metadata whose `commondir` +and `gitdir` point back. Shared Git storage paths granted for writes must be +real directories, not symlinks into sibling metadata; optional log and LFS +paths may be absent. A lane's sandbox can write only its worktree, the +shared object and ref storage (`.git/objects`, `.git/refs`, `.git/logs/refs`, +`.git/lfs`) and its own `.git/worktrees/` metadata. Everything else in +`.git` stays read-only: configuration, hooks and `info`, the checkout's own +`HEAD`, index and merge or rebase state, and sibling metadata. Automatic `gc` +and maintenance are disabled, and `git gc` itself cannot run in a lane (it +needs to write `.git/gc.pid` and `packed-refs`); run storage maintenance from +the checkout. Explicit `git prune --expire now`, `git repack -ad`, or LFS +pruning can still delete objects another lane is writing; the sandbox does +**not** prevent this race. Do not enable lanes for agents that run destructive +maintenance until those commands are blocked or serialized. Deleting a branch +or tag also needs the checkout, because Git locks `packed-refs` for every ref +deletion. Each lane has its own durable quarantine guard. A lane +cannot start while its checkout is quarantined, and a checkout cannot start +while any lane beneath it is. + +Lanes require native-srt commands and at least two lease slots, and cannot yet +be combined with conversation worktrees. Code API must advertise +`supportedWorkspaceScopes`; older deployments do not, and the worker omits the +scope for them. Deploy consumers that read worker status (such as LibreChat) +with support for `workspaceScopes` before enabling lanes on a worker. + On an updated Code API, admission waits up to 30 seconds without the `X-LibreChat-Workspace-Queue-Wait-Ms` request header. A caller may advertise a positive integer millisecond allowance up to five minutes, capped by any server @@ -850,6 +901,11 @@ To recover a quarantined native root: 3. Run the normal worker command with all its root/slot options plus `--reset-workspace-quarantine second`. This verifies the local guard is cleared, resets the server fence, then exits. 4. Restart the normal worker command without the reset option. +A linked-worktree lane keeps its own guard and fence. Inspect or restore +`/.worktrees/`, clear its guard with +`--worker-dir /.worktrees/`, then add +`--reset-workspace-worktree ` to the reset command in step 3. + The workspace selector in LibreChat must preserve these registered IDs. Adding roots here does not grant a principal access or change an agent's selected root. # Named project environments diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 6e3a0062..135b9f8e 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -1,8 +1,8 @@ #!/usr/bin/env node import { createHash, createHmac, randomBytes } from 'node:crypto'; import { readFileSync } from 'node:fs'; -import { realpath, stat } from 'node:fs/promises'; -import { basename, resolve, relative, isAbsolute, sep } from 'node:path'; +import { readdir, realpath, stat } from 'node:fs/promises'; +import { basename, join, resolve, relative, isAbsolute, sep } from 'node:path'; import { pairBridgeWorker } from './pairing.js'; import { discoverProjects } from './projects.js'; @@ -37,6 +37,7 @@ import { RuntimeWorkspaceCommandSandbox } from './workspace-runtime.js'; import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; import { NativeWorkspaceCommandPool } from './native-pool.js'; import { GitWorktreeWorkspaceTools, internalWorkspaceId } from './workspace-instances.js'; +import { LINKED_WORKTREE_DIRECTORY, LinkedWorktreeWorkspaceTools } from './linked-worktrees.js'; import { GitWorktreeManager } from './worktrees.js'; import { captureWorkspaceRootIdentity } from './root-identity.js'; import { @@ -66,6 +67,7 @@ import { BridgeProtocolError, isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, + isValidLinkedWorktreeName, workspaceIsolationKey, } from './protocol.js'; @@ -514,6 +516,11 @@ async function run( (args.includes('--allow-workspace-commands') || process.env.LIBRECHAT_CODE_ALLOW_WORKSPACE_COMMANDS?.trim().toLowerCase() === 'true'); + const linkedWorktreeLanes = + runtimeSessionId == null && + (args.includes('--linked-worktree-lanes') || + process.env.LIBRECHAT_CODE_LINKED_WORKTREE_LANES?.trim().toLowerCase() === + 'true'); const commandSandboxMode = option(args, '--command-sandbox') ?? process.env.LIBRECHAT_CODE_COMMAND_SANDBOX?.trim().toLowerCase() ?? @@ -783,6 +790,21 @@ async function run( 'Conversation worktrees require native-srt commands and at least two workspace lease slots', ); } + if ( + linkedWorktreeLanes && + (!allowWorkspaceCommands || + commandSandboxMode !== 'native-srt' || + workspaceLeaseSlots < 2) + ) { + throw new Error( + 'Linked worktree lanes require native-srt commands and at least two workspace lease slots', + ); + } + if (linkedWorktreeLanes && conversationWorktreeRoot) { + throw new Error( + 'Linked worktree lanes cannot be combined with conversation worktrees', + ); + } if ( roots.length > 1 && process.env.LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE?.trim() @@ -1073,7 +1095,7 @@ async function run( }; const nativeCommandSandbox = allowWorkspaceCommands && commandSandboxMode === 'native-srt' - ? roots.length > 1 || workspaceLeaseSlots > 1 || conversationWorktreeRoot + ? roots.length > 1 || workspaceLeaseSlots > 1 || conversationWorktreeRoot || linkedWorktreeLanes ? new NativeWorkspaceCommandPool( new Map( roots.map(root => [ @@ -1199,6 +1221,41 @@ async function run( ), }); workspaceTools = conversationWorkspaceTools; + } + let linkedWorktreeTools: LinkedWorktreeWorkspaceTools | undefined; + if (linkedWorktreeLanes && workspaceTools) { + if (!(nativeCommandSandbox instanceof NativeWorkspaceCommandPool)) { + throw new Error('Linked worktree lanes require a native command pool'); + } + linkedWorktreeTools = new LinkedWorktreeWorkspaceTools({ + commandPool: nativeCommandSandbox, + delegate: workspaceTools, + programmaticDelegate: conversationWorkspaceTools ?? nativeCommandSandbox, + onResolve(workspaceId, root) { + if (admittedGitHubRepositories) { + admittedGitHubRepositories.set( + root, + repositoriesByWorkspace?.get(workspaceId), + ); + } + }, + onRelease(root) { + admittedGitHubRepositories?.delete(root); + }, + sources: new Map( + roots.map((root) => [ + root.id, + { + root: root.root, + identity: root.identity, + command: nativeOptions, + repositoryInstructions: args.includes('--repository-instructions'), + writable: root.writable ?? false, + }, + ]), + ), + }); + workspaceTools = linkedWorktreeTools; } if (workspaceTools && environments.length) { workspaceTools = new EnvironmentWorkspaceTools( @@ -1304,7 +1361,7 @@ async function run( ...(nativeProgrammaticEnabled && nativeCommandSandbox ? { workspaceProgrammatic: - conversationWorkspaceTools ?? nativeCommandSandbox, + linkedWorktreeTools ?? conversationWorkspaceTools ?? nativeCommandSandbox, } : {}), ...(conversationWorktrees @@ -1331,6 +1388,40 @@ async function run( ), } : {}), + ...(linkedWorktreeTools + ? { + linkedWorktreeQuarantineResolver: ( + selectedWorkspaceId: string, + worktree: string, + ) => { + const source = roots.find((root) => root.id === selectedWorkspaceId); + if (!source) { + throw new BridgeProtocolError('Linked worktree source is not registered'); + } + return workspaceMutationGuard( + defaultWorkspaceQuarantinePath({ + codeApiUrl, + workerId, + workspaceRoot: join(source.root, LINKED_WORKTREE_DIRECTORY, worktree), + }), + workerId, + workspaceIsolationKey(selectedWorkspaceId, undefined, worktree), + incarnationId, + ); + }, + linkedWorktreeNames: async (selectedWorkspaceId: string) => { + const source = roots.find((root) => root.id === selectedWorkspaceId); + if (!source) return []; + try { + const entries = await readdir(join(source.root, LINKED_WORKTREE_DIRECTORY)); + return entries.filter(isValidLinkedWorktreeName); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; + throw error; + } + }, + } + : {}), ...(workspaceLeaseSlots > 1 || roots.length > 1 ? { workspaceQuarantines: new Map( @@ -1445,15 +1536,20 @@ async function run( args, '--reset-workspace-instance', ); + const resetWorkspaceWorktree = option( + args, + '--reset-workspace-worktree', + ); await worker.refreshCredential(controller.signal); await worker.registerForMaintenance(controller.signal); await worker.resetNativeWorkspace( resetNativeRoot, controller.signal, resetWorkspaceInstance, + resetWorkspaceWorktree, ); process.stdout.write( - `librechat-code: reset acknowledged for native workspace ${resetNativeRoot}${resetWorkspaceInstance ? ` instance ${resetWorkspaceInstance}` : ''}\n`, + `librechat-code: reset acknowledged for native workspace ${resetNativeRoot}${resetWorkspaceInstance ? ` instance ${resetWorkspaceInstance}` : ''}${resetWorkspaceWorktree ? ` worktree ${resetWorkspaceWorktree}` : ''}\n`, ); return; } diff --git a/packages/code/src/linked-worktrees.test.ts b/packages/code/src/linked-worktrees.test.ts new file mode 100644 index 00000000..53ec156e --- /dev/null +++ b/packages/code/src/linked-worktrees.test.ts @@ -0,0 +1,294 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { + mkdir, + mkdtemp, + readFile, + realpath, + rm, + stat, + symlink, + writeFile, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import test from 'node:test'; + +import { + LinkedWorktreeWorkspaceTools, + linkedWorktreeWorkspaceId, + verifyLinkedWorktree, +} from './linked-worktrees.js'; +import { LocalWorkspaceTools, WorkspaceToolError } from './workspace.js'; + +import type { NativeWorkspaceCommandPool } from './native-pool.js'; +import type { NativeProcessSandboxOptions } from './native-process.js'; +import type { WorkspaceToolRequest } from './protocol.js'; + +const execFileAsync = promisify(execFile); + +async function git(cwd: string, ...args: string[]): Promise { + await execFileAsync( + 'git', + ['-c', 'user.name=test', '-c', 'user.email=test@example.com', '-c', 'commit.gpgsign=false', ...args], + { cwd }, + ); +} + +async function checkout(): Promise<{ parent: string; root: string }> { + const parent = await realpath(await mkdtemp(join(tmpdir(), 'linked-worktree-'))); + const root = join(parent, 'repo'); + await mkdir(root); + await git(root, 'init', '-q', '-b', 'main'); + await writeFile(join(root, 'README.md'), 'root\n'); + await git(root, 'add', '.'); + await git(root, 'commit', '-q', '-m', 'init'); + await mkdir(join(root, '.worktrees')); + await git(root, 'worktree', 'add', '-q', '-b', 'task-a', '.worktrees/task-a'); + return { parent, root }; +} + +async function rejects(promise: Promise, code = 'INVALID_REQUEST'): Promise { + await assert.rejects(promise, (error: unknown) => error instanceof WorkspaceToolError && error.code === code); +} + +test('verifies a linked worktree of the checkout and lists the only Git paths it may write', async (t) => { + const { parent, root } = await checkout(); + t.after(() => rm(parent, { recursive: true, force: true })); + await git(root, 'worktree', 'add', '-q', '-b', 'task-b', '.worktrees/task-b'); + + const lane = await verifyLinkedWorktree(root, 'task-a'); + + assert.equal(lane.root, join(root, '.worktrees', 'task-a')); + assert.equal(lane.checkoutRoot, root); + assert.equal(lane.commonGitDir, join(root, '.git')); + assert.deepEqual(lane.writableGitPaths, [ + join(root, '.git', 'objects'), + join(root, '.git', 'refs'), + join(root, '.git', 'logs', 'refs'), + join(root, '.git', 'lfs'), + join(root, '.git', 'worktrees', 'task-a'), + ]); +}); + + +test('rejects directories that are not linked worktrees of this checkout', async (t) => { + const { parent, root } = await checkout(); + t.after(() => rm(parent, { recursive: true, force: true })); + + await mkdir(join(root, '.worktrees', 'plain')); + await rejects(verifyLinkedWorktree(root, 'plain')); + + await mkdir(join(root, '.worktrees', 'borrowed')); + await writeFile( + join(root, '.worktrees', 'borrowed', '.git'), + `gitdir: ${join(root, '.git', 'worktrees', 'task-a')}\n`, + ); + await rejects(verifyLinkedWorktree(root, 'borrowed')); + + await symlink(join(root, '.worktrees', 'task-a'), join(root, '.worktrees', 'alias')); + await rejects(verifyLinkedWorktree(root, 'alias')); + + const other = join(parent, 'other'); + await mkdir(other); + await git(other, 'init', '-q', '-b', 'main'); + await writeFile(join(other, 'file'), 'x\n'); + await git(other, 'add', '.'); + await git(other, 'commit', '-q', '-m', 'other'); + await git(other, 'worktree', 'add', '-q', '-b', 'foreign', join(root, '.worktrees', 'foreign')); + await rejects(verifyLinkedWorktree(root, 'foreign')); + + for (const name of ['..', '.git', 'a/b', 'task.lock']) { + await rejects(verifyLinkedWorktree(root, name)); + } + await rejects(verifyLinkedWorktree(root, 'missing')); +}); + +test('rejects a checkout whose .worktrees directory is a symlink', async (t) => { + const { parent, root } = await checkout(); + t.after(() => rm(parent, { recursive: true, force: true })); + const elsewhere = join(parent, 'elsewhere'); + await git(root, 'worktree', 'move', '.worktrees/task-a', elsewhere); + await rm(join(root, '.worktrees'), { recursive: true }); + await symlink(parent, join(root, '.worktrees')); + await rejects(verifyLinkedWorktree(root, 'elsewhere')); +}); + +test('rejects shared Git storage symlinks into sibling worktree metadata', async (t) => { + const { parent, root } = await checkout(); + t.after(() => rm(parent, { recursive: true, force: true })); + await git(root, 'worktree', 'add', '-q', '-b', 'task-b', '.worktrees/task-b'); + const commonGitDir = join(root, '.git'); + const siblingMetadata = join(commonGitDir, 'worktrees', 'task-b'); + await rm(join(commonGitDir, 'objects'), { recursive: true }); + await symlink(siblingMetadata, join(commonGitDir, 'objects')); + + await rejects(verifyLinkedWorktree(root, 'task-a')); +}); + +function recordingPool(): { + pool: NativeWorkspaceCommandPool; + calls: Array<{ action: string; id: string; options?: NativeProcessSandboxOptions }>; +} { + const calls: Array<{ action: string; id: string; options?: NativeProcessSandboxOptions }> = []; + const pool = { + async registerRoot(id: string, options: NativeProcessSandboxOptions) { + calls.push({ action: 'register', id, options }); + }, + async unregisterRoot(id: string) { + calls.push({ action: 'unregister', id }); + }, + async execute(request: WorkspaceToolRequest) { + calls.push({ action: 'execute', id: request.workspaceId }); + return { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: request.workspaceId, + exitCode: 0, + stdout: '', + stderr: '', + truncated: false, + timedOut: false, + }; + }, + async executeProgrammatic(id: string) { + calls.push({ action: 'programmatic', id }); + return {}; + }, + } as unknown as NativeWorkspaceCommandPool; + return { pool, calls }; +} + +async function laneTools( + root: string, + pool?: NativeWorkspaceCommandPool, + onRelease?: (root: string) => void, +) { + const delegate = await LocalWorkspaceTools.create({ + repositoryInstructions: false, + workspaces: [{ id: 'repo', root, writable: true }], + }); + return new LinkedWorktreeWorkspaceTools({ + commandPool: pool, + delegate, + onRelease, + sources: new Map([ + [ + 'repo', + { + root, + command: { workspaceRoot: root } as NativeProcessSandboxOptions, + repositoryInstructions: false, + writable: true, + }, + ], + ]), + }); +} + +test('lane file tools are confined to the worktree and report the public workspace', async (t) => { + const { parent, root } = await checkout(); + t.after(() => rm(parent, { recursive: true, force: true })); + const tools = await laneTools(root); + + assert.deepEqual(tools.capabilities.workspaces[0]?.workspaceScopes, ['git_linked_worktree']); + const written = await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'repo', + worktree: 'task-a', + path: 'lane.txt', + content: 'from the lane\n', + }); + assert.equal(written.workspaceId, 'repo'); + assert.equal(await readFile(join(root, '.worktrees', 'task-a', 'lane.txt'), 'utf8'), 'from the lane\n'); + await assert.rejects(stat(join(root, 'lane.txt'))); + + const read = await tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'repo', + path: 'README.md', + }); + assert.equal(read.operation === 'read_file' && read.content.trimEnd(), 'root'); + + await rejects( + tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'repo', + worktree: 'task-a', + workspaceInstanceId: 'a'.repeat(64), + path: 'README.md', + }), + ); +}); + +test('lane commands register a confined root once, whatever siblings come and go', async (t) => { + const { parent, root } = await checkout(); + t.after(() => rm(parent, { recursive: true, force: true })); + const { pool, calls } = recordingPool(); + const tools = await laneTools(root, pool); + const command = { + protocolVersion: 1 as const, + operation: 'execute_command' as const, + workspaceId: 'repo', + worktree: 'task-a', + command: 'git status', + }; + const id = linkedWorktreeWorkspaceId('repo', 'task-a'); + + const result = await tools.execute(command); + await tools.execute(command); + assert.equal(result.workspaceId, 'repo'); + assert.deepEqual( + calls.map((call) => call.action), + ['register', 'execute', 'execute'], + ); + const registered = calls[0]!.options!; + assert.equal(calls[0]!.id, id); + assert.equal(registered.workspaceRoot, join(root, '.worktrees', 'task-a')); + assert.equal(registered.linkedWorktree?.commonGitDir, join(root, '.git')); + assert.equal(registered.linkedWorktree?.checkoutRoot, root); + + assert.ok(!registered.linkedWorktree?.writableGitPaths.includes(join(root, '.git'))); + + await rejects(tools.execute({ + ...command, + environmentAction: { name: 'unresolved', fingerprint: 'a'.repeat(64) }, + })); + assert.equal(calls.length, 3, 'an unresolved action must never reach the command pool'); + + await git(root, 'worktree', 'add', '-q', '-b', 'task-b', '.worktrees/task-b'); + await tools.execute(command); + assert.deepEqual( + calls.slice(3).map((call) => call.action), + ['execute'], + ); +}); + +test('a removed lane releases its command root and credential route', async (t) => { + const { parent, root } = await checkout(); + t.after(() => rm(parent, { recursive: true, force: true })); + const { pool, calls } = recordingPool(); + const released: string[] = []; + const tools = await laneTools(root, pool, (lane) => released.push(lane)); + const command = { + protocolVersion: 1 as const, + operation: 'execute_command' as const, + workspaceId: 'repo', + worktree: 'task-a', + command: 'git status', + }; + await tools.execute(command); + + await git(root, 'worktree', 'remove', '.worktrees/task-a'); + await rejects(tools.execute(command)); + + assert.deepEqual( + calls.map((call) => call.action), + ['register', 'execute', 'unregister'], + ); + assert.deepEqual(released, [join(root, '.worktrees', 'task-a')]); +}); diff --git a/packages/code/src/linked-worktrees.ts b/packages/code/src/linked-worktrees.ts new file mode 100644 index 00000000..e049aa6e --- /dev/null +++ b/packages/code/src/linked-worktrees.ts @@ -0,0 +1,386 @@ +import { createHash } from 'node:crypto'; +import { lstat, open, realpath } from 'node:fs/promises'; +import { isAbsolute, join, resolve } from 'node:path'; + +import { isValidLinkedWorktreeName } from './protocol.js'; +import { + captureWorkspaceRootIdentity, + matchesWorkspaceRoot, +} from './root-identity.js'; +import { LocalWorkspaceTools, WorkspaceToolError } from './workspace.js'; + +import type { NativeWorkspaceCommandPool } from './native-pool.js'; +import type { NativeProcessSandboxOptions } from './native-process.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; +import type { + BridgeWorkspaceProgrammaticRequest, + WorkspaceExecuteCommandRequest, + WorkspaceToolRequest, + WorkspaceToolResult, +} from './protocol.js'; +import type { WorkspaceToolExecutor } from './workspace.js'; + +/** Linked worktrees are only admitted from this directory beneath a checkout. */ +export const LINKED_WORKTREE_DIRECTORY = '.worktrees'; +/** + * Shared Git storage a lane may write beneath the common Git directory. Every + * other path there (config, hooks, the checkout's own HEAD, index and + * operation state, sibling metadata) stays read-only. + */ +const LINKED_WORKTREE_SHARED_GIT_PATHS = ['objects', 'refs', join('logs', 'refs'), 'lfs']; +/** Lane registrations kept per worker; the least recently used idle lanes are released first. */ +const LINKED_WORKTREE_LANE_LIMIT = 32; +/** Git pointer files are a single line; anything larger is not one. */ +const GIT_POINTER_MAX_BYTES = 4096; + +export interface LinkedWorktreeSource { + root: string; + identity?: WorkspaceRootIdentity; + command?: NativeProcessSandboxOptions; + repositoryInstructions: boolean; + writable: boolean; +} + +export interface VerifiedLinkedWorktree { + root: string; + identity: WorkspaceRootIdentity; + checkoutRoot: string; + commonGitDir: string; + /** Shared object and ref storage plus the lane's own metadata; nothing else in the common Git directory. */ + writableGitPaths: string[]; +} + +export interface LinkedWorktreeWorkspaceToolsOptions { + commandPool?: NativeWorkspaceCommandPool; + delegate: WorkspaceToolExecutor; + /** Called with each verified lane root, e.g. to route credentials for its repository. */ + onResolve?: (workspaceId: string, root: string) => void; + /** Called when a lane root is released, e.g. to drop its credential route. */ + onRelease?: (root: string) => void; + programmaticDelegate?: { + executeProgrammatic( + workspaceId: string, + request: BridgeWorkspaceProgrammaticRequest, + signal?: AbortSignal, + ): Promise; + }; + sources: ReadonlyMap; +} + +export function linkedWorktreeWorkspaceId(workspaceId: string, worktree: string): string { + return `lane-${createHash('sha256').update(`${workspaceId}\0${worktree}`).digest('hex')}`; +} + +function rejected(message: string): WorkspaceToolError { + return new WorkspaceToolError(message, 'INVALID_REQUEST'); +} + +async function realDirectory(path: string): Promise { + try { + const status = await lstat(path); + return status.isDirectory() && !status.isSymbolicLink(); + } catch { + return false; + } +} + +/** Optional Git paths may be absent, but cannot redirect a write grant into sibling metadata. */ +async function safeSharedGitStorage(commonGitDir: string): Promise { + for (const path of ['objects', 'refs', 'logs', join('logs', 'refs'), 'lfs']) { + try { + const status = await lstat(join(commonGitDir, path)); + if (!status.isDirectory() || status.isSymbolicLink()) return false; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT' || + path === 'objects' || path === 'refs') return false; + } + } + return true; +} + +async function readPointer(path: string): Promise { + let handle; + try { + const status = await lstat(path); + if (!status.isFile() || status.isSymbolicLink() || status.size > GIT_POINTER_MAX_BYTES) { + return undefined; + } + handle = await open(path, 'r'); + const buffer = Buffer.alloc(GIT_POINTER_MAX_BYTES + 1); + const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0); + if (bytesRead > GIT_POINTER_MAX_BYTES) return undefined; + return buffer.subarray(0, bytesRead).toString('utf8'); + } catch { + return undefined; + } finally { + await handle?.close().catch(() => undefined); + } +} + +async function canonicalOrUndefined(path: string): Promise { + try { + return await realpath(path); + } catch { + return undefined; + } +} + +function singleLine(value: string | undefined): string | undefined { + const line = value?.replace(/\r?\n$/, ''); + return line == null || line.length === 0 || /[\r\n\0]/.test(line) ? undefined : line; +} + +/** + * Verify, without running Git, that `/.worktrees/` is a linked + * worktree of that checkout: a real directory whose `.git` file points at + * `/.git/worktrees/`, whose metadata points back at it, and + * whose common directory is the checkout's own `.git`. Neither the lane path + * nor a writable shared Git storage directory may be a symlink, so a forged + * worktree cannot borrow a lane or redirect its Git write grants. + */ +export async function verifyLinkedWorktree( + checkoutRoot: string, + name: string, + checkoutIdentity?: WorkspaceRootIdentity, +): Promise { + if (!isValidLinkedWorktreeName(name)) { + throw rejected('Invalid linked worktree name'); + } + const checkout = await canonicalOrUndefined(checkoutRoot); + if (checkout == null || !(await realDirectory(checkout))) { + throw new WorkspaceToolError('Selected project is unavailable', 'REGISTRATION_INVALID'); + } + if (checkoutIdentity != null && !(await matchesWorkspaceRoot(checkout, checkoutIdentity))) { + throw new WorkspaceToolError('Selected project changed before lane admission', 'REGISTRATION_INVALID'); + } + const commonGitDir = join(checkout, '.git'); + const worktreesDirectory = join(checkout, LINKED_WORKTREE_DIRECTORY); + const root = join(worktreesDirectory, name); + const metadata = join(commonGitDir, 'worktrees', name); + if ( + !(await realDirectory(commonGitDir)) || + !(await realDirectory(worktreesDirectory)) || + !(await realDirectory(root)) || + (await canonicalOrUndefined(root)) !== root || + !(await realDirectory(metadata)) || + (await canonicalOrUndefined(metadata)) !== metadata || + !(await safeSharedGitStorage(commonGitDir)) + ) { + throw rejected(`No linked worktree named ${name} in ${LINKED_WORKTREE_DIRECTORY}`); + } + const dotGit = join(root, '.git'); + const pointer = singleLine(await readPointer(dotGit))?.match(/^gitdir: (.+)$/)?.[1]; + const gitDir = pointer == null ? undefined : isAbsolute(pointer) ? pointer : resolve(root, pointer); + const commonPointer = singleLine(await readPointer(join(metadata, 'commondir'))); + const backPointer = singleLine(await readPointer(join(metadata, 'gitdir'))); + if ( + gitDir == null || + (await canonicalOrUndefined(gitDir)) !== metadata || + commonPointer == null || + (await canonicalOrUndefined(resolve(metadata, commonPointer))) !== commonGitDir || + backPointer == null || + (await canonicalOrUndefined(resolve(metadata, backPointer))) !== dotGit + ) { + throw rejected(`${LINKED_WORKTREE_DIRECTORY}/${name} is not a linked worktree of this project`); + } + let identity: WorkspaceRootIdentity; + try { + identity = await captureWorkspaceRootIdentity(root); + } catch { + throw rejected(`${LINKED_WORKTREE_DIRECTORY}/${name} is unavailable`); + } + const writableGitPaths = [ + ...LINKED_WORKTREE_SHARED_GIT_PATHS.map((path) => join(commonGitDir, path)), + metadata, + ]; + return { root, identity, checkoutRoot: checkout, commonGitDir, writableGitPaths }; +} + +function publicResult(result: WorkspaceToolResult, workspaceId: string): WorkspaceToolResult { + return { ...result, workspaceId }; +} + +/** + * Route requests that name a linked worktree into their own isolated executor. + * Code API schedules each `.worktrees/` as its own lane beneath the + * checkout, so file tools and commands here run confined to that worktree while + * sibling lanes run concurrently. Requests without a worktree pass through. + */ +export class LinkedWorktreeWorkspaceTools implements WorkspaceToolExecutor { + readonly mutationFailuresAreAtomic?: true; + readonly capabilities: WorkspaceToolExecutor['capabilities']; + private readonly executors = new Map< + string, + { fingerprint: string; value: Promise } + >(); + private readonly commandRoots = new Map(); + /** Verified lane roots by internal ID, least recently used first. */ + private readonly lanes = new Map(); + + constructor(private readonly options: LinkedWorktreeWorkspaceToolsOptions) { + this.mutationFailuresAreAtomic = options.delegate.mutationFailuresAreAtomic; + this.capabilities = { + ...options.delegate.capabilities, + workspaces: options.delegate.capabilities.workspaces.map((workspace) => ({ + ...workspace, + ...(options.sources.has(workspace.id) + ? { workspaceScopes: ['git_linked_worktree' as const] } + : {}), + })), + }; + } + + private async resolveLane( + workspaceId: string, + worktree: string, + workspaceInstanceId: string | undefined, + ): Promise<{ lane: VerifiedLinkedWorktree; source: LinkedWorktreeSource; internalId: string }> { + if (workspaceInstanceId != null) { + throw rejected('Linked worktree lanes are not available inside conversation worktrees'); + } + const source = this.options.sources.get(workspaceId); + if (!source) { + throw rejected('Workspace does not allow linked worktree lanes'); + } + const internalId = linkedWorktreeWorkspaceId(workspaceId, worktree); + let lane: VerifiedLinkedWorktree; + try { + lane = await verifyLinkedWorktree(source.root, worktree, source.identity); + } catch (error) { + await this.release(internalId); + throw error; + } + this.lanes.delete(internalId); + this.lanes.set(internalId, lane.root); + this.options.onResolve?.(workspaceId, lane.root); + await this.releaseIdleLanes(internalId); + return { lane, source, internalId }; + } + + /** Forget a lane's executors and routes; a lane still running a command is kept. */ + private async release(internalId: string): Promise { + const root = this.lanes.get(internalId); + if (root == null) return; + if (this.commandRoots.has(internalId)) { + try { + await this.options.commandPool?.unregisterRoot(internalId); + } catch { + return; + } + this.commandRoots.delete(internalId); + } + this.executors.delete(internalId); + this.lanes.delete(internalId); + this.options.onRelease?.(root); + } + + private async releaseIdleLanes(current: string): Promise { + for (const internalId of [...this.lanes.keys()]) { + if (this.lanes.size <= LINKED_WORKTREE_LANE_LIMIT) return; + if (internalId !== current) await this.release(internalId); + } + } + + private async fileExecutor( + internalId: string, + lane: VerifiedLinkedWorktree, + source: LinkedWorktreeSource, + ): Promise { + const fingerprint = `${lane.identity.path}\0${lane.identity.dev}\0${lane.identity.ino}`; + let cached = this.executors.get(internalId); + if (cached == null || cached.fingerprint !== fingerprint) { + cached = { + fingerprint, + value: LocalWorkspaceTools.create({ + repositoryInstructions: source.repositoryInstructions, + workspaces: [ + { id: internalId, identity: lane.identity, root: lane.root, writable: source.writable }, + ], + }), + }; + this.executors.set(internalId, cached); + } + return await cached.value; + } + + /** Register the lane's command root, replacing it when its identity or writable Git paths changed. */ + private async registerCommandRoot( + internalId: string, + lane: VerifiedLinkedWorktree, + source: LinkedWorktreeSource, + ): Promise { + const pool = this.options.commandPool; + if (!source.command || !pool) { + throw new WorkspaceToolError('Linked worktree commands are unavailable', 'COMMAND_DISABLED'); + } + const fingerprint = JSON.stringify([ + lane.identity.path, + lane.identity.dev, + lane.identity.ino, + lane.writableGitPaths, + ]); + const registered = this.commandRoots.get(internalId); + if (registered !== fingerprint) { + if (registered != null) await pool.unregisterRoot(internalId); + await pool.registerRoot(internalId, { + ...source.command, + workspaceIdentity: lane.identity, + workspaceRoot: lane.root, + linkedWorktree: { + checkoutRoot: lane.checkoutRoot, + commonGitDir: lane.commonGitDir, + writableGitPaths: lane.writableGitPaths, + }, + }); + this.commandRoots.set(internalId, fingerprint); + } + return pool; + } + + async execute(request: WorkspaceToolRequest, signal?: AbortSignal): Promise { + if (request.worktree == null) { + return await this.options.delegate.execute(request, signal); + } + if (request.operation === 'execute_command' && request.environmentAction) { + throw rejected('Environment action was not resolved by this worker'); + } + const { worktree, ...baseRequest } = request; + const { lane, source, internalId } = await this.resolveLane( + request.workspaceId, + worktree, + request.workspaceInstanceId, + ); + const laneRequest = { ...baseRequest, workspaceId: internalId } as WorkspaceToolRequest; + if (request.operation === 'execute_command') { + const pool = await this.registerCommandRoot(internalId, lane, source); + return publicResult( + await pool.execute(laneRequest as WorkspaceExecuteCommandRequest, signal), + request.workspaceId, + ); + } + const executor = await this.fileExecutor(internalId, lane, source); + return publicResult(await executor.execute(laneRequest, signal), request.workspaceId); + } + + async executeProgrammatic( + workspaceId: string, + request: BridgeWorkspaceProgrammaticRequest, + signal?: AbortSignal, + ): Promise { + const worktree = request.body.workspace_worktree; + if (worktree == null) { + if (!this.options.programmaticDelegate) { + throw new WorkspaceToolError('Workspace programmatic execution is unavailable', 'COMMAND_DISABLED'); + } + return await this.options.programmaticDelegate.executeProgrammatic(workspaceId, request, signal); + } + const { lane, source, internalId } = await this.resolveLane( + workspaceId, + worktree, + request.body.workspace_instance_id, + ); + const pool = await this.registerCommandRoot(internalId, lane, source); + const { workspace_worktree: _worktree, ...body } = request.body; + return await pool.executeProgrammatic(internalId, { ...request, body }, signal); + } +} diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts index 5f7eedbb..8abf263c 100644 --- a/packages/code/src/native-process.test.ts +++ b/packages/code/src/native-process.test.ts @@ -150,6 +150,22 @@ test('executor bootstrap excludes bridge credentials and Node injection variable await sandbox.close(); }); +test('executor forwards the linked worktree policy to the sandbox process', async () => { + const fake = fixture(); + const linkedWorktree = { + checkoutRoot: '/checkout', + commonGitDir: '/checkout/.git', + writableGitPaths: ['/checkout/.git/objects', '/checkout/.git/refs'], + }; + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { workspaceRoot: '/checkout/.worktrees/task-a', linkedWorktree }, + fake.fork, + ); + await sandbox.prepare(); + assert.deepEqual(fake.messages[0].options.linkedWorktree, linkedWorktree); + await sandbox.close(); +}); + test('executor forwards the resolved command policy without worker credentials', async () => { const fake = fixture(); const sandbox = new NativeProcessWorkspaceCommandSandbox( diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index ba411042..77fe8269 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -343,6 +343,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan homeDirectory, shellPath, programmaticFileUpstream, + linkedWorktree, } = this.options; await this.rpc( 'prepare', @@ -356,6 +357,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan homeDirectory, shellPath, programmaticFileUpstream, + linkedWorktree, variables: this.options.maskedEnvironment?.variables, }, }, diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index eab72966..7ecb06f3 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -1598,3 +1598,64 @@ test('cleans allocated command state exactly once on every execution exit', asyn } } }); + +test('a linked worktree lane may write only shared Git storage and its own metadata', async t => { + const checkoutRoot = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-lane-'))); + t.after(() => rm(checkoutRoot, { recursive: true, force: true })); + const commonGitDir = join(checkoutRoot, '.git'); + const lane = join(checkoutRoot, '.worktrees', 'task-a'); + const writableGitPaths = [ + join(commonGitDir, 'objects'), + join(commonGitDir, 'refs'), + join(commonGitDir, 'worktrees', 'task-a'), + ]; + await Promise.all( + [lane, ...writableGitPaths].map(path => mkdir(path, { recursive: true })), + ); + const prepare = async (paths: string[]) => { + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: lane, + linkedWorktree: { checkoutRoot, commonGitDir, writableGitPaths: paths }, + environment: { PATH: '/usr/bin' }, + manager: fake.manager, + }); + t.after(() => sandbox.close()); + await sandbox.prepare(); + return fake.config!; + }; + + const config = await prepare(writableGitPaths); + assert.deepEqual(config.filesystem.allowWrite.slice(0, 4), [lane, ...writableGitPaths]); + assert.ok(!config.filesystem.allowWrite.includes(commonGitDir)); + assert.ok(config.filesystem.allowRead?.includes(commonGitDir)); + + const probed = fakeManager(); + const prober = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: lane, + linkedWorktree: { checkoutRoot, commonGitDir, writableGitPaths }, + environment: { PATH: '/usr/bin' }, + manager: probed.manager, + }); + t.after(() => prober.close()); + const dataDirectory = await prober.createExecutionDirectory(); + await prober.executeProgrammatic(request, dataDirectory, undefined, { probe: true }); + assert.ok(probed.customConfigSeenDuringWrap?.filesystem?.allowRead?.includes(commonGitDir)); + assert.ok(!probed.customConfigSeenDuringWrap?.filesystem?.allowWrite?.includes(commonGitDir)); + + await assert.rejects( + prepare([commonGitDir]), + (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'REGISTRATION_INVALID', + ); + + const siblingMetadata = join(commonGitDir, 'worktrees', 'task-b'); + await mkdir(siblingMetadata, { recursive: true }); + await rm(join(commonGitDir, 'objects'), { recursive: true }); + await symlink(siblingMetadata, join(commonGitDir, 'objects')); + await assert.rejects( + prepare(writableGitPaths), + (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'REGISTRATION_INVALID', + ); +}); diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 5bf61c50..6ac51acc 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -106,10 +106,15 @@ const TRUSTED_GIT_ENVIRONMENT = { GIT_CONFIG_KEY_3: 'filter.lfs.required', GIT_CONFIG_VALUE_3: 'true', } as const; -const { - GIT_CONFIG_COUNT: TRUSTED_GIT_CONFIG_COUNT, - ...TRUSTED_GIT_CONFIG_ENTRIES -} = TRUSTED_GIT_ENVIRONMENT; +/** Sibling lanes share object storage, so a lane never starts automatic gc or maintenance. */ +const LINKED_WORKTREE_GIT_ENVIRONMENT = { + ...TRUSTED_GIT_ENVIRONMENT, + GIT_CONFIG_COUNT: '6', + GIT_CONFIG_KEY_4: 'gc.auto', + GIT_CONFIG_VALUE_4: '0', + GIT_CONFIG_KEY_5: 'maintenance.auto', + GIT_CONFIG_VALUE_5: 'false', +} as const; const NATIVE_SANDBOX_SCRATCH_PREFIX = 'librechat-code-srt-'; // SRT grants these shared compatibility paths by default. A worker-specific @@ -162,6 +167,15 @@ type SpawnCommand = ( export interface NativeSrtWorkspaceCommandSandboxOptions { workspaceIdentity?: WorkspaceRootIdentity; workspaceRoot: string; + /** Present when `workspaceRoot` is a verified linked worktree lane of a checkout. */ + linkedWorktree?: { + /** The checkout that owns the worktree; trusted as a Git safe directory. */ + checkoutRoot: string; + /** `/.git`: readable, but writable only at `writableGitPaths`. */ + commonGitDir: string; + /** Shared objects and refs plus the lane's own metadata beneath `commonGitDir`. */ + writableGitPaths: string[]; + }; commandPolicy?: NativeSrtCommandPolicy; /** Trusted worker files that must never become workspace-readable or writable. */ protectedPaths?: string[]; @@ -295,9 +309,18 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox private readonly platform: NodeJS.Platform; private initialized?: Promise; private canonicalRoot?: string; + /** A lane's shared Git directory; replay probes of the lane must read it too. */ + private canonicalCommonGitDir?: string; private runtimeConfig?: SandboxRuntimeConfig; private denyReadPaths: string[] = []; private denyWritePaths: string[] = []; + private get gitEnvironment(): + | typeof TRUSTED_GIT_ENVIRONMENT + | typeof LINKED_WORKTREE_GIT_ENVIRONMENT { + return this.options.linkedWorktree + ? LINKED_WORKTREE_GIT_ENVIRONMENT + : TRUSTED_GIT_ENVIRONMENT; + } private scratchDirectory?: string; private scratchHandle?: FileHandle; private execution?: Promise; @@ -424,6 +447,33 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); } } + const lane = this.options.linkedWorktree; + const commonGitDir = lane ? await canonicalPath(lane.commonGitDir) : undefined; + const checkoutRoot = lane ? await canonicalPath(lane.checkoutRoot) : undefined; + const writableGitPaths = lane + ? await Promise.all(lane.writableGitPaths.map(canonicalPath)) + : []; + if ( + lane && + (commonGitDir == null || + checkoutRoot == null || + !isWithin(checkoutRoot, commonGitDir) || + !isWithin(checkoutRoot, root) || + isWithin(commonGitDir, home) || + protectedPaths.some(path => isWithin(commonGitDir, path)) || + writableGitPaths.some( + (path, index) => + path !== resolve(lane.writableGitPaths[index]!) || + path === commonGitDir || + !isWithin(commonGitDir, path), + )) + ) { + throw new WorkspaceToolError( + 'Linked worktree Git storage is outside its checkout', + 'REGISTRATION_INVALID', + ); + } + const laneGitPaths = commonGitDir ? [commonGitDir] : []; const canonicalScratchDirectory = await this.createScratchDirectory(sharedScratchPaths); if ( @@ -459,17 +509,22 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ], allowRead: [ root, + ...laneGitPaths, ...(canonicalScratchDirectory ? [canonicalScratchDirectory] : []), ], allowWrite: [ root, + ...writableGitPaths, ...(canonicalScratchDirectory ? [canonicalScratchDirectory] : []), ], - denyWrite: [...protectedPaths, ...deniedInheritedWritablePaths], + denyWrite: [ + ...protectedPaths, + ...deniedInheritedWritablePaths, + ], allowGitConfig: false, }, credentials: { @@ -520,13 +575,14 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox allowAppleEvents: false, enableWeakerNestedSandbox: false, enableWeakerNetworkIsolation: false, - git: { safeDirectories: [root] }, + git: { safeDirectories: checkoutRoot ? [root, checkoutRoot] : [root] }, }; await this.manager.initialize( config, unrestrictedNetwork ? async () => true : undefined, ); this.canonicalRoot = root; + this.canonicalCommonGitDir = commonGitDir; this.runtimeConfig = config; this.denyReadPaths = [ home, @@ -758,6 +814,9 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox allowRead: [ canonicalWorkspaceRoot ?? this.canonicalRoot!, canonicalDataDirectory, + ...(this.canonicalCommonGitDir + ? [this.canonicalCommonGitDir] + : []), ], allowWrite: [ ...(canonicalWorkspaceRoot != null @@ -894,7 +953,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox : request.command; wrapped = await this.withTemporaryHostEnvironment( { - ...TRUSTED_GIT_ENVIRONMENT, + ...this.gitEnvironment, ...(credentialEnvironment ?? {}), ...this.scratchSelectorEnvironment(sandboxScratchDirectory), }, @@ -999,10 +1058,10 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ...wrapped.env, ...this.scratchEnvironment(), ...trustedEnvironment, - ...TRUSTED_GIT_CONFIG_ENTRIES, + ...this.gitEnvironment, GIT_CONFIG_COUNT: wrapped.env.GIT_CONFIG_COUNT ?? - TRUSTED_GIT_CONFIG_COUNT, + this.gitEnvironment.GIT_CONFIG_COUNT, GIT_CONFIG_GLOBAL: this.platform === 'win32' ? 'NUL' diff --git a/packages/code/src/protocol.test.ts b/packages/code/src/protocol.test.ts index 5c26c462..729b20e2 100644 --- a/packages/code/src/protocol.test.ts +++ b/packages/code/src/protocol.test.ts @@ -11,6 +11,8 @@ import { isWorkspaceToolRequest, isWorkspaceToolResult, workspaceIsolationKey, + workspaceIsolationKeysConflict, + workspaceIsolationParent, } from './protocol.js'; import type { WorkspaceEditFileRequest, @@ -816,3 +818,62 @@ test('bridge artifact policy and media types match the hardened gateway contract assert.equal(bridgeArtifactMediaType('reports/result.json'), 'application/json'); assert.equal(bridgeArtifactMediaType('Dockerfile'), 'application/octet-stream'); }); + +test('linked-worktree lanes nest beneath their checkout key and conflict only with it', () => { + const instanceId = 'a'.repeat(64); + const lane = workspaceIsolationKey('repo', undefined, 'task-a'); + const sibling = workspaceIsolationKey('repo', undefined, 'task-b'); + const nested = workspaceIsolationKey('repo', instanceId, 'task-a'); + assert.notEqual(lane, workspaceIsolationKey('repo')); + assert.notEqual(lane, workspaceIsolationKey('repo\0task-a')); + assert.notEqual(nested, lane); + assert.equal(workspaceIsolationParent(lane), 'repo'); + assert.equal(workspaceIsolationParent(nested), workspaceIsolationKey('repo', instanceId)); + assert.equal(workspaceIsolationParent('repo'), undefined); + assert.equal(workspaceIsolationParent(workspaceIsolationKey('repo', instanceId)), undefined); + assert.equal(workspaceIsolationKeysConflict(lane, 'repo'), true); + assert.equal(workspaceIsolationKeysConflict('repo', lane), true); + assert.equal(workspaceIsolationKeysConflict(lane, lane), true); + assert.equal(workspaceIsolationKeysConflict(lane, sibling), false); + assert.equal(workspaceIsolationKeysConflict(nested, 'repo'), false); + assert.equal(workspaceIsolationKeysConflict(nested, workspaceIsolationKey('repo', instanceId)), true); +}); + +test('workspace requests accept only a single safe worktree name', () => { + const base = { protocolVersion: 1, operation: 'read_file', workspaceId: 'repo', path: 'README.md' }; + for (const worktree of ['task-a', 'fix_16464', 'v2.0']) { + assert.equal(isWorkspaceToolRequest({ ...base, worktree }), true, worktree); + } + for (const worktree of ['', '.hidden', '..', 'a/b', 'a\\b', 'task.lock', 'x'.repeat(129), 7, null]) { + assert.equal(isWorkspaceToolRequest({ ...base, worktree }), false, String(worktree)); + } + const programmatic = (workspace_worktree: string) => ({ + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'session', + files: [{ name: 'main.sh', content: 'true' }], + workspace_worktree, + }, + }); + assert.equal(isBridgeWorkspaceProgrammaticRequest(programmatic('task-a')), true); + assert.equal(isBridgeWorkspaceProgrammaticRequest(programmatic('../x')), false); +}); + +test('workspace capabilities advertise linked-worktree scopes exactly', () => { + const capabilities = (workspaceScopes: unknown) => ({ + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: { + protocolVersion: 1, + operations: ['read_file'], + workspaces: [{ id: 'repo', workspaceScopes }], + }, + }); + assert.equal(isValidBridgeWorkerCapabilities(capabilities(['git_linked_worktree'])), true); + assert.equal(isValidBridgeWorkerCapabilities(capabilities(['git_worktree'])), false); + assert.equal(isValidBridgeWorkerCapabilities(capabilities([])), false); + assert.equal(isValidBridgeWorkerCapabilities(capabilities('git_linked_worktree')), false); +}); diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index 7657fc9c..b23ad327 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -259,14 +259,51 @@ export function bridgeArtifactMediaType(name: string): string { export type BridgeProtocolVersion = typeof BRIDGE_PROTOCOL_VERSION; -/** Collision-free identity shared by scheduling and worker quarantine state. */ +/** One path segment naming a linked worktree at `/.worktrees/`. */ +export const BRIDGE_LINKED_WORKTREE_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; + +export function isValidLinkedWorktreeName(value: unknown): value is string { + return ( + typeof value === 'string' && + BRIDGE_LINKED_WORKTREE_NAME_PATTERN.test(value) && + !value.endsWith('.lock') + ); +} + +const LINKED_WORKTREE_KEY_PREFIX = '\0linked-worktree\0'; + +/** Collision-free identity shared by scheduling and worker quarantine state. + * A linked worktree lane nests beneath the key of the checkout that owns it. */ export function workspaceIsolationKey( workspaceId: string, instanceId?: string, + worktree?: string, ): string { - return instanceId === undefined + const base = instanceId === undefined ? workspaceId : `\0git-worktree\0${workspaceId}\0${instanceId}`; + return worktree === undefined + ? base + : `${LINKED_WORKTREE_KEY_PREFIX}${base}\0${worktree}`; +} + +/** The checkout key a linked-worktree lane nests beneath, or undefined for a root key. + * Scheduling treats a lane and its parent as conflicting; sibling lanes do not. */ +export function workspaceIsolationParent(key: string): string | undefined { + if (!key.startsWith(LINKED_WORKTREE_KEY_PREFIX)) return undefined; + const separator = key.lastIndexOf('\0'); + return separator <= LINKED_WORKTREE_KEY_PREFIX.length + ? undefined + : key.slice(LINKED_WORKTREE_KEY_PREFIX.length, separator); +} + +/** Two isolation keys may not execute concurrently when either nests the other. */ +export function workspaceIsolationKeysConflict(left: string, right: string): boolean { + return ( + left === right || + workspaceIsolationParent(left) === right || + workspaceIsolationParent(right) === left + ); } export type BridgeWorkspaceToolOperation = @@ -292,6 +329,9 @@ export interface BridgeWorkspaceDescriptor { operations?: BridgeWorkspaceToolOperation[]; /** Worker-owned isolation schemes available beneath this selected root. */ workspaceInstances?: ['git_worktree']; + /** Scheduling scopes available beneath this root. `git_linked_worktree` gives each + * verified `.worktrees/` linked worktree its own lane. */ + workspaceScopes?: ['git_linked_worktree']; environment?: { fingerprint: string; repo?: string; @@ -321,6 +361,8 @@ export interface WorkspaceReadFileRequest { operation: 'read_file'; workspaceId: string; workspaceInstanceId?: string; + /** Linked worktree lane at `.worktrees/`; paths and cwd are relative to it. */ + worktree?: string; path: string; startLine?: number; maxLines?: number; @@ -364,6 +406,8 @@ export interface WorkspaceSearchTextRequest { operation: 'search_text'; workspaceId: string; workspaceInstanceId?: string; + /** Linked worktree lane at `.worktrees/`; paths and cwd are relative to it. */ + worktree?: string; query: string; path?: string; maxResults?: number; @@ -389,6 +433,8 @@ export interface WorkspaceListFilesRequest { operation: 'list_files'; workspaceId: string; workspaceInstanceId?: string; + /** Linked worktree lane at `.worktrees/`; paths and cwd are relative to it. */ + worktree?: string; path?: string; maxResults?: number; /** Continue strictly after this canonical path from a previous page. */ @@ -410,6 +456,8 @@ export interface WorkspaceWriteFileRequest { operation: 'write_file'; workspaceId: string; workspaceInstanceId?: string; + /** Linked worktree lane at `.worktrees/`; paths and cwd are relative to it. */ + worktree?: string; path: string; content: string; /** False requires an atomic create and refuses to replace an existing file. */ @@ -430,6 +478,8 @@ interface WorkspaceEditFileRequestBase { operation: 'edit_file'; workspaceId: string; workspaceInstanceId?: string; + /** Linked worktree lane at `.worktrees/`; paths and cwd are relative to it. */ + worktree?: string; path: string; /** Refuses the mutation unless current file bytes match this preview revision. */ expectedBaseSha256?: string; @@ -475,6 +525,8 @@ interface WorkspacePreviewEditRequestBase { operation: 'preview_edit'; workspaceId: string; workspaceInstanceId?: string; + /** Linked worktree lane at `.worktrees/`; paths and cwd are relative to it. */ + worktree?: string; path: string; } @@ -513,6 +565,8 @@ export interface WorkspaceExecuteCommandRequest { operation: 'execute_command'; workspaceId: string; workspaceInstanceId?: string; + /** Linked worktree lane at `.worktrees/`; paths and cwd are relative to it. */ + worktree?: string; /** Shell source evaluated only inside the selected sandbox runtime. */ command: string; /** Portable path relative to the workspace root; defaults to '.'. */ @@ -558,6 +612,7 @@ const WORKSPACE_READ_REQUEST_KEYS = new Set([ 'operation', 'workspaceId', 'workspaceInstanceId', + 'worktree', 'path', 'startLine', 'maxLines', @@ -567,6 +622,7 @@ const WORKSPACE_SEARCH_REQUEST_KEYS = new Set([ 'operation', 'workspaceId', 'workspaceInstanceId', + 'worktree', 'query', 'path', 'maxResults', @@ -576,6 +632,7 @@ const WORKSPACE_LIST_REQUEST_KEYS = new Set([ 'operation', 'workspaceId', 'workspaceInstanceId', + 'worktree', 'path', 'maxResults', 'afterPath', @@ -585,6 +642,7 @@ const WORKSPACE_WRITE_REQUEST_KEYS = new Set([ 'operation', 'workspaceId', 'workspaceInstanceId', + 'worktree', 'path', 'content', 'overwrite', @@ -594,6 +652,7 @@ const WORKSPACE_EDIT_REQUEST_KEYS = new Set([ 'operation', 'workspaceId', 'workspaceInstanceId', + 'worktree', 'path', 'oldText', 'newText', @@ -605,6 +664,7 @@ const WORKSPACE_PREVIEW_EDIT_REQUEST_KEYS = new Set([ 'operation', 'workspaceId', 'workspaceInstanceId', + 'worktree', 'path', 'oldText', 'newText', @@ -617,6 +677,7 @@ const WORKSPACE_COMMAND_REQUEST_KEYS = new Set([ 'operation', 'workspaceId', 'workspaceInstanceId', + 'worktree', 'command', 'cwd', 'timeoutMs', @@ -730,6 +791,8 @@ export interface BridgeWorkerRegistrationResponse { supportedWorkspaceProgrammaticLanguages?: WorkspaceProgrammaticLanguage[]; /** Workspace isolation schemes this Code API understands and can route. */ supportedWorkspaceInstanceTypes?: ['git_worktree']; + /** Scheduling scopes this Code API can admit as independent lanes. */ + supportedWorkspaceScopes?: ['git_linked_worktree']; } /** Administrator-visible liveness for a configured worker. Credentials, @@ -803,6 +866,8 @@ export interface BridgeWorkspaceProgrammaticBody { language: 'bash'; version: string; workspace_instance_id?: string; + /** Linked worktree lane at `.worktrees/` beneath the selected checkout. */ + workspace_worktree?: string; /** Stable identity shared by every replay iteration of one execution. */ execution_id?: string; /** Declared replay tools; zero allows the worker to skip the probe pass. */ @@ -970,6 +1035,8 @@ export function isBridgeWorkspaceProgrammaticRequest( (body.workspace_instance_id !== undefined && (typeof body.workspace_instance_id !== 'string' || !/^[a-f0-9]{64}$/.test(body.workspace_instance_id))) || + (body.workspace_worktree !== undefined && + !isValidLinkedWorktreeName(body.workspace_worktree)) || (body.execution_id !== undefined && (typeof body.execution_id !== 'string' || !/^[A-Za-z0-9_-]{1,128}$/.test(body.execution_id))) || @@ -1222,7 +1289,8 @@ export function isWorkspaceToolRequest( !isValidBridgeWorkerId(request.workspaceId) || (request.workspaceInstanceId !== undefined && (typeof request.workspaceInstanceId !== 'string' || - !/^[a-f0-9]{64}$/.test(request.workspaceInstanceId))) + !/^[a-f0-9]{64}$/.test(request.workspaceInstanceId))) || + (request.worktree !== undefined && !isValidLinkedWorktreeName(request.worktree)) ) { return false; } @@ -1676,6 +1744,7 @@ export function isValidBridgeWorkspaceToolCapabilities( key !== 'name' && key !== 'operations' && key !== 'workspaceInstances' && + key !== 'workspaceScopes' && key !== 'instructions' && key !== 'environment', ) || @@ -1686,6 +1755,10 @@ export function isValidBridgeWorkspaceToolCapabilities( (!Array.isArray(descriptor.workspaceInstances) || descriptor.workspaceInstances.length !== 1 || descriptor.workspaceInstances[0] !== 'git_worktree')) || + (descriptor.workspaceScopes !== undefined && + (!Array.isArray(descriptor.workspaceScopes) || + descriptor.workspaceScopes.length !== 1 || + descriptor.workspaceScopes[0] !== 'git_linked_worktree')) || (descriptor.instructions !== undefined && (!Array.isArray(descriptor.instructions) || descriptor.instructions.length > 1 || !descriptor.instructions.every(isRepositoryInstructionDescriptor))) || (descriptor.environment !== undefined && !isValidCodeEnvironmentDescriptor(descriptor.environment)) || diff --git a/packages/code/src/worker-slots.test.ts b/packages/code/src/worker-slots.test.ts index fda6c659..72052974 100644 --- a/packages/code/src/worker-slots.test.ts +++ b/packages/code/src/worker-slots.test.ts @@ -377,3 +377,204 @@ test('programmatic work on an independent workspace bypasses another root cleanu 'previous', ); }); + +test('sibling linked-worktree lanes run past each other, but a lane and its checkout wait for one another', async () => { + const worker = new BridgeWorker({ + codeApiUrl: 'http://localhost:1', + token: 'fixture', + workerId: 'worker', + sandboxEndpoint: 'http://localhost:2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'fixture', + runtimes: [], + }, + }); + const internals = worker as unknown as { + activeWorkspaceAssignments: Map }>; + executeOwned: (assignment: BridgeAssignment) => Promise; + }; + const executed: string[] = []; + internals.executeOwned = async (assignment) => { + executed.push(assignment.assignmentId); + }; + const assignment = (assignmentId: string, worktree?: string) => + ({ + assignmentId, + executionKind: 'workspace_tool', + remainingMs: 1_000, + request: { + protocolVersion: 1, + workspaceId: 'repo', + operation: 'read_file', + path: 'README.md', + ...(worktree ? { worktree } : {}), + }, + }) as BridgeAssignment; + let releaseLane!: () => void; + internals.activeWorkspaceAssignments.set(workspaceIsolationKey('repo', undefined, 'task-a'), { + id: 'lane-a', + done: new Promise((resolve) => { + releaseLane = resolve; + }), + }); + + await worker.executeAndSettle(assignment('lane-b', 'task-b')); + assert.deepEqual(executed, ['lane-b']); + + const checkout = worker.executeAndSettle(assignment('checkout')); + await new Promise((resolve) => setTimeout(resolve, 5)); + assert.deepEqual(executed, ['lane-b']); + internals.activeWorkspaceAssignments.delete(workspaceIsolationKey('repo', undefined, 'task-a')); + releaseLane(); + await checkout; + assert.deepEqual(executed, ['lane-b', 'checkout']); + + let releaseCheckout!: () => void; + internals.activeWorkspaceAssignments.set(workspaceIsolationKey('repo'), { + id: 'root', + done: new Promise((resolve) => { + releaseCheckout = resolve; + }), + }); + const lane = worker.executeAndSettle(assignment('lane-c', 'task-c')); + await new Promise((resolve) => setTimeout(resolve, 5)); + assert.deepEqual(executed, ['lane-b', 'checkout']); + internals.activeWorkspaceAssignments.delete(workspaceIsolationKey('repo')); + releaseCheckout(); + await lane; + assert.deepEqual(executed, ['lane-b', 'checkout', 'lane-c']); +}); + +test('a linked worktree lane fence is reset through its own guard and isolation key', async () => { + const bodies: Record[] = []; + const guarded: string[] = []; + const worker = new BridgeWorker({ + codeApiUrl: 'http://localhost:1', + token: 'fixture', + workerId: 'worker', + incarnationId: 'incarnation-reset', + sandboxEndpoint: 'http://localhost:2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + workspaceLeaseSlots: 2, + requiresReadyConfirmation: true, + workspaceTools: { + protocolVersion: 1, + operations: ['read_file'], + workspaces: [{ id: 'repo', workspaceScopes: ['git_linked_worktree'] }, { id: 'plain' }], + }, + }, + workspaceQuarantines: new Map( + ['repo', 'plain'].map((id) => [ + id, + { + assertAvailable: async () => undefined, + arm: async () => undefined, + clear: async () => undefined, + quarantine: async () => undefined, + }, + ]), + ), + workspaceTools: { + capabilities: { + protocolVersion: 1, + operations: ['read_file'], + workspaces: [{ id: 'repo', workspaceScopes: ['git_linked_worktree'] }, { id: 'plain' }], + }, + async execute() { + throw new Error('must not execute'); + }, + }, + linkedWorktreeQuarantineResolver: (workspaceId, worktree) => { + guarded.push(`${workspaceId}/${worktree}`); + return { + assertAvailable: async () => undefined, + arm: async () => undefined, + clear: async () => undefined, + quarantine: async () => undefined, + }; + }, + fetchImpl: async (url, init) => { + assert.ok(new URL(String(url)).pathname.endsWith('/workspaces/reset')); + bodies.push(JSON.parse(String(init?.body))); + return Response.json({ protocolVersion: 1, reset: true }); + }, + }); + + await worker.resetNativeWorkspace('repo', undefined, undefined, 'task-a'); + + assert.deepEqual(guarded, ['repo/task-a']); + assert.equal( + bodies[0]?.runtimeSessionId, + `native-workspace:${workspaceIsolationKey('repo', undefined, 'task-a')}`, + ); + await assert.rejects( + worker.resetNativeWorkspace('plain', undefined, undefined, 'task-a'), + /worktree lanes/, + ); + await assert.rejects( + worker.resetNativeWorkspace('repo', undefined, 'a'.repeat(64), 'task-a'), + /worktree lanes/, + ); + await assert.rejects( + worker.resetNativeWorkspace('repo', undefined, undefined, '../task-a'), + /worktree lanes/, + ); + assert.equal(bodies.length, 1); +}); + +test('a checkout waits on quarantined linked worktrees beneath it', async () => { + const quarantinedLanes = new Set(['task-b']); + const worker = new BridgeWorker({ + codeApiUrl: 'http://localhost:1', + token: 'fixture', + workerId: 'worker', + sandboxEndpoint: 'http://localhost:2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'fixture', + runtimes: [], + }, + linkedWorktreeQuarantineResolver: (_workspaceId, worktree) => ({ + assertAvailable: async () => { + if (quarantinedLanes.has(worktree)) throw new Error(`lane ${worktree} is quarantined`); + }, + arm: async () => undefined, + clear: async () => undefined, + quarantine: async () => undefined, + }), + linkedWorktreeNames: async () => ['task-a', 'task-b'], + }); + const internals = worker as unknown as { + quarantinedWorkspaces: Set; + assertLaneFamilyAvailable: (key: string, assignment: BridgeAssignment) => Promise; + }; + const assignment = (worktree?: string) => + ({ + assignmentId: 'probe', + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + workspaceId: 'repo', + operation: 'read_file', + path: 'README.md', + ...(worktree ? { worktree } : {}), + }, + }) as BridgeAssignment; + const check = (worktree?: string) => + internals.assertLaneFamilyAvailable( + workspaceIsolationKey('repo', undefined, worktree), + assignment(worktree), + ); + + await assert.rejects(check(), /task-b is quarantined/); + await check('task-a'); + + quarantinedLanes.clear(); + await check(); + internals.quarantinedWorkspaces.add(workspaceIsolationKey('repo', undefined, 'task-c')); + await assert.rejects(check(), /linked worktree/); +}); diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 2a5896fd..37223b34 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -6,8 +6,11 @@ import { BridgeProtocolError, bridgeWorkerPath, isBridgeWorkspaceProgrammaticRequest, + isValidLinkedWorktreeName, isWorkspaceToolResult, workspaceIsolationKey, + workspaceIsolationKeysConflict, + workspaceIsolationParent, } from './protocol.js'; import { EndpointRuntimeSupervisor } from './runtime.js'; import { signBridgeRequest } from './identity.js'; @@ -62,6 +65,15 @@ export interface BridgeWorkerOptions { ) => | WorkspaceMutationQuarantine | Promise; + /** Resolve a durable guard for a linked-worktree lane beneath a registered root. */ + linkedWorktreeQuarantineResolver?: ( + workspaceId: string, + worktree: string, + ) => + | WorkspaceMutationQuarantine + | Promise; + /** Names of the linked worktrees currently beneath a registered root. */ + linkedWorktreeNames?: (workspaceId: string) => Promise; leaseWaitMs?: number; leaseTransportGraceMs?: number; registrationTransportTimeoutMs?: number; @@ -223,7 +235,13 @@ function workspaceCapabilitiesMatch( (instanceType, instanceIndex) => instanceType === executor.workspaces[index]?.workspaceInstances?.[instanceIndex], - ) ?? executor.workspaces[index]?.workspaceInstances == null), + ) ?? executor.workspaces[index]?.workspaceInstances == null) && + workspace.workspaceScopes?.length === + executor.workspaces[index]?.workspaceScopes?.length && + (workspace.workspaceScopes?.every( + (scope, scopeIndex) => + scope === executor.workspaces[index]?.workspaceScopes?.[scopeIndex], + ) ?? executor.workspaces[index]?.workspaceScopes == null), ) ); } @@ -239,7 +257,9 @@ function registrationCompatibleCapabilities( ) && workspaceTools.workspaces.every( (workspace) => - workspace.operations == null && workspace.workspaceInstances == null, + workspace.operations == null && + workspace.workspaceInstances == null && + workspace.workspaceScopes == null, )) ) { return capabilities; @@ -263,6 +283,7 @@ function registrationCompatibleCapabilities( const { operations: _operations, workspaceInstances: _workspaceInstances, + workspaceScopes: _workspaceScopes, ...compatibleWorkspace } = workspace; return [{ ...compatibleWorkspace, ...(workspace.environment ? { @@ -355,6 +376,10 @@ function supportedWorkspaceCapabilities( ) ? { workspaceInstances: workspace.workspaceInstances } : { workspaceInstances: undefined }), + ...(workspace.workspaceScopes != null && + registration.supportedWorkspaceScopes?.includes('git_linked_worktree') + ? { workspaceScopes: workspace.workspaceScopes } + : { workspaceScopes: undefined }), ...(workspace.operations ? { operations: workspaceOperations } : {}), ...(workspace.environment && !workspaceOperations.includes('execute_command') ? { environment: { ...workspace.environment, actions: [] }, @@ -506,7 +531,8 @@ export class BridgeWorker { ) === true && options.workspaceMutationQuarantine == null && options.workspaceQuarantines == null && - options.workspaceQuarantineResolver == null + options.workspaceQuarantineResolver == null && + options.linkedWorktreeQuarantineResolver == null ) { throw new BridgeProtocolError( 'Workspace mutation capabilities require durable quarantine storage', @@ -522,6 +548,17 @@ export class BridgeWorker { 'Workspace instance capabilities require a durable quarantine resolver', ); } + if ( + options.capabilities.workspaceTools?.workspaces.some( + (root) => (root.workspaceScopes?.length ?? 0) > 0, + ) && + (options.linkedWorktreeQuarantineResolver == null || + (options.capabilities.workspaceLeaseSlots ?? 1) < 2) + ) { + throw new BridgeProtocolError( + 'Linked worktree lanes require a durable quarantine resolver and at least two lease slots', + ); + } if ((options.capabilities.workspaceLeaseSlots ?? 1) > 1) { if ( options.capabilities.requiresReadyConfirmation !== true || @@ -823,18 +860,33 @@ export class BridgeWorker { workspaceId: string, signal?: AbortSignal, workspaceInstanceId?: string, + worktree?: string, ): Promise { const workspace = this.options.capabilities.workspaceTools?.workspaces.find( (root) => root.id === workspaceId, ); - const key = workspaceIsolationKey(workspaceId, workspaceInstanceId); - const guard = - workspaceInstanceId == null - ? this.options.workspaceQuarantines?.get(workspaceId) - : await this.options.workspaceQuarantineResolver?.( - workspaceId, - workspaceInstanceId, - ); + if ( + worktree != null && + (workspaceInstanceId != null || + !isValidLinkedWorktreeName(worktree) || + workspace?.workspaceScopes?.includes('git_linked_worktree') !== true) + ) { + throw new BridgeProtocolError( + 'Linked worktree reset requires a registered checkout with worktree lanes', + ); + } + const key = workspaceIsolationKey(workspaceId, workspaceInstanceId, worktree); + let guard: WorkspaceMutationQuarantine | undefined; + if (worktree != null) { + guard = await this.options.linkedWorktreeQuarantineResolver?.(workspaceId, worktree); + } else if (workspaceInstanceId != null) { + guard = await this.options.workspaceQuarantineResolver?.( + workspaceId, + workspaceInstanceId, + ); + } else { + guard = this.options.workspaceQuarantines?.get(workspaceId); + } if ( !guard || this.activeWorkspaceAssignments.size > 0 || @@ -1313,8 +1365,11 @@ export class BridgeWorker { ): Promise { const root = this.assignmentWorkspaceId(assignment); const waitingAt = Date.now(); - while (root != null && this.activeWorkspaceAssignments.has(root)) { - const active = this.activeWorkspaceAssignments.get(root)!; + for ( + let active = root == null ? undefined : this.conflictingActiveAssignment(root); + root != null && active != null; + active = this.conflictingActiveAssignment(root) + ) { if (active.id === assignment.assignmentId) throw new BridgeProtocolError( 'Code API replayed an active workspace assignment', @@ -1386,6 +1441,50 @@ export class BridgeWorker { } } + /** An active assignment on the same key, the key's parent checkout, or a lane beneath it. */ + private conflictingActiveAssignment( + key: string, + ): { id: string; done: Promise } | undefined { + for (const [activeKey, active] of this.activeWorkspaceAssignments) { + if (workspaceIsolationKeysConflict(activeKey, key)) return active; + } + return undefined; + } + + /** A lane may not run while the checkout it belongs to is quarantined. */ + /** + * A lane may not start while its checkout is quarantined, and a checkout may + * not start while any linked worktree beneath it is: root commands can reach + * `.worktrees/*`, including `git worktree remove`. + */ + private async assertLaneFamilyAvailable( + workspaceKey: string, + assignment: BridgeAssignment, + ): Promise { + const workspaceId = this.assignmentBaseWorkspaceId(assignment); + const parent = workspaceIsolationParent(workspaceKey); + if (parent != null) { + if (this.quarantinedWorkspaces.has(parent)) { + throw new Error('Parent workspace requires an explicit quarantine reset'); + } + if (workspaceId != null && parent === workspaceId) { + await this.options.workspaceQuarantines?.get(workspaceId)?.assertAvailable(); + } + return; + } + const resolveLaneGuard = this.options.linkedWorktreeQuarantineResolver; + if (workspaceId == null || workspaceKey !== workspaceId || resolveLaneGuard == null) return; + for (const key of this.quarantinedWorkspaces) { + if (workspaceIsolationParent(key) === workspaceKey) { + throw new Error('A linked worktree in this workspace requires an explicit quarantine reset'); + } + } + const names = (await this.options.linkedWorktreeNames?.(workspaceId)) ?? []; + await Promise.all( + names.map(async (name) => (await resolveLaneGuard(workspaceId, name)).assertAvailable()), + ); + } + private workspaceGuard( assignment: BridgeAssignment, ): @@ -1394,6 +1493,10 @@ export class BridgeWorker { | undefined { const workspaceId = this.assignmentBaseWorkspaceId(assignment); const instanceId = this.assignmentWorkspaceInstanceId(assignment); + const worktree = this.assignmentWorktree(assignment); + if (workspaceId != null && worktree != null) { + return this.options.linkedWorktreeQuarantineResolver?.(workspaceId, worktree); + } if (workspaceId != null && instanceId != null) { return this.options.workspaceQuarantineResolver?.( workspaceId, @@ -1412,7 +1515,27 @@ export class BridgeWorker { const workspaceId = this.assignmentBaseWorkspaceId(assignment); if (workspaceId == null) return undefined; const instanceId = this.assignmentWorkspaceInstanceId(assignment); - return workspaceIsolationKey(workspaceId, instanceId); + return workspaceIsolationKey( + workspaceId, + instanceId, + this.assignmentWorktree(assignment), + ); + } + + private assignmentWorktree(assignment: BridgeAssignment): string | undefined { + if ( + assignment.executionKind === 'workspace_tool' && + isWorkspaceToolRequest(assignment.request) + ) { + return assignment.request.worktree; + } + if ( + assignment.executionKind === 'workspace_programmatic' && + isBridgeWorkspaceProgrammaticRequest(assignment.request) + ) { + return assignment.request.body.workspace_worktree; + } + return undefined; } private assignmentBaseWorkspaceId( @@ -1580,9 +1703,11 @@ export class BridgeWorker { } if ( this.options.workspaceQuarantines != null || - this.options.workspaceQuarantineResolver != null + this.options.workspaceQuarantineResolver != null || + this.options.linkedWorktreeQuarantineResolver != null ) { await guard?.assertAvailable(); + await this.assertLaneFamilyAvailable(workspaceKey, assignment); } } catch (error) { throw new BridgeWorkspaceQuarantinedError( @@ -1615,6 +1740,14 @@ export class BridgeWorker { 'Workspace instance type is not advertised', ); } + if ( + workspaceRequest.worktree != null && + workspace.workspaceScopes?.includes('git_linked_worktree') !== true + ) { + throw new BridgeProtocolError( + 'Linked worktree lanes are not advertised for workspace', + ); + } if ( workspace.operations != null && !workspace.operations.includes(workspaceRequest.operation) @@ -1739,9 +1872,11 @@ export class BridgeWorker { } if ( this.options.workspaceQuarantines != null || - this.options.workspaceQuarantineResolver != null + this.options.workspaceQuarantineResolver != null || + this.options.linkedWorktreeQuarantineResolver != null ) { await guard?.assertAvailable(); + await this.assertLaneFamilyAvailable(workspaceKey, assignment); } } catch (error) { throw new BridgeWorkspaceQuarantinedError( @@ -1772,6 +1907,14 @@ export class BridgeWorker { 'Workspace instance type is not advertised', ); } + if ( + assignment.request.body.workspace_worktree != null && + workspace.workspaceScopes?.includes('git_linked_worktree') !== true + ) { + throw new BridgeProtocolError( + 'Linked worktree lanes are not advertised for workspace', + ); + } this.mutationGuardArmed = true; try { this.armedWorkspaces.add(workspaceKey); diff --git a/packages/code/src/workspace-cli.test.ts b/packages/code/src/workspace-cli.test.ts index 1de68818..52743bae 100644 --- a/packages/code/src/workspace-cli.test.ts +++ b/packages/code/src/workspace-cli.test.ts @@ -147,6 +147,47 @@ test('CLI requires concurrent native slots for conversation worktrees', async (t assert.match(result.stderr, /at least two workspace lease slots/i); }); +test('CLI requires concurrent native slots and no conversation worktrees for linked worktree lanes', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-lanes-')); + const workspaceRoot = join(root, 'workspace'); + await mkdir(workspaceRoot); + t.after(() => rm(root, { recursive: true, force: true })); + const run = (...extra: string[]) => + spawnSync( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'run', + '--worker-dir', + workspaceRoot, + '--allow-workspace-writes', + '--allow-workspace-commands', + '--linked-worktree-lanes', + ...extra, + ], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + }, + }, + ); + const serial = run(); + assert.notEqual(serial.status, 0); + assert.match(serial.stderr, /Linked worktree lanes require .*at least two workspace lease slots/i); + const combined = run( + '--workspace-lease-slots', + '2', + '--conversation-worktree-root', + join(root, 'conversations'), + ); + assert.notEqual(combined.status, 0); + assert.match(combined.stderr, /cannot be combined with conversation worktrees/i); +}); + test('CLI advertises explicitly enabled writes without exposing the workspace root', async (t) => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-cli-')); const workspaceRoot = join(root, ' '); diff --git a/service/src/bridge/linked-worktree.test.ts b/service/src/bridge/linked-worktree.test.ts new file mode 100644 index 00000000..5a286726 --- /dev/null +++ b/service/src/bridge/linked-worktree.test.ts @@ -0,0 +1,197 @@ +import { createHash, randomUUID } from 'node:crypto'; +import { expect, test } from 'bun:test'; +import Redis from 'ioredis'; +import { RedisBridgeStore } from './store'; +import type { CodeBridgeAssignment } from './store'; +import type { WorkspaceToolRequest } from '../../../packages/code/src/protocol'; + +const redisUrl = process.env.BRIDGE_TEST_REDIS_URL; +const incarnationId = 'lane-incarnation'; + +function fenceKey(workerId: string, fence: string): string { + const hash = createHash('sha256').update(fence).digest('hex'); + return `codeapi:bridge:v1:worker:${encodeURIComponent(workerId)}:workspace:${hash}:quarantined`; +} + +async function withLaneWorker( + run: (context: { + store: RedisBridgeStore; + redis: Redis; + workerId: string; + dispatch: ( + request: Partial, + budgetMs?: number, + ) => Promise; + lease: (slot: number) => Promise; + settle: (assignment: CodeBridgeAssignment) => Promise; + }) => Promise, + scopes = true, +): Promise { + const redis = new Redis(redisUrl!); + const store = new RedisBridgeStore(redis, 60, 1000, 3); + const workerId = `lanes-${randomUUID()}`; + const controller = new AbortController(); + const pending: Promise[] = []; + try { + const generation = await store.register({ + protocolVersion: 1, + workerId, + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + workspaceLeaseSlots: 3, + requiresReadyConfirmation: true, + workspaceTools: { + protocolVersion: 1, + operations: ['read_file'], + workspaces: [ + { + id: 'repo', + ...(scopes ? { workspaceScopes: ['git_linked_worktree' as const] } : {}), + }, + ], + }, + }, + }); + await store.confirmReady(workerId, incarnationId, generation); + await run({ + store, + redis, + workerId, + dispatch(request, budgetMs = 3000) { + const result = store.dispatchWorkspaceTool({ + workerId, + signal: controller.signal, + deadlineAtMs: Date.now() + budgetMs, + executionTimeoutMs: 5000, + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'repo', + path: 'probe', + ...request, + } as WorkspaceToolRequest, + }); + void result.catch(() => undefined); + pending.push(result); + return result; + }, + lease: (slot) => store.lease(workerId, incarnationId, 1000, undefined, undefined, slot), + async settle(assignment) { + await store.acknowledgeLease( + workerId, + incarnationId, + assignment.assignmentId, + assignment.generation, + assignment.leaseToken, + ); + await store.settle(workerId, assignment.assignmentId, { + protocolVersion: 1, + incarnationId, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + status: 'rejected', + error: 'probe complete', + }); + await store.confirmWorkspaceCleanup(workerId, assignment.assignmentId, { + protocolVersion: 1, + incarnationId, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + status: 'rejected', + error: 'local cleanup confirmed', + }); + }, + }); + } finally { + controller.abort(); + await Promise.allSettled(pending); + await redis.quit(); + } +} + +test.skipIf(!redisUrl)( + 'sibling linked-worktree lanes run together while their checkout waits for both', + async () => { + await withLaneWorker(async ({ dispatch, lease, settle }) => { + const laneA = dispatch({ worktree: 'task-a' }); + const laneB = dispatch({ worktree: 'task-b' }); + const first = await lease(0); + const second = await lease(1); + expect( + [first?.request, second?.request].map( + (request) => (request as WorkspaceToolRequest).worktree, + ).sort(), + ).toEqual(['task-a', 'task-b']); + + const checkout = dispatch({}, 400); + expect(await lease(2)).toBeUndefined(); + await expect(checkout).rejects.toMatchObject({ code: 'WORKSPACE_QUEUE_TIMEOUT' }); + + await settle(first!); + await settle(second!); + await Promise.allSettled([laneA, laneB]); + }); + }, +); + +test.skipIf(!redisUrl)('a busy checkout holds every lane beneath it', async () => { + await withLaneWorker(async ({ dispatch, lease, settle }) => { + const checkout = dispatch({}); + const held = await lease(0); + expect((held?.request as WorkspaceToolRequest).worktree).toBeUndefined(); + + const lane = dispatch({ worktree: 'task-a' }); + expect(await lease(1)).toBeUndefined(); + + await settle(held!); + await Promise.allSettled([checkout]); + const admitted = await lease(0); + expect((admitted?.request as WorkspaceToolRequest).worktree).toBe('task-a'); + await settle(admitted!); + await Promise.allSettled([lane]); + }); +}); + +test.skipIf(!redisUrl)('a lane may not start while its checkout is quarantined', async () => { + await withLaneWorker(async ({ redis, workerId, dispatch }) => { + await redis.set(fenceKey(workerId, 'native-workspace:repo'), 'quarantined:earlier'); + await expect(dispatch({ worktree: 'task-a' })).rejects.toMatchObject({ + code: 'WORKSPACE_QUARANTINED', + }); + expect(await redis.exists(fenceKey(workerId, 'native-workspace:\0linked-worktree\0repo\0task-a'))).toBe(0); + }); +}); + +test.skipIf(!redisUrl)('a checkout is refused while a lane beneath it keeps a stuck fence', async () => { + await withLaneWorker(async ({ redis, workerId, dispatch, lease, settle }) => { + const settleNext = async (request: Partial) => { + const pending = dispatch(request); + const assignment = await lease(0); + expect((assignment?.request as WorkspaceToolRequest).worktree).toBe(request.worktree); + await settle(assignment!); + await Promise.allSettled([pending]); + }; + await settleNext({ worktree: 'task-a' }); + await settleNext({}); + + // A lane whose cleanup was never acknowledged keeps its fence after its slot is released. + await settleNext({ worktree: 'task-a' }); + await redis.set( + fenceKey(workerId, 'native-workspace:\0linked-worktree\0repo\0task-a'), + 'quarantined:cleanup-unacknowledged', + ); + + await expect(dispatch({})).rejects.toMatchObject({ code: 'WORKSPACE_QUARANTINED' }); + }); +}); + +test.skipIf(!redisUrl)('a lane is refused unless the worker advertises linked-worktree scopes', async () => { + await withLaneWorker(async ({ dispatch }) => { + await expect(dispatch({ worktree: 'task-a' })).rejects.toMatchObject({ + code: 'WORKER_MISMATCH', + }); + }, false); +}); diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index b3d549b1..962c6242 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -539,6 +539,7 @@ router.post( supportedWorkspaceListFileFeatures: ['after_path'], supportedWorkspaceProgrammaticLanguages: ['bash'], supportedWorkspaceInstanceTypes: ['git_worktree'], + supportedWorkspaceScopes: ['git_linked_worktree'], }); } catch (error) { if (error instanceof BridgeStoreError) { diff --git a/service/src/bridge/slots.test.ts b/service/src/bridge/slots.test.ts index dfe89495..1efe3b97 100644 --- a/service/src/bridge/slots.test.ts +++ b/service/src/bridge/slots.test.ts @@ -3,6 +3,7 @@ import RedisMock from 'ioredis-mock'; import type Redis from 'ioredis'; import { BridgeAdmissionQueue } from './admission'; import { BridgeWorkspaceSlots } from './slots'; +import { workspaceIsolationKey } from '../../../packages/code/src/protocol'; const redis = new RedisMock() as unknown as Redis; const admission = new BridgeAdmissionQueue(redis); @@ -13,20 +14,27 @@ const prefix = `codeapi:bridge:v1:worker:${workerId}`; afterEach(async () => { await redis.flushall(); }); -async function enqueue(assignmentId: string, workspaceId: string) { +async function enqueue(assignmentId: string, workspaceId: string, capacity = 2) { await redis.set(`${prefix}:incarnation`, incarnationId); - await redis.set(`${prefix}:workspace-slot-capacity`, '2'); + await redis.set(`${prefix}:workspace-slot-capacity`, String(capacity)); await admission.enter(workerId, assignmentId, Date.now() + 5000, workspaceId); return { workerId, incarnationId, assignmentId, workspaceId, - capacity: 2, + capacity, expiresAtMs: Date.now() + 10000, }; } +async function finish(assignmentId: string) { + await slots.release(workerId, incarnationId, assignmentId); + await admission.leave(workerId, assignmentId); +} + +const lane = (name: string, parent = 'repo') => workspaceIsolationKey(parent, undefined, name); + test('slots admit independent workspaces, skip a busy root, and bound capacity', async () => { const a = await enqueue('a', 'root-a'); const a2 = await enqueue('a2', 'root-a'); @@ -91,3 +99,51 @@ test('releasing a long slot shortens the aggregate expiry to remaining work', as await slots.release(workerId, incarnationId, 'a'); expect(await redis.pttl(`${prefix}:lock`)).toBeLessThanOrEqual(3000); }); + +test('sibling linked-worktree lanes share their checkout, which waits for both', async () => { + const a = await enqueue('a', lane('task-a'), 3); + const b = await enqueue('b', lane('task-b'), 3); + const root = await enqueue('root', 'repo', 3); + expect(await slots.reserve(a)).toBe(0); + expect(await slots.reserve(b)).toBe(1); + expect(await slots.reserve(root)).toBeUndefined(); + await finish('a'); + expect(await slots.reserve(root)).toBeUndefined(); + await finish('b'); + expect(await slots.reserve(root)).toBe(0); +}); + +test('a busy checkout holds every lane beneath it but not other checkouts', async () => { + const root = await enqueue('root', 'repo', 3); + const a = await enqueue('a', lane('task-a'), 3); + const other = await enqueue('other', lane('task-a', 'other-repo'), 3); + expect(await slots.reserve(root)).toBe(0); + expect(await slots.reserve(a)).toBeUndefined(); + expect(await slots.reserve(other)).toBe(1); + await finish('root'); + expect(await slots.reserve(a)).toBe(0); +}); + +test('a waiting checkout holds back newer lanes so root work cannot starve', async () => { + const a = await enqueue('a', lane('task-a'), 3); + expect(await slots.reserve(a)).toBe(0); + const root = await enqueue('root', 'repo', 3); + const b = await enqueue('b', lane('task-b'), 3); + const other = await enqueue('other', 'other-repo', 3); + expect(await slots.reserve(root)).toBeUndefined(); + expect(await slots.reserve(b)).toBeUndefined(); + expect(await slots.reserve(other)).toBe(1); + await finish('a'); + expect(await slots.reserve(b)).toBeUndefined(); + expect(await slots.reserve(root)).toBe(0); +}); + +test('lanes nest beneath a conversation checkout, not the source root', async () => { + const instance = workspaceIsolationKey('repo', 'c'.repeat(64)); + const conversation = await enqueue('conversation', instance, 3); + const nested = await enqueue('nested', workspaceIsolationKey('repo', 'c'.repeat(64), 'task-a'), 3); + const source = await enqueue('source', lane('task-a'), 3); + expect(await slots.reserve(conversation)).toBe(0); + expect(await slots.reserve(nested)).toBeUndefined(); + expect(await slots.reserve(source)).toBe(1); +}); diff --git a/service/src/bridge/slots.ts b/service/src/bridge/slots.ts index 9c9e7d3b..81f60ebe 100644 --- a/service/src/bridge/slots.ts +++ b/service/src/bridge/slots.ts @@ -6,6 +6,12 @@ export const MAX_WORKSPACE_LEASE_SLOTS = 8; /** Reservations share the legacy admission queue and aggregate worker lock. * Thus a serial dispatcher or replacement incarnation cannot race active slots. * Workspace mutation uncertainty is fenced separately by the assignment store. + * + * Keys are hierarchical: a linked-worktree lane (`\0linked-worktree\0\0`) + * conflicts with itself and with its parent checkout, and a checkout conflicts + * with every lane beneath it. Sibling lanes run concurrently. A waiting checkout + * request also holds back newer lanes beneath it, so root operations such as + * `git worktree add` cannot be starved by a steady stream of lane work. */ export class BridgeWorkspaceSlots { constructor(private readonly redis: Redis) {} @@ -47,10 +53,26 @@ export class BridgeWorkspaceSlots { [ "if redis.call('GET', KEYS[4]) ~= ARGV[1] then return -2 end", "if (redis.call('GET', KEYS[8]) or '1') ~= ARGV[4] then return -2 end", + // Parent of a linked-worktree lane key; nil for a checkout key. + 'local lanePrefix = "\\0linked-worktree\\0"', + 'local function parentOf(key)', + ' if string.sub(key, 1, #lanePrefix) ~= lanePrefix then return nil end', + ' local last = nil', + ' local cursor = #lanePrefix + 1', + ' while true do', + ' local found = string.find(key, "\\0", cursor, true)', + ' if not found then break end', + ' last = found', + ' cursor = found + 1', + ' end', + ' if last == nil or last <= #lanePrefix + 1 then return nil end', + ' return string.sub(key, #lanePrefix + 1, last - 1)', + 'end', "local lock = redis.call('GET', KEYS[2])", "local owner = 'workspace-slots:' .. ARGV[1]", 'if lock and lock ~= owner then return -1 end', 'local busy = {}', + 'local busyChildren = {}', 'local free = nil', 'local latest = tonumber(ARGV[5])', `for slot = 0, ${MAX_WORKSPACE_LEASE_SLOTS - 1} do`, @@ -63,12 +85,20 @@ export class BridgeWorkspaceSlots { ' else', ' if entry[1] == ARGV[2] then return slot end', ' busy[entry[3]] = true', + ' local busyParent = parentOf(entry[3])', + ' if busyParent then busyChildren[busyParent] = true end', ' latest = math.max(latest, tonumber(entry[4]))', ' end', ' end', ' if not occupied and free == nil and slot < tonumber(ARGV[4]) then free = slot end', 'end', - 'if free == nil or busy[ARGV[3]] then return -1 end', + 'local function conflicts(key)', + ' if busy[key] then return true end', + ' local parent = parentOf(key)', + ' if parent then return busy[parent] == true end', + ' return busyChildren[key] == true', + 'end', + 'if free == nil or conflicts(ARGV[3]) then return -1 end', // Expiry removes queue metadata, never a workspace uncertainty fence. "local expired = redis.call('ZRANGEBYSCORE', KEYS[6], '-inf', ARGV[6])", 'for _, id in ipairs(expired) do', @@ -78,11 +108,15 @@ export class BridgeWorkspaceSlots { 'end', "local pending = redis.call('ZRANGE', KEYS[5], 0, 31)", 'local selected = nil', + 'local waitingCheckouts = {}', 'for _, id in ipairs(pending) do', " local workspace = redis.call('HGET', KEYS[7], id)", // An older serial request remains a barrier until its dispatcher finishes. ' if not workspace then return -1 end', - ' if not busy[workspace] then selected = id; break end', + ' local parent = parentOf(workspace)', + ' local held = conflicts(workspace) or (parent ~= nil and waitingCheckouts[parent] == true)', + ' if not held then selected = id; break end', + ' if parent == nil then waitingCheckouts[workspace] = true end', 'end', 'if selected ~= ARGV[2] then return -1 end', "redis.call('HSET', KEYS[1], 'a:' .. free, ARGV[2], 'i:' .. free, ARGV[1], 'w:' .. free, ARGV[3], 'e:' .. free, ARGV[5])", diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 0ca75566..123e7ee2 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -18,6 +18,7 @@ import { isWorkspaceToolRequest, isWorkspaceToolResult, workspaceIsolationKey, + workspaceIsolationParent, } from '../../../packages/code/src/protocol'; import type { BridgeWorkerBinding } from './pairing'; import { BridgeAdmissionQueue } from './admission'; @@ -105,6 +106,25 @@ type AssignmentOwnership = Pick< | 'runtimeSessionId' >; +const NATIVE_WORKSPACE_FENCE_PREFIX = 'native-workspace:'; + +/** Fence keys of the linked-worktree lanes enqueued beneath a checkout fence. */ +function workspaceLaneFenceIndexKey(checkoutFenceKey: string): string { + return `${checkoutFenceKey}:lanes`; +} + +/** The fence of the checkout a linked-worktree lane nests beneath. While that + * checkout is quarantined, its lanes may not start either. */ +function workspaceFenceParent(fence: string): string | undefined { + if (!fence.startsWith(NATIVE_WORKSPACE_FENCE_PREFIX)) return undefined; + const parent = workspaceIsolationParent( + fence.slice(NATIVE_WORKSPACE_FENCE_PREFIX.length), + ); + return parent === undefined + ? undefined + : `${NATIVE_WORKSPACE_FENCE_PREFIX}${parent}`; +} + function workspaceFenceReceiptKey(assignmentId: string): string { return `${assignmentKey(assignmentId)}:workspace-fence-owner`; } @@ -151,6 +171,12 @@ function supportsWorkspaceTool( ) { return false; } + if ( + request.worktree !== undefined && + workspace.workspaceScopes?.includes('git_linked_worktree') !== true + ) { + return false; + } if (request.operation === 'list_files' && request.afterPath !== undefined) { return capabilities?.listFileFeatures?.includes('after_path') === true; } @@ -184,6 +210,7 @@ function supportsWorkspaceProgrammatic( workspaceId: string, language: string, workspaceInstanceId?: string, + worktree?: string, ): boolean { const capabilities = registration.capabilities.workspaceTools; const workspace = capabilities?.workspaces.find( @@ -193,6 +220,8 @@ function supportsWorkspaceProgrammatic( workspace != null && (workspaceInstanceId === undefined || workspace.workspaceInstances?.includes('git_worktree') === true) && + (worktree === undefined || + workspace.workspaceScopes?.includes('git_linked_worktree') === true) && capabilities?.operations.includes('execute_command') === true && (workspace.operations == null || workspace.operations.includes('execute_command')) && @@ -214,11 +243,24 @@ function workspaceInstanceId(body: t.PayloadBody): string | undefined { return undefined; } +function workspaceWorktree(body: t.PayloadBody): string | undefined { + if ( + typeof body === 'object' && + body != null && + 'workspace_worktree' in body && + typeof body.workspace_worktree === 'string' + ) { + return body.workspace_worktree; + } + return undefined; +} + export function workspaceAdmissionId( workspaceId: string, instanceId?: string, + worktree?: string, ): string { - return workspaceIsolationKey(workspaceId, instanceId); + return workspaceIsolationKey(workspaceId, instanceId, worktree); } function workerKey(workerId: string): string { @@ -862,6 +904,7 @@ export class RedisBridgeStore { args.workspaceId, args.body.language, workspaceInstanceId(args.body), + workspaceWorktree(args.body), ) ) { throw new BridgeStoreError( @@ -899,12 +942,15 @@ export class RedisBridgeStore { args.workspaceRequest?.workspaceId ?? args.workspaceId; const selectedWorkspaceInstanceId = args.workspaceRequest?.workspaceInstanceId ?? workspaceInstanceId(args.body); + const selectedWorktree = + args.workspaceRequest?.worktree ?? workspaceWorktree(args.body); const selectedWorkspaceAdmissionId = selectedWorkspaceId == null ? undefined : workspaceAdmissionId( selectedWorkspaceId, selectedWorkspaceInstanceId, + selectedWorktree, ); const workspaceSlots = selectedWorkspaceId != null && @@ -1033,6 +1079,7 @@ export class RedisBridgeStore { args.workspaceId, args.body.language, workspaceInstanceId(args.body), + workspaceWorktree(args.body), )) ) { throw new BridgeStoreError( @@ -1059,13 +1106,13 @@ export class RedisBridgeStore { leaseToken, leaseTokenHash: tokenHash(leaseToken), ...(selectedWorkspaceAdmissionId == null ? {} : { - workspaceFence: `native-workspace:${selectedWorkspaceAdmissionId}`, + workspaceFence: `${NATIVE_WORKSPACE_FENCE_PREFIX}${selectedWorkspaceAdmissionId}`, }), ...(workspaceLeaseSlot === undefined ? {} : { workspaceLeaseSlot, - workspaceFence: `native-workspace:${selectedWorkspaceAdmissionId!}`, + workspaceFence: `${NATIVE_WORKSPACE_FENCE_PREFIX}${selectedWorkspaceAdmissionId!}`, }), ...(registration.identityId != null ? { workerIdentityId: registration.identityId } @@ -1155,6 +1202,7 @@ export class RedisBridgeStore { args.workspaceId, args.body.language, workspaceInstanceId(args.body), + workspaceWorktree(args.body), ) ) { throw new BridgeStoreError( @@ -2185,6 +2233,13 @@ export class RedisBridgeStore { "if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end", 'if ARGV[7] ~= "" and redis.call(\'GET\', KEYS[6]) ~= ARGV[7] then return 0 end', "if #KEYS >= 7 and redis.call('EXISTS', KEYS[7]) == 1 then return -1 end", + // A lane refuses a quarantined checkout. A checkout reaches enqueue only once no + // lane beneath it holds a slot, so any lane fence still indexed here is stuck. + "if ARGV[9] == 'lane' and redis.call('EXISTS', KEYS[10]) == 1 then return -1 end", + "if ARGV[9] == 'checkout' then", + " for i = 11, #KEYS do if redis.call('EXISTS', KEYS[i]) == 1 then return -1 end end", + " for i = 11, #KEYS do redis.call('SREM', KEYS[10], KEYS[i]) end", + 'end', 'redis.call(\'SET\', KEYS[2], ARGV[2], \"EX\", ARGV[3])', "redis.call('RPUSH', KEYS[3], ARGV[4])", "redis.call('EXPIRE', KEYS[3], ARGV[3])", @@ -2193,7 +2248,8 @@ export class RedisBridgeStore { : []), 'redis.call(\'SET\', KEYS[5], "1", \"PXAT\", ARGV[6])', "if #KEYS >= 7 then redis.call('SET', KEYS[7], ARGV[4]) end", - 'if #KEYS == 9 then', + "if ARGV[9] == 'lane' then redis.call('SADD', KEYS[11], KEYS[7]) end", + 'if #KEYS >= 9 then', " local epoch = redis.call('GET', KEYS[9])", " if type(epoch) ~= 'string' then epoch = '0'; redis.call('SET', KEYS[9], epoch, 'EX', ARGV[3]) end", " if redis.call('PTTL', KEYS[9]) < tonumber(ARGV[3]) * 1000 then redis.call('EXPIRE', KEYS[9], ARGV[3]) end", @@ -2233,11 +2289,29 @@ export class RedisBridgeStore { workerIdentityId: assignment.workerIdentityId, expiresAt: assignment.expiresAt, }; + let fenceScope: '' | 'lane' | 'checkout' = ''; if (assignment.workspaceLeaseSlot !== undefined) { keys.push( workspaceFenceReceiptKey(assignment.assignmentId), `${workspaceQuarantineKey(assignment.workerId, assignment.workspaceFence!)}:epoch`, ); + const parent = workspaceFenceParent(assignment.workspaceFence!); + if (parent !== undefined) { + const parentFence = workspaceQuarantineKey(assignment.workerId, parent); + keys.push(parentFence, workspaceLaneFenceIndexKey(parentFence)); + fenceScope = 'lane'; + } else { + const index = workspaceLaneFenceIndexKey( + workspaceQuarantineKey(assignment.workerId, assignment.workspaceFence!), + ); + const laneFences = await boundedCommand( + this.redis.smembers(index), + this.redisCommandTimeoutMs, + 'Bridge linked worktree fence index read', + ); + keys.push(index, ...laneFences); + fenceScope = 'checkout'; + } } const result = await this.redis.eval( script, @@ -2251,6 +2325,7 @@ export class RedisBridgeStore { String(Date.parse(assignment.expiresAt)), readyToken ?? '', JSON.stringify(receipt), + fenceScope, ); if (Number(result) === -1) { throw new BridgeStoreError( diff --git a/service/src/service/programmatic-router.ts b/service/src/service/programmatic-router.ts index a646b9ad..1e2a367c 100644 --- a/service/src/service/programmatic-router.ts +++ b/service/src/service/programmatic-router.ts @@ -92,7 +92,11 @@ import { CODEAPI_BRIDGE_WORKSPACE_HEADER, resolveBridgeWorkerSelection, } from '../bridge/selection'; -import { isValidBridgeWorkerId, BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_INPUT_FILES } from '../../../packages/code/src/protocol'; +import { + isValidBridgeWorkerId, + isValidLinkedWorktreeName, + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_INPUT_FILES, +} from '../../../packages/code/src/protocol'; import logger from '../logger'; import { type ExecutionState, @@ -505,6 +509,7 @@ async function handleReplayInitial( bridgeWorkerId?: string; workspaceId?: string; workspaceInstanceId?: string; + workspaceWorktree?: string; }, cancellation: ReplayRequestCancellation, ): Promise { @@ -514,6 +519,7 @@ async function handleReplayInitial( bridgeWorkerId, workspaceId, workspaceInstanceId, + workspaceWorktree, } = params; const { code, tools, user_id, files } = req.body as t.ProgrammaticRequestBody; @@ -671,6 +677,7 @@ async function handleReplayInitial( bridgeWorkerId, workspaceId, workspaceInstanceId, + workspaceWorktree, executionProfile: env.EXECUTION_PROFILE, executionProfileSource: env.EXECUTION_PROFILE_SOURCE, sandboxBackend: resolveReplayStateSandboxBackend({ @@ -1291,6 +1298,7 @@ router.post( let bridgeWorkerId: string | undefined; let workspaceId: string | undefined; let workspaceInstanceId: string | undefined; + let workspaceWorktree: string | undefined; if (continuation_token == null || continuation_token === '') { try { const bridgeSelection = resolveBridgeWorkerSelection({ @@ -1341,6 +1349,15 @@ router.post( principalId: principal.userId, }); } + const requestedWorktree = rawBody.workspace_worktree; + if (requestedWorktree !== undefined) { + if (workspaceId == null || !isValidLinkedWorktreeName(requestedWorktree)) { + return res.status(400).json({ + error: 'Invalid code workspace worktree', + }); + } + workspaceWorktree = requestedWorktree; + } } catch (error) { if (error instanceof BridgeWorkerSelectionError) { return res @@ -1458,6 +1475,7 @@ router.post( bridgeWorkerId, workspaceId, workspaceInstanceId, + workspaceWorktree, }, cancellation); } if (workspaceId != null) { diff --git a/service/src/service/programmatic-state.ts b/service/src/service/programmatic-state.ts index 38c72486..056fcbb5 100644 --- a/service/src/service/programmatic-state.ts +++ b/service/src/service/programmatic-state.ts @@ -40,6 +40,7 @@ export interface BuildReplayExecutionStateParams { bridgeWorkerId?: string; workspaceId?: string; workspaceInstanceId?: string; + workspaceWorktree?: string; sandboxBackend?: SandboxBackendName; executionProfile: ExecutionProfile; executionProfileSource: ExecutionProfileSource; @@ -70,6 +71,7 @@ export function buildReplayExecutionState( bridgeWorkerId: params.bridgeWorkerId, workspaceId: params.workspaceId, workspaceInstanceId: params.workspaceInstanceId, + workspaceWorktree: params.workspaceWorktree, sandboxBackend: params.sandboxBackend, executionProfile: params.executionProfile, executionProfileSource: params.executionProfileSource, @@ -86,12 +88,21 @@ export function buildReplayExecutionState( }; } -/** Bind the authenticated conversation checkout to every replay iteration. */ +/** Bind the authenticated conversation checkout and linked-worktree lane to every replay iteration. */ export function bindReplayWorkspaceInstance( payload: t.PayloadBody, - state: Pick, + state: Pick, ): t.PayloadBody { - return state.workspaceInstanceId == null - ? payload - : { ...payload, workspace_instance_id: state.workspaceInstanceId }; + if (state.workspaceInstanceId == null && state.workspaceWorktree == null) { + return payload; + } + return { + ...payload, + ...(state.workspaceInstanceId == null + ? {} + : { workspace_instance_id: state.workspaceInstanceId }), + ...(state.workspaceWorktree == null + ? {} + : { workspace_worktree: state.workspaceWorktree }), + }; } diff --git a/service/src/service/replay-state.ts b/service/src/service/replay-state.ts index fd4e90e3..003ce6b5 100644 --- a/service/src/service/replay-state.ts +++ b/service/src/service/replay-state.ts @@ -120,6 +120,8 @@ export interface ExecutionState { workspaceId?: string; /** Selected conversation checkout retained across every replay iteration. */ workspaceInstanceId?: string; + /** Selected linked-worktree lane retained across every replay iteration. */ + workspaceWorktree?: string; /** Original queue/backend target retained across replay continuations. */ sandboxBackend?: SandboxBackendName; /** Original producer profile retained so continuations use the same queue. */ diff --git a/service/src/types/service.ts b/service/src/types/service.ts index 91c3ed89..b0375b38 100644 --- a/service/src/types/service.ts +++ b/service/src/types/service.ts @@ -206,6 +206,8 @@ export interface PayloadBody { version: string; /** Opaque conversation checkout selected and authenticated by the API. */ workspace_instance_id?: string; + /** Linked worktree lane at `.worktrees/` beneath the selected checkout. */ + workspace_worktree?: string; /** Stable identity shared by all replay iterations of one execution. */ execution_id?: string; replay_tool_count?: number; @@ -397,6 +399,8 @@ export interface ProgrammaticRequestBody { lang?: 'python' | 'bash'; /** Opaque conversation checkout binding for a selected native workspace. */ workspace_instance_id?: string; + /** Linked worktree lane at `.worktrees/` beneath the selected checkout. */ + workspace_worktree?: string; } export interface ProgrammaticToolCall { From dbb84779312d473a3f3eee763e36c7f92804aab3 Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Tue, 29 Sep 2026 12:05:10 -0400 Subject: [PATCH 15/48] =?UTF-8?q?=F0=9F=9B=A1=EF=B8=8F=20fix:=20Guard=20Li?= =?UTF-8?q?nked-Worktree=20Lanes=20Against=20Accidental=20Git=20Pruning=20?= =?UTF-8?q?(#272)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Schedule Linked Worktrees as Their Own Workspace Lanes * fix: Refuse quarantined-parent lanes before enqueue, forward lane policy, reset lane fences - Check the parent checkout fence before the assignment is stored or queued. - Forward the linked worktree policy to the forked native sandbox. - Protect shared Git config, hooks and info even before they exist. - Reset a lane fence with --reset-workspace-worktree. * fix: Grant lanes an explicit Git write allowlist, hold checkouts on quarantined lanes, release stale lanes - A lane writes only shared objects, refs, ref logs, LFS storage and its own worktree metadata; the checkout HEAD, index, operation state, config and hooks stay read-only without per-path denies. - A checkout assignment waits on any quarantined lane beneath it. - Lane registrations are released when their worktree disappears and capped at 32, dropping command roots and credential routes. * fix: Refuse a checkout while a lane keeps a stuck fence; let lane probes read shared Git - Code API indexes the lane fences enqueued beneath each checkout. A checkout reaches enqueue only once no lane holds a slot, so an indexed fence that still exists is stuck and the checkout is refused, surviving worker restarts. - Replay probes of a lane may read its common Git directory (read-only). * fix: Guard linked-worktree Git grants and unresolved actions * fix: Guard linked-worktree lanes against accidental Git pruning * fix: Refuse Git Aliases and Allow Pathspec Flags in Worktree Lanes * fix: Guard Worktree Git Against Indirect Pruning * fix: Refuse Git Repository Dispatchers in Worktree Lanes --------- Co-authored-by: Danny Avila Co-authored-by: Lia --- packages/code/README.md | 37 ++- packages/code/src/cli.ts | 3 + .../src/linked-worktree-git-guard.test.ts | 276 ++++++++++++++++++ .../code/src/linked-worktree-git-guard.ts | 123 ++++++++ packages/code/src/native-sandbox.test.ts | 81 +++++ packages/code/src/native-sandbox.ts | 46 ++- 6 files changed, 556 insertions(+), 10 deletions(-) create mode 100644 packages/code/src/linked-worktree-git-guard.test.ts create mode 100644 packages/code/src/linked-worktree-git-guard.ts diff --git a/packages/code/README.md b/packages/code/README.md index e3051930..a38523a3 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -826,15 +826,34 @@ shared object and ref storage (`.git/objects`, `.git/refs`, `.git/logs/refs`, `.git` stays read-only: configuration, hooks and `info`, the checkout's own `HEAD`, index and merge or rebase state, and sibling metadata. Automatic `gc` and maintenance are disabled, and `git gc` itself cannot run in a lane (it -needs to write `.git/gc.pid` and `packed-refs`); run storage maintenance from -the checkout. Explicit `git prune --expire now`, `git repack -ad`, or LFS -pruning can still delete objects another lane is writing; the sandbox does -**not** prevent this race. Do not enable lanes for agents that run destructive -maintenance until those commands are blocked or serialized. Deleting a branch -or tag also needs the checkout, because Git locks `packed-refs` for every ref -deletion. Each lane has its own durable quarantine guard. A lane -cannot start while its checkout is quarantined, and a checkout cannot start -while any lane beneath it is. +needs to write `.git/gc.pid` and `packed-refs`). A lane's `PATH` starts with +a read-only Git wrapper that refuses `prune`, `gc`, `repack`, `prune-packed`, +`maintenance`, `multi-pack-index`, `for-each-repo` and `git lfs prune`, +including after Git options such as `-C` or `-c`. `for-each-repo` is refused +entirely because Git dispatches its child commands without re-entering the +wrapper. It rejects `git lfs fetch` or `pull` with +`--prune`/`-p`, and `git fetch` or `pull` with `--auto-maintenance`/`--auto-gc`. +It enforces `maintenance.auto=false`, `gc.auto=0` and `help.autocorrect=0` +after caller-supplied Git options, preventing those overrides from restoring +automatic maintenance or correcting a misspelled command to `prune`. +It also refuses **all configured Git aliases**, +including harmless ones: aliases can hide destructive maintenance through +local config, `-c`, `--config-env`, includes or shell commands. Call the +underlying Git command instead, or run the alias from the checkout. Git's +safe pathspec options (`--literal-pathspecs`, `--glob-pathspecs`, +`--noglob-pathspecs`, `--icase-pathspecs`) remain usable in lanes. Run storage +maintenance from the checkout. + +This is a guardrail against accidental commands, **not** a filesystem +boundary: invoking Git by an absolute path, resetting `PATH`, or using a +script that does either bypasses it. Such commands can still delete a sibling +lane's unpublished objects. Do not enable concurrent lanes for agents or +scripts that deliberately bypass the wrapper. The guard requires `/bin/bash` +and a trusted system Git executable; without either, lane commands are not +admitted. Deleting a branch or tag also needs the checkout, because Git locks +`packed-refs` for every ref deletion. Each lane has its own durable quarantine +guard. A lane cannot start while its checkout is quarantined, and a checkout +cannot start while any lane beneath it is. Lanes require native-srt commands and at least two lease slots, and cannot yet be combined with conversation worktrees. Code API must advertise diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 135b9f8e..410caf25 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -805,6 +805,9 @@ async function run( 'Linked worktree lanes cannot be combined with conversation worktrees', ); } + if (linkedWorktreeLanes && process.platform === 'win32') { + throw new Error('Linked worktree Git guard requires a POSIX host'); + } if ( roots.length > 1 && process.env.LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE?.trim() diff --git a/packages/code/src/linked-worktree-git-guard.test.ts b/packages/code/src/linked-worktree-git-guard.test.ts new file mode 100644 index 00000000..96f16b4a --- /dev/null +++ b/packages/code/src/linked-worktree-git-guard.test.ts @@ -0,0 +1,276 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { promisify } from 'node:util'; + +import { writeLinkedWorktreeGitGuard } from './linked-worktree-git-guard.js'; + +const execFileAsync = promisify(execFile); + +test('lane Git guard prevents destructive maintenance after global options without breaking ordinary Git', async t => { + if (process.platform === 'win32') return t.skip('the lane Git guard requires POSIX'); + const parent = await mkdtemp(join(tmpdir(), 'linked-worktree-git-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const bin = join(parent, 'bin'); + const repo = join(parent, 'repo'); + await Promise.all([mkdir(bin), mkdir(repo)]); + await writeLinkedWorktreeGitGuard(bin); + const wrapper = join(bin, 'git'); + assert.equal((await stat(wrapper)).mode & 0o222, 0, 'the script is not writable'); + assert.match(await readFile(wrapper, 'utf8'), /^#!\/bin\/bash/); + await execFileAsync('/bin/bash', ['-n', wrapper]); + assert.match(await readFile(wrapper, 'utf8'), /run storage maintenance from the checkout/); + + const run = (args: string[]) => execFileAsync('git', args, { + cwd: repo, + env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? '/usr/bin:/bin'}` }, + }); + await run(['init', '-q', '-b', 'main']); + assert.match((await run(['--version'])).stdout, /git version/); + assert.equal((await run(['-C', repo, '-c', 'core.quotepath=false', 'status', '--short'])).stdout, ''); + for (const option of ['--literal-pathspecs', '--glob-pathspecs', '--noglob-pathspecs', '--icase-pathspecs']) { + assert.equal((await run([option, '-C', repo, 'status', '--short'])).stdout, '', option); + } + await writeFile(join(repo, 'literal[abc].txt'), 'literal\n'); + await run(['--literal-pathspecs', 'add', '--', 'literal[abc].txt']); + assert.match((await run(['status', '--short'])).stdout, /A literal\[abc\]\.txt/); + + // Git has created this object, but no ref points to it yet. Another lane can be + // in exactly this window between writing objects and updating a ref. + await writeFile(join(repo, 'unpublished.txt'), 'from a second lane\n'); + const object = (await run(['hash-object', '-w', 'unpublished.txt'])).stdout.trim(); + const blocked = [ + ['prune', '--expire', 'now'], + ['-C', repo, 'prune', '--expire=now'], + ['-C' + repo, '-c', 'gc.auto=0', '--git-dir=' + join(repo, '.git'), 'prune', '--expire=now'], + ['--no-pager', '-cgc.auto=0', 'gc', '--force'], + ['repack', '-ad'], + ['prune-packed'], + ['maintenance', 'run', '--task=gc'], + ['multi-pack-index', 'expire'], + ['lfs', 'prune'], + ['--literal-pathspecs', 'prune', '--expire', 'now'], + ]; + for (const args of blocked) { + await assert.rejects(run(args), (error: unknown) => { + const result = error as { code?: number; stderr?: string }; + return result.code === 1 && /run storage maintenance from the checkout/.test(result.stderr ?? ''); + }, `must reject git ${args.join(' ')}`); + assert.equal((await run(['cat-file', '-t', object])).stdout.trim(), 'blob'); + } + await assert.rejects(run(['--unrecognized-option', 'prune']), /unsupported global option/); + assert.match((await run(['-C', repo, 'status', '--short'])).stdout, /\?\? unpublished\.txt\n/); + + // The guard is deliberately not a sandbox boundary: resetting PATH or using + // an absolute Git binary can still prune shared objects. + await execFileAsync('git', ['prune', '--expire', 'now'], { cwd: repo }); + await assert.rejects(run(['cat-file', '-e', object])); +}); + +test('lane Git guard rejects aliases from checkout config, -c, include.path and --config-env', async t => { + if (process.platform === 'win32') return t.skip('the lane Git guard requires POSIX'); + const parent = await mkdtemp(join(tmpdir(), 'linked-worktree-alias-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const bin = join(parent, 'bin'); + const repo = join(parent, 'repo'); + const otherRepo = join(parent, 'other'); + await Promise.all([mkdir(bin), mkdir(repo), mkdir(otherRepo)]); + await writeLinkedWorktreeGitGuard(bin); + const run = (args: string[], extraEnv: Record = {}) => execFileAsync('git', args, { + cwd: repo, + env: { ...process.env, ...extraEnv, PATH: `${bin}:${process.env.PATH ?? '/usr/bin:/bin'}` }, + }); + await run(['init', '-q', '-b', 'main']); + await run(['config', 'alias.cleanup', 'prune --expire now']); + await run(['config', 'alias.inspect', 'status --short']); + await run(['config', 'alias.indirect', 'cleanup']); + await run(['-C', otherRepo, 'init', '-q', '-b', 'main']); + await run(['-C', otherRepo, 'config', 'alias.othercleanup', 'prune --expire now']); + await writeFile(join(parent, 'aliases.cfg'), '[alias]\n frominclude = prune --expire now\n'); + await writeFile(join(repo, 'unpublished.txt'), 'a different lane wrote this object\n'); + const object = (await run(['hash-object', '-w', 'unpublished.txt'])).stdout.trim(); + const blocked: Array<[string[], Record?]> = [ + [['cleanup']], + [['-C', repo, 'cleanup']], + [['-C', otherRepo, 'othercleanup']], + [['--git-dir=' + join(otherRepo, '.git'), 'othercleanup']], + [['inspect']], + [['indirect']], + [['-c', 'alias.fromoption=prune --expire now', 'fromoption']], + [['-c', 'alias.fromshell=!git prune --expire now', 'fromshell']], + [['--git-dir=' + join(repo, '.git'), '-c', 'alias.fromgitdir=prune --expire now', 'fromgitdir']], + [['-c', `include.path=${join(parent, 'aliases.cfg')}`, 'frominclude']], + [['--config-env=alias.fromenv=LANE_TEST_GIT_ALIAS', 'fromenv'], { LANE_TEST_GIT_ALIAS: 'prune --expire now' }], + ]; + for (const [args, environment] of blocked) { + await assert.rejects(run(args, environment), (error: unknown) => { + const result = error as { code?: number; stderr?: string }; + return result.code === 1 && /Git aliases are unavailable in linked worktree lanes/.test(result.stderr ?? ''); + }, `must reject Git alias in ${JSON.stringify(args)}`); + assert.equal((await run(['cat-file', '-t', object])).stdout.trim(), 'blob'); + } + assert.match((await run(['status', '--short'])).stdout, /unpublished\.txt/); +}); + +test('a lane rejects a checkout-defined maintenance alias before pruning another lane’s unpublished object', async t => { + if (process.platform === 'win32') return t.skip('the lane Git guard requires POSIX'); + const parent = await mkdtemp(join(tmpdir(), 'linked-worktree-shared-alias-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const checkout = join(parent, 'checkout'); + const bin = join(parent, 'bin'); + await Promise.all([mkdir(checkout), mkdir(bin)]); + await execFileAsync('git', ['init', '-q', '-b', 'main'], { cwd: checkout }); + await execFileAsync('git', [ + '-c', 'user.name=test', '-c', 'user.email=test@example.test', + '-c', 'commit.gpgsign=false', 'commit', '--allow-empty', '-q', '-m', 'seed', + ], { cwd: checkout }); + await mkdir(join(checkout, '.worktrees')); + const lane = join(checkout, '.worktrees', 'task'); + await execFileAsync('git', ['worktree', 'add', '-q', '-b', 'task', lane], { cwd: checkout }); + await execFileAsync('git', ['config', 'alias.cleanup', 'prune --expire now'], { cwd: checkout }); + await writeLinkedWorktreeGitGuard(bin); + await writeFile(join(lane, 'pending.txt'), 'another lane has not updated its ref\n'); + const run = (args: string[]) => execFileAsync('git', args, { + cwd: lane, + env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? '/usr/bin:/bin'}` }, + }); + const object = (await run(['hash-object', '-w', 'pending.txt'])).stdout.trim(); + await assert.rejects(run(['cleanup']), (error: unknown) => { + const result = error as { code?: number; stderr?: string }; + return result.code === 1 && /Git aliases are unavailable in linked worktree lanes/.test(result.stderr ?? ''); + }); + assert.equal((await run(['cat-file', '-t', object])).stdout.trim(), 'blob'); +}); + +test('a lane never lets a misspelled maintenance command autocorrect to prune', async t => { + if (process.platform === 'win32') return t.skip('lane Git guard requires POSIX'); + const parent = await mkdtemp(join(tmpdir(), 'linked-worktree-autocorrect-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const bin = join(parent, 'bin'); + const repo = join(parent, 'repo'); + await Promise.all([mkdir(bin), mkdir(repo)]); + await writeLinkedWorktreeGitGuard(bin); + const run = (args: string[]) => execFileAsync('git', args, { + cwd: repo, + env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? '/usr/bin:/bin'}` }, + }); + await run(['init', '-q', '-b', 'main']); + await run(['config', 'help.autocorrect', 'immediate']); + await writeFile(join(repo, 'unpublished'), 'a sibling has not made this blob reachable'); + const object = (await run(['hash-object', '-w', 'unpublished'])).stdout.trim(); + for (const args of [ + ['prun', '--expire', 'now'], + ['-c', 'help.autocorrect=immediate', 'prun', '--expire=now'], + ]) { + await assert.rejects(run(args), /not a git command|not a Git command|unknown Git command/i); + assert.equal((await run(['cat-file', '-t', object])).stdout.trim(), 'blob'); + } +}); + +test('Git options cannot re-enable automatic maintenance for lane commands', async t => { + if (process.platform === 'win32') return t.skip('lane Git guard requires POSIX'); + const parent = await mkdtemp(join(tmpdir(), 'linked-worktree-maintenance-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const bin = join(parent, 'bin'); + const repo = join(parent, 'repo'); + const remote = join(parent, 'remote.git'); + await Promise.all([mkdir(bin), mkdir(repo)]); + await writeLinkedWorktreeGitGuard(bin); + const run = (args: string[], additionalEnv: NodeJS.ProcessEnv = {}) => execFileAsync('git', args, { + cwd: repo, + env: { ...process.env, ...additionalEnv, PATH: `${bin}:${process.env.PATH ?? '/usr/bin:/bin'}` }, + }); + await run(['init', '-q', '-b', 'main']); + await execFileAsync('git', ['init', '-q', '--bare', remote]); + for (const [key, supplied, enforced] of [ + ['maintenance.auto', 'true', 'false'], + ['gc.auto', '1', '0'], + ['help.autocorrect', 'immediate', '0'], + ]) { + const { stdout } = await run(['-c', `${key}=${supplied}`, 'config', '--get', key]); + assert.equal(stdout.trim(), enforced, `${key} must be enforced after caller-supplied -c`); + } + const { stdout } = await run(['config', '--get', 'maintenance.auto'], { + GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'maintenance.auto', GIT_CONFIG_VALUE_0: 'true', + }); + assert.equal(stdout.trim(), 'false', 'inherited configuration must not re-enable maintenance'); + const configuredGc = await run(['--config-env=gc.auto=LANE_TEST_GC_AUTO', 'config', '--get', 'gc.auto'], { + LANE_TEST_GC_AUTO: '1', + }); + assert.equal(configuredGc.stdout.trim(), '0', 'config-env cannot override the enforced setting'); + await writeFile(join(repo, 'unpublished'), 'not referenced yet'); + const object = (await run(['hash-object', '-w', 'unpublished'])).stdout.trim(); + for (const args of [ + ['fetch', '--auto-maintenance', remote], + ['fetch', '--auto-maintenance=true', remote], + ['-c', 'maintenance.auto=true', 'fetch', '--auto-gc', remote], + ['fetch', '--auto-gc=true', remote], + ['-c', 'gc.auto=1', 'fetch', '--auto-maintenance', remote], + ['pull', '--auto-maintenance', remote], + ]) { + await assert.rejects(run(args), /run storage maintenance from the checkout/); + assert.equal((await run(['cat-file', '-t', object])).stdout.trim(), 'blob'); + } + await run(['-c', 'maintenance.auto=true', '-c', 'gc.auto=1', 'fetch', '--no-auto-maintenance', remote, 'refs/heads/*:refs/remotes/origin/*']); + assert.equal((await run(['cat-file', '-t', object])).stdout.trim(), 'blob'); +}); + +test('for-each-repo cannot dispatch maintenance behind the lane Git guard', async t => { + if (process.platform === 'win32') return t.skip('lane Git guard requires POSIX'); + const parent = await mkdtemp(join(tmpdir(), 'linked-worktree-for-each-repo-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const bin = join(parent, 'bin'); + const repo = join(parent, 'repo'); + await Promise.all([mkdir(bin), mkdir(repo)]); + await writeLinkedWorktreeGitGuard(bin); + const run = (args: string[]) => execFileAsync('git', args, { + cwd: repo, + env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? '/usr/bin:/bin'}` }, + }); + await run(['init', '-q', '-b', 'main']); + await writeFile(join(repo, 'pending'), 'a sibling has not published this object yet\n'); + const object = (await run(['hash-object', '-w', 'pending'])).stdout.trim(); + const dispatcher = ['-c', `maintenance.repo=${repo}`, 'for-each-repo', '--config=maintenance.repo']; + for (const args of [ + [...dispatcher, 'prune', '--expire', 'now'], + [...dispatcher, 'status', '--short'], + ]) { + await assert.rejects(run(args), (error: unknown) => { + const result = error as { code?: number; stderr?: string }; + return result.code === 1 && /run storage maintenance from the checkout/.test(result.stderr ?? ''); + }, 'Git dispatchers cannot be inspected safely inside a lane'); + assert.equal((await run(['cat-file', '-t', object])).stdout.trim(), 'blob'); + } + assert.match((await run(['status', '--short'])).stdout, /pending/); +}); + +test('Git LFS fetch and pull cannot request pruning in a lane', async t => { + if (process.platform === 'win32') return t.skip('lane Git guard requires POSIX'); + const parent = await mkdtemp(join(tmpdir(), 'linked-worktree-lfs-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const bin = join(parent, 'bin'); + const repo = join(parent, 'repo'); + await Promise.all([mkdir(bin), mkdir(repo)]); + await writeLinkedWorktreeGitGuard(bin); + const run = (args: string[]) => execFileAsync('git', args, { + cwd: repo, + env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? '/usr/bin:/bin'}` }, + }); + await run(['init', '-q', '-b', 'main']); + await writeFile(join(repo, 'unpublished'), 'an LFS sibling has no ref yet'); + const object = (await run(['hash-object', '-w', 'unpublished'])).stdout.trim(); + for (const args of [ + ['lfs', 'fetch', '--prune'], + ['lfs', 'fetch', '-p'], + ['lfs', 'fetch', '-rp'], + ['lfs', 'fetch', '--prune', '--recent'], + ['lfs', 'pull', '--prune'], + ['-C', repo, 'lfs', 'fetch', '--prune'], + ]) { + await assert.rejects(run(args), /run storage maintenance from the checkout/); + assert.equal((await run(['cat-file', '-t', object])).stdout.trim(), 'blob'); + } +}); diff --git a/packages/code/src/linked-worktree-git-guard.ts b/packages/code/src/linked-worktree-git-guard.ts new file mode 100644 index 00000000..227bfd3f --- /dev/null +++ b/packages/code/src/linked-worktree-git-guard.ts @@ -0,0 +1,123 @@ +import { constants as fsConstants } from 'node:fs'; +import { access, realpath, stat, writeFile } from 'node:fs/promises'; +import { join, sep } from 'node:path'; + +import { WorkspaceToolError } from './workspace.js'; + +// Match the system-only executable search used for native programmatic tools. +const GIT_BIN_DIRS = [ + '/opt/homebrew/bin', + '/usr/local/bin', + '/usr/bin', + '/bin', + '/home/linuxbrew/.linuxbrew/bin', +]; +const TRUSTED_GIT_DIRS = [ + ...GIT_BIN_DIRS, + '/opt/homebrew/Cellar', + '/usr/local/Cellar', + '/home/linuxbrew/.linuxbrew/Cellar', + '/usr/lib/git-core', +]; + +async function systemGitExecutable(): Promise { + for (const directory of GIT_BIN_DIRS) { + try { + const executable = await realpath(join(directory, 'git')); + if (!TRUSTED_GIT_DIRS.some(path => executable.startsWith(`${path}${sep}`))) continue; + if (!(await stat(executable)).isFile()) continue; + await access(executable, fsConstants.X_OK); + return executable; + } catch { + // A missing system installation is not a reason to run Git from the workspace or PATH. + } + } + throw new WorkspaceToolError('Trusted Git executable is unavailable for linked worktree lanes', 'COMMAND_UNAVAILABLE'); +} + +/** This guards accidental `git` calls, not absolute executable paths or caller shell aliases. */ +export async function writeLinkedWorktreeGitGuard(directory: string): Promise { + try { + await access('/bin/bash', fsConstants.X_OK); + } catch { + throw new WorkspaceToolError('Linked worktree Git guard requires /bin/bash', 'COMMAND_UNAVAILABLE'); + } + const git = await systemGitExecutable(); + const quotedGit = `'${git.replaceAll("'", "'\\''")}'`; + const script = [ + // Bash arrays preserve global option arguments when checking the same + // effective Git configuration without eval or interpreting alias contents. + '#!/bin/bash', + 'check() {', + ' local -a globals=() forwarded=()', + ' local aliasStatus=0', + ' while [ "$#" -gt 0 ]; do', + ' case "$1" in', + ' -C|-c|--git-dir|--work-tree|--namespace|--config-env)', + ' if [ "$#" -lt 2 ]; then echo "git: missing global option argument" >&2; return 2; fi', + ' globals+=("$1" "$2"); forwarded+=("$1" "$2"); shift 2 ;;', + ' -C?*|-c?*|--git-dir=*|--work-tree=*|--namespace=*|--config-env=*)', + ' globals+=("$1"); forwarded+=("$1"); shift ;;', + ' --bare)', + ' globals+=("$1"); forwarded+=("$1"); shift ;;', + ' -p|-P|--paginate|--no-pager|--no-replace-objects|--no-optional-locks|--exec-path=*)', + ' forwarded+=("$1"); shift ;;', + ' --literal-pathspecs|--glob-pathspecs|--noglob-pathspecs|--icase-pathspecs)', + ' forwarded+=("$1"); shift ;;', + ' --) forwarded+=("$1"); shift; break ;;', + ' -v|--version|-h|--help|--exec-path|--html-path|--man-path|--info-path)', + ' return 0 ;;', + ' -*) echo "git: unsupported global option in linked worktree lane" >&2; return 2 ;;', + ' *) break ;;', + ' esac', + ' done', + ' case "${1:-}" in', + ' prune|gc|repack|prune-packed|maintenance|multi-pack-index|for-each-repo)', + ' echo "git: run storage maintenance from the checkout, not a linked worktree lane" >&2', + ' return 1 ;;', + ' fetch|pull)', + ' for option in "${@:2}"; do', + ' case "$option" in', + ' --auto-maintenance|--auto-maintenance=*|--auto-gc|--auto-gc=*)', + ' echo "git: run storage maintenance from the checkout, not a linked worktree lane" >&2', + ' return 1 ;;', + ' esac', + ' done ;;', + ' lfs)', + ' if [ "${2:-}" = prune ]; then', + ' echo "git: run storage maintenance from the checkout, not a linked worktree lane" >&2', + ' return 1', + ' fi', + ' if [ "${2:-}" = fetch ] || [ "${2:-}" = pull ]; then', + ' for option in "${@:3}"; do', + ' case "$option" in', + ' --prune|--prune=*|-p|-p?*|-[!-]*p*)', + ' echo "git: run storage maintenance from the checkout, not a linked worktree lane" >&2', + ' return 1 ;;', + ' esac', + ' done', + ' fi ;;', + ' esac', + ' if [ "$#" -eq 0 ]; then return 0; fi', + // Git itself loads local config, -C, -c, includes and --config-env, so an + // alias from any of those sources is rejected even when it hides a prune. + // Reject all aliases, including harmless ones: Git shell aliases and nested + // aliases cannot be proven safe by inspecting the first token. + ` ${quotedGit} "\${globals[@]}" config --get "alias.\${1}" >/dev/null 2>&1 || aliasStatus=$?`, + ' case "$aliasStatus" in', + ' 0) echo "git: Git aliases are unavailable in linked worktree lanes; run the underlying command or use the checkout" >&2; return 1 ;;', + ' 1) ;;', + ' *) echo "git: cannot verify Git aliases in linked worktree lane" >&2; return 2 ;;', + ' esac', + // Command-line -c values override both Git config files and GIT_CONFIG_COUNT. + // Add these after every caller global option: fetch can start maintenance + // internally without invoking the PATH guard a second time. + ` exec ${quotedGit} "\${forwarded[@]}" -c maintenance.auto=false -c gc.auto=0 -c help.autocorrect=0 "$@"`, + '}', + 'check "$@" || exit "$?"', + `exec ${quotedGit} "$@"`, + '', + ].join('\n'); + await writeFile(join(directory, 'git'), script, { flag: 'wx', mode: 0o500 }); + await access(join(directory, 'git'), fsConstants.X_OK); +} diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index 7ecb06f3..b6fb25bc 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -1629,6 +1629,11 @@ test('a linked worktree lane may write only shared Git storage and its own metad assert.deepEqual(config.filesystem.allowWrite.slice(0, 4), [lane, ...writableGitPaths]); assert.ok(!config.filesystem.allowWrite.includes(commonGitDir)); assert.ok(config.filesystem.allowRead?.includes(commonGitDir)); + const gitGuard = config.filesystem.allowRead?.find(path => path.includes('librechat-code-git-')); + assert.ok(gitGuard, 'lane Git guard must be readable'); + assert.ok(!config.filesystem.allowWrite.includes(gitGuard)); + assert.ok(config.filesystem.denyWrite.includes(gitGuard)); + assert.equal((await stat(join(gitGuard, 'git'))).mode & 0o222, 0); const probed = fakeManager(); const prober = new NativeSrtWorkspaceCommandSandbox({ @@ -1642,6 +1647,10 @@ test('a linked worktree lane may write only shared Git storage and its own metad await prober.executeProgrammatic(request, dataDirectory, undefined, { probe: true }); assert.ok(probed.customConfigSeenDuringWrap?.filesystem?.allowRead?.includes(commonGitDir)); assert.ok(!probed.customConfigSeenDuringWrap?.filesystem?.allowWrite?.includes(commonGitDir)); + const probeGuard = probed.config?.filesystem.allowRead?.find(path => path.includes('librechat-code-git-')); + assert.ok(probeGuard); + assert.ok(probed.customConfigSeenDuringWrap?.filesystem?.allowRead?.includes(probeGuard)); + assert.ok(probed.customConfigSeenDuringWrap?.filesystem?.denyWrite?.includes(probeGuard)); await assert.rejects( prepare([commonGitDir]), @@ -1659,3 +1668,75 @@ test('a linked worktree lane may write only shared Git storage and its own metad error instanceof WorkspaceToolError && error.code === 'REGISTRATION_INVALID', ); }); + +test('lane commands put the read-only Git guard ahead of the ordinary PATH', async t => { + if (process.platform === 'win32') return t.skip('lane Git guard requires POSIX'); + const checkoutRoot = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-guard-'))); + t.after(() => rm(checkoutRoot, { recursive: true, force: true })); + const commonGitDir = join(checkoutRoot, '.git'); + const lane = join(checkoutRoot, '.worktrees', 'task-a'); + const writableGitPaths = [join(commonGitDir, 'objects'), join(commonGitDir, 'worktrees', 'task-a')]; + await Promise.all([lane, ...writableGitPaths].map(path => mkdir(path, { recursive: true }))); + const fake = fakeManager(); + let commandPath: string | undefined; + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: lane, + linkedWorktree: { checkoutRoot, commonGitDir, writableGitPaths }, + environment: { PATH: '/usr/bin:/bin' }, + manager: fake.manager, + spawnCommand(command, args, options) { + commandPath = options.env?.PATH; + return spawn(command, args, options); + }, + }); + t.after(() => sandbox.close()); + const result = await sandbox.execute({ ...request, command: 'git --version' }); + assert.equal(result.exitCode, 0); + assert.match(result.stdout, /git version/); + const guard = fake.config?.filesystem.allowRead?.find(path => path.includes('librechat-code-git-')); + assert.ok(guard); + assert.ok(commandPath?.startsWith(`${guard}:`)); + await sandbox.close(); + await assert.rejects(stat(guard), { code: 'ENOENT' }); +}); + +test('linked worktree Git guard is removed when sandbox initialization fails', async t => { + if (process.platform === 'win32') return t.skip('lane Git guard requires POSIX'); + const checkoutRoot = await mkdtemp(join(tmpdir(), 'librechat-code-failed-guard-')); + t.after(() => rm(checkoutRoot, { recursive: true, force: true })); + const commonGitDir = join(checkoutRoot, '.git'); + const lane = join(checkoutRoot, '.worktrees', 'task-a'); + const objects = join(commonGitDir, 'objects'); + await Promise.all([lane, objects].map(path => mkdir(path, { recursive: true }))); + const fake = fakeManager({ initializeError: new Error('init failed') }); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: lane, + linkedWorktree: { checkoutRoot, commonGitDir, writableGitPaths: [objects] }, + manager: fake.manager, + }); + await assert.rejects(sandbox.prepare(), /init failed/); + const guard = fake.config?.filesystem.allowRead?.find(path => path.includes('librechat-code-git-')); + assert.ok(guard); + await assert.rejects(stat(guard), { code: 'ENOENT' }); + await sandbox.close(); +}); + +test('linked worktree Git guard refuses Windows rather than admitting an unguarded lane', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-win-guard-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + linkedWorktree: { + checkoutRoot: root, + commonGitDir: join(root, '.git'), + writableGitPaths: [join(root, '.git', 'objects')], + }, + platform: 'win32', + manager: fakeManager().manager, + }); + await assert.rejects(sandbox.prepare(), (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'COMMAND_UNAVAILABLE' && + /POSIX host/.test(error.message), + ); + await sandbox.close(); +}); diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 6ac51acc..e635f8e7 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -32,6 +32,7 @@ import { } from './private-storage.js'; import { WorkspaceToolError } from './workspace.js'; import { restoreScratchTraversal } from './native-scratch.js'; +import { writeLinkedWorktreeGitGuard } from './linked-worktree-git-guard.js'; import type { ChildProcessWithoutNullStreams, @@ -323,6 +324,8 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox } private scratchDirectory?: string; private scratchHandle?: FileHandle; + /** A private sibling of scratch, readable but never writable by lane commands. */ + private gitGuardDirectory?: string; private execution?: Promise; private closing?: Promise; private resetFailed = false; @@ -361,6 +364,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox await this.manager.reset().catch(() => { this.resetFailed = true; }); + await this.removeLinkedWorktreeGitGuard().catch(() => undefined); await this.removeScratchDirectory().catch(() => undefined); if (!this.resetFailed) managerOwners.delete(this.manager); this.initialized = undefined; @@ -448,6 +452,9 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox } } const lane = this.options.linkedWorktree; + if (lane && this.platform === 'win32') { + throw new WorkspaceToolError('Linked worktree Git guard requires a POSIX host', 'COMMAND_UNAVAILABLE'); + } const commonGitDir = lane ? await canonicalPath(lane.commonGitDir) : undefined; const checkoutRoot = lane ? await canonicalPath(lane.checkoutRoot) : undefined; const writableGitPaths = lane @@ -485,6 +492,12 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox 'REGISTRATION_INVALID', ); } + if (lane && !canonicalScratchDirectory) { + throw new WorkspaceToolError('Linked worktree Git guard requires private scratch storage', 'COMMAND_UNAVAILABLE'); + } + const gitGuardDirectory = lane + ? await this.createLinkedWorktreeGitGuard(canonicalScratchDirectory!) + : undefined; const commandPolicy = normalizeNativeSrtCommandPolicy( this.options.commandPolicy, ); @@ -510,6 +523,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox allowRead: [ root, ...laneGitPaths, + ...(gitGuardDirectory ? [gitGuardDirectory] : []), ...(canonicalScratchDirectory ? [canonicalScratchDirectory] : []), @@ -524,6 +538,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox denyWrite: [ ...protectedPaths, ...deniedInheritedWritablePaths, + ...(gitGuardDirectory ? [gitGuardDirectory] : []), ], allowGitConfig: false, }, @@ -593,6 +608,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox this.denyWritePaths = [ ...protectedPaths, ...deniedInheritedWritablePaths, + ...(gitGuardDirectory ? [gitGuardDirectory] : []), ]; } @@ -817,6 +833,9 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ...(this.canonicalCommonGitDir ? [this.canonicalCommonGitDir] : []), + ...(this.gitGuardDirectory + ? [this.gitGuardDirectory] + : []), ], allowWrite: [ ...(canonicalWorkspaceRoot != null @@ -1067,6 +1086,9 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ? 'NUL' : '/dev/null', GIT_CONFIG_NOSYSTEM: '1', + ...(this.gitGuardDirectory ? { + PATH: `${this.gitGuardDirectory}:${wrapped.env.PATH || this.environment.PATH || '/usr/bin:/bin'}`, + } : {}), }, detached: this.platform !== 'win32', shell: false, @@ -1297,6 +1319,24 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); } + private async createLinkedWorktreeGitGuard(scratchDirectory: string): Promise { + if (this.gitGuardDirectory) { + throw new Error('Linked worktree Git guard cleanup is still pending'); + } + // Unlike scratch, this sibling directory is not in filesystem.allowWrite. + const directory = await mkdtemp(join(dirname(scratchDirectory), 'librechat-code-git-')); + this.gitGuardDirectory = directory; + await assertPrivateStorageAncestors(directory); + await writeLinkedWorktreeGitGuard(directory); + return directory; + } + + private async removeLinkedWorktreeGitGuard(): Promise { + if (!this.gitGuardDirectory) return; + await rm(this.gitGuardDirectory, { recursive: true, force: true }); + this.gitGuardDirectory = undefined; + } + private async removeScratchDirectory(): Promise { const scratchDirectory = this.scratchDirectory; if (!scratchDirectory) return; @@ -1340,6 +1380,10 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox } this.initialized = undefined; this.canonicalRoot = undefined; - await this.removeScratchDirectory(); + try { + await this.removeLinkedWorktreeGitGuard(); + } finally { + await this.removeScratchDirectory(); + } } } From 3189cfd8ee7f81cf90671f9b91b1d52ae9d3693d Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 29 Sep 2026 15:47:33 -0400 Subject: [PATCH 16/48] =?UTF-8?q?=F0=9F=94=93=20fix:=20Let=20Linked-Worktr?= =?UTF-8?q?ee=20Lanes=20Commit=20When=20the=20Checkout=20Lives=20in=20the?= =?UTF-8?q?=20Worker=20Home=20(#274)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: Grant lane Git reads entry by entry so write binds survive a read-denied home A read grant on the whole common Git directory masked the write binds beneath it whenever the checkout lived inside the worker home, which SRT re-binds under a tmpfs (writes first, then reads). Lanes could not create their own index.lock or ref locks. Grant each entry off the writable paths instead, and add a real-SRT regression test with a Linux CI job. * fix: Re-bind lane Git writes after the common read bind instead of granting entries Per-entry read grants left the common Git directory as a writable tmpfs placeholder inside the worker home (pack-refs could strand refs there) and pinned replaced files to stale inodes. Grant the whole directory read-only again and, on Linux, list each existing writable Git directory as a deeper read-deny so SRT re-binds its write mount after the ancestor read bind. --- .github/workflows/ci.yml | 24 ++++ .../code/src/linked-worktrees-live.test.ts | 108 ++++++++++++++++++ packages/code/src/native-sandbox.test.ts | 13 ++- packages/code/src/native-sandbox.ts | 20 ++++ 4 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 packages/code/src/linked-worktrees-live.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4076d8f6..f7393284 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -235,6 +235,30 @@ jobs: LIBRECHAT_CODE_LIVE_SRT_TESTS: '1' run: node --test dist/environment-live.test.js + linux-native-sandbox-tests: + name: Linux Native Sandbox Tests + runs-on: ubuntu-24.04 + defaults: + run: + working-directory: packages/code + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + with: + node-version: 24.16.0 + - name: Install bubblewrap + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq bubblewrap socat ripgrep + # Ubuntu 24.04 blocks unprivileged user namespaces through AppArmor; bwrap needs them. + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + - run: npm ci + - run: npm run build + - name: Linked worktree lane containment + env: + LIBRECHAT_CODE_LIVE_SRT_TESTS: '1' + run: node --test dist/linked-worktrees-live.test.js + lambda-microvm-provisioning: name: Lambda MicroVM Provisioning runs-on: ubuntu-latest diff --git a/packages/code/src/linked-worktrees-live.test.ts b/packages/code/src/linked-worktrees-live.test.ts new file mode 100644 index 00000000..c9052a35 --- /dev/null +++ b/packages/code/src/linked-worktrees-live.test.ts @@ -0,0 +1,108 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { mkdir, mkdtemp, readFile, realpath, rm, stat, writeFile } from 'node:fs/promises'; +import { homedir, tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import test from 'node:test'; + +import { verifyLinkedWorktree } from './linked-worktrees.js'; +import { NativeSrtWorkspaceCommandSandbox } from './native-sandbox.js'; +import { resolveNativeSrtCommandPolicy } from './native-policy.js'; + +const execFileAsync = promisify(execFile); +const IDENTITY = ['-c', 'user.name=lane', '-c', 'user.email=lane@example.com', '-c', 'commit.gpgsign=false']; + +async function git(cwd: string, ...args: string[]): Promise { + return (await execFileAsync('git', [...IDENTITY, ...args], { cwd })).stdout.trim(); +} + +async function snapshot(paths: string[]): Promise> { + const entries = await Promise.all( + paths.map(async (path) => [path, await readFile(path, 'utf8').catch(() => null)] as const), + ); + return Object.fromEntries(entries); +} + +/** + * The worker's home is read-denied, so a checkout beneath it exercises the + * sandbox's tmpfs re-binding; one beneath the system temporary directory does not. + */ +for (const [location, parent] of [ + ['beneath the worker home', homedir()], + ['outside the worker home', tmpdir()], +] as const) { + test(`real SRT lets a linked worktree lane commit while checkout and sibling Git metadata stay intact (${location})`, { + skip: process.env.LIBRECHAT_CODE_LIVE_SRT_TESTS !== '1', + timeout: 60_000, + }, async (t) => { + const base = await realpath(await mkdtemp(join(parent, 'lane-live-'))); + t.after(() => rm(base, { recursive: true, force: true })); + const root = join(base, 'repo'); + await mkdir(root); + await git(root, 'init', '-q', '-b', 'main'); + await writeFile(join(root, 'tracked.txt'), 'checkout\n'); + await git(root, 'add', 'tracked.txt'); + await git(root, 'commit', '-qm', 'init'); + await mkdir(join(root, '.worktrees')); + await git(root, 'worktree', 'add', '-q', '-b', 'task-a', '.worktrees/task-a'); + await git(root, 'worktree', 'add', '-q', '-b', 'task-b', '.worktrees/task-b'); + + const commonGitDir = join(root, '.git'); + const protectedFiles = [ + join(commonGitDir, 'HEAD'), + join(commonGitDir, 'index'), + join(commonGitDir, 'config'), + join(commonGitDir, 'MERGE_HEAD'), + join(commonGitDir, 'packed-refs'), + join(commonGitDir, 'hooks', 'pre-commit'), + join(commonGitDir, 'worktrees', 'task-b', 'HEAD'), + join(root, 'tracked.txt'), + ]; + const before = await snapshot(protectedFiles); + const lane = await verifyLinkedWorktree(root, 'task-a'); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: lane.root, + workspaceIdentity: lane.identity, + linkedWorktree: { + checkoutRoot: lane.checkoutRoot, + commonGitDir: lane.commonGitDir, + writableGitPaths: lane.writableGitPaths, + }, + commandPolicy: resolveNativeSrtCommandPolicy('trusted-vm'), + environment: { PATH: process.env.PATH, LANG: 'C.UTF-8' }, + }); + t.after(() => sandbox.close()); + const run = (command: string) => + sandbox.execute({ + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'lane', + command, + timeoutMs: 30_000, + maxOutputBytes: 16_384, + }); + const gitInLane = `git ${IDENTITY.join(' ')}`; + + const committed = await run( + `printf lane > lane.txt && ${gitInLane} add lane.txt && ${gitInLane} commit -qm lane && ${gitInLane} branch lane-extra`, + ); + assert.equal(committed.exitCode, 0, committed.stderr); + + for (const path of protectedFiles) { + await run(`printf tampered > '${path}'`); + } + // Packing must not move refs into storage that vanishes with the sandbox. + await run(`${gitInLane} pack-refs --all`); + await sandbox.close(); + + for (const branch of ['main', 'task-a', 'task-b', 'lane-extra']) { + assert.ok(await git(root, 'rev-parse', '--verify', '-q', `refs/heads/${branch}`), branch); + } + + assert.equal(await git(root, 'log', '-1', '--format=%s', 'task-a'), 'lane'); + assert.deepEqual(await snapshot(protectedFiles), before); + assert.equal(await git(root, 'status', '--porcelain', '--untracked-files=no'), ''); + await assert.rejects(stat(join(commonGitDir, 'MERGE_HEAD')), { code: 'ENOENT' }); + }); +} diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index b6fb25bc..5b79aa1a 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -1612,13 +1612,14 @@ test('a linked worktree lane may write only shared Git storage and its own metad await Promise.all( [lane, ...writableGitPaths].map(path => mkdir(path, { recursive: true })), ); - const prepare = async (paths: string[]) => { + const prepare = async (paths: string[], platform: NodeJS.Platform = 'linux') => { const fake = fakeManager(); const sandbox = new NativeSrtWorkspaceCommandSandbox({ workspaceRoot: lane, linkedWorktree: { checkoutRoot, commonGitDir, writableGitPaths: paths }, environment: { PATH: '/usr/bin' }, manager: fake.manager, + platform, }); t.after(() => sandbox.close()); await sandbox.prepare(); @@ -1629,6 +1630,16 @@ test('a linked worktree lane may write only shared Git storage and its own metad assert.deepEqual(config.filesystem.allowWrite.slice(0, 4), [lane, ...writableGitPaths]); assert.ok(!config.filesystem.allowWrite.includes(commonGitDir)); assert.ok(config.filesystem.allowRead?.includes(commonGitDir)); + // Deeper read denies make SRT re-bind each writable Git directory after the + // read-only bind of the common directory (Linux tmpfs re-binding order). + for (const path of writableGitPaths) { + assert.ok(config.filesystem.denyRead.includes(path), path); + } + assert.ok(!config.filesystem.denyRead.includes(join(commonGitDir, 'lfs'))); + const darwin = await prepare(writableGitPaths, 'darwin'); + for (const path of writableGitPaths) { + assert.ok(!darwin.filesystem.denyRead.includes(path), path); + } const gitGuard = config.filesystem.allowRead?.find(path => path.includes('librechat-code-git-')); assert.ok(gitGuard, 'lane Git guard must be readable'); assert.ok(!config.filesystem.allowWrite.includes(gitGuard)); diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index e635f8e7..49fe6ecf 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -481,6 +481,25 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); } const laneGitPaths = commonGitDir ? [commonGitDir] : []; + /** + * On Linux, SRT hides a read-denied directory (such as the worker home) under a + * tmpfs and then re-binds writes before reads, so the read-only bind of the whole + * common Git directory would mask its writable descendants. SRT processes read + * denies shallow-first, re-binding each one's writes on top, so listing every + * existing writable Git directory as a deeper deny restores its write bind after + * the ancestor read bind. The common directory stays a live, read-only host + * directory: nothing can be created at its top level. + */ + const laneWriteRebinds = + this.platform === 'linux' + ? ( + await Promise.all( + writableGitPaths.map(async path => + (await stat(path).catch(() => undefined))?.isDirectory() ? path : undefined, + ), + ) + ).filter((path): path is string => path != null) + : []; const canonicalScratchDirectory = await this.createScratchDirectory(sharedScratchPaths); if ( @@ -519,6 +538,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ...sharedScratchPaths.filter(path => deniedInheritedWritablePaths.includes(path), ), + ...laneWriteRebinds, ], allowRead: [ root, From 1e683ce749278788288a541364688077ebc595de Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 30 Sep 2026 06:30:16 -0400 Subject: [PATCH 17/48] =?UTF-8?q?=F0=9F=8E=AF=20feat:=20Diagnose=20Every?= =?UTF-8?q?=20Failing=20Workspace=20Edit=20and=20Negotiate=20Tolerant=20Ma?= =?UTF-8?q?tching=20(#271)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🎯 feat: Diagnose Every Failing Workspace Edit and Negotiate Tolerant Matching A rejected edit batch now reports every failing edit by position: missing edits name the nearest candidate line and flag elided, line-numbered, whitespace-only or CRLF mismatches, and ambiguous edits give their match count and line numbers. Two negotiated edit features add tolerant matching (line-trimmed, indentation-flexible, whitespace-normalized) and replaceAll, with per-edit match reporting only for requests that opt in. * fix: Honor Edit Boundaries and Negotiated Capabilities * fix: Preserve Matched Line Endings in Tolerant Edits * fix: Reject Unrepresentable Tolerant Edit Boundaries * fix: Require Tolerant Edit Boundary Line Breaks * fix: Bound Edit Matching and Redact Edit-Only Diagnostics * fix: Stream Workspace Replace-All Results Within Memory Bounds * fix: Preserve Extra Blank Lines in Tolerant Edits * fix: Honor Full Edit Wrappers and Linear Candidate Budgets * fix: Retain Every Failed Edit Position in Bounded Diagnostics --------- Co-authored-by: Lia --- docs/remote-bridge/README.md | 5 +- packages/code/README.md | 48 +- packages/code/src/edits.test.ts | 721 +++++++++++++++++++++ packages/code/src/edits.ts | 712 ++++++++++++++++++++ packages/code/src/protocol.test.ts | 158 +++++ packages/code/src/protocol.ts | 119 +++- packages/code/src/worker.ts | 39 +- packages/code/src/workspace-worker.test.ts | 136 ++++ packages/code/src/workspace.test.ts | 390 ++++++++++- packages/code/src/workspace.ts | 56 +- service/src/bridge/router.test.ts | 6 +- service/src/bridge/router.ts | 3 +- service/src/bridge/store.ts | 15 +- service/src/bridge/workspace-store.test.ts | 128 +++- 14 files changed, 2493 insertions(+), 43 deletions(-) create mode 100644 packages/code/src/edits.test.ts create mode 100644 packages/code/src/edits.ts diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index a37745a2..b2d33db7 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -165,7 +165,10 @@ backslashes, symlink escapes, unexpected fields, and host roots are rejected. Workspace mutation remains disabled unless the operator starts the worker with `--allow-workspace-writes` (or `LIBRECHAT_CODE_ALLOW_WORKSPACE_WRITES=true`). That adds bounded `write_file` -and exact-match `edit_file` operations. Writes are limited to 1 MiB of UTF-8 +and `edit_file` operations. `edit_file` matches exactly by default; the +negotiated `tolerant_match` and `replace_all` edit features add +whitespace-tolerant matching and multi-location replacement (see +`packages/code/README.md`). Writes are limited to 1 MiB of UTF-8 text, require an existing in-workspace parent directory, reject symlinks, and commit atomically. The worker capability is an enforcement boundary; LibreChat should still route every mutation through its configurable tool-approval hooks. diff --git a/packages/code/README.md b/packages/code/README.md index a38523a3..ba7ec4d7 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -590,6 +590,48 @@ installed as one atomic mutation. Code API dispatches the batch form only after the worker and server negotiate `batch` in `editFileModes`. Revision-fenced edits likewise require the negotiated `expected_base_sha256` entry in `editFileFeatures`. + +Edits apply in order, each to the text the earlier ones produced. When any edit +fails, the worker still checks the rest and rejects the whole batch with one +`EDIT_CONFLICT` whose message lists every failing edit by position. Large +batch messages shorten reasons and source excerpts to stay within the bound, +but never omit failing edit positions. A missing edit names the nearest +candidate line and flags elided (`...`) or line-numbered `oldText`, a +whitespace-only difference, or CRLF line endings. +An ambiguous edit gives its match count and line numbers. Overlapping +occurrences count as separate locations. Detailed source-line excerpts require +`read_file` or `preview_edit` on the same workspace; edit-only workers return a +generic failure and its error code without revealing file contents. A preview +itself exposes the resulting file text, so it is read-capable. + +Two optional features change matching, each negotiated in `editFileFeatures` +before Code API dispatches it: + +- `tolerant_match`: a request-level `matching: 'tolerant'` falls back from an + exact match to, in order, `line-trimmed` (ignores trailing whitespace and + CRLF), `indentation-flexible` (a uniformly shifted block, with `newText` + moved to the file's indentation) and `whitespace-normalized` (any whitespace + run between complete whitespace-delimited tokens, never a prefix or suffix + of another token). Without `replaceAll`, a match must still be unique; + replacements use the matched line's ending even in mixed-ending files. The + whitespace-normalized tier peels the complete shared newline-and-indentation + wrapper from `newText` even when CRLF/LF or nearby spaces differ, without + removing intentional extra line breaks or duplicating the source line ending. + Boundary whitespace claimed by `oldText`, including the number of line breaks, + must exist beside the matched tokens in the source; an attempt to remove it + with a token-only fallback fails rather than silently preserving it. Exact and + line-window matches can still replace terminators. + Excessively repetitive indentation candidates fail closed with a request + for more context rather than scanning every long window. Dense files reuse a + compact newline index for matching and diagnostics; overlapping exact matches + are counted without restarting a scan at each offset. +- `replace_all`: a batch edit's `replaceAll: true` replaces every + non-overlapping match instead of requiring exactly one, and still fails when + nothing matches. + +A request that sets `matching` or any `replaceAll` receives `matches`, one +`{ strategy, occurrences }` entry per edit. Requests that set neither receive +exactly the legacy result. Only IDs, names, protocol version, supported operations, and negotiated write modes appear in worker capabilities; absolute host paths remain local to the worker process. @@ -655,8 +697,10 @@ non-regular files, and commit through an owner-only temporary file followed by an atomic rename. The worker syncs the containing directory and verifies that the installed inode still contains the requested bytes before reporting success. Edits replace text only when the requested old text occurs exactly -once and reject if the file changes before commit. These operations do not -create directories or execute commands. +once (unless negotiated `replaceAll` selects every non-overlapping match) and +reject if the file changes before commit. Intermediate replacements are bounded +before construction, including `replaceAll`; these operations do not create +directories or execute commands. Register one directory already present on the worker machine with the worker-directory option: diff --git a/packages/code/src/edits.test.ts b/packages/code/src/edits.test.ts new file mode 100644 index 00000000..3ef9428d --- /dev/null +++ b/packages/code/src/edits.test.ts @@ -0,0 +1,721 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { + applyTextEdits, + EDIT_DIAGNOSTIC_MAX_CHARS, + WorkspaceEditMatchError, + WorkspaceEditOutputLimitError, +} from './edits.js'; + +function rejection(run: () => unknown): WorkspaceEditMatchError { + try { + run(); + } catch (error) { + assert.ok(error instanceof WorkspaceEditMatchError); + return error; + } + assert.fail('expected the edits to be rejected'); +} + +test('exact matching keeps its single-location contract', () => { + const applied = applyTextEdits('alpha\nbeta\n', [{ oldText: 'beta', newText: 'gamma' }]); + assert.deepEqual(applied, { + text: 'alpha\ngamma\n', + matches: [{ strategy: 'exact', occurrences: 1 }], + }); +}); + +test('overlapping occurrences make an exact edit ambiguous', () => { + const error = rejection(() => applyTextEdits('aaa', [{ oldText: 'aa', newText: 'b' }])); + assert.match(error.message, /matched 2 locations/); + const applied = applyTextEdits('aaaa', [{ oldText: 'aa', newText: 'b', replaceAll: true }]); + assert.deepEqual(applied, { text: 'bb', matches: [{ strategy: 'exact', occurrences: 2 }] }); +}); + +test('later edits see the result of earlier ones', () => { + const applied = applyTextEdits('one', [ + { oldText: 'one', newText: 'two' }, + { oldText: 'two', newText: 'three' }, + ]); + assert.equal(applied.text, 'three'); +}); + +test('an ambiguous edit names every location instead of a bare conflict', () => { + const text = 'const a = 1;\nreturn a;\nconst b = 2;\nreturn a;\n'; + const error = rejection(() => + applyTextEdits(text, [{ oldText: 'return a;', newText: 'return b;' }]), + ); + assert.match(error.message, /matched 2 locations at lines 2, 4/); + assert.match(error.message, /include more surrounding lines/); + assert.match(error.message, /nothing was written/); +}); + +test('a batch reports every failing edit with its position in one rejection', () => { + const text = 'a\nb\nc\nb\n'; + const error = rejection(() => + applyTextEdits(text, [ + { oldText: 'a', newText: 'A' }, + { oldText: 'missing', newText: 'x' }, + { oldText: 'c', newText: 'C' }, + { oldText: 'b', newText: 'B' }, + ]), + ); + assert.deepEqual( + error.failures.map((failure) => failure.index), + [1, 3], + ); + assert.match(error.message, /^2 of 4 workspace edits did not apply/); + assert.match(error.message, /\nEdit 2: old_text was not found/); + assert.match(error.message, /\nEdit 4: old_text matched 2 locations at lines 2, 4/); +}); + +test('a missing edit points at the line it most likely meant', () => { + const text = 'function load(user) {\n return fetchUser(user.id);\n}\n'; + const error = rejection(() => + applyTextEdits(text, [ + { oldText: 'function load(user) {\n return fetchUser(user);\n}', newText: 'x' }, + ]), + ); + assert.match(error.message, /its first line appears at line 1, but the lines after it differ/); +}); + +test('a missing edit explains elided and line-numbered old_text', () => { + const text = 'start\nmiddle\nend\n'; + const elided = rejection(() => + applyTextEdits(text, [{ oldText: 'start\n// ...\nend', newText: 'x' }]), + ); + assert.match(elided.message, /elision placeholder/); + const numbered = rejection(() => + applyTextEdits(text, [{ oldText: '1 | start\n2 | middle', newText: 'x' }]), + ); + assert.match(numbered.message, /line-number prefixes/); +}); + +test('exact mode says when only the whitespace differs', () => { + const shifted = rejection(() => + applyTextEdits('class A {\n a();\n b();\n}\n', [{ oldText: 'a();\nb();', newText: 'x' }]), + ); + assert.match(shifted.message, /exists at line 2 with different whitespace \(indentation-flexible\)/); + const reflowed = rejection(() => + applyTextEdits('if (ready) {\n run();\n}\n', [ + { oldText: 'if (ready) {\n run();\n}', newText: 'x' }, + ]), + ); + assert.match(reflowed.message, /exists at line 1 with different whitespace \(whitespace-normalized\)/); +}); + +test('exact-mode diagnostics detect normalized whitespace with both boundary wrappers intact', () => { + for (const [source, oldText] of [ + ['foo bar\n', 'foo bar\n'], + ['header\n foo bar\n', '\nfoo bar'], + ['header\r\n foo bar\r\nnext', '\r\nfoo bar\r\n'], + [' foo bar \n', ' foo bar '], + ]) { + const error = rejection(() => applyTextEdits(source, [{ oldText, newText: '' }])); + assert.match(error.message, /exists at line \d+ with different whitespace \(whitespace-normalized\)/, + JSON.stringify({ source, oldText })); + } + const missingBoundary = rejection(() => applyTextEdits('foo bar', [ + { oldText: 'foo bar\n', newText: 'unchanged' }, + ])); + assert.doesNotMatch(missingBoundary.message, /exists at.*different whitespace/); +}); + +test('tolerant matching ignores trailing whitespace and keeps CRLF line endings', () => { + const text = 'first \r\nsecond\t\r\nthird\r\n'; + const applied = applyTextEdits( + text, + [{ oldText: 'first\nsecond', newText: 'one\ntwo' }], + 'tolerant', + ); + assert.equal(applied.text, 'one\r\ntwo\r\nthird\r\n'); + assert.deepEqual(applied.matches, [{ strategy: 'line-trimmed', occurrences: 1 }]); +}); + +test('a trailing newline in old_text matches through the line terminator', () => { + const text = 'keep\ndrop\nkeep too\n'; + const applied = applyTextEdits(text, [{ oldText: 'drop \n', newText: '' }], 'tolerant'); + assert.equal(applied.text, 'keep\nkeep too\n'); +}); + +test('indentation-flexible matches move new_text to the file indentation', () => { + const text = 'class A {\n method() {\n return 1;\n }\n}\n'; + const applied = applyTextEdits( + text, + [ + { + oldText: 'method() {\n return 1;\n}', + newText: 'method() {\n const value = 2;\n return value;\n}', + }, + ], + 'tolerant', + ); + assert.equal( + applied.text, + 'class A {\n method() {\n const value = 2;\n return value;\n }\n}\n', + ); + assert.deepEqual(applied.matches, [{ strategy: 'indentation-flexible', occurrences: 1 }]); +}); + +test('whitespace-normalized matches do not splice prefixes or suffixes of tokens', () => { + for (const [source, oldText] of [ + ['prereturn value;\n', 'return value;'], + ['return valueSuffix\n', 'return value'], + ['return value;postfix\n', 'return value;'], + ]) { + const error = rejection(() => applyTextEdits(source, [{ oldText, newText: 'changed' }], 'tolerant')); + assert.match(error.message, /old_text was not found/); + } + const valid = applyTextEdits('return value;\n', [ + { oldText: 'return value;', newText: 'return changed;' }, + ], 'tolerant'); + assert.equal(valid.text, 'return changed;\n'); + assert.deepEqual(valid.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +test('large whitespace-normalized edits match without compiling a request-sized regular expression', () => { + const oldText = `head ${'part '.repeat(16_000)}tail`; + const source = `before ${oldText.replace(/ /g, '\t')} after`; + const applied = applyTextEdits(source, [{ oldText, newText: 'result' }], 'tolerant'); + assert.equal(applied.text, 'before result after'); + assert.deepEqual(applied.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +test('whitespace-normalized matches count overlapping token sequences but replaceAll does not overlap', () => { + const source = 'a b a\tb a'; + const edit = { oldText: 'a b a', newText: 'x' }; + const error = rejection(() => applyTextEdits(source, [edit], 'tolerant')); + assert.match(error.message, /matched 2 locations/); + const replaced = applyTextEdits(source, [{ ...edit, replaceAll: true }], 'tolerant'); + assert.equal(replaced.text, 'x\tb a'); + assert.deepEqual(replaced.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +for (const fileEnding of ['\n', '\r\n'] as const) { + for (const [oldEnding, newEnding] of [['\r\n', '\n'], ['\n', '\r\n']] as const) { + test(`whitespace-normalized matching peels ${JSON.stringify(oldEnding)} to ${JSON.stringify(newEnding)} in ${JSON.stringify(fileEnding)} files`, () => { + const source = `foo bar${fileEnding}next`; + const result = applyTextEdits(source, [{ + oldText: `foo bar${oldEnding}`, + newText: `baz qux${newEnding}`, + }], 'tolerant'); + assert.equal(result.text, `baz qux${fileEnding}next`); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); + }); + } +} + +test('whitespace-normalized matching peels leading and trailing CRLF/LF wrappers in a batch', () => { + const result = applyTextEdits('header\r\n \tfoo bar\r\nnext\r\n', [ + { oldText: '\n foo bar\r\n', newText: '\r\n baz qux\n' }, + { oldText: 'next', newText: 'done' }, + ], 'tolerant'); + assert.equal(result.text, 'header\r\n \tbaz qux\r\ndone\r\n'); + assert.deepEqual(result.matches, [ + { strategy: 'whitespace-normalized', occurrences: 1 }, + { strategy: 'exact', occurrences: 1 }, + ]); +}); + +test('whitespace-normalized matching peels a trailing newline even if adjacent spaces differ', () => { + const result = applyTextEdits('foo bar \r\nnext', [{ + oldText: 'foo bar \r\n', newText: 'baz qux\n', + }], 'tolerant'); + assert.equal(result.text, 'baz qux \r\nnext'); +}); + +test('whitespace-normalized matching does not prepend new indentation beside preserved source indentation', () => { + const source = 'header\n foo bar\n'; + const result = applyTextEdits(source, [{ + oldText: '\n foo bar', newText: '\n\tbaz qux', + }], 'tolerant'); + assert.equal(result.text, 'header\n baz qux\n'); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +test('newline-only boundary prefixes peel the complete equivalent indentation run', () => { + for (const newline of ['\n', '\r\n']) { + const result = applyTextEdits(`header${newline} foo bar${newline}next`, [{ + oldText: '\nfoo bar\n', newText: '\n\tbaz qux\t\n', + }], 'tolerant'); + assert.equal(result.text, `header${newline} baz qux${newline}next`); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); + } +}); + +test('wrapper peeling preserves source whitespace across complete prefixes and extra-newline combinations', () => { + for (const newline of ['\n', '\r\n']) { + for (const oldIndent of ['', ' ', '\t']) { + for (const newIndent of ['', ' ', '\t']) { + for (let extra = 0; extra < 3; extra++) { + const extraLines = `${newline}${newIndent}`.repeat(extra); + const leading = applyTextEdits(`header${newline} foo bar${newline}`, [{ + oldText: `${newline}${oldIndent}foo bar`, + newText: `${newline}${newIndent}${extraLines}baz qux`, + }], 'tolerant'); + assert.equal(leading.text, `header${newline} ${extraLines}baz qux${newline}`); + const trailing = applyTextEdits(`foo bar${newline} next`, [{ + oldText: `foo bar${newline}${oldIndent}`, + newText: `baz qux${extraLines}${newline}${newIndent}`, + }], 'tolerant'); + assert.equal(trailing.text, `baz qux${extraLines}${newline} next`); + } + } + } + } +}); + +test('horizontal boundary prefixes do not silently duplicate mismatched spaces', () => { + const error = rejection(() => applyTextEdits(' foo bar', [ + { oldText: ' foo bar', newText: ' baz qux' }, + ], 'tolerant')); + assert.match(error.message, /old_text was not found/); +}); + +test('whitespace-normalized matching preserves extra leading blank lines when indentation changes', () => { + const source = 'header\n foo bar\n'; + const result = applyTextEdits(source, [{ + oldText: '\n foo bar', newText: '\n\t\n\tbaz qux', + }], 'tolerant'); + assert.equal(result.text, 'header\n \n\tbaz qux\n'); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +test('whitespace-normalized replaceAll preserves extra leading blank lines independently', () => { + const source = 'one\n foo bar\ntwo\n foo bar\n'; + const result = applyTextEdits(source, [{ + oldText: '\n foo bar', newText: '\n\t\n\tbaz qux', replaceAll: true, + }], 'tolerant'); + assert.equal(result.text, 'one\n \n\tbaz qux\ntwo\n \n\tbaz qux\n'); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 2 }]); +}); + +test('whitespace-normalized matching preserves extra lines after an identical leading wrapper', () => { + const result = applyTextEdits('header\r\n foo bar', [{ + oldText: '\r\n foo bar', newText: '\r\n \r\n\tbaz qux', + }], 'tolerant'); + assert.equal(result.text, 'header\r\n \r\n\tbaz qux'); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +test('whitespace-normalized matching preserves multiple extra leading blank lines', () => { + const source = 'header\r\n \r\n foo bar\r\n'; + const result = applyTextEdits(source, [{ + oldText: '\n \n foo bar', newText: '\r\n\t\r\n\t\r\n\t\r\n\tbaz qux', + }], 'tolerant'); + assert.equal(result.text, 'header\r\n \r\n \r\n\t\r\n\tbaz qux\r\n'); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +test('whitespace-normalized matching preserves extra trailing blank lines when boundary indentation changes', () => { + const source = 'foo bar\r\n next'; + const result = applyTextEdits(source, [{ + oldText: 'foo bar\n ', newText: 'baz qux\n\t\n\t', + }], 'tolerant'); + assert.equal(result.text, 'baz qux\r\n\t\r\n next'); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +test('whitespace-normalized matching leaves intentional indentation on an extra blank line', () => { + const result = applyTextEdits('foo bar\r\nnext', [{ + oldText: 'foo bar\n', newText: 'baz qux\n\t\n', + }], 'tolerant'); + assert.equal(result.text, 'baz qux\r\n\t\r\nnext'); +}); + +test('whitespace-normalized matching does not duplicate differing trailing spaces', () => { + const result = applyTextEdits('foo bar \nnext', [{ + oldText: 'foo bar \n', newText: 'baz qux\t\n', + }], 'tolerant'); + assert.equal(result.text, 'baz qux \nnext'); +}); + +test('whitespace-normalized matching peels a leading newline even if adjacent spaces differ', () => { + const result = applyTextEdits('header\r\n foo bar\r\nnext', [{ + oldText: '\n foo bar', newText: '\nchanged', + }], 'tolerant'); + assert.equal(result.text, 'header\r\n changed\r\nnext'); +}); + +test('whitespace-normalized matching rejects absent required boundary whitespace', () => { + const cases = [ + { source: 'foo bar', oldText: 'foo bar\r\n', newText: 'baz qux\n' }, + { source: 'foo bar next', oldText: 'foo bar\n', newText: 'baz qux\n' }, + { source: 'previous foo bar', oldText: '\nfoo bar', newText: '\nbaz qux' }, + { source: 'foo bar', oldText: ' foo bar', newText: ' baz qux' }, + { source: 'foo bar', oldText: 'foo bar ', newText: 'baz qux ' }, + { source: 'foo bar\nnext', oldText: 'foo bar\n\n', newText: 'baz qux\n\n' }, + { source: 'header\nfoo bar\nnext', oldText: '\n\nfoo bar', newText: '\n\nbaz qux' }, + { source: 'header\n \n foo bar', oldText: '\n \n foo bar', newText: '\n\tbaz qux' }, + ]; + for (const edit of cases) { + const { source, ...request } = edit; + const error = rejection(() => applyTextEdits(source, [request], 'tolerant')); + assert.match(error.message, /old_text was not found/, `unmatched boundary: ${JSON.stringify(edit)}`); + } +}); + +test('whitespace-normalized scanning skips boundary-invalid matches and keeps later valid ones', () => { + const source = 'header foo bar header\nfoo bar\nend'; + const result = applyTextEdits(source, [{ + oldText: '\nfoo bar\n', newText: '\nbaz qux\n', replaceAll: true, + }], 'tolerant'); + assert.equal(result.text, 'header foo bar header\nbaz qux\nend'); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +test('whitespace-normalized matching rejects boundary removal outside its token range', () => { + const cases = [ + { source: 'foo bar\r\nnext', oldText: 'foo bar\r\n', newText: 'baz qux' }, + { source: 'header\r\n foo bar', oldText: '\n foo bar', newText: 'baz qux' }, + { source: ' foo bar', oldText: ' foo bar', newText: 'baz qux' }, + { source: 'foo bar ', oldText: 'foo bar ', newText: 'baz qux' }, + { source: 'header\nfoo bar\nnext', oldText: '\nfoo bar\n', newText: '\n' }, + { source: 'foo bar\n\nnext', oldText: 'foo bar\n\n', newText: 'baz qux\n' }, + ]; + for (const edit of cases) { + const { source, ...request } = edit; + const error = rejection(() => applyTextEdits(source, [request], 'tolerant')); + assert.match(error.message, /old_text was not found/, `unremovable boundary: ${JSON.stringify(edit)}`); + } + const exact = applyTextEdits('foo bar\r\nnext', [{ + oldText: 'foo bar\r\n', newText: 'baz qux', + }], 'tolerant'); + assert.equal(exact.text, 'baz quxnext'); + assert.deepEqual(exact.matches, [{ strategy: 'exact', occurrences: 1 }]); +}); + +test('whitespace-normalized matching preserves two present boundary line breaks', () => { + const result = applyTextEdits('foo bar\n\nnext', [{ + oldText: 'foo bar\r\n\r\n', newText: 'baz qux\n\n', + }], 'tolerant'); + assert.equal(result.text, 'baz qux\n\nnext'); +}); + +test('whitespace-normalized matching preserves an intentional extra line break', () => { + const result = applyTextEdits('foo bar\r\nnext', [{ + oldText: 'foo bar\r\n', newText: 'baz qux\n\n', + }], 'tolerant'); + assert.equal(result.text, 'baz qux\r\n\r\nnext'); +}); + +test('line-trimmed replacements adopt the matched line ending in mixed files', () => { + const source = 'header\r\nfoo \nbar \nnext\n'; + const result = applyTextEdits(source, [{ oldText: 'foo\nbar', newText: 'baz\nqux' }], 'tolerant'); + assert.equal(result.text, 'header\r\nbaz\nqux\nnext\n'); + assert.deepEqual(result.matches, [{ strategy: 'line-trimmed', occurrences: 1 }]); +}); + +test('indentation-flexible replacements keep the selected line ending in mixed files', () => { + const source = 'header\r\n method() {\n return 1;\n }\nend\n'; + const result = applyTextEdits(source, [{ + oldText: 'method() {\n return 1;\n}', + newText: 'method() {\r\n return 2;\r\n}', + }], 'tolerant'); + assert.equal(result.text, 'header\r\n method() {\n return 2;\n }\nend\n'); + assert.deepEqual(result.matches, [{ strategy: 'indentation-flexible', occurrences: 1 }]); +}); + +test('line-trimmed replaceAll keeps each local ending in mixed files', () => { + const source = 'foo \r\nbar \r\nfoo \nbar \nend'; + const result = applyTextEdits(source, [{ + oldText: 'foo\nbar', newText: 'baz\nqux', replaceAll: true, + }], 'tolerant'); + assert.equal(result.text, 'baz\r\nqux\r\nbaz\nqux\nend'); + assert.deepEqual(result.matches, [{ strategy: 'line-trimmed', occurrences: 2 }]); +}); + +test('exact replacements preserve caller line endings rather than normalizing them', () => { + const source = 'foo bar\r\nnext\r\n'; + const result = applyTextEdits(source, [{ oldText: 'foo bar\r\n', newText: 'baz qux\n' }], 'tolerant'); + assert.equal(result.text, 'baz qux\nnext\r\n'); + assert.deepEqual(result.matches, [{ strategy: 'exact', occurrences: 1 }]); +}); + +test('whitespace-normalized replacement adopts the matched line ending in mixed files', () => { + const original = 'header\r\nfoo bar\nnext\n'; + const result = applyTextEdits(original, [{ + oldText: 'foo bar\n', newText: 'baz\nqux\n', + }], 'tolerant'); + assert.equal(result.text, 'header\r\nbaz\nqux\nnext\n'); +}); + +test('whitespace-normalized replaceAll peels boundary newlines on every mixed-ending match', () => { + const source = 'foo bar\r\nfoo bar\nend'; + const result = applyTextEdits(source, [{ + oldText: 'foo bar\r\n', newText: 'baz qux\n', replaceAll: true, + }], 'tolerant'); + assert.equal(result.text, 'baz qux\r\nbaz qux\nend'); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 2 }]); +}); + +test('whitespace-normalized replaceAll uses each matched line ending in mixed files', () => { + const source = 'foo bar\r\nfoo bar\nend'; + const result = applyTextEdits(source, [{ + oldText: 'foo bar', newText: 'baz\nqux', replaceAll: true, + }], 'tolerant'); + assert.equal(result.text, 'baz\r\nqux\r\nbaz\nqux\nend'); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 2 }]); +}); + +test('whitespace-normalized replaceAll finds many same-line matches without rescanning newline tails', () => { + const source = 'foo bar '.repeat(16_000); + const start = performance.now(); + const result = applyTextEdits(source, [{ + oldText: 'foo bar', newText: 'baz\nqux', replaceAll: true, + }], 'tolerant'); + assert.equal(result.text, 'baz\nqux '.repeat(16_000)); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 16_000 }]); + assert.ok(performance.now() - start < 2_000, 'line endings must not be searched from each match'); +}); + +test('whitespace-normalized matches do not indent new_text twice', () => { + const text = ' total = price *\n quantity;\n'; + const applied = applyTextEdits( + text, + [{ oldText: ' total = price * quantity;', newText: ' total = price * quantity * rate;' }], + 'tolerant', + ); + assert.equal(applied.text, ' total = price * quantity * rate;\n'); + assert.deepEqual(applied.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +test('tolerant matching still refuses an ambiguous edit', () => { + const error = rejection(() => + applyTextEdits('x = 1 \ny = 2\nx = 1\n', [{ oldText: 'x = 1', newText: 'x = 3' }], 'tolerant'), + ); + assert.match(error.message, /matched 2 locations at lines 1, 3/); +}); + +test('exact overlap counts UTF-16 offsets while replaceAll consumes whole characters', () => { + const error = rejection(() => applyTextEdits('😀😀😀', [ + { oldText: '😀😀', newText: 'x' }, + ])); + assert.match(error.message, /matched 2 locations/); + const result = applyTextEdits('😀😀😀', [{ oldText: '😀😀', newText: 'x', replaceAll: true }]); + assert.equal(result.text, 'x😀'); + assert.deepEqual(result.matches, [{ strategy: 'exact', occurrences: 1 }]); +}); + +test('long overlapping exact matches are counted without repeatedly rescanning the input', () => { + const text = 'a'.repeat(450_000); + const oldText = 'a'.repeat(220_000); + const started = performance.now(); + const error = rejection(() => applyTextEdits(text, [{ oldText, newText: 'b' }])); + assert.match(error.message, /matched 230001 locations/); + assert.ok(performance.now() - started < 3_000, 'overlapping ambiguity must be counted in linear time'); +}); + +test('highly repeated exact matches report the count with bounded line samples', () => { + const error = rejection(() => applyTextEdits('z'.repeat(200_000), [ + { oldText: 'z', newText: 'y' }, + ])); + assert.match(error.message, /matched 200000 locations at lines 1, 1, 1, 1, 1 and 199995 more/); +}); + +test('replaceAll replaces every location and reports the count', () => { + const applied = applyTextEdits('foo(); bar(); foo();', [ + { oldText: 'foo()', newText: 'baz()', replaceAll: true }, + ]); + assert.equal(applied.text, 'baz(); bar(); baz();'); + assert.deepEqual(applied.matches, [{ strategy: 'exact', occurrences: 2 }]); +}); + +test('replaceAll over line windows never overlaps its own matches', () => { + const applied = applyTextEdits( + 'a\na\na\na\n', + [{ oldText: 'a\na', newText: 'b', replaceAll: true }], + 'tolerant', + ); + assert.equal(applied.text, 'b\nb\n'); + assert.deepEqual(applied.matches, [{ strategy: 'exact', occurrences: 2 }]); +}); + +test('replaceAll bounds memory for a million identical and changed matches', () => { + const source = 'x'.repeat(1024 * 1024); + const noop = applyTextEdits(source, [ + { oldText: 'x', newText: 'x', replaceAll: true }, + { oldText: 'x', newText: 'y', replaceAll: true }, + ]); + assert.equal(noop.text, 'y'.repeat(source.length)); + assert.deepEqual(noop.matches, [ + { strategy: 'exact', occurrences: source.length }, + { strategy: 'exact', occurrences: source.length }, + ]); +}); + +test('replaceAll preserves non-overlapping literal edits when identical hits precede changed hits', () => { + for (const [source, oldText, newText] of [ + ['aaaaa', 'aa', 'a'], + ['foo foo foo', 'foo', 'bar'], + ['😀😀😀', '😀😀', 'x'], + ['x\nx\nx', 'x', ''], + ['aba aba', 'aba', 'aba'], + ]) { + const result = applyTextEdits(source, [{ oldText, newText, replaceAll: true }]); + assert.equal(result.text, source.split(oldText).join(newText)); + assert.deepEqual(result.matches, [{ strategy: 'exact', occurrences: source.split(oldText).length - 1 }]); + } + const source = 'foo\tbar foo bar'; + const tolerant = applyTextEdits(source, [{ + oldText: 'foo bar', newText: 'foo\tbar', replaceAll: true, + }], 'tolerant'); + assert.equal(tolerant.text, 'foo\tbar foo\tbar'); + assert.deepEqual(tolerant.matches, [{ strategy: 'whitespace-normalized', occurrences: 2 }]); +}); + +test('replaceAll handles repeated tolerant and non-overlapping line-window matches', () => { + const lines = 'foo \r\nbar \r\n'.repeat(4000); + const result = applyTextEdits(lines, [ + { oldText: 'foo\nbar', newText: 'baz\nqux', replaceAll: true }, + { oldText: 'baz qux', newText: 'updated', replaceAll: true }, + ], 'tolerant'); + assert.equal(result.text, 'updated\r\n'.repeat(4000)); + assert.deepEqual(result.matches, [ + { strategy: 'line-trimmed', occurrences: 4000 }, + { strategy: 'whitespace-normalized', occurrences: 4000 }, + ]); +}); + +test('replaceAll rejects oversized intermediate output before constructing it', () => { + assert.throws( + () => applyTextEdits('x'.repeat(1_000_000), [ + { oldText: 'x', newText: 'y'.repeat(100_000), replaceAll: true }, + ]), + WorkspaceEditOutputLimitError, + ); +}); + +test('replaceAll still fails when nothing matches', () => { + const error = rejection(() => + applyTextEdits('abc', [{ oldText: 'xyz', newText: '', replaceAll: true }]), + ); + assert.match(error.message, /old_text was not found/); +}); + +test('line diagnostic index follows earlier successful batch edits', () => { + const error = rejection(() => applyTextEdits('alpha\nbeta\n', [ + { oldText: 'not present', newText: 'skip' }, + { oldText: 'alpha', newText: 'first\nsecond' }, + { oldText: 'beta\nmissing', newText: 'nope' }, + ])); + assert.match(error.message, /Edit 3: old_text was not found; its first line appears at line 3/); +}); + +test('newline-dense files do not materialize millions of line objects across failed batch edits', () => { + const source = 'a\n'.repeat(450_000); + const edits = Array.from({ length: 80 }, () => ({ oldText: 'missing\ntext', newText: 'other' })); + const started = performance.now(); + const error = rejection(() => applyTextEdits(source, edits)); + assert.equal(error.failures.length, edits.length); + assert.ok(performance.now() - started < 10_000, 'a bounded file must not re-index every failed diagnostic'); +}); + +test('first-line hints report bounded samples even when the line repeats throughout a file', () => { + const error = rejection(() => applyTextEdits('a\n'.repeat(100_000), [ + { oldText: 'a\nmissing', newText: 'replacement' }, + ])); + assert.match(error.message, /its first line appears at lines 1, 2, 3, 4, 5 and 99995 more/); +}); + +test('one uniquely anchored indentation candidate is allowed on both sides of the verification threshold', () => { + for (const length of [100_000, 100_001]) { + const oldText = ['anchor', ...Array.from({ length: length - 2 }, () => 'x'), 'end'].join('\n'); + const source = ` ${oldText.replaceAll('\n', '\n ')}\n`; + const result = applyTextEdits(source, [{ oldText, newText: 'changed' }], 'tolerant'); + assert.equal(result.text, ' changed\n'); + assert.deepEqual(result.matches, [{ strategy: 'indentation-flexible', occurrences: 1 }]); + } +}); + +test('large indentation verification still bounds repeated candidates', () => { + const oldText = `${'x\n'.repeat(100_000)}x`; + const source = ' x\n'.repeat(100_002); + const error = rejection(() => applyTextEdits(source, [{ oldText, newText: 'changed' }], 'tolerant')); + assert.match(error.message, /too many repetitive line-window candidates/); +}); + +test('repetitive indentation candidates fail closed after a bounded comparison budget', () => { + const error = rejection(() => applyTextEdits(' a\n'.repeat(1_200), [ + { oldText: `${'a\n'.repeat(199)}a`, newText: 'changed' }, + ], 'tolerant')); + assert.match(error.message, /too many repetitive line-window candidates/); +}); + +test('repetitive long line windows return a missing-edit diagnosis without quadratic scans', () => { + const text = 'line\n'.repeat(24_000); + const oldText = `${'line\n'.repeat(6_000)}missing`; + const started = performance.now(); + const error = rejection(() => applyTextEdits(text, [{ oldText, newText: 'replacement' }])); + assert.match(error.message, /did not apply and nothing was written/); + assert.ok(performance.now() - started < 2_000, 'a bounded edit must not compare every long window'); +}); + +test('a hundred failing multi-line edits on a large file are diagnosed quickly', () => { + const text = Array.from({ length: 30_000 }, (_, index) => ` line ${index} = value;`).join('\n'); + const edits = Array.from({ length: 100 }, (_, index) => ({ + oldText: `line ${index} = value;\n other ${index};\n more ${index};`, + newText: 'y', + })); + const started = performance.now(); + rejection(() => applyTextEdits(text, edits)); + assert.ok(performance.now() - started < 5_000); +}); + +test('diagnostics for a hundred failing edits stay within the settlement bound', () => { + const edits = Array.from({ length: 100 }, (_, index) => ({ + oldText: `missing ${'x'.repeat(200)} ${index}`, + newText: 'y', + })); + const error = rejection(() => applyTextEdits('content\n'.repeat(1000), edits)); + assert.ok(error.message.length <= EDIT_DIAGNOSTIC_MAX_CHARS); + assert.match(error.message, /^100 of 100 workspace edits did not apply/); + assert.deepEqual( + [...error.message.matchAll(/\nEdit (\d+):/g)].map((match) => Number(match[1])), + Array.from({ length: 100 }, (_, index) => index + 1), + ); + assert.doesNotMatch(error.message, /failing edits not shown/); +}); + +test('bounded batch diagnostics preserve sparse failing positions rather than only their count', () => { + const source = `KEEP\nconst shared = '${'x'.repeat(180)}';\n`; + const edits = Array.from({ length: 60 }, (_, index) => index % 2 === 0 + ? { oldText: `const shared missing_${index}`, newText: 'replacement' } + : { oldText: 'KEEP', newText: 'KEEP' }); + const error = rejection(() => applyTextEdits(source, edits)); + assert.equal(error.failures.length, 30); + assert.match(error.message, /^30 of 60 workspace edits did not apply/); + assert.ok(error.message.length <= EDIT_DIAGNOSTIC_MAX_CHARS); + assert.deepEqual( + [...error.message.matchAll(/\nEdit (\d+):/g)].map((match) => Number(match[1])), + Array.from({ length: 30 }, (_, index) => 2 * index + 1), + ); + assert.doesNotMatch(error.message, /failing edits not shown/); +}); + +test('bounded diagnostics keep all positions and well-formed Unicode with mixed reason lengths', () => { + for (const count of [2, 30, 100]) { + const failures = Array.from({ length: count }, (_, index) => ({ + index, + reason: index % 3 === 0 ? 'missing' : `source excerpt ${'😀'.repeat(300)}`, + })); + const error = new WorkspaceEditMatchError(failures, count); + assert.ok(error.message.length <= EDIT_DIAGNOSTIC_MAX_CHARS); + assert.equal(Buffer.from(error.message).toString('utf8'), error.message); + assert.deepEqual( + [...error.message.matchAll(/\nEdit (\d+):/g)].map((match) => Number(match[1])), + Array.from({ length: count }, (_, index) => index + 1), + ); + } +}); + +test('short batch diagnostic reasons remain complete when the message fits', () => { + const error = rejection(() => applyTextEdits('a\nb\nb\n', [ + { oldText: 'missing', newText: 'x' }, + { oldText: 'b', newText: 'B' }, + ])); + for (const failure of error.failures) { + assert.ok(error.message.includes(`\nEdit ${failure.index + 1}: ${failure.reason}.`)); + } +}); diff --git a/packages/code/src/edits.ts b/packages/code/src/edits.ts new file mode 100644 index 00000000..a6d54269 --- /dev/null +++ b/packages/code/src/edits.ts @@ -0,0 +1,712 @@ +import { BRIDGE_WORKSPACE_WRITE_MAX_BYTES } from './protocol.js'; + +import type { + WorkspaceEditMatch, + WorkspaceEditMatching, + WorkspaceEditMatchStrategy, + WorkspaceTextEdit, +} from './protocol.js'; + +/** + * The Code API rejects a settlement whose error exceeds 4096 characters, and + * callers prefix their own context, so diagnostics stay well below that. + */ +export const EDIT_DIAGNOSTIC_MAX_CHARS = 3000; +const MAX_REPORTED_LINES = 5; +const MAX_SNIPPET_CHARS = 120; +/** A highly repetitive indentation candidate must not monopolize the worker. */ +const MAX_LINE_WINDOW_VERIFICATIONS = 100_000; +const MAX_REPLACEMENT_CHUNK_CHARS = 16 * 1024; + +interface MatchedRange { + start: number; + end: number; + /** Replacement for this range, already adapted to its indentation and line endings. */ + replacement: string; +} + +type MatchOutcome = + | { status: 'matched'; strategy: WorkspaceEditMatchStrategy; ranges: MatchedRange[]; occurrences: number; updated?: string } + | { status: 'ambiguous'; strategy: WorkspaceEditMatchStrategy; count: number; starts: number[] } + | { status: 'limit' } + | { status: 'none' }; + +interface CollectedMatches { + count: number; + source: string; + projectedLength: number; + /** Only the first range is needed for a unique edit or an exact-mode hint. */ + ranges: MatchedRange[]; + starts: number[]; + output?: { chunks: string[]; pending: string; cursor: number }; +} + +export interface EditFailure { + /** Zero-based position of the edit in the request. */ + index: number; + reason: string; +} + +export class WorkspaceEditMatchError extends Error { + constructor( + readonly failures: EditFailure[], + readonly editCount: number, + ) { + super(formatEditFailures(failures, editCount)); + this.name = 'WorkspaceEditMatchError'; + } +} + +export class WorkspaceEditOutputLimitError extends Error { + constructor() { + super('Workspace file exceeds write limit'); + this.name = 'WorkspaceEditOutputLimitError'; + } +} + +export interface AppliedEdits { + text: string; + matches: WorkspaceEditMatch[]; +} + +/** + * Applies ordered replacements to `text`. Each edit sees the result of the + * edits before it. Every edit is attempted even after a failure, so a caller + * learns about all unmatched or ambiguous edits from one rejection; nothing is + * returned unless every edit applied. + */ +export function applyTextEdits( + text: string, + edits: readonly WorkspaceTextEdit[], + matching: WorkspaceEditMatching = 'exact', +): AppliedEdits { + let working = text; + let lineIndex: LineIndex | undefined; + const lines = (): LineIndex => (lineIndex ??= new LineIndex(working)); + const matches: WorkspaceEditMatch[] = []; + const failures: EditFailure[] = []; + edits.forEach((edit, index) => { + const outcome = findEditMatch(working, edit, matching, lines); + if (outcome.status === 'matched') { + working = outcome.updated ?? replaceRanges(working, outcome.ranges); + lineIndex = undefined; + matches.push({ strategy: outcome.strategy, occurrences: outcome.occurrences }); + return; + } + failures.push({ + index, + reason: + outcome.status === 'ambiguous' + ? describeAmbiguous(working, outcome.strategy, outcome.count, outcome.starts) + : outcome.status === 'limit' + ? 'old_text has too many repetitive line-window candidates; include more surrounding lines or use an exact match' + : describeMissing(working, edit.oldText, matching, lines), + }); + }); + if (failures.length > 0) { + throw new WorkspaceEditMatchError(failures, edits.length); + } + return { text: working, matches }; +} + +function findEditMatch( + text: string, + edit: WorkspaceTextEdit, + matching: WorkspaceEditMatching, + lines: () => LineIndex, +): MatchOutcome { + const strategies = matching === 'tolerant' ? TOLERANT_STRATEGIES : EXACT_STRATEGIES; + for (const find of strategies) { + const outcome = find(text, edit, lines); + if (outcome.status !== 'none') return outcome; + } + return { status: 'none' }; +} + +function collectedMatches(text: string, replaceAll?: boolean): CollectedMatches { + return { + count: 0, source: text, projectedLength: text.length, starts: [], ranges: [], + ...(replaceAll === true ? { output: { chunks: [], pending: '', cursor: 0 } } : {}), + }; +} + +function appendReplacement(output: NonNullable, part: string): void { + if (!part) return; + output.pending += part; + if (output.pending.length >= MAX_REPLACEMENT_CHUNK_CHARS) { + output.chunks.push(output.pending); + output.pending = ''; + } +} + +function finishReplacement(matches: CollectedMatches): string { + const output = matches.output!; + if (output.cursor === 0) return matches.source; + appendReplacement(output, matches.source.slice(output.cursor)); + return output.chunks.join('') + output.pending; +} + +function collectMatch( + matches: CollectedMatches, + start: number, + end: number, + replacement: string, + replaceAll?: boolean, +): void { + if (replaceAll === true) { + matches.projectedLength += replacement.length - (end - start); + // Even deleting every remaining source character cannot bring this + // intermediate below the 1 MiB limit. Do not retain more matches. + if (matches.projectedLength - (matches.source.length - end) > BRIDGE_WORKSPACE_WRITE_MAX_BYTES) { + throw new WorkspaceEditOutputLimitError(); + } + // Leave unchanged ranges in the source suffix rather than constructing a + // million identical output pieces for a no-op replaceAll edit. + if (replacement !== matches.source.slice(start, end)) { + const output = matches.output!; + appendReplacement(output, matches.source.slice(output.cursor, start)); + appendReplacement(output, replacement); + output.cursor = end; + } + } + matches.count++; + if (matches.starts.length < MAX_REPORTED_LINES) matches.starts.push(start); + if (matches.count === 1) matches.ranges.push({ start, end, replacement }); +} + +function resolve( + strategy: WorkspaceEditMatchStrategy, + matches: CollectedMatches, + replaceAll: boolean | undefined, +): MatchOutcome { + if (matches.count === 0) return { status: 'none' }; + if (matches.count === 1 || replaceAll === true) { + return { + status: 'matched', strategy, ranges: matches.ranges, occurrences: matches.count, + ...(replaceAll === true ? { updated: finishReplacement(matches) } : {}), + }; + } + return { status: 'ambiguous', strategy, count: matches.count, starts: matches.starts }; +} + +/** + * Overlapping occurrences count toward ambiguity (`aa` in `aaa` is two + * locations), while `replaceAll` replaces non-overlapping occurrences. + */ +function findExact(text: string, edit: WorkspaceTextEdit): MatchOutcome { + if (edit.oldText.length === 0) return { status: 'none' }; + const matches = collectedMatches(text, edit.replaceAll); + const prefix = prefixTable(edit.oldText); + let matched = 0; + for (let index = 0; index < text.length; index++) { + while (matched > 0 && text[index] !== edit.oldText[matched]) matched = prefix[matched - 1]; + if (text[index] === edit.oldText[matched]) matched++; + if (matched !== edit.oldText.length) continue; + collectMatch(matches, index - matched + 1, index + 1, edit.newText, edit.replaceAll); + // Ambiguity counts overlaps, but replaceAll must consume disjoint ranges. + matched = edit.replaceAll === true ? 0 : prefix[matched - 1]; + } + return resolve('exact', matches, edit.replaceAll); +} + +interface Line { + /** Offset of the first character of the line. */ + start: number; + /** Offset just past the line's content, excluding `\r\n` or `\n`. */ + end: number; + /** Offset just past the line terminator, or `undefined` for an unterminated last line. */ + next: number | undefined; + text: string; +} + +/** One bounded index per source revision, shared by matching and diagnostics. */ +class LineIndex { + private readonly newlines: Uint32Array; + readonly length: number; + + constructor(private readonly source: string) { + this.newlines = new Uint32Array(newlineCount(source)); + let index = 0; + for (let offset = source.indexOf('\n'); offset >= 0; offset = source.indexOf('\n', offset + 1)) { + this.newlines[index++] = offset; + } + // A trailing terminator leaves an empty final line, just as before. + this.length = this.newlines.length + 1; + } + + start(index: number): number { + return index === 0 ? 0 : this.newlines[index - 1] + 1; + } + + next(index: number): number | undefined { + return index < this.newlines.length ? this.newlines[index] + 1 : undefined; + } + + end(index: number): number { + const start = this.start(index); + const end = index < this.newlines.length ? this.newlines[index] : this.source.length; + return end > start && this.source[end - 1] === '\r' ? end - 1 : end; + } + + text(index: number): string { + return this.source.slice(this.start(index), this.end(index)); + } + + at(index: number): Line { + return { start: this.start(index), end: this.end(index), next: this.next(index), text: this.text(index) }; + } +} + +/** + * Lines a line-window strategy must match. A trailing line break means "through + * the end of the last line", not "followed by an empty line". + */ +function neededLines(oldText: string): { lines: string[]; throughTerminator: boolean } { + const normalized = oldText.replace(/\r\n/g, '\n'); + const throughTerminator = normalized.endsWith('\n'); + return { + lines: (throughTerminator ? normalized.slice(0, -1) : normalized).split('\n'), + throughTerminator, + }; +} + +/** Fallback when neither the matched line nor its neighbors have a terminator. */ +function fileLineEnding(text: string): '\r\n' | '\n' { + return text.includes('\r\n') ? '\r\n' : '\n'; +} + +function lineEndingAt(text: string, newline: number): '\r\n' | '\n' { + return newline > 0 && text[newline - 1] === '\r' ? '\r\n' : '\n'; +} + +function leadingWhitespace(line: string): string { + return /^[ \t]*/.exec(line)?.[0] ?? ''; +} + +function commonIndent(lines: Iterable): string { + let common: string | undefined; + for (const line of lines) { + if (line.trim().length === 0) continue; + const indent = leadingWhitespace(line); + if (common === undefined) { + common = indent; + continue; + } + let shared = 0; + while (shared < common.length && shared < indent.length && common[shared] === indent[shared]) { + shared++; + } + common = common.slice(0, shared); + } + return common ?? ''; +} + +function withLineEnding(value: string, ending: '\r\n' | '\n'): string { + const normalized = value.replace(/\r\n/g, '\n'); + return ending === '\r\n' ? normalized.replace(/\n/g, '\r\n') : normalized; +} + +function prefixTable(values: ArrayLike): Uint32Array { + const prefix = new Uint32Array(values.length); + for (let index = 1, matched = 0; index < values.length; index++) { + while (matched > 0 && values[index] !== values[matched]) matched = prefix[matched - 1]; + if (values[index] === values[matched]) matched++; + prefix[index] = matched; + } + return prefix; +} + +/** + * Line-window strategies compare whole lines, so a match always spans from the + * start of its first line to the end of its last line's content. The file's + * own line terminators are kept, and the replacement adopts them. + */ +function findLineWindows( + text: string, + edit: WorkspaceTextEdit, + strategy: 'line-trimmed' | 'indentation-flexible', + getLines: () => LineIndex, +): MatchOutcome { + const { lines: needle, throughTerminator } = neededLines(edit.oldText); + if (needle.every((line) => line.trim().length === 0)) return { status: 'none' }; + const lines = getLines(); + const ending = fileLineEnding(text); + const needleIndent = strategy === 'indentation-flexible' ? commonIndent(needle) : ''; + const normalizedNeedle = needle.map((line) => + (strategy === 'indentation-flexible' ? stripIndent(line, needleIndent) : line).trimEnd(), + ); + if (needle.length > lines.length) return { status: 'none' }; + // For line-trimmed matching, the normalized lines are the complete comparison. + // For indentation-flexible matching, whole-line content is a linear-time + // prefilter; only complete candidates need their relative indent verified. + const sought = strategy === 'line-trimmed' + ? normalizedNeedle + : normalizedNeedle.map((line) => line.trimStart()); + const prefix = prefixTable(sought); + const collected = collectedMatches(text, edit.replaceAll); + let matched = 0; + let verifications = 0; + // Allow one complete linear verification even for a large valid block. + const verificationBudget = Math.max(MAX_LINE_WINDOW_VERIFICATIONS, needle.length); + for (let index = 0; index < lines.length; index++) { + const value = strategy === 'line-trimmed' ? lines.text(index).trimEnd() : lines.text(index).trim(); + while (matched > 0 && value !== sought[matched]) matched = prefix[matched - 1]; + if (value === sought[matched]) matched++; + if (matched !== sought.length) continue; + + const first = index - sought.length + 1; + const end = throughTerminator ? lines.next(index) : lines.end(index); + let windowIndent = ''; + let valid = end !== undefined; + if (valid && strategy === 'indentation-flexible') { + if (verifications + needle.length > verificationBudget) return { status: 'limit' }; + verifications += needle.length; + windowIndent = commonIndent((function* () { + for (let offset = 0; offset < needle.length; offset++) yield lines.text(first + offset); + })()); + valid = normalizedNeedle.every((expected, offset) => + stripIndent(lines.text(first + offset), windowIndent).trimEnd() === expected, + ); + } + if (valid) { + const replacement = strategy === 'indentation-flexible' + ? reindent(edit.newText, needleIndent, windowIndent) + : edit.newText; + const lineFeed = lines.next(first) ?? (first > 0 ? lines.next(first - 1) : undefined); + const localEnding = lineFeed === undefined ? ending : lineEndingAt(text, lineFeed - 1); + collectMatch(collected, lines.start(first), end!, withLineEnding(replacement, localEnding), edit.replaceAll); + } + // A replacement cannot consume overlapping lines; ambiguity still counts them. + matched = valid && edit.replaceAll === true ? 0 : prefix[matched - 1]; + } + return resolve(strategy, collected, edit.replaceAll); +} + +function stripIndent(line: string, indent: string): string { + return line.startsWith(indent) ? line.slice(indent.length) : line.trimStart(); +} + +/** + * Moves `value` from the indentation the caller wrote to the file's own. Lines + * the caller indented less than its `old_text` keep their indentation as written. + */ +function reindent(value: string, from: string, to: string): string { + if (from === to) return value; + return value + .replace(/\r\n/g, '\n') + .split('\n') + .map((line) => + line.trim().length > 0 && line.startsWith(from) ? to + line.slice(from.length) : line, + ) + .join('\n'); +} + +function findLineTrimmed(text: string, edit: WorkspaceTextEdit, lines: () => LineIndex): MatchOutcome { + return findLineWindows(text, edit, 'line-trimmed', lines); +} + +function findIndentationFlexible(text: string, edit: WorkspaceTextEdit, lines: () => LineIndex): MatchOutcome { + return findLineWindows(text, edit, 'indentation-flexible', lines); +} + +function newlineCount(text: string): number { + let count = 0; + for (let index = text.indexOf('\n'); index >= 0; index = text.indexOf('\n', index + 1)) count++; + return count; +} + +/** A boundary supplied by oldText must exist beside the candidate tokens. */ +function hasBoundaryWhitespace( + text: string, + position: number, + direction: -1 | 1, + requiredNewlines: number, +): boolean { + for (let index = position; index >= 0 && index < text.length; index += direction) { + const char = text[index]; + if (!/\s/.test(char)) break; + if (char === '\n') requiredNewlines--; + if (requiredNewlines <= 0) return true; + } + return false; +} + +/** A token-only match leaves the source's outer whitespace untouched. */ +function peelBoundaryWhitespace( + text: string, + boundary: string, + side: 'leading' | 'trailing', +): string | undefined { + if (!boundary) return text; + const whitespace = (side === 'leading' ? /^\s*/ : /\s*$/).exec(text)?.[0] ?? ''; + if (whitespace === boundary) { + return side === 'leading' ? text.slice(boundary.length) : text.slice(0, -boundary.length); + } + + const requiredNewlines = newlineCount(boundary); + if (!requiredNewlines) return undefined; + if (newlineCount(whitespace) < requiredNewlines) return undefined; + + if (side === 'leading') { + let end = 0; + for (let count = 0; count < requiredNewlines; count++) { + end = whitespace.indexOf('\n', end) + 1; + } + // Consume the equivalent indentation, stopping before the first extra + // newline. Extra blank lines belong to the replacement, not the wrapper. + while (end < whitespace.length && whitespace[end] !== '\n') end++; + return text.slice(end); + } + + if (newlineCount(whitespace) === requiredNewlines) { + return text.slice(0, text.length - whitespace.length); + } + let start = whitespace.length; + for (let count = 0; count < requiredNewlines; count++) { + start = whitespace.lastIndexOf('\n', start - 1); + } + // Indentation before this newline belongs to the extra blank line. + return text.slice(0, text.length - (whitespace.length - start)); +} + +/** + * Tolerates any run of whitespace, including line breaks, between tokens. The + * match starts and ends on a token, so whitespace the caller wrapped around + * `old_text` is also peeled off `new_text` rather than inserted twice. + */ +function findWhitespaceNormalized(text: string, edit: WorkspaceTextEdit): MatchOutcome { + const oldText = edit.oldText.replace(/\r\n/g, '\n'); + const tokens = oldText.trim().split(/\s+/).filter(Boolean); + if (tokens.length < 2) return { status: 'none' }; + // The matched range contains tokens only; leave the file's boundary whitespace + // outside it. Normalize the caller's line endings *before* peeling equivalent + // wrappers, otherwise CRLF/LF differences insert a second line break. + const leading = /^\s*/.exec(oldText)?.[0] ?? ''; + const trailing = /\s*$/.exec(oldText)?.[0] ?? ''; + const leadingNewlines = newlineCount(leading); + const trailingNewlines = newlineCount(trailing); + const normalizedNewText = edit.newText.replace(/\r\n/g, '\n'); + const withoutLeading = peelBoundaryWhitespace(normalizedNewText, leading, 'leading'); + if (withoutLeading === undefined) return { status: 'none' }; + const newText = peelBoundaryWhitespace(withoutLeading, trailing, 'trailing'); + if (newText === undefined) return { status: 'none' }; + const lfReplacement = newText; + const crlfReplacement = withLineEnding(newText, '\r\n'); + // Match entire whitespace-delimited tokens, never an identifier prefix or + // suffix. A fixed-size regex tokenizes the file; KMP keeps repetitive input + // linear without compiling user-provided text as a regular expression. + const prefix = prefixTable(tokens); + const tokenStarts = new Uint32Array(tokens.length); + const collected = collectedMatches(text, edit.replaceAll); + const words = /\S+/g; + let matched = 0; + let tokenIndex = 0; + // Advance the newline cursor only forwards. replaceAll may encounter many + // matches on one long line, so searching from each match would be quadratic. + let nextNewline = text.indexOf('\n'); + let previousNewline = -1; + for (let word = words.exec(text); word != null; word = words.exec(text)) { + tokenStarts[tokenIndex % tokens.length] = word.index; + while (matched > 0 && word[0] !== tokens[matched]) matched = prefix[matched - 1]; + if (word[0] === tokens[matched]) matched++; + if (matched === tokens.length) { + const start = tokenStarts[(tokenIndex + 1) % tokens.length]; + const end = word.index + word[0].length; + const boundariesMatch = + (leading.length === 0 || hasBoundaryWhitespace(text, start - 1, -1, leadingNewlines)) && + (trailing.length === 0 || hasBoundaryWhitespace(text, end, 1, trailingNewlines)); + if (boundariesMatch) { + while (nextNewline >= 0 && nextNewline < start) { + previousNewline = nextNewline; + nextNewline = text.indexOf('\n', nextNewline + 1); + } + const nearestNewline = nextNewline >= 0 ? nextNewline : previousNewline; + const replacement = nearestNewline >= 0 && lineEndingAt(text, nearestNewline) === '\r\n' + ? crlfReplacement + : lfReplacement; + collectMatch(collected, start, end, replacement, edit.replaceAll); + } + matched = boundariesMatch && edit.replaceAll === true ? 0 : prefix[matched - 1]; + } + tokenIndex++; + } + return resolve('whitespace-normalized', collected, edit.replaceAll); +} + +type Strategy = (text: string, edit: WorkspaceTextEdit, lines: () => LineIndex) => MatchOutcome; + +/** + * Loosest last. Line-window strategies run before whitespace normalization + * because they can carry `new_text` over to the file's indentation; a + * whitespace-normalized match would insert it exactly as written. + */ +const RELAXED_STRATEGIES: readonly Strategy[] = [ + findLineTrimmed, + findIndentationFlexible, + findWhitespaceNormalized, +]; +const EXACT_STRATEGIES: readonly Strategy[] = [findExact]; +const TOLERANT_STRATEGIES: readonly Strategy[] = [findExact, ...RELAXED_STRATEGIES]; + +function replaceRanges(text: string, ranges: readonly MatchedRange[]): string { + const length = ranges.reduce( + (total, range) => total + range.replacement.length - (range.end - range.start), + text.length, + ); + // Every valid UTF-8 string has at least this many encoded bytes. Reject a + // pathological replaceAll before allocating an unbounded intermediate string. + if (length > BRIDGE_WORKSPACE_WRITE_MAX_BYTES) throw new WorkspaceEditOutputLimitError(); + let result = ''; + let cursor = 0; + for (const range of ranges) { + result += text.slice(cursor, range.start) + range.replacement; + cursor = range.end; + } + return result + text.slice(cursor); +} + +function lineNumberAt(text: string, offset: number): number { + let line = 1; + for (let index = text.indexOf('\n'); index >= 0 && index < offset; index = text.indexOf('\n', index + 1)) { + line++; + } + return line; +} + +function formatLineList(text: string, starts: readonly number[], count = starts.length): string { + const shown = starts.slice(0, MAX_REPORTED_LINES).map((start) => lineNumberAt(text, start)); + const more = count - shown.length; + return `line${shown.length === 1 ? '' : 's'} ${shown.join(', ')}${more > 0 ? ` and ${more} more` : ''}`; +} + +function describeAmbiguous( + text: string, + strategy: WorkspaceEditMatchStrategy, + count: number, + starts: readonly number[], +): string { + const how = strategy === 'exact' ? '' : ` (${strategy})`; + return `old_text matched ${count} locations${how} at ${formatLineList(text, starts, count)}; include more surrounding lines so it matches exactly one`; +} + +function snippet(line: string): string { + const trimmed = line.trim(); + return JSON.stringify( + trimmed.length > MAX_SNIPPET_CHARS ? `${trimmed.slice(0, MAX_SNIPPET_CHARS)}…` : trimmed, + ); +} + +const ELISION_LINE = /^\s*(?:(?:\/\/|#|--|\/\*|\*|))?\s*$/; +const LINE_NUMBER_PREFIX = /^\s*\d+\s*(?:\||:|\t)/; + +function describeMissing( + text: string, + oldText: string, + matching: WorkspaceEditMatching, + lines: () => LineIndex, +): string { + const hints: string[] = []; + const nonBlank = neededLines(oldText).lines.filter((line) => line.trim().length > 0); + if (nonBlank.some((line) => ELISION_LINE.test(line))) { + hints.push('it contains an elision placeholder ("..."); copy the exact lines instead of abbreviating'); + } + if (nonBlank.length > 0 && nonBlank.every((line) => LINE_NUMBER_PREFIX.test(line))) { + hints.push('it appears to include line-number prefixes from read_file output; remove them'); + } + if (matching === 'exact') { + let tolerant: MatchOutcome | undefined; + for (const find of RELAXED_STRATEGIES) { + // Diagnose match existence, not whether the requested replacement can + // preserve its out-of-range wrappers. A no-op keeps those wrappers valid. + tolerant = find(text, { oldText, newText: oldText }, lines); + if (tolerant.status !== 'none') break; + } + if (tolerant?.status === 'matched') { + hints.push( + `the same text exists at ${formatLineList(text, [tolerant.ranges[0].start])} with different whitespace (${tolerant.strategy}); copy that whitespace exactly`, + ); + } else if (text.includes('\r\n') && !oldText.includes('\r\n') && oldText.includes('\n')) { + hints.push('the file uses CRLF line endings'); + } + } + const nearest = nearestLine(nonBlank[0], lines); + if (nearest != null && hints.length === 0) { + hints.push( + nearest.exact + ? `its first line appears at ${formatLineList(text, nearest.starts, nearest.count)}, but the lines after it differ` + : `the closest line is ${formatLineList(text, nearest.starts)}: ${snippet(nearest.text)}`, + ); + } + return `old_text was not found${hints.length > 0 ? `; ${hints.join('; ')}` : ''}`; +} + +/** Finds where the first line of a failed edit most likely belongs. */ +function nearestLine( + firstLine: string | undefined, + getLines: () => LineIndex, +): { exact: boolean; count: number; starts: number[]; text: string } | undefined { + const target = firstLine?.trim(); + if (!target) return undefined; + const lines = getLines(); + const starts: number[] = []; + let count = 0; + let firstMatch = ''; + for (let index = 0; index < lines.length; index++) { + const content = lines.text(index); + if (content.trim() !== target) continue; + if (count++ === 0) firstMatch = content; + if (starts.length < MAX_REPORTED_LINES) starts.push(lines.start(index)); + } + if (count > 0) { + return { exact: true, count, starts, text: firstMatch }; + } + const tokens = new Set(target.split(/\W+/).filter((token) => token.length > 1)); + if (tokens.size < 2) return undefined; + let best: Line | undefined; + let bestScore = 0; + for (let index = 0; index < lines.length; index++) { + const content = lines.text(index); + let score = 0; + for (const token of new Set(content.split(/\W+/))) { + if (tokens.has(token)) score++; + } + if (score > bestScore) { + best = { start: lines.start(index), end: lines.end(index), next: lines.next(index), text: content }; + bestScore = score; + } + } + return best != null && bestScore / tokens.size >= 0.5 + ? { exact: false, count: 1, starts: [best.start], text: best.text } + : undefined; +} + +function formatEditFailures(failures: readonly EditFailure[], editCount: number): string { + if (editCount === 1) { + return `Workspace edit did not apply and nothing was written: ${failures[0]?.reason ?? 'no match'}.`.slice( + 0, + EDIT_DIAGNOSTIC_MAX_CHARS, + ); + } + const header = `${failures.length} of ${editCount} workspace edits did not apply, so nothing was written. Every other edit matched.`; + const footer = failures.some((failure) => failure.index > 0) + ? '\nLine numbers account for the earlier edits in this batch.' + : ''; + const prefixes = failures.map((failure) => `\nEdit ${failure.index + 1}: `); + // Labels and punctuation for every failing position take priority over long + // reasons or source excerpts. Never leave callers guessing which edits failed. + const available = EDIT_DIAGNOSTIC_MAX_CHARS - header.length - footer.length - + prefixes.reduce((total, prefix) => total + prefix.length + 1, 0); + const reasonLength = failures.reduce((total, failure) => total + failure.reason.length, 0); + const reasonLimit = reasonLength <= available ? Infinity : Math.floor(available / failures.length); + const details = failures.map((failure, index) => { + let reason = failure.reason; + if (reason.length > reasonLimit) { + let end = reasonLimit - 1; + // Do not split a surrogate pair in a shortened source excerpt. + if (end > 0 && /[\uD800-\uDBFF]/.test(reason[end - 1]) && /[\uDC00-\uDFFF]/.test(reason[end])) end--; + reason = `${reason.slice(0, end)}…`; + } + return `${prefixes[index]}${reason}.`; + }); + return header + details.join('') + footer; +} diff --git a/packages/code/src/protocol.test.ts b/packages/code/src/protocol.test.ts index 729b20e2..340ddfdf 100644 --- a/packages/code/src/protocol.test.ts +++ b/packages/code/src/protocol.test.ts @@ -545,6 +545,164 @@ test('workspace mutations accept bounded UTF-8 requests and exact result shapes' ); }); +test('edit matching and replaceAll are opt-in and reported only when requested', () => { + const tolerant = { + protocolVersion: 1 as const, + operation: 'edit_file' as const, + workspaceId: 'primary', + path: 'notes.txt', + matching: 'tolerant' as const, + edits: [ + { oldText: 'hello', newText: 'goodbye' }, + { oldText: 'world', newText: 'BYOM', replaceAll: true }, + ], + }; + const result = { + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + replacements: 2, + bytesWritten: 12, + }; + const matches = [ + { strategy: 'line-trimmed', occurrences: 1 }, + { strategy: 'exact', occurrences: 3 }, + ]; + assert.equal(isWorkspaceToolRequest(tolerant), true); + assert.equal(isWorkspaceToolRequest({ ...tolerant, operation: 'preview_edit' }), true); + assert.equal(isWorkspaceToolRequest({ ...tolerant, matching: 'fuzzy' }), false); + assert.equal( + isWorkspaceToolRequest({ + ...tolerant, + edits: [{ oldText: 'a', newText: 'b', replaceAll: 'yes' }], + }), + false, + ); + assert.equal( + isWorkspaceToolRequest({ + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + oldText: 'a', + newText: 'b', + replaceAll: true, + }), + false, + 'replaceAll is only accepted per batch edit', + ); + + assert.equal(isWorkspaceToolResult(tolerant, { ...result, matches }), true); + assert.equal(isWorkspaceToolResult(tolerant, result), false, 'an opted-in request must report matches'); + assert.equal( + isWorkspaceToolResult(tolerant, { ...result, matches: matches.slice(0, 1) }), + false, + ); + assert.equal( + isWorkspaceToolResult(tolerant, { + ...result, + matches: [matches[0], { strategy: 'exact', occurrences: 0 }], + }), + false, + ); + assert.equal( + isWorkspaceToolResult(tolerant, { + ...result, + matches: [{ strategy: 'exact', occurrences: 2 }, matches[1]], + }), + false, + 'only replaceAll edits may replace more than one location', + ); + + const exactReplaceAll = { + ...tolerant, + matching: undefined, + edits: [{ oldText: 'a', newText: 'b', replaceAll: true }], + }; + delete exactReplaceAll.matching; + assert.equal( + isWorkspaceToolResult(exactReplaceAll, { + ...result, + replacements: 1, + matches: [{ strategy: 'line-trimmed', occurrences: 2 }], + }), + false, + 'without tolerant matching every edit matches exactly', + ); + + const legacy = { ...tolerant, edits: [{ oldText: 'a', newText: 'b' }] }; + delete (legacy as { matching?: string }).matching; + assert.equal( + isWorkspaceToolResult(legacy, { + ...result, + replacements: 1, + matches: [{ strategy: 'exact', occurrences: 1 }], + }), + false, + 'legacy requests never receive a matches field', + ); +}); + +test('edit features advertise any unique subset of the known features', () => { + const valid = { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: 1, + operations: ['read_file', 'edit_file'], + workspaces: [{ id: 'primary' }], + editFileModes: ['single', 'batch'], + }, + }; + const withFeatures = (editFileFeatures: unknown) => ({ + ...valid, + workspaceTools: { ...valid.workspaceTools, editFileFeatures }, + }); + for (const features of [ + ['expected_base_sha256'], + ['tolerant_match'], + ['expected_base_sha256', 'tolerant_match', 'replace_all'], + ]) { + assert.equal(isValidBridgeWorkerCapabilities(withFeatures(features)), true, features.join(',')); + } + for (const features of [[], ['fuzzy'], ['replace_all', 'replace_all']]) { + assert.equal(isValidBridgeWorkerCapabilities(withFeatures(features)), false, features.join(',')); + } +}); + +test('preview-only workers may advertise tolerant matching and replace-all, but not edit-only hashes', () => { + const previewOnly = { + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + workspaceTools: { + protocolVersion: 1, + operations: ['read_file', 'preview_edit'], + workspaces: [{ id: 'primary' }], + editFileModes: ['single', 'batch'], + }, + }; + for (const features of [ + ['tolerant_match'], + ['replace_all'], + ['tolerant_match', 'replace_all'], + ]) { + assert.equal(isValidBridgeWorkerCapabilities({ + ...previewOnly, + workspaceTools: { ...previewOnly.workspaceTools, editFileFeatures: features }, + }), true, features.join(',')); + } + assert.equal(isValidBridgeWorkerCapabilities({ + ...previewOnly, + workspaceTools: { + ...previewOnly.workspaceTools, + editFileFeatures: ['expected_base_sha256', 'tolerant_match'], + }, + }), false); +}); + test('workspace commands require bounded sandbox inputs and outputs', () => { const request = { protocolVersion: 1 as const, diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index b23ad327..26111273 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -317,7 +317,29 @@ export type BridgeWorkspaceToolOperation = export type WorkspaceWriteFileMode = 'replace' | 'create'; export type WorkspaceEditFileMode = 'single' | 'batch'; -export type WorkspaceEditFileFeature = 'expected_base_sha256'; +export type WorkspaceEditFileFeature = + | 'expected_base_sha256' + | 'tolerant_match' + | 'replace_all'; +/** Every edit feature this protocol version defines, for capability validation. */ +export const WORKSPACE_EDIT_FILE_FEATURES: readonly WorkspaceEditFileFeature[] = [ + 'expected_base_sha256', + 'tolerant_match', + 'replace_all', +]; +/** `tolerant` falls back from exact matching to whitespace-tolerant strategies. */ +export type WorkspaceEditMatching = 'exact' | 'tolerant'; +export type WorkspaceEditMatchStrategy = + | 'exact' + | 'line-trimmed' + | 'whitespace-normalized' + | 'indentation-flexible'; +const WORKSPACE_EDIT_MATCH_STRATEGIES = new Set([ + 'exact', + 'line-trimmed', + 'whitespace-normalized', + 'indentation-flexible', +]); export type WorkspaceListFileFeature = 'after_path'; export type WorkspaceProgrammaticLanguage = 'bash'; @@ -483,6 +505,8 @@ interface WorkspaceEditFileRequestBase { path: string; /** Refuses the mutation unless current file bytes match this preview revision. */ expectedBaseSha256?: string; + /** Requires the `tolerant_match` edit feature. Omitted means `exact`. */ + matching?: WorkspaceEditMatching; } export interface WorkspaceSingleEditFileRequest @@ -509,6 +533,15 @@ export type WorkspaceEditFileRequest = export interface WorkspaceTextEdit { oldText: string; newText: string; + /** Replaces every match instead of requiring exactly one. Requires `replace_all`. */ + replaceAll?: boolean; +} + +/** How one edit matched. Present only when the request set `matching` or `replaceAll`. */ +export interface WorkspaceEditMatch { + strategy: WorkspaceEditMatchStrategy; + /** Locations replaced; always 1 unless the edit set `replaceAll`. */ + occurrences: number; } export interface WorkspaceEditFileResult { @@ -516,8 +549,10 @@ export interface WorkspaceEditFileResult { operation: 'edit_file'; workspaceId: string; path: string; + /** Number of edits applied, which is always the number requested. */ replacements: number; bytesWritten: number; + matches?: WorkspaceEditMatch[]; } interface WorkspacePreviewEditRequestBase { @@ -528,6 +563,8 @@ interface WorkspacePreviewEditRequestBase { /** Linked worktree lane at `.worktrees/`; paths and cwd are relative to it. */ worktree?: string; path: string; + /** Requires the `tolerant_match` edit feature. Omitted means `exact`. */ + matching?: WorkspaceEditMatching; } export interface WorkspaceSinglePreviewEditRequest @@ -558,6 +595,7 @@ export interface WorkspacePreviewEditResult { baseSha256: string; replacements: number; bytesWritten: number; + matches?: WorkspaceEditMatch[]; } export interface WorkspaceExecuteCommandRequest { @@ -658,6 +696,7 @@ const WORKSPACE_EDIT_REQUEST_KEYS = new Set([ 'newText', 'edits', 'expectedBaseSha256', + 'matching', ]); const WORKSPACE_PREVIEW_EDIT_REQUEST_KEYS = new Set([ 'protocolVersion', @@ -669,8 +708,10 @@ const WORKSPACE_PREVIEW_EDIT_REQUEST_KEYS = new Set([ 'oldText', 'newText', 'edits', + 'matching', ]); -const WORKSPACE_TEXT_EDIT_KEYS = new Set(['oldText', 'newText']); +const WORKSPACE_TEXT_EDIT_KEYS = new Set(['oldText', 'newText', 'replaceAll']); +const WORKSPACE_EDIT_MATCH_KEYS = new Set(['strategy', 'occurrences']); const WORKSPACE_COMMAND_REQUEST_KEYS = new Set([ 'environmentAction', 'protocolVersion', @@ -724,6 +765,7 @@ const WORKSPACE_EDIT_RESULT_KEYS = new Set([ 'path', 'replacements', 'bytesWritten', + 'matches', ]); const WORKSPACE_PREVIEW_EDIT_RESULT_KEYS = new Set([ 'protocolVersion', @@ -735,6 +777,7 @@ const WORKSPACE_PREVIEW_EDIT_RESULT_KEYS = new Set([ 'baseSha256', 'replacements', 'bytesWritten', + 'matches', ]); const WORKSPACE_COMMAND_RESULT_KEYS = new Set([ 'protocolVersion', @@ -1212,6 +1255,44 @@ function isWithinRequestedPath(candidate: string, requested?: string): boolean { ); } +/** Whether a request asked for per-edit match reporting (and so must receive it). */ +export function workspaceEditRequestReportsMatches( + request: WorkspaceEditFileRequest | WorkspacePreviewEditRequest, +): boolean { + return ( + request.matching !== undefined || + (request.edits?.some((edit) => edit.replaceAll !== undefined) ?? false) + ); +} + +function isValidWorkspaceEditMatches( + request: WorkspaceEditFileRequest | WorkspacePreviewEditRequest, + matches: unknown, +): boolean { + if (!workspaceEditRequestReportsMatches(request)) return matches === undefined; + const edits: WorkspaceTextEdit[] = request.edits ?? [ + { oldText: request.oldText ?? '', newText: request.newText ?? '' }, + ]; + return ( + Array.isArray(matches) && + matches.length === edits.length && + matches.every((match: unknown, index) => { + if (typeof match !== 'object' || match === null) return false; + const candidate = match as Record; + return ( + hasOnlyKeys(candidate, WORKSPACE_EDIT_MATCH_KEYS) && + WORKSPACE_EDIT_MATCH_STRATEGIES.has( + candidate.strategy as WorkspaceEditMatchStrategy, + ) && + (request.matching === 'tolerant' || candidate.strategy === 'exact') && + Number.isSafeInteger(candidate.occurrences) && + Number(candidate.occurrences) >= 1 && + (edits[index]?.replaceAll === true || candidate.occurrences === 1) + ); + }) + ); +} + function isValidWorkspaceEditRequest( request: Record, ): boolean { @@ -1222,6 +1303,13 @@ function isValidWorkspaceEditRequest( ) { return false; } + if ( + request.matching !== undefined && + request.matching !== 'exact' && + request.matching !== 'tolerant' + ) { + return false; + } const edits = hasBatch ? request.edits : [{ oldText: request.oldText, newText: request.newText }]; @@ -1252,7 +1340,9 @@ function isValidWorkspaceEditRequest( candidate.oldText || typeof candidate.newText !== 'string' || Buffer.from(candidate.newText).toString('utf8') !== - candidate.newText + candidate.newText || + (candidate.replaceAll !== undefined && + typeof candidate.replaceAll !== 'boolean') ) { return false; } @@ -1561,7 +1651,8 @@ export function isWorkspaceToolResult( result.replacements === replacements && Number.isSafeInteger(result.bytesWritten) && Number(result.bytesWritten) >= 0 && - Number(result.bytesWritten) <= BRIDGE_WORKSPACE_WRITE_MAX_BYTES + Number(result.bytesWritten) <= BRIDGE_WORKSPACE_WRITE_MAX_BYTES && + isValidWorkspaceEditMatches(request, result.matches) ); } @@ -1582,7 +1673,8 @@ export function isWorkspaceToolResult( Number(result.bytesWritten) === new TextEncoder().encode(content).byteLength + (result.hasUtf8Bom ? 3 : 0) && - Number(result.bytesWritten) <= BRIDGE_WORKSPACE_WRITE_MAX_BYTES + Number(result.bytesWritten) <= BRIDGE_WORKSPACE_WRITE_MAX_BYTES && + isValidWorkspaceEditMatches(request, result.matches) ); } @@ -1706,9 +1798,20 @@ export function isValidBridgeWorkspaceToolCapabilities( if ( capabilities.editFileFeatures !== undefined && (!Array.isArray(capabilities.editFileFeatures) || - capabilities.editFileFeatures.length !== 1 || - !capabilities.operations.includes('edit_file') || - capabilities.editFileFeatures[0] !== 'expected_base_sha256') + capabilities.editFileFeatures.length < 1 || + capabilities.editFileFeatures.length > + WORKSPACE_EDIT_FILE_FEATURES.length || + (!capabilities.operations.includes('edit_file') && + !capabilities.operations.includes('preview_edit')) || + !capabilities.editFileFeatures.every((feature: unknown) => + WORKSPACE_EDIT_FILE_FEATURES.includes( + feature as WorkspaceEditFileFeature, + ) && + (feature !== 'expected_base_sha256' || + (capabilities.operations as string[]).includes('edit_file')), + ) || + new Set(capabilities.editFileFeatures).size !== + capabilities.editFileFeatures.length) ) { return false; } diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 37223b34..217ca2c7 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -173,6 +173,19 @@ function errorCode(value: object): string | undefined { return undefined; } +/** Edit conflicts can include source excerpts, so disclosure needs read or full-preview access. */ +function workspaceCanReadSource( + capabilities: BridgeWorkerCapabilities['workspaceTools'], + workspaceId: string, +): boolean { + const workspace = capabilities?.workspaces.find((entry) => entry.id === workspaceId); + if (!workspace || !capabilities) return false; + return (['read_file', 'preview_edit'] as const).some( + (operation) => capabilities.operations.includes(operation) && + (workspace.operations == null || workspace.operations.includes(operation)), + ); +} + function workspaceCapabilitiesMatch( advertised: NonNullable, executor: NonNullable, @@ -388,7 +401,8 @@ function supportedWorkspaceCapabilities( }); if (workspaces.length === 0) return undefined; const editFileFeatures = desired.editFileFeatures?.filter((feature) => - registration.supportedWorkspaceEditFileFeatures?.includes(feature), + registration.supportedWorkspaceEditFileFeatures?.includes(feature) && + (feature !== 'expected_base_sha256' || operations.includes('edit_file')), ); const listFileFeatures = desired.listFileFeatures?.filter((feature) => registration.supportedWorkspaceListFileFeatures?.includes(feature), @@ -417,7 +431,7 @@ function supportedWorkspaceCapabilities( ...(supportsEditRequests && editFileModes?.length ? { editFileModes } : {}), - ...(operations.includes('edit_file') && editFileFeatures?.length + ...(supportsEditRequests && editFileFeatures?.length ? { editFileFeatures } : {}), ...(operations.includes('list_files') && listFileFeatures?.length @@ -1785,9 +1799,13 @@ export class BridgeWorker { ); } if ( - workspaceRequest.operation === 'edit_file' && - workspaceRequest.expectedBaseSha256 !== undefined && - !advertised.editFileFeatures?.includes('expected_base_sha256') + (workspaceRequest.operation === 'edit_file' && + workspaceRequest.expectedBaseSha256 !== undefined && + !advertised.editFileFeatures?.includes('expected_base_sha256')) || + (workspaceRequest.matching !== undefined && + !advertised.editFileFeatures?.includes('tolerant_match')) || + (workspaceRequest.edits?.some((edit) => edit.replaceAll !== undefined) === true && + !advertised.editFileFeatures?.includes('replace_all')) ) { throw new BridgeProtocolError( 'Workspace edit feature is not advertised', @@ -2069,9 +2087,14 @@ export class BridgeWorker { error instanceof WorkspaceToolError ? { errorCode: error.code } : {}), - error: (error instanceof Error - ? error.message - : 'Sandbox execution failed' + error: (assignment.executionKind === 'workspace_tool' && + isWorkspaceToolRequest(assignment.request) && + assignment.request.operation === 'edit_file' && + !workspaceCanReadSource(this.activeCapabilities.workspaceTools, assignment.request.workspaceId) + ? 'Workspace edit failed; source diagnostics require read access' + : error instanceof Error + ? error.message + : 'Sandbox execution failed' ).slice(0, MAX_SETTLEMENT_ERROR_LENGTH), }; } diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index 96a5d1f0..71ef2cd3 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { BridgeProtocolError } from './protocol.js'; +import type { WorkspaceToolRequest } from './protocol.js'; import { BridgeWorker, BridgeWorkspaceQuarantinedError } from './worker.js'; import { SandboxWorkspaceTools, WorkspaceToolError } from './workspace.js'; @@ -659,6 +660,49 @@ test('worker advertises only edit modes and features negotiated by Code API', as ]); }); +test('worker negotiates preview-only tolerant features and drops edit-only hashes', async () => { + const registrations: Array> = []; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['read_file' as const, 'preview_edit' as const, 'edit_file' as const], + workspaces: [{ id: 'primary' }], + editFileModes: ['single' as const, 'batch' as const], + editFileFeatures: ['expected_base_sha256' as const, 'tolerant_match' as const, 'replace_all' as const], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'native-srt', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute() { throw new Error('must not execute'); }, + }, + workspaceMutationQuarantine: mutationQuarantine(), + fetchImpl: async (_input, init) => { + const capabilities = JSON.parse(String(init?.body)).capabilities.workspaceTools; + registrations.push(capabilities); + return Response.json({ + protocolVersion: 1, workerId: 'vm-1', incarnationId, + registeredAt: new Date().toISOString(), leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: ['read_file', 'preview_edit'], + supportedWorkspaceEditFileModes: ['single', 'batch'], + supportedWorkspaceEditFileFeatures: ['tolerant_match', 'replace_all', 'expected_base_sha256'], + }); + }, + }); + await worker.register(); + assert.deepEqual(registrations.at(-1)?.operations, ['read_file', 'preview_edit']); + assert.deepEqual(registrations.at(-1)?.editFileFeatures, ['tolerant_match', 'replace_all']); +}); + test('worker drops file operations when no request mode is compatible', async () => { const registrations: Array> = []; const workspaceCapabilities = { @@ -3165,6 +3209,98 @@ test('worker rejects workspace operations outside its advertised capability', as assert.match(String(settlement?.error), /operation is not advertised/i); }); +for (const [operation, extras] of [ + ['preview_edit', { matching: 'tolerant', oldText: 'a', newText: 'b' }], + ['preview_edit', { matching: 'exact', oldText: 'a', newText: 'b' }], + ['preview_edit', { edits: [{ oldText: 'a', newText: 'b', replaceAll: false }] }], + ['edit_file', { edits: [{ oldText: 'a', newText: 'b', replaceAll: true }] }], +] as const) { + test(`worker refuses unnegotiated ${operation} feature before dispatch`, async () => { + let executions = 0; + let armed = 0; + let settlement: Record | undefined; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['preview_edit' as const, 'edit_file' as const], + editFileModes: ['single' as const, 'batch' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', + incarnationId, sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: true, sandboxProfile: 'native-srt', runtimes: ['bash'], + workspaceTools: workspaceCapabilities }, + workspaceTools: { capabilities: workspaceCapabilities, async execute() { + executions++; + throw new Error('must not execute'); + } }, + workspaceMutationQuarantine: mutationQuarantine(undefined, () => armed++), + fetchImpl: async (_input, init) => { + settlement = JSON.parse(String(init?.body)) as Record; + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: `assignment-${operation}-${String('matching' in extras ? extras.matching : 'replace')}`, + workerId: 'vm-1', incarnationId, generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { protocolVersion: 1, operation, workspaceId: 'primary', path: 'notes.txt', ...extras } as WorkspaceToolRequest, + }); + assert.equal(executions, 0); + assert.equal(armed, 0); + assert.equal(settlement?.status, 'rejected'); + assert.match(String(settlement?.error), /edit feature is not advertised/i); + }); +} + +for (const [label, operations, allowed, mayRead] of [ + ['edit-only worker', ['edit_file'], ['edit_file'], false], + ['read denied by workspace', ['read_file', 'edit_file'], ['edit_file'], false], + ['read permitted by workspace', ['read_file', 'edit_file'], ['read_file', 'edit_file'], true], + ['preview permits full file reads', ['preview_edit', 'edit_file'], ['preview_edit', 'edit_file'], true], +] as const) { + test(`edit diagnostics respect ${label} authorization`, async () => { + const source = 'confidential-source-line'; + let settlement: Record | undefined; + const capabilities = { + protocolVersion: 1 as const, + operations: [...operations], + editFileModes: ['single' as const], + workspaces: [{ id: 'primary', operations: [...allowed] }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', + incarnationId, sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', runtimes: ['bash'], + workspaceTools: capabilities }, + workspaceTools: { capabilities, mutationFailuresAreAtomic: true, async execute() { + throw new WorkspaceToolError(`old_text was not found; closest line: "${source}"`, 'EDIT_CONFLICT'); + } }, + workspaceMutationQuarantine: mutationQuarantine(), + fetchImpl: async (_url, init) => { + settlement = JSON.parse(String(init?.body)) as Record; + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'edit-source-authorization', + workerId: 'vm-1', incarnationId, generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { protocolVersion: 1, operation: 'edit_file', workspaceId: 'primary', + path: 'notes.txt', oldText: 'missing', newText: 'replacement' }, + }); + assert.equal(settlement?.status, 'rejected'); + assert.equal(settlement?.errorCode, 'EDIT_CONFLICT'); + assert.equal(String(settlement?.error).includes(source), mayRead); + }); +} + test('worker rejects legacy replacement writes outside its advertised mode', async () => { let executions = 0; let settlement: Record | undefined; diff --git a/packages/code/src/workspace.test.ts b/packages/code/src/workspace.test.ts index 3dd7d2f4..4a0ecb3e 100644 --- a/packages/code/src/workspace.test.ts +++ b/packages/code/src/workspace.test.ts @@ -939,7 +939,7 @@ test('writable workspaces create, replace, and exactly edit files', async (t) => ], writeFileModes: ['replace', 'create'], editFileModes: ['single', 'batch'], - editFileFeatures: ['expected_base_sha256'], + editFileFeatures: ['expected_base_sha256', 'tolerant_match', 'replace_all'], listFileFeatures: ['after_path'], }); await tools.execute({ @@ -1351,6 +1351,392 @@ test('exact edits reject missing or repeated text without changing the file', as assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), 'aaa'); }); +test('a failed edit batch reports every failing edit and writes nothing', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = 'alpha\nbeta\nbeta\ngamma\n'; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + edits: [ + { oldText: 'alpha', newText: 'ALPHA' }, + { oldText: 'beta', newText: 'BETA' }, + { oldText: 'delta', newText: 'DELTA' }, + ], + }), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'EDIT_CONFLICT' && + /^2 of 3 workspace edits did not apply/.test(error.message) && + /Edit 2: old_text matched 2 locations at lines 2, 3/.test(error.message) && + /Edit 3: old_text was not found/.test(error.message), + ); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); +}); + +test('long preview and edit batch errors expose every failing edit position without writing', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-batch-positions-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = `KEEP\nconst shared = '${'x'.repeat(180)}';\n`; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ workspaces: [{ id: 'primary', root, writable: true }] }); + const edits = Array.from({ length: 60 }, (_, index) => index % 2 === 0 + ? { oldText: `const shared missing_${index}`, newText: 'replacement' } + : { oldText: 'KEEP', newText: 'KEEP' }); + for (const operation of ['preview_edit', 'edit_file'] as const) { + await assert.rejects(tools.execute({ + protocolVersion: 1, operation, workspaceId: 'primary', path: 'notes.txt', edits, + }), (error: unknown) => { + assert.ok(error instanceof WorkspaceToolError); + assert.equal(error.code, 'EDIT_CONFLICT'); + assert.equal(error.mutationMayHaveCommitted, false); + assert.ok(error.message.length <= 3_000); + assert.deepEqual( + [...error.message.matchAll(/\nEdit (\d+):/g)].map((match) => Number(match[1])), + Array.from({ length: 30 }, (_, index) => 2 * index + 1), + ); + return true; + }); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + } +}); + +test('tolerant edits and replaceAll report how each edit matched', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile(join(root, 'app.ts'), 'if (ok) { \r\n run();\r\n}\r\nrun();\r\n'); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + + const preview = await tools.execute({ + protocolVersion: 1, + operation: 'preview_edit', + workspaceId: 'primary', + path: 'app.ts', + matching: 'tolerant', + edits: [{ oldText: 'if (ok) {\n run();\n}', newText: 'if (ok) {\n go();\n}' }], + }); + assert.equal(preview.operation, 'preview_edit'); + assert.deepEqual(preview.operation === 'preview_edit' && preview.matches, [ + { strategy: 'line-trimmed', occurrences: 1 }, + ]); + + const edit = await tools.execute({ + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'app.ts', + matching: 'tolerant', + edits: [ + { oldText: 'if (ok) {\n run();\n}', newText: 'if (ok) {\n go();\n}' }, + { oldText: 'run();', newText: 'stop();', replaceAll: true }, + ], + }); + assert.deepEqual(edit, { + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'app.ts', + replacements: 2, + bytesWritten: 32, + matches: [ + { strategy: 'line-trimmed', occurrences: 1 }, + { strategy: 'exact', occurrences: 1 }, + ], + }); + assert.equal( + await readFile(join(root, 'app.ts'), 'utf8'), + 'if (ok) {\r\n go();\r\n}\r\nstop();\r\n', + ); +}); + +test('preview and edit peel equivalent newline wrappers without doubling line breaks or losing BOM', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-newlines-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = '\uFEFFheader\r\n \tfoo bar\r\nnext\r\n'; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + const request = { + protocolVersion: 1 as const, + operation: 'preview_edit' as const, + workspaceId: 'primary', + path: 'notes.txt', + matching: 'tolerant' as const, + edits: [ + { oldText: '\n foo bar\r\n', newText: '\r\n baz qux\n' }, + { oldText: 'next', newText: 'done' }, + ], + }; + const preview = await tools.execute(request); + assert.equal(preview.operation, 'preview_edit'); + if (preview.operation !== 'preview_edit') assert.fail('expected preview result'); + assert.equal(preview.content, 'header\r\n \tbaz qux\r\ndone\r\n'); + assert.equal(preview.hasUtf8Bom, true); + assert.equal(preview.bytesWritten, Buffer.byteLength(`\uFEFF${preview.content}`)); + assert.equal(isWorkspaceToolResult(request, preview), true); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + const editRequest = { + ...request, + operation: 'edit_file' as const, + expectedBaseSha256: preview.baseSha256, + }; + const edited = await tools.execute(editRequest); + assert.equal(edited.operation, 'edit_file'); + if (edited.operation !== 'edit_file') assert.fail('expected edit result'); + assert.deepEqual(edited.matches, [ + { strategy: 'whitespace-normalized', occurrences: 1 }, + { strategy: 'exact', occurrences: 1 }, + ]); + assert.equal(isWorkspaceToolResult(editRequest, edited), true); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), `\uFEFF${preview.content}`); +}); + +test('tolerant previews and edits keep source indentation when newText uses a different indent', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-indent-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = 'header\n foo bar\n'; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + const edit = { + protocolVersion: 1 as const, workspaceId: 'primary', path: 'notes.txt', + matching: 'tolerant' as const, oldText: '\n foo bar', newText: '\n\tbaz qux', + }; + const preview = await tools.execute({ ...edit, operation: 'preview_edit' }); + assert.equal(preview.operation === 'preview_edit' && preview.content, 'header\n baz qux\n'); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + const result = await tools.execute({ ...edit, operation: 'edit_file' }); + assert.equal(result.operation === 'edit_file' && result.matches?.[0]?.strategy, 'whitespace-normalized'); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), 'header\n baz qux\n'); +}); + +test('tolerant previews and edits retain an extra leading blank line across a fenced batch', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-extra-line-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = '\uFEFFheader\r\n foo bar\r\nend\r\n'; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + const request = { + protocolVersion: 1 as const, workspaceId: 'primary', path: 'notes.txt', + matching: 'tolerant' as const, + edits: [ + { oldText: '\n foo bar', newText: '\r\n\t\r\n\tbaz qux' }, + { oldText: 'end', newText: 'done' }, + ], + }; + const previewRequest = { ...request, operation: 'preview_edit' as const }; + const preview = await tools.execute(previewRequest); + if (preview.operation !== 'preview_edit') assert.fail('expected preview result'); + assert.equal(preview.content, 'header\r\n \r\n\tbaz qux\r\ndone\r\n'); + assert.equal(preview.hasUtf8Bom, true); + assert.equal(isWorkspaceToolResult(previewRequest, preview), true); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + const editRequest = { ...request, operation: 'edit_file' as const, + expectedBaseSha256: preview.baseSha256 }; + const edited = await tools.execute(editRequest); + if (edited.operation !== 'edit_file') assert.fail('expected edit result'); + assert.deepEqual(edited.matches, [ + { strategy: 'whitespace-normalized', occurrences: 1 }, + { strategy: 'exact', occurrences: 1 }, + ]); + assert.equal(isWorkspaceToolResult(editRequest, edited), true); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), `\uFEFF${preview.content}`); +}); + +test('exact preview and edit failures retain whitespace hints without writing the file', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-probe-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = '\uFEFFfoo bar\r\n'; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ workspaces: [{ id: 'primary', root, writable: true }] }); + for (const operation of ['preview_edit', 'edit_file'] as const) { + await assert.rejects(tools.execute({ + protocolVersion: 1, operation, workspaceId: 'primary', path: 'notes.txt', + oldText: 'foo bar\n', newText: '', + }), (error: unknown) => error instanceof WorkspaceToolError && error.code === 'EDIT_CONFLICT' && + /different whitespace \(whitespace-normalized\)/.test(error.message) && !error.mutationMayHaveCommitted); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + } +}); + +test('tolerant preview and edit peel newline-only wrappers and preserve source indentation', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-newline-only-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = '\uFEFFheader\r\n foo bar\r\nnext\r\n'; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ workspaces: [{ id: 'primary', root, writable: true }] }); + const request = { protocolVersion: 1 as const, workspaceId: 'primary', path: 'notes.txt', + matching: 'tolerant' as const, oldText: '\nfoo bar\n', newText: '\n\tbaz qux\t\n' }; + const previewRequest = { ...request, operation: 'preview_edit' as const }; + const preview = await tools.execute(previewRequest); + if (preview.operation !== 'preview_edit') assert.fail('expected preview result'); + assert.equal(preview.content, 'header\r\n baz qux\r\nnext\r\n'); + assert.equal(preview.hasUtf8Bom, true); + assert.equal(isWorkspaceToolResult(previewRequest, preview), true); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + const editRequest = { ...request, operation: 'edit_file' as const, expectedBaseSha256: preview.baseSha256 }; + const edited = await tools.execute(editRequest); + assert.equal(isWorkspaceToolResult(editRequest, edited), true); + assert.deepEqual(edited.operation === 'edit_file' && edited.matches, + [{ strategy: 'whitespace-normalized', occurrences: 1 }]); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), `\uFEFF${preview.content}`); +}); + +test('preview and edit verify a uniquely anchored indentation block larger than the repeat budget', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-large-indent-')); + t.after(() => rm(root, { recursive: true, force: true })); + const oldText = ['anchor', ...Array.from({ length: 99_999 }, () => 'x'), 'end'].join('\n'); + const original = ` ${oldText.replaceAll('\n', '\n ')}\n`; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ workspaces: [{ id: 'primary', root, writable: true }] }); + const request = { protocolVersion: 1 as const, workspaceId: 'primary', path: 'notes.txt', + matching: 'tolerant' as const, oldText, newText: 'changed' }; + const previewRequest = { ...request, operation: 'preview_edit' as const }; + const preview = await tools.execute(previewRequest); + if (preview.operation !== 'preview_edit') assert.fail('expected preview result'); + assert.equal(preview.content, ' changed\n'); + assert.equal(isWorkspaceToolResult(previewRequest, preview), true); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + const editRequest = { ...request, operation: 'edit_file' as const, expectedBaseSha256: preview.baseSha256 }; + const edited = await tools.execute(editRequest); + assert.equal(isWorkspaceToolResult(editRequest, edited), true); + assert.deepEqual(edited.operation === 'edit_file' && edited.matches, + [{ strategy: 'indentation-flexible', occurrences: 1 }]); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), preview.content); +}); + +test('tolerant previews and edits reject missing source boundary newlines without changing the file', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-missing-newline-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = 'foo bar'; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + for (const operation of ['preview_edit', 'edit_file'] as const) { + await assert.rejects(tools.execute({ + protocolVersion: 1, operation, workspaceId: 'primary', path: 'notes.txt', + matching: 'tolerant', oldText: 'foo bar\r\n', newText: 'baz qux\n', + }), (error: unknown) => error instanceof WorkspaceToolError && error.code === 'EDIT_CONFLICT'); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + } +}); + +test('tolerant previews and edits reject boundary removal outside the token match', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-remove-newline-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = 'foo bar\r\nnext'; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + for (const operation of ['preview_edit', 'edit_file'] as const) { + await assert.rejects(tools.execute({ + protocolVersion: 1, operation, workspaceId: 'primary', path: 'notes.txt', + matching: 'tolerant', oldText: 'foo bar\r\n', newText: 'baz qux', + }), (error: unknown) => error instanceof WorkspaceToolError && error.code === 'EDIT_CONFLICT'); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + } +}); + +test('tolerant previews and edits refuse partial tokens without changing the file', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-boundary-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = 'prereturn value;\n'; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + for (const operation of ['preview_edit', 'edit_file'] as const) { + await assert.rejects(tools.execute({ + protocolVersion: 1, operation, workspaceId: 'primary', path: 'notes.txt', + matching: 'tolerant', oldText: 'return value;', newText: 'return changed;', + }), (error: unknown) => error instanceof WorkspaceToolError && error.code === 'EDIT_CONFLICT'); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + } +}); + +test('large whitespace-only differences work for preview and edit_file', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-large-')); + t.after(() => rm(root, { recursive: true, force: true })); + const oldText = `head ${'part '.repeat(16_000)}tail`; + const original = `before ${oldText.replace(/ /g, '\t')} after`; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + const base = { protocolVersion: 1 as const, workspaceId: 'primary', path: 'notes.txt', + matching: 'tolerant' as const, oldText, newText: 'result' }; + const preview = await tools.execute({ ...base, operation: 'preview_edit' }); + assert.equal(preview.operation === 'preview_edit' && preview.content, 'before result after'); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + const edited = await tools.execute({ ...base, operation: 'edit_file' }); + assert.equal(edited.operation === 'edit_file' && edited.matches?.[0]?.strategy, 'whitespace-normalized'); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), 'before result after'); +}); + +test('large replaceAll previews and edits return bounded content and accurate match counts', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-stream-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = 'x'.repeat(256 * 1024); + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + const request = { protocolVersion: 1 as const, workspaceId: 'primary', path: 'notes.txt', + edits: [{ oldText: 'x', newText: 'y', replaceAll: true }] }; + const previewRequest = { ...request, operation: 'preview_edit' as const }; + const preview = await tools.execute(previewRequest); + if (preview.operation !== 'preview_edit') assert.fail('expected preview result'); + assert.equal(preview.content, 'y'.repeat(original.length)); + assert.equal(preview.bytesWritten, original.length); + assert.deepEqual(preview.matches, [{ strategy: 'exact', occurrences: original.length }]); + assert.equal(isWorkspaceToolResult(previewRequest, preview), true); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + const editRequest = { ...request, operation: 'edit_file' as const, + expectedBaseSha256: preview.baseSha256 }; + const edited = await tools.execute(editRequest); + assert.equal(edited.operation === 'edit_file' && edited.bytesWritten, original.length); + assert.deepEqual(edited.operation === 'edit_file' && edited.matches, preview.matches); + assert.equal(isWorkspaceToolResult(editRequest, edited), true); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), preview.content); +}); + +test('oversized replaceAll previews and edits fail before writing the source file', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-limit-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = 'x'.repeat(10_000); + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + for (const operation of ['preview_edit', 'edit_file'] as const) { + await assert.rejects(tools.execute({ + protocolVersion: 1, + operation, + workspaceId: 'primary', + path: 'notes.txt', + edits: [{ oldText: 'x', newText: 'y'.repeat(100_000), replaceAll: true }], + }), (error: unknown) => error instanceof WorkspaceToolError && + error.code === 'WRITE_LIMIT_EXCEEDED' && !error.mutationMayHaveCommitted); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + } +}); + test('writes reject symlink targets and missing parent directories', async (t) => { const parent = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); t.after(() => rm(parent, { recursive: true, force: true })); @@ -1930,6 +2316,8 @@ test('composes sandboxed commands without exposing them on unconfigured workspac assert.deepEqual(tools.capabilities.editFileModes, ['single', 'batch']); assert.deepEqual(tools.capabilities.editFileFeatures, [ 'expected_base_sha256', + 'tolerant_match', + 'replace_all', ]); assert.deepEqual(tools.capabilities.listFileFeatures, ['after_path']); assert.deepEqual( diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index 65757d39..0b9328d0 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -20,10 +20,19 @@ import { isValidBridgeWorkspaceToolCapabilities, isWorkspaceToolRequest, isWorkspaceToolResult, + WORKSPACE_EDIT_FILE_FEATURES, + workspaceEditRequestReportsMatches, } from './protocol.js'; +import { + applyTextEdits, + WorkspaceEditMatchError, + WorkspaceEditOutputLimitError, +} from './edits.js'; +import type { AppliedEdits } from './edits.js'; import type { BridgeWorkspaceDescriptor, + WorkspaceEditMatch, BridgeWorkspaceToolCapabilities, WorkspaceReadFileRequest, WorkspaceReadFileResult, @@ -722,7 +731,10 @@ async function editWorkspaceFile( 'EDIT_CONFLICT', ); } - const { updated, replacements } = applyWorkspaceEdits(original, request); + const { updated, replacements, matches } = applyWorkspaceEdits( + original, + request, + ); await atomicWriteConfinedFile( root, request.path, @@ -741,6 +753,7 @@ async function editWorkspaceFile( path: request.path, replacements, bytesWritten: updated.byteLength, + ...(matches ? { matches } : {}), }; } catch (error) { if (error instanceof WorkspaceToolError) throw error; @@ -753,7 +766,7 @@ async function editWorkspaceFile( function applyWorkspaceEdits( original: Buffer, request: WorkspaceEditFileRequest | WorkspacePreviewEditRequest, -): { updated: Buffer; replacements: number } { +): { updated: Buffer; replacements: number; matches?: WorkspaceEditMatch[] } { const hasBom = original[0] === 0xef && original[1] === 0xbb && original[2] === 0xbf; const body = hasBom ? original.subarray(3) : original; @@ -767,20 +780,19 @@ function applyWorkspaceEdits( const edits = request.edits ?? [ { oldText: request.oldText ?? '', newText: request.newText ?? '' }, ]; - let updatedText = text; - for (const edit of edits) { - const first = updatedText.indexOf(edit.oldText); - if (first < 0 || updatedText.indexOf(edit.oldText, first + 1) >= 0) { - throw new WorkspaceToolError( - 'Workspace edit must match exactly once', - 'EDIT_CONFLICT', - ); + let applied: AppliedEdits; + try { + applied = applyTextEdits(text, edits, request.matching ?? 'exact'); + } catch (error) { + if (error instanceof WorkspaceEditMatchError) { + throw new WorkspaceToolError(error.message, 'EDIT_CONFLICT'); + } + if (error instanceof WorkspaceEditOutputLimitError) { + throw new WorkspaceToolError(error.message, 'WRITE_LIMIT_EXCEEDED'); } - updatedText = - updatedText.slice(0, first) + - edit.newText + - updatedText.slice(first + edit.oldText.length); + throw error; } + const updatedText = applied.text; const updatedBody = Buffer.from(updatedText, 'utf8'); const updated = hasBom ? Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), updatedBody]) @@ -791,7 +803,13 @@ function applyWorkspaceEdits( 'WRITE_LIMIT_EXCEEDED', ); } - return { updated, replacements: edits.length }; + return { + updated, + replacements: edits.length, + ...(workspaceEditRequestReportsMatches(request) + ? { matches: applied.matches } + : {}), + }; } async function previewWorkspaceEdit( @@ -806,7 +824,10 @@ async function previewWorkspaceEdit( 'EXECUTION_ABORTED', ); } - const { updated, replacements } = applyWorkspaceEdits(original, request); + const { updated, replacements, matches } = applyWorkspaceEdits( + original, + request, + ); if (signal?.aborted) { throw new WorkspaceToolError( 'Workspace tool execution aborted', @@ -825,6 +846,7 @@ async function previewWorkspaceEdit( baseSha256: createHash('sha256').update(original).digest('hex'), replacements, bytesWritten: updated.byteLength, + ...(matches ? { matches } : {}), }; } @@ -1497,7 +1519,7 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { ...(anyWritable ? { writeFileModes: ['replace', 'create'] } : {}), ...(anyWritable ? { editFileModes: ['single', 'batch'] } : {}), ...(anyWritable - ? { editFileFeatures: ['expected_base_sha256'] } + ? { editFileFeatures: [...WORKSPACE_EDIT_FILE_FEATURES] } : {}), listFileFeatures: ['after_path'], }; diff --git a/service/src/bridge/router.test.ts b/service/src/bridge/router.test.ts index 63640111..28035847 100644 --- a/service/src/bridge/router.test.ts +++ b/service/src/bridge/router.test.ts @@ -432,7 +432,11 @@ describe('paired bridge HTTP API', () => { ], supportedWorkspaceWriteFileModes: ['replace', 'create'], supportedWorkspaceEditFileModes: ['single', 'batch'], - supportedWorkspaceEditFileFeatures: ['expected_base_sha256'], + supportedWorkspaceEditFileFeatures: [ + 'expected_base_sha256', + 'tolerant_match', + 'replace_all', + ], supportedWorkspaceListFileFeatures: ['after_path'], }); diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 962c6242..8fdc700e 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -12,6 +12,7 @@ import { isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, isWorkspaceToolErrorCode, + WORKSPACE_EDIT_FILE_FEATURES, } from '../../../packages/code/src/protocol'; import { BridgePairingError, RedisBridgePairingStore } from './pairing'; import { BridgeStoreError, RedisBridgeStore } from './store'; @@ -535,7 +536,7 @@ router.post( ], supportedWorkspaceWriteFileModes: ['replace', 'create'], supportedWorkspaceEditFileModes: ['single', 'batch'], - supportedWorkspaceEditFileFeatures: ['expected_base_sha256'], + supportedWorkspaceEditFileFeatures: [...WORKSPACE_EDIT_FILE_FEATURES], supportedWorkspaceListFileFeatures: ['after_path'], supportedWorkspaceProgrammaticLanguages: ['bash'], supportedWorkspaceInstanceTypes: ['git_worktree'], diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 123e7ee2..f64e748a 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -194,12 +194,21 @@ function supportsWorkspaceTool( const mode = request.edits === undefined ? 'single' : 'batch'; const modes = capabilities?.editFileModes; const supportsMode = modes == null ? mode === 'single' : modes.includes(mode); - if (request.operation === 'preview_edit') return supportsMode; + const features = capabilities?.editFileFeatures ?? []; + const supportsMatching = + request.matching === undefined || features.includes('tolerant_match'); + const supportsReplaceAll = + request.edits?.some((edit) => edit.replaceAll !== undefined) !== true || + features.includes('replace_all'); + if (request.operation === 'preview_edit') { + return supportsMode && supportsMatching && supportsReplaceAll; + } return ( supportsMode && + supportsMatching && + supportsReplaceAll && (request.expectedBaseSha256 === undefined || - capabilities?.editFileFeatures?.includes('expected_base_sha256') === - true) + features.includes('expected_base_sha256')) ); } return true; diff --git a/service/src/bridge/workspace-store.test.ts b/service/src/bridge/workspace-store.test.ts index f248aa40..90f68da3 100644 --- a/service/src/bridge/workspace-store.test.ts +++ b/service/src/bridge/workspace-store.test.ts @@ -2,8 +2,9 @@ import { afterEach, expect, test } from 'bun:test'; import RedisMock from 'ioredis-mock'; import type Redis from 'ioredis'; +import type { WorkspaceToolRequest } from '../../../packages/code/src/protocol'; -import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; +import { BRIDGE_PROTOCOL_VERSION, isValidBridgeWorkerCapabilities } from '../../../packages/code/src/protocol'; import { RedisBridgeStore } from './store'; const redis = new RedisMock() as unknown as Redis; @@ -643,6 +644,131 @@ test('rejects fenced edits from workers without the negotiated feature', async ( expect(await redis.keys('codeapi:bridge:v1:assignment:*')).toHaveLength(0); }); +const featureGatedEdits: Array<[string, WorkspaceToolRequest]> = [ + [ + 'tolerant matching', + { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + matching: 'tolerant', + edits: [{ oldText: 'before', newText: 'after' }], + }, + ], + [ + 'tolerant previews', + { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'preview_edit', + workspaceId: 'primary', + path: 'notes.txt', + matching: 'tolerant', + edits: [{ oldText: 'before', newText: 'after' }], + }, + ], + [ + 'replaceAll edits', + { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + edits: [{ oldText: 'before', newText: 'after', replaceAll: true }], + }, + ], +]; + +test.each(featureGatedEdits)( + 'rejects %s from workers without the negotiated feature', + async (_label, request) => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'workspace-worker', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['preview_edit', 'edit_file'], + editFileModes: ['single', 'batch'], + editFileFeatures: ['expected_base_sha256'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + + await expect( + store.dispatchWorkspaceTool({ + workerId: 'workspace-worker', + request, + deadlineAtMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).rejects.toMatchObject({ code: 'WORKER_MISMATCH' }); + expect(await redis.keys('codeapi:bridge:v1:assignment:*')).toHaveLength(0); + }, +); + +test('preview-only workers can dispatch negotiated tolerant and replace-all previews', async () => { + expect(isValidBridgeWorkerCapabilities({ + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['preview_edit'], + editFileModes: ['single', 'batch'], + editFileFeatures: ['tolerant_match', 'replace_all'], + workspaces: [{ id: 'primary' }], + }, + })).toBe(true); + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'workspace-worker', + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['preview_edit'], + editFileModes: ['single', 'batch'], + editFileFeatures: ['tolerant_match', 'replace_all'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + const request = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'preview_edit' as const, + workspaceId: 'primary', + path: 'notes.txt', + matching: 'tolerant' as const, + edits: [{ oldText: 'before', newText: 'after', replaceAll: true }], + }; + const completion = store.dispatchWorkspaceTool({ + workerId: 'workspace-worker', + request, + deadlineAtMs: Date.now() + 5_000, + signal: new AbortController().signal, + }); + const assignment = await store.lease('workspace-worker', incarnationId, 1_000); + expect(assignment).toMatchObject({ executionKind: 'workspace_tool', request }); + await store.settle('workspace-worker', assignment!.assignmentId, { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment!.generation, + leaseToken: assignment!.leaseToken, + incarnationId, + status: 'rejected', + error: 'preview tested', + }); + await expect(completion).resolves.toMatchObject({ status: 'rejected', error: 'preview tested' }); +}); + test('rejects batch previews from workers without the negotiated mode', async () => { await store.register({ protocolVersion: BRIDGE_PROTOCOL_VERSION, From 836d001319015072f9493df5d7710fc371403962 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Wed, 30 Sep 2026 08:27:56 -0400 Subject: [PATCH 18/48] =?UTF-8?q?=F0=9F=9B=A1=EF=B8=8F=20fix:=20Deny=20a?= =?UTF-8?q?=20Non-Lane=20Root's=20Own=20Git=20Metadata=20in=20the=20Native?= =?UTF-8?q?=20SRT=20Sandbox=20(#276)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🛡️ fix: Deny a Non-Lane Root's Own Git Metadata in the Native SRT Sandbox A trusted-vm command in a native (non-lane) SRT workspace could write the registered root's own `.git/hooks/*` and `.git/config`, planting a hook or a `core.fsmonitor` / `filter.*` entry that then runs unsandboxed the next time the user — or trusted worker code — invokes Git in that checkout. The non-lane config relied on SRT's mandatory `.git/hooks` and `.git/config` denies, which on Linux are computed from the worker *process's* current directory (see `sandbox/linux-sandbox-utils.js`). The worker's cwd is its home (the systemd user unit sets no WorkingDirectory), never the registered root, so those denies landed elsewhere and the root's `.git` was writable. macOS was unaffected because it applies global `**/.git/hooks/**` and `**/.git/config` Seatbelt patterns. `NativeSrtWorkspaceCommandSandbox` now adds explicit `denyWrite` entries for the registered root's own Git metadata when `/.git` is a directory (lanes are untouched — they already keep the whole common Git directory read-only): - `/.git/hooks` and `/.git/config` unconditionally; - `/.git/config.worktree` and `/.git/commondir` where they exist; - the same files for every submodule Git directory under `.git/modules/**` (including nested submodules) and every `.git/worktrees/*`. `commondir` and `config.worktree` are denied only where present: Git reads them strictly and SRT would otherwise mask an absent target with an empty `/dev/null` bind Git cannot parse, breaking ordinary commands. Denied files that exist are re-bound read-only, so `.git/config` stays readable (remotes keep working) while writes fail; a benign `git commit` in the workspace is unaffected. Tests: a live SRT test gated on `LIBRECHAT_CODE_LIVE_SRT_TESTS=1` runs the sandbox with `process.cwd()` outside the root and asserts hooks, config, an existing per-worktree config, a submodule config, and a linked-worktree commondir cannot be tampered while a normal commit still succeeds; wired into the Linux native-sandbox CI job. A fast unit test asserts the computed `denyWrite` set. `packages/code/README.md` documents the guarantee and the residual writable-`.git` vectors (whole-`.git` replacement, a fresh top-level commondir, a nested repo) that the personal-machine SRT trust model accepts. * 🔁 fix: Refresh Root Git Metadata Denies per Command and Fail Closed The root's Git metadata denies were a snapshot taken at sandbox initialization, but the worker is long-lived: a repository, submodule, or linked worktree created on the host afterwards stayed writable until restart. Ordinary root commands now recompute the set and pass it as a per-command `denyWrite` override (every other filesystem field is the session policy), the way SRT recomputes its own mandatory denies on every wrap. Native Windows keeps the initialization snapshot because srt-win rejects per-command allowRead/allowWrite. Inspecting the metadata now treats only ENOENT/ENOTDIR as absence; any other failure propagates, so an unreadable `.git` fails the command closed (COMMAND_UNAVAILABLE) instead of silently dropping a deny. Corrects the `config.worktree` comment: in a repository that already enables `worktreeConfig`, a missing per-worktree config stays creatable, because denying an absent file makes every Git command fail. The README lists it with the other writable-workspace residuals. --- .github/workflows/ci.yml | 7 +- packages/code/README.md | 28 +++ packages/code/src/native-sandbox.test.ts | 122 ++++++++++++ packages/code/src/native-sandbox.ts | 178 +++++++++++++++++- .../code/src/root-git-denies-live.test.ts | 140 ++++++++++++++ 5 files changed, 471 insertions(+), 4 deletions(-) create mode 100644 packages/code/src/root-git-denies-live.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f7393284..e6a23b61 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -254,10 +254,13 @@ jobs: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - run: npm ci - run: npm run build - - name: Linked worktree lane containment + - name: Linked worktree lane and root Git metadata containment env: LIBRECHAT_CODE_LIVE_SRT_TESTS: '1' - run: node --test dist/linked-worktrees-live.test.js + run: >- + node --test + dist/linked-worktrees-live.test.js + dist/root-git-denies-live.test.js lambda-microvm-provisioning: name: Lambda MicroVM Provisioning diff --git a/packages/code/README.md b/packages/code/README.md index ba7ec4d7..d421e444 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -213,12 +213,40 @@ SRT with: worker's private scratch directory; - read access denied to the worker's home directory except for that workspace; - paired identity and mutation-quarantine files explicitly denied; +- the registered workspace's own Git metadata denied writes — `.git/hooks` + and `.git/config` always, plus `.git/config.worktree`, `.git/commondir`, + and the equivalent files under `.git/modules/*` and `.git/worktrees/*` + where they already exist — so a sandboxed command cannot plant a hook or a + `filter`/`fsmonitor`/`diff` config entry that would run unsandboxed the + next time Git runs in the checkout; - `LIBRECHAT_CODE_*` and nonessential inherited environment variables removed; - network egress denied by default, local binding denied, and Unix sockets denied; and - bounded time and aggregate output, with best-effort process-group termination on cancellation, timeout, and completion. +The Git-metadata denies are applied to the registered root directly rather +than relying on SRT's Linux mandatory denies, which are derived from the +worker process's own current directory — the worker home, not the workspace — +and so never covered the registered root; macOS already enforced the +equivalent through global Seatbelt patterns, and this keeps the guarantee +identical on both platforms regardless of the worker's cwd. The set is +recomputed before every command, so a repository, submodule, or linked +worktree created on the host after the worker starts is covered from the next +command, and metadata the worker cannot inspect fails the command closed. +`.git/commondir` and `.git/config.worktree` are denied only where they already +exist, because Git reads them strictly and SRT would otherwise mask an absent +one with a stub Git cannot open, failing every Git command. Because the whole +workspace stays writable, a sandboxed command can still stage Git +configuration Git will honor later by other means — for example replacing +the entire `.git` directory, writing a new `commondir` that redirects the +common directory, creating a missing `config.worktree` in a repository that +already enables the `worktreeConfig` extension, or initializing a fresh +nested repository. Denying those safely would require +making the workspace's Git storage structurally read-only, which the +personal-machine SRT trust model does not; use the Docker/NsJail backend or a +dedicated VM boundary when a workspace command must be treated as adversarial. + SRT restrictions remain inherited by descendants. Windows additionally uses a kill-on-close Job Object. Native macOS does not provide an equivalent hard process-lifetime boundary: a deliberately daemonized descendant can outlive diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index 5b79aa1a..9001102f 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -1751,3 +1751,125 @@ test('linked worktree Git guard refuses Windows rather than admitting an unguard ); await sandbox.close(); }); + +test('non-lane roots deny their own executable Git metadata at initialization', async t => { + const root = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-gitmeta-'))); + t.after(() => rm(root, { recursive: true, force: true })); + const gitDir = join(root, '.git'); + await mkdir(join(gitDir, 'hooks'), { recursive: true }); + await writeFile(join(gitDir, 'config'), '[core]\n'); + await writeFile(join(gitDir, 'config.worktree'), '[core]\n'); + // A submodule Git directory (identified by its HEAD file) and a nested one. + // This submodule has no config.worktree, so it is not denied one. + await mkdir(join(gitDir, 'modules', 'sub', 'modules', 'inner'), { recursive: true }); + await writeFile(join(gitDir, 'modules', 'sub', 'HEAD'), 'ref: refs/heads/main\n'); + await writeFile(join(gitDir, 'modules', 'sub', 'config'), '[core]\n'); + await writeFile( + join(gitDir, 'modules', 'sub', 'modules', 'inner', 'HEAD'), + 'ref: refs/heads/main\n', + ); + // A linked worktree's per-worktree metadata: an existing commondir and + // config.worktree are both denied. + await mkdir(join(gitDir, 'worktrees', 'wt'), { recursive: true }); + await writeFile(join(gitDir, 'worktrees', 'wt', 'commondir'), '../..\n'); + await writeFile(join(gitDir, 'worktrees', 'wt', 'config.worktree'), '[core]\n'); + + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + environment: { PATH: '/usr/bin', LANG: 'C.UTF-8' }, + manager: fake.manager, + }); + t.after(() => sandbox.close()); + await sandbox.prepare(); + + const denyWrite = fake.config?.filesystem.denyWrite ?? []; + for (const relativePath of [ + '.git/hooks', + '.git/config', + '.git/config.worktree', + '.git/modules/sub/hooks', + '.git/modules/sub/config', + '.git/modules/sub/modules/inner/hooks', + '.git/modules/sub/modules/inner/config', + '.git/worktrees/wt/config.worktree', + '.git/worktrees/wt/commondir', + ]) { + assert.ok(denyWrite.includes(join(root, relativePath)), relativePath); + } + // Deny-if-exists paths are skipped when absent: masking them with an empty + // bind would make Git read a broken redirect or config from every command. + assert.ok(!denyWrite.includes(join(gitDir, 'commondir'))); + assert.ok(!denyWrite.includes(join(gitDir, 'modules', 'sub', 'config.worktree'))); +}); + +test('ordinary root commands deny Git metadata created after initialization', async t => { + const root = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-gitmeta-late-'))); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + environment: { PATH: '/usr/bin', LANG: 'C.UTF-8' }, + manager: fake.manager, + }); + t.after(() => sandbox.close()); + await sandbox.prepare(); + const session = fake.config!.filesystem; + const gitDir = join(root, '.git'); + assert.ok(!session.denyWrite.some(path => path.startsWith(gitDir))); + + // The host initializes the repository and adds a submodule and a linked + // worktree while the worker keeps running. + await mkdir(join(gitDir, 'hooks'), { recursive: true }); + await writeFile(join(gitDir, 'config'), '[core]\n'); + await mkdir(join(gitDir, 'modules', 'late'), { recursive: true }); + await writeFile(join(gitDir, 'modules', 'late', 'HEAD'), 'ref: refs/heads/main\n'); + await mkdir(join(gitDir, 'worktrees', 'late'), { recursive: true }); + await writeFile(join(gitDir, 'worktrees', 'late', 'commondir'), '../..\n'); + + const result = await sandbox.execute(request); + assert.equal(result.exitCode, 0); + const filesystem = fake.customConfigSeenDuringWrap?.filesystem; + for (const relativePath of [ + '.git/hooks', + '.git/config', + '.git/modules/late/hooks', + '.git/modules/late/config', + '.git/worktrees/late/commondir', + ]) { + assert.ok(filesystem?.denyWrite.includes(join(root, relativePath)), relativePath); + } + // Only denyWrite differs from the session filesystem policy. + assert.deepEqual({ ...filesystem, denyWrite: session.denyWrite }, session); +}); + +test('a root command fails closed when its Git metadata cannot be inspected', async t => { + if (process.getuid?.() === 0) { + t.skip('directory permissions do not restrict root'); + return; + } + const root = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-gitmeta-denied-'))); + t.after(() => rm(root, { recursive: true, force: true })); + const modules = join(root, '.git', 'modules'); + await mkdir(modules, { recursive: true }); + await writeFile(join(root, '.git', 'config'), '[core]\n'); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + environment: { PATH: '/usr/bin', LANG: 'C.UTF-8' }, + manager: fake.manager, + }); + t.after(() => sandbox.close()); + await sandbox.prepare(); + + await chmod(modules, 0o000); + try { + await assert.rejects( + sandbox.execute(request), + (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'COMMAND_UNAVAILABLE', + ); + } finally { + await chmod(modules, 0o700); + } +}); diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 49fe6ecf..5d39d405 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -11,7 +11,8 @@ import { sep, } from 'node:path'; import { constants as fsConstants } from 'node:fs'; -import { access, mkdtemp, open, realpath, rm, stat } from 'node:fs/promises'; +import type { Dirent, Stats } from 'node:fs'; +import { access, mkdtemp, open, readdir, realpath, rm, stat } from 'node:fs/promises'; import type { FileHandle } from 'node:fs/promises'; import { matchesWorkspaceRoot } from './root-identity.js'; import type { WorkspaceRootIdentity } from './root-identity.js'; @@ -234,6 +235,130 @@ async function canonicalPath(path: string): Promise { } } +/** + * Paths inside a repository's own Git directory that Git treats as executable + * configuration: hooks run on the next Git invocation, and `config` / + * an existing `config.worktree` can define filter, diff, or fsmonitor commands + * that later run unsandboxed. An existing `commondir` is denied too, since it + * redirects the whole common Git directory to attacker-controlled config. + * + * A sandboxed command may write anywhere in the workspace root, so these are + * denied explicitly for the registered root's `.git`. SRT's Linux mandatory + * denies derive equivalents from the worker process's current directory, which + * is not the registered root, so they never cover it; macOS instead applies + * global `.git/hooks` and `.git/config` Seatbelt patterns. Denying the root's + * own metadata directly keeps the guarantee identical on both platforms and + * independent of the worker's cwd. Lane workspaces keep the whole common Git + * directory read-only already, so this augments only the non-lane root. + * + * The set is recomputed before every ordinary root command, so a repository, + * submodule, or linked worktree created after initialization is covered from + * the next command on. Only submodule and linked-worktree Git directories + * that exist at that moment are enumerated: masking a non-existent + * `.git/modules` would block a later `git submodule add` from creating it + * during the same command. `.git/info` + * is intentionally omitted — its attributes reference filter/diff drivers by + * name, but the commands those names resolve to live in the denied config, so + * `info` alone cannot introduce a new executable. + */ +async function collectGitMetadataDenies(root: string): Promise { + const gitDir = join(root, '.git'); + if (!(await statIfPresent(gitDir))?.isDirectory()) return []; + const denies = [join(gitDir, 'hooks'), join(gitDir, 'config')]; + await pushExistingDenies(denies, gitDir); + for (const submoduleGitDir of await collectSubmoduleGitDirs( + join(gitDir, 'modules'), + )) { + denies.push(join(submoduleGitDir, 'hooks'), join(submoduleGitDir, 'config')); + await pushExistingDenies(denies, submoduleGitDir); + } + const worktreesDir = join(gitDir, 'worktrees'); + for (const entry of await readdirIfPresent(worktreesDir)) { + if (entry.isDirectory()) { + await pushExistingDenies(denies, join(worktreesDir, entry.name)); + } + } + return denies; +} + +/** + * `commondir` and `config.worktree` are read strictly by Git when present — + * `commondir` at every startup, `config.worktree` whenever `worktreeConfig` + * is enabled — and SRT masks a non-existent deny target with an empty + * `/dev/null` bind that Git cannot open, failing every Git command. Denying + * them only where they already exist keeps the mask a read-only bind of the + * real file. `config` and `hooks` stay unconditional: they always exist in a + * real repository. In a repository that already enables `worktreeConfig`, a + * missing `config.worktree` therefore stays creatable, like a new + * `commondir`; the package README lists both with the other residuals of a + * writable workspace. + */ +async function pushExistingDenies(paths: string[], gitDir: string): Promise { + for (const name of ['config.worktree', 'commondir']) { + const candidate = join(gitDir, name); + if (await statIfPresent(candidate)) paths.push(candidate); + } +} + +/** + * Enumerate every submodule Git directory beneath `.git/modules`, following the + * `/modules/` nesting Git uses for recursive submodules. A + * directory holding a `HEAD` file is a Git directory; any other directory is an + * intermediate segment of a submodule path that itself contains slashes. + */ +async function collectSubmoduleGitDirs(modulesDir: string): Promise { + const found: string[] = []; + const stack = [modulesDir]; + while (stack.length > 0) { + const dir = stack.pop()!; + const entries = await readdirIfPresent(dir); + if (entries.some(entry => entry.isFile() && entry.name === 'HEAD')) { + found.push(dir); + if ( + entries.some(entry => entry.isDirectory() && entry.name === 'modules') + ) { + stack.push(join(dir, 'modules')); + } + continue; + } + for (const entry of entries) { + if (entry.isDirectory()) stack.push(join(dir, entry.name)); + } + } + return found; +} + +/** + * Absence is an expected answer while inspecting Git metadata. Any other + * failure (for example `EACCES`) propagates, so metadata that cannot be + * inspected fails the command closed instead of silently dropping a deny. + */ +function isMissing(error: unknown): boolean { + return ( + error instanceof Error && + 'code' in error && + (error.code === 'ENOENT' || error.code === 'ENOTDIR') + ); +} + +async function statIfPresent(path: string): Promise { + try { + return await stat(path); + } catch (error) { + if (isMissing(error)) return undefined; + throw error; + } +} + +async function readdirIfPresent(path: string): Promise { + try { + return await readdir(path, { withFileTypes: true }); + } catch (error) { + if (isMissing(error)) return []; + throw error; + } +} + function boundedUtf8(buffer: Buffer, budget: number): string { let end = Math.min(buffer.byteLength, budget); while (end > 0) { @@ -315,6 +440,8 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox private runtimeConfig?: SandboxRuntimeConfig; private denyReadPaths: string[] = []; private denyWritePaths: string[] = []; + /** Worker-owned write denies that precede each command's live Git metadata denies. */ + private baseDenyWritePaths: string[] = []; private get gitEnvironment(): | typeof TRUSTED_GIT_ENVIRONMENT | typeof LINKED_WORKTREE_GIT_ENVIRONMENT { @@ -517,6 +644,12 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox const gitGuardDirectory = lane ? await this.createLinkedWorktreeGitGuard(canonicalScratchDirectory!) : undefined; + // A lane already keeps the shared common Git directory read-only; only + // the non-lane root exposes a writable `.git` whose executable metadata + // SRT's cwd-derived mandatory denies do not reach. + const rootGitMetadataDenies = lane + ? [] + : await collectGitMetadataDenies(root); const commandPolicy = normalizeNativeSrtCommandPolicy( this.options.commandPolicy, ); @@ -558,6 +691,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox denyWrite: [ ...protectedPaths, ...deniedInheritedWritablePaths, + ...rootGitMetadataDenies, ...(gitGuardDirectory ? [gitGuardDirectory] : []), ], allowGitConfig: false, @@ -625,9 +759,14 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox deniedInheritedWritablePaths.includes(path), ), ]; + this.baseDenyWritePaths = [ + ...protectedPaths, + ...deniedInheritedWritablePaths, + ]; this.denyWritePaths = [ ...protectedPaths, ...deniedInheritedWritablePaths, + ...rootGitMetadataDenies, ...(gitGuardDirectory ? [gitGuardDirectory] : []), ]; } @@ -937,6 +1076,40 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox } } + /** + * The session policy snapshots the root's Git metadata at initialization, but + * the worker outlives that snapshot: a repository, submodule, or linked + * worktree created on the host afterwards must be protected without a + * restart, the way SRT recomputes its own mandatory denies on every wrap. + * Only `denyWrite` differs from the session filesystem policy. Lanes keep + * the whole common Git directory read-only already. Native Windows keeps the + * initialization snapshot, because `srt-win` rejects the per-command + * `allowRead`/`allowWrite` a full filesystem override carries. + */ + private async rootCommandConfig(): Promise< + Partial | undefined + > { + const config = this.runtimeConfig; + const root = this.canonicalRoot; + if ( + !config || + !root || + this.options.linkedWorktree || + this.platform === 'win32' + ) { + return undefined; + } + return { + filesystem: { + ...config.filesystem, + denyWrite: [ + ...this.baseDenyWritePaths, + ...(await collectGitMetadataDenies(root)), + ], + }, + }; + } + private async executeBound( request: WorkspaceExecuteCommandRequest, signal?: AbortSignal, @@ -981,6 +1154,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ReturnType >; try { + const commandConfig = customConfig ?? (await this.rootCommandConfig()); const credentialEnvironment = await this.options.maskedEnvironment?.resolve(signal, cwd); const sandboxedCommand = this.options.maskedEnvironment?.wrapCommand @@ -1002,7 +1176,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox this.platform === 'win32' ? undefined : (this.options.shellPath ?? '/bin/bash'), - customConfig, + commandConfig, signal, cwd, { commandId, commandText: request.command }, diff --git a/packages/code/src/root-git-denies-live.test.ts b/packages/code/src/root-git-denies-live.test.ts new file mode 100644 index 00000000..12dc820c --- /dev/null +++ b/packages/code/src/root-git-denies-live.test.ts @@ -0,0 +1,140 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { mkdir, mkdtemp, readFile, realpath, rm, stat, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import test from 'node:test'; + +import { NativeSrtWorkspaceCommandSandbox } from './native-sandbox.js'; +import { resolveNativeSrtCommandPolicy } from './native-policy.js'; + +const execFileAsync = promisify(execFile); +const IDENTITY = ['-c', 'user.name=t', '-c', 'user.email=t@example.com', '-c', 'commit.gpgsign=false']; + +async function git(cwd: string, ...args: string[]): Promise { + return (await execFileAsync('git', [...IDENTITY, ...args], { cwd })).stdout.trim(); +} + +async function snapshot(paths: string[]): Promise> { + const entries = await Promise.all( + paths.map(async (path) => [path, await readFile(path, 'utf8').catch(() => null)] as const), + ); + return Object.fromEntries(entries); +} + +async function exists(path: string): Promise { + return stat(path).then(() => true, () => false); +} + +/** + * SRT's Linux mandatory denies for `.git/hooks` and `.git/config` are computed + * from the worker process's own current directory, which is never the registered + * workspace root. This exercises the sandbox from a cwd outside the root and + * asserts the root's executable Git metadata is unwritable anyway — hooks, + * config, an existing per-worktree config, and a submodule's config cannot be + * tampered, so nothing an attacker writes runs the next time git is invoked on + * the host. + */ +test('real SRT denies a non-lane root its own Git hooks and config even when the worker cwd is elsewhere', { + skip: process.env.LIBRECHAT_CODE_LIVE_SRT_TESTS !== '1', + timeout: 90_000, +}, async (t) => { + const outside = await realpath(await mkdtemp(join(tmpdir(), 'git-denies-cwd-'))); + const base = await realpath(await mkdtemp(join(tmpdir(), 'git-denies-'))); + t.after(() => rm(base, { recursive: true, force: true })); + t.after(() => rm(outside, { recursive: true, force: true })); + + // A standalone repo used only as a local submodule source. + const upstream = join(base, 'sub-src'); + await mkdir(upstream); + await git(upstream, 'init', '-q', '-b', 'main'); + await writeFile(join(upstream, 's.txt'), 'sub\n'); + await git(upstream, 'add', 's.txt'); + await git(upstream, 'commit', '-qm', 'sub'); + + const root = join(base, 'repo'); + await mkdir(root); + await git(root, 'init', '-q', '-b', 'main'); + await writeFile(join(root, 'tracked.txt'), 'checkout\n'); + await git(root, 'add', 'tracked.txt'); + await git(root, 'commit', '-qm', 'init'); + await git(root, '-c', 'protocol.file.allow=always', 'submodule', 'add', '-q', upstream, 'sub'); + await git(root, 'commit', '-qm', 'add submodule'); + await mkdir(join(root, '.worktrees')); + await git(root, 'worktree', 'add', '-q', '.worktrees/wt', '-b', 'wt'); + // An existing per-worktree config for the linked worktree; --worktree needs + // the repository extension enabled first. + await git(root, 'config', 'extensions.worktreeConfig', 'true'); + await git(join(root, '.worktrees', 'wt'), 'config', '--worktree', 'core.testflag', 'true'); + + const gitDir = join(root, '.git'); + const submoduleGitDir = join(gitDir, 'modules', 'sub'); + const worktreeGitDir = join(gitDir, 'worktrees', 'wt'); + const guarded = [ + join(gitDir, 'hooks', 'post-checkout'), + join(gitDir, 'config'), + join(submoduleGitDir, 'config'), + join(submoduleGitDir, 'hooks', 'post-checkout'), + join(worktreeGitDir, 'config.worktree'), + join(worktreeGitDir, 'commondir'), + ]; + for (const path of guarded) { + assert.ok(await exists(path) || path.endsWith('post-checkout'), `precondition: ${path}`); + } + const before = await snapshot(guarded); + + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + commandPolicy: resolveNativeSrtCommandPolicy('trusted-vm'), + environment: { PATH: process.env.PATH, LANG: 'C.UTF-8' }, + }); + t.after(() => sandbox.close()); + const run = (command: string) => + sandbox.execute({ + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'root', + command, + timeoutMs: 30_000, + maxOutputBytes: 16_384, + }); + + // The worker process runs from outside the registered root, the way the + // systemd user unit does (its cwd is the worker home, not the workspace). + const previousCwd = process.cwd(); + process.chdir(outside); + t.after(() => process.chdir(previousCwd)); + + for (const path of guarded) { + const attempt = await run(`printf tampered > '${path}'`); + assert.notEqual(attempt.exitCode, 0, `expected a write to ${path} to be denied`); + } + // Planting a fresh hook file (not just overwriting one) must also be blocked. + const plantHook = await run( + `printf '#!/bin/sh\\ntouch ${join(base, 'PWNED')}\\n' > .git/hooks/pre-commit && chmod +x .git/hooks/pre-commit`, + ); + assert.notEqual(plantHook.exitCode, 0); + + // Metadata the host creates while the worker runs is covered from the next + // command, not only what existed when the sandbox initialized. + await git(root, 'worktree', 'add', '-q', '.worktrees/late', '-b', 'late'); + const lateCommondir = join(gitDir, 'worktrees', 'late', 'commondir'); + const lateBefore = await readFile(lateCommondir, 'utf8'); + const lateAttempt = await run(`printf tampered > '${lateCommondir}'`); + assert.notEqual(lateAttempt.exitCode, 0, 'expected the new worktree metadata to be denied'); + assert.equal(await readFile(lateCommondir, 'utf8'), lateBefore); + + // A benign commit in the workspace must still succeed: the guard makes Git + // metadata read-only, not the checkout or the writable object store. + const committed = await run( + `printf work > work.txt && git ${IDENTITY.join(' ')} add work.txt && git ${IDENTITY.join(' ')} commit -qm work`, + ); + assert.equal(committed.exitCode, 0, committed.stderr); + + await sandbox.close(); + + assert.equal(await exists(join(base, 'PWNED')), false); + assert.deepEqual(await snapshot(guarded), before); + assert.equal(await git(root, 'log', '-1', '--format=%s'), 'work'); +}); From 05bad85d10197832b0fa472e00687b699beec680 Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 20:52:30 -0400 Subject: [PATCH 19/48] =?UTF-8?q?=F0=9F=A7=B7=20test:=20Retain=20the=20Con?= =?UTF-8?q?tention=20Lock=20Owner=20Through=20Garbage=20Collection=20(#283?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Lia --- packages/code/src/process-lock.test.ts | 51 ++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 7 deletions(-) diff --git a/packages/code/src/process-lock.test.ts b/packages/code/src/process-lock.test.ts index 3594a1a9..77091db3 100644 --- a/packages/code/src/process-lock.test.ts +++ b/packages/code/src/process-lock.test.ts @@ -17,25 +17,60 @@ test( const child = spawn( process.execPath, [ + '--expose-gc', '--input-type=module', '-e', ` + import { setTimeout as delay } from 'node:timers/promises'; import { withProcessLock } from ${JSON.stringify(moduleUrl)}; await withProcessLock(${JSON.stringify(path)}, async () => { - process.stdout.write('locked'); - await new Promise(() => { setInterval(() => {}, 1000); }); + setImmediate(() => { + global.gc(); + setImmediate(() => process.send('locked')); + }); + // The timer retains its resolver and the lock-owning async frame. + await delay(60_000); }); `, ], - { stdio: ['ignore', 'pipe', 'pipe'] } + { stdio: ['ignore', 'pipe', 'pipe', 'ipc'] } ); + let stdout = ''; + let stderr = ''; + child.stdout!.setEncoding('utf8').on('data', (chunk: string) => { + stdout = (stdout + chunk).slice(-4096); + }); + child.stderr!.setEncoding('utf8').on('data', (chunk: string) => { + stderr = (stderr + chunk).slice(-4096); + }); const closed = once(child, 'close'); t.after(async () => { child.kill('SIGKILL'); - await closed; - await rm(directory, { recursive: true, force: true }); + try { + await closed; + } finally { + t.diagnostic( + JSON.stringify({ + stdout, + stderr, + exitCode: child.exitCode, + signal: child.signalCode, + }) + ); + await rm(directory, { recursive: true, force: true }); + } }); - await once(child.stdout!, 'data'); + await Promise.race([ + once(child, 'message', { signal: t.signal }).then(([marker]) => { + assert.equal(marker, 'locked'); + t.diagnostic('Lock owner ready: locked (after GC)'); + }), + closed.then(([code, signal]) => { + throw new Error( + `Lock owner exited before readiness: ${code}/${signal}` + ); + }), + ]); let entered = false; await assert.rejects( withProcessLock( @@ -47,8 +82,10 @@ test( ) ); assert.equal(entered, false); + assert.equal(child.exitCode, null); + assert.equal(child.signalCode, null); child.kill('SIGKILL'); - await closed; + assert.deepEqual(await closed, [null, 'SIGKILL']); await withProcessLock( path, async () => { From e46fca4ceb175c7cb9d46f8e3fca9d4224e39ccd Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Thu, 1 Oct 2026 22:48:41 -0400 Subject: [PATCH 20/48] =?UTF-8?q?=F0=9F=AA=9F=20feat:=20Preserve=20Command?= =?UTF-8?q?=20Output=20Heads=20and=20Tails=20(#275)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🪟 feat: Preserve Command Output Heads and Tails * 🪟 fix: Retain Logical Command Output Suffixes --------- Co-authored-by: Lia --- packages/code/README.md | 15 + packages/code/src/native-sandbox.test.ts | 142 ++++++++- packages/code/src/native-sandbox.ts | 57 +--- packages/code/src/output.test.ts | 353 +++++++++++++++++++++++ packages/code/src/output.ts | 182 ++++++++++++ 5 files changed, 702 insertions(+), 47 deletions(-) create mode 100644 packages/code/src/output.test.ts create mode 100644 packages/code/src/output.ts diff --git a/packages/code/README.md b/packages/code/README.md index d421e444..acae9bb8 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -225,6 +225,21 @@ SRT with: - bounded time and aggregate output, with best-effort process-group termination on cancellation, timeout, and completion. +Native command output keeps a prefix and rolling suffix for each stream, so late +summaries and errors survive truncation. Each stream stores at most +`maxOutputBytes` of copied raw bytes while the command runs, independent of output +volume or chunk count. When both streams are noisy they split the existing combined +response budget equally, with the odd byte reserved for stderr; a quiet stream gives +its unused allowance to the other based on rendered UTF-8 bytes, including replacement +characters for malformed input. Sandbox violation annotations enter the same +stderr window before rendering. UTF-8 boundaries and inline +`[... N bytes omitted ...]` markers count toward the combined byte limit. The count +reports omitted raw bytes for that stream, including annotation bytes. If a stream's +allowance cannot fit its marker, only the retained text and the existing `truncated` +flag are returned. Truncation does not stop execution. Exit codes, timeout/signal +fields, and cancellation errors keep their existing semantics, and no new request, +result, or capability keys are introduced. + The Git-metadata denies are applied to the registered root directly rather than relying on SRT's Linux mandatory denies, which are derived from the worker process's own current directory — the worker home, not the workspace — diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index 9001102f..90215156 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -33,6 +33,7 @@ import { } from './native-sandbox.js'; import { restoreScratchTraversal } from './native-scratch.js'; import { WorkspaceToolError } from './workspace.js'; +import { isWorkspaceToolResult } from './protocol.js'; const request = { protocolVersion: 1 as const, @@ -1355,14 +1356,151 @@ test('executes in the canonical workspace and bounds aggregate output', async t operation: 'execute_command', workspaceId: 'primary', exitCode: 0, - stdout: '1234567890', - stderr: 'ab', + stdout: '123890', + stderr: 'abchij', truncated: true, timedOut: false, }, ); }); +test('retains real command summaries on both streams under the legacy combined budget', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fakeManager().manager, + }); + t.after(() => sandbox.close()); + const commandRequest = { + ...request, + maxOutputBytes: 256, + command: + "printf 'OUT\\n'; printf '%20000d' 0; printf '\\n42 tests passed\\n'; printf 'ERR\\n' >&2; printf '%20000d' 0 >&2; printf '\\nlate stderr summary\\n' >&2", + }; + const result = await sandbox.execute(commandRequest); + assert.equal(result.exitCode, 0); + assert.equal(result.timedOut, false); + assert.equal(result.truncated, true); + assert.ok(result.stdout.startsWith('OUT\n')); + assert.ok(result.stderr.startsWith('ERR\n')); + assert.ok(result.stdout.endsWith('\n42 tests passed\n')); + assert.ok(result.stderr.endsWith('\nlate stderr summary\n')); + assert.ok(result.stdout.includes('bytes omitted')); + assert.ok(result.stderr.includes('bytes omitted')); + assert.equal(isWorkspaceToolResult(commandRequest, result), true); +}); + +test('partly filled command buffers retain final summaries regardless of stream order', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fakeManager().manager, + }); + t.after(() => sandbox.close()); + const summary = 'late stderr summary\n'; + const quiet = `printf '%109d' 0; printf 'late stderr summary\\n'`; + const noisy = `printf '%20000d' 0`; + for (const [command, stream] of [ + [`{ ${quiet}; } >&2; ${noisy}`, 'stderr'], + [`${noisy}; { ${quiet}; } >&2`, 'stderr'], + [`{ ${quiet}; }; ${noisy} >&2`, 'stdout'], + [`${noisy} >&2; { ${quiet}; }`, 'stdout'], + ] as const) { + const commandRequest = { ...request, command, maxOutputBytes: 256 }; + const result = await sandbox.execute(commandRequest); + assert.ok(result[stream].endsWith(summary)); + assert.equal(result.truncated, true); + assert.equal(result.exitCode, 0); + assert.equal(isWorkspaceToolResult(commandRequest, result), true); + } +}); + +test('quiet malformed command output is retained beside a noisy stream', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fakeManager().manager, + }); + t.after(() => sandbox.close()); + for (const [command, stream] of [ + ["printf '%20000d' 0; printf '\\377\\377' >&2", 'stderr'], + ["printf '\\377\\377'; printf '%20000d' 0 >&2", 'stdout'], + ] as const) { + const commandRequest = { ...request, command, maxOutputBytes: 32 }; + const result = await sandbox.execute(commandRequest); + assert.equal(result[stream], '\ufffd\ufffd'); + assert.equal(result.truncated, true); + assert.equal(result.exitCode, 0); + assert.equal(isWorkspaceToolResult(commandRequest, result), true); + } +}); + +test('sandbox annotations survive full stdout and remain bounded when stderr is noisy', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager(); + fake.manager.annotateStderrWithSandboxFailures = (_commandId, stderr) => + stderr + '\n\ndenied write\n'; + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fake.manager, + }); + t.after(() => sandbox.close()); + for (const stderrCommand of ['', "printf '%20000d' 0 >&2;"]) { + const commandRequest = { + ...request, + maxOutputBytes: 512, + command: `printf '%20000d' 0; ${stderrCommand} true`, + }; + const result = await sandbox.execute(commandRequest); + assert.ok( + result.stderr.endsWith( + '\ndenied write\n' + ) + ); + assert.equal(isWorkspaceToolResult(commandRequest, result), true); + assert.equal(result.truncated, true); + } + fake.manager.annotateStderrWithSandboxFailures = () => { + throw new Error('annotation unavailable'); + }; + const result = await sandbox.execute({ + ...request, + maxOutputBytes: 128, + command: "printf '%20000d' 0; printf 'child failure' >&2", + }); + assert.equal(result.stderr, 'child failure'); +}); + +test('timeout settlement keeps late diagnostics, signal, and truncation without widening the result', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fakeManager().manager, + }); + t.after(() => sandbox.close()); + const commandRequest = { + ...request, + maxOutputBytes: 128, + timeoutMs: 100, + command: + "printf 'START\\n'; printf '%20000d' 0; printf '\\nlast progress\\n'; printf 'last failure' >&2; sleep 30", + }; + const result = await sandbox.execute(commandRequest); + assert.ok(result.stdout.startsWith('START\n')); + assert.ok(result.stdout.endsWith('\nlast progress\n')); + assert.equal(result.stderr, 'last failure'); + assert.equal(result.timedOut, true); + assert.equal(result.truncated, true); + assert.equal(result.exitCode, null); + assert.equal(result.signal, 'SIGKILL'); + assert.equal(isWorkspaceToolResult(commandRequest, result), true); +}); + test('rejects an escaping or unavailable command working directory', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 5d39d405..26c848e5 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -32,6 +32,7 @@ import { removePrivateStorageAcl, } from './private-storage.js'; import { WorkspaceToolError } from './workspace.js'; +import { OutputBuffer, renderCommandOutput } from './output.js'; import { restoreScratchTraversal } from './native-scratch.js'; import { writeLinkedWorktreeGitGuard } from './linked-worktree-git-guard.js'; @@ -1301,28 +1302,10 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox } let settled = false; let timedOut = false; - let outputBytes = 0; - let truncated = false; - const stdout: Buffer[] = []; - const stderr: Buffer[] = []; - const append = (target: Buffer[], chunk: Buffer): void => { - const remaining = outputLimit - outputBytes; - if (remaining <= 0) { - truncated = true; - return; - } - const accepted = chunk.subarray(0, remaining); - target.push(accepted); - outputBytes += accepted.byteLength; - if (accepted.byteLength !== chunk.byteLength) - truncated = true; - }; - child.stdout.on('data', (chunk: Buffer) => - append(stdout, chunk), - ); - child.stderr.on('data', (chunk: Buffer) => - append(stderr, chunk), - ); + const stdout = new OutputBuffer(outputLimit); + const stderr = new OutputBuffer(outputLimit); + child.stdout.on('data', (chunk: Buffer) => stdout.append(chunk)); + child.stderr.on('data', (chunk: Buffer) => stderr.append(chunk)); const abort = (): void => { if (settled) return; this.killCommandTree(child); @@ -1371,29 +1354,17 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); return; } - const stdoutValue = boundedUtf8( - Buffer.concat(stdout), - outputLimit, - ); - const stderrBudget = Math.max( - 0, - outputLimit - Buffer.byteLength(stdoutValue), - ); - const rawStderr = Buffer.concat(stderr).toString('utf8'); - let annotatedStderr = rawStderr; try { - annotatedStderr = - this.manager.annotateStderrWithSandboxFailures( - commandId, - rawStderr, + // SRT appends violations to its input. Capture them in the same bounded stderr window. + stderr.append( + Buffer.from( + this.manager.annotateStderrWithSandboxFailures(commandId, ''), + ), ); } catch { // Preserve the bounded child error if optional violation annotation fails. } - const stderrValue = boundedUtf8( - Buffer.from(annotatedStderr), - stderrBudget, - ); + const output = renderCommandOutput(stdout, stderr, outputLimit); resolvePromise({ protocolVersion: BRIDGE_PROTOCOL_VERSION, operation: 'execute_command', @@ -1403,11 +1374,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ? null : this.protocolExitCode(code), ...(childSignal ? { signal: childSignal } : {}), - stdout: stdoutValue, - stderr: stderrValue, - truncated: - truncated || - Buffer.byteLength(annotatedStderr) > stderrBudget, + ...output, timedOut, }); }); diff --git a/packages/code/src/output.test.ts b/packages/code/src/output.test.ts new file mode 100644 index 00000000..f78b0993 --- /dev/null +++ b/packages/code/src/output.test.ts @@ -0,0 +1,353 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { OutputBuffer, renderCommandOutput } from './output.js'; + +function collect( + content: Buffer, + budget: number, + chunkSize = content.length || 1 +): OutputBuffer { + const output = new OutputBuffer(budget); + for (let offset = 0; offset < content.length; offset += chunkSize) { + output.append(content.subarray(offset, offset + chunkSize)); + } + return output; +} + +function assertBounded(text: string, budget: number): void { + assert.ok(Buffer.byteLength(text) <= budget); + assert.equal(Buffer.from(text).toString('utf8'), text); +} + +test('small output stays byte-for-byte unchanged, including split UTF-8 and exact limits', () => { + for (const content of ['', '1234567', 'hello\n世界🌍\n']) { + const bytes = Buffer.from(content); + const budget = Math.max(1, bytes.length); + for (const chunkSize of [1, 2, 7, 64]) { + assert.deepEqual(collect(bytes, budget, chunkSize).render(budget), { + text: content, + truncated: false, + }); + } + } +}); + +test('a rolling suffix keeps the last summary and reports the exact dropped-byte count', () => { + const budget = 128; + const content = Buffer.from( + 'START\n' + 'x'.repeat(10_000) + '\n42 tests passed\n' + ); + for (const chunkSize of [1, 3, 63, 64, 65, 127, 1024, content.length]) { + const result = collect(content, budget, chunkSize).render(budget); + assert.equal(result.truncated, true); + assert.ok(result.text.startsWith('START\n')); + assert.ok(result.text.endsWith('\n42 tests passed\n')); + const marker = /\n\[\.\.\. (\d+) bytes omitted \.\.\.\]\n/.exec( + result.text + ); + assert.ok(marker); + assert.equal( + Number(marker[1]), + content.length - + Buffer.byteLength(result.text.replace(marker[0], '')) + ); + assertBounded(result.text, budget); + } +}); + +test('either quiet stream donates its budget, and stdout cannot starve stderr', () => { + const budget = 256; + const empty = collect(Buffer.alloc(0), budget); + const exact = collect(Buffer.alloc(budget, 'a'), budget, 1); + assert.deepEqual(renderCommandOutput(exact, empty, budget), { + stdout: 'a'.repeat(budget), + stderr: '', + truncated: false, + }); + assert.deepEqual(renderCommandOutput(empty, exact, budget), { + stdout: '', + stderr: 'a'.repeat(budget), + truncated: false, + }); + const stdout = collect( + Buffer.from('OUT\n' + 'x'.repeat(4096) + '\nstdout summary'), + budget + ); + const stderr = collect( + Buffer.from('ERR\n' + 'y'.repeat(4096) + '\nstderr summary'), + budget + ); + const result = renderCommandOutput(stdout, stderr, budget); + assert.ok(result.stdout.startsWith('OUT\n')); + assert.ok(result.stderr.startsWith('ERR\n')); + assert.ok(result.stdout.endsWith('stdout summary')); + assert.ok(result.stderr.endsWith('stderr summary')); + assert.ok(result.stdout.includes('bytes omitted')); + assert.ok(result.stderr.includes('bytes omitted')); + assert.ok( + Buffer.byteLength(result.stdout) + Buffer.byteLength(result.stderr) <= + budget + ); + assert.equal(result.truncated, true); + + const shortError = collect(Buffer.from('late failure'), budget); + assert.equal( + renderCommandOutput(stdout, shortError, budget).stderr, + 'late failure' + ); + assert.equal( + renderCommandOutput(shortError, stdout, budget).stdout, + 'late failure' + ); +}); + +test('partly filled buffers retain final summaries on either stream', () => { + const capacity = 256; + const summary = 'late stderr summary\n'; + const content = Buffer.from('y'.repeat(129 - summary.length) + summary); + for (const chunkSize of [1, 7, 128, content.length]) { + const quiet = collect(content, capacity, chunkSize); + const noisy = collect(Buffer.alloc(20_000, 'x'), capacity, chunkSize); + for (const [stdout, stderr, stream] of [ + [noisy, quiet, 'stderr'], + [quiet, noisy, 'stdout'], + ] as const) { + const result = renderCommandOutput(stdout, stderr, capacity); + assert.ok(result[stream].endsWith(summary)); + assert.ok( + Buffer.byteLength(result.stdout) + + Buffer.byteLength(result.stderr) <= + capacity + ); + } + } +}); + +test('rendered windows match logical stream ends across capture and allowance boundaries', () => { + for (const capacity of [32, 63, 64, 127, 128, 255, 256, 257]) { + const half = Math.floor(capacity / 2); + for (const length of [ + 0, + 1, + half - 1, + half, + half + 1, + capacity - 1, + capacity, + capacity + 1, + capacity * 4, + ]) { + const content = Buffer.from( + Array.from({ length }, (_, index) => + String.fromCharCode(65 + (index % 26)) + ).join('') + ); + for (const budget of [ + 0, + 1, + Math.floor(half / 2), + half, + capacity - 1, + capacity, + ]) { + let expected = content.toString('utf8'); + if (length > budget) { + const marker = (bytes: number): string => + `\n[... ${bytes} bytes omitted ...]\n`; + const markerBytes = Buffer.byteLength(marker(length)); + const includeMarker = budget >= markerBytes; + const remaining = includeMarker + ? budget - markerBytes + : budget; + const prefix = Math.floor(remaining / 2); + const suffix = Math.ceil(remaining / 2); + expected = + content.subarray(0, prefix).toString('utf8') + + (includeMarker + ? marker(length - prefix - suffix) + : '') + + content.subarray(length - suffix).toString('utf8'); + } + for (const chunkSize of [1, half, capacity + 7]) { + assert.deepEqual( + collect(content, capacity, chunkSize).render(budget), + { + text: expected, + truncated: length > budget, + }, + `capacity ${capacity}, length ${length}, budget ${budget}, chunk ${chunkSize}` + ); + } + } + } + } +}); + +test('UTF-8 summaries survive capture transitions and split code-point chunks', () => { + const summary = 'end 世界🌍\n'; + for (const capacity of [128, 255, 256, 257]) { + const half = Math.floor(capacity / 2); + for (const length of [ + half - 1, + half, + half + 1, + capacity - 1, + capacity, + capacity + 1, + ]) { + const content = Buffer.from( + 'x'.repeat(length - Buffer.byteLength(summary)) + summary + ); + for (const chunkSize of [1, 2, content.length]) { + const quiet = collect(content, capacity, chunkSize); + const noisy = collect( + Buffer.alloc(capacity * 4, 'y'), + capacity, + chunkSize + ); + for (const [stdout, stderr, stream] of [ + [noisy, quiet, 'stderr'], + [quiet, noisy, 'stdout'], + ] as const) { + const result = renderCommandOutput( + stdout, + stderr, + capacity + ); + assert.ok(result[stream].endsWith(summary)); + assert.ok(!result[stream].includes('\ufffd')); + assert.ok( + Buffer.byteLength(result.stdout) + + Buffer.byteLength(result.stderr) <= + capacity + ); + } + } + } + } +}); + +test('budgeting includes UTF-8 boundaries and marker overhead for every small limit', () => { + const content = Buffer.from('世界🌍 café Ελληνικά\n'.repeat(100)); + for (let budget = 1; budget <= 256; budget += 1) { + const result = collect(content, budget, 1).render(budget); + assertBounded(result.text, budget); + assert.equal(result.truncated, true); + assert.ok( + !result.text.includes('\ufffd'), + `split code point at budget ${budget}` + ); + const marker = /\n\[\.\.\. (\d+) bytes omitted \.\.\.\]\n/.exec( + result.text + ); + if (marker) { + assert.equal( + Number(marker[1]), + content.length - + Buffer.byteLength(result.text.replace(marker[0], '')) + ); + } + const streams = renderCommandOutput( + collect(content, budget, 7), + collect(content, budget, 3), + budget + ); + assert.ok( + Buffer.byteLength(streams.stdout) + + Buffer.byteLength(streams.stderr) <= + budget + ); + assert.ok( + !streams.stdout.includes('\ufffd') && + !streams.stderr.includes('\ufffd') + ); + } +}); + +test('malformed output cannot expand past the byte budget during decoding', () => { + for (const content of [ + Buffer.alloc(1024, 0xff), + Buffer.from([0xf0, 0x80, 0xff, 0xc2, 0xa2, 0xe0, 0xa0]), + Buffer.from('a\ufffdb'.repeat(300)), + ]) { + for (let budget = 1; budget <= 128; budget += 1) { + const result = collect(content, budget, 1).render(budget); + assertBounded(result.text, budget); + if (content.length > budget) assert.equal(result.truncated, true); + } + } +}); + +test('short decoded output is preserved when replacement characters fit the combined budget', () => { + const budget = 32; + assert.deepEqual( + renderCommandOutput( + collect(Buffer.from([0xff, 0xff]), budget), + collect(Buffer.from('child failure'), budget), + budget + ), + { stdout: '\ufffd\ufffd', stderr: 'child failure', truncated: false } + ); +}); + +test('quiet-stream donation counts decoded bytes beside noisy output', () => { + for (const bytes of [ + Buffer.from([0xff, 0xff]), + Buffer.from([0x80, 0x80]), + Buffer.from([0xf0, 0x9f]), + Buffer.from([0x61, 0xff, 0x62]), + ]) { + const decoded = bytes.toString('utf8'); + for (const budget of [16, 31, 32, 63, 64, 128]) { + const quiet = collect(bytes, budget, 1); + const noisy = collect(Buffer.alloc(budget * 4, 'x'), budget, 7); + for (const [stdout, stderr, stream] of [ + [noisy, quiet, 'stderr'], + [quiet, noisy, 'stdout'], + ] as const) { + const result = renderCommandOutput(stdout, stderr, budget); + assert.equal(result[stream], decoded); + assert.equal(result.truncated, true); + assert.ok( + Buffer.byteLength(result.stdout) + + Buffer.byteLength(result.stderr) <= + budget + ); + } + } + } +}); + +test('tiny budgets still flag truncation and give stderr the odd byte', () => { + for (let budget = 1; budget < 32; budget += 1) { + const result = renderCommandOutput( + collect(Buffer.from('a'.repeat(256)), budget), + collect(Buffer.from('b'.repeat(256)), budget), + budget + ); + assert.equal(result.truncated, true); + assert.ok(result.stderr.length > 0); + assert.ok( + Buffer.byteLength(result.stdout) + + Buffer.byteLength(result.stderr) <= + budget + ); + } +}); + +test('copies bounded windows instead of retaining or repeatedly concatenating source chunks', () => { + const budget = 128; + const output = new OutputBuffer(budget); + const oversized = Buffer.alloc(1024 * 1024, 'a'); + output.append(oversized); + oversized.fill('z'); + const byte = Buffer.from('b'); + for (let i = 0; i < 100_000; i += 1) output.append(byte); + byte[0] = 0x7a; + const result = output.render(budget); + assert.ok(result.text.startsWith('a'.repeat(40))); + assert.ok(result.text.endsWith('b'.repeat(40))); + assert.ok(!result.text.includes('z')); + assert.equal(output.bytes, 1024 * 1024 + 100_000); + assertBounded(result.text, budget); +}); diff --git a/packages/code/src/output.ts b/packages/code/src/output.ts new file mode 100644 index 00000000..4cb34211 --- /dev/null +++ b/packages/code/src/output.ts @@ -0,0 +1,182 @@ +interface OutputWindow { + text: string; + bytes: number; +} + +function utf8Window( + buffer: Buffer, + length: number, + tail: boolean +): OutputWindow { + let start = tail ? buffer.length - length : 0; + let end = tail ? buffer.length : length; + if (tail) { + while (start < end && (buffer[start] & 0xc0) === 0x80) start += 1; + } else if (end > 0) { + let last = end - 1; + while (last > 0 && (buffer[last] & 0xc0) === 0x80) last -= 1; + const lead = buffer[last]; + const width = + lead >= 0xc2 && lead <= 0xdf + ? 2 + : lead >= 0xe0 && lead <= 0xef + ? 3 + : lead >= 0xf0 && lead <= 0xf4 + ? 4 + : 1; + if (end - last < width) end = last; + } + return { + text: buffer.subarray(start, end).toString('utf8'), + bytes: end - start, + }; +} + +function boundedWindow( + buffer: Buffer, + budget: number, + tail: boolean +): OutputWindow { + const length = Math.min(buffer.length, budget); + const window = utf8Window(buffer, length, tail); + if (Buffer.byteLength(window.text) <= budget) return window; + + // Malformed bytes expand to replacement characters. Valid UTF-8 takes the fast path. + let low = 0; + let high = length; + while (low < high) { + const middle = Math.ceil((low + high) / 2); + if ( + Buffer.byteLength(utf8Window(buffer, middle, tail).text) <= budget + ) { + low = middle; + } else { + high = middle - 1; + } + } + return utf8Window(buffer, low, tail); +} + +/** Copies a prefix and rolling suffix, never retaining child-process chunk buffers. */ +export class OutputBuffer { + private readonly headCapacity: number; + private readonly tailCapacity: number; + private head?: Buffer; + private tail?: Buffer; + private headLength = 0; + private tailLength = 0; + private tailOffset = 0; + private totalBytes = 0; + + constructor(capacity: number) { + this.headCapacity = Math.floor(capacity / 2); + this.tailCapacity = capacity - this.headCapacity; + } + + get bytes(): number { + return this.totalBytes; + } + + append(chunk: Buffer): void { + this.totalBytes += chunk.length; + const headBytes = Math.min( + chunk.length, + this.headCapacity - this.headLength + ); + if (headBytes > 0) { + this.head ??= Buffer.allocUnsafe(this.headCapacity); + chunk.copy(this.head, this.headLength, 0, headBytes); + this.headLength += headBytes; + } + const remaining = chunk.length - headBytes; + if (remaining === 0) return; + this.tail ??= Buffer.allocUnsafe(this.tailCapacity); + if (remaining >= this.tailCapacity) { + chunk.copy(this.tail, 0, chunk.length - this.tailCapacity); + this.tailOffset = 0; + this.tailLength = this.tailCapacity; + return; + } + const first = Math.min(remaining, this.tailCapacity - this.tailOffset); + chunk.copy(this.tail, this.tailOffset, headBytes, headBytes + first); + chunk.copy(this.tail, 0, headBytes + first); + this.tailOffset = (this.tailOffset + remaining) % this.tailCapacity; + this.tailLength = Math.min( + this.tailCapacity, + this.tailLength + remaining + ); + } + + private suffix(): Buffer { + if (!this.tail) return Buffer.alloc(0); + if (this.tailLength < this.tailCapacity) + return this.tail.subarray(0, this.tailLength); + if (this.tailOffset === 0) return this.tail; + return Buffer.concat([ + this.tail.subarray(this.tailOffset), + this.tail.subarray(0, this.tailOffset), + ]); + } + + render(budget: number): { text: string; truncated: boolean } { + const head = this.head?.subarray(0, this.headLength) ?? Buffer.alloc(0); + const tail = this.suffix(); + // Before overflow the buffers are adjacent; either window may cross their boundary. + const contiguous = + this.totalBytes === head.length + tail.length + ? tail.length === 0 + ? head + : Buffer.concat([head, tail]) + : undefined; + if (contiguous && this.totalBytes <= budget) { + const text = contiguous.toString('utf8'); + if (Buffer.byteLength(text) <= budget) + return { text, truncated: false }; + } + const marker = (bytes: number): string => + `\n[... ${bytes} bytes omitted ...]\n`; + const markerBytes = Buffer.byteLength(marker(this.totalBytes)); + const includeMarker = budget >= markerBytes; + const contentBudget = includeMarker ? budget - markerBytes : budget; + const prefix = boundedWindow( + contiguous ?? head, + Math.floor(contentBudget / 2), + false + ); + const suffix = boundedWindow( + contiguous ?? tail, + Math.ceil(contentBudget / 2), + true + ); + const omitted = this.totalBytes - prefix.bytes - suffix.bytes; + return { + text: + prefix.text + + (includeMarker ? marker(omitted) : '') + + suffix.text, + truncated: omitted > 0, + }; + } +} + +/** Each stream can use the whole budget when the other is quiet; stderr cannot be starved. */ +export function renderCommandOutput( + stdout: OutputBuffer, + stderr: OutputBuffer, + budget: number +): { + stdout: string; + stderr: string; + truncated: boolean; +} { + let err = stderr.render(Math.ceil(budget / 2)); + const out = stdout.render(budget - Buffer.byteLength(err.text)); + if (!out.truncated) { + err = stderr.render(budget - Buffer.byteLength(out.text)); + } + return { + stdout: out.text, + stderr: err.text, + truncated: out.truncated || err.truncated, + }; +} From 3db258469569658480b51e618d9c078b2dda8509 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Thu, 1 Oct 2026 23:07:09 -0400 Subject: [PATCH 21/48] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20feat:=20Reuse=20Veri?= =?UTF-8?q?fied=20Environment=20Preparation=20(#286)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Reuse Verified Environment Preparation * fix: Preserve Environment Setup Recovery Diagnostics --- docs/remote-bridge/worker-runbook.md | 5 +- packages/code/README.md | 51 ++- packages/code/src/cli.ts | 68 ++-- .../code/src/environment-preparation.test.ts | 367 ++++++++++++++++++ packages/code/src/environment-preparation.ts | 150 +++++++ packages/code/src/environment.ts | 30 +- packages/code/src/storage.ts | 46 ++- 7 files changed, 688 insertions(+), 29 deletions(-) create mode 100644 packages/code/src/environment-preparation.test.ts create mode 100644 packages/code/src/environment-preparation.ts diff --git a/docs/remote-bridge/worker-runbook.md b/docs/remote-bridge/worker-runbook.md index aeeef4c4..8cbeda98 100644 --- a/docs/remote-bridge/worker-runbook.md +++ b/docs/remote-bridge/worker-runbook.md @@ -255,7 +255,10 @@ Important semantics: existing ID such as `primary` to preserve agent/conversation bindings. - `repo` and `ref` are labels. They do not clone, fetch, or check out anything. - `root` must already exist. Relative roots resolve from the definition file. -- Setup runs before registration on every worker start. It must be idempotent. +- Setup runs before registration on every worker start by default. It must be + idempotent. Optional `setup.reuse` declares fingerprint inputs and a sandboxed + readiness check to avoid reinstalling an unchanged, still-ready checkout. See + [preparation reuse](../../packages/code/README.md#reusing-a-prepared-checkout). - A setup failure or timeout prevents registration and leaves a durable quarantine marker for operator inspection. - Actions are fixed operator commands. The model selects only the action name diff --git a/packages/code/README.md b/packages/code/README.md index acae9bb8..47771057 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -1050,7 +1050,7 @@ worker runs. This inspection happens at startup, not on the command hot path. Setup is an operator-authorized startup command under the configured native sandbox policy. It requires commands to be enabled, runs once per worker startup before -registration, and must be idempotent for restarts. Its timeout is bounded to five +registration by default, and must be idempotent for restarts. Its timeout is bounded to five minutes and captured output to 8 KiB. Setup failure prevents registration. A nonzero exit, timeout, crash or uncertain termination retains the workspace quarantine marker; inspect the workspace before running `librechat-code clear-workspace-quarantine @@ -1058,6 +1058,55 @@ inspect the workspace before running `librechat-code clear-workspace-quarantine deployment and identity configuration. Only use the separate `--reset-workspace-quarantine ` run option afterward if a server fence also needs clearing. Only successful setup automatically clears its marker. + +## Reusing a prepared checkout + +Opt in to checkout-local preparation reuse when setup is an installation rather +than work that must run on every startup: + +```yaml +setup: + command: npm ci + timeoutMs: 300000 + reuse: + inputs: + - package.json + - package-lock.json + - packages/api/package.json + - packages/data-provider/package.json + checkCommand: test -f node_modules/.package-lock.json && test -x node_modules/.bin/tsc + checkTimeoutMs: 10000 +``` + +Declare **all** relevant manifests, installation configuration and lifecycle-script +inputs. There is no globbing or automatic monorepo discovery. Inputs must be +existing root-confined regular files: at most 32, 8 MiB per file and 32 MiB total. +Include an operator-maintained toolchain/version file if installation uses tools +other than the worker's Node runtime. Lockfile equality alone does not prove that +arbitrary postinstall scripts are reusable. + +The worker fingerprints declared file bytes, the setup recipe, checkout inode, +Node version/ABI, platform/architecture and native command policy. A matching +worker-owned receipt runs the readiness check instead of setup. A nonzero check +reruns setup; a timed-out, signalled or aborted check fails without starting a +replacement command. After successful setup, the check must pass and inputs must +remain unchanged before the worker publishes a receipt. Receipts are bounded, +owner-only files alongside the identity, outside all registered roots and denied +to native commands. Deleting one causes setup to run again; it never clears a +quarantine. Checks are operator commands under the same sandbox and mutation guard +as setup, not unsandboxed host scripts. Keep them cheap and non-mutating. + +Existing definitions without `reuse` retain the startup behavior. Fresh conversation +instances use the same preparation contract, with independent checkout receipts. +This does not attach another checkout's `node_modules`, provision linked lanes on +command admission, or recheck existing instances on every command. It does not +deduplicate installed dependencies between worktrees or enforce disk quotas. + +The next resource-store slice must explicitly grant shared cache paths under SRT, +keep monorepo links and mutable outputs checkout-local, and bound retention. Do not +work around that missing grant by broadening the sandbox root or symlinking another +branch's full installation. Shared download caches alone do not reduce installed +`node_modules` copies. No setup output is sent to the model. Named actions are fixed commands without model-supplied substitution. The bridge diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 410caf25..054074ed 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -2,7 +2,7 @@ import { createHash, createHmac, randomBytes } from 'node:crypto'; import { readFileSync } from 'node:fs'; import { readdir, realpath, stat } from 'node:fs/promises'; -import { basename, join, resolve, relative, isAbsolute, sep } from 'node:path'; +import { basename, dirname, join, resolve, relative, isAbsolute, sep } from 'node:path'; import { pairBridgeWorker } from './pairing.js'; import { discoverProjects } from './projects.js'; @@ -12,6 +12,7 @@ import { assertEnvironmentDefinitionsOutsideRoots, EnvironmentWorkspaceTools, } from './environment.js'; +import { prepareCodeEnvironment } from './environment-preparation.js'; import { startFileRelay } from './relay.js'; import { DockerFileRelaySupervisor } from './relay-runtime.js'; import { @@ -26,6 +27,7 @@ import { loadWorkspaceMutationQuarantine, saveBridgeIdentity, saveWorkspaceMutationQuarantine, + prepareEnvironmentPreparationDirectory, } from './storage.js'; import { BridgeWorker } from './worker.js'; import { LocalWorkspaceTools, SandboxWorkspaceTools } from './workspace.js'; @@ -826,6 +828,17 @@ async function run( }), ]), ); + const preparationDirectory = join(dirname(identityPath ?? defaultBridgeIdentityPath(workerId)), 'environment-preparation'); + if (environments.some(environment => environment.definition.setup?.reuse)) { + const sourceParents = await prepareEnvironmentPreparationDirectory(preparationDirectory); + // Reuse the mount/ancestor isolation checks for this worker-owned state directory. + await assertEnvironmentDefinitionsOutsideRoots([{ + path: preparationDirectory, sourceParents, + definition: { name: 'preparation-state', root: preparationDirectory }, fingerprint: '', + }], roots); + } + const preparationReceipt = (root: string) => join(preparationDirectory, + `${createHash('sha256').update(JSON.stringify([codeApiUrl, workerId, root])).digest('hex')}.json`); // Keep an admission boundary even when trusted-VM checkout routing uses a // nested repository's remote for the current command. const admittedGitHubRepositories = github.provider && github.repositoryRouting @@ -1048,6 +1061,7 @@ async function run( commandPolicy, protectedPaths: [ identityPath, + ...(environments.some(environment => environment.definition.setup?.reuse) ? [preparationDirectory] : []), ...environments.map(environment => environment.path), ...rootQuarantinePaths.values(), github.privateKeyPath, @@ -1151,22 +1165,20 @@ async function run( workspaceIdentity: instance.identity, workspaceRoot: instance.root, }); - const result = await nativeCommandSandbox.execute( - { + await prepareCodeEnvironment({ + root: instance.root, identity: instance.identity, setup, + receiptPath: preparationReceipt(instance.root), + context: JSON.stringify([serializeNativeSrtCommandPolicy(commandPolicy), commandAllowedDomains, github.policyIdentity]), + signal, + execute: (command, timeoutMs) => nativeCommandSandbox.execute({ protocolVersion: 1, operation: 'execute_command', workspaceId: id, - command: setup.command, - timeoutMs: setup.timeoutMs, + command, + timeoutMs, maxOutputBytes: 8192, - }, - signal, - ); - if (result.exitCode !== 0 || result.timedOut) { - throw new Error( - `Environment ${instance.sourceWorkspaceId} setup failed for its conversation worktree`, - ); - } + }, signal), + }); }, discardInstance: async (instance) => { await nativeCommandSandbox.unregisterRoot( @@ -1323,26 +1335,36 @@ async function run( incarnationId, ); await guard.assertAvailable(); - await guard.arm('Environment setup did not settle', 'setup'); - const result = await nativeCommandSandbox.execute( - { + let armed = false; + const preparation = await prepareCodeEnvironment({ + root: environment.definition.root, + identity: roots.find(root => root.id === id)!.identity!, + setup, receiptPath: preparationReceipt(environment.definition.root), + context: JSON.stringify([serializeNativeSrtCommandPolicy(commandPolicy), commandAllowedDomains, github.policyIdentity]), + signal: controller.signal, + execute: async (command, timeoutMs) => { + if (!armed) { + await guard.arm('Environment preparation did not settle', 'setup'); + armed = true; + } + return nativeCommandSandbox.execute({ protocolVersion: 1, operation: 'execute_command', workspaceId: id, - command: setup.command, - timeoutMs: setup.timeoutMs, + command, + timeoutMs, maxOutputBytes: 8192, + }, controller.signal); }, - controller.signal, - ); - if (result.exitCode !== 0 || result.timedOut) { + }).catch(error => { throw new Error( `Environment ${id} setup failed; inspect the workspace and use clear-workspace-quarantine with its root and workspace ID before restarting`, + { cause: error }, ); - } + }); await guard.clear('setup'); process.stdout.write( - `librechat-code: environment ${id} prepared\n`, + `librechat-code: environment ${id} ${preparation}\n`, ); } } catch (error) { diff --git a/packages/code/src/environment-preparation.test.ts b/packages/code/src/environment-preparation.test.ts new file mode 100644 index 00000000..51149441 --- /dev/null +++ b/packages/code/src/environment-preparation.test.ts @@ -0,0 +1,367 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { + chmod, + mkdtemp, + mkdir, + readFile, + rm, + symlink, + writeFile, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { promisify } from 'node:util'; +import test from 'node:test'; +import { prepareCodeEnvironment } from './environment-preparation.js'; +import { parseCodeEnvironment } from './environment.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; +import { prepareEnvironmentPreparationDirectory } from './storage.js'; +import type { EnvironmentPreparationOptions } from './environment-preparation.js'; +import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; + +async function fixture(t: test.TestContext) { + const directory = await mkdtemp(join(tmpdir(), 'code-preparation-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const root = join(directory, 'checkout'); + const state = join(directory, 'private'); + await mkdir(root); + await prepareEnvironmentPreparationDirectory(state); + await writeFile(join(root, 'package-lock.json'), 'lock-v1'); + const commands: string[] = []; + const options: EnvironmentPreparationOptions = { + root: (await captureWorkspaceRootIdentity(root)).path, + identity: await captureWorkspaceRootIdentity(root), + receiptPath: join(state, 'receipt.json'), + context: 'policy-v1', + setup: { + command: 'install', + timeoutMs: 1000, + reuse: { + inputs: ['package-lock.json'], + checkCommand: 'check', + checkTimeoutMs: 100, + }, + }, + async execute(command) { + commands.push(command); + return { exitCode: 0, timedOut: false }; + }, + }; + return { root, state, commands, options }; +} + +test('unchanged checkout checks readiness without repeating installation, including after restart', async t => { + const { options, commands } = await fixture(t); + assert.equal(await prepareCodeEnvironment(options), 'prepared'); + assert.equal(await prepareCodeEnvironment({ ...options }), 'reused'); + assert.deepEqual(commands, ['install', 'check', 'check']); + assert.equal( + (await readFile(options.receiptPath, 'utf8')).includes('lock-v1'), + false, + ); +}); + +test('changed inputs, recipe and toolchain/policy context invalidate preparation', async t => { + const { options, root, commands } = await fixture(t); + await prepareCodeEnvironment(options); + await writeFile(join(root, 'package-lock.json'), 'lock-v2'); + await prepareCodeEnvironment(options); + await prepareCodeEnvironment({ ...options, context: 'policy-v2' }); + await prepareCodeEnvironment({ + ...options, + context: 'policy-v2', + setup: { ...options.setup, command: 'install-v2' }, + }); + assert.deepEqual(commands, [ + 'install', + 'check', + 'install', + 'check', + 'install', + 'check', + 'install-v2', + 'check', + ]); +}); + +test('missing installed artifacts trigger repair; a failed setup is never stamped', async t => { + const { options, commands } = await fixture(t); + await prepareCodeEnvironment(options); + let installed = false; + await prepareCodeEnvironment({ + ...options, + async execute(command) { + commands.push(command); + if (command === 'install') installed = true; + return { exitCode: installed ? 0 : 1, timedOut: false }; + }, + }); + assert.deepEqual(commands, [ + 'install', + 'check', + 'check', + 'install', + 'check', + ]); + const failed = { + ...options, + receiptPath: join(options.receiptPath, '..', 'failed.json'), + execute: async () => ({ exitCode: 1, timedOut: false }), + }; + await assert.rejects(prepareCodeEnvironment(failed), /setup failed/); + await assert.rejects(readFile(failed.receiptPath), { code: 'ENOENT' }); +}); + +test('timed-out readiness and cancellation do not fall through to another setup', async t => { + const { options, commands } = await fixture(t); + await prepareCodeEnvironment(options); + await assert.rejects( + prepareCodeEnvironment({ + ...options, + execute: async command => { + commands.push(command); + return { exitCode: 0, timedOut: true }; + }, + }), + /did not settle/, + ); + assert.deepEqual(commands, ['install', 'check', 'check']); + const controller = new AbortController(); + await assert.rejects( + prepareCodeEnvironment({ + ...options, + signal: controller.signal, + execute: async () => { + controller.abort(); + return { exitCode: 0, timedOut: false }; + }, + }), + { name: 'AbortError' }, + ); +}); + +test('changing inputs during setup or readiness never publishes success', async t => { + const { options, root } = await fixture(t); + await assert.rejects( + prepareCodeEnvironment({ + ...options, + execute: async () => { + await writeFile(join(root, 'package-lock.json'), 'changed'); + return { exitCode: 0, timedOut: false }; + }, + }), + /inputs changed/, + ); + await assert.rejects(readFile(options.receiptPath), { code: 'ENOENT' }); + await assert.rejects( + prepareCodeEnvironment({ + ...options, + execute: async command => { + if (command === 'check') + await writeFile( + join(root, 'package-lock.json'), + 'changed-again', + ); + return { exitCode: 0, timedOut: false }; + }, + }), + /inputs changed/, + ); +}); + +test('input hashing is bounded and refuses symlink traversal and non-files', async t => { + const { options, root } = await fixture(t); + const outside = join(root, '..', 'outside'); + await mkdir(outside); + await writeFile(join(outside, 'secret'), 'do-not-read'); + await symlink(outside, join(root, 'escape')); + for (const path of ['escape/secret', 'escape']) { + await assert.rejects( + prepareCodeEnvironment({ + ...options, + setup: { + ...options.setup, + reuse: { ...options.setup.reuse!, inputs: [path] }, + }, + }), + ); + } + await writeFile(join(root, 'big'), Buffer.alloc(8 * 1024 * 1024 + 1)); + await assert.rejects( + prepareCodeEnvironment({ + ...options, + setup: { + ...options.setup, + reuse: { ...options.setup.reuse!, inputs: ['big'] }, + }, + }), + /bounded regular-file/, + ); +}); + +test('legacy setup always runs and does not create preparation state', async t => { + const { options, commands } = await fixture(t); + const legacy = { + ...options, + setup: { command: 'install', timeoutMs: 1000 }, + }; + await prepareCodeEnvironment(legacy); + await prepareCodeEnvironment(legacy); + assert.deepEqual(commands, ['install', 'install']); + await assert.rejects(readFile(options.receiptPath), { code: 'ENOENT' }); +}); + +test('a foreign checkout receipt is not installation evidence and exposed state fails closed', async t => { + const { options, commands } = await fixture(t); + await prepareCodeEnvironment(options); + const other = join(options.root, '..', 'other'); + await mkdir(other); + await writeFile(join(other, 'package-lock.json'), 'lock-v1'); + await prepareCodeEnvironment({ + ...options, + root: (await captureWorkspaceRootIdentity(other)).path, + identity: await captureWorkspaceRootIdentity(other), + }); + assert.deepEqual(commands, ['install', 'check', 'install', 'check']); + await chmod(options.receiptPath, 0o644); + await assert.rejects( + prepareCodeEnvironment({ + ...options, + root: (await captureWorkspaceRootIdentity(other)).path, + identity: await captureWorkspaceRootIdentity(other), + }), + /owner-only/, + ); +}); + +test('reuse YAML requires bounded explicit inputs and readiness check', () => { + assert.ok( + parseCodeEnvironment( + 'name: app\nroot: app\nsetup:\n command: npm ci\n reuse:\n inputs: [package.json, package-lock.json]\n checkCommand: test -d node_modules\n', + ).setup?.reuse, + ); + for (const reuse of [ + 'inputs: []\ncheckCommand: ready', + 'inputs: [../outside]\ncheckCommand: ready', + 'inputs: [package.json, package.json]\ncheckCommand: ready', + 'inputs: [package.json]', + 'inputs: [package.json]\ncheckCommand: ready\ncheckTimeoutMs: 0', + 'inputs: [package.json]\ncheckCommand: ready\nunknown: true', + ]) + assert.throws(() => + parseCodeEnvironment( + `name: app\nroot: app\nsetup:\n command: install\n reuse:\n${reuse + .split('\n') + .map(line => ` ${line}`) + .join('\n')}\n`, + ), + ); +}); + +test('real local shell canary: installation reused, deleted output repaired, changed lockfile reinstalled', async t => { + const { options } = await fixture(t); + const run = promisify(execFile); + const ready = join(options.root, 'installed'); + const configured: EnvironmentPreparationOptions = { + ...options, + setup: { + command: 'printf "install\\n" >> installs.log; mkdir -p installed', + timeoutMs: 1000, + reuse: { + inputs: ['package-lock.json'], + checkCommand: 'test -d installed', + checkTimeoutMs: 1000, + }, + }, + async execute(command, timeout) { + try { + await run('/bin/sh', ['-c', command], { + cwd: options.root, + timeout, + }); + return { exitCode: 0, timedOut: false }; + } catch (error) { + return { + exitCode: 1, + timedOut: Boolean((error as { killed?: boolean }).killed), + }; + } + }, + }; + await prepareCodeEnvironment(configured); + assert.equal(await prepareCodeEnvironment(configured), 'reused'); + await rm(ready, { recursive: true }); + await prepareCodeEnvironment(configured); + await writeFile(resolve(options.root, 'package-lock.json'), 'new-lock'); + await prepareCodeEnvironment(configured); + assert.equal( + await readFile(join(options.root, 'installs.log'), 'utf8'), + 'install\ninstall\ninstall\n', + ); +}); + +test( + 'real native SRT preparation reuse keeps receipts inaccessible to checkout commands', + { + skip: process.env.LIBRECHAT_CODE_LIVE_SRT_TESTS !== '1', + timeout: 30_000, + }, + async t => { + const { options, state } = await fixture(t); + const sandbox = new NativeProcessWorkspaceCommandSandbox({ + workspaceRoot: options.root, + workspaceIdentity: options.identity, + protectedPaths: [state], + }); + t.after(() => sandbox.close()); + await sandbox.prepare(); + const execute: EnvironmentPreparationOptions['execute'] = ( + command, + timeoutMs, + ) => + sandbox.execute({ + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command, + timeoutMs, + maxOutputBytes: 1024, + }); + const configured = { + ...options, + execute, + setup: { + command: + 'printf "install\\n" >> installs.log; mkdir -p installed', + timeoutMs: 5000, + reuse: { + inputs: ['package-lock.json'], + checkCommand: 'test -d installed', + checkTimeoutMs: 5000, + }, + }, + }; + assert.equal(await prepareCodeEnvironment(configured), 'prepared'); + assert.equal(await prepareCodeEnvironment(configured), 'reused'); + assert.notEqual( + (await execute(`cat '${options.receiptPath}'`, 5000)).exitCode, + 0, + ); + assert.notEqual( + (await execute(`printf forged > '${options.receiptPath}'`, 5000)) + .exitCode, + 0, + ); + assert.equal( + await readFile(join(options.root, 'installs.log'), 'utf8'), + 'install\n', + ); + await rm(join(options.root, 'installed'), { recursive: true }); + assert.equal(await prepareCodeEnvironment(configured), 'prepared'); + assert.equal( + await readFile(join(options.root, 'installs.log'), 'utf8'), + 'install\ninstall\n', + ); + }, +); diff --git a/packages/code/src/environment-preparation.ts b/packages/code/src/environment-preparation.ts new file mode 100644 index 00000000..c2597f3b --- /dev/null +++ b/packages/code/src/environment-preparation.ts @@ -0,0 +1,150 @@ +import { createHash } from 'node:crypto'; +import { constants } from 'node:fs'; +import { resolve } from 'node:path'; +import { open, withWorkspaceRoot } from './root-access.js'; +import { + loadEnvironmentPreparationKey, + saveEnvironmentPreparationKey, +} from './storage.js'; +import type { CodeEnvironmentDefinition } from './environment.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; + +// Safety bounds on operator-declared hashing, not a dependency-store quota. +const MAX_INPUT_BYTES = 8 * 1024 * 1024; +const MAX_TOTAL_INPUT_BYTES = 32 * 1024 * 1024; + +export interface EnvironmentPreparationOptions { + root: string; + identity: WorkspaceRootIdentity; + setup: NonNullable; + receiptPath: string; + /** Includes the worker's policy and toolchain configuration. */ + context: string; + execute( + command: string, + timeoutMs: number, + ): Promise<{ exitCode: number | null; timedOut: boolean }>; + signal?: AbortSignal; +} + +/** Checkout-local reuse. Never transfers mutable installations between worktrees. */ +export async function prepareCodeEnvironment( + options: EnvironmentPreparationOptions, +): Promise<'prepared' | 'reused'> { + options.signal?.throwIfAborted(); + const key = await preparationKey(options); + if ( + key && + (await loadEnvironmentPreparationKey(options.receiptPath)) === key + ) { + const reuse = options.setup.reuse!; + const check = await options.execute( + reuse.checkCommand, + reuse.checkTimeoutMs, + ); + options.signal?.throwIfAborted(); + if (check.timedOut || check.exitCode === null) + throw new Error('Environment readiness check did not settle'); + if (check.exitCode === 0 && (await preparationKey(options)) === key) + return 'reused'; + } + const result = await options.execute( + options.setup.command, + options.setup.timeoutMs, + ); + options.signal?.throwIfAborted(); + if (result.timedOut || result.exitCode !== 0) + throw new Error('Environment setup failed'); + if (key) { + // Do not stamp an installation against inputs that changed during setup. + if ((await preparationKey(options)) !== key) + throw new Error( + 'Environment preparation inputs changed during setup', + ); + const reuse = options.setup.reuse!; + const check = await options.execute( + reuse.checkCommand, + reuse.checkTimeoutMs, + ); + options.signal?.throwIfAborted(); + if (check.timedOut || check.exitCode !== 0) + throw new Error('Environment readiness check failed after setup'); + if ((await preparationKey(options)) !== key) + throw new Error( + 'Environment preparation inputs changed during readiness check', + ); + await saveEnvironmentPreparationKey(options.receiptPath, key); + } + return 'prepared'; +} + +async function preparationKey( + options: EnvironmentPreparationOptions, +): Promise { + if (!options.setup.reuse) return undefined; + return withWorkspaceRoot(options.root, options.identity, async () => { + const hash = createHash('sha256').update( + JSON.stringify({ + version: 1, + root: options.identity, + setup: options.setup, + context: options.context, + node: process.version, + abi: process.versions.modules, + platform: process.platform, + arch: process.arch, + }), + ); + let total = 0; + for (const path of options.setup.reuse!.inputs) { + options.signal?.throwIfAborted(); + const handle = await open( + resolve(options.root, path), + constants.O_RDONLY | + constants.O_NOFOLLOW | + constants.O_NONBLOCK, + ); + try { + const before = await handle.stat(); + if ( + !before.isFile() || + before.size > MAX_INPUT_BYTES || + total + before.size > MAX_TOTAL_INPUT_BYTES + ) + throw new Error( + 'Environment preparation inputs exceed the bounded regular-file contract', + ); + const buffer = Buffer.alloc(before.size + 1); + let length = 0; + while (length < buffer.length) { + options.signal?.throwIfAborted(); + const read = await handle.read( + buffer, + length, + buffer.length - length, + length, + ); + if (!read.bytesRead) break; + length += read.bytesRead; + } + const after = await handle.stat(); + if ( + length !== before.size || + after.size !== before.size || + after.mtimeMs !== before.mtimeMs || + after.ctimeMs !== before.ctimeMs + ) + throw new Error( + 'Environment preparation input changed while hashing', + ); + total += length; + hash.update(JSON.stringify([path, length])).update( + buffer.subarray(0, length), + ); + } finally { + await handle.close(); + } + } + return hash.digest('hex'); + }); +} diff --git a/packages/code/src/environment.ts b/packages/code/src/environment.ts index c2b2cf1f..406e3850 100644 --- a/packages/code/src/environment.ts +++ b/packages/code/src/environment.ts @@ -10,6 +10,7 @@ import { import { BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, BRIDGE_WORKSPACE_COMMAND_MAX_BYTES, + isSafePortableRelativePath, } from './protocol.js'; import type { LocalWorkspaceConfig } from './workspace.js'; import { WorkspaceToolError } from './workspace.js'; @@ -25,7 +26,12 @@ export interface CodeEnvironmentDefinition { root: string; repo?: string; ref?: string; - setup?: { command: string; timeoutMs: number }; + setup?: { + command: string; + timeoutMs: number; + /** Explicit readiness contract; absent preserves startup setup behavior. */ + reuse?: { inputs: string[]; checkCommand: string; checkTimeoutMs: number }; + }; actions?: { name: string; command: string; timeoutMs: number }[]; } @@ -89,7 +95,7 @@ export function parseCodeEnvironment( if ( !record(value.setup) || Object.keys(value.setup).some( - key => !['command', 'timeoutMs'].includes(key), + key => !['command', 'timeoutMs', 'reuse'].includes(key), ) || !text(value.setup.command, 16_384) || Buffer.byteLength(value.setup.command) > @@ -109,7 +115,25 @@ export function parseCodeEnvironment( `Environment setup timeout must be between 1 and ${BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS} ms`, ); } - setup = { command: value.setup.command, timeoutMs }; + let reuse: NonNullable['reuse']; + if (value.setup.reuse !== undefined) { + const candidate = value.setup.reuse; + if ( + !record(candidate) || + Object.keys(candidate).some(key => !['inputs', 'checkCommand', 'checkTimeoutMs'].includes(key)) || + !Array.isArray(candidate.inputs) || + candidate.inputs.length < 1 || candidate.inputs.length > 32 || + candidate.inputs.some(path => typeof path !== 'string' || !isSafePortableRelativePath(path) || path === '.') || + new Set(candidate.inputs).size !== candidate.inputs.length || + !text(candidate.checkCommand, 16_384) || + Buffer.byteLength(candidate.checkCommand) > BRIDGE_WORKSPACE_COMMAND_MAX_BYTES + ) throw new Error('Invalid environment setup reuse'); + const checkTimeoutMs = candidate.checkTimeoutMs ?? 10_000; + if (typeof checkTimeoutMs !== 'number' || !Number.isSafeInteger(checkTimeoutMs) || checkTimeoutMs < 1 || checkTimeoutMs > BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS) + throw new Error('Invalid environment readiness check timeout'); + reuse = { inputs: candidate.inputs as string[], checkCommand: candidate.checkCommand, checkTimeoutMs }; + } + setup = { command: value.setup.command, timeoutMs, ...(reuse ? { reuse } : {}) }; } let actions: CodeEnvironmentDefinition['actions']; if (value.actions !== undefined) { diff --git a/packages/code/src/storage.ts b/packages/code/src/storage.ts index 1cbb7093..904a8562 100644 --- a/packages/code/src/storage.ts +++ b/packages/code/src/storage.ts @@ -1,4 +1,5 @@ import { createHash, randomBytes } from 'node:crypto'; +import { constants } from 'node:fs'; import { chmod, lstat, @@ -201,11 +202,15 @@ async function assertOwnedByWorker(path: string): Promise { async function readGuardedFile( path: string, exposed: (mode: string) => string, + maxBytes = Infinity, ): Promise { await assertReadPathPrivate(path); const handle = await open(path, 'r'); try { const stats = await handle.stat(); + if (!stats.isFile() || stats.size > maxBytes) { + throw new BridgeProtocolError('Invalid private storage file size or type'); + } const self = process.getuid?.(); if (self !== undefined && !isTrustedOwner(stats.uid, self)) { throw new BridgeProtocolError( @@ -423,6 +428,44 @@ export async function saveBridgeIdentity( path: string, identity: PairedBridgeWorkerIdentity, ): Promise { + return savePrivateJson(path, identity); +} + +/** Preparation receipts are worker state, never a workspace-authored success marker. */ +export async function prepareEnvironmentPreparationDirectory(path: string): Promise { + await assertPrivateStorageAncestors(path, true); + await ensureDurableDirectory(path); + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const metadata = await handle.stat(); + if (!metadata.isDirectory() || (metadata.mode & 0o077) !== 0) + throw new BridgeProtocolError('Environment preparation directory must be owner-only'); + await assertPrivateStorageAcl(handle, path, true); + } finally { await handle.close(); } + return assertPrivateStorageAncestors(path); +} + +export async function loadEnvironmentPreparationKey(path: string): Promise { + let content: string; + try { + content = await readGuardedFile(path, () => 'Environment preparation receipt must be owner-only', 256); + } catch (error) { + if (isMissingPathError(error)) return undefined; + throw error; + } + try { + const value: unknown = JSON.parse(content); + if (isRecord(value) && value.version === 1 && typeof value.key === 'string' && /^[a-f0-9]{64}$/.test(value.key)) return value.key; + } catch { /* Invalid receipts are cache misses, not readiness evidence. */ } + return undefined; +} + +export async function saveEnvironmentPreparationKey(path: string, key: string): Promise { + if (!/^[a-f0-9]{64}$/.test(key)) throw new Error('Invalid environment preparation key'); + await savePrivateJson(path, { version: 1, key }); +} + +async function savePrivateJson(path: string, value: unknown): Promise { assertPrivateStorageSupported(); await assertWriteContainerPrivate(path); await mkdir(dirname(path), { recursive: true, mode: 0o700 }); @@ -435,13 +478,14 @@ export async function saveBridgeIdentity( await removePrivateStorageAcl(file, path); await file.chmod(0o600); await assertOwnerOnlyFile(file, path); - await file.writeFile(`${JSON.stringify(identity, null, 2)}\n`, 'utf8'); + await file.writeFile(`${JSON.stringify(value, null, 2)}\n`, 'utf8'); await file.sync(); } finally { await file.close(); } await rename(temporaryPath, path); await chmod(path, 0o600); + await syncParentDirectory(path); } catch (error) { await rm(temporaryPath, { force: true }); throw error; From 42a7fcd7cb02596cbb9ee861e1e24315b1163a24 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 2 Oct 2026 07:24:03 -0400 Subject: [PATCH 22/48] =?UTF-8?q?=F0=9F=A7=B0=20feat:=20Share=20Explicit?= =?UTF-8?q?=20Environment=20Tool=20Resources=20(#287)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Grant Explicit Shared Tool Cache Resources * fix: Reject Replaced Tool Stores Before Dispatch * test: Verify Protected Receipt Contents Across Native Platforms --- packages/code/README.md | 46 +++- packages/code/src/cli.ts | 31 ++- .../code/src/environment-preparation.test.ts | 24 +- .../code/src/environment-resources.test.ts | 210 ++++++++++++++++++ packages/code/src/environment-resources.ts | 164 ++++++++++++++ packages/code/src/environment.ts | 8 +- packages/code/src/native-process.ts | 2 + packages/code/src/native-sandbox.ts | 23 ++ 8 files changed, 482 insertions(+), 26 deletions(-) create mode 100644 packages/code/src/environment-resources.test.ts create mode 100644 packages/code/src/environment-resources.ts diff --git a/packages/code/README.md b/packages/code/README.md index 47771057..47d3c3a0 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -1102,11 +1102,47 @@ This does not attach another checkout's `node_modules`, provision linked lanes o command admission, or recheck existing instances on every command. It does not deduplicate installed dependencies between worktrees or enforce disk quotas. -The next resource-store slice must explicitly grant shared cache paths under SRT, -keep monorepo links and mutable outputs checkout-local, and bound retention. Do not -work around that missing grant by broadening the sandbox root or symlinking another -branch's full installation. Shared download caches alone do not reduce installed -`node_modules` copies. +Shared tool cache grants below do not attach another checkout's installed dependency +tree. Keep monorepo links and mutable outputs checkout-local. Do not broaden the +sandbox root or symlink another branch's full installation. Shared download caches +alone do not reduce installed `node_modules` copies. + +## Shared tool and download resources + +Explicit operator-managed stores can live outside the checkouts: + +```yaml +resources: + - kind: npm-cache + path: /srv/lia-resources/npm + access: read-write + - kind: uv-cache + path: /srv/lia-resources/uv + access: read-write + - kind: playwright-browsers + path: /srv/lia-resources/playwright + access: read-only +``` + +Create each directory as the worker service account with mode `0700`, then populate +browser binaries using the matching Playwright version and `PLAYWRIGHT_BROWSERS_PATH` +outside the coding session. Roots must exist and may not be symlinks or overlap any +registered workspace or worker control state. Linux mount-alias checks cover both +directions. Keep the mount namespace stable while the worker runs. + +The worker grants only these paths and injects `npm_config_cache`, `UV_CACHE_DIR` +and `PLAYWRIGHT_BROWSERS_PATH` from the loaded definition, including in linked lanes +and fresh conversation worktrees. An undeclared process environment variable never +grants filesystem access. Each store root is inode-bound and revalidated before +commands. Read-only stores stay read-only, and speculative programmatic probes +cannot write any shared store. + +Sharing is explicit within one worker's trust domain. Do not share mutable caches +between unrelated principals, store credentials in them, or treat their contents +as trusted worker code. Separate package caches from browsers and mutable browser +profiles, Redis data, build output and test state. Stores currently have no automatic +eviction; the storage-lifecycle slice adds that separately. This shares downloads +and browser binaries, not installed `node_modules` trees. No setup output is sent to the model. Named actions are fixed commands without model-supplied substitution. The bridge diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 054074ed..04d94aaa 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -13,6 +13,7 @@ import { EnvironmentWorkspaceTools, } from './environment.js'; import { prepareCodeEnvironment } from './environment-preparation.js'; +import { assertEnvironmentResourceIsolation } from './environment-resources.js'; import { startFileRelay } from './relay.js'; import { DockerFileRelaySupervisor } from './relay-runtime.js'; import { @@ -837,6 +838,16 @@ async function run( definition: { name: 'preparation-state', root: preparationDirectory }, fingerprint: '', }], roots); } + // Cache contents may be shared explicitly; their grants must never authorize source or control state. + const resourceRoots = environments.flatMap(environment => (environment.resources ?? []).map(resource => ({ + path: resource.path, sourceParents: resource.controlPaths, + definition: { name: 'shared-resource', root: resource.path }, fingerprint: '', + }))); + await assertEnvironmentResourceIsolation(environments.flatMap(environment => environment.resources ?? []), roots.map(root => root.root), [ + identityPath, preparationDirectory, github.privateKeyPath, + ...environments.map(environment => environment.path), ...rootQuarantinePaths.values(), + ].filter((path): path is string => path != null)); + await assertEnvironmentDefinitionsOutsideRoots(resourceRoots, roots); const preparationReceipt = (root: string) => join(preparationDirectory, `${createHash('sha256').update(JSON.stringify([codeApiUrl, workerId, root])).digest('hex')}.json`); // Keep an admission boundary even when trusted-VM checkout routing uses a @@ -1110,6 +1121,10 @@ async function run( } : {}), }; + const nativeOptionsForWorkspace = (workspaceId: string): NativeProcessSandboxOptions => ({ + ...nativeOptions, + resources: environments.find(environment => environment.definition.name === workspaceId)?.resources, + }); const nativeCommandSandbox = allowWorkspaceCommands && commandSandboxMode === 'native-srt' ? roots.length > 1 || workspaceLeaseSlots > 1 || conversationWorktreeRoot || linkedWorktreeLanes @@ -1117,12 +1132,12 @@ async function run( new Map( roots.map(root => [ root.id, - { ...nativeOptions, workspaceRoot: root.root, workspaceIdentity: root.identity }, + { ...nativeOptionsForWorkspace(root.id), workspaceRoot: root.root, workspaceIdentity: root.identity }, ]), ), workspaceLeaseSlots, ) - : new NativeProcessWorkspaceCommandSandbox(nativeOptions) + : new NativeProcessWorkspaceCommandSandbox(nativeOptionsForWorkspace(roots[0].id)) : undefined; if (allowWorkspaceCommands && workspaceTools) { workspaceTools = new SandboxWorkspaceTools({ @@ -1161,14 +1176,15 @@ async function run( } const id = internalWorkspaceId(instance.sourceWorkspaceId, instance.id); await nativeCommandSandbox.registerRoot(id, { - ...nativeOptions, + ...nativeOptionsForWorkspace(instance.sourceWorkspaceId), workspaceIdentity: instance.identity, workspaceRoot: instance.root, }); await prepareCodeEnvironment({ root: instance.root, identity: instance.identity, setup, receiptPath: preparationReceipt(instance.root), - context: JSON.stringify([serializeNativeSrtCommandPolicy(commandPolicy), commandAllowedDomains, github.policyIdentity]), + context: JSON.stringify([serializeNativeSrtCommandPolicy(commandPolicy), commandAllowedDomains, github.policyIdentity, + nativeOptionsForWorkspace(instance.sourceWorkspaceId).resources]), signal, execute: (command, timeoutMs) => nativeCommandSandbox.execute({ protocolVersion: 1, @@ -1225,7 +1241,7 @@ async function run( root.id, { command: { - ...nativeOptions, + ...nativeOptionsForWorkspace(root.id), workspaceIdentity: root.identity, workspaceRoot: root.root, }, @@ -1263,7 +1279,7 @@ async function run( { root: root.root, identity: root.identity, - command: nativeOptions, + command: nativeOptionsForWorkspace(root.id), repositoryInstructions: args.includes('--repository-instructions'), writable: root.writable ?? false, }, @@ -1340,7 +1356,8 @@ async function run( root: environment.definition.root, identity: roots.find(root => root.id === id)!.identity!, setup, receiptPath: preparationReceipt(environment.definition.root), - context: JSON.stringify([serializeNativeSrtCommandPolicy(commandPolicy), commandAllowedDomains, github.policyIdentity]), + context: JSON.stringify([serializeNativeSrtCommandPolicy(commandPolicy), commandAllowedDomains, github.policyIdentity, + nativeOptionsForWorkspace(id).resources]), signal: controller.signal, execute: async (command, timeoutMs) => { if (!armed) { diff --git a/packages/code/src/environment-preparation.test.ts b/packages/code/src/environment-preparation.test.ts index 51149441..55186064 100644 --- a/packages/code/src/environment-preparation.test.ts +++ b/packages/code/src/environment-preparation.test.ts @@ -25,7 +25,7 @@ async function fixture(t: test.TestContext) { t.after(() => rm(directory, { recursive: true, force: true })); const root = join(directory, 'checkout'); const state = join(directory, 'private'); - await mkdir(root); + await mkdir(root, { mode: 0o700 }); await prepareEnvironmentPreparationDirectory(state); await writeFile(join(root, 'package-lock.json'), 'lock-v1'); const commands: string[] = []; @@ -316,9 +316,9 @@ test( }); t.after(() => sandbox.close()); await sandbox.prepare(); - const execute: EnvironmentPreparationOptions['execute'] = ( - command, - timeoutMs, + const execute = ( + command: string, + timeoutMs: number, ) => sandbox.execute({ protocolVersion: 1, @@ -344,15 +344,13 @@ test( }; assert.equal(await prepareCodeEnvironment(configured), 'prepared'); assert.equal(await prepareCodeEnvironment(configured), 'reused'); - assert.notEqual( - (await execute(`cat '${options.receiptPath}'`, 5000)).exitCode, - 0, - ); - assert.notEqual( - (await execute(`printf forged > '${options.receiptPath}'`, 5000)) - .exitCode, - 0, - ); + const receipt = await readFile(options.receiptPath, 'utf8'); + const protectedRead = await execute(`cat '${options.receiptPath}'`, 5000); + // Linux SRT may mask denied reads with an empty file. Exit status alone + // is not evidence that the protected receipt was exposed. + assert.ok(!protectedRead.stdout?.includes(receipt)); + await execute(`printf forged > '${options.receiptPath}'`, 5000); + assert.equal(await readFile(options.receiptPath, 'utf8'), receipt); assert.equal( await readFile(join(options.root, 'installs.log'), 'utf8'), 'install\n', diff --git a/packages/code/src/environment-resources.test.ts b/packages/code/src/environment-resources.test.ts new file mode 100644 index 00000000..32d1d390 --- /dev/null +++ b/packages/code/src/environment-resources.test.ts @@ -0,0 +1,210 @@ +import assert from 'node:assert/strict'; +import { + chmod, + mkdir, + mkdtemp, + readFile, + rename, + rm, + symlink, + writeFile, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { + parseCodeEnvironment, + loadCodeEnvironment, + assertEnvironmentDefinitionsOutsideRoots, +} from './environment.js'; +import { + parseEnvironmentResources, + loadEnvironmentResource, + assertEnvironmentResourcesStable, +} from './environment-resources.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; +import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; + +test('shared resources are explicit bounded known cache kinds, not arbitrary environment or filesystem grants', () => { + assert.equal( + parseCodeEnvironment( + 'name: app\nroot: app\nresources:\n - kind: npm-cache\n path: /cache/npm\n access: read-write\n', + ).resources?.[0].kind, + 'npm-cache', + ); + for (const value of [ + [], + [{ kind: 'secrets', path: '/cache', access: 'read-write' }], + [{ kind: 'npm-cache', path: '../cache', access: 'read-write' }], + [{ kind: 'uv-cache', path: '/cache', access: 'all' }], + [ + { + kind: 'uv-cache', + path: '/cache', + access: 'read-only', + env: 'GH_TOKEN', + }, + ], + [1], + [ + { kind: 'uv-cache', path: '/cache', access: 'read-only' }, + { kind: 'uv-cache', path: '/other', access: 'read-only' }, + ], + ]) + assert.throws(() => parseEnvironmentResources(value)); +}); + +test('resource roots must be stable private directories and outside source grants', async t => { + const directory = await mkdtemp(join(tmpdir(), 'code-resources-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const root = join(directory, 'root'); + const cache = join(directory, 'cache'); + await mkdir(root, { mode: 0o700 }); + await mkdir(cache, { mode: 0o700 }); + const resource = { + kind: 'npm-cache' as const, + path: cache, + access: 'read-write' as const, + }; + const loaded = await loadEnvironmentResource(resource); + await assertEnvironmentResourcesStable([loaded]); + await chmod(cache, 0o755); + await assert.rejects(loadEnvironmentResource(resource), /owner-only/); + await chmod(cache, 0o700); + await symlink(cache, join(directory, 'alias')); + await assert.rejects( + loadEnvironmentResource({ + ...resource, + path: join(directory, 'alias'), + }), + /symlink/, + ); + await rename(cache, join(directory, 'old')); + await mkdir(cache, { mode: 0o700 }); + await assert.rejects(assertEnvironmentResourcesStable([loaded]), /changed/); + const file = join(directory, 'app.yaml'); + await writeFile( + file, + `name: app\nroot: ${root}\nresources:\n - kind: npm-cache\n path: ${cache}\n access: read-write\n`, + { mode: 0o600 }, + ); + const environment = await loadCodeEnvironment(file); + const [cacheRoot] = environment.resources!; + await assert.rejects( + assertEnvironmentDefinitionsOutsideRoots( + [ + { + path: cacheRoot.path, + sourceParents: cacheRoot.controlPaths, + fingerprint: '', + definition: { name: 'cache', root: cacheRoot.path }, + }, + ], + [{ id: 'root', root: directory }], + ), + /workspace|root|definition/i, + ); +}); + +test( + 'real native SRT shares only declared stores across independent checkouts and refuses readonly writes', + { + skip: process.env.LIBRECHAT_CODE_LIVE_SRT_TESTS !== '1', + timeout: 30_000, + }, + async t => { + const directory = await mkdtemp(join(tmpdir(), 'code-resources-live-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const cache = join(directory, 'cache'); + const browser = join(directory, 'browsers'); + await mkdir(cache, { mode: 0o700 }); + await mkdir(browser, { mode: 0o700 }); + await writeFile(join(browser, 'version'), 'pinned-browser'); + const resources = await Promise.all([ + loadEnvironmentResource({ + kind: 'npm-cache', + path: cache, + access: 'read-write', + }), + loadEnvironmentResource({ + kind: 'playwright-browsers', + path: browser, + access: 'read-only', + }), + ]); + const sandboxes = await Promise.all( + ['a', 'b'].map(async name => { + const root = join(directory, name); + await mkdir(root); + const identity = await captureWorkspaceRootIdentity(root); + await writeFile(join(root, 'private'), 'other-checkout'); + const sandbox = new NativeProcessWorkspaceCommandSandbox({ + workspaceRoot: identity.path, + workspaceIdentity: identity, + homeDirectory: directory, + resources, + }); + await sandbox.prepare(); + return sandbox; + }), + ); + t.after(async () => { + for (const sandbox of sandboxes) await sandbox.close(); + }); + const execute = (index: number, command: string) => + sandboxes[index].execute({ + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command, + timeoutMs: 5000, + }); + const results = await Promise.all( + sandboxes.map((_, i) => + execute( + i, + 'printf "%s\\n" "$npm_config_cache"; printf once >> "$npm_config_cache/marker"; cat "$PLAYWRIGHT_BROWSERS_PATH/version"', + ), + ), + ); + for (const result of results) { + assert.equal(result.exitCode, 0, result.stderr); + assert.match(result.stdout, /pinned-browser/); + } + assert.equal(await readFile(join(cache, 'marker'), 'utf8'), 'onceonce'); + assert.notEqual( + ( + await execute( + 0, + 'printf forbidden > "$PLAYWRIGHT_BROWSERS_PATH/version"', + ) + ).exitCode, + 0, + ); + assert.equal( + await readFile(join(browser, 'version'), 'utf8'), + 'pinned-browser', + ); + assert.notEqual( + (await execute(0, `cat '${join(directory, 'b', 'private')}'`)) + .exitCode, + 0, + ); + await rename(cache, join(directory, 'old-cache')); + await mkdir(cache, { mode: 0o700 }); + await assert.rejects( + execute(0, 'printf not-started > side-effect'), + error => { + assert.equal( + (error as { mutationMayHaveCommitted: boolean }) + .mutationMayHaveCommitted, + false, + ); + return /resource changed/.test((error as Error).message); + }, + ); + await assert.rejects(readFile(join(directory, 'a', 'side-effect')), { + code: 'ENOENT', + }); + }, +); diff --git a/packages/code/src/environment-resources.ts b/packages/code/src/environment-resources.ts new file mode 100644 index 00000000..1bb57440 --- /dev/null +++ b/packages/code/src/environment-resources.ts @@ -0,0 +1,164 @@ +import { lstat, open, realpath } from 'node:fs/promises'; +import { constants } from 'node:fs'; +import { isAbsolute, relative, sep } from 'node:path'; +import { + assertPrivateStorageAcl, + assertPrivateStorageAncestors, +} from './private-storage.js'; +import { + captureWorkspaceRootIdentity, + matchesWorkspaceRoot, +} from './root-identity.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; +import { + createEnvironmentMountIsolation, + readEnvironmentMountTable, +} from './environment-mount.js'; + +export type EnvironmentResourceKind = + | 'npm-cache' + | 'uv-cache' + | 'playwright-browsers'; +export interface EnvironmentResource { + kind: EnvironmentResourceKind; + path: string; + access: 'read-only' | 'read-write'; +} +export interface LoadedEnvironmentResource extends EnvironmentResource { + identity: WorkspaceRootIdentity; + controlPaths: string[]; +} +export const RESOURCE_ENVIRONMENT_NAMES: Record< + EnvironmentResourceKind, + string +> = { + 'npm-cache': 'npm_config_cache', + 'uv-cache': 'UV_CACHE_DIR', + 'playwright-browsers': 'PLAYWRIGHT_BROWSERS_PATH', +}; + +export function parseEnvironmentResources( + value: unknown, +): EnvironmentResource[] { + if (!Array.isArray(value) || value.length < 1 || value.length > 3) + throw new Error('Invalid environment resources'); + const kinds = new Set(); + return value.map(raw => { + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) + throw new Error('Invalid environment resource'); + const entry = raw as Record; + if ( + Object.keys(entry).some( + key => !['kind', 'path', 'access'].includes(key), + ) || + typeof entry.kind !== 'string' || + !Object.hasOwn(RESOURCE_ENVIRONMENT_NAMES, entry.kind) || + kinds.has(entry.kind) || + typeof entry.path !== 'string' || + !isAbsolute(entry.path) || + entry.path.length > 4096 || + /[\0\r\n]/.test(entry.path) || + (entry.access !== 'read-only' && entry.access !== 'read-write') + ) + throw new Error('Invalid environment resource'); + kinds.add(entry.kind); + return { + kind: entry.kind as EnvironmentResourceKind, + path: entry.path, + access: entry.access, + }; + }); +} + +/** Operator-owned roots only. Resource contents are still untrusted tool data. */ +export async function loadEnvironmentResource( + resource: EnvironmentResource, +): Promise { + const controlPaths = await assertPrivateStorageAncestors(resource.path); + if ((await lstat(resource.path)).isSymbolicLink()) + throw new Error('Environment resource root cannot be a symlink'); + const path = await realpath(resource.path); + const identity = await captureWorkspaceRootIdentity(path); + const handle = await open( + path, + constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW, + ); + try { + const metadata = await handle.stat({ bigint: true }); + if ( + (metadata.mode & 0o077n) !== 0n || + metadata.dev.toString() !== identity.dev || + metadata.ino.toString() !== identity.ino + ) + throw new Error( + 'Environment resource directory must be stable and owner-only', + ); + await assertPrivateStorageAcl(handle, path, true); + } finally { + await handle.close(); + } + return { ...resource, path, identity, controlPaths }; +} + +export async function assertEnvironmentResourcesStable( + resources: readonly LoadedEnvironmentResource[], +): Promise { + for (const resource of resources) { + if (!(await matchesWorkspaceRoot(resource.path, resource.identity))) + throw new Error( + 'Environment resource root changed after admission', + ); + } +} + +export function environmentResourceVariables( + resources: readonly EnvironmentResource[], +): Record { + return Object.fromEntries( + resources.map(resource => [ + RESOURCE_ENVIRONMENT_NAMES[resource.kind], + resource.path, + ]), + ); +} + +export function assertEnvironmentResourceSeparation( + resources: readonly EnvironmentResource[], + roots: readonly string[], +): void { + const contains = (parent: string, child: string) => { + const path = relative(parent, child); + return ( + path === '' || + (!isAbsolute(path) && path !== '..' && !path.startsWith(`..${sep}`)) + ); + }; + for (const resource of resources) { + if ( + roots.some( + root => + contains(root, resource.path) || + contains(resource.path, root), + ) + ) + throw new Error( + 'Environment resources must not overlap any registered workspace', + ); + } +} + +export async function assertEnvironmentResourceIsolation( + resources: readonly EnvironmentResource[], + roots: readonly string[], + controls: readonly string[], +): Promise { + if (!resources.length) return; + assertEnvironmentResourceSeparation(resources, [...roots, ...controls]); + const table = await readEnvironmentMountTable(); + if (table !== undefined) { + const check = createEnvironmentMountIsolation(table); + const stores = resources.map(resource => resource.path); + check(stores, roots); + check([...roots, ...controls], stores); + } +} diff --git a/packages/code/src/environment.ts b/packages/code/src/environment.ts index 406e3850..976ffe6d 100644 --- a/packages/code/src/environment.ts +++ b/packages/code/src/environment.ts @@ -3,6 +3,8 @@ import { constants } from 'node:fs'; import { open, realpath, stat } from 'node:fs/promises'; import { dirname, isAbsolute, relative, resolve, sep } from 'node:path'; import { parseDocument } from 'yaml'; +import { parseEnvironmentResources, loadEnvironmentResource } from './environment-resources.js'; +import type { EnvironmentResource, LoadedEnvironmentResource } from './environment-resources.js'; import { assertPrivateStorageAcl, assertPrivateStorageAncestors, @@ -33,6 +35,7 @@ export interface CodeEnvironmentDefinition { reuse?: { inputs: string[]; checkCommand: string; checkTimeoutMs: number }; }; actions?: { name: string; command: string; timeoutMs: number }[]; + resources?: EnvironmentResource[]; } export interface LoadedCodeEnvironment { @@ -41,6 +44,7 @@ export interface LoadedCodeEnvironment { rootPaths?: string[]; definition: CodeEnvironmentDefinition; fingerprint: string; + resources?: LoadedEnvironmentResource[]; } function record(value: unknown): value is Record { @@ -73,7 +77,7 @@ export function parseCodeEnvironment( !record(value) || Object.keys(value).some( key => - !['name', 'root', 'repo', 'ref', 'setup', 'actions'].includes( + !['name', 'root', 'repo', 'ref', 'setup', 'actions', 'resources'].includes( key, ), ) || @@ -173,6 +177,7 @@ export function parseCodeEnvironment( ...(typeof value.ref === 'string' ? { ref: value.ref } : {}), ...(setup ? { setup } : {}), ...(actions ? { actions } : {}), + ...(value.resources !== undefined ? { resources: parseEnvironmentResources(value.resources) } : {}), }; } @@ -328,6 +333,7 @@ export async function loadCodeEnvironment( sourceParents, rootPaths, definition, + ...(definition.resources ? { resources: await Promise.all(definition.resources.map(loadEnvironmentResource)) } : {}), fingerprint: createHash('sha256') .update(JSON.stringify(definition)) .digest('hex'), diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index 77fe8269..588f4c33 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -344,6 +344,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan shellPath, programmaticFileUpstream, linkedWorktree, + resources, } = this.options; await this.rpc( 'prepare', @@ -358,6 +359,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan shellPath, programmaticFileUpstream, linkedWorktree, + resources, variables: this.options.maskedEnvironment?.variables, }, }, diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 26c848e5..2701fe44 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -15,6 +15,7 @@ import type { Dirent, Stats } from 'node:fs'; import { access, mkdtemp, open, readdir, realpath, rm, stat } from 'node:fs/promises'; import type { FileHandle } from 'node:fs/promises'; import { matchesWorkspaceRoot } from './root-identity.js'; +import { assertEnvironmentResourcesStable, environmentResourceVariables } from './environment-resources.js'; import type { WorkspaceRootIdentity } from './root-identity.js'; import { withWorkspaceRoot, WorkspaceRootAccessError, spawnWithinWorkspace, realpath as rootedRealpath, stat as rootedStat } from './root-access.js'; @@ -168,6 +169,8 @@ type SpawnCommand = ( ) => ChildProcessWithoutNullStreams; export interface NativeSrtWorkspaceCommandSandboxOptions { + /** Explicit operator-approved stores, never inferred from process environment. */ + resources?: import('./environment-resources.js').LoadedEnvironmentResource[]; workspaceIdentity?: WorkspaceRootIdentity; workspaceRoot: string; /** Present when `workspaceRoot` is a verified linked worktree lane of a checkout. */ @@ -530,6 +533,14 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox const protectedPaths = await Promise.all( (this.options.protectedPaths ?? []).map(canonicalPath), ); + const resources = this.options.resources ?? []; + await assertEnvironmentResourcesStable(resources); + for (const resource of resources) { + if (isWithin(root, resource.path) || isWithin(resource.path, root) || + protectedPaths.some(path => isWithin(resource.path, path) || isWithin(path, resource.path))) { + throw new WorkspaceToolError('Environment resource overlaps a workspace or worker control path', 'REGISTRATION_INVALID'); + } + } if (protectedPaths.some(path => isWithin(root, path))) { throw new WorkspaceToolError( 'Native sandbox workspace cannot contain worker control files', @@ -676,6 +687,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ], allowRead: [ root, + ...resources.map(resource => resource.path), ...laneGitPaths, ...(gitGuardDirectory ? [gitGuardDirectory] : []), ...(canonicalScratchDirectory @@ -684,6 +696,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ], allowWrite: [ root, + ...resources.filter(resource => resource.access === 'read-write').map(resource => resource.path), ...writableGitPaths, ...(canonicalScratchDirectory ? [canonicalScratchDirectory] @@ -691,6 +704,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ], denyWrite: [ ...protectedPaths, + ...resources.filter(resource => resource.access === 'read-only').map(resource => resource.path), ...deniedInheritedWritablePaths, ...rootGitMetadataDenies, ...(gitGuardDirectory ? [gitGuardDirectory] : []), @@ -762,10 +776,12 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ]; this.baseDenyWritePaths = [ ...protectedPaths, + ...resources.filter(resource => resource.access === 'read-only').map(resource => resource.path), ...deniedInheritedWritablePaths, ]; this.denyWritePaths = [ ...protectedPaths, + ...resources.filter(resource => resource.access === 'read-only').map(resource => resource.path), ...deniedInheritedWritablePaths, ...rootGitMetadataDenies, ...(gitGuardDirectory ? [gitGuardDirectory] : []), @@ -988,6 +1004,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ? { filesystem: { allowRead: [ + ...(this.options.resources ?? []).map(resource => resource.path), canonicalWorkspaceRoot ?? this.canonicalRoot!, canonicalDataDirectory, ...(this.canonicalCommonGitDir @@ -1005,6 +1022,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ], denyRead: this.denyReadPaths, denyWrite: [ + ...(this.options.resources ?? []).map(resource => resource.path), this.canonicalRoot!, ...this.denyWritePaths, ], @@ -1138,6 +1156,9 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); } await this.initialize(); + await assertEnvironmentResourcesStable(this.options.resources ?? []).catch(() => { + throw new WorkspaceToolError('Environment resource changed before command dispatch', 'REGISTRATION_INVALID'); + }); const root = workspaceRoot ?? this.canonicalRoot!; let cwd: string; try { @@ -1167,6 +1188,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox : request.command; wrapped = await this.withTemporaryHostEnvironment( { + ...environmentResourceVariables(this.options.resources ?? []), ...this.gitEnvironment, ...(credentialEnvironment ?? {}), ...this.scratchSelectorEnvironment(sandboxScratchDirectory), @@ -1270,6 +1292,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox cwd, env: { ...wrapped.env, + ...environmentResourceVariables(this.options.resources ?? []), ...this.scratchEnvironment(), ...trustedEnvironment, ...this.gitEnvironment, From cdb457d024cde52967a01b96a39d7733dc7dc135 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 2 Oct 2026 07:41:15 -0400 Subject: [PATCH 23/48] =?UTF-8?q?=F0=9F=93=A6=20feat:=20Restore=20Isolated?= =?UTF-8?q?=20Dependency=20Snapshots=20(#288)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Restore Isolated Dependency Snapshots * fix: Reject Dependency Hardlink and Control Path Aliases --- packages/code/README.md | 48 +- packages/code/src/cli.ts | 20 +- .../code/src/dependency-snapshots.test.ts | 267 ++++++++++ packages/code/src/dependency-snapshots.ts | 457 ++++++++++++++++++ packages/code/src/environment-preparation.ts | 77 ++- packages/code/src/environment.ts | 11 +- packages/code/src/root-access.ts | 67 ++- 7 files changed, 936 insertions(+), 11 deletions(-) create mode 100644 packages/code/src/dependency-snapshots.test.ts create mode 100644 packages/code/src/dependency-snapshots.ts diff --git a/packages/code/README.md b/packages/code/README.md index 47d3c3a0..a861efd3 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -1100,13 +1100,59 @@ Existing definitions without `reuse` retain the startup behavior. Fresh conversa instances use the same preparation contract, with independent checkout receipts. This does not attach another checkout's `node_modules`, provision linked lanes on command admission, or recheck existing instances on every command. It does not -deduplicate installed dependencies between worktrees or enforce disk quotas. +deduplicate installed dependencies between worktrees unless snapshots below are +configured, or enforce disk quotas. Shared tool cache grants below do not attach another checkout's installed dependency tree. Keep monorepo links and mutable outputs checkout-local. Do not broaden the sandbox root or symlink another branch's full installation. Shared download caches alone do not reduce installed `node_modules` copies. +## Copy-on-write installed dependencies + +For matching fresh checkouts on the **same clone-capable filesystem**, add a +private snapshot store to the readiness contract: + +```yaml +setup: + command: npm ci + timeoutMs: 300000 + reuse: + inputs: [package.json, package-lock.json] + checkCommand: test -x node_modules/.bin/tsc + snapshot: + store: /srv/lia-state/dependency-snapshots + paths: [node_modules] + maxBytes: 4294967296 + maxFiles: 200000 +``` + +Create the external store as the worker account with mode `0700`. It must not +overlap any source root, definition, credential or shared tool cache. Commands +cannot read or write it. Use a separate store per project/trust domain. The +portable key includes project identity, declared input bytes, recipe, policy, +Node ABI and platform, but not the checkout inode. Kernel locks serialize setup +for one key; different keys remain independent. Incomplete clones are never +published. Cancellation is checked during bounded traversal. + +Matching snapshots restore only when **every** declared `node_modules` directory +is missing. The sandboxed readiness check must pass before accepting the restore. +Existing directories are never replaced by restoration; ordinary setup handles +repair. Include nested workspace installations explicitly. Relative checkout-local +package links are preserved; absolute/escaping links, hard-linked files, links into +Git/worktree control paths and special files are rejected. Hardlink-based package +manager layouts need a different adapter; this snapshot mode targets npm copies. +Changing one restored installation cannot modify the snapshot or another checkout. + +This requires APFS clones or Linux reflinks (for example a suitably configured +XFS/Btrfs volume). The worker verifies cloning before installation and rejects +unsupported filesystems instead of silently making full copies or writable hard +links. Ordinary ext4 workers should leave snapshots disabled and can still use +checkout-local preparation receipts and shared downloads. This is **not** Python +virtualenv relocation, automatic preparation of manually created linked lanes, +or a hard quota on arbitrary commands. Validate all install/postinstall inputs +and path-independent artifacts before opting in. + ## Shared tool and download resources Explicit operator-managed stores can live outside the checkouts: diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 04d94aaa..4c64c0ae 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -848,6 +848,13 @@ async function run( ...environments.map(environment => environment.path), ...rootQuarantinePaths.values(), ].filter((path): path is string => path != null)); await assertEnvironmentDefinitionsOutsideRoots(resourceRoots, roots); + const snapshotStores = environments.flatMap(environment => environment.snapshotStore ? [environment.snapshotStore] : []); + await assertEnvironmentResourceIsolation(snapshotStores.map(store => ({ kind: 'npm-cache' as const, path: store.store, access: 'read-only' as const })), roots.map(root => root.root), [ + identityPath, preparationDirectory, github.privateKeyPath, ...environments.map(environment => environment.path), + ...rootQuarantinePaths.values(), ...environments.flatMap(environment => (environment.resources ?? []).map(resource => resource.path)), + ].filter((path): path is string => path != null)); + await assertEnvironmentDefinitionsOutsideRoots(snapshotStores.map(store => ({ path: store.store, sourceParents: store.controlPaths, + definition: { name: 'dependency-store', root: store.store }, fingerprint: '' })), roots); const preparationReceipt = (root: string) => join(preparationDirectory, `${createHash('sha256').update(JSON.stringify([codeApiUrl, workerId, root])).digest('hex')}.json`); // Keep an admission boundary even when trusted-VM checkout routing uses a @@ -1073,6 +1080,7 @@ async function run( protectedPaths: [ identityPath, ...(environments.some(environment => environment.definition.setup?.reuse) ? [preparationDirectory] : []), + ...snapshotStores.map(store => store.store), ...environments.map(environment => environment.path), ...rootQuarantinePaths.values(), github.privateKeyPath, @@ -1163,10 +1171,11 @@ async function run( ...(nativeCommandSandbox instanceof NativeWorkspaceCommandPool ? { prepareInstance: async (instance, signal) => { - const setup = environments.find( + const environment = environments.find( (environment) => environment.definition.name === instance.sourceWorkspaceId, - )?.definition.setup; + ); + const setup = environment?.definition.setup; if (!setup) return; if (admittedGitHubRepositories) { admittedGitHubRepositories.set( @@ -1181,6 +1190,8 @@ async function run( workspaceRoot: instance.root, }); await prepareCodeEnvironment({ + snapshotStore: environment!.snapshotStore, + snapshotScope: environment!.definition.repo ?? environment!.definition.name, root: instance.root, identity: instance.identity, setup, receiptPath: preparationReceipt(instance.root), context: JSON.stringify([serializeNativeSrtCommandPolicy(commandPolicy), commandAllowedDomains, github.policyIdentity, @@ -1353,6 +1364,11 @@ async function run( await guard.assertAvailable(); let armed = false; const preparation = await prepareCodeEnvironment({ + snapshotStore: environment.snapshotStore, + snapshotScope: environment.definition.repo ?? environment.definition.name, + beforeMutation: async () => { + if (!armed) { await guard.arm('Dependency restoration did not settle', 'setup'); armed = true; } + }, root: environment.definition.root, identity: roots.find(root => root.id === id)!.identity!, setup, receiptPath: preparationReceipt(environment.definition.root), diff --git a/packages/code/src/dependency-snapshots.test.ts b/packages/code/src/dependency-snapshots.test.ts new file mode 100644 index 00000000..b23ae70f --- /dev/null +++ b/packages/code/src/dependency-snapshots.test.ts @@ -0,0 +1,267 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + mkdtemp, + mkdir, + writeFile, + readFile, + lstat, + link, + realpath, + rm, + symlink, + rename, + readdir, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { + parseDependencySnapshot, + loadDependencySnapshot, + publishDependencySnapshot, + restoreDependencySnapshot, + withDependencySnapshot, +} from './dependency-snapshots.js'; +import { prepareCodeEnvironment } from './environment-preparation.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; +import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; +import type { EnvironmentPreparationOptions } from './environment-preparation.js'; + +async function fixture(t: test.TestContext) { + const directory = await realpath( + await mkdtemp(join(tmpdir(), 'dependency-snapshot-')), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const a = join(directory, 'a'), + b = join(directory, 'b'), + storePath = join(directory, 'store'); + for (const path of [a, b, storePath]) await mkdir(path, { mode: 0o700 }); + const store = await loadDependencySnapshot( + parseDependencySnapshot({ store: storePath, paths: ['node_modules'] }), + ); + const ai = await captureWorkspaceRootIdentity(a), + bi = await captureWorkspaceRootIdentity(b); + return { directory, a, b, store, ai, bi, key: 'a'.repeat(64) }; +} + +test('snapshot schema restricts installed directories, overlapping paths and finite bounds', () => { + const valid = { + store: '/private/store', + paths: ['node_modules', 'packages/api/node_modules'], + }; + assert.equal(parseDependencySnapshot(valid).maxFiles, 200_000); + for (const paths of [ + ['.git'], + ['../node_modules'], + ['node_modules', 'node_modules/pkg/node_modules'], + ['node_modules', 'node_modules'], + ]) + assert.throws(() => parseDependencySnapshot({ ...valid, paths })); + assert.throws(() => + parseDependencySnapshot({ ...valid, maxBytes: Infinity }), + ); + assert.throws(() => + parseDependencySnapshot({ ...valid, store: 'relative' }), + ); +}); + +// These assertions exercise real filesystem cloning, not a mocked successful copy. +test( + 'real clone snapshots isolate writable files, preserve checkout-local links and never replace installations', + { + skip: + process.platform !== 'darwin' && + process.env.LIBRECHAT_CODE_LIVE_SNAPSHOT_TESTS !== '1', + }, + async t => { + const { a, b, store, ai, bi, key } = await fixture(t); + await mkdir(join(a, 'node_modules')); + await writeFile( + join(a, 'node_modules', 'package.js'), + 'module.exports = 42', + ); + await mkdir(join(a, 'packages')); + await mkdir(join(b, 'packages')); + await symlink('../packages', join(a, 'node_modules', 'local')); + await withDependencySnapshot(store, key, () => + publishDependencySnapshot(store, key, a, ai), + ); + assert.equal( + await withDependencySnapshot(store, key, () => + restoreDependencySnapshot(store, key, b, bi), + ), + true, + ); + assert.notEqual( + (await lstat(join(a, 'node_modules', 'package.js'))).ino, + (await lstat(join(b, 'node_modules', 'package.js'))).ino, + ); + await writeFile(join(b, 'node_modules', 'package.js'), 'changed'); + assert.equal( + await readFile(join(a, 'node_modules', 'package.js'), 'utf8'), + 'module.exports = 42', + ); + assert.equal( + await readFile(join(store.store, key, '0', 'package.js'), 'utf8'), + 'module.exports = 42', + ); + assert.equal(await restoreDependencySnapshot(store, key, b, bi), false); + assert.equal( + await readFile(join(b, 'node_modules', 'package.js'), 'utf8'), + 'changed', + ); + }, +); + +test('unsafe links and bounded traversal never publish a partial snapshot', async t => { + const { a, store, ai, key } = await fixture(t); + await mkdir(join(a, 'node_modules')); + await symlink('/etc/passwd', join(a, 'node_modules', 'escape')); + await assert.rejects( + publishDependencySnapshot(store, key, a, ai), + /checkout-local/, + ); + assert.deepEqual(await readdir(store.store), []); + await rm(join(a, 'node_modules', 'escape')); + await writeFile(join(a, 'node_modules', 'large'), 'large'); + await assert.rejects( + publishDependencySnapshot({ ...store, maxBytes: 1 }, key, a, ai), + /maxBytes/, + ); + assert.deepEqual(await readdir(store.store), []); +}); + +test('hardlinks cannot copy outside file contents into a readable dependency snapshot', async t => { + const { a, store, ai, key, directory } = await fixture(t); + await mkdir(join(a, 'node_modules')); + const outside = join(directory, 'private-input'); + await writeFile(outside, 'private'); + await link(outside, join(a, 'node_modules', 'alias')); + await assert.rejects( + publishDependencySnapshot(store, key, a, ai), + /hard-linked/, + ); + assert.deepEqual(await readdir(store.store), []); +}); + +test('kernel lock serializes publishers, supports cancellation and rejects a replaced store', async t => { + const { store, key } = await fixture(t); + let release!: () => void; + const blocked = new Promise(resolve => { + release = resolve; + }); + let entered!: () => void; + const started = new Promise(resolve => { + entered = resolve; + }); + const first = withDependencySnapshot(store, key, async () => { + entered(); + await blocked; + }); + await started; + const controller = new AbortController(); + const second = withDependencySnapshot( + store, + key, + async () => assert.fail('cancelled lock ran'), + controller.signal, + ); + controller.abort(); + await assert.rejects(second); + release(); + await first; + await rename(store.store, `${store.store}-old`); + await mkdir(store.store, { mode: 0o700 }); + await assert.rejects( + withDependencySnapshot(store, key, async () => {}), + /changed after admission/, + ); +}); + +test( + 'real native preparation shares dependencies between checkouts, invalidates changed inputs and denies store access', + { + skip: + process.env.LIBRECHAT_CODE_LIVE_SRT_TESTS !== '1' || + (process.platform !== 'darwin' && + process.env.LIBRECHAT_CODE_LIVE_SNAPSHOT_TESTS !== '1'), + }, + async t => { + const { a, b, store, directory, ai, bi } = await fixture(t); + for (const root of [a, b]) + await writeFile(join(root, 'package-lock.json'), 'version-one'); + const commands: string[] = []; + const setup: EnvironmentPreparationOptions['setup'] = { + command: + "mkdir -p node_modules; printf 'module.exports = 42' > node_modules/package.js", + timeoutMs: 5000, + reuse: { + inputs: ['package-lock.json'], + checkCommand: 'test -f node_modules/package.js', + checkTimeoutMs: 3000, + snapshot: { + store: store.store, + paths: store.paths, + maxBytes: store.maxBytes, + maxFiles: store.maxFiles, + }, + }, + }; + const sandboxes = [ai, bi].map( + identity => + new NativeProcessWorkspaceCommandSandbox({ + workspaceRoot: identity.path, + workspaceIdentity: identity, + homeDirectory: directory, + protectedPaths: [store.store], + allowedDomains: [], + }), + ); + t.after(async () => { + for (const sandbox of sandboxes) await sandbox.close(); + }); + const run = (index: number) => { + const identity = [ai, bi][index]; + return prepareCodeEnvironment({ + root: identity.path, + identity, + setup, + context: 'policy', + snapshotStore: store, + snapshotScope: 'project', + receiptPath: join(store.store, `receipt-${index}.json`), + execute: async (command, timeoutMs) => { + commands.push(command); + return sandboxes[index].execute({ + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command, + timeoutMs, + maxOutputBytes: 8192, + }); + }, + }); + }; + assert.equal(await run(0), 'prepared'); + assert.equal(await run(1), 'restored'); + assert.equal( + commands.filter(command => command === setup.command).length, + 1, + ); + await writeFile(join(b, 'package-lock.json'), 'version-two'); + assert.equal(await run(1), 'prepared'); + assert.equal( + commands.filter(command => command === setup.command).length, + 2, + ); + const denied = await sandboxes[0].execute({ + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command: `ls '${store.store}'`, + timeoutMs: 3000, + }); + assert.notEqual(denied.exitCode, 0); + }, +); diff --git a/packages/code/src/dependency-snapshots.ts b/packages/code/src/dependency-snapshots.ts new file mode 100644 index 00000000..5c9ab173 --- /dev/null +++ b/packages/code/src/dependency-snapshots.ts @@ -0,0 +1,457 @@ +import { randomUUID } from 'node:crypto'; +import { constants } from 'node:fs'; +import * as fs from 'node:fs/promises'; +import { + basename, + dirname, + isAbsolute, + join, + relative, + resolve, + sep, +} from 'node:path'; +import { createRequire } from 'node:module'; +import * as rooted from './root-access.js'; +import { withProcessLock } from './process-lock.js'; +import { + captureWorkspaceRootIdentity, + matchesWorkspaceRoot, +} from './root-identity.js'; +import { loadEnvironmentResource } from './environment-resources.js'; +import { isSafePortableRelativePath } from './protocol.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; + +export interface DependencySnapshotConfig { + store: string; + paths: string[]; + maxBytes: number; + maxFiles: number; +} +export interface DependencySnapshotStore extends DependencySnapshotConfig { + identity: WorkspaceRootIdentity; + controlPaths: string[]; +} +export function parseDependencySnapshot( + value: unknown, +): DependencySnapshotConfig { + if (!value || typeof value !== 'object' || Array.isArray(value)) + throw new Error('Invalid dependency snapshot'); + const v = value as Record; + const maxBytes = v.maxBytes ?? 4 * 1024 ** 3; + const maxFiles = v.maxFiles ?? 200_000; + if ( + Object.keys(v).some( + k => !['store', 'paths', 'maxBytes', 'maxFiles'].includes(k), + ) || + typeof v.store !== 'string' || + !isAbsolute(v.store) || + v.store.length > 4096 || + /[\0\r\n]/.test(v.store) || + !Array.isArray(v.paths) || + !v.paths.length || + v.paths.length > 32 || + v.paths.some( + p => + typeof p !== 'string' || + !isSafePortableRelativePath(p) || + basename(p) !== 'node_modules' || + p.split('/').some(c => c === '.git' || c === '.worktrees'), + ) || + new Set(v.paths).size !== v.paths.length || + v.paths.some(p => + (v.paths as unknown[]).some( + q => p !== q && (p as string).startsWith(`${q}/`), + ), + ) || + !Number.isSafeInteger(maxBytes) || + (maxBytes as number) < 1 || + !Number.isSafeInteger(maxFiles) || + (maxFiles as number) < 1 + ) + throw new Error('Invalid dependency snapshot'); + return { + store: v.store, + paths: v.paths as string[], + maxBytes: maxBytes as number, + maxFiles: maxFiles as number, + }; +} +export async function loadDependencySnapshot( + config: DependencySnapshotConfig, +): Promise { + const resource = await loadEnvironmentResource({ + kind: 'npm-cache', + path: config.store, + access: 'read-only', + }); + return { + ...config, + store: resource.path, + identity: resource.identity, + controlPaths: resource.controlPaths, + }; +} + +let native: + | { clone: (...args: any[]) => number; errno: () => number } + | undefined; +/** No copy fallback and no hardlinks: failure means the host cannot meet the contract. */ +async function cloneFile(source: fs.FileHandle, target: string): Promise { + if (!native) { + const koffi = createRequire(import.meta.url)('koffi'); + const library = koffi.load(null); + native = { + errno: () => koffi.errno(), + clone: + process.platform === 'darwin' + ? library.func( + 'int fclonefileat(int srcfd, int dstfd, const char *name, unsigned int flags)', + ) + : library.func( + 'int ioctl(int fd, unsigned long request, ...)', + ), + }; + } + if (process.platform === 'darwin') { + const parent = await rooted.open( + dirname(target), + constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW, + ); + try { + if (native.clone(source.fd, parent.fd, basename(target), 0) !== 0) + throw new Error( + `Copy-on-write dependency cloning unavailable (errno ${native.errno()})`, + ); + } finally { + await parent.close(); + } + } else if (process.platform === 'linux') { + const output = await rooted.open( + target, + constants.O_CREAT | + constants.O_EXCL | + constants.O_WRONLY | + constants.O_NOFOLLOW, + 0o600, + ); + try { + if (native.clone(output.fd, 0x40049409, 'int', source.fd) !== 0) + throw new Error( + `Copy-on-write dependency cloning unavailable (errno ${native.errno()})`, + ); + } finally { + await output.close(); + } + } else + throw new Error( + 'Dependency snapshots require a clone-capable POSIX filesystem', + ); +} + +function inside(root: string, path: string): boolean { + const p = relative(root, path); + return !isAbsolute(p) && p !== '..' && !p.startsWith(`..${sep}`); +} +async function removeTree(path: string): Promise { + const stat = await rooted.lstat(path).catch((e: NodeJS.ErrnoException) => { + if (e.code === 'ENOENT') return undefined; + throw e; + }); + if (!stat) return; + if (!stat.isDirectory()) return rooted.unlink(path); + for (const name of await rooted.readdir(path)) + await removeTree(join(path, name)); + await rooted.rmdir(path); +} +async function cloneTree( + sourceRoot: string, + sourceIdentity: WorkspaceRootIdentity, + targetRoot: string, + targetIdentity: WorkspaceRootIdentity, + paths: string[], + config: DependencySnapshotConfig, + checkout: string, + signal?: AbortSignal, +): Promise { + let bytes = 0, + files = 0; + const target = (operation: () => Promise) => + rooted.withWorkspaceRoot(targetRoot, targetIdentity, operation); + await rooted.withWorkspaceRoot(sourceRoot, sourceIdentity, async () => { + const visit = async ( + path: string, + out: string, + checkoutPath: string, + ): Promise => { + signal?.throwIfAborted(); + if (++files > config.maxFiles) + throw new Error('Dependency snapshot exceeds maxFiles'); + const before = await rooted.lstat(path); + if (before.isDirectory()) { + await target(() => rooted.mkdir(out)); + for (const name of await rooted.readdir( + path, + config.maxFiles - files, + )) + await visit( + join(path, name), + join(out, name), + join(checkoutPath, name), + ); + const after = await rooted.lstat(path); + if ( + before.ino !== after.ino || + before.dev !== after.dev || + before.mtimeMs !== after.mtimeMs || + before.ctimeMs !== after.ctimeMs + ) + throw new Error( + 'Dependencies changed during snapshot traversal', + ); + } else if (before.isSymbolicLink()) { + const link = await rooted.readlink(path); + const resolved = resolve(dirname(checkoutPath), link); + if ( + isAbsolute(link) || + !inside(checkout, resolved) || + relative(checkout, resolved).split(sep).includes('.git') || + relative(checkout, resolved) + .split(sep) + .includes('.worktrees') + ) + throw new Error( + 'Dependency snapshot link must remain checkout-local', + ); + await target(() => rooted.symlink(link, out)); + } else if (before.isFile()) { + if (before.nlink !== 1) + throw new Error( + 'Dependency snapshots reject hard-linked files', + ); + bytes += Number(before.size); + if (bytes > config.maxBytes) + throw new Error('Dependency snapshot exceeds maxBytes'); + const handle = await rooted.open( + path, + constants.O_RDONLY | + constants.O_NOFOLLOW | + constants.O_NONBLOCK, + ); + try { + const opened = await handle.stat(); + if ( + !opened.isFile() || + opened.nlink !== 1 || + opened.ino !== before.ino || + opened.dev !== before.dev + ) + throw new Error( + 'Dependency entry changed before cloning', + ); + await target(async () => { + await cloneFile(handle, out); + const output = await rooted.open( + out, + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + try { + await output.chmod(opened.mode & 0o777); + } finally { + await output.close(); + } + }); + const after = await handle.stat(); + if ( + opened.size !== after.size || + after.nlink !== 1 || + opened.mtimeMs !== after.mtimeMs || + opened.ctimeMs !== after.ctimeMs + ) + throw new Error( + 'Dependency file changed during cloning', + ); + } finally { + await handle.close(); + } + } else + throw new Error( + 'Dependency snapshots accept regular files, directories and relative links only', + ); + }; + for (let i = 0; i < paths.length; i++) + await visit( + join(sourceRoot, paths[i]), + join(targetRoot, String(i)), + join(checkout, config.paths[i]), + ); + }); +} + +export async function withDependencySnapshot( + store: DependencySnapshotStore, + key: string, + operation: () => Promise, + signal?: AbortSignal, + verifyCloneSupport = false, +): Promise { + if (!/^[a-f0-9]{64}$/.test(key)) + throw new Error('Invalid dependency snapshot key'); + if (!(await matchesWorkspaceRoot(store.store, store.identity))) + throw new Error('Dependency snapshot store changed after admission'); + return withProcessLock( + join(store.store, `${key}.lock`), + async () => { + if (!(await matchesWorkspaceRoot(store.store, store.identity))) + throw new Error( + 'Dependency snapshot store changed while waiting', + ); + signal?.throwIfAborted(); + // Fail before installation, rather than discovering unsupported reflinks after npm ci. + if (verifyCloneSupport) { + const probe = await fs.mkdtemp(join(store.store, '.probe-')); + try { + await fs.writeFile(join(probe, 'source'), 'clone-support', { + mode: 0o600, + }); + const input = await fs.open( + join(probe, 'source'), + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + try { + await rooted.withWorkspaceRoot( + store.store, + store.identity, + () => cloneFile(input, join(probe, 'target')), + ); + } finally { + await input.close(); + } + } finally { + await fs.rm(probe, { recursive: true, force: true }); + } + } + signal?.throwIfAborted(); + return operation(); + }, + signal, + ); +} +/** Restores only when every declared installation is absent. Existing files are never replaced. */ +export async function restoreDependencySnapshot( + store: DependencySnapshotStore, + key: string, + checkout: string, + identity: WorkspaceRootIdentity, + signal?: AbortSignal, +): Promise { + return rooted.withWorkspaceRoot(checkout, identity, async () => { + for (const path of store.paths) { + const exists = await rooted.lstat(join(checkout, path)).then( + () => true, + (e: NodeJS.ErrnoException) => { + if (e.code === 'ENOENT') return false; + throw e; + }, + ); + if (exists) return false; + } + const source = join(store.store, key); + const metadata = await fs + .lstat(source) + .catch((e: NodeJS.ErrnoException) => { + if (e.code === 'ENOENT') return undefined; + throw e; + }); + if (!metadata) return false; + if (!metadata.isDirectory() || metadata.isSymbolicLink()) + throw new Error('Invalid dependency snapshot entry'); + const staging = join( + checkout, + `.librechat-dependencies-${randomUUID()}`, + ); + await rooted.mkdir(staging); + const installed: string[] = []; + try { + const stagingIdentity = await captureWorkspaceRootIdentity(staging); + await cloneTree( + source, + await captureWorkspaceRootIdentity(source), + staging, + stagingIdentity, + store.paths.map((_, i) => String(i)), + store, + checkout, + signal, + ); + for (let i = 0; i < store.paths.length; i++) { + signal?.throwIfAborted(); + const destination = join(checkout, store.paths[i]); + // rename is descriptor-anchored; reject a newly appeared entry instead of replacing it. + if ( + await rooted.lstat(destination).then( + () => true, + (e: NodeJS.ErrnoException) => { + if (e.code === 'ENOENT') return false; + throw e; + }, + ) + ) + throw new Error( + 'Dependency destination appeared during restore', + ); + await rooted.renameExclusive( + join(staging, String(i)), + destination, + ); + installed.push(destination); + } + return true; + } catch (error) { + // A partial restore is inspection-worthy; do not delete a destination another process may have changed. + if (installed.length) + throw new Error( + 'Dependency restore partially committed; inspect the checkout', + { cause: error }, + ); + throw error; + } finally { + await removeTree(staging); + } + }); +} +export async function publishDependencySnapshot( + store: DependencySnapshotStore, + key: string, + checkout: string, + identity: WorkspaceRootIdentity, + signal?: AbortSignal, +): Promise { + const destination = join(store.store, key); + if ( + await fs.lstat(destination).then( + () => true, + (e: NodeJS.ErrnoException) => { + if (e.code === 'ENOENT') return false; + throw e; + }, + ) + ) + return; + const staging = await fs.mkdtemp(join(store.store, '.staging-')); + try { + await cloneTree( + checkout, + identity, + staging, + await captureWorkspaceRootIdentity(staging), + store.paths, + store, + checkout, + signal, + ); + signal?.throwIfAborted(); + await fs.rename(staging, destination); + } finally { + await fs.rm(staging, { recursive: true, force: true }); + } +} diff --git a/packages/code/src/environment-preparation.ts b/packages/code/src/environment-preparation.ts index c2597f3b..73df85f3 100644 --- a/packages/code/src/environment-preparation.ts +++ b/packages/code/src/environment-preparation.ts @@ -8,6 +8,12 @@ import { } from './storage.js'; import type { CodeEnvironmentDefinition } from './environment.js'; import type { WorkspaceRootIdentity } from './root-identity.js'; +import { + withDependencySnapshot, + restoreDependencySnapshot, + publishDependencySnapshot, +} from './dependency-snapshots.js'; +import type { DependencySnapshotStore } from './dependency-snapshots.js'; // Safety bounds on operator-declared hashing, not a dependency-store quota. const MAX_INPUT_BYTES = 8 * 1024 * 1024; @@ -25,12 +31,40 @@ export interface EnvironmentPreparationOptions { timeoutMs: number, ): Promise<{ exitCode: number | null; timedOut: boolean }>; signal?: AbortSignal; + snapshotStore?: DependencySnapshotStore; + snapshotScope?: string; + beforeMutation?(): Promise; } /** Checkout-local reuse. Never transfers mutable installations between worktrees. */ export async function prepareCodeEnvironment( options: EnvironmentPreparationOptions, -): Promise<'prepared' | 'reused'> { +): Promise<'prepared' | 'reused' | 'restored'> { + if (options.snapshotStore) { + if (!options.setup.reuse?.snapshot || !options.snapshotScope) + throw new Error( + 'Dependency snapshot requires an explicit preparation scope', + ); + if (options.snapshotStore.identity.dev !== options.identity.dev) + throw new Error( + 'Dependency snapshots and checkouts must use the same clone-capable filesystem', + ); + const portable = await preparationKey(options, true); + return withDependencySnapshot( + options.snapshotStore, + portable!, + () => prepareInLock(options, portable), + options.signal, + true, + ); + } + return prepareInLock(options); +} + +async function prepareInLock( + options: EnvironmentPreparationOptions, + portable?: string, +): Promise<'prepared' | 'reused' | 'restored'> { options.signal?.throwIfAborted(); const key = await preparationKey(options); if ( @@ -48,6 +82,36 @@ export async function prepareCodeEnvironment( if (check.exitCode === 0 && (await preparationKey(options)) === key) return 'reused'; } + if (options.snapshotStore && portable) { + await options.beforeMutation?.(); + if ( + await restoreDependencySnapshot( + options.snapshotStore, + portable, + options.root, + options.identity, + options.signal, + ) + ) { + const reuse = options.setup.reuse!; + const check = await options.execute( + reuse.checkCommand, + reuse.checkTimeoutMs, + ); + options.signal?.throwIfAborted(); + if (check.timedOut || check.exitCode === null) + throw new Error( + 'Restored dependency readiness check did not settle', + ); + if ( + check.exitCode === 0 && + (await preparationKey(options)) === key + ) { + await saveEnvironmentPreparationKey(options.receiptPath, key!); + return 'restored'; + } + } + } const result = await options.execute( options.setup.command, options.setup.timeoutMs, @@ -74,19 +138,28 @@ export async function prepareCodeEnvironment( 'Environment preparation inputs changed during readiness check', ); await saveEnvironmentPreparationKey(options.receiptPath, key); + if (options.snapshotStore && portable) + await publishDependencySnapshot( + options.snapshotStore, + portable, + options.root, + options.identity, + options.signal, + ); } return 'prepared'; } async function preparationKey( options: EnvironmentPreparationOptions, + portable = false, ): Promise { if (!options.setup.reuse) return undefined; return withWorkspaceRoot(options.root, options.identity, async () => { const hash = createHash('sha256').update( JSON.stringify({ version: 1, - root: options.identity, + root: portable ? options.snapshotScope : options.identity, setup: options.setup, context: options.context, node: process.version, diff --git a/packages/code/src/environment.ts b/packages/code/src/environment.ts index 976ffe6d..ca859a2e 100644 --- a/packages/code/src/environment.ts +++ b/packages/code/src/environment.ts @@ -4,6 +4,8 @@ import { open, realpath, stat } from 'node:fs/promises'; import { dirname, isAbsolute, relative, resolve, sep } from 'node:path'; import { parseDocument } from 'yaml'; import { parseEnvironmentResources, loadEnvironmentResource } from './environment-resources.js'; +import { parseDependencySnapshot, loadDependencySnapshot } from './dependency-snapshots.js'; +import type { DependencySnapshotConfig, DependencySnapshotStore } from './dependency-snapshots.js'; import type { EnvironmentResource, LoadedEnvironmentResource } from './environment-resources.js'; import { assertPrivateStorageAcl, @@ -32,7 +34,7 @@ export interface CodeEnvironmentDefinition { command: string; timeoutMs: number; /** Explicit readiness contract; absent preserves startup setup behavior. */ - reuse?: { inputs: string[]; checkCommand: string; checkTimeoutMs: number }; + reuse?: { inputs: string[]; checkCommand: string; checkTimeoutMs: number; snapshot?: DependencySnapshotConfig }; }; actions?: { name: string; command: string; timeoutMs: number }[]; resources?: EnvironmentResource[]; @@ -45,6 +47,7 @@ export interface LoadedCodeEnvironment { definition: CodeEnvironmentDefinition; fingerprint: string; resources?: LoadedEnvironmentResource[]; + snapshotStore?: DependencySnapshotStore; } function record(value: unknown): value is Record { @@ -124,7 +127,7 @@ export function parseCodeEnvironment( const candidate = value.setup.reuse; if ( !record(candidate) || - Object.keys(candidate).some(key => !['inputs', 'checkCommand', 'checkTimeoutMs'].includes(key)) || + Object.keys(candidate).some(key => !['inputs', 'checkCommand', 'checkTimeoutMs', 'snapshot'].includes(key)) || !Array.isArray(candidate.inputs) || candidate.inputs.length < 1 || candidate.inputs.length > 32 || candidate.inputs.some(path => typeof path !== 'string' || !isSafePortableRelativePath(path) || path === '.') || @@ -135,7 +138,8 @@ export function parseCodeEnvironment( const checkTimeoutMs = candidate.checkTimeoutMs ?? 10_000; if (typeof checkTimeoutMs !== 'number' || !Number.isSafeInteger(checkTimeoutMs) || checkTimeoutMs < 1 || checkTimeoutMs > BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS) throw new Error('Invalid environment readiness check timeout'); - reuse = { inputs: candidate.inputs as string[], checkCommand: candidate.checkCommand, checkTimeoutMs }; + reuse = { inputs: candidate.inputs as string[], checkCommand: candidate.checkCommand, checkTimeoutMs, + ...(candidate.snapshot !== undefined ? { snapshot: parseDependencySnapshot(candidate.snapshot) } : {}) }; } setup = { command: value.setup.command, timeoutMs, ...(reuse ? { reuse } : {}) }; } @@ -334,6 +338,7 @@ export async function loadCodeEnvironment( rootPaths, definition, ...(definition.resources ? { resources: await Promise.all(definition.resources.map(loadEnvironmentResource)) } : {}), + ...(definition.setup?.reuse?.snapshot ? { snapshotStore: await loadDependencySnapshot(definition.setup.reuse.snapshot) } : {}), fingerprint: createHash('sha256') .update(JSON.stringify(definition)) .digest('hex'), diff --git a/packages/code/src/root-access.ts b/packages/code/src/root-access.ts index 4aeb6100..b06de8d0 100644 --- a/packages/code/src/root-access.ts +++ b/packages/code/src/root-access.ts @@ -62,10 +62,16 @@ const openAt = nativeOpenAt const renameAt = bind( 'int renameat(int fromfd, const char *from, int tofd, const char *to)', ); +const renameExclusiveAt = bind(process.platform === 'darwin' + ? 'int renameatx_np(int fromfd, const char *from, int tofd, const char *to, unsigned int flags)' + : 'int renameat2(int fromfd, const char *from, int tofd, const char *to, unsigned int flags)'); const linkAt = bind( 'int linkat(int fromfd, const char *from, int tofd, const char *to, int flags)', ); const unlinkAt = bind('int unlinkat(int dirfd, const char *path, int flags)'); +const mkdirAt = bind('int mkdirat(int dirfd, const char *path, unsigned int mode)'); +const symlinkAt = bind('int symlinkat(const char *target, int dirfd, const char *path)'); +const readlinkAt = bind('int readlinkat(int dirfd, const char *path, void *buffer, size_t size)'); const getPath = process.platform === 'darwin' ? bind('int fcntl(int fd, int command, ...)') @@ -328,13 +334,14 @@ export class WorkspaceRootAccess { } } - install(from: string, to: string, link: boolean): void { + install(from: string, to: string, link: boolean, exclusive = false): void { const source = this.parent(from); let target: ReturnType | undefined; try { target = this.parent(to); const result = link ? linkAt!(source.fd, source.name, target.fd, target.name, 0) + : exclusive ? renameExclusiveAt!(source.fd, source.name, target.fd, target.name, process.platform === 'darwin' ? 4 : 1) : renameAt!(source.fd, source.name, target.fd, target.name); if (result !== 0) throw nativeError(); } finally { @@ -343,10 +350,27 @@ export class WorkspaceRootAccess { } } - unlink(path: string): void { + entry(path: string, operation: 'mkdir' | 'symlink' | 'readlink', target?: string): string | void { const parent = this.parent(path); try { - if (unlinkAt!(parent.fd, parent.name, 0) !== 0) throw nativeError(); + if (operation === 'readlink') { + const buffer = Buffer.alloc(4097); + const length = readlinkAt!(parent.fd, parent.name, buffer, buffer.length); + if (length < 0) throw nativeError(); + if (length === buffer.length) throw new Error('Workspace link exceeds its bounded contract'); + return buffer.subarray(0, length).toString(); + } + const result = operation === 'mkdir' + ? mkdirAt!(parent.fd, parent.name, 0o700) + : symlinkAt!(target!, parent.fd, parent.name); + if (result !== 0) throw nativeError(); + } finally { closeSync(parent.fd); } + } + + unlink(path: string, directory = false): void { + const parent = this.parent(path); + try { + if (unlinkAt!(parent.fd, parent.name, directory ? (process.platform === 'darwin' ? 0x80 : 0x200) : 0) !== 0) throw nativeError(); } finally { closeSync(parent.fd); } @@ -448,6 +472,11 @@ export const rename = async (from: string, to: string): Promise => { if (access) access.install(from, to, false); else await fs.rename(from, to); }; +export const renameExclusive = async (from: string, to: string): Promise => { + const access = context.getStore(); + if (!access) throw new Error('Exclusive installation requires a selected root'); + access.install(from, to, false, true); +}; export const link = async (from: string, to: string): Promise => { const access = context.getStore(); if (access) access.install(from, to, true); @@ -458,3 +487,35 @@ export const unlink = async (path: string): Promise => { if (access) access.unlink(path); else await fs.unlink(path); }; + +/** Enumeration and entry mutation retain the selected root descriptor. */ +export const readdir = async (path: string, maxEntries = 200_000): Promise => { + const access = context.getStore(); + const handle = access ? await access.openFile(path, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW) : undefined; + try { + const entries: string[] = []; + const directory = await fs.opendir(handle ? descriptorPath(handle.fd) : path); + for await (const entry of directory) { + if (entries.length >= maxEntries) throw new Error('Dependency directory exceeds maxFiles'); + entries.push(entry.name); + } + return entries; + } finally { await handle?.close(); } +}; +export const readlink = async (path: string): Promise => + context.getStore()?.entry(path, 'readlink') as string ?? fs.readlink(path); +export const mkdir = async (path: string): Promise => { + const access = context.getStore(); + if (access) access.entry(path, 'mkdir'); + else await fs.mkdir(path, { mode: 0o700 }); +}; +export const symlink = async (target: string, path: string): Promise => { + const access = context.getStore(); + if (access) access.entry(path, 'symlink', target); + else await fs.symlink(target, path); +}; +export const rmdir = async (path: string): Promise => { + const access = context.getStore(); + if (access) access.unlink(path, true); + else await fs.rmdir(path); +}; From 572ccbacf88de56f5257184942e17cd6d9e1b97c Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 2 Oct 2026 07:47:50 -0400 Subject: [PATCH 24/48] feat: Bound Managed Environment Storage (#289) --- docs/remote-bridge/worker-runbook.md | 16 +- packages/code/README.md | 66 +++- packages/code/src/cli.ts | 25 ++ .../code/src/dependency-snapshots.test.ts | 17 + packages/code/src/dependency-snapshots.ts | 93 +++++- packages/code/src/environment-preparation.ts | 7 + packages/code/src/environment.ts | 6 +- packages/code/src/process-lock.ts | 11 +- packages/code/src/snapshot-lifecycle.test.ts | 226 +++++++++++++ packages/code/src/snapshot-lifecycle.ts | 316 ++++++++++++++++++ 10 files changed, 773 insertions(+), 10 deletions(-) create mode 100644 packages/code/src/snapshot-lifecycle.test.ts create mode 100644 packages/code/src/snapshot-lifecycle.ts diff --git a/docs/remote-bridge/worker-runbook.md b/docs/remote-bridge/worker-runbook.md index 8cbeda98..51018940 100644 --- a/docs/remote-bridge/worker-runbook.md +++ b/docs/remote-bridge/worker-runbook.md @@ -533,7 +533,21 @@ command, cancellation, or settlement whose effects may be incomplete. - **Worker online but not ready:** check native sandbox preparation, definition validation, setup, quarantine, and readiness logs. - **Setup repeats on restart:** setup is intentionally per-start; make it - idempotent or remove it. + idempotent, remove it, or opt in to the bounded `setup.reuse` readiness + contract in the [worker package guide](../../packages/code/README.md#reusing-a-prepared-checkout). +- **Dependency copies fill the disk:** declare shared npm/uv/browser resources + instead of per-worktree downloads. On a verified clone-capable filesystem, + configure private copy-on-write snapshots and their lifecycle budget. Do not + symlink another branch's mutable `node_modules` or hardlink writable installs. +- **Managed preparation deferred for low space:** `storage.minFreeBytes` plus + `setupReserveBytes` is a soft pre-setup floor, not a hard quota. Expand the + volume or clean reproducible artifacts; do not clear quarantine as a disk fix. +- **Snapshot maintenance:** preview with `prune-environment-storage + --environment ` and use `--apply` only after reviewing its JSON. Active, + unknown and unmarked data stays intact. Include all environment definitions + for root-isolation checks. This does not archive source worktrees or prune + mutable tool caches. Keep control state on separate storage when hard + protection from arbitrary build writes is required. - **Git works on the host but not in tools:** verify the App installation, permissions, private-key mode/owner, and allowed GitHub domains. - **Repository label is present but files are absent:** `repo`/`ref` are diff --git a/packages/code/README.md b/packages/code/README.md index a861efd3..38a22d5d 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -1187,10 +1187,74 @@ Sharing is explicit within one worker's trust domain. Do not share mutable cache between unrelated principals, store credentials in them, or treat their contents as trusted worker code. Separate package caches from browsers and mutable browser profiles, Redis data, build output and test state. Stores currently have no automatic -eviction; the storage-lifecycle slice adds that separately. This shares downloads +eviction. The snapshot lifecycle below does not delete these mutable stores. This shares downloads and browser binaries, not installed `node_modules` trees. No setup output is sent to the model. +## Storage admission and snapshot lifecycle + +Use an explicit free-space floor before managed setup or restoration: + +```yaml +storage: + minFreeBytes: 5368709120 # 5 GiB left for the worker and other activity + setupReserveBytes: 2147483648 # estimated installation headroom +``` + +This works without dependency snapshots. Absent `storage`, preparation retains +today's behavior. A ready checkout can still pass its readiness check when disk +is low. When installation is needed, low space rejects it **before** starting the +setup command, with a specific remediation message. These are soft admission +checks, not allocated reservations: concurrent processes and arbitrary shell +writes can still consume space after admission. Put worker identity/quarantine +state on a separate small volume and use filesystem/project quotas for hard +containment. This option neither clears quarantine nor deletes source to recover. + +The private snapshot store can bound reproducible dependency versions: + +```yaml +setup: + # command and reuse inputs/readiness omitted here; retain the full contract above + reuse: + snapshot: + store: /srv/lia-state/dependency-snapshots + paths: [node_modules] + lifecycle: + maxStoreBytes: 21474836480 + maxEntries: 8 + retentionMs: 432000000 + scanLimit: 4096 +``` + +Cleanup runs during managed preparation and publication. Oldest inactive snapshots +are reclaimed by last-use age and logical payload/count budgets. Per-key kernel +locks protect installs, restores and publishers; maintenance never waits on or +deletes an active key. A short store-wide lock makes budget checks and publication +atomic, without serializing installations for different keys. Abandoned staging +directories require a worker ownership manifest and a free key lock before removal. +Unknown/unmarked entries are reported and preserved. Lock files are kept to avoid +splitting lock ownership. Scan limits fail closed instead of crawling unbounded data. + +The byte budget counts snapshot file lengths, not deduplicated physical blocks, +metadata overhead or working checkout copies. An otherwise valid prepared checkout +does not fail when the cache cannot fit another entry: publication is skipped with +a worker diagnostic, and later low-space admission still applies. + +Preview or explicitly apply maintenance without restarting the worker: + +```sh +librechat-code prune-environment-storage --environment /etc/librechat-code/app.yaml +librechat-code prune-environment-storage --environment /etc/librechat-code/app.yaml --apply +``` + +Pass all relevant definitions with repeated `--environment` flags so their roots +participate in isolation checks. Preview is the default and the JSON identifies +`dryRun`, proposed removals, active keys, unknown data and retained logical usage. +The command is host-operator-only, not an agent workspace action. It does not touch +source worktrees (including dirty or unpushed branches), `.verification`, build +outputs, arbitrary installations or mutable npm/uv caches. Source worktree archival +requires the separate worktree ownership/binding lifecycle, not an mtime heuristic. + Named actions are fixed commands without model-supplied substitution. The bridge advertises only their names and the definition fingerprint, never their shell source or host root. A command request can select `environmentAction: { name, fingerprint }`; diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 4c64c0ae..cf05e263 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -14,6 +14,7 @@ import { } from './environment.js'; import { prepareCodeEnvironment } from './environment-preparation.js'; import { assertEnvironmentResourceIsolation } from './environment-resources.js'; +import { pruneDependencySnapshots } from './snapshot-lifecycle.js'; import { startFileRelay } from './relay.js'; import { DockerFileRelaySupervisor } from './relay-runtime.js'; import { @@ -1191,6 +1192,7 @@ async function run( }); await prepareCodeEnvironment({ snapshotStore: environment!.snapshotStore, + storage: environment!.definition.storage, snapshotScope: environment!.definition.repo ?? environment!.definition.name, root: instance.root, identity: instance.identity, setup, receiptPath: preparationReceipt(instance.root), @@ -1365,6 +1367,7 @@ async function run( let armed = false; const preparation = await prepareCodeEnvironment({ snapshotStore: environment.snapshotStore, + storage: environment.definition.storage, snapshotScope: environment.definition.repo ?? environment.definition.name, beforeMutation: async () => { if (!armed) { await guard.arm('Dependency restoration did not settle', 'setup'); armed = true; } @@ -1687,6 +1690,28 @@ async function clearMutationQuarantine(args: string[]): Promise { async function main(): Promise { const args = process.argv.slice(2); + if (args[0] === 'prune-environment-storage') { + const paths: string[] = []; + for (let i = 1; i < args.length; i++) { + if (args[i] === '--apply') continue; + if (args[i] !== '--environment' || !args[i + 1] || args[i + 1].startsWith('--')) + throw new Error('Usage: librechat-code prune-environment-storage --environment [--environment ...] [--apply]'); + paths.push(args[++i]); + } + if (!paths.length || paths.length > 32) throw new Error('Supply between one and 32 environment definitions'); + const loaded = await Promise.all(paths.map(loadCodeEnvironment)); + const roots = loaded.map(environment => ({ id: environment.definition.name, root: environment.definition.root })); + const stores = loaded.flatMap(environment => environment.snapshotStore ? [environment.snapshotStore] : []); + if (!stores.length) throw new Error('No dependency snapshot stores configured'); + await assertEnvironmentDefinitionsOutsideRoots(loaded, roots); + await assertEnvironmentDefinitionsOutsideRoots(stores.map(store => ({ path: store.store, sourceParents: store.controlPaths, + definition: { name: 'dependency-store', root: store.store }, fingerprint: '' })), roots); + await assertEnvironmentResourceIsolation(stores.map(store => ({ kind: 'npm-cache' as const, path: store.store, access: 'read-only' as const })), + roots.map(root => root.root), [...loaded.map(environment => environment.path), + ...loaded.flatMap(environment => (environment.resources ?? []).map(resource => resource.path))]); + for (const store of stores) process.stdout.write(`${JSON.stringify(await pruneDependencySnapshots(store, { dryRun: !args.includes('--apply') }))}\n`); + return; + } if (args[0] === 'projects') { const root = option(args, '--root'); if (!root || args.slice(1).some((arg, index, rest) => diff --git a/packages/code/src/dependency-snapshots.test.ts b/packages/code/src/dependency-snapshots.test.ts index b23ae70f..f7d59681 100644 --- a/packages/code/src/dependency-snapshots.test.ts +++ b/packages/code/src/dependency-snapshots.test.ts @@ -188,6 +188,12 @@ test( }, async t => { const { a, b, store, directory, ai, bi } = await fixture(t); + store.lifecycle = { + maxStoreBytes: 10000, + maxEntries: 1, + retentionMs: 5 * 24 * 60 * 60 * 1000, + scanLimit: 100, + }; for (const root of [a, b]) await writeFile(join(root, 'package-lock.json'), 'version-one'); const commands: string[] = []; @@ -204,6 +210,7 @@ test( paths: store.paths, maxBytes: store.maxBytes, maxFiles: store.maxFiles, + lifecycle: store.lifecycle, }, }, }; @@ -255,6 +262,16 @@ test( commands.filter(command => command === setup.command).length, 2, ); + assert.equal( + (await readdir(store.store)).filter(name => + /^[a-f0-9]{64}$/.test(name), + ).length, + 1, + ); + assert.equal( + await readFile(join(a, 'node_modules', 'package.js'), 'utf8'), + 'module.exports = 42', + ); const denied = await sandboxes[0].execute({ protocolVersion: 1, operation: 'execute_command', diff --git a/packages/code/src/dependency-snapshots.ts b/packages/code/src/dependency-snapshots.ts index 5c9ab173..89a76935 100644 --- a/packages/code/src/dependency-snapshots.ts +++ b/packages/code/src/dependency-snapshots.ts @@ -20,12 +20,21 @@ import { import { loadEnvironmentResource } from './environment-resources.js'; import { isSafePortableRelativePath } from './protocol.js'; import type { WorkspaceRootIdentity } from './root-identity.js'; +import { + parseSnapshotLifecycle, + pruneDependencySnapshots, + SNAPSHOT_MANIFEST, + STAGING_MANIFEST, + SnapshotBudgetFullError, +} from './snapshot-lifecycle.js'; +import type { SnapshotLifecyclePolicy } from './snapshot-lifecycle.js'; export interface DependencySnapshotConfig { store: string; paths: string[]; maxBytes: number; maxFiles: number; + lifecycle?: SnapshotLifecyclePolicy; } export interface DependencySnapshotStore extends DependencySnapshotConfig { identity: WorkspaceRootIdentity; @@ -41,7 +50,14 @@ export function parseDependencySnapshot( const maxFiles = v.maxFiles ?? 200_000; if ( Object.keys(v).some( - k => !['store', 'paths', 'maxBytes', 'maxFiles'].includes(k), + k => + ![ + 'store', + 'paths', + 'maxBytes', + 'maxFiles', + 'lifecycle', + ].includes(k), ) || typeof v.store !== 'string' || !isAbsolute(v.store) || @@ -74,6 +90,9 @@ export function parseDependencySnapshot( paths: v.paths as string[], maxBytes: maxBytes as number, maxFiles: maxFiles as number, + ...(v.lifecycle !== undefined + ? { lifecycle: parseSnapshotLifecycle(v.lifecycle) } + : {}), }; } export async function loadDependencySnapshot( @@ -172,7 +191,7 @@ async function cloneTree( config: DependencySnapshotConfig, checkout: string, signal?: AbortSignal, -): Promise { +): Promise<{ bytes: number; files: number }> { let bytes = 0, files = 0; const target = (operation: () => Promise) => @@ -285,6 +304,7 @@ async function cloneTree( join(checkout, config.paths[i]), ); }); + return { bytes, files }; } export async function withDependencySnapshot( @@ -306,6 +326,14 @@ export async function withDependencySnapshot( 'Dependency snapshot store changed while waiting', ); signal?.throwIfAborted(); + if (store.lifecycle) + await pruneDependencySnapshots(store, { + currentKey: key, + signal, + }).catch(error => { + if (!(error instanceof SnapshotBudgetFullError)) + throw error; + }); // Fail before installation, rather than discovering unsupported reflinks after npm ci. if (verifyCloneSupport) { const probe = await fs.mkdtemp(join(store.store, '.probe-')); @@ -331,7 +359,25 @@ export async function withDependencySnapshot( } } signal?.throwIfAborted(); - return operation(); + const result = await operation(); + const manifest = await fs + .open( + join(store.store, key, SNAPSHOT_MANIFEST), + constants.O_RDONLY | constants.O_NOFOLLOW, + ) + .catch((e: NodeJS.ErrnoException) => { + if (e.code === 'ENOENT') return undefined; + throw e; + }); + if (manifest) { + try { + const now = new Date(); + await manifest.utimes(now, now); + } finally { + await manifest.close(); + } + } + return result; }, signal, ); @@ -437,9 +483,20 @@ export async function publishDependencySnapshot( ) ) return; - const staging = await fs.mkdtemp(join(store.store, '.staging-')); + const staging = await fs.mkdtemp(join(store.store, `.staging-${key}-`)); try { - await cloneTree( + await fs.writeFile( + join(staging, STAGING_MANIFEST), + JSON.stringify({ + version: 1, + key, + bytes: 0, + files: 1, + createdAt: Date.now(), + }), + { mode: 0o600 }, + ); + const measured = await cloneTree( checkout, identity, staging, @@ -450,7 +507,31 @@ export async function publishDependencySnapshot( signal, ); signal?.throwIfAborted(); - await fs.rename(staging, destination); + await fs.writeFile( + join(staging, SNAPSHOT_MANIFEST), + JSON.stringify({ + version: 1, + key, + ...measured, + createdAt: Date.now(), + }), + { mode: 0o600 }, + ); + const publish = () => fs.rename(staging, destination); + if (store.lifecycle) + await pruneDependencySnapshots(store, { + currentKey: key, + incomingBytes: measured.bytes, + incomingEntries: 1, + signal, + publish, + }).catch(error => { + if (!(error instanceof SnapshotBudgetFullError)) throw error; + process.stderr.write( + 'librechat-code: dependency snapshot not cached because the store budget is full; prepared checkout remains valid\n', + ); + }); + else await publish(); } finally { await fs.rm(staging, { recursive: true, force: true }); } diff --git a/packages/code/src/environment-preparation.ts b/packages/code/src/environment-preparation.ts index 73df85f3..8dbb8751 100644 --- a/packages/code/src/environment-preparation.ts +++ b/packages/code/src/environment-preparation.ts @@ -14,6 +14,8 @@ import { publishDependencySnapshot, } from './dependency-snapshots.js'; import type { DependencySnapshotStore } from './dependency-snapshots.js'; +import { assertPreparationSpace } from './snapshot-lifecycle.js'; +import type { EnvironmentStoragePolicy } from './snapshot-lifecycle.js'; // Safety bounds on operator-declared hashing, not a dependency-store quota. const MAX_INPUT_BYTES = 8 * 1024 * 1024; @@ -34,6 +36,7 @@ export interface EnvironmentPreparationOptions { snapshotStore?: DependencySnapshotStore; snapshotScope?: string; beforeMutation?(): Promise; + storage?: EnvironmentStoragePolicy; } /** Checkout-local reuse. Never transfers mutable installations between worktrees. */ @@ -83,6 +86,8 @@ async function prepareInLock( return 'reused'; } if (options.snapshotStore && portable) { + if (options.storage) + await assertPreparationSpace(options.root, options.storage); await options.beforeMutation?.(); if ( await restoreDependencySnapshot( @@ -112,6 +117,8 @@ async function prepareInLock( } } } + if (options.storage) + await assertPreparationSpace(options.root, options.storage); const result = await options.execute( options.setup.command, options.setup.timeoutMs, diff --git a/packages/code/src/environment.ts b/packages/code/src/environment.ts index ca859a2e..576a149f 100644 --- a/packages/code/src/environment.ts +++ b/packages/code/src/environment.ts @@ -6,6 +6,8 @@ import { parseDocument } from 'yaml'; import { parseEnvironmentResources, loadEnvironmentResource } from './environment-resources.js'; import { parseDependencySnapshot, loadDependencySnapshot } from './dependency-snapshots.js'; import type { DependencySnapshotConfig, DependencySnapshotStore } from './dependency-snapshots.js'; +import { parseEnvironmentStorage } from './snapshot-lifecycle.js'; +import type { EnvironmentStoragePolicy } from './snapshot-lifecycle.js'; import type { EnvironmentResource, LoadedEnvironmentResource } from './environment-resources.js'; import { assertPrivateStorageAcl, @@ -38,6 +40,7 @@ export interface CodeEnvironmentDefinition { }; actions?: { name: string; command: string; timeoutMs: number }[]; resources?: EnvironmentResource[]; + storage?: EnvironmentStoragePolicy; } export interface LoadedCodeEnvironment { @@ -80,7 +83,7 @@ export function parseCodeEnvironment( !record(value) || Object.keys(value).some( key => - !['name', 'root', 'repo', 'ref', 'setup', 'actions', 'resources'].includes( + !['name', 'root', 'repo', 'ref', 'setup', 'actions', 'resources', 'storage'].includes( key, ), ) || @@ -182,6 +185,7 @@ export function parseCodeEnvironment( ...(setup ? { setup } : {}), ...(actions ? { actions } : {}), ...(value.resources !== undefined ? { resources: parseEnvironmentResources(value.resources) } : {}), + ...(value.storage !== undefined ? { storage: parseEnvironmentStorage(value.storage) } : {}), }; } diff --git a/packages/code/src/process-lock.ts b/packages/code/src/process-lock.ts index aeca561a..6e456d50 100644 --- a/packages/code/src/process-lock.ts +++ b/packages/code/src/process-lock.ts @@ -5,6 +5,7 @@ import { setTimeout as delay } from 'node:timers/promises'; const LOCK_EX = 2; const LOCK_NB = 4; const LOCK_UN = 8; +export class ProcessLockBusyError extends Error {} let binding: | Promise<{ flock: (fd: number, operation: number) => number; @@ -26,7 +27,8 @@ async function lockBinding() { export async function withProcessLock( path: string, operation: () => Promise, - signal?: AbortSignal + signal?: AbortSignal, + wait = true, ): Promise { signal?.throwIfAborted(); if (process.platform !== 'darwin' && process.platform !== 'linux') { @@ -48,6 +50,7 @@ export async function withProcessLock( `Conversation worktree lock failed with errno ${errno}` ); } + if (!wait) throw new ProcessLockBusyError('Process lock is active'); await delay(50, undefined, { signal }); } signal?.throwIfAborted(); @@ -57,3 +60,9 @@ export async function withProcessLock( await handle.close(); } } + +/** Non-blocking maintenance admission. Never masks operation errors as lock contention. */ +export async function tryWithProcessLock(path: string, operation: () => Promise, signal?: AbortSignal): Promise<{ acquired: true; value: T } | { acquired: false }> { + try { return { acquired: true, value: await withProcessLock(path, operation, signal, false) }; } + catch (error) { if (error instanceof ProcessLockBusyError) return { acquired: false }; throw error; } +} diff --git a/packages/code/src/snapshot-lifecycle.test.ts b/packages/code/src/snapshot-lifecycle.test.ts new file mode 100644 index 00000000..33e37c2d --- /dev/null +++ b/packages/code/src/snapshot-lifecycle.test.ts @@ -0,0 +1,226 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + mkdtemp, + mkdir, + realpath, + writeFile, + readFile, + rm, + lstat, + utimes, + symlink, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { + loadDependencySnapshot, + parseDependencySnapshot, +} from './dependency-snapshots.js'; +import { + assertPreparationSpace, + parseEnvironmentStorage, + parseSnapshotLifecycle, + pruneDependencySnapshots, + SNAPSHOT_MANIFEST, + STAGING_MANIFEST, +} from './snapshot-lifecycle.js'; +import { withProcessLock } from './process-lock.js'; +import { prepareCodeEnvironment } from './environment-preparation.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; + +async function fixture(t: test.TestContext) { + const root = await realpath( + await mkdtemp(join(tmpdir(), 'snapshot-lifecycle-')), + ); + t.after(() => rm(root, { recursive: true, force: true })); + const path = join(root, 'store'); + await mkdir(path, { mode: 0o700 }); + const store = await loadDependencySnapshot( + parseDependencySnapshot({ + store: path, + paths: ['node_modules'], + lifecycle: { maxStoreBytes: 10, maxEntries: 2, retentionMs: 1000 }, + }), + ); + const add = async (key: string, bytes = 4, age = 0) => { + const directory = join(store.store, key); + await mkdir(directory, { mode: 0o700 }); + await writeFile( + join(directory, SNAPSHOT_MANIFEST), + JSON.stringify({ + version: 1, + key, + bytes, + files: 1, + createdAt: Date.now(), + }), + { mode: 0o600 }, + ); + const lastUsed = new Date(Date.now() - age); + await utimes(join(directory, SNAPSHOT_MANIFEST), lastUsed, lastUsed); + return directory; + }; + return { + root, + store, + add, + a: 'a'.repeat(64), + b: 'b'.repeat(64), + c: 'c'.repeat(64), + }; +} + +test('storage policy is opt-in, bounded and fails before a setup command', async t => { + assert.equal(parseEnvironmentStorage({}).minFreeBytes, 5 * 1024 ** 3); + assert.throws(() => parseEnvironmentStorage({ minFreeBytes: -1 })); + assert.throws(() => parseSnapshotLifecycle({ retentionMs: Infinity })); + await assertPreparationSpace( + '/', + { minFreeBytes: 5, setupReserveBytes: 2 }, + async () => 7n, + ); + await assert.rejects( + assertPreparationSpace( + '/', + { minFreeBytes: 5, setupReserveBytes: 2 }, + async () => 6n, + ), + /no setup command was started/, + ); + const { root } = await fixture(t); + const checkout = join(root, 'checkout'); + await mkdir(checkout); + const identity = await captureWorkspaceRootIdentity(checkout); + let executions = 0; + await assert.rejects( + prepareCodeEnvironment({ + root: checkout, + identity, + setup: { command: 'install', timeoutMs: 1000 }, + receiptPath: join(root, 'receipt'), + context: '', + storage: { + minFreeBytes: Number.MAX_SAFE_INTEGER, + setupReserveBytes: 0, + }, + execute: async () => { + executions++; + return { exitCode: 0, timedOut: false }; + }, + }), + /deferred/, + ); + assert.equal(executions, 0); +}); + +test('dry-run is non-destructive; cleanup removes only expired owned entries and retains unknown or linked data', async t => { + const { store, add, a, b, c, root } = await fixture(t); + const old = await add(a, 4, 10_000); + await add(b); + await mkdir(join(store.store, 'unrecognized')); + await writeFile(join(store.store, 'unrecognized', 'source'), 'keep'); + await symlink(root, join(store.store, c)); + const preview = await pruneDependencySnapshots(store, { dryRun: true }); + assert.deepEqual(preview.removed, [a]); + assert.equal((await lstat(old)).isDirectory(), true); + const actual = await pruneDependencySnapshots(store); + assert.deepEqual(actual.removed, [a]); + await assert.rejects(lstat(old), { code: 'ENOENT' }); + assert.equal( + await readFile(join(store.store, 'unrecognized', 'source'), 'utf8'), + 'keep', + ); + assert.equal((await lstat(join(store.store, c))).isSymbolicLink(), true); + assert.equal(actual.retainedBytes, 4); +}); + +test('cleanup preserves active keys and current preparation even when over budget', async t => { + const { store, add, a, b } = await fixture(t); + await add(a, 4, 10_000); + await add(b, 4, 10_000); + await withProcessLock(join(store.store, `${a}.lock`), async () => { + const result = await pruneDependencySnapshots(store, { currentKey: b }); + assert.deepEqual(result.removed, []); + assert.deepEqual(result.skippedActive, [a]); + await assert.rejects( + pruneDependencySnapshots(store, { + currentKey: b, + incomingBytes: 10, + }), + /budget is full/, + ); + }); + assert.equal((await lstat(join(store.store, a))).isDirectory(), true); +}); + +test('publication budgets evict LRU entries and serialize commits without deleting source', async t => { + const { store, add, a, b, c, root } = await fixture(t); + await add(a, 4, 500); + await add(b, 4, 100); + const source = join(root, 'dirty-worktree'); + await mkdir(source); + await writeFile(join(source, 'uncommitted'), 'keep'); + let published = false; + const result = await pruneDependencySnapshots(store, { + currentKey: c, + incomingBytes: 4, + incomingEntries: 1, + publish: async () => { + published = true; + }, + }); + assert.deepEqual(result.removed, [a]); + assert.equal(published, true); + assert.equal(await readFile(join(source, 'uncommitted'), 'utf8'), 'keep'); +}); + +test('operator CLI previews by default and applies only after explicit flag', async t => { + const { store, add, a, root } = await fixture(t); + await add(a, 4, 10_000); + const checkout = join(root, 'checkout'); + await mkdir(checkout); + const config = join(root, 'environment.yaml'); + await writeFile( + config, + `name: app\nroot: ${checkout}\nsetup:\n command: npm ci\n reuse:\n inputs: [package-lock.json]\n checkCommand: test -d node_modules\n snapshot:\n store: ${store.store}\n paths: [node_modules]\n lifecycle:\n retentionMs: 1000\n`, + { mode: 0o600 }, + ); + const run = promisify(execFile); + const cli = new URL('./cli.js', import.meta.url).pathname; + const args = [cli, 'prune-environment-storage', '--environment', config]; + const preview = await run(process.execPath, args); + assert.deepEqual(JSON.parse(preview.stdout).removed, [a]); + assert.equal((await lstat(join(store.store, a))).isDirectory(), true); + const applied = await run(process.execPath, [...args, '--apply']); + assert.deepEqual(JSON.parse(applied.stdout).removed, [a]); + await assert.rejects(lstat(join(store.store, a)), { code: 'ENOENT' }); +}); + +test('abandoned staging is reclaimed only with an ownership manifest and a free key lock', async t => { + const { store, a } = await fixture(t); + const name = `.staging-${a}-abcdef`; + const directory = join(store.store, name); + await mkdir(directory, { mode: 0o700 }); + await writeFile( + join(directory, STAGING_MANIFEST), + JSON.stringify({ + version: 1, + key: a, + bytes: 0, + files: 1, + createdAt: Date.now(), + }), + { mode: 0o600 }, + ); + await withProcessLock(join(store.store, `${a}.lock`), async () => { + const active = await pruneDependencySnapshots(store); + assert.deepEqual(active.skippedActive, [name]); + assert.deepEqual(active.removed, []); + }); + const result = await pruneDependencySnapshots(store); + assert.deepEqual(result.removed, [name]); + await assert.rejects(lstat(directory), { code: 'ENOENT' }); +}); diff --git a/packages/code/src/snapshot-lifecycle.ts b/packages/code/src/snapshot-lifecycle.ts new file mode 100644 index 00000000..dfa5926e --- /dev/null +++ b/packages/code/src/snapshot-lifecycle.ts @@ -0,0 +1,316 @@ +import { constants } from 'node:fs'; +import { lstat, open, rm, statfs } from 'node:fs/promises'; +import { join } from 'node:path'; +import { withProcessLock, tryWithProcessLock } from './process-lock.js'; +import { matchesWorkspaceRoot } from './root-identity.js'; +import { readdir, withWorkspaceRoot } from './root-access.js'; +import type { DependencySnapshotStore } from './dependency-snapshots.js'; + +export interface SnapshotLifecyclePolicy { + maxStoreBytes: number; + maxEntries: number; + retentionMs: number; + scanLimit: number; +} +export interface EnvironmentStoragePolicy { + minFreeBytes: number; + setupReserveBytes: number; +} +function positive(value: unknown, zero = false): value is number { + return ( + typeof value === 'number' && + Number.isSafeInteger(value) && + value >= (zero ? 0 : 1) + ); +} +export function parseSnapshotLifecycle( + value: unknown, +): SnapshotLifecyclePolicy { + if (!value || typeof value !== 'object' || Array.isArray(value)) + throw new Error('Invalid snapshot lifecycle'); + const v = value as Record; + const policy = { + maxStoreBytes: v.maxStoreBytes ?? 20 * 1024 ** 3, + maxEntries: v.maxEntries ?? 8, + retentionMs: v.retentionMs ?? 5 * 24 * 60 * 60 * 1000, + scanLimit: v.scanLimit ?? 4096, + }; + if ( + Object.keys(v).some(key => !Object.hasOwn(policy, key)) || + !Object.values(policy).every(value => positive(value)) + ) + throw new Error('Invalid snapshot lifecycle'); + return policy as SnapshotLifecyclePolicy; +} +export function parseEnvironmentStorage( + value: unknown, +): EnvironmentStoragePolicy { + if (!value || typeof value !== 'object' || Array.isArray(value)) + throw new Error('Invalid environment storage policy'); + const v = value as Record; + const policy = { + minFreeBytes: v.minFreeBytes ?? 5 * 1024 ** 3, + setupReserveBytes: v.setupReserveBytes ?? 2 * 1024 ** 3, + }; + if ( + Object.keys(v).some(key => !Object.hasOwn(policy, key)) || + !Object.values(policy).every(value => positive(value, true)) || + !Number.isSafeInteger( + (policy.minFreeBytes as number) + + (policy.setupReserveBytes as number), + ) + ) + throw new Error('Invalid environment storage policy'); + return policy as EnvironmentStoragePolicy; +} +/** Soft admission for managed preparation, not a reservation or arbitrary-write quota. */ +export async function assertPreparationSpace( + root: string, + policy: EnvironmentStoragePolicy, + available: (root: string) => Promise = async path => { + const status = await statfs(path, { bigint: true }); + return status.bavail * status.bsize; + }, +): Promise { + if ( + (await available(root)) < + BigInt(policy.minFreeBytes) + BigInt(policy.setupReserveBytes) + ) + throw new Error( + 'Managed environment preparation deferred: free disk is below the configured floor plus setup reserve. Clean reproducible artifacts or expand storage; no setup command was started.', + ); +} + +export const SNAPSHOT_MANIFEST = '.snapshot.json'; +export const STAGING_MANIFEST = '.staging.json'; +export class SnapshotBudgetFullError extends Error {} +export interface SnapshotManifest { + version: 1; + key: string; + bytes: number; + files: number; + createdAt: number; +} +async function manifest( + store: DependencySnapshotStore, + key: string, + name = key, + file = SNAPSHOT_MANIFEST, +): Promise { + const directory = join(store.store, name); + const status = await lstat(directory); + if (!status.isDirectory() || status.isSymbolicLink() || status.mode & 0o077) + return; + const handle = await open( + join(directory, file), + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ).catch((e: NodeJS.ErrnoException) => { + if (e.code === 'ENOENT' || e.code === 'ELOOP') return undefined; + throw e; + }); + if (!handle) return; + try { + const metadata = await handle.stat(); + if ( + !metadata.isFile() || + metadata.size > 4096 || + metadata.mode & 0o077 || + (process.getuid && metadata.uid !== process.getuid()) + ) + return; + const buffer = Buffer.alloc(4097); + const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0); + if (bytesRead > 4096) return; + let value: unknown; + try { + value = JSON.parse(buffer.subarray(0, bytesRead).toString()); + } catch { + return; + } + if (!value || typeof value !== 'object' || Array.isArray(value)) return; + const entry = value as Record; + if ( + entry.version !== 1 || + entry.key !== key || + !positive(entry.bytes, true) || + !positive(entry.files) || + !positive(entry.createdAt) + ) + return; + return value as SnapshotManifest; + } finally { + await handle.close(); + } +} +export interface SnapshotPruneResult { + dryRun: boolean; + removed: string[]; + skippedActive: string[]; + unknown: string[]; + retainedBytes: number; + retainedEntries: number; +} + +/** Only worker-published manifests authorize deletion. Source worktrees and unknown entries never qualify. */ +export async function pruneDependencySnapshots( + store: DependencySnapshotStore, + options: { + dryRun?: boolean; + currentKey?: string; + incomingBytes?: number; + incomingEntries?: number; + now?: number; + signal?: AbortSignal; + publish?: () => Promise; + } = {}, +): Promise { + const policy = store.lifecycle; + if (!policy) throw new Error('Snapshot lifecycle is not configured'); + return withProcessLock( + join(store.store, '.maintenance.lock'), + async () => { + if (!(await matchesWorkspaceRoot(store.store, store.identity))) + throw new Error( + 'Dependency snapshot store changed before cleanup', + ); + const result: SnapshotPruneResult = { + dryRun: options.dryRun === true, + removed: [], + skippedActive: [], + unknown: [], + retainedBytes: 0, + retainedEntries: 0, + }; + const candidates: { + key: string; + data: SnapshotManifest; + lastUsed: number; + }[] = []; + const names = await withWorkspaceRoot( + store.store, + store.identity, + () => readdir(store.store, policy.scanLimit), + ); + for (const key of names) { + options.signal?.throwIfAborted(); + const staging = /^\.staging-([a-f0-9]{64})-[A-Za-z0-9]+$/.exec( + key, + ); + if (staging) { + const owner = staging[1]; + const data = await manifest( + store, + owner, + key, + STAGING_MANIFEST, + ); + if (!data) { + result.unknown.push(key); + continue; + } + if (owner === options.currentKey) continue; + const lock = await tryWithProcessLock( + join(store.store, `${owner}.lock`), + async () => { + if ( + !(await manifest( + store, + owner, + key, + STAGING_MANIFEST, + )) + ) + return; + if (!options.dryRun) + await rm(join(store.store, key), { + recursive: true, + force: false, + }); + result.removed.push(key); + }, + options.signal, + ); + if (!lock.acquired) result.skippedActive.push(key); + continue; + } + if (!/^[a-f0-9]{64}$/.test(key)) { + if ( + !/^[a-f0-9]{64}\.lock$/.test(key) && + key !== '.maintenance.lock' + ) + result.unknown.push(key); + continue; + } + const data = await manifest(store, key); + if (!data) { + result.unknown.push(key); + continue; + } + candidates.push({ + key, + data, + lastUsed: ( + await lstat(join(store.store, key, SNAPSHOT_MANIFEST)) + ).mtimeMs, + }); + result.retainedBytes += data.bytes; + result.retainedEntries++; + } + const now = options.now ?? Date.now(); + candidates.sort( + (a, b) => a.lastUsed - b.lastUsed || a.key.localeCompare(b.key), + ); + for (const candidate of candidates) { + options.signal?.throwIfAborted(); + if (candidate.key === options.currentKey) continue; + if ( + now - candidate.lastUsed < policy.retentionMs && + result.retainedBytes + (options.incomingBytes ?? 0) <= + policy.maxStoreBytes && + result.retainedEntries + (options.incomingEntries ?? 0) <= + policy.maxEntries + ) + continue; + const admission = await tryWithProcessLock( + join(store.store, `${candidate.key}.lock`), + async () => { + const fresh = await manifest(store, candidate.key); + const updated = await lstat( + join(store.store, candidate.key, SNAPSHOT_MANIFEST), + ); + if ( + !fresh || + fresh.createdAt !== candidate.data.createdAt || + updated.mtimeMs !== candidate.lastUsed + ) + return; + if (!options.dryRun) + await rm(join(store.store, candidate.key), { + recursive: true, + force: false, + }); + result.removed.push(candidate.key); + result.retainedBytes -= candidate.data.bytes; + result.retainedEntries--; + }, + options.signal, + ); + if (!admission.acquired) + result.skippedActive.push(candidate.key); + } + if ( + !options.dryRun && + (result.retainedBytes + (options.incomingBytes ?? 0) > + policy.maxStoreBytes || + result.retainedEntries + (options.incomingEntries ?? 0) > + policy.maxEntries) + ) + throw new SnapshotBudgetFullError( + 'Dependency snapshot budget is full; active or unknown data was not deleted', + ); + if (!options.dryRun) await options.publish?.(); + return result; + }, + options.signal, + ); +} From a9528bc53918d63f324cfc6376bcf120a57cbcc7 Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Fri, 2 Oct 2026 07:48:06 -0400 Subject: [PATCH 25/48] =?UTF-8?q?=F0=9F=94=AC=20feat:=20Begin=20the=20Open?= =?UTF-8?q?Shell=20Backend=20Evaluation=20Track=20(#282)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Lia --- .github/workflows/ci.yml | 3 + README.md | 6 + docs/openshell/README.md | 189 +++++++++++++++++++ scripts/openshell/probe-policy.yaml | 11 ++ scripts/openshell/probe.mjs | 217 ++++++++++++++++++++++ scripts/openshell/probe.test.mjs | 273 ++++++++++++++++++++++++++++ 6 files changed, 699 insertions(+) create mode 100644 docs/openshell/README.md create mode 100644 scripts/openshell/probe-policy.yaml create mode 100644 scripts/openshell/probe.mjs create mode 100644 scripts/openshell/probe.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6a23b61..87aaa066 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,6 +46,9 @@ jobs: - name: Compose bridge configuration run: node tests/compose-bridge-config.cjs + - name: OpenShell evaluation probe (no gateway) + run: node --test scripts/openshell/probe.test.mjs + - name: Release versioning run: tests/release-versioning.sh diff --git a/README.md b/README.md index 3856b254..b746e3c0 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,12 @@ Omitting the expected-profile header remains supported for older clients, but provides no wrong-endpoint protection. There is deliberately no silent fallback between profiles and no automatic workspace or file migration. +## OpenShell evaluation + +An opt-in OpenShell backend is under [evaluation](docs/openshell/README.md). +The initial track provides a pinned lifecycle probe and security/compatibility +gates. It does not add a backend or change execution routing. + ## Sandbox Isolation Two modes are supported: diff --git a/docs/openshell/README.md b/docs/openshell/README.md new file mode 100644 index 00000000..a139ab7c --- /dev/null +++ b/docs/openshell/README.md @@ -0,0 +1,189 @@ +# OpenShell backend evaluation + +Status: Evaluation. No OpenShell backend is registered or enabled. + +Goal: add an opt-in, operator-hosted OpenShell backend behind Code API's existing +execution contract. Keep LibreChat clients, the default HTTP/libkrun backend, +Lambda MicroVMs, and outbound BYOM workers unchanged. + +## Baseline and upstream pin + +- Code API baseline: `836d001319015072f9493df5d7710fc371403962`. +- OpenShell evaluation release: `v0.1.2`, commit + `6648bd0c290efbc41ba131ee9831ee45cd431f94`. +- Pin the CLI, gateway, supervisor, and compute driver to that release. Record + image digests, effective policy, runtime/kernel, and resource limits with each + live result. The probe checks the CLI version, not the gateway's version. +- Revalidate this track against each proposed upgrade. Do not use rolling `dev` + tags for acceptance results. + +Upstream references at the evaluation commit: +[license](https://github.com/NVIDIA/OpenShell/blob/6648bd0c290efbc41ba131ee9831ee45cd431f94/LICENSE), +[third-party notices](https://github.com/NVIDIA/OpenShell/blob/6648bd0c290efbc41ba131ee9831ee45cd431f94/THIRD-PARTY-NOTICES), +[SDK](https://github.com/NVIDIA/OpenShell/blob/6648bd0c290efbc41ba131ee9831ee45cd431f94/sdk/typescript/README.md), +[policy schema](https://github.com/NVIDIA/OpenShell/blob/6648bd0c290efbc41ba131ee9831ee45cd431f94/docs/how-it-works/policies/schema.mdx). + +## Integration boundary + +Current flow: + +```text +LibreChat -> Code API auth/authorization -> Redis job + -> worker -> signed execute request -> SandboxBackend.execute + -> runner -> Code API egress gateway -> artifacts/tools + -> worker result finalization -> client +``` + +Proposed flow changes only the sandbox backend: + +```text +worker -> OpenShell adapter -> private OpenShell gateway + -> isolated workload + compatible Code API runner + -> unchanged signed execute contract and Code API egress gateway +``` + +The backend seam is +[`SandboxBackend`](../../service/src/sandbox-backend/types.ts). Its existing +consumers already own deadlines, queue/backend fencing, artifact restoration, +and result mapping. OpenShell's lifecycle API is not a substitute for the +runner's `/api/v2/execute` protocol. + +### Options + +| Approach | Trade-off | Decision | +| -------------------------------------------- | ------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------- | +| Existing backends only | No new control plane; no OpenShell policy/lifecycle integration | Keep as defaults and rollback targets | +| Adapter retaining the signed runner contract | Reuses Code API authorization, artifacts, tools, and finalization; runner compatibility is unproven | Evaluate first | +| Execute code directly with OpenShell exec | Avoids nested NsJail; requires rebuilding file/tool delivery, manifest validation, and result semantics | Defer unless the runner approach cannot retain its guarantees | + +Do not put the LibreChat application, the BYOM identity process, or Code API +control-plane credentials inside an agent workload. The gateway remains private; +Code API authenticates users and authorizes tenant/session access. + +## Invariants and owners + +| Invariant | Owner | +| ------------------------------------------------------------------------------ | ------------------------------------------------------------ | +| Tenant/user/session authorization and quotas | Code API | +| Signed body is immutable; `inputDelivery` storage IDs never reach the workload | Worker and adapter | +| Only scoped egress grants authorize artifact and tool access | Code API egress gateway and runner | +| Queue wait, provisioning, and execution share the original job deadline | Worker and adapter | +| Abort cannot settle as success; remote execution must terminate | Adapter and compute runtime | +| Retried jobs cannot create duplicate active execution or repeat mutations | Adapter, queue fencing, and runtime registry | +| Stateful workspace reuse waits for successful `sessionResultFinalizer` | Worker and adapter | +| Filesystem, process identity, network, and resource restrictions fail closed | OpenShell policy and selected runtime, plus runner hardening | +| Cleanup is confirmed, not merely requested; orphans remain observable/reapable | Adapter and lifecycle reconciler | + +OpenShell Docker/Podman policy rejects root workloads. The current runner +entrypoint performs mounts, ownership changes, cgroup setup, and an NsJail smoke +test. Compatibility is a blocker to resolve, not permission to run privileged, +disable manifests, relax Landlock, or bypass hardened startup. OpenShell policy +alone does not establish a separate-kernel tenant boundary. + +## Work sequence + +### 1. Lifecycle spike (this slice) + +- [x] Pin an upstream release and identify the existing backend seam. +- [x] Add an explicit, dependency-free CLI probe with strict filesystem policy, + no network allowances, no attached providers, resource limits, and cleanup. +- [x] Test validation, cancellation, partial failure, redaction, and nonterminal + cleanup without a live gateway. +- [ ] Run the probe against an isolated operator-owned gateway. Record results. + +This probe runs only a fixed shell/file round trip. It does **not** exercise Code +API, verify effective policy enforcement, or prove tenant isolation. A gateway +administrator's global policy can override the supplied sandbox policy. + +### 2. Hardened runner compatibility + +- [ ] Identify the supported compute/runtime boundary and prove startup without + weakening the existing security invariants. +- [ ] Forward the private runner listener through authenticated OpenShell + transport. Never expose its execute endpoint publicly. +- [ ] Execute a real worker-built signed request unchanged. Test Python/Bash, + input/output artifacts, tool-call replay, and error/result parity. +- [ ] Reject tampered/expired manifests, unauthorized storage handles, access to + supervisor/control-plane credentials, forbidden destinations, metadata + endpoints, and cross-tenant workspace reuse. +- [ ] Verify host/kernel filesystem protection and resource enforcement with + negative tests, not only effective-policy inspection. + +If nesting cannot work securely, stop and design the direct-exec replacement's +full authorization and delivery contract before implementation. + +### 3. Opt-in adapter + +- [ ] Use the same-release SDK behind lazy backend loading. Verify package + availability and locking; this release's TypeScript SDK uses GitHub Packages. + The CLI is a spike dependency, not a per-job production transport. +- [ ] Add an explicit backend config gate, isolated queues, and producer/consumer + backend checks. Old consumers must never pick up OpenShell jobs, including + legacy jobs without a backend marker. +- [ ] Start stateless. Reject affinity/strict mode until tenant-bound session + locks, fencing, finalization, and persistence are implemented and tested. +- [ ] Cover enqueue-anchored deadlines, cancellation while provisioning/running, + retry idempotency, gateway failure, ambiguous create responses, accepted/pending + deletion, worker crashes, orphan reconciliation, and shutdown. +- [ ] Keep auth tokens and provider credentials out of logs, argv, and workload + environment. Do not silently fall back to a different backend. + +### 4. Acceptance and rollout + +- [ ] Measure concurrency, queue/provision/execute/cleanup latency, resources, + artifact throughput, and failure rates against the existing backend using the + same workloads. Record measurements; define targets before promotion. +- [ ] Exercise gateway restart, worker restart, disconnects, upgrades, cleanup + backlogs, and quota pressure. Test database/runtime topology before claiming HA. +- [ ] Pin deployment artifacts, collect required notices/SBOMs, and document + private gateway authentication, backups, upgrades, and orphan operations. +- [ ] Canary a separate endpoint and worker pool only after compatibility and + security gates pass. Roll back routing, then drain the new queues. No automatic + file/workspace migration or production-default change is part of this track. + +## Run the lifecycle probe + +Requirements: Node 20+, an OpenShell `0.1.2` CLI, a registered named evaluation +gateway using the matching release, and a digest-pinned shell/coreutils image +pullable by that gateway. Install and configure these separately on dedicated +infrastructure. This repository does not install or start a gateway. + +```bash +node --test scripts/openshell/probe.test.mjs + +node scripts/openshell/probe.mjs \ + --gateway codeapi-evaluation \ + --workspace default \ + --image 'REGISTRY/IMAGE@sha256:REPLACE_WITH_64_HEX_DIGEST' +``` + +The live command creates and deletes one uniquely named `codeapi-probe-*` +sandbox. It uses the bundled policy, one CPU, 256 MiB, manual approvals, no +credential auto-discovery, and a fixed ten-second exec timeout. Each CLI phase +has a 120-second timeout (`--timeout-ms`, maximum 300000); this is not the +production job deadline. SIGINT/SIGTERM abort work but allow an independent +bounded cleanup attempt. SIGKILL or host/process failure cannot run cleanup. + +Only the pinned CLI's terminal `Deleted` or `already deleted` acknowledgement +confirms cleanup. Accepted/pending, unfamiliar, failed, and timed-out deletion +results fail the probe. CLI stdout/stderr are not emitted, except sanitized +phase/version/timing results. The live probe's overall success requires both +the file round trip and confirmed cleanup. + +On failure, use the emitted sandbox name and the **same gateway/workspace** to +inspect and reconcile it. A timed-out create can finish after cleanup; even an +`already deleted` acknowledgement does not resolve that race. Review the +gateway's resources after any ambiguous create or process failure. Do not retry +blindly or interpret `cleanup=confirmed` on a failed run as an orphan-free +fleet. Automated reconciliation is an adapter acceptance gate. + +## License implications + +OpenShell is Apache-2.0 at the pinned commit. That permits commercial hosted use +and modification without a network-use source-disclosure obligation. It does +not grant NVIDIA trademark rights or a support/warranty commitment. + +For distributed images or on-prem packages, include the license, preserve +applicable attribution/notices, mark modified upstream files, and audit bundled +runtime/SDK/image/model dependencies separately. This slice distributes no +OpenShell binaries or SDK and introduces no paid NVIDIA service dependency. diff --git a/scripts/openshell/probe-policy.yaml b/scripts/openshell/probe-policy.yaml new file mode 100644 index 00000000..bef0a892 --- /dev/null +++ b/scripts/openshell/probe-policy.yaml @@ -0,0 +1,11 @@ +version: 1 +filesystem_policy: + include_workdir: false + read_only: [/bin, /usr, /lib, /lib64, /etc] + read_write: [/tmp, /dev/null] +landlock: + compatibility: hard_requirement +process: + run_as_user: '1000' + run_as_group: '1000' +network_policies: {} diff --git a/scripts/openshell/probe.mjs b/scripts/openshell/probe.mjs new file mode 100644 index 00000000..116beb56 --- /dev/null +++ b/scripts/openshell/probe.mjs @@ -0,0 +1,217 @@ +import { execFile } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { parseArgs } from 'node:util'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +export const OPENSHELL_VERSION = '0.1.2'; +export const OPENSHELL_COMMIT = '6648bd0c290efbc41ba131ee9831ee45cd431f94'; +const POLICY = fileURLToPath(new URL('./probe-policy.yaml', import.meta.url)); +const MARKER = 'codeapi-openshell-probe-ok'; + +export class ProbeError extends Error { + constructor(stage) { + super(`OpenShell probe failed at ${stage}`); + this.stage = stage; + } +} + +export function commandRunner(binary = 'openshell') { + const env = { ...process.env, NO_COLOR: '1' }; + // Named gateway selection must not inherit an endpoint or insecure override. + delete env.OPENSHELL_GATEWAY_ENDPOINT; + delete env.OPENSHELL_GATEWAY_INSECURE; + return (args, { stage, timeoutMs, signal }) => + new Promise((resolve, reject) => { + const child = execFile( + binary, + args, + { + env, + timeout: timeoutMs, + killSignal: 'SIGKILL', + maxBuffer: 65536, + signal, + }, + (error, stdout) => { + // CLI errors can contain credentials, policy contents, or provider details. + if (error) reject(new ProbeError(stage)); + else resolve(stdout); + } + ); + child.stdin.end(); + }); +} + +export async function runProbe(options, run = commandRunner()) { + const { + gateway, + image, + workspace = 'default', + timeoutMs = 120000, + signal, + } = options; + if (!/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}$/.test(gateway ?? '')) { + throw new ProbeError('gateway validation'); + } + if (!/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}$/.test(workspace)) { + throw new ProbeError('workspace validation'); + } + if ( + !/^\S+@sha256:[a-f0-9]{64}$/.test(image ?? '') || + image.startsWith('-') + ) { + throw new ProbeError('image validation'); + } + if ( + !Number.isSafeInteger(timeoutMs) || + timeoutMs < 100 || + timeoutMs > 300000 + ) { + throw new ProbeError('timeout validation'); + } + const invoke = (args, stage, useSignal = true) => + run(args, { stage, timeoutMs, signal: useSignal ? signal : undefined }); + const version = await invoke(['--version'], 'version'); + if (version.trim() !== `openshell ${OPENSHELL_VERSION}`) { + throw new ProbeError('version mismatch'); + } + if (signal?.aborted) throw new ProbeError('cancelled'); + + const name = `codeapi-probe-${randomUUID()}`; + const scope = ['--gateway', gateway, '--workspace', workspace]; + const timings = {}; + const timed = async (stage, args, useSignal = true) => { + const start = performance.now(); + try { + return await invoke([...scope, ...args], stage, useSignal); + } finally { + timings[stage] = Math.round(performance.now() - start); + } + }; + let failure; + let cleanup = 'unconfirmed'; + try { + await timed('create', [ + 'sandbox', + 'create', + '--name', + name, + '--from', + image, + '--policy', + POLICY, + '--no-auto-providers', + '--approval-mode', + 'manual', + '--cpu', + '1', + '--memory', + '256Mi', + '--keep', + '--detach', + '--no-tty', + '--output', + 'json', + '--', + '/bin/sh', + '-c', + 'sleep 300', + ]); + const output = await timed('exec', [ + 'sandbox', + 'exec', + '--name', + name, + '--timeout', + '10', + '--no-tty', + '--no-login-shell', + '--', + '/bin/sh', + '-c', + `printf '%s' '${MARKER}' > /tmp/codeapi-probe && cat /tmp/codeapi-probe`, + ]); + if (output.trim() !== MARKER) throw new ProbeError('exec output'); + } catch (error) { + failure = error instanceof ProbeError ? error.stage : 'execution'; + } finally { + try { + const output = await timed( + 'delete', + ['sandbox', 'delete', name], + false + ); + const text = output.replace(/\x1b\[[0-9;]*m/g, '').trim(); + // Exit zero also covers accepted/pending deletion. Only terminal outcomes pass. + if ( + text === `✓ Deleted sandbox ${name}` || + text === `✓ Sandbox ${name} already deleted` + ) { + cleanup = 'confirmed'; + } + } catch { + // Keep the execution failure and report cleanup independently. + } + } + if (signal?.aborted && failure === undefined) failure = 'cancelled'; + return { + ok: failure === undefined && cleanup === 'confirmed', + version: OPENSHELL_VERSION, + upstreamCommit: OPENSHELL_COMMIT, + sandbox: name, + workspace, + cleanup, + ...(failure ? { failedStage: failure } : {}), + timingsMs: timings, + }; +} + +async function main() { + const { values } = parseArgs({ + options: { + gateway: { type: 'string' }, + workspace: { type: 'string', default: 'default' }, + image: { type: 'string' }, + 'timeout-ms': { type: 'string', default: '120000' }, + help: { type: 'boolean' }, + }, + }); + if (values.help) { + console.log( + 'node scripts/openshell/probe.mjs --gateway NAME --image REGISTRY/IMAGE@sha256:DIGEST [--workspace NAME] [--timeout-ms 120000]' + ); + return; + } + const controller = new AbortController(); + const abort = () => controller.abort(); + process.on('SIGINT', abort); + process.on('SIGTERM', abort); + try { + const result = await runProbe({ + gateway: values.gateway, + workspace: values.workspace, + image: values.image, + timeoutMs: Number(values['timeout-ms']), + signal: controller.signal, + }); + console.log(JSON.stringify(result, null, 2)); + process.exitCode = result.ok ? 0 : 1; + } finally { + process.off('SIGINT', abort); + process.off('SIGTERM', abort); + } +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + main().catch(error => { + console.error( + error instanceof ProbeError + ? error.message + : 'Invalid OpenShell probe invocation' + ); + process.exitCode = 1; + }); +} diff --git a/scripts/openshell/probe.test.mjs b/scripts/openshell/probe.test.mjs new file mode 100644 index 00000000..2ec95997 --- /dev/null +++ b/scripts/openshell/probe.test.mjs @@ -0,0 +1,273 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { + commandRunner, + OPENSHELL_VERSION, + ProbeError, + runProbe, +} from './probe.mjs'; + +const options = { + gateway: 'evaluation', + workspace: 'codeapi-test', + image: `registry.test/shell@sha256:${'a'.repeat(64)}`, +}; +function fixture(overrides = {}) { + const calls = []; + const run = async (args, call) => { + calls.push({ args, ...call }); + if (overrides[call.stage]) return overrides[call.stage](args, call); + if (call.stage === 'version') return `openshell ${OPENSHELL_VERSION}\n`; + if (call.stage === 'exec') return 'codeapi-openshell-probe-ok'; + if (call.stage === 'delete') + return `✓ Deleted sandbox ${args.at(-1)}\n`; + return '{}'; + }; + return { run, calls }; +} + +test('scopes every operation and creates a bounded, credential-free sandbox', async () => { + const { run, calls } = fixture(); + const result = await runProbe(options, run); + assert.equal(result.ok, true); + assert.equal(result.cleanup, 'confirmed'); + assert.match(result.sandbox, /^codeapi-probe-[a-f0-9-]{36}$/); + assert.deepEqual( + calls.map(call => call.stage), + ['version', 'create', 'exec', 'delete'] + ); + for (const call of calls.slice(1)) { + assert.deepEqual(call.args.slice(0, 4), [ + '--gateway', + 'evaluation', + '--workspace', + 'codeapi-test', + ]); + assert.equal(call.timeoutMs, 120000); + assert.ok(call.args.includes(result.sandbox)); + } + const create = calls[1].args; + for (const flag of [ + '--policy', + '--no-auto-providers', + '--keep', + '--detach', + '--no-tty', + ]) + assert.ok(create.includes(flag)); + assert.equal(create.includes('--provider'), false); + assert.equal(create.includes('--env'), false); + assert.equal(create[create.indexOf('--from') + 1], options.image); + assert.equal(create[create.indexOf('--cpu') + 1], '1'); + assert.equal(create[create.indexOf('--memory') + 1], '256Mi'); + assert.equal(create[create.indexOf('--approval-mode') + 1], 'manual'); +}); + +test('rejects mutable images, ambiguous scope, and invalid timeouts before invoking the CLI', async () => { + for (const invalid of [ + { image: 'shell:latest' }, + { image: `-shell@sha256:${'a'.repeat(64)}` }, + { gateway: undefined }, + { workspace: '../default' }, + { timeoutMs: 0 }, + { timeoutMs: 300001 }, + ]) { + const { run, calls } = fixture(); + await assert.rejects( + runProbe({ ...options, ...invalid }, run), + ProbeError + ); + assert.equal(calls.length, 0); + } +}); + +test('rejects an unpinned CLI without creating or deleting a sandbox', async () => { + const { run, calls } = fixture({ version: () => 'openshell 0.1.3' }); + await assert.rejects(runProbe(options, run), /version mismatch/); + assert.equal(calls.length, 1); +}); + +for (const stage of ['create', 'exec']) { + test(`attempts cleanup after ${stage} failure without exposing upstream details`, async () => { + const { run, calls } = fixture({ + [stage]: () => { + throw new Error('secret provider credential'); + }, + }); + const result = await runProbe(options, run); + assert.equal(result.ok, false); + assert.equal(result.failedStage, 'execution'); + assert.equal(result.cleanup, 'confirmed'); + assert.equal(calls.at(-1).stage, 'delete'); + assert.equal(JSON.stringify(result).includes('secret'), false); + }); +} + +test('does not accept unexpected command output as success', async () => { + const { run } = fixture({ exec: () => 'unexpected' }); + const result = await runProbe(options, run); + assert.equal(result.ok, false); + assert.equal(result.failedStage, 'exec output'); + assert.equal(result.cleanup, 'confirmed'); +}); + +for (const output of [ + 'deletion accepted; cleanup is pending', + 'unsupported outcome', + '', + '✓ Deleted sandbox someone-else', +]) { + test(`does not report terminal cleanup for ${JSON.stringify( + output + )}`, async () => { + const { run } = fixture({ delete: () => output }); + const result = await runProbe(options, run); + assert.equal(result.ok, false); + assert.equal(result.cleanup, 'unconfirmed'); + }); +} + +test('accepts an already-absent sandbox as terminal cleanup', async () => { + const { run } = fixture({ + delete: args => `✓ Sandbox ${args.at(-1)} already deleted\n`, + }); + assert.equal((await runProbe(options, run)).cleanup, 'confirmed'); +}); + +test('preserves execution failure when cleanup also fails', async () => { + const { run } = fixture({ + exec: () => { + throw new ProbeError('exec'); + }, + delete: () => { + throw new Error('secret'); + }, + }); + const result = await runProbe(options, run); + assert.equal(result.failedStage, 'exec'); + assert.equal(result.cleanup, 'unconfirmed'); + assert.equal(result.ok, false); +}); + +test('cancellation before create does not launch work', async () => { + const controller = new AbortController(); + controller.abort(); + const { run, calls } = fixture(); + await assert.rejects( + runProbe({ ...options, signal: controller.signal }, run), + /cancelled/ + ); + assert.equal(calls.length, 1); +}); + +test('cancellation during create still uses an independent cleanup attempt', async () => { + const controller = new AbortController(); + const { run, calls } = fixture({ + create: () => { + controller.abort(); + throw new ProbeError('create'); + }, + }); + const result = await runProbe( + { ...options, signal: controller.signal }, + run + ); + assert.equal(result.ok, false); + assert.equal(calls[1].signal, controller.signal); + assert.equal(calls.at(-1).signal, undefined); + assert.equal(calls.at(-1).stage, 'delete'); +}); + +test('command runner redacts stdout and stderr on failure', async () => { + await assert.rejects( + commandRunner(process.execPath)( + [ + '-e', + 'console.log("secret"); console.error("private-key"); process.exit(2)', + ], + { stage: 'exec', timeoutMs: 1000 } + ), + error => { + assert.equal(error.message, 'OpenShell probe failed at exec'); + assert.equal(error.cause, undefined); + assert.equal(error.stdout, undefined); + assert.equal(error.stderr, undefined); + return true; + } + ); +}); + +test('command runner kills a stalled CLI within its timeout', async () => { + await assert.rejects( + commandRunner(process.execPath)(['-e', 'setInterval(() => {}, 1000)'], { + stage: 'create', + timeoutMs: 100, + }), + /failed at create/ + ); +}); + +test('command runner removes inherited endpoint and insecure overrides', async () => { + const savedEndpoint = process.env.OPENSHELL_GATEWAY_ENDPOINT; + const savedInsecure = process.env.OPENSHELL_GATEWAY_INSECURE; + try { + process.env.OPENSHELL_GATEWAY_ENDPOINT = 'http://wrong-gateway'; + process.env.OPENSHELL_GATEWAY_INSECURE = 'true'; + const output = await commandRunner(process.execPath)( + [ + '-e', + 'console.log(JSON.stringify([process.env.OPENSHELL_GATEWAY_ENDPOINT, process.env.OPENSHELL_GATEWAY_INSECURE]))', + ], + { stage: 'version', timeoutMs: 1000 } + ); + assert.deepEqual(JSON.parse(output), [null, null]); + } finally { + if (savedEndpoint === undefined) + delete process.env.OPENSHELL_GATEWAY_ENDPOINT; + else process.env.OPENSHELL_GATEWAY_ENDPOINT = savedEndpoint; + if (savedInsecure === undefined) + delete process.env.OPENSHELL_GATEWAY_INSECURE; + else process.env.OPENSHELL_GATEWAY_INSECURE = savedInsecure; + } +}); + +test('late cancellation cannot report success while cleanup completes', async () => { + const controller = new AbortController(); + const { run } = fixture({ + delete: args => { + controller.abort(); + return `✓ Deleted sandbox ${args.at(-1)}\n`; + }, + }); + const result = await runProbe( + { ...options, signal: controller.signal }, + run + ); + assert.equal(result.ok, false); + assert.equal(result.failedStage, 'cancelled'); + assert.equal(result.cleanup, 'confirmed'); +}); + +test('command runner rejects oversized output without exposing it', async () => { + await assert.rejects( + commandRunner(process.execPath)( + ['-e', 'process.stdout.write("x".repeat(100000))'], + { stage: 'exec', timeoutMs: 1000 } + ), + error => { + assert.equal(error.message, 'OpenShell probe failed at exec'); + assert.equal(error.stdout, undefined); + return true; + } + ); +}); + +test('command runner handles a missing CLI without exposing process details', async () => { + await assert.rejects( + commandRunner('codeapi-nonexistent-probe-binary')([], { + stage: 'version', + timeoutMs: 1000, + }), + /failed at version/ + ); +}); From 032a93fec88716a819ef33483448f403d1c17729 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 2 Oct 2026 22:29:58 -0400 Subject: [PATCH 26/48] feat: Retire Stale Linked Worktrees Automatically (#299) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🧹 feat: Retire Stale Linked Worktrees Automatically Agents create one linked worktree per task under /.worktrees and nothing ever removed them. Each keeps its own dependencies and build output, so a trusted-VM checkout accumulated 231 worktrees and filled its disk to 100% three times, quarantining workspaces. A worker with linked worktree lanes now retires stale worktrees in the background: two minutes after start, every six hours, sooner while a backlog remains, and before managed setup would be deferred for low disk space. A worktree is retired only when it verifies as a linked worktree under .worktrees, no request uses its lane or checkout, it has been idle for seven days by both this worker's lane use and on-disk Git activity, it is clean, unlocked, outside any merge/rebase/cherry-pick/revert/bisect, unquarantined, and its work is published or merged: HEAD is contained in a remote-tracking ref, or, with no live upstream (detached, never pushed, or upstream deleted after merge), its commits are already in the remote's default branch by content (patch-equivalent per git cherry, or identical content at every path the branch changed). Squash-merged branches and abandoned review worktrees therefore qualify; only local Git is consulted. Removal is `git worktree remove` without --force followed by `git worktree prune`; branches are kept so `git worktree add` restores any worktree. Retirement is on by default. --no-worktree-retirement or LIBRECHAT_CODE_WORKTREE_RETIREMENT=false disables it, and --worktree-idle-days / LIBRECHAT_CODE_WORKTREE_IDLE_DAYS change the threshold. LIBRECHAT_CODE_LOG_LEVEL=debug logs every kept worktree with its reason. * fix: Fence Checkout Requests During Retirement and Never Cut Removal Short - Checkout requests wait for an in-progress retirement of any lane beneath them, since root commands can reach .worktrees/*. - Requests waiting on a retirement honor their own cancellation. - git worktree remove runs without a timeout; a killed removal could leave a half-deleted worktree that no later pass can verify. - Drop the repository-wide git worktree prune: remove already deletes the retired worktree's metadata, and prune would also expire other registered worktrees that are only temporarily unavailable. - Read every .worktrees entry each pass instead of the first 4096. * fix: Recheck Quarantine Under the Lane Reservation A checkout or lane request that finished between inspection and the lane reservation may have left an uncertain mutation quarantined. Re-read both quarantine markers inside the reservation, immediately before removal. * fix: Defer Retirement When the Checkout Was Used During Inspection A checkout request can write ignored files inside .worktrees/ that git status never reports, and finish before the lane reservation. Count checkout request starts and finishes, capture the count before inspecting each worktree, and keep the worktree for a later pass when it changed by the time the reservation is held. * test: Clone the Merge-Simulation Remote on main Regardless of init.defaultBranch On CI the bare remote's HEAD names master, so the hosting clone had an empty HEAD and `git merge --squash` refused to run. --- docs/remote-bridge/worker-runbook.md | 38 +- packages/code/README.md | 57 ++ packages/code/src/cli.ts | 52 ++ packages/code/src/environment-preparation.ts | 16 +- packages/code/src/linked-worktrees.ts | 198 ++++- packages/code/src/snapshot-lifecycle.test.ts | 48 ++ packages/code/src/snapshot-lifecycle.ts | 24 +- packages/code/src/worktree-retirement.test.ts | 592 +++++++++++++++ packages/code/src/worktree-retirement.ts | 697 ++++++++++++++++++ 9 files changed, 1685 insertions(+), 37 deletions(-) create mode 100644 packages/code/src/worktree-retirement.test.ts create mode 100644 packages/code/src/worktree-retirement.ts diff --git a/docs/remote-bridge/worker-runbook.md b/docs/remote-bridge/worker-runbook.md index 51018940..4026a755 100644 --- a/docs/remote-bridge/worker-runbook.md +++ b/docs/remote-bridge/worker-runbook.md @@ -521,6 +521,35 @@ replace the local clear command. Never clear quarantine merely to make the worker start. It represents a setup, command, cancellation, or settlement whose effects may be incomplete. +### Disk filled by task worktrees + +Agents that create one linked worktree per task under `/.worktrees/` +leave each one behind with its own dependencies and build output. A worker with +`--linked-worktree-lanes` retires stale ones automatically: two minutes after +startup, every six hours, and before managed setup would be deferred for low +space. Only verified linked worktrees in writable roots that are idle for seven +days, clean, unlocked, outside any merge or rebase, unquarantined, and whose +`HEAD` is on a remote-tracking ref or already in the default branch by content +(a squash- or rebase-merged branch whose remote branch was deleted) are +removed, with `git worktree remove` without `--force`. Merged content is judged +against the checkout's last fetch of the default branch; the worker never +fetches. Branches are kept; restore one with +`git worktree add .worktrees/ `. The +[worker package guide](../../packages/code/README.md#retiring-stale-linked-worktrees) +lists every condition. + +Each pass logs `worktree retirement: retired N, kept M (reason counts)`. To see +why a particular worktree is kept, restart once with +`LIBRECHAT_CODE_LOG_LEVEL=debug`. Worktrees kept as `dirty` or `unpushed` hold +work nobody has published; push or discard it from the checkout before removing +them by hand, never with `rm -rf`. If the disk is already full and setup keeps +failing, free space by hand first: retirement runs only after the worker starts, +or during setup that has a `storage` floor configured. + +To keep every worktree, add `--no-worktree-retirement` (or +`LIBRECHAT_CODE_WORKTREE_RETIREMENT=false`). To keep them longer, set +`--worktree-idle-days ` (or `LIBRECHAT_CODE_WORKTREE_IDLE_DAYS`). + ## 15. Common failures - **`--environment cannot be combined...`:** remove old workspace flags and @@ -540,8 +569,13 @@ command, cancellation, or settlement whose effects may be incomplete. configure private copy-on-write snapshots and their lifecycle budget. Do not symlink another branch's mutable `node_modules` or hardlink writable installs. - **Managed preparation deferred for low space:** `storage.minFreeBytes` plus - `setupReserveBytes` is a soft pre-setup floor, not a hard quota. Expand the - volume or clean reproducible artifacts; do not clear quarantine as a disk fix. + `setupReserveBytes` is a soft pre-setup floor, not a hard quota. A worker + with linked worktree lanes first retires stale task worktrees and measures + again. If space is still short, expand the volume or clean reproducible + artifacts; do not clear quarantine as a disk fix. +- **Many `.worktrees/*` directories remain:** see + [Disk filled by task worktrees](#disk-filled-by-task-worktrees). Count the + kept reasons in the retirement summary before removing anything by hand. - **Snapshot maintenance:** preview with `prune-environment-storage --environment ` and use `--apply` only after reviewing its JSON. Active, unknown and unmarked data stays intact. Include all environment definitions diff --git a/packages/code/README.md b/packages/code/README.md index 38a22d5d..55f5c668 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -948,6 +948,63 @@ be combined with conversation worktrees. Code API must advertise scope for them. Deploy consumers that read worker status (such as LibreChat) with support for `workspaceScopes` before enabling lanes on a worker. +##### Retiring stale linked worktrees + +Nothing else removes a task's worktree once its work is pushed, and each one +keeps its own dependencies and build output. A worker with lanes therefore +retires stale ones itself, with no configuration: a pass runs two minutes after +startup and every six hours after that, sooner while a backlog remains, and +also before managed environment setup would be deferred for low disk space. +Passes run in the background and never overlap. Only an actual removal holds +back requests, and only those for that lane or its checkout. + +A worktree in a writable registered root is retired only when **all** of these +hold; otherwise it is kept and the reason is counted: + +- It verifies as a linked worktree of the checkout under `.worktrees/`, as for + lane admission. The checkout itself and worktrees elsewhere are never + touched. +- Neither the lane nor its checkout has a request in flight or ran one while + the worktree was being inspected, and both this + worker's last use of the lane and the newest on-disk activity (the worktree + directory and its Git `HEAD`, `index`, `logs/HEAD`, `ORIG_HEAD` and + `FETCH_HEAD`) are older than the idle threshold, seven days by default. +- It has no modified tracked files and no untracked files the repository does + not ignore; no merge, rebase, cherry-pick, revert or bisect in progress; no + `git worktree lock`; and no quarantine on the lane or its checkout. +- Its `HEAD` commit, including a detached one, is contained in at least one + remote-tracking ref (`refs/remotes/*`), or its work is already in the + remote's default branch by content, as after a squash or rebase merge whose + remote branch was deleted. Content counts only when `HEAD` has no live + upstream (it is detached, never pushed, or its upstream ref is gone) and + either every commit beyond the default branch is patch-equivalent to one in + it (`git cherry` shows only `-`, with no merge commits), or the default + branch has identical content at every path the branch changed since their + merge base. The default branch is the remote's `HEAD`, else `main` or + `master`, as last fetched; the worker never fetches or calls a hosting API. + Commits beyond a live upstream are never treated as merged. + +Removal is `git worktree remove` **without** `--force`, so Git re-checks for +changes itself and deletes only that worktree's metadata. No repository-wide +`git worktree prune` runs, so other registered worktrees that are temporarily +unavailable stay registered. Removal has no timeout, because a half-deleted +worktree could no longer be recognized. Ignored files such as `node_modules`, +build output and ignored `.env` files go with the worktree; that is the space +being reclaimed. The branch is kept, so +`git worktree add .worktrees/ ` restores the worktree. Lane and +checkout requests that arrive during a removal wait for it, or for their own +cancellation; a lane request then fails as an unknown worktree. Each pass reads +every `.worktrees` entry, inspects at most 128 idle worktrees and removes at +most 32, oldest first, rotating so that worktrees kept for lasting reasons +cannot hide the rest. Each pass logs one summary line, for example +`worktree retirement: retired 3, kept 12 (dirty 2, recent 8, unpushed 2), freed +about 4.1 GiB`; set `LIBRECHAT_CODE_LOG_LEVEL=debug` to log every kept worktree +and its reason. + +Pass `--no-worktree-retirement` or set `LIBRECHAT_CODE_WORKTREE_RETIREMENT=false` +to disable retirement. Change the idle threshold with `--worktree-idle-days ` +or `LIBRECHAT_CODE_WORKTREE_IDLE_DAYS=` (1 to 3650 days). + On an updated Code API, admission waits up to 30 seconds without the `X-LibreChat-Workspace-Queue-Wait-Ms` request header. A caller may advertise a positive integer millisecond allowance up to five minutes, capped by any server diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index cf05e263..6b450b64 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -43,6 +43,10 @@ import { NativeWorkspaceCommandPool } from './native-pool.js'; import { GitWorktreeWorkspaceTools, internalWorkspaceId } from './workspace-instances.js'; import { LINKED_WORKTREE_DIRECTORY, LinkedWorktreeWorkspaceTools } from './linked-worktrees.js'; import { GitWorktreeManager } from './worktrees.js'; +import { + WorktreeRetirementScheduler, + worktreeRetirementSettings, +} from './worktree-retirement.js'; import { captureWorkspaceRootIdentity } from './root-identity.js'; import { resolveNativeSrtCommandPolicy, @@ -812,6 +816,13 @@ async function run( if (linkedWorktreeLanes && process.platform === 'win32') { throw new Error('Linked worktree Git guard requires a POSIX host'); } + const worktreeRetirement = worktreeRetirementSettings({ + optOut: args.includes('--no-worktree-retirement'), + enabled: process.env.LIBRECHAT_CODE_WORKTREE_RETIREMENT, + idleDays: + option(args, '--worktree-idle-days') ?? + process.env.LIBRECHAT_CODE_WORKTREE_IDLE_DAYS, + }); if ( roots.length > 1 && process.env.LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE?.trim() @@ -1301,6 +1312,41 @@ async function run( }); workspaceTools = linkedWorktreeTools; } + const retirementSources = roots.filter((root) => root.writable); + const debugLogs = + process.env.LIBRECHAT_CODE_LOG_LEVEL?.trim().toLowerCase() === 'debug'; + const worktreeRetirementScheduler = + linkedWorktreeTools && + worktreeRetirement.enabled && + retirementSources.length > 0 && + option(args, '--reset-workspace-quarantine') == null + ? new WorktreeRetirementScheduler({ + activity: linkedWorktreeTools, + idleMs: worktreeRetirement.idleMs, + sources: retirementSources.map((root) => ({ + workspaceId: root.id, + root: root.root, + identity: root.identity, + })), + async isQuarantined(selectedWorkspaceId, worktree) { + const source = roots.find((root) => root.id === selectedWorkspaceId); + if (!source) return true; + const path = + worktree == null + ? rootQuarantinePaths.get(selectedWorkspaceId)! + : defaultWorkspaceQuarantinePath({ + codeApiUrl, + workerId, + workspaceRoot: join(source.root, LINKED_WORKTREE_DIRECTORY, worktree), + }); + return (await loadWorkspaceMutationQuarantine(path)) != null; + }, + log(level, message) { + if (level === 'debug' && !debugLogs) return; + process.stdout.write(`librechat-code: ${message}\n`); + }, + }) + : undefined; if (workspaceTools && environments.length) { workspaceTools = new EnvironmentWorkspaceTools( workspaceTools, @@ -1375,6 +1421,9 @@ async function run( root: environment.definition.root, identity: roots.find(root => root.id === id)!.identity!, setup, receiptPath: preparationReceipt(environment.definition.root), + reclaimSpace: worktreeRetirementScheduler + ? () => worktreeRetirementScheduler.runNow() + : undefined, context: JSON.stringify([serializeNativeSrtCommandPolicy(commandPolicy), commandAllowedDomains, github.policyIdentity, nativeOptionsForWorkspace(id).resources]), signal: controller.signal, @@ -1404,10 +1453,12 @@ async function run( ); } } catch (error) { + await worktreeRetirementScheduler?.stop().catch(() => undefined); await nativeCommandSandbox?.close().catch(() => undefined); await fileRelaySupervisor?.stop().catch(() => undefined); throw error; } + worktreeRetirementScheduler?.start(); try { const worker = new BridgeWorker({ instructionDescriptors: () => localWorkspaceTools?.instructionDescriptors() ?? Promise.resolve(undefined), @@ -1617,6 +1668,7 @@ async function run( await worker.run(controller.signal); } finally { try { + await worktreeRetirementScheduler?.stop(); await nativeCommandSandbox?.close(); } finally { await fileRelaySupervisor?.stop(); diff --git a/packages/code/src/environment-preparation.ts b/packages/code/src/environment-preparation.ts index 8dbb8751..b17b4d32 100644 --- a/packages/code/src/environment-preparation.ts +++ b/packages/code/src/environment-preparation.ts @@ -37,6 +37,8 @@ export interface EnvironmentPreparationOptions { snapshotScope?: string; beforeMutation?(): Promise; storage?: EnvironmentStoragePolicy; + /** Frees reproducible storage when the storage floor would defer setup. */ + reclaimSpace?(): Promise; } /** Checkout-local reuse. Never transfers mutable installations between worktrees. */ @@ -87,7 +89,12 @@ async function prepareInLock( } if (options.snapshotStore && portable) { if (options.storage) - await assertPreparationSpace(options.root, options.storage); + await assertPreparationSpace( + options.root, + options.storage, + undefined, + options.reclaimSpace, + ); await options.beforeMutation?.(); if ( await restoreDependencySnapshot( @@ -118,7 +125,12 @@ async function prepareInLock( } } if (options.storage) - await assertPreparationSpace(options.root, options.storage); + await assertPreparationSpace( + options.root, + options.storage, + undefined, + options.reclaimSpace, + ); const result = await options.execute( options.setup.command, options.setup.timeoutMs, diff --git a/packages/code/src/linked-worktrees.ts b/packages/code/src/linked-worktrees.ts index e049aa6e..225abebd 100644 --- a/packages/code/src/linked-worktrees.ts +++ b/packages/code/src/linked-worktrees.ts @@ -32,6 +32,27 @@ const LINKED_WORKTREE_SHARED_GIT_PATHS = ['objects', 'refs', join('logs', 'refs' const LINKED_WORKTREE_LANE_LIMIT = 32; /** Git pointer files are a single line; anything larger is not one. */ const GIT_POINTER_MAX_BYTES = 4096; +/** Last-use timestamps kept per worker; the oldest are forgotten first. */ +const LINKED_WORKTREE_ACTIVITY_LIMIT = 4096; + +/** What this worker knows about lane use, for deciding when a worktree may be retired. */ +export interface LinkedWorktreeActivity { + /** When a request for the lane last started or finished in this process. */ + lastUsed(workspaceId: string, worktree: string): number | undefined; + /** Changes whenever a checkout request (outside any lane) starts or finishes. */ + checkoutActivity(workspaceId: string): number; + /** + * Run `task` only while neither the lane nor its checkout has a request in + * flight. Requests for the lane or its checkout that arrive meanwhile wait + * for it to finish (or for their own cancellation); a lane request then + * verifies the lane again. + */ + whileIdle( + workspaceId: string, + worktree: string, + task: () => Promise, + ): Promise<{ ran: true; value: T } | { ran: false }>; +} export interface LinkedWorktreeSource { root: string; @@ -206,7 +227,7 @@ function publicResult(result: WorkspaceToolResult, workspaceId: string): Workspa * checkout, so file tools and commands here run confined to that worktree while * sibling lanes run concurrently. Requests without a worktree pass through. */ -export class LinkedWorktreeWorkspaceTools implements WorkspaceToolExecutor { +export class LinkedWorktreeWorkspaceTools implements WorkspaceToolExecutor, LinkedWorktreeActivity { readonly mutationFailuresAreAtomic?: true; readonly capabilities: WorkspaceToolExecutor['capabilities']; private readonly executors = new Map< @@ -216,6 +237,16 @@ export class LinkedWorktreeWorkspaceTools implements WorkspaceToolExecutor { private readonly commandRoots = new Map(); /** Verified lane roots by internal ID, least recently used first. */ private readonly lanes = new Map(); + /** Requests in flight by lane internal ID. */ + private readonly laneRequests = new Map(); + /** Requests in flight by checkout workspace ID, outside any lane. */ + private readonly checkoutRequests = new Map(); + /** Last request start or finish by lane internal ID, oldest first. */ + private readonly used = new Map(); + /** Checkout request starts and finishes by workspace ID. */ + private readonly checkoutEvents = new Map(); + /** Lanes being retired by internal ID; requests for the lane or its checkout wait for them to settle. */ + private readonly retiring = new Map }>(); constructor(private readonly options: LinkedWorktreeWorkspaceToolsOptions) { this.mutationFailuresAreAtomic = options.delegate.mutationFailuresAreAtomic; @@ -337,29 +368,139 @@ export class LinkedWorktreeWorkspaceTools implements WorkspaceToolExecutor { return pool; } + lastUsed(workspaceId: string, worktree: string): number | undefined { + return this.used.get(linkedWorktreeWorkspaceId(workspaceId, worktree)); + } + + checkoutActivity(workspaceId: string): number { + return this.checkoutEvents.get(workspaceId) ?? 0; + } + + async whileIdle( + workspaceId: string, + worktree: string, + task: () => Promise, + ): Promise<{ ran: true; value: T } | { ran: false }> { + const internalId = linkedWorktreeWorkspaceId(workspaceId, worktree); + if ( + this.laneRequests.has(internalId) || + this.checkoutRequests.has(workspaceId) || + this.retiring.has(internalId) + ) { + return { ran: false }; + } + let settle!: () => void; + const done = new Promise((resolve) => { settle = resolve; }); + this.retiring.set(internalId, { workspaceId, done }); + try { + return { ran: true, value: await task() }; + } finally { + await this.release(internalId); + this.retiring.delete(internalId); + settle(); + } + } + + private touch(internalId: string): void { + this.used.delete(internalId); + this.used.set(internalId, Date.now()); + if (this.used.size <= LINKED_WORKTREE_ACTIVITY_LIMIT) return; + const oldest = this.used.keys().next().value; + if (oldest != null) this.used.delete(oldest); + } + + /** Count a request from its first synchronous step, so `whileIdle` never races its admission. */ + private async tracked( + requests: Map, + key: string, + task: () => Promise, + lane: boolean, + ): Promise { + const record = (): void => { + if (lane) this.touch(key); + else this.checkoutEvents.set(key, (this.checkoutEvents.get(key) ?? 0) + 1); + }; + requests.set(key, (requests.get(key) ?? 0) + 1); + record(); + try { + return await task(); + } finally { + const remaining = (requests.get(key) ?? 1) - 1; + if (remaining > 0) requests.set(key, remaining); + else requests.delete(key); + record(); + } + } + + /** Wait out retirements a request could collide with, unless the request is cancelled first. */ + private async afterRetirement( + collides: (internalId: string, workspaceId: string) => boolean, + signal: AbortSignal | undefined, + ): Promise { + const pending = [...this.retiring] + .filter(([internalId, entry]) => collides(internalId, entry.workspaceId)) + .map(([, entry]) => entry.done); + if (pending.length === 0) return; + signal?.throwIfAborted(); + await new Promise((resolve, reject) => { + const onAbort = (): void => reject(signal?.reason); + signal?.addEventListener('abort', onAbort, { once: true }); + void Promise.all(pending).then(() => { + signal?.removeEventListener('abort', onAbort); + resolve(); + }); + }); + } + + private async inCheckout(workspaceId: string, task: () => Promise, signal?: AbortSignal): Promise { + return await this.tracked(this.checkoutRequests, workspaceId, async () => { + await this.afterRetirement((_internalId, laneWorkspaceId) => laneWorkspaceId === workspaceId, signal); + return await task(); + }, false); + } + + private async inLane( + workspaceId: string, + worktree: string, + task: () => Promise, + signal?: AbortSignal, + ): Promise { + const internalId = linkedWorktreeWorkspaceId(workspaceId, worktree); + return await this.tracked(this.laneRequests, internalId, async () => { + await this.afterRetirement((retiringId) => retiringId === internalId, signal); + return await task(); + }, true); + } + async execute(request: WorkspaceToolRequest, signal?: AbortSignal): Promise { if (request.worktree == null) { - return await this.options.delegate.execute(request, signal); + return await this.inCheckout( + request.workspaceId, + () => this.options.delegate.execute(request, signal), + signal, + ); } if (request.operation === 'execute_command' && request.environmentAction) { throw rejected('Environment action was not resolved by this worker'); } const { worktree, ...baseRequest } = request; - const { lane, source, internalId } = await this.resolveLane( - request.workspaceId, - worktree, - request.workspaceInstanceId, - ); - const laneRequest = { ...baseRequest, workspaceId: internalId } as WorkspaceToolRequest; - if (request.operation === 'execute_command') { - const pool = await this.registerCommandRoot(internalId, lane, source); - return publicResult( - await pool.execute(laneRequest as WorkspaceExecuteCommandRequest, signal), + return await this.inLane(request.workspaceId, worktree, async () => { + const { lane, source, internalId } = await this.resolveLane( request.workspaceId, + worktree, + request.workspaceInstanceId, ); - } - const executor = await this.fileExecutor(internalId, lane, source); - return publicResult(await executor.execute(laneRequest, signal), request.workspaceId); + const laneRequest = { ...baseRequest, workspaceId: internalId } as WorkspaceToolRequest; + if (request.operation === 'execute_command') { + const pool = await this.registerCommandRoot(internalId, lane, source); + return publicResult( + await pool.execute(laneRequest as WorkspaceExecuteCommandRequest, signal), + request.workspaceId, + ); + } + const executor = await this.fileExecutor(internalId, lane, source); + return publicResult(await executor.execute(laneRequest, signal), request.workspaceId); + }, signal); } async executeProgrammatic( @@ -369,18 +510,25 @@ export class LinkedWorktreeWorkspaceTools implements WorkspaceToolExecutor { ): Promise { const worktree = request.body.workspace_worktree; if (worktree == null) { - if (!this.options.programmaticDelegate) { + const delegate = this.options.programmaticDelegate; + if (!delegate) { throw new WorkspaceToolError('Workspace programmatic execution is unavailable', 'COMMAND_DISABLED'); } - return await this.options.programmaticDelegate.executeProgrammatic(workspaceId, request, signal); + return await this.inCheckout( + workspaceId, + () => delegate.executeProgrammatic(workspaceId, request, signal), + signal, + ); } - const { lane, source, internalId } = await this.resolveLane( - workspaceId, - worktree, - request.body.workspace_instance_id, - ); - const pool = await this.registerCommandRoot(internalId, lane, source); - const { workspace_worktree: _worktree, ...body } = request.body; - return await pool.executeProgrammatic(internalId, { ...request, body }, signal); + return await this.inLane(workspaceId, worktree, async () => { + const { lane, source, internalId } = await this.resolveLane( + workspaceId, + worktree, + request.body.workspace_instance_id, + ); + const pool = await this.registerCommandRoot(internalId, lane, source); + const { workspace_worktree: _worktree, ...body } = request.body; + return await pool.executeProgrammatic(internalId, { ...request, body }, signal); + }, signal); } } diff --git a/packages/code/src/snapshot-lifecycle.test.ts b/packages/code/src/snapshot-lifecycle.test.ts index 33e37c2d..8074fd11 100644 --- a/packages/code/src/snapshot-lifecycle.test.ts +++ b/packages/code/src/snapshot-lifecycle.test.ts @@ -116,6 +116,54 @@ test('storage policy is opt-in, bounded and fails before a setup command', async assert.equal(executions, 0); }); +test('low space reclaims once before deferring setup', async t => { + const policy = { minFreeBytes: 5, setupReserveBytes: 2 }; + let free = 6n; + let reclaims = 0; + await assertPreparationSpace('/', policy, async () => free, async () => { + reclaims++; + free = 7n; + }); + assert.equal(reclaims, 1); + await assertPreparationSpace('/', policy, async () => free, async () => { + reclaims++; + }); + assert.equal(reclaims, 1, 'enough space never triggers reclamation'); + await assert.rejects( + assertPreparationSpace('/', policy, async () => 6n, async () => { + reclaims++; + }), + /no setup command was started/, + ); + assert.equal(reclaims, 2); + + const { root } = await fixture(t); + const checkout = join(root, 'checkout'); + await mkdir(checkout); + let executions = 0; + let reclaimed = 0; + await assert.rejects( + prepareCodeEnvironment({ + root: checkout, + identity: await captureWorkspaceRootIdentity(checkout), + setup: { command: 'install', timeoutMs: 1000 }, + receiptPath: join(root, 'receipt'), + context: '', + storage: { minFreeBytes: Number.MAX_SAFE_INTEGER, setupReserveBytes: 0 }, + reclaimSpace: async () => { + reclaimed++; + }, + execute: async () => { + executions++; + return { exitCode: 0, timedOut: false }; + }, + }), + /deferred/, + ); + assert.equal(reclaimed, 1); + assert.equal(executions, 0); +}); + test('dry-run is non-destructive; cleanup removes only expired owned entries and retains unknown or linked data', async t => { const { store, add, a, b, c, root } = await fixture(t); const old = await add(a, 4, 10_000); diff --git a/packages/code/src/snapshot-lifecycle.ts b/packages/code/src/snapshot-lifecycle.ts index dfa5926e..66ed39dd 100644 --- a/packages/code/src/snapshot-lifecycle.ts +++ b/packages/code/src/snapshot-lifecycle.ts @@ -63,7 +63,11 @@ export function parseEnvironmentStorage( throw new Error('Invalid environment storage policy'); return policy as EnvironmentStoragePolicy; } -/** Soft admission for managed preparation, not a reservation or arbitrary-write quota. */ +/** + * Soft admission for managed preparation, not a reservation or arbitrary-write + * quota. Below the floor, `reclaim` (when supplied) may free reproducible + * storage once before the space is measured again. + */ export async function assertPreparationSpace( root: string, policy: EnvironmentStoragePolicy, @@ -71,14 +75,18 @@ export async function assertPreparationSpace( const status = await statfs(path, { bigint: true }); return status.bavail * status.bsize; }, + reclaim?: () => Promise, ): Promise { - if ( - (await available(root)) < - BigInt(policy.minFreeBytes) + BigInt(policy.setupReserveBytes) - ) - throw new Error( - 'Managed environment preparation deferred: free disk is below the configured floor plus setup reserve. Clean reproducible artifacts or expand storage; no setup command was started.', - ); + const floor = + BigInt(policy.minFreeBytes) + BigInt(policy.setupReserveBytes); + if ((await available(root)) >= floor) return; + if (reclaim) { + await reclaim(); + if ((await available(root)) >= floor) return; + } + throw new Error( + 'Managed environment preparation deferred: free disk is below the configured floor plus setup reserve. Clean reproducible artifacts or expand storage; no setup command was started.', + ); } export const SNAPSHOT_MANIFEST = '.snapshot.json'; diff --git a/packages/code/src/worktree-retirement.test.ts b/packages/code/src/worktree-retirement.test.ts new file mode 100644 index 00000000..f10ed8f3 --- /dev/null +++ b/packages/code/src/worktree-retirement.test.ts @@ -0,0 +1,592 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { mkdir, mkdtemp, readFile, realpath, rename, rm, stat, utimes, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import test from 'node:test'; + +import { LinkedWorktreeWorkspaceTools } from './linked-worktrees.js'; +import { LocalWorkspaceTools, WorkspaceToolError } from './workspace.js'; +import { + WorktreeRetirementScheduler, + describeWorktreeRetirement, + retireStaleWorktrees, + worktreeRetirementSettings, +} from './worktree-retirement.js'; + +import type { NativeWorkspaceCommandPool } from './native-pool.js'; +import type { NativeProcessSandboxOptions } from './native-process.js'; +import type { WorktreeRetirementOptions } from './worktree-retirement.js'; + +const execFileAsync = promisify(execFile); +const DAY_MS = 24 * 60 * 60 * 1000; + +async function git(cwd: string, ...args: string[]): Promise { + const { stdout } = await execFileAsync( + 'git', + ['-c', 'user.name=test', '-c', 'user.email=test@example.com', '-c', 'commit.gpgsign=false', ...args], + { cwd }, + ); + return stdout.trim(); +} + +async function exists(path: string): Promise { + try { + await stat(path); + return true; + } catch { + return false; + } +} + +/** A checkout on `main` with an `origin` bare remote and an ignored `node_modules`. */ +async function repository(t: test.TestContext): Promise { + const parent = await realpath(await mkdtemp(join(tmpdir(), 'worktree-retirement-'))); + t.after(() => rm(parent, { recursive: true, force: true })); + const remote = join(parent, 'remote.git'); + await git(parent, 'init', '-q', '--bare', remote); + const root = join(parent, 'repo'); + await mkdir(root); + await git(root, 'init', '-q', '-b', 'main'); + await writeFile(join(root, 'README.md'), 'root\n'); + await writeFile(join(root, '.gitignore'), 'node_modules/\n.worktrees/\n'); + await git(root, 'add', '.'); + await git(root, 'commit', '-q', '-m', 'init'); + await git(root, 'remote', 'add', 'origin', remote); + await git(root, 'push', '-q', 'origin', 'main'); + await mkdir(join(root, '.worktrees')); + return root; +} + +/** A task worktree with one commit of its own and ignored dependencies. */ +async function worktree(root: string, name: string, push = true): Promise { + await git(root, 'worktree', 'add', '-q', '-b', name, `.worktrees/${name}`); + const path = join(root, '.worktrees', name); + await writeFile(join(path, `${name}.txt`), `${name}\n`); + await git(path, 'add', '.'); + await git(path, 'commit', '-q', '-m', name); + if (push) await git(path, 'push', '-q', 'origin', name); + await mkdir(join(path, 'node_modules', 'dependency'), { recursive: true }); + await writeFile(join(path, 'node_modules', 'dependency', 'index.js'), 'module.exports = 1;\n'); + return path; +} + +/** Backdate every on-disk activity signal, as if nobody had touched the worktree for `days`. */ +async function age(root: string, name: string, days = 30): Promise { + const when = new Date(Date.now() - days * DAY_MS); + const metadata = join(root, '.git', 'worktrees', name); + const paths = [ + join(root, '.worktrees', name), + metadata, + ...['HEAD', 'index', join('logs', 'HEAD'), 'ORIG_HEAD', 'FETCH_HEAD'].map((path) => join(metadata, path)), + ]; + for (const path of paths) { + await utimes(path, when, when).catch(() => undefined); + } +} + +function sources(root: string): WorktreeRetirementOptions['sources'] { + return [{ workspaceId: 'repo', root }]; +} + +/** A command pool whose lane commands wait until released. */ +function blockingPool(): { + pool: NativeWorkspaceCommandPool; + started: Promise; + finish: () => void; +} { + let started!: () => void; + let finish!: () => void; + const startedPromise = new Promise((resolve) => { started = resolve; }); + const finished = new Promise((resolve) => { finish = resolve; }); + const pool = { + async registerRoot() {}, + async unregisterRoot() {}, + async execute(request: { workspaceId: string }) { + started(); + await finished; + return { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: request.workspaceId, + exitCode: 0, + stdout: '', + stderr: '', + truncated: false, + timedOut: false, + }; + }, + } as unknown as NativeWorkspaceCommandPool; + return { pool, started: startedPromise, finish }; +} + +async function laneTools(root: string, pool?: NativeWorkspaceCommandPool): Promise { + const delegate = await LocalWorkspaceTools.create({ + repositoryInstructions: false, + workspaces: [{ id: 'repo', root, writable: true }], + }); + return new LinkedWorktreeWorkspaceTools({ + commandPool: pool, + delegate, + sources: new Map([ + ['repo', { + root, + command: { workspaceRoot: root } as NativeProcessSandboxOptions, + repositoryInstructions: false, + writable: true, + }], + ]), + }); +} + +const laneCommand = (worktree: string) => ({ + protocolVersion: 1 as const, + operation: 'execute_command' as const, + workspaceId: 'repo', + worktree, + command: 'true', +}); + +const pause = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +test('retires a clean, pushed, idle worktree and keeps its branch for recovery', async (t) => { + const root = await repository(t); + const path = await worktree(root, 'done'); + await age(root, 'done'); + + const summary = await retireStaleWorktrees({ sources: sources(root) }); + + assert.deepEqual(summary.retired, ['repo:done']); + assert.deepEqual(summary.kept, {}); + assert.equal(await exists(path), false, 'the worktree and its ignored files are gone'); + assert.equal(await exists(join(root, '.git', 'worktrees', 'done')), false); + assert.doesNotMatch(await git(root, 'worktree', 'list', '--porcelain'), /\.worktrees\/done/); + const branch = await git(root, 'rev-parse', '--verify', 'refs/heads/done'); + assert.equal(branch, await git(root, 'rev-parse', '--verify', 'refs/remotes/origin/done')); + + await git(root, 'worktree', 'add', '-q', '.worktrees/done', 'done'); + assert.equal(await readFile(join(path, 'done.txt'), 'utf8'), 'done\n'); + assert.equal(await git(root, 'status', '--porcelain'), '', 'the main checkout is untouched'); + assert.equal(await git(root, 'branch', '--show-current'), 'main'); +}); + +test('keeps every worktree that could lose work or is not a linked worktree', async (t) => { + const root = await repository(t); + const worktrees = join(root, '.worktrees'); + + const dirty = await worktree(root, 'dirty'); + await writeFile(join(dirty, 'dirty.txt'), 'changed\n'); + const untracked = await worktree(root, 'untracked'); + await writeFile(join(untracked, 'notes.txt'), 'not yet added\n'); + await worktree(root, 'unpushed', false); + await worktree(root, 'locked'); + await git(root, 'worktree', 'lock', '.worktrees/locked'); + const merging = await worktree(root, 'merging'); + await worktree(root, 'side'); + await git(merging, 'merge', '-q', '--no-commit', '--no-ff', 'side'); + await worktree(root, 'recent'); + const detached = await worktree(root, 'detached'); + await git(detached, 'checkout', '-q', '--detach'); + await writeFile(join(detached, 'local.txt'), 'local\n'); + await git(detached, 'add', '.'); + await git(detached, 'commit', '-q', '-m', 'detached and unpushed'); + await mkdir(join(worktrees, 'forged')); + await writeFile(join(worktrees, 'forged', '.git'), `gitdir: ${join(root, '.git', 'worktrees', 'side')}\n`); + await mkdir(join(worktrees, 'plain')); + for (const name of ['dirty', 'untracked', 'unpushed', 'locked', 'merging', 'side', 'detached']) { + await age(root, name); + } + await age(root, 'forged'); + await age(root, 'plain'); + + const summary = await retireStaleWorktrees({ sources: sources(root) }); + + assert.deepEqual(summary.retired, ['repo:side']); + assert.deepEqual(summary.kept, { + dirty: 2, + unpushed: 2, + locked: 1, + operation: 1, + recent: 1, + unverified: 2, + }); + for (const name of ['dirty', 'untracked', 'unpushed', 'locked', 'merging', 'recent', 'detached', 'forged', 'plain']) { + assert.ok(await exists(join(worktrees, name)), `${name} is kept`); + } + assert.equal(await readFile(join(dirty, 'dirty.txt'), 'utf8'), 'changed\n'); + assert.equal(await readFile(join(untracked, 'notes.txt'), 'utf8'), 'not yet added\n'); + assert.ok(await exists(join(root, '.git', 'worktrees', 'merging', 'MERGE_HEAD'))); + assert.equal(await git(root, 'status', '--porcelain'), ''); + assert.equal(await readFile(join(root, 'README.md'), 'utf8'), 'root\n'); +}); + +test('a detached HEAD is retired only when a remote-tracking ref contains it', async (t) => { + const root = await repository(t); + const path = await worktree(root, 'review'); + await git(path, 'checkout', '-q', '--detach'); + await age(root, 'review'); + + const summary = await retireStaleWorktrees({ sources: sources(root) }); + + assert.deepEqual(summary.retired, ['repo:review']); + assert.equal(await exists(path), false); +}); + +test('a squash- or rebase-merged branch whose remote branch was deleted is retired; unmerged work is kept', async (t) => { + const root = await repository(t); + const parent = join(root, '..'); + const tracked = async (name: string): Promise => { + const path = await worktree(root, name); + await git(path, 'branch', '-q', '--set-upstream-to', `origin/${name}`); + return path; + }; + const squashed = await tracked('squashed'); + await writeFile(join(squashed, 'second.txt'), 'second\n'); + await git(squashed, 'add', '.'); + await git(squashed, 'commit', '-q', '-m', 'second'); + await git(squashed, 'push', '-q', 'origin', 'squashed'); + await tracked('rebased'); + await tracked('abandoned'); + const review = 'review-16677-1f35df8'; + await git(root, 'worktree', 'add', '-q', '--detach', `.worktrees/${review}`, await git(squashed, 'rev-parse', 'HEAD')); + await git(root, 'worktree', 'add', '-q', '-b', 'ahead', '.worktrees/ahead'); + const ahead = join(root, '.worktrees', 'ahead'); + await git(ahead, 'push', '-q', '-u', 'origin', 'ahead'); + await writeFile(join(ahead, 'later.txt'), 'later\n'); + await git(ahead, 'add', '.'); + await git(ahead, 'commit', '-q', '-m', 'later, also landed on main but never pushed here'); + + const hosting = join(parent, 'hosting'); + await git(parent, 'clone', '-q', '-b', 'main', join(parent, 'remote.git'), hosting); + await git(hosting, 'merge', '-q', '--squash', 'origin/squashed'); + await git(hosting, 'commit', '-q', '-m', 'squash merge'); + await git(hosting, 'cherry-pick', 'origin/rebased'); + await writeFile(join(hosting, 'later.txt'), 'later\n'); + await git(hosting, 'add', '.'); + await git(hosting, 'commit', '-q', '-m', 'later'); + await git(hosting, 'push', '-q', 'origin', 'main'); + await git(hosting, 'push', '-q', 'origin', '--delete', 'squashed', 'rebased', 'abandoned'); + await git(root, 'fetch', '-q', '--prune', 'origin'); + for (const name of ['squashed', 'rebased', 'abandoned', review, 'ahead']) await age(root, name); + + const summary = await retireStaleWorktrees({ sources: sources(root) }); + + assert.deepEqual([...summary.retired].sort(), ['repo:rebased', `repo:${review}`, 'repo:squashed']); + assert.deepEqual(summary.kept, { unpushed: 2 }); + assert.ok(await exists(join(root, '.worktrees', 'abandoned')), 'unmerged work with a deleted remote is kept'); + assert.ok(await exists(ahead), 'commits beyond a live upstream are kept even when main has the same change'); + for (const branch of ['squashed', 'rebased', 'abandoned', 'ahead']) { + await git(root, 'rev-parse', '--verify', `refs/heads/${branch}`); + } +}); + +test('never touches the main checkout or worktrees outside .worktrees', async (t) => { + const root = await repository(t); + const outside = join(root, '..', 'outside'); + await git(root, 'worktree', 'add', '-q', '-b', 'outside', outside); + await git(outside, 'push', '-q', 'origin', 'outside'); + const nested = join(root, 'nested'); + await git(root, 'worktree', 'add', '-q', '-b', 'nested', nested); + await git(nested, 'push', '-q', 'origin', 'nested'); + const old = new Date(Date.now() - 30 * DAY_MS); + for (const path of [root, join(root, '.git'), join(root, '.git', 'HEAD'), join(root, '.git', 'index'), outside, nested]) { + await utimes(path, old, old); + } + + const summary = await retireStaleWorktrees({ sources: sources(root) }); + + assert.deepEqual(summary.retired, []); + assert.ok(await exists(join(root, 'README.md'))); + assert.ok(await exists(outside)); + assert.ok(await exists(nested)); + assert.equal((await git(root, 'worktree', 'list', '--porcelain')).match(/^worktree /gm)?.length, 3); +}); + +test('this worker\'s own lane use counts as activity, and an active lane is never retired', async (t) => { + const root = await repository(t); + const path = await worktree(root, 'busy'); + await age(root, 'busy'); + const { pool, started, finish } = blockingPool(); + const tools = await laneTools(root, pool); + + const running = tools.execute(laneCommand('busy')); + await started; + await pause(5); + const during = await retireStaleWorktrees({ sources: sources(root), activity: tools, idleMs: 1 }); + assert.deepEqual(during.kept, { active: 1 }); + assert.ok(await exists(path)); + + finish(); + await running; + const recent = await retireStaleWorktrees({ sources: sources(root), activity: tools }); + assert.deepEqual(recent.kept, { recent: 1 }, 'the default threshold counts this process\'s last use'); + assert.ok(await exists(path)); + + await pause(5); + const after = await retireStaleWorktrees({ sources: sources(root), activity: tools, idleMs: 1 }); + assert.deepEqual(after.retired, ['repo:busy']); + assert.equal(await exists(path), false); + await assert.rejects( + tools.execute(laneCommand('busy')), + (error: unknown) => error instanceof WorkspaceToolError && error.code === 'INVALID_REQUEST', + ); +}); + +test('a lane request that arrives during retirement waits, then finds the lane gone', async (t) => { + const root = await repository(t); + await worktree(root, 'leaving'); + let executions = 0; + const pool = { + async registerRoot() {}, + async unregisterRoot() {}, + async execute() { + executions += 1; + throw new Error('a retired lane must not run commands'); + }, + } as unknown as NativeWorkspaceCommandPool; + const tools = await laneTools(root, pool); + let release!: () => void; + const gate = new Promise((resolve) => { release = resolve; }); + + const retiring = tools.whileIdle('repo', 'leaving', async () => { + await gate; + await git(root, 'worktree', 'remove', '.worktrees/leaving'); + return 'removed'; + }); + let settled = false; + const request = tools.execute(laneCommand('leaving')); + request.catch(() => undefined).finally(() => { settled = true; }); + let checkoutSettled = false; + const checkout = tools.execute({ protocolVersion: 1, operation: 'read_file', workspaceId: 'repo', path: 'README.md' }); + checkout.finally(() => { checkoutSettled = true; }); + const cancelled = new AbortController(); + const abandoned = tools.execute(laneCommand('leaving'), cancelled.signal); + await pause(20); + assert.equal(settled, false, 'the lane request waits for retirement'); + assert.equal(checkoutSettled, false, 'a checkout request waits too: it can reach the lane being removed'); + assert.deepEqual(await tools.whileIdle('repo', 'leaving', async () => 'again'), { ran: false }); + cancelled.abort(new Error('caller gave up')); + await assert.rejects(abandoned, /caller gave up/, 'cancellation ends the wait without the retirement'); + + release(); + assert.deepEqual(await retiring, { ran: true, value: 'removed' }); + await assert.rejects( + request, + (error: unknown) => error instanceof WorkspaceToolError && error.code === 'INVALID_REQUEST', + ); + const read = await checkout; + assert.equal(read.operation === 'read_file' && read.content.trimEnd(), 'root'); + assert.equal(executions, 0); +}); + +test('retirement never prunes other registered worktrees that are temporarily missing', async (t) => { + const root = await repository(t); + const external = join(root, '..', 'external'); + await git(root, 'worktree', 'add', '-q', '-b', 'external', external); + await rename(external, `${external}.unmounted`); + await worktree(root, 'done'); + await age(root, 'done'); + + const summary = await retireStaleWorktrees({ sources: sources(root) }); + + assert.deepEqual(summary.retired, ['repo:done']); + assert.ok(await exists(join(root, '.git', 'worktrees', 'external')), 'the unavailable worktree stays registered'); + assert.equal(await exists(join(root, '.git', 'worktrees', 'done')), false, 'removal deletes its own metadata'); + await rename(`${external}.unmounted`, external); + assert.equal(await git(external, 'branch', '--show-current'), 'external'); +}); + +test('a checkout request in flight defers retirement of its lanes', async (t) => { + const root = await repository(t); + const path = await worktree(root, 'quiet'); + await age(root, 'quiet'); + let release!: () => void; + let entered!: () => void; + const inside = new Promise((resolve) => { entered = resolve; }); + const gate = new Promise((resolve) => { release = resolve; }); + const delegate = await LocalWorkspaceTools.create({ + repositoryInstructions: false, + workspaces: [{ id: 'repo', root, writable: true }], + }); + const tools = new LinkedWorktreeWorkspaceTools({ + delegate: { + capabilities: delegate.capabilities, + async execute(request, signal) { + entered(); + await gate; + return await delegate.execute(request, signal); + }, + }, + sources: new Map([['repo', { root, repositoryInstructions: false, writable: true }]]), + }); + + const reading = tools.execute({ protocolVersion: 1, operation: 'read_file', workspaceId: 'repo', path: 'README.md' }); + await inside; + const during = await retireStaleWorktrees({ sources: sources(root), activity: tools }); + assert.deepEqual(during.kept, { active: 1 }); + release(); + await reading; + assert.ok(await exists(path)); + + const after = await retireStaleWorktrees({ sources: sources(root), activity: tools }); + assert.deepEqual(after.retired, ['repo:quiet']); +}); + +test('a checkout request that runs during inspection defers retirement', async (t) => { + const root = await repository(t); + await worktree(root, 'touched'); + await age(root, 'touched'); + const tools = await laneTools(root); + const before = tools.checkoutActivity('repo'); + await tools.execute({ protocolVersion: 1, operation: 'read_file', workspaceId: 'repo', path: 'README.md' }); + assert.equal(tools.checkoutActivity('repo'), before + 2, 'a checkout request counts at start and finish'); + + let reads = 0; + const duringInspection = { + lastUsed: (workspaceId: string, name: string) => tools.lastUsed(workspaceId, name), + // The second read, under the reservation, sees a checkout request that ran during inspection. + checkoutActivity: () => reads++, + whileIdle: (workspaceId: string, name: string, task: () => Promise) => tools.whileIdle(workspaceId, name, task), + }; + const deferred = await retireStaleWorktrees({ sources: sources(root), activity: duringInspection }); + assert.deepEqual(deferred.kept, { changed: 1 }); + assert.ok(await exists(join(root, '.worktrees', 'touched'))); + + const quiet = await retireStaleWorktrees({ sources: sources(root), activity: tools }); + assert.deepEqual(quiet.retired, ['repo:touched']); +}); + +test('quarantined lanes and checkouts are left for the operator', async (t) => { + const root = await repository(t); + await worktree(root, 'held'); + await worktree(root, 'free'); + await age(root, 'held'); + await age(root, 'free'); + + const checkoutHeld = await retireStaleWorktrees({ + sources: sources(root), + isQuarantined: async (_workspaceId, worktree) => worktree == null, + }); + assert.deepEqual(checkoutHeld, { retired: [], kept: {}, freedBytes: 0, truncated: false }); + + const laneHeld = await retireStaleWorktrees({ + sources: sources(root), + isQuarantined: async (_workspaceId, worktree) => worktree === 'held', + }); + assert.deepEqual(laneHeld.retired, ['repo:free']); + assert.deepEqual(laneHeld.kept, { quarantined: 1 }); + assert.ok(await exists(join(root, '.worktrees', 'held'))); +}); + +test('a quarantine that appears after inspection is caught under the lane reservation', async (t) => { + const root = await repository(t); + await worktree(root, 'late'); + await age(root, 'late'); + for (const quarantinedLater of [undefined, 'late']) { + const asked: string[] = []; + const summary = await retireStaleWorktrees({ + sources: sources(root), + activity: await laneTools(root), + isQuarantined: async (_workspaceId, worktree) => { + asked.push(worktree ?? ''); + // Clear for the scan and inspection; quarantined once the lane is reserved. + return asked.length > 2 && worktree === quarantinedLater; + }, + }); + assert.deepEqual(summary.kept, { quarantined: 1 }); + assert.deepEqual(asked, ['', 'late', '', ...(quarantinedLater ? ['late'] : [])]); + assert.ok(await exists(join(root, '.worktrees', 'late'))); + } +}); + +test('one worktree failing does not stop the pass', async (t) => { + const root = await repository(t); + await worktree(root, 'broken'); + await worktree(root, 'fine'); + await rm(join(root, '.git', 'worktrees', 'broken', 'HEAD')); + await age(root, 'broken'); + await age(root, 'fine'); + + const summary = await retireStaleWorktrees({ sources: sources(root) }); + + assert.deepEqual(summary.retired, ['repo:fine']); + assert.equal(Object.values(summary.kept).reduce((total, count) => total + (count ?? 0), 0), 1); + assert.ok(await exists(join(root, '.worktrees', 'broken'))); +}); + +test('worktrees kept for lasting reasons cannot starve the rest of a backlog', async (t) => { + const root = await repository(t); + for (const [index, name] of ['first', 'second', 'third'].entries()) { + await worktree(root, name, false); + await age(root, name, 40 - index); + } + await worktree(root, 'newest'); + await age(root, 'newest', 10); + const rotation = new Set(); + const options = { sources: sources(root), rotation, limits: { inspect: 2 } }; + + const first = await retireStaleWorktrees(options); + assert.deepEqual(first.retired, []); + assert.deepEqual(first.kept, { unpushed: 2, deferred: 2 }); + assert.equal(first.truncated, true, 'uninspected worktrees remain in this rotation'); + + const second = await retireStaleWorktrees(options); + assert.deepEqual(second.retired, ['repo:newest']); + assert.deepEqual(second.kept, { unpushed: 1, deferred: 2 }); + assert.equal(second.truncated, false, 'the rotation is complete'); + + const third = await retireStaleWorktrees(options); + assert.deepEqual(third.kept, { unpushed: 2, deferred: 1 }, 'a new rotation starts from the oldest'); + assert.equal(third.truncated, true); +}); + +test('retirement is on by default and can be disabled or retuned', () => { + assert.deepEqual(worktreeRetirementSettings({ optOut: false }), { enabled: true, idleMs: 7 * DAY_MS }); + assert.equal(worktreeRetirementSettings({ optOut: true }).enabled, false); + assert.equal(worktreeRetirementSettings({ optOut: false, enabled: 'FALSE' }).enabled, false); + assert.equal(worktreeRetirementSettings({ optOut: true, enabled: 'true' }).enabled, false); + assert.equal(worktreeRetirementSettings({ optOut: false, enabled: ' ' }).enabled, true); + assert.equal(worktreeRetirementSettings({ optOut: false, idleDays: '14' }).idleMs, 14 * DAY_MS); + assert.throws(() => worktreeRetirementSettings({ optOut: false, enabled: 'off' }), /must be true or false/); + for (const idleDays of ['0', '-1', '1.5', 'week', '3651']) { + assert.throws(() => worktreeRetirementSettings({ optOut: false, idleDays }), /IDLE_DAYS/); + } +}); + +test('scheduled passes never overlap and stop cleanly', async (t) => { + const root = await repository(t); + await worktree(root, 'scheduled'); + await age(root, 'scheduled'); + const messages: string[] = []; + const scheduler = new WorktreeRetirementScheduler({ + sources: sources(root), + startDelayMs: 60_000, + log: (level, message) => { + if (level === 'info') messages.push(message); + }, + }); + scheduler.start(); + + const first = scheduler.runNow(); + assert.equal(scheduler.runNow(), first, 'a pass requested while one runs joins it'); + const summary = await first; + assert.deepEqual(summary?.retired, ['repo:scheduled']); + assert.equal(messages.length, 1); + assert.match(messages[0]!, /^worktree retirement: retired 1, kept 0(, freed about .+)?$/); + + await scheduler.stop(); + assert.equal(await scheduler.runNow(), undefined, 'a stopped scheduler runs nothing'); +}); + +test('summaries count reasons and note a remaining backlog', () => { + assert.equal( + describeWorktreeRetirement({ + retired: ['repo:a', 'repo:b'], + kept: { unpushed: 2, dirty: 1, recent: 4 }, + freedBytes: 3 * 1024 ** 3, + truncated: true, + }), + 'worktree retirement: retired 2, kept 7 (dirty 1, recent 4, unpushed 2), freed about 3.0 GiB, more next pass', + ); +}); diff --git a/packages/code/src/worktree-retirement.ts b/packages/code/src/worktree-retirement.ts new file mode 100644 index 00000000..39a7b60c --- /dev/null +++ b/packages/code/src/worktree-retirement.ts @@ -0,0 +1,697 @@ +import { execFile } from 'node:child_process'; +import { lstat, opendir, statfs } from 'node:fs/promises'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; + +import { LINKED_WORKTREE_DIRECTORY, verifyLinkedWorktree } from './linked-worktrees.js'; +import { isValidLinkedWorktreeName } from './protocol.js'; + +import type { LinkedWorktreeActivity, VerifiedLinkedWorktree } from './linked-worktrees.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; + +const execFileAsync = promisify(execFile); + +const DAY_MS = 24 * 60 * 60 * 1000; +export const DEFAULT_WORKTREE_IDLE_DAYS = 7; +const MAX_WORKTREE_IDLE_DAYS = 3650; +export const WORKTREE_RETIREMENT_INTERVAL_MS = 6 * 60 * 60 * 1000; +const WORKTREE_RETIREMENT_START_DELAY_MS = 2 * 60 * 1000; +/** A pass that hit a bound continues soon instead of waiting a full interval. */ +const WORKTREE_RETIREMENT_CONTINUATION_MS = 5 * 60 * 1000; +/** + * Directory entries read beneath one checkout's `.worktrees` per pass: a + * memory bound far above any real checkout, so every pass sees every entry. + * Git inspection is what is bounded per pass, and rotation keeps it fair. + */ +const SCAN_LIMIT = 65_536; +/** Idle worktrees whose Git state is inspected per pass, oldest first. */ +const INSPECT_LIMIT = 128; +/** Worktrees removed per pass. */ +const RETIRE_LIMIT = 32; +const GIT_TIMEOUT_MS = 30_000; +/** + * Removal deletes ignored build output too, which can take a while. It has no + * timeout and ignores cancellation: a half-deleted worktree loses its `.git` + * file and can no longer be verified, retired or recognized. + */ +const GIT_REMOVE_NO_TIMEOUT = 0; +const GIT_OUTPUT_LIMIT = 64 * 1024; +/** Path and commit lists between a stale branch and its default branch can be long. */ +const GIT_LIST_LIMIT = 16 * 1024 * 1024; +/** Per-worktree Git state that marks a merge, rebase, cherry-pick, revert or bisect in progress. */ +const OPERATION_STATE = [ + 'MERGE_HEAD', + 'CHERRY_PICK_HEAD', + 'REVERT_HEAD', + 'BISECT_LOG', + 'BISECT_START', + 'rebase-merge', + 'rebase-apply', + 'sequencer', +]; +/** Per-worktree Git metadata that Git rewrites whenever the worktree is used. */ +const ACTIVITY_STATE = ['HEAD', 'index', join('logs', 'HEAD'), 'ORIG_HEAD', 'FETCH_HEAD']; + +export type WorktreeKeptReason = + | 'unverified' + | 'recent' + | 'quarantined' + | 'locked' + | 'operation' + | 'no-commit' + | 'dirty' + | 'unpushed' + | 'active' + | 'changed' + | 'failed' + | 'deferred'; + +export interface WorktreeRetirementSource { + workspaceId: string; + root: string; + identity?: WorkspaceRootIdentity; +} + +export interface WorktreeRetirementOptions { + sources: readonly WorktreeRetirementSource[]; + /** Lane use in this worker; without it, only on-disk activity counts. */ + activity?: LinkedWorktreeActivity; + idleMs?: number; + /** A quarantined checkout or lane awaits an operator and is never retired. */ + isQuarantined?: (workspaceId: string, worktree?: string) => Promise; + log?: (level: 'debug' | 'info', message: string) => void; + /** + * Idle worktrees already inspected in the current rotation. A pass inspects + * the others first and records what it inspects, so worktrees kept for a + * lasting reason cannot starve the rest of a backlog larger than one pass. + */ + rotation?: Set; + /** Per-pass bounds; the defaults suit production. */ + limits?: { inspect?: number; retire?: number }; +} + +export interface WorktreeRetirementSummary { + /** `:` for each retired worktree. */ + retired: string[]; + kept: Partial>; + /** Free-space gain on the checkouts' filesystems across the pass; other writers make it approximate. */ + freedBytes: number; + /** Idle worktrees remain that this rotation has not inspected yet. */ + truncated: boolean; +} + +export interface WorktreeRetirementSettings { + enabled: boolean; + idleMs: number; +} + +interface Candidate { + source: WorktreeRetirementSource; + name: string; + lane: VerifiedLinkedWorktree; + metadata: string; + /** Newest on-disk activity, re-read under the reservation to detect use during inspection. */ + diskActiveAt: number; + /** This worker's own last use of the lane, re-read under the reservation. */ + usedAt?: number; + /** Checkout request activity when inspection began, re-read under the reservation. */ + checkoutAt?: number; + /** Newest of on-disk activity and this worker's own lane use. */ + activeAt: number; +} + +type Kept = { kept: WorktreeKeptReason; detail?: string }; +type Verdict = Kept | { head: string; basis: string }; + +/** + * Resolve the operator's choice. Retirement is on unless explicitly disabled; + * malformed values fail startup rather than silently choosing either way. + */ +export function worktreeRetirementSettings(input: { + optOut: boolean; + enabled?: string; + idleDays?: string; +}): WorktreeRetirementSettings { + const enabled = input.enabled?.trim().toLowerCase() ?? ''; + if (enabled !== '' && enabled !== 'true' && enabled !== 'false') { + throw new Error('LIBRECHAT_CODE_WORKTREE_RETIREMENT must be true or false'); + } + const idleDays = input.idleDays?.trim() ?? ''; + const days = idleDays === '' ? DEFAULT_WORKTREE_IDLE_DAYS : Number(idleDays); + if (!Number.isSafeInteger(days) || days < 1 || days > MAX_WORKTREE_IDLE_DAYS) { + throw new Error( + `LIBRECHAT_CODE_WORKTREE_IDLE_DAYS must be an integer between 1 and ${MAX_WORKTREE_IDLE_DAYS}`, + ); + } + return { enabled: !input.optOut && enabled !== 'false', idleMs: days * DAY_MS }; +} + +function gitEnvironment(): NodeJS.ProcessEnv { + return { + PATH: process.env.PATH, + SYSTEMROOT: process.env.SYSTEMROOT, + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_TERMINAL_PROMPT: '0', + GIT_OPTIONAL_LOCKS: '0', + LC_ALL: 'C', + }; +} + +async function git( + cwd: string, + args: string[], + signal?: AbortSignal, + timeout = GIT_TIMEOUT_MS, + maxBuffer = GIT_OUTPUT_LIMIT, +): Promise { + const { stdout } = await execFileAsync( + 'git', + ['--no-optional-locks', '-C', cwd, '-c', 'core.fsmonitor=false', ...args], + { encoding: 'utf8', env: gitEnvironment(), maxBuffer, signal, timeout }, + ); + return stdout; +} + +/** The commit a ref names, or undefined when it does not resolve. */ +async function resolveCommit(cwd: string, ref: string, signal?: AbortSignal): Promise { + try { + const commit = (await git(cwd, ['rev-parse', '--verify', '--quiet', `${ref}^{commit}`], signal)).trim(); + return /^[0-9a-f]{40,64}$/.test(commit) ? commit : undefined; + } catch { + signal?.throwIfAborted(); + return undefined; + } +} + +function pathList(output: string): string[] { + return output.split('\0').filter(Boolean); +} + +/** The remote's default branch as last fetched: its `HEAD` symref, else `main` or `master`. */ +async function remoteDefaultBranch( + checkout: string, + remote: string, + signal?: AbortSignal, +): Promise<{ ref: string; commit: string } | undefined> { + let symbolic: string | undefined; + try { + symbolic = (await git(checkout, ['symbolic-ref', '-q', `refs/remotes/${remote}/HEAD`], signal)).trim(); + } catch { + signal?.throwIfAborted(); + } + const refs = [ + ...(symbolic?.startsWith(`refs/remotes/${remote}/`) ? [symbolic] : []), + `refs/remotes/${remote}/main`, + `refs/remotes/${remote}/master`, + ]; + for (const ref of refs) { + const commit = await resolveCommit(checkout, ref, signal); + if (commit) return { ref: ref.replace(/^refs\/remotes\//, ''), commit }; + } + return undefined; +} + +/** + * Whether a HEAD that no remote-tracking ref contains is nonetheless already + * in its remote's default branch by content, as after a squash or rebase merge + * whose remote branch was then deleted. Only a HEAD without a live upstream + * qualifies: detached, never pushed, or whose upstream ref is gone. A live + * upstream that lacks HEAD means unpushed commits. Content counts when every + * commit beyond the default branch is patch-equivalent to one in it (and none + * is a merge, which could carry its own changes), or when the default branch + * has identical content at every path the branch changed since their merge + * base. Local refs only; nothing is fetched. + */ +async function mergedByContent( + lane: VerifiedLinkedWorktree, + head: string, + signal?: AbortSignal, +): Promise { + const checkout = lane.checkoutRoot; + let branch: string | undefined; + try { + branch = (await git(lane.root, ['symbolic-ref', '-q', 'HEAD'], signal)).trim() || undefined; + } catch { + signal?.throwIfAborted(); + } + let remote = 'origin'; + if (branch?.startsWith('refs/heads/')) { + const [upstream = '', upstreamRemote = ''] = ( + await git(checkout, ['for-each-ref', '--format=%(upstream)%00%(upstream:remotename)', branch], signal) + ) + .trim() + .split('\0'); + if (upstream) { + if (await resolveCommit(checkout, upstream, signal)) return undefined; + if (/^[A-Za-z0-9._-]+$/.test(upstreamRemote)) remote = upstreamRemote; + } + } + const target = await remoteDefaultBranch(checkout, remote, signal); + if (!target) return undefined; + const merges = (await git(checkout, ['rev-list', '--count', '--merges', `${target.commit}..${head}`], signal)).trim(); + if (merges === '0') { + const cherry = (await git(checkout, ['cherry', target.commit, head], signal, GIT_TIMEOUT_MS, GIT_LIST_LIMIT)) + .split('\n') + .filter(Boolean); + if (cherry.every((line) => line.startsWith('- '))) return `patch-equivalent to ${target.ref}`; + } + let base: string; + try { + base = (await git(checkout, ['merge-base', target.commit, head], signal)).trim(); + } catch { + signal?.throwIfAborted(); + return undefined; + } + const diff = (from: string): Promise => + git( + checkout, + ['diff', '--name-only', '-z', '--no-renames', '--no-ext-diff', '--no-textconv', from, head], + signal, + GIT_TIMEOUT_MS, + GIT_LIST_LIMIT, + ); + const touched = pathList(await diff(base)); + const differing = new Set(pathList(await diff(target.commit))); + return touched.every((path) => !differing.has(path)) ? `same content as ${target.ref}` : undefined; +} + +function errorDetail(error: unknown): string { + const stderr = (error as { stderr?: unknown }).stderr; + const text = typeof stderr === 'string' && stderr.trim() ? stderr : error instanceof Error ? error.message : String(error); + return text.trim().split('\n')[0]!.slice(0, 200); +} + +/** Absent is the only answer that clears a check; anything unreadable counts as present. */ +async function present(path: string): Promise { + try { + await lstat(path); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code !== 'ENOENT'; + } +} + +async function modifiedAt(path: string): Promise { + try { + return (await lstat(path)).mtimeMs; + } catch { + return 0; + } +} + +/** The newest on-disk sign of use: the worktree directory itself and its Git metadata. */ +async function lastActivity(root: string, metadata: string): Promise { + const times = await Promise.all( + [root, metadata, ...ACTIVITY_STATE.map((path) => join(metadata, path))].map(modifiedAt), + ); + return Math.max(...times); +} + +async function worktreeNames(checkout: string): Promise<{ names: string[]; truncated: boolean }> { + let directory; + try { + directory = await opendir(join(checkout, LINKED_WORKTREE_DIRECTORY)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return { names: [], truncated: false }; + throw error; + } + const names: string[] = []; + try { + for await (const entry of directory) { + if (names.length >= SCAN_LIMIT) return { names, truncated: true }; + if (isValidLinkedWorktreeName(entry.name)) names.push(entry.name); + } + } finally { + await directory.close().catch(() => undefined); + } + return { names, truncated: false }; +} + +async function freeSpace(paths: readonly string[]): Promise> { + const space = new Map(); + for (const path of paths) { + try { + const device = String((await lstat(path)).dev); + if (space.has(device)) continue; + const status = await statfs(path, { bigint: true }); + space.set(device, Number(status.bavail * status.bsize)); + } catch { + // Space reporting is informational; it never decides a retirement. + } + } + return space; +} + +function describeChanges(status: string): string { + const entries = status.split('\0').filter(Boolean); + const untracked = entries.filter((entry) => entry.startsWith('?? ')).length; + return untracked === entries.length + ? 'untracked files' + : untracked > 0 + ? 'tracked changes and untracked files' + : 'tracked changes'; +} + +function formatBytes(bytes: number): string { + const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB']; + let value = bytes; + let unit = 0; + while (value >= 1024 && unit < units.length - 1) { + value /= 1024; + unit += 1; + } + return `${unit === 0 ? value : value.toFixed(1)} ${units[unit]}`; +} + +export function describeWorktreeRetirement(summary: WorktreeRetirementSummary): string { + const kept = Object.entries(summary.kept) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([reason, count]) => `${reason} ${count}`); + const keptTotal = Object.values(summary.kept).reduce((total, count) => total + (count ?? 0), 0); + return [ + `worktree retirement: retired ${summary.retired.length}`, + `kept ${keptTotal}${kept.length ? ` (${kept.join(', ')})` : ''}`, + ...(summary.freedBytes > 0 ? [`freed about ${formatBytes(summary.freedBytes)}`] : []), + ...(summary.truncated ? ['more next pass'] : []), + ].join(', '); +} + +/** Everything short of the worker's own activity that could make removal lose work. */ +async function inspect( + candidate: Candidate, + options: WorktreeRetirementOptions, + signal?: AbortSignal, +): Promise { + const { lane, metadata, name, source } = candidate; + try { + if (await options.isQuarantined?.(source.workspaceId, name)) return { kept: 'quarantined' }; + } catch { + return { kept: 'quarantined', detail: 'quarantine state is unreadable' }; + } + if (await present(join(metadata, 'locked'))) return { kept: 'locked' }; + for (const state of OPERATION_STATE) { + if (await present(join(metadata, state))) return { kept: 'operation', detail: state }; + } + let head: string; + try { + head = (await git(lane.root, ['rev-parse', '--verify', '--quiet', 'HEAD^{commit}'], signal)).trim(); + } catch { + signal?.throwIfAborted(); + return { kept: 'no-commit' }; + } + if (!/^[0-9a-f]{40,64}$/.test(head)) return { kept: 'no-commit' }; + let status: string; + try { + status = await git( + lane.root, + ['status', '--porcelain', '-z', '--untracked-files=normal', '--ignore-submodules=none'], + signal, + ); + } catch (error) { + signal?.throwIfAborted(); + if ((error as NodeJS.ErrnoException).code === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER') { + return { kept: 'dirty', detail: 'too many changes to list' }; + } + return { kept: 'failed', detail: `status: ${errorDetail(error)}` }; + } + if (status.length > 0) return { kept: 'dirty', detail: describeChanges(status) }; + let remote: string; + try { + remote = ( + await git( + lane.checkoutRoot, + ['for-each-ref', '--count=1', '--contains', head, '--format=%(refname)', 'refs/remotes/'], + signal, + ) + ).trim(); + } catch (error) { + signal?.throwIfAborted(); + return { kept: 'failed', detail: `remote containment: ${errorDetail(error)}` }; + } + if (remote) return { head, basis: `contained in ${remote.replace(/^refs\/remotes\//, '')}` }; + let merged: string | undefined; + try { + merged = await mergedByContent(lane, head, signal); + } catch (error) { + signal?.throwIfAborted(); + return { kept: 'failed', detail: `merged content: ${errorDetail(error)}` }; + } + return merged + ? { head, basis: merged } + : { kept: 'unpushed', detail: 'not on a remote-tracking ref or merged into the default branch' }; +} + +/** Re-checked under the lane reservation: anything that moved since inspection defers to the next pass. */ +async function remove( + candidate: Candidate, + head: string, + options: WorktreeRetirementOptions, + signal?: AbortSignal, +): Promise { + const { lane, metadata, name, source } = candidate; + if (options.activity?.lastUsed(source.workspaceId, name) !== candidate.usedAt) { + return { kept: 'changed', detail: 'used during inspection' }; + } + // A checkout request can reach `.worktrees/*`, including ignored files that `git status` never reports. + if (options.activity?.checkoutActivity(source.workspaceId) !== candidate.checkoutAt) { + return { kept: 'changed', detail: 'checkout used during inspection' }; + } + // A request that finished between inspection and this reservation may have quarantined either. + try { + if ( + (await options.isQuarantined?.(source.workspaceId)) || + (await options.isQuarantined?.(source.workspaceId, name)) + ) { + return { kept: 'quarantined' }; + } + } catch { + return { kept: 'quarantined', detail: 'quarantine state is unreadable' }; + } + let current: VerifiedLinkedWorktree; + try { + current = await verifyLinkedWorktree(source.root, name, source.identity); + } catch { + return { kept: 'changed', detail: 'no longer a verified linked worktree' }; + } + if (current.identity.dev !== lane.identity.dev || current.identity.ino !== lane.identity.ino) { + return { kept: 'changed', detail: 'worktree directory was replaced' }; + } + if ((await lastActivity(lane.root, metadata)) !== candidate.diskActiveAt) { + return { kept: 'changed', detail: 'used during inspection' }; + } + if (await present(join(metadata, 'locked'))) return { kept: 'locked' }; + try { + const now = (await git(lane.root, ['rev-parse', '--verify', '--quiet', 'HEAD^{commit}'], signal)).trim(); + if (now !== head) return { kept: 'changed', detail: 'HEAD moved during inspection' }; + } catch { + signal?.throwIfAborted(); + return { kept: 'changed', detail: 'HEAD is unreadable' }; + } + signal?.throwIfAborted(); + try { + // Never --force: Git re-checks for changes and untracked files itself and refuses a dirty or locked worktree. + await git(lane.checkoutRoot, ['worktree', 'remove', lane.root], undefined, GIT_REMOVE_NO_TIMEOUT); + } catch (error) { + return { kept: 'failed', detail: `remove: ${errorDetail(error)}` }; + } + return 'retired'; +} + +/** + * Retire linked worktrees beneath each checkout's `.worktrees` that are + * verified, idle, clean, free of in-progress operations and locks, and whose + * HEAD is contained in a remote-tracking ref or already merged into the + * remote's default branch by content. Removal is `git worktree remove` + * without `--force`, which also deletes that worktree's own metadata; no + * repository-wide `git worktree prune` runs, since it would also expire other + * registered worktrees that are only temporarily unavailable. Branches are + * kept, so `git worktree add` restores any of them. Each worktree is judged + * independently; one failure never ends the pass. + */ +export async function retireStaleWorktrees( + options: WorktreeRetirementOptions, + signal?: AbortSignal, +): Promise { + const now = Date.now(); + const idleMs = options.idleMs ?? DEFAULT_WORKTREE_IDLE_DAYS * DAY_MS; + const summary: WorktreeRetirementSummary = { retired: [], kept: {}, freedBytes: 0, truncated: false }; + const keep = (source: WorktreeRetirementSource, name: string, reason: WorktreeKeptReason, detail?: string): void => { + summary.kept[reason] = (summary.kept[reason] ?? 0) + 1; + options.log?.('debug', `worktree retirement kept ${source.workspaceId}:${name}: ${reason}${detail ? ` (${detail})` : ''}`); + }; + const candidates: Candidate[] = []; + for (const source of options.sources) { + signal?.throwIfAborted(); + try { + if (await options.isQuarantined?.(source.workspaceId)) { + options.log?.('debug', `worktree retirement skipped ${source.workspaceId}: checkout is quarantined`); + continue; + } + const { names, truncated } = await worktreeNames(source.root); + if (truncated) { + options.log?.('debug', `worktree retirement read only the first ${SCAN_LIMIT} entries of ${source.workspaceId}`); + } + for (const name of names) { + signal?.throwIfAborted(); + let lane: VerifiedLinkedWorktree; + try { + lane = await verifyLinkedWorktree(source.root, name, source.identity); + } catch (error) { + keep(source, name, 'unverified', errorDetail(error)); + continue; + } + const metadata = join(lane.commonGitDir, 'worktrees', name); + const diskActiveAt = await lastActivity(lane.root, metadata); + const usedAt = options.activity?.lastUsed(source.workspaceId, name); + const activeAt = Math.max(diskActiveAt, usedAt ?? 0); + if (now - activeAt < idleMs) { + keep(source, name, 'recent'); + continue; + } + candidates.push({ source, name, lane, metadata, diskActiveAt, usedAt, activeAt }); + } + } catch (error) { + signal?.throwIfAborted(); + options.log?.('debug', `worktree retirement skipped ${source.workspaceId}: ${errorDetail(error)}`); + } + } + if (candidates.length === 0) { + options.rotation?.clear(); + return summary; + } + const { rotation } = options; + const key = (candidate: Candidate): string => `${candidate.source.workspaceId}\0${candidate.name}`; + if (rotation) { + const current = new Set(candidates.map(key)); + for (const entry of rotation) { + if (!current.has(entry)) rotation.delete(entry); + } + if (rotation.size === current.size) rotation.clear(); + } + const visited = (candidate: Candidate): number => (rotation?.has(key(candidate)) ? 1 : 0); + candidates.sort((a, b) => visited(a) - visited(b) || a.activeAt - b.activeAt); + const checkouts = [...new Set(candidates.map((candidate) => candidate.lane.checkoutRoot))]; + const before = await freeSpace(checkouts); + const retiredCheckouts = new Set(); + let inspected = 0; + for (const candidate of candidates) { + signal?.throwIfAborted(); + const { source, name } = candidate; + if ( + inspected >= (options.limits?.inspect ?? INSPECT_LIMIT) || + summary.retired.length >= (options.limits?.retire ?? RETIRE_LIMIT) + ) { + summary.truncated ||= visited(candidate) === 0; + keep(source, name, 'deferred'); + continue; + } + inspected += 1; + rotation?.add(key(candidate)); + try { + candidate.checkoutAt = options.activity?.checkoutActivity(source.workspaceId); + const verdict = await inspect(candidate, options, signal); + if ('kept' in verdict) { + keep(source, name, verdict.kept, verdict.detail); + continue; + } + const retire = (): Promise => remove(candidate, verdict.head, options, signal); + const reserved = options.activity + ? await options.activity.whileIdle(source.workspaceId, name, retire) + : { ran: true as const, value: await retire() }; + if (!reserved.ran) { + keep(source, name, 'active'); + continue; + } + if (reserved.value !== 'retired') { + keep(source, name, reserved.value.kept, reserved.value.detail); + continue; + } + summary.retired.push(`${source.workspaceId}:${name}`); + retiredCheckouts.add(candidate.lane.checkoutRoot); + options.log?.('debug', `worktree retirement retired ${source.workspaceId}:${name} (${verdict.basis}); its branch is kept`); + } catch (error) { + signal?.throwIfAborted(); + keep(source, name, 'failed', errorDetail(error)); + } + } + if (retiredCheckouts.size > 0) { + const after = await freeSpace(checkouts); + for (const [device, free] of after) { + summary.freedBytes += Math.max(0, free - (before.get(device) ?? free)); + } + } + return summary; +} + +export interface WorktreeRetirementSchedulerOptions extends WorktreeRetirementOptions { + intervalMs?: number; + startDelayMs?: number; + continuationMs?: number; +} + +/** + * Runs retirement passes in the background: once shortly after start, then + * every interval, sooner while a backlog remains. Passes never overlap; a pass + * requested while one runs joins it. + */ +export class WorktreeRetirementScheduler { + private readonly controller = new AbortController(); + private readonly rotation = new Set(); + private running?: Promise; + private timer?: NodeJS.Timeout; + private started = false; + + constructor(private readonly options: WorktreeRetirementSchedulerOptions) {} + + start(): void { + if (this.started || this.controller.signal.aborted) return; + this.started = true; + if (this.running == null) this.schedule(this.options.startDelayMs ?? WORKTREE_RETIREMENT_START_DELAY_MS); + } + + /** Run a pass now, or join the one already running. */ + runNow(): Promise { + if (this.controller.signal.aborted) return Promise.resolve(undefined); + this.running ??= this.pass().finally(() => { + this.running = undefined; + }); + return this.running; + } + + /** Stop scheduling; a removal already under way finishes first. */ + async stop(): Promise { + this.controller.abort(); + clearTimeout(this.timer); + await this.running; + } + + private schedule(delayMs: number): void { + clearTimeout(this.timer); + if (!this.started || this.controller.signal.aborted) return; + this.timer = setTimeout(() => void this.runNow(), delayMs); + this.timer.unref(); + } + + private async pass(): Promise { + clearTimeout(this.timer); + let summary: WorktreeRetirementSummary | undefined; + try { + summary = await retireStaleWorktrees( + { ...this.options, rotation: this.rotation }, + this.controller.signal, + ); + this.options.log?.('info', describeWorktreeRetirement(summary)); + } catch (error) { + if (!this.controller.signal.aborted) { + this.options.log?.('info', `worktree retirement pass failed: ${errorDetail(error)}`); + } + } + this.schedule( + summary?.truncated + ? (this.options.continuationMs ?? WORKTREE_RETIREMENT_CONTINUATION_MS) + : (this.options.intervalMs ?? WORKTREE_RETIREMENT_INTERVAL_MS), + ); + return summary; + } +} From d6ac1b63b82c908db74c7c38f8358d8972d0c4fc Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 2 Oct 2026 22:30:23 -0400 Subject: [PATCH 27/48] fix: Report Missing Workspace Paths as NOT_FOUND and Create Missing Parents on Write (#301) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🧭 fix: Report Missing Workspace Paths as NOT_FOUND and Create Missing Parents on Write A missing file, directory or search scope no longer reports INVALID_PATH. When every existing ancestor resolves to a directory inside the workspace, the worker returns NOT_FOUND ("Workspace path does not exist"); escapes, symlinks, files used as directories and other path-safety failures keep INVALID_PATH. write_file creates missing parent directories beneath the workspace or lane root, one verified level at a time, never through a symlink or an existing file, and removes the directories it created if the write then fails. Code API advertises the added code in registration and maps it to HTTP 422 like INVALID_PATH. A worker reports NOT_FOUND only to a Code API that advertised it, and resends a refused settlement with INVALID_PATH, so older Code API replicas keep accepting settlements. * fix: Anchor Parent Creation to a Held Root, Sync It, and Surface Cleanup Failure Parent directories are created only under a held root descriptor, where mkdirat cannot follow a swapped path out of the root. A pathname-only root creates nothing and reports the missing parent as NOT_FOUND. Each created directory is synced into its parent before the write is acknowledged, and a rejected write that cannot remove a directory it created now reports a possible mutation instead of an atomic failure. * fix: Recheck Ancestors Before NOT_FOUND and Sync Parent Cleanup A miss is reported as NOT_FOUND only when the walk reached it through real directories that still have the same identity afterwards; any symlink in the chain stays INVALID_PATH. Cleanup syncs each removal into its parent and treats any directory it could not durably remove as a possible mutation. * fix: Keep Created Parents as a Committed Side Effect Directories a write creates are no longer rolled back. Once one exists the write finishes despite cancellation, and any later failure is reported as a possible mutation. A raced EEXIST parent is synced too, and the NOT_FOUND ancestry recheck now includes the workspace root. --- packages/code/src/linked-worktrees.test.ts | 53 +++++ packages/code/src/protocol.test.ts | 13 ++ packages/code/src/protocol.ts | 13 ++ packages/code/src/root-access.ts | 12 +- packages/code/src/worker.ts | 31 ++- packages/code/src/workspace-worker.test.ts | 106 ++++++++++ packages/code/src/workspace.test.ts | 231 ++++++++++++++++++++- packages/code/src/workspace.ts | 206 ++++++++++++++++-- service/src/bridge/router.test.ts | 1 + service/src/bridge/router.ts | 1 + service/src/workspace-tools/router.test.ts | 2 + 11 files changed, 646 insertions(+), 23 deletions(-) diff --git a/packages/code/src/linked-worktrees.test.ts b/packages/code/src/linked-worktrees.test.ts index 53ec156e..30c5b7e6 100644 --- a/packages/code/src/linked-worktrees.test.ts +++ b/packages/code/src/linked-worktrees.test.ts @@ -225,6 +225,59 @@ test('lane file tools are confined to the worktree and report the public workspa ); }); +test('lane writes create missing parents inside the worktree and report missing files', async (t) => { + const { parent, root } = await checkout(); + t.after(() => rm(parent, { recursive: true, force: true })); + const tools = await laneTools(root); + const lane = join(root, '.worktrees', 'task-a'); + + await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'repo', + worktree: 'task-a', + path: 'packages/new-package/package.json', + content: '{}\n', + }); + assert.equal(await readFile(join(lane, 'packages', 'new-package', 'package.json'), 'utf8'), '{}\n'); + await assert.rejects(stat(join(root, 'packages'))); + + await rejects( + tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'repo', + worktree: 'task-a', + path: '.checks/tests.log', + }), + 'NOT_FOUND', + ); + + await symlink(root, join(lane, 'checkout-link')); + await rejects( + tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'repo', + worktree: 'task-a', + path: 'checkout-link/escaped/notes.txt', + content: 'blocked', + }), + 'INVALID_PATH', + ); + await rejects( + tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'repo', + worktree: 'task-a', + path: 'checkout-link/missing.txt', + }), + 'INVALID_PATH', + ); + await assert.rejects(stat(join(root, 'escaped'))); +}); + test('lane commands register a confined root once, whatever siblings come and go', async (t) => { const { parent, root } = await checkout(); t.after(() => rm(parent, { recursive: true, force: true })); diff --git a/packages/code/src/protocol.test.ts b/packages/code/src/protocol.test.ts index 340ddfdf..25c00cf3 100644 --- a/packages/code/src/protocol.test.ts +++ b/packages/code/src/protocol.test.ts @@ -8,8 +8,10 @@ import { isSupportedBridgeArtifactName, isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, + isWorkspaceToolErrorCode, isWorkspaceToolRequest, isWorkspaceToolResult, + WORKSPACE_TOOL_ERROR_CODE_FALLBACKS, workspaceIsolationKey, workspaceIsolationKeysConflict, workspaceIsolationParent, @@ -19,6 +21,17 @@ import type { WorkspacePreviewEditRequest, } from './protocol.js'; +test('every added workspace error code falls back to a legacy code', () => { + assert.equal(isWorkspaceToolErrorCode('NOT_FOUND'), true); + assert.equal(isWorkspaceToolErrorCode('MISSING'), false); + for (const [code, legacy] of Object.entries(WORKSPACE_TOOL_ERROR_CODE_FALLBACKS)) { + assert.equal(isWorkspaceToolErrorCode(code), true); + assert.equal(isWorkspaceToolErrorCode(legacy), true); + assert.equal(legacy in WORKSPACE_TOOL_ERROR_CODE_FALLBACKS, false); + } + assert.equal(WORKSPACE_TOOL_ERROR_CODE_FALLBACKS.NOT_FOUND, 'INVALID_PATH'); +}); + test('accepts gateway directory markers as artifacts', () => { assert.equal(isSupportedBridgeArtifactName('.dirkeep'), true); assert.equal(isSupportedBridgeArtifactName('nested/.dirkeep'), true); diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index 26111273..216784a4 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -836,6 +836,8 @@ export interface BridgeWorkerRegistrationResponse { supportedWorkspaceInstanceTypes?: ['git_worktree']; /** Scheduling scopes this Code API can admit as independent lanes. */ supportedWorkspaceScopes?: ['git_linked_worktree']; + /** Added workspace tool error codes this Code API accepts in settlements. */ + supportedWorkspaceToolErrorCodes?: WorkspaceToolErrorCode[]; } /** Administrator-visible liveness for a configured worker. Credentials, @@ -976,6 +978,7 @@ export interface BridgeRejectedSettlement { export type WorkspaceToolErrorCode = | 'INVALID_PATH' + | 'NOT_FOUND' | 'INVALID_REQUEST' | 'READ_LIMIT_EXCEEDED' | 'WRITE_LIMIT_EXCEEDED' @@ -994,6 +997,7 @@ export type WorkspaceToolErrorCode = const WORKSPACE_TOOL_ERROR_CODES = new Set([ 'INVALID_PATH', + 'NOT_FOUND', 'INVALID_REQUEST', 'READ_LIMIT_EXCEEDED', 'WRITE_LIMIT_EXCEEDED', @@ -1011,6 +1015,15 @@ const WORKSPACE_TOOL_ERROR_CODES = new Set([ 'COMMAND_DISABLED', ]); +/** + * Codes added after the original settlement contract, each with the legacy code + * an older Code API accepts in its place. A worker reports an added code only + * after registration advertises it, and falls back if a settlement is refused. + */ +export const WORKSPACE_TOOL_ERROR_CODE_FALLBACKS: Readonly< + Partial> +> = { NOT_FOUND: 'INVALID_PATH' }; + export function isWorkspaceToolErrorCode( value: unknown, ): value is WorkspaceToolErrorCode { diff --git a/packages/code/src/root-access.ts b/packages/code/src/root-access.ts index b06de8d0..5bdddca9 100644 --- a/packages/code/src/root-access.ts +++ b/packages/code/src/root-access.ts @@ -350,7 +350,7 @@ export class WorkspaceRootAccess { } } - entry(path: string, operation: 'mkdir' | 'symlink' | 'readlink', target?: string): string | void { + entry(path: string, operation: 'mkdir' | 'symlink' | 'readlink', target?: string, mode = 0o700): string | void { const parent = this.parent(path); try { if (operation === 'readlink') { @@ -361,7 +361,7 @@ export class WorkspaceRootAccess { return buffer.subarray(0, length).toString(); } const result = operation === 'mkdir' - ? mkdirAt!(parent.fd, parent.name, 0o700) + ? mkdirAt!(parent.fd, parent.name, mode) : symlinkAt!(target!, parent.fd, parent.name); if (result !== 0) throw nativeError(); } finally { closeSync(parent.fd); } @@ -378,6 +378,8 @@ export class WorkspaceRootAccess { } const context = new AsyncLocalStorage(); +/** Whether filesystem adapters in this call are anchored to a held root descriptor. */ +export const holdsWorkspaceRoot = (): boolean => context.getStore() != null; export async function withWorkspaceRoot( root: string, identity: WorkspaceRootIdentity | undefined, @@ -504,10 +506,10 @@ export const readdir = async (path: string, maxEntries = 200_000): Promise => context.getStore()?.entry(path, 'readlink') as string ?? fs.readlink(path); -export const mkdir = async (path: string): Promise => { +export const mkdir = async (path: string, mode = 0o700): Promise => { const access = context.getStore(); - if (access) access.entry(path, 'mkdir'); - else await fs.mkdir(path, { mode: 0o700 }); + if (access) access.entry(path, 'mkdir', undefined, mode); + else await fs.mkdir(path, { mode }); }; export const symlink = async (target: string, path: string): Promise => { const access = context.getStore(); diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 217ca2c7..17a9a6e1 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -11,6 +11,7 @@ import { workspaceIsolationKey, workspaceIsolationKeysConflict, workspaceIsolationParent, + WORKSPACE_TOOL_ERROR_CODE_FALLBACKS, } from './protocol.js'; import { EndpointRuntimeSupervisor } from './runtime.js'; import { signBridgeRequest } from './identity.js'; @@ -28,6 +29,7 @@ import type { BridgeWorkspaceToolOperation, BridgeWorkspaceProgrammaticRequest, RepositoryInstructionDescriptor, + WorkspaceToolErrorCode, } from './protocol.js'; import type { RuntimeLease, RuntimeSupervisor } from './runtime.js'; import type { WorkspaceToolExecutor } from './workspace.js'; @@ -464,6 +466,8 @@ export class BridgeWorker { private registrationCapabilities: BridgeWorkerCapabilities; private activeCapabilities: BridgeWorkerCapabilities; private instructionMetadataSupported = true; + /** Added error codes the current Code API registration accepts in settlements. */ + private settlementErrorCodes: ReadonlySet = new Set(); private registrationTtlMs = DEFAULT_REGISTRATION_TTL_MS; private lastRegisteredAtMs = 0; private maintenanceOnly = false; @@ -803,6 +807,11 @@ export class BridgeWorker { } this.registrationTtlMs = registration.leaseTtlMs; this.activeCapabilities = this.registrationCapabilities; + this.settlementErrorCodes = new Set( + Array.isArray(registration.supportedWorkspaceToolErrorCodes) + ? registration.supportedWorkspaceToolErrorCodes + : [], + ); await this.options.onRegistered?.(registration); if ( !this.maintenanceOnly && @@ -2085,7 +2094,7 @@ export class BridgeWorker { ...((assignment.executionKind === 'workspace_tool' || assignment.executionKind === 'workspace_programmatic') && error instanceof WorkspaceToolError - ? { errorCode: error.code } + ? { errorCode: this.settlementErrorCode(error.code) } : {}), error: (assignment.executionKind === 'workspace_tool' && isWorkspaceToolRequest(assignment.request) && @@ -2474,6 +2483,12 @@ export class BridgeWorker { ); } + /** An older Code API refuses a settlement carrying a code it does not know. */ + private settlementErrorCode(code: WorkspaceToolErrorCode): WorkspaceToolErrorCode { + if (this.settlementErrorCodes.has(code)) return code; + return WORKSPACE_TOOL_ERROR_CODE_FALLBACKS[code] ?? code; + } + private async settleWithRetry( assignment: BridgeAssignment, settlement: BridgeSettlement, @@ -2525,6 +2540,20 @@ export class BridgeWorker { } catch (error) { lastError = error; if (signal?.aborted) break; + const legacyErrorCode = + settlement.status === 'rejected' && settlement.errorCode != null + ? WORKSPACE_TOOL_ERROR_CODE_FALLBACKS[settlement.errorCode] + : undefined; + if ( + settlement.status === 'rejected' && + legacyErrorCode != null && + error instanceof BridgeProtocolError && + error.status === 400 + ) { + /** A replica that predates the added code can still serve this settlement. */ + settlement = { ...settlement, errorCode: legacyErrorCode }; + continue; + } if ( error instanceof BridgeProtocolError && error.status != null && diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index 71ef2cd3..99fcdd02 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -237,6 +237,112 @@ test('worker omits pagination fields until Code API negotiates them', async () = }); }); +async function settleMissingFile( + registeredCodes: string[] | undefined, + rejectSettlement: (body: Record) => boolean = () => false, +): Promise<{ settlements: Array>; failure?: unknown }> { + const settlements: Array> = []; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['read_file' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute() { + throw new WorkspaceToolError('Workspace path does not exist', 'NOT_FOUND'); + }, + }, + fetchImpl: async (input, init) => { + if (!String(input).endsWith('/settle')) { + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: ['read_file'], + ...(registeredCodes ? { supportedWorkspaceToolErrorCodes: registeredCodes } : {}), + }); + } + const body = JSON.parse(String(init?.body)) as Record; + settlements.push(body); + if (rejectSettlement(body)) { + return Response.json({ error: 'Invalid bridge settlement' }, { status: 400 }); + } + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.register(); + const failure = await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-missing-file', + workerId: 'vm-1', + incarnationId, + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'logs/pending.log', + }, + }).then(() => undefined, (error: unknown) => error); + return { settlements, failure }; +} + +test('worker reports NOT_FOUND only to a Code API that advertised it', async () => { + const { settlements: supported } = await settleMissingFile(['NOT_FOUND']); + assert.deepEqual( + supported.map(({ status, errorCode, error }) => ({ status, errorCode, error })), + [{ status: 'rejected', errorCode: 'NOT_FOUND', error: 'Workspace path does not exist' }], + ); + + for (const registeredCodes of [undefined, [], ['SOMETHING_ELSE']]) { + const { settlements: legacy } = await settleMissingFile(registeredCodes); + assert.deepEqual( + legacy.map(({ status, errorCode, error }) => ({ status, errorCode, error })), + [{ status: 'rejected', errorCode: 'INVALID_PATH', error: 'Workspace path does not exist' }], + ); + } +}); + +test('worker resends a refused NOT_FOUND settlement with its legacy code', async () => { + const { settlements, failure } = await settleMissingFile( + ['NOT_FOUND'], + (body) => body.errorCode === 'NOT_FOUND', + ); + + assert.equal(failure, undefined); + assert.deepEqual( + settlements.map(({ errorCode }) => errorCode), + ['NOT_FOUND', 'INVALID_PATH'], + ); +}); + +test('worker does not resend a refused legacy settlement', async () => { + const { settlements, failure } = await settleMissingFile(undefined, () => true); + + assert.deepEqual(settlements.map(({ errorCode }) => errorCode), ['INVALID_PATH']); + assert.ok(failure instanceof BridgeProtocolError); + assert.equal(failure.status, 400); +}); + test('worker omits restricted workspaces that legacy registration would widen', async () => { const registrations: Array<{ operations: string[]; diff --git a/packages/code/src/workspace.test.ts b/packages/code/src/workspace.test.ts index 4a0ecb3e..787fbe7d 100644 --- a/packages/code/src/workspace.test.ts +++ b/packages/code/src/workspace.test.ts @@ -15,6 +15,7 @@ import { unlink, writeFile, } from 'node:fs/promises'; +import { existsSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, sep } from 'node:path'; import test from 'node:test'; @@ -28,6 +29,7 @@ import { SandboxWorkspaceTools, WorkspaceToolError, } from './workspace.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; import { BRIDGE_WORKSPACE_COMMAND_MAX_BYTES, @@ -1737,7 +1739,7 @@ test('oversized replaceAll previews and edits fail before writing the source fil } }); -test('writes reject symlink targets and missing parent directories', async (t) => { +test('writes reject symlink targets and symlinked parent directories', async (t) => { const parent = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); t.after(() => rm(parent, { recursive: true, force: true })); const root = join(parent, 'root'); @@ -1752,8 +1754,8 @@ test('writes reject symlink targets and missing parent directories', async (t) = for (const path of [ 'link.txt', - 'missing/notes.txt', 'directory-link/notes.txt', + 'directory-link/missing/notes.txt', ]) { await assert.rejects( tools.execute({ @@ -1768,6 +1770,231 @@ test('writes reject symlink targets and missing parent directories', async (t) = ); } assert.equal(await readFile(join(parent, 'outside.txt'), 'utf8'), 'outside'); + assert.deepEqual(await readdir(join(root, 'real-directory')), []); +}); + +async function heldRootTools(root: string): Promise { + return LocalWorkspaceTools.create({ + workspaces: [ + { id: 'primary', root, identity: await captureWorkspaceRootIdentity(root), writable: true }, + ], + }); +} + +test('writes create missing parent directories inside a held workspace root', async (t) => { + const root = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-workspace-'))); + t.after(() => rm(root, { recursive: true, force: true })); + await mkdir(join(root, 'src')); + const tools = await heldRootTools(root); + + for (const [path, overwrite] of [ + ['src/feature/deep/new.ts', undefined], + ['fresh/created.ts', false], + ] as const) { + const result = await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path, + content: 'export {};\n', + ...(overwrite === undefined ? {} : { overwrite }), + }); + assert.deepEqual(result, { + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path, + created: true, + bytesWritten: 11, + }); + assert.equal(await readFile(join(root, ...path.split('/')), 'utf8'), 'export {};\n'); + } + assert.equal((await stat(join(root, 'src', 'feature', 'deep'))).isDirectory(), true); +}); + +test('parent creation never follows symlinks, crosses files, or leaves the workspace', async (t) => { + const parent = await realpath( + await mkdtemp(join(tmpdir(), 'librechat-code-workspace-parent-')), + ); + t.after(() => rm(parent, { recursive: true, force: true })); + const root = join(parent, 'root'); + const outside = join(parent, 'outside'); + await mkdir(join(root, 'src'), { recursive: true }); + await mkdir(outside); + await writeFile(join(root, 'file.txt'), 'file'); + await symlink(outside, join(root, 'linked-outside')); + await symlink(join(root, 'src'), join(root, 'alias')); + await symlink(join(parent, 'missing-target'), join(root, 'dangling')); + const tools = await heldRootTools(root); + + for (const [path, code] of [ + ['linked-outside/new/notes.txt', 'INVALID_PATH'], + ['alias/new/notes.txt', 'INVALID_PATH'], + ['dangling/new/notes.txt', 'INVALID_PATH'], + ['file.txt/new/notes.txt', 'INVALID_PATH'], + ['../outside/new/notes.txt', 'INVALID_REQUEST'], + ['new/../../outside/notes.txt', 'INVALID_REQUEST'], + ] as const) { + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path, + content: 'blocked', + }), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === code && + !error.mutationMayHaveCommitted, + path, + ); + } + assert.deepEqual(await readdir(outside), []); + assert.deepEqual(await readdir(join(root, 'src')), []); + assert.deepEqual((await readdir(root)).sort(), [ + 'alias', + 'dangling', + 'file.txt', + 'linked-outside', + 'src', + ]); + await assert.rejects(stat(join(parent, 'missing-target'))); +}); + +test('a write finishes once it has created parents, even if cancelled', async (t) => { + const root = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-workspace-'))); + t.after(() => rm(root, { recursive: true, force: true })); + const tools = await heldRootTools(root); + /** Reports cancellation as soon as the parents exist. */ + const signal = { + get aborted() { + return existsSync(join(root, 'fresh', 'nested')); + }, + } as AbortSignal; + + const result = await tools.execute( + { + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'fresh/nested/file.txt', + content: 'installed', + }, + signal, + ); + + assert.equal(result.operation === 'write_file' && result.created, true); + assert.equal(await readFile(join(root, 'fresh', 'nested', 'file.txt'), 'utf8'), 'installed'); +}); + +test('a write cancelled before creating parents leaves the workspace unchanged', async (t) => { + const root = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-workspace-'))); + t.after(() => rm(root, { recursive: true, force: true })); + const tools = await heldRootTools(root); + const controller = new AbortController(); + controller.abort(); + + await assert.rejects( + tools.execute( + { + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'fresh/nested/file.txt', + content: 'never installed', + }, + controller.signal, + ), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'EXECUTION_ABORTED' && + !error.mutationMayHaveCommitted, + ); + assert.deepEqual(await readdir(root), []); +}); + +test('a pathname-only workspace root reports a missing parent instead of creating it', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'missing/notes.txt', + content: 'blocked', + }), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'NOT_FOUND' && + error.message === 'Workspace path does not exist', + ); + assert.deepEqual(await readdir(root), []); +}); + +test('a missing workspace path is NOT_FOUND only when reached through the workspace', async (t) => { + const parent = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-parent-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const root = join(parent, 'root'); + const outside = join(parent, 'outside'); + await mkdir(join(root, 'src'), { recursive: true }); + await mkdir(outside); + await writeFile(join(root, 'file.txt'), 'file'); + await symlink(join(root, 'src'), join(root, 'alias')); + await symlink(outside, join(root, 'linked-outside')); + await symlink(join(parent, 'missing-target'), join(root, 'dangling')); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + const edits = [{ oldText: 'a', newText: 'b' }]; + const requests = (path: string): WorkspaceToolRequest[] => [ + { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path }, + { protocolVersion: 1, operation: 'edit_file', workspaceId: 'primary', path, edits }, + { protocolVersion: 1, operation: 'preview_edit', workspaceId: 'primary', path, edits }, + { protocolVersion: 1, operation: 'search_text', workspaceId: 'primary', query: 'x', path }, + { protocolVersion: 1, operation: 'list_files', workspaceId: 'primary', path }, + ]; + + for (const [path, code] of [ + ['missing.log', 'NOT_FOUND'], + ['src/missing.ts', 'NOT_FOUND'], + ['not-yet/created/output.log', 'NOT_FOUND'], + ['alias/missing.ts', 'INVALID_PATH'], + ['linked-outside/missing.txt', 'INVALID_PATH'], + ['dangling', 'INVALID_PATH'], + ['file.txt/missing.txt', 'INVALID_PATH'], + ] as const) { + for (const request of requests(path)) { + await assert.rejects( + tools.execute(request), + (error: unknown) => { + assert.ok(error instanceof WorkspaceToolError); + assert.equal(error.code, code, `${request.operation} ${path}`); + assert.equal(error.message.includes(parent), false); + if (code === 'NOT_FOUND') { + assert.equal(error.message, 'Workspace path does not exist'); + } + return true; + }, + ); + } + } + for (const path of ['../outside/missing.txt', 'src/../../outside/missing.txt']) { + for (const request of requests(path)) { + await assert.rejects( + tools.execute(request), + (error: unknown) => + error instanceof WorkspaceToolError && + (error.code === 'INVALID_REQUEST' || error.code === 'INVALID_PATH'), + `${request.operation} ${path}`, + ); + } + } }); test('workspace mutations preserve existing file permissions', async (t) => { diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index 0b9328d0..e48365c8 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -1,6 +1,6 @@ import { createHash, randomBytes } from 'node:crypto'; import { constants } from 'node:fs'; -import { link, lstat, open, realpath, rename, stat, unlink, spawn, withWorkspaceRoot, WorkspaceRootAccessError } from './root-access.js'; +import { holdsWorkspaceRoot, link, lstat, mkdir, open, realpath, rename, stat, unlink, spawn, withWorkspaceRoot, WorkspaceRootAccessError } from './root-access.js'; import { basename, dirname, isAbsolute, relative, resolve, sep } from 'node:path'; import type { FileHandle } from 'node:fs/promises'; @@ -324,6 +324,9 @@ async function readConfinedFileBuffer( return buffer.subarray(0, bytesRead); } catch (error) { if (error instanceof WorkspaceToolError) throw error; + if (isMissingEntry(error)) { + throw await classifyMissingWorkspacePath(root, candidate); + } throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); } finally { await handle?.close(); @@ -369,6 +372,147 @@ async function verifyDirectoryPathHasNoSymlinks( return currentIdentity; } +function isMissingEntry(error: unknown): boolean { + return (error as NodeJS.ErrnoException | undefined)?.code === 'ENOENT'; +} + +interface DirectoryIdentity { + path: string; + dev: bigint | number; + ino: bigint | number; +} + +/** Whether each ancestor observed before a miss is still the same real directory. */ +async function ancestorsUnchanged(ancestors: readonly DirectoryIdentity[]): Promise { + for (const ancestor of ancestors) { + try { + const current = await lstat(ancestor.path); + if ( + current.isSymbolicLink() || + !current.isDirectory() || + current.dev !== ancestor.dev || + current.ino !== ancestor.ino + ) { + return false; + } + } catch { + return false; + } + } + return true; +} + +/** + * Reports an absent target as `NOT_FOUND` only when the walk from the root + * reaches the missing entry through real directories, the root included, that + * are unchanged after the miss. A symlink, a file used as a directory, or an ancestor replaced + * during the walk stays a path-safety rejection, so the distinction never + * describes a path beyond the root. + */ +async function classifyMissingWorkspacePath( + root: string, + candidate: string, +): Promise { + const invalid = new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + const segments = relative(root, candidate).split(sep).filter(Boolean); + const ancestors: DirectoryIdentity[] = []; + try { + const rootEntry = await lstat(root); + if (rootEntry.isSymbolicLink() || !rootEntry.isDirectory()) return invalid; + ancestors.push({ path: root, dev: rootEntry.dev, ino: rootEntry.ino }); + } catch { + return invalid; + } + let current = root; + for (const [index, segment] of segments.entries()) { + current = resolve(current, segment); + let entry: Awaited>; + try { + entry = await lstat(current); + } catch (error) { + return isMissingEntry(error) && (await ancestorsUnchanged(ancestors)) + ? new WorkspaceToolError('Workspace path does not exist', 'NOT_FOUND') + : invalid; + } + if (index === segments.length - 1) return invalid; + if (entry.isSymbolicLink() || !entry.isDirectory()) return invalid; + ancestors.push({ path: current, dev: entry.dev, ino: entry.ino }); + } + return invalid; +} + +/** + * Creates each missing ancestor of a write target as a real directory beneath + * the root, one verified level at a time, and syncs each new entry into its + * parent. Creation needs a held root descriptor, so `mkdirat` is anchored to + * the root; a pathname-only root creates nothing and its write reports the + * missing parent. An existing symlink or file stops the walk. + * + * Created directories are a committed side effect, never rolled back: once one + * exists, a later failure of this write is reported as a possible mutation, so + * the worker quarantines rather than claiming an atomic rejection. + */ +async function createMissingParentDirectories( + root: string, + candidate: string, +): Promise { + const created: string[] = []; + if (!holdsWorkspaceRoot()) return created; + const segments = relative(root, dirname(candidate)).split(sep).filter(Boolean); + let current = root; + try { + for (const segment of segments) { + current = resolve(current, segment); + let entry: Awaited> | undefined; + try { + entry = await lstat(current); + } catch (error) { + if (!isMissingEntry(error)) throw error; + } + let createdHere = false; + if (entry == null) { + try { + await mkdir(current, 0o777); + created.push(current); + createdHere = true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + } + entry = await lstat(current); + /** Also after a raced `EEXIST`: the write relies on that entry too. */ + await syncWorkspaceDirectory(dirname(current)); + } + if ( + entry.isSymbolicLink() || + !entry.isDirectory() || + (createdHere && !isWithinRoot(root, await realpath(current))) + ) { + throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + } + } + return created; + } catch (error) { + throw afterCreatedDirectories(created, classifyWritePathValidationError(error)); + } +} + +function afterCreatedDirectories( + created: readonly string[], + error: WorkspaceToolError, +): WorkspaceToolError { + if (created.length === 0 || error.mutationMayHaveCommitted) return error; + return new WorkspaceToolError(error.message, error.code, true); +} + +function assertWithinWriteLimit(content: Buffer): void { + if (content.byteLength > BRIDGE_WORKSPACE_WRITE_MAX_BYTES) { + throw new WorkspaceToolError( + 'Workspace file exceeds write limit', + 'WRITE_LIMIT_EXCEEDED', + ); + } +} + function classifyWritePathValidationError(error: unknown): WorkspaceToolError { if (error instanceof WorkspaceToolError) return error; const code = (error as NodeJS.ErrnoException).code; @@ -481,12 +625,7 @@ async function atomicWriteConfinedFile( allowOverwrite = true, ): Promise<{ created: boolean }> { throwIfAborted(signal); - if (content.byteLength > BRIDGE_WORKSPACE_WRITE_MAX_BYTES) { - throw new WorkspaceToolError( - 'Workspace file exceeds write limit', - 'WRITE_LIMIT_EXCEEDED', - ); - } + assertWithinWriteLimit(content); const candidate = resolveWorkspacePath(root, requestedPath); const parent = dirname(candidate); let canonicalParent: string; @@ -498,6 +637,9 @@ async function atomicWriteConfinedFile( throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); } } catch (error) { + if (isMissingEntry(error)) { + throw await classifyMissingWorkspacePath(root, parent); + } throw classifyWritePathValidationError(error); } @@ -671,14 +813,26 @@ async function writeWorkspaceFile( signal?: AbortSignal, ): Promise { const content = Buffer.from(request.content, 'utf8'); - const { created } = await atomicWriteConfinedFile( + throwIfAborted(signal); + assertWithinWriteLimit(content); + const directories = await createMissingParentDirectories( root, - request.path, - content, - signal, - undefined, - request.overwrite !== false, + resolveWorkspacePath(root, request.path), ); + let created: boolean; + try { + /** Once parents exist, finish the write rather than abandon them to a cancellation. */ + ({ created } = await atomicWriteConfinedFile( + root, + request.path, + content, + directories.length === 0 ? signal : undefined, + undefined, + request.overwrite !== false, + )); + } catch (error) { + throw afterCreatedDirectories(directories, classifyWritePathValidationError(error)); + } return { protocolVersion: BRIDGE_PROTOCOL_VERSION, operation: 'write_file', @@ -757,6 +911,9 @@ async function editWorkspaceFile( }; } catch (error) { if (error instanceof WorkspaceToolError) throw error; + if (isMissingEntry(error)) { + throw await classifyMissingWorkspacePath(root, candidate); + } throw classifyWritePathValidationError(error); } finally { await opened?.close().catch(() => undefined); @@ -1009,7 +1166,10 @@ async function searchWorkspace( let canonicalTarget: string; try { canonicalTarget = await realpath(target); - } catch { + } catch (error) { + if (isMissingEntry(error)) { + throw await classifyMissingWorkspacePath(root, target); + } throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); } if (!isWithinRoot(root, canonicalTarget)) @@ -1060,7 +1220,9 @@ async function searchWorkspace( } catch (error) { if ( error instanceof WorkspaceToolError && - (error.code === 'INVALID_PATH' || error.code === 'READ_LIMIT_EXCEEDED') + (error.code === 'INVALID_PATH' || + error.code === 'NOT_FOUND' || + error.code === 'READ_LIMIT_EXCEEDED') ) { continue; } @@ -1167,6 +1329,13 @@ async function listWorkspaceFiles( ); } catch (error) { if (error instanceof WorkspaceToolError) throw error; + if (isMissingEntry(error)) { + throw await withinListDeadline( + classifyMissingWorkspacePath(root, target), + signal, + deadline, + ); + } throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); } if (!isWithinRoot(root, canonicalTarget)) { @@ -1181,6 +1350,13 @@ async function listWorkspaceFiles( ).isDirectory(); } catch (error) { if (error instanceof WorkspaceToolError) throw error; + if (isMissingEntry(error)) { + throw await withinListDeadline( + classifyMissingWorkspacePath(root, target), + signal, + deadline, + ); + } throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); } const normalizedRequestedResultPath = request.path diff --git a/service/src/bridge/router.test.ts b/service/src/bridge/router.test.ts index 28035847..360414a0 100644 --- a/service/src/bridge/router.test.ts +++ b/service/src/bridge/router.test.ts @@ -438,6 +438,7 @@ describe('paired bridge HTTP API', () => { 'replace_all', ], supportedWorkspaceListFileFeatures: ['after_path'], + supportedWorkspaceToolErrorCodes: ['NOT_FOUND'], }); const crossDeploymentRevoke = await fetch( diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 8fdc700e..0bc20a1e 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -541,6 +541,7 @@ router.post( supportedWorkspaceProgrammaticLanguages: ['bash'], supportedWorkspaceInstanceTypes: ['git_worktree'], supportedWorkspaceScopes: ['git_linked_worktree'], + supportedWorkspaceToolErrorCodes: ['NOT_FOUND'], }); } catch (error) { if (error instanceof BridgeStoreError) { diff --git a/service/src/workspace-tools/router.test.ts b/service/src/workspace-tools/router.test.ts index d3cb032e..8a753c08 100644 --- a/service/src/workspace-tools/router.test.ts +++ b/service/src/workspace-tools/router.test.ts @@ -223,6 +223,8 @@ test.each([ ['COMMAND_TIMEOUT', 504], ['COMMAND_UNAVAILABLE', 503], ['COMMAND_DISABLED', 403], + ['INVALID_PATH', 422], + ['NOT_FOUND', 422], ] as const)('maps worker %s rejections to HTTP %i', async (errorCode, expectedStatus) => { const app = express(); app.use(json()); From 893ad3603f223f279439b8ae764f2860b8ac82bd Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Fri, 2 Oct 2026 23:06:38 -0400 Subject: [PATCH 28/48] feat: Quote the Current Text Where a Missing Workspace Edit Belongs (#300) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 📍 feat: Quote the Current Text Where a Missing Workspace Edit Belongs A missing edit's diagnosis now ends with a bounded excerpt of the region old_text most likely meant: the alignment where most of its distinctive lines fall into place (found in one pass with bounded votes), else its unique first line or closest line. Each of at most 8 rows is `|`, with a space for identical lines, `~` for whitespace-only differences and `!` for changed text; lines are shortened to 160 characters and the excerpt to 1,600. The excerpt is a final `; the current text at lines A-B (...) is ""` hint, so hosts that parse the existing grammar drop it and keep every other hint. Excerpts are the first detail dropped to fit the 3,000-character message bound and never shorten another edit's reason. * 🧭 fix: Bound Every Alignment Offset and Anchor Excerpts Where old_text Starts - Count every inspected anchor offset against the region vote budget, and stop at the first offset that would start before the file (offsets ascend), so a repetitive old_text cannot escape the bound. - Start a first-line or closest-line excerpt at the line where old_text starts, not at its first non-blank line. - Escape U+2028/U+2029 in quoted text, which JSON.stringify leaves raw, so each batch edit stays on one line for hosts that read the message line by line. * 📏 fix: Bound Edit Excerpts by Their Encoded Error-Body Size Code API returns the diagnostic inside a JSON error body, and LibreChat reads that body through a 4,096-byte buffer, discarding a truncated one. A 3,000-character message with multibyte or escape-heavy excerpts could exceed it. Excerpts are now limited to 1,600 bytes and granted only while the whole message stays within 3,800 bytes, both measured as UTF-8 once JSON-encoded. * 📐 fix: Shorten Edit Diagnostics by Encoded Size as Well as Length Batch reasons that still exceed the bound after excerpts are declined are now shortened against both the 3,000-character limit and the 3,800-byte encoded budget, and a single edit's reason is shortened the same way instead of being cut by character count. Multibyte closest-line quotes can no longer push the Code API's error body past the 4,096 bytes hosts read. --- packages/code/README.md | 13 +- packages/code/src/edits.test.ts | 243 ++++++++++++++++++++++++++++ packages/code/src/edits.ts | 270 ++++++++++++++++++++++++++++---- 3 files changed, 491 insertions(+), 35 deletions(-) diff --git a/packages/code/README.md b/packages/code/README.md index 55f5c668..636cd1a8 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -640,7 +640,18 @@ fails, the worker still checks the rest and rejects the whole batch with one batch messages shorten reasons and source excerpts to stay within the bound, but never omit failing edit positions. A missing edit names the nearest candidate line and flags elided (`...`) or line-numbered `oldText`, a -whitespace-only difference, or CRLF line endings. +whitespace-only difference, or CRLF line endings. When the file has a region +that `oldText` most likely meant (the alignment where the most of its +distinctive lines fall into place, or its unique first line or closest line), +the reason ends with `the current text at lines A-B (~ whitespace differs, ! +text differs) is ""`: a JSON string of at most 8 rows of the form +`|`, each line shortened to 160 characters, where the mark is +a space for a line identical to `oldText` at that position, `~` when only its +whitespace differs and `!` when its text differs. Excerpts are the first detail +dropped to fit the message bound, and never shorten another edit's reason. Both +the excerpt (1,600 bytes) and the message that carries it (3,800 bytes) are +measured as UTF-8 once JSON-encoded, so the Code API's error body stays within +the 4,096 bytes hosts such as LibreChat read. An ambiguous edit gives its match count and line numbers. Overlapping occurrences count as separate locations. Detailed source-line excerpts require `read_file` or `preview_edit` on the same workspace; edit-only workers return a diff --git a/packages/code/src/edits.test.ts b/packages/code/src/edits.test.ts index 3ef9428d..5bcabf49 100644 --- a/packages/code/src/edits.test.ts +++ b/packages/code/src/edits.test.ts @@ -719,3 +719,246 @@ test('short batch diagnostic reasons remain complete when the message fits', () assert.ok(error.message.includes(`\nEdit ${failure.index + 1}: ${failure.reason}.`)); } }); + +const EXCERPT = /the current text at lines? (\d+)(?:-(\d+))? \(~ whitespace differs, ! text differs\) is ("(?:[^"\\]|\\.)*")/; + +function excerptOf(message: string): { first: number; last: number; rows: string[] } | undefined { + const match = EXCERPT.exec(message); + if (!match) return undefined; + const first = Number(match[1]); + return { first, last: Number(match[2] ?? first), rows: (JSON.parse(match[3]) as string).split('\n') }; +} + +test('a missing edit quotes the current text of the region it meant, marking what differs', () => { + const text = 'export function load(user) {\n const id = user.id;\n return fetchUser(id);\n}\n'; + const error = rejection(() => applyTextEdits(text, [{ + oldText: 'export function load(user) {\n const id = user.id;\n return fetchUser(user);\n}', + newText: 'x', + }], 'tolerant')); + assert.match(error.message, /old_text was not found; its first line appears at line 1, but the lines after it differ; the current text/); + assert.deepEqual(excerptOf(error.message), { + first: 1, + last: 4, + rows: [ + '1| export function load(user) {', + '2|~ const id = user.id;', + '3|! return fetchUser(id);', + '4| }', + ], + }); + assert.match(error.message, /\.$/); +}); + +test('the region is found from later lines when the first line itself changed', () => { + const text = 'header();\nfunction save(record) {\n validate(record);\n persist(record);\n}\nfooter();\n'; + const error = rejection(() => applyTextEdits(text, [{ + oldText: 'function save(item) {\n validate(record);\n persist(record);\n}', + newText: 'x', + }])); + assert.deepEqual(excerptOf(error.message), { + first: 2, + last: 5, + rows: ['2|!function save(record) {', '3| validate(record);', '4| persist(record);', '5| }'], + }); +}); + +test('the region with the most aligned lines wins, and ties prefer the earliest', () => { + const block = (name: string) => `function ${name}() {\n prepare();\n run();\n cleanup();\n}\n`; + const text = `${block('first')}${block('second')}`; + const error = rejection(() => applyTextEdits(text, [{ + oldText: 'function second() {\n prepare();\n execute();\n cleanup();\n}', + newText: 'x', + }])); + assert.equal(excerptOf(error.message)?.first, 6); + const tied = rejection(() => applyTextEdits(text, [{ + oldText: 'function third() {\n prepare();\n execute();\n cleanup();\n}', + newText: 'x', + }])); + assert.equal(excerptOf(tied.message)?.first, 1); +}); + +test('a missing one-line edit quotes its closest line', () => { + const error = rejection(() => applyTextEdits('const total = items.length;\nreturn total;\n', [ + { oldText: 'const total = items.size;', newText: 'x' }, + ])); + assert.match(error.message, /the closest line is line 1: "const total = items.length;"/); + assert.deepEqual(excerptOf(error.message), { first: 1, last: 1, rows: ['1|!const total = items.length;'] }); +}); + +test('exact mode quotes the whitespace to copy alongside the whitespace hint', () => { + const error = rejection(() => applyTextEdits('class A {\n a();\n b();\n}\n', [ + { oldText: 'a();\nb();', newText: 'x' }, + ])); + assert.match(error.message, /exists at line 2 with different whitespace \(indentation-flexible\); copy that whitespace exactly; the current text at lines 2-3/); + assert.deepEqual(excerptOf(error.message)?.rows, ['2|~ a();', '3|~ b();']); +}); + +test('a long region is quoted from just before its first difference', () => { + const source = Array.from({ length: 40 }, (_, index) => `step(${index});`); + const needle = source.slice(5, 25); + needle[8] = 'step(changed);'; + const error = rejection(() => applyTextEdits(`${source.join('\n')}\n`, [{ oldText: needle.join('\n'), newText: 'x' }])); + const excerpt = excerptOf(error.message); + assert.equal(excerpt?.first, 13); + assert.equal(excerpt?.last, 20); + assert.equal(excerpt?.rows[1], '14|!step(13);'); + assert.ok(excerpt?.rows.every((row, index) => index === 1 || row.includes('| '))); +}); + +test('excerpts shorten long lines and stay bounded without splitting characters', () => { + const wide = `x${'😀'.repeat(400)}`; + const controls = '\u0001'.repeat(2_000); + const text = Array.from({ length: 10 }, (_, index) => `key_${index} = ${index % 2 === 0 ? wide : controls}`).join('\n'); + const oldText = Array.from({ length: 10 }, (_, index) => `key_${index} = other`).join('\n'); + const error = rejection(() => applyTextEdits(text, [{ oldText, newText: 'x' }])); + const excerpt = excerptOf(error.message); + assert.ok(excerpt != null); + assert.ok(excerpt.rows.length >= 1 && excerpt.rows.length <= 8); + assert.ok(EXCERPT.exec(error.message)![0].length <= 1_600); + for (const row of excerpt.rows) { + assert.ok(row.length <= 175, row.slice(0, 40)); + assert.equal(Buffer.from(row).toString('utf8'), row); + } +}); + +test('an edit that resembles nothing in the file gets no excerpt', () => { + const error = rejection(() => applyTextEdits('alpha\nbeta\n', [ + { oldText: 'completely different text\nwith nothing shared', newText: 'x' }, + ])); + assert.equal(excerptOf(error.message), undefined); + assert.match(error.message, /old_text was not found\.$/); +}); + +test('ambiguous and line-numbered edits keep their messages without an excerpt', () => { + const ambiguous = rejection(() => applyTextEdits('return a;\nreturn a;\n', [{ oldText: 'return a;', newText: 'x' }])); + assert.equal(excerptOf(ambiguous.message), undefined); + const numbered = rejection(() => applyTextEdits('start\nmiddle\nend\n', [{ oldText: '1 | start\n2 | middle', newText: 'x' }])); + assert.equal(excerptOf(numbered.message), undefined); +}); + +test('batch excerpts are granted in edit order only while every position and reason still fits', () => { + const lines = Array.from({ length: 400 }, (_, index) => `value_${index} = compute_${index}(input_${index}, ${'p'.repeat(120)});`); + const edits = Array.from({ length: 12 }, (_, index) => ({ + oldText: `${lines[index * 10]}\n${lines[index * 10 + 1].replace('compute', 'changed')}\n${lines[index * 10 + 2]}`, + newText: 'x', + })); + const error = rejection(() => applyTextEdits(`${lines.join('\n')}\n`, edits)); + assert.ok(error.message.length <= EDIT_DIAGNOSTIC_MAX_CHARS); + assert.deepEqual( + [...error.message.matchAll(/\nEdit (\d+):/g)].map((match) => Number(match[1])), + Array.from({ length: 12 }, (_, index) => index + 1), + ); + const quoted = [...error.message.matchAll(/\nEdit (\d+): [^\n]*the current text at/g)].map((match) => Number(match[1])); + assert.ok(quoted.length >= 1 && quoted.length < 12); + assert.deepEqual(quoted, Array.from({ length: quoted.length }, (_, index) => index + 1)); + for (const failure of error.failures) { + assert.ok(error.message.includes(`\nEdit ${failure.index + 1}: ${failure.reason}`)); + } +}); + +test('a single edit drops its excerpt rather than exceed the bound', () => { + const error = new WorkspaceEditMatchError([ + { index: 0, reason: `old_text was not found${'; x'.repeat(1_200)}`, excerpt: 'the current text at line 1 (~ whitespace differs, ! text differs) is "1|!y"' }, + ], 1); + assert.ok(error.message.length <= EDIT_DIAGNOSTIC_MAX_CHARS); + assert.doesNotMatch(error.message, /the current text/); +}); + +test('region votes stay bounded on highly repetitive files', () => { + const text = 'item = value;\nother = thing;\n'.repeat(150_000); + const oldText = 'item = value;\nother = thing;\nmissing = line;\nitem = value;'; + const started = performance.now(); + const error = rejection(() => applyTextEdits(text, [{ oldText, newText: 'x' }])); + assert.ok(performance.now() - started < 3_000, 'a repetitive file must not cast unbounded votes'); + assert.match(error.message, /did not apply and nothing was written/); +}); + +test('an excerpt anchored on a first line after blank lines starts where old_text starts', () => { + const error = rejection(() => applyTextEdits('header();\nconst total = items.length;\nreturn total;\n', [ + { oldText: '\nconst total = items.size;', newText: 'x' }, + ])); + assert.deepEqual(excerptOf(error.message), { + first: 1, + last: 2, + rows: ['1|!header();', '2|!const total = items.length;'], + }); +}); + +test('region votes count every inspected anchor offset against the budget', () => { + const oldText = `${'repeat_line();\n'.repeat(5_000)}missing();`; + const text = 'repeat_line();\n'.repeat(20_000); + const started = performance.now(); + const error = rejection(() => applyTextEdits(text, [{ oldText, newText: 'x' }])); + assert.ok(performance.now() - started < 3_000, 'repeated anchor offsets must not escape the vote budget'); + assert.match(error.message, /did not apply and nothing was written/); +}); + +test('quoted text escapes line separators so every batch edit stays on one line', () => { + const separator = String.fromCharCode(0x2028); + const text = `const label = "first${separator}second";\nconst other = "value";\n`; + const error = rejection(() => applyTextEdits(text, [ + { oldText: `const label = "first${separator}third";`, newText: 'x' }, + { oldText: 'const other = "changed";', newText: 'y' }, + ])); + assert.ok(!error.message.includes(separator)); + const lines = error.message.split('\n'); + assert.match(lines[1], /^Edit 1: old_text was not found; the closest line is line 1: .*\\u2028.*the current text at line 1/); + assert.match(lines[2], /^Edit 2: /); + assert.deepEqual(excerptOf(lines[1])?.rows, [`1|!const label = "first${separator}second";`]); +}); + +function errorBodyBytes(message: string): number { + return Buffer.byteLength(JSON.stringify({ error: message, code: 'EDIT_CONFLICT' })); +} + +test('excerpts of multibyte source stay within the 4096-byte error body hosts read', () => { + const text = Array.from({ length: 12 }, (_, index) => `名前_${index} = "${'漢字'.repeat(80)}";`).join('\n'); + const oldText = Array.from({ length: 12 }, (_, index) => `名前_${index} = "changed";`).join('\n'); + const error = rejection(() => applyTextEdits(text, [{ oldText, newText: 'x' }])); + const excerpt = excerptOf(error.message); + assert.ok(excerpt != null && excerpt.rows.length >= 1 && excerpt.rows.length < 8); + assert.ok(Buffer.byteLength(JSON.stringify(EXCERPT.exec(error.message)![0])) - 2 <= 1_600); + assert.ok(errorBodyBytes(error.message) <= 4_096); +}); + +test('batch excerpts are granted by encoded size, so escapes cannot overflow the error body', () => { + const lines = Array.from({ length: 200 }, (_, index) => `path_${index} = "C:\\\\dir\\\\${'"q"'.repeat(48)}";`); + const edits = Array.from({ length: 6 }, (_, index) => ({ + oldText: lines.slice(index * 20, index * 20 + 8).map((line, offset) => (offset === 3 ? 'changed();' : line)).join('\n'), + newText: 'x', + })); + const error = rejection(() => applyTextEdits(`${lines.join('\n')}\n`, edits)); + assert.ok(errorBodyBytes(error.message) <= 4_096, String(errorBodyBytes(error.message))); + assert.match(error.message, /\nEdit 1: [^\n]*the current text at/); + assert.deepEqual( + [...error.message.matchAll(/\nEdit (\d+):/g)].map((match) => Number(match[1])), + [1, 2, 3, 4, 5, 6], + ); +}); + +test('shortened batch reasons also respect the encoded error-body size', () => { + const emojis = '😀'.repeat(200); + const text = Array.from({ length: 20 }, (_, index) => `item_${index} emoji_${index} = "${emojis}";`).join('\n'); + const edits = Array.from({ length: 20 }, (_, index) => ({ + oldText: `item_${index} emoji_${index} = "plain";`, + newText: 'x', + })); + const error = rejection(() => applyTextEdits(text, edits)); + assert.ok(error.message.length <= EDIT_DIAGNOSTIC_MAX_CHARS); + assert.ok(errorBodyBytes(error.message) <= 4_096, String(errorBodyBytes(error.message))); + assert.equal(Buffer.from(error.message).toString('utf8'), error.message); + assert.deepEqual( + [...error.message.matchAll(/\nEdit (\d+):/g)].map((match) => Number(match[1])), + Array.from({ length: 20 }, (_, index) => index + 1), + ); +}); + +test('a single edit with a long multibyte reason stays within both bounds', () => { + const error = new WorkspaceEditMatchError([ + { index: 0, reason: `old_text was not found; the closest line is line 1: "${'😀'.repeat(1_400)}"` }, + ], 1); + assert.ok(error.message.length <= EDIT_DIAGNOSTIC_MAX_CHARS); + assert.ok(errorBodyBytes(error.message) <= 4_096); + assert.equal(Buffer.from(error.message).toString('utf8'), error.message); + assert.match(error.message, /…\.$/); +}); diff --git a/packages/code/src/edits.ts b/packages/code/src/edits.ts index a6d54269..8205b1d7 100644 --- a/packages/code/src/edits.ts +++ b/packages/code/src/edits.ts @@ -12,8 +12,24 @@ import type { * callers prefix their own context, so diagnostics stay well below that. */ export const EDIT_DIAGNOSTIC_MAX_CHARS = 3000; +/** + * Code API returns the message inside a JSON error body, and hosts read that + * body through a bounded buffer (4096 bytes in LibreChat). Optional excerpts + * are granted only while the JSON-encoded UTF-8 message stays within this size. + */ +export const EDIT_DIAGNOSTIC_MAX_BODY_BYTES = 3_800; const MAX_REPORTED_LINES = 5; const MAX_SNIPPET_CHARS = 120; +/** A missing edit's current-text excerpt: a few lines, each shortened, in one bounded hint. */ +const MAX_EXCERPT_LINES = 8; +const MAX_EXCERPT_LINE_CHARS = 160; +/** Measured like the message bound: UTF-8 bytes once JSON-encoded into the error body. */ +const MAX_EXCERPT_BYTES = 1_600; +/** Alignment votes cast while locating the region a missing edit most likely meant. */ +const MAX_REGION_VOTES = 200_000; +/** Lines too generic to anchor a region on their own (`}`, `);`, blank lines). */ +const ANCHOR_LINE = /[\p{L}\p{N}_]{2}/u; +const LINE_SEPARATORS = /[\u2028\u2029]/g; /** A highly repetitive indentation candidate must not monopolize the worker. */ const MAX_LINE_WINDOW_VERIFICATIONS = 100_000; const MAX_REPLACEMENT_CHUNK_CHARS = 16 * 1024; @@ -45,6 +61,12 @@ export interface EditFailure { /** Zero-based position of the edit in the request. */ index: number; reason: string; + /** + * A final `; the current text at ...` hint quoting the region the edit most + * likely meant. It is appended to `reason` only while the whole message stays + * within its bound, and is the first detail dropped when it would not. + */ + excerpt?: string; } export class WorkspaceEditMatchError extends Error { @@ -93,14 +115,16 @@ export function applyTextEdits( matches.push({ strategy: outcome.strategy, occurrences: outcome.occurrences }); return; } + if (outcome.status === 'none') { + failures.push({ index, ...describeMissing(working, edit.oldText, matching, lines) }); + return; + } failures.push({ index, reason: outcome.status === 'ambiguous' ? describeAmbiguous(working, outcome.strategy, outcome.count, outcome.starts) - : outcome.status === 'limit' - ? 'old_text has too many repetitive line-window candidates; include more surrounding lines or use an exact match' - : describeMissing(working, edit.oldText, matching, lines), + : 'old_text has too many repetitive line-window candidates; include more surrounding lines or use an exact match', }); }); if (failures.length > 0) { @@ -589,11 +613,21 @@ function describeAmbiguous( return `old_text matched ${count} locations${how} at ${formatLineList(text, starts, count)}; include more surrounding lines so it matches exactly one`; } +/** + * JSON quoting that also escapes U+2028/U+2029, which JSON.stringify leaves + * raw: hosts read a batch diagnostic one line per edit, and a regular + * expression `.` stops at those separators. + */ +function quote(value: string): string { + return JSON.stringify(value).replace( + LINE_SEPARATORS, + (separator) => `\\u${separator.charCodeAt(0).toString(16)}`, + ); +} + function snippet(line: string): string { const trimmed = line.trim(); - return JSON.stringify( - trimmed.length > MAX_SNIPPET_CHARS ? `${trimmed.slice(0, MAX_SNIPPET_CHARS)}…` : trimmed, - ); + return quote(trimmed.length > MAX_SNIPPET_CHARS ? `${trimmed.slice(0, MAX_SNIPPET_CHARS)}…` : trimmed); } const ELISION_LINE = /^\s*(?:(?:\/\/|#|--|\/\*|\*|))?\s*$/; @@ -604,9 +638,121 @@ function describeMissing( oldText: string, matching: WorkspaceEditMatching, lines: () => LineIndex, +): { reason: string; excerpt?: string } { + const needle = neededLines(oldText).lines; + const firstOffset = needle.findIndex((line) => line.trim().length > 0); + const nearest = nearestLine(needle[firstOffset], lines); + const reason = describeMissingReason(text, oldText, needle, matching, lines, nearest); + // The anchor is the first non-blank line, so the region starts that many lines earlier. + const anchor = nearest != null && (!nearest.exact || nearest.count === 1) + ? Math.max(0, nearest.index - firstOffset) + : undefined; + const start = closestRegion(needle, lines()) ?? anchor; + const excerpt = start == null ? undefined : currentTextExcerpt(needle, lines(), start); + return excerpt == null ? { reason } : { reason, excerpt }; +} + +type ExcerptMark = ' ' | '~' | '!'; + +/** + * Finds where a missing multi-line edit most likely belongs: the alignment on + * which the most of its distinctive lines, compared without surrounding + * whitespace, fall into place. One pass over the file with bounded votes. + */ +function closestRegion(needle: readonly string[], lines: LineIndex): number | undefined { + const offsets = new Map(); + let anchors = 0; + needle.forEach((line, offset) => { + const key = line.trim(); + if (!ANCHOR_LINE.test(key)) return; + anchors++; + const known = offsets.get(key); + if (known) known.push(offset); + else offsets.set(key, [offset]); + }); + if (anchors < 2) return undefined; + const votes = new Map(); + let budget = MAX_REGION_VOTES; + let best: number | undefined; + let bestVotes = 0; + for (let index = 0; index < lines.length && budget > 0; index++) { + const hits = offsets.get(lines.text(index).trim()); + if (!hits) continue; + // Offsets ascend, so every later one would start before the file too. + for (const offset of hits) { + if (budget-- <= 0) break; + const start = index - offset; + if (start < 0) break; + const count = (votes.get(start) ?? 0) + 1; + votes.set(start, count); + if (count > bestVotes || (count === bestVotes && best !== undefined && start < best)) { + best = start; + bestVotes = count; + } + } + } + // One shared line is too weak to claim a region; the first-line anchor covers that case. + return bestVotes >= 2 ? best : undefined; +} + +function excerptMark(current: string, expected: string | undefined): ExcerptMark { + if (expected === undefined) return '!'; + if (current === expected) return ' '; + return current.trim() === expected.trim() ? '~' : '!'; +} + +function shortenLine(line: string): string { + if (line.length <= MAX_EXCERPT_LINE_CHARS) return line; + let end = MAX_EXCERPT_LINE_CHARS; + if (/[\uD800-\uDBFF]/.test(line[end - 1]) && /[\uDC00-\uDFFF]/.test(line[end])) end--; + return `${line.slice(0, end)}…`; +} + +/** + * Quotes the current text where a missing edit most likely belongs, so the next + * attempt can copy it instead of re-reading the file. Each line is marked + * against the `old_text` line at the same position: a space when identical, + * `~` when only whitespace differs and `!` when the text differs. A region + * longer than the excerpt is shown from just before its first difference. + */ +function currentTextExcerpt(needle: readonly string[], lines: LineIndex, start: number): string | undefined { + const span = Math.max(1, Math.min(needle.length, lines.length - start)); + const marks = Array.from({ length: span }, (_, offset) => + excerptMark(lines.text(start + offset), needle[offset]), + ); + const shown = Math.min(span, MAX_EXCERPT_LINES); + const firstDifference = Math.max(0, marks.findIndex((mark) => mark !== ' ')); + const first = start + Math.min(Math.max(0, firstDifference - 1), span - shown); + const rows = marks + .slice(first - start, first - start + shown) + .map((mark, offset) => `${first + offset + 1}|${mark}${shortenLine(lines.text(first + offset))}`); + for (; rows.length > 0; rows.pop()) { + const where = rows.length === 1 ? `line ${first + 1}` : `lines ${first + 1}-${first + rows.length}`; + const excerpt = `the current text at ${where} (~ whitespace differs, ! text differs) is ${quote(rows.join('\n'))}`; + if (encodedBytes(excerpt) <= MAX_EXCERPT_BYTES) return excerpt; + } + return undefined; +} + +interface NearestLine { + exact: boolean; + count: number; + /** Line index of the first exact candidate, or of the best token match. */ + index: number; + starts: number[]; + text: string; +} + +function describeMissingReason( + text: string, + oldText: string, + needle: readonly string[], + matching: WorkspaceEditMatching, + lines: () => LineIndex, + nearest: NearestLine | undefined, ): string { const hints: string[] = []; - const nonBlank = neededLines(oldText).lines.filter((line) => line.trim().length > 0); + const nonBlank = needle.filter((line) => line.trim().length > 0); if (nonBlank.some((line) => ELISION_LINE.test(line))) { hints.push('it contains an elision placeholder ("..."); copy the exact lines instead of abbreviating'); } @@ -629,7 +775,6 @@ function describeMissing( hints.push('the file uses CRLF line endings'); } } - const nearest = nearestLine(nonBlank[0], lines); if (nearest != null && hints.length === 0) { hints.push( nearest.exact @@ -644,48 +789,106 @@ function describeMissing( function nearestLine( firstLine: string | undefined, getLines: () => LineIndex, -): { exact: boolean; count: number; starts: number[]; text: string } | undefined { +): NearestLine | undefined { const target = firstLine?.trim(); if (!target) return undefined; const lines = getLines(); const starts: number[] = []; let count = 0; - let firstMatch = ''; + let first = 0; for (let index = 0; index < lines.length; index++) { - const content = lines.text(index); - if (content.trim() !== target) continue; - if (count++ === 0) firstMatch = content; + if (lines.text(index).trim() !== target) continue; + if (count++ === 0) first = index; if (starts.length < MAX_REPORTED_LINES) starts.push(lines.start(index)); } if (count > 0) { - return { exact: true, count, starts, text: firstMatch }; + return { exact: true, count, index: first, starts, text: lines.text(first) }; } const tokens = new Set(target.split(/\W+/).filter((token) => token.length > 1)); if (tokens.size < 2) return undefined; - let best: Line | undefined; + let best: number | undefined; let bestScore = 0; for (let index = 0; index < lines.length; index++) { - const content = lines.text(index); let score = 0; - for (const token of new Set(content.split(/\W+/))) { + for (const token of new Set(lines.text(index).split(/\W+/))) { if (tokens.has(token)) score++; } if (score > bestScore) { - best = { start: lines.start(index), end: lines.end(index), next: lines.next(index), text: content }; + best = index; bestScore = score; } } return best != null && bestScore / tokens.size >= 0.5 - ? { exact: false, count: 1, starts: [best.start], text: best.text } + ? { exact: false, count: 1, index: best, starts: [lines.start(best)], text: lines.text(best) } : undefined; } +/** UTF-8 bytes `value` occupies once JSON-encoded as a string in an error body. */ +function encodedBytes(value: string): number { + return Buffer.byteLength(JSON.stringify(value)) - 2; +} + +/** A failure's reason, followed by its excerpt when the excerpt was granted room. */ +function failureText(failure: EditFailure, withExcerpt: boolean): string { + return withExcerpt && failure.excerpt ? `${failure.reason}; ${failure.excerpt}` : failure.reason; +} + +/** + * Grants excerpts in edit order while the message still fits. Excerpts are the + * most expendable detail, so granting one never shortens another reason. + */ +function grantExcerpts( + failures: readonly EditFailure[], + availableChars: number, + availableBytes: number, +): boolean[] { + let chars = availableChars - failures.reduce((total, failure) => total + failure.reason.length, 0); + let bytes = availableBytes - failures.reduce((total, failure) => total + encodedBytes(failure.reason), 0); + return failures.map((failure) => { + if (!failure.excerpt) return false; + const addition = `; ${failure.excerpt}`; + const byteCost = encodedBytes(addition); + if (addition.length > chars || byteCost > bytes) return false; + chars -= addition.length; + bytes -= byteCost; + return true; + }); +} + +/** + * The longest prefix of `reason` within both limits, ending in an ellipsis. It + * never splits a surrogate pair in a shortened source excerpt. + */ +function shortenReason(reason: string, charLimit: number, byteLimit: number): string { + if (reason.length <= charLimit && encodedBytes(reason) <= byteLimit) return reason; + const budget = byteLimit - encodedBytes('…'); + let end = 0; + let bytes = 0; + while (end < reason.length) { + const width = /[\uD800-\uDBFF]/.test(reason[end]) && /[\uDC00-\uDFFF]/.test(reason[end + 1] ?? '') ? 2 : 1; + const cost = encodedBytes(reason.slice(end, end + width)); + if (end + width > charLimit - 1 || bytes + cost > budget) break; + bytes += cost; + end += width; + } + return `${reason.slice(0, end)}…`; +} + +function fitsDiagnosticBounds(message: string): boolean { + return message.length <= EDIT_DIAGNOSTIC_MAX_CHARS && encodedBytes(message) <= EDIT_DIAGNOSTIC_MAX_BODY_BYTES; +} + function formatEditFailures(failures: readonly EditFailure[], editCount: number): string { if (editCount === 1) { - return `Workspace edit did not apply and nothing was written: ${failures[0]?.reason ?? 'no match'}.`.slice( - 0, - EDIT_DIAGNOSTIC_MAX_CHARS, + const prefix = 'Workspace edit did not apply and nothing was written: '; + const detailed = `${prefix}${failures[0] ? failureText(failures[0], true) : 'no match'}.`; + if (fitsDiagnosticBounds(detailed)) return detailed; + const reason = shortenReason( + failures[0]?.reason ?? 'no match', + EDIT_DIAGNOSTIC_MAX_CHARS - prefix.length - 1, + EDIT_DIAGNOSTIC_MAX_BODY_BYTES - encodedBytes(prefix) - 1, ); + return `${prefix}${reason}.`; } const header = `${failures.length} of ${editCount} workspace edits did not apply, so nothing was written. Every other edit matched.`; const footer = failures.some((failure) => failure.index > 0) @@ -696,17 +899,16 @@ function formatEditFailures(failures: readonly EditFailure[], editCount: number) // reasons or source excerpts. Never leave callers guessing which edits failed. const available = EDIT_DIAGNOSTIC_MAX_CHARS - header.length - footer.length - prefixes.reduce((total, prefix) => total + prefix.length + 1, 0); - const reasonLength = failures.reduce((total, failure) => total + failure.reason.length, 0); - const reasonLimit = reasonLength <= available ? Infinity : Math.floor(available / failures.length); - const details = failures.map((failure, index) => { - let reason = failure.reason; - if (reason.length > reasonLimit) { - let end = reasonLimit - 1; - // Do not split a surrogate pair in a shortened source excerpt. - if (end > 0 && /[\uD800-\uDBFF]/.test(reason[end - 1]) && /[\uDC00-\uDFFF]/.test(reason[end])) end--; - reason = `${reason.slice(0, end)}…`; - } - return `${prefixes[index]}${reason}.`; - }); + const frame = header + footer + prefixes.join('') + '.'.repeat(failures.length); + const availableBytes = EDIT_DIAGNOSTIC_MAX_BODY_BYTES - encodedBytes(frame); + const granted = grantExcerpts(failures, available, availableBytes); + const reasons = failures.map((failure, index) => failureText(failure, granted[index])); + const reasonLength = reasons.reduce((total, reason) => total + reason.length, 0); + const reasonBytes = reasons.reduce((total, reason) => total + encodedBytes(reason), 0); + const charLimit = reasonLength <= available ? Infinity : Math.floor(available / failures.length); + const byteLimit = reasonBytes <= availableBytes ? Infinity : Math.floor(availableBytes / failures.length); + const details = reasons.map((reason, index) => + `${prefixes[index]}${shortenReason(reason, charLimit, byteLimit)}.`, + ); return header + details.join('') + footer; } From 961777e3fd1347e9c0b571c02292371e4a6d8d86 Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 11:33:34 -0400 Subject: [PATCH 29/48] feat: Persist Workspace Admission Across API Restarts (#302) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🧾 feat: Persist Workspace Admission Across API Restarts * 🧾 fix: Fence Durable Admission Recovery Across Replicas * 🧾 fix: Fence Durable Capacity Acquisition Before Dispatch * 🧾 fix: Retain Durable Quarantine Outcomes Across Restarts * 🧾 fix: Preserve Durable Admission and Read Cancellation --------- Co-authored-by: Lia --- .github/workflows/ci.yml | 2 +- docs/remote-bridge/README.md | 55 ++ service/openapi.yml | 105 +++ service/src/bridge/admission.ts | 55 ++ service/src/bridge/slots.ts | 22 +- service/src/bridge/store.ts | 235 +++++- service/src/workspace-tools/coordination.ts | 21 + service/src/workspace-tools/index.ts | 49 +- service/src/workspace-tools/outcome.ts | 4 +- .../workspace-tools/requests-router.test.ts | 96 +++ service/src/workspace-tools/requests.test.ts | 671 ++++++++++++++++++ service/src/workspace-tools/requests.ts | 251 +++++++ service/src/workspace-tools/router.ts | 320 +++++---- 13 files changed, 1718 insertions(+), 168 deletions(-) create mode 100644 service/src/workspace-tools/coordination.ts create mode 100644 service/src/workspace-tools/requests-router.test.ts create mode 100644 service/src/workspace-tools/requests.test.ts create mode 100644 service/src/workspace-tools/requests.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 87aaa066..5ee14372 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -170,7 +170,7 @@ jobs: redis_socket="$RUNNER_TEMP/byom-admission.sock" redis-server --port 0 --unixsocket "$redis_socket" --save '' --appendonly no --daemonize yes trap 'redis-cli -s "$redis_socket" shutdown nosave' EXIT - BRIDGE_TEST_REDIS_URL="$redis_socket" bun test src/bridge/fleet.test.ts + BRIDGE_TEST_REDIS_URL="$redis_socket" bun test src/bridge/fleet.test.ts src/workspace-tools/requests.test.ts - name: Build service run: bun run build diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index b2d33db7..dc7cb98b 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -384,3 +384,58 @@ capabilities. LibreChat's owner-scoped environment registry can issue these principal-bound pairings without changing the worker execution protocol or moving code tools into the Agents SDK. + +### Durable workspace requests + +New clients can probe authenticated `GET /v1/workspace-tools/capabilities` for +`durableWorkspaceRequests: 1`. Older servers return 404 or advertise 0. Keep the +legacy synchronous `/workspace-tools/execute` path for those servers. Do not +fall back to synchronous execution after submitting durable work. + +Submit the same workspace-tool body to `POST /v1/workspace-tools/requests`, with +`X-LibreChat-Workspace-Request-Id` set to a unique 16–128 character identifier +(letters, digits, `_`, `-`). The queue-wait header and command timeout retain their +existing limits. HTTP 202 acknowledges durable acceptance, not execution. +Disconnecting does not cancel accepted work. + +- Repeat an identical submission with the same ID to recover a lost response. + An ID reused for different work returns 409 `REQUEST_CONFLICT`. +- Poll `GET /v1/workspace-tools/requests/:requestId`. States are `queued`, + `admitted`, `completed`, `failed`, and `cancelled`. Lookup is tenant/user scoped; + unknown or another principal's IDs return 404. Results are non-consuming reads. +- Cancel with `DELETE /v1/workspace-tools/requests/:requestId`. Cancellation is a + request, not proof of termination. All operations notify the worker. A committed + result can win the race. Without a confirmed settlement, read-only calls retain + `cancelled`; mutations retain an unknown-outcome failure and their safety fence. + `ASSIGNMENT_EXPIRED` means execution may have occurred. Never replay it. +- Status includes worker/workspace/lane metadata, queue position when available, + measured queue wait, and the execution deadline after admission. Queue position + is advisory: negotiated independent roots may run concurrently. + +Requests and results are retained for 24 hours from acceptance. Idempotency is +bounded by that retention window. A client must not resubmit an expired ID after +404; its outcome is unknown. Automatic result claims and wake-ups remain the +client's responsibility. + +Bridge-enabled API replicas reconcile accepted work from Redis within a shared +bridge-policy scope. Disabled bridges and different scheduling policies never +claim that work. Keep backend/profile, bridge auth mode, worker selection, slot +ceiling and command-timeout policy identical on replicas behind one endpoint. +Drain durable requests before changing those settings. FIFO admission position +survives API restarts and temporary worker registration/readiness loss until the +queue deadline. Capacity acquisition and assignment enqueue atomically check the coordinator +claim, queued state, cancellation and admission deadline. Delayed callbacks +cannot reserve capacity after their claim ends. Assignment enqueue also checks +capacity ownership and atomically commits the durable `admitted` transition. +Recovery resumes only queued work and observes admitted assignments without +creating another command. Worker identity/incarnation changes fail pending work +rather than transferring it to another machine. Execution gets its full budget +after admission. Existing lease fencing, cancellation and workspace quarantine +continue to apply. + +This capability covers workspace tools, including `execute_command`, not +programmatic `/execute` jobs. It requires Redis state to survive the API restart; +Redis data loss is not recoverable. All API replicas behind an endpoint must +support the capability before a client enables it. Roll back clients to the legacy +path only for new work; drain durable requests before rolling back servers. +Worker protocol, worker concurrency and deployed rate limits are unchanged. diff --git a/service/openapi.yml b/service/openapi.yml index a257fd18..07d2d7e3 100644 --- a/service/openapi.yml +++ b/service/openapi.yml @@ -64,6 +64,40 @@ components: - $ref: '#/components/schemas/ExecutionProfileError' schemas: + WorkspaceToolRequest: + type: object + description: Workspace tool protocol v1 body; operation-specific fields follow the worker protocol. + required: [protocolVersion, operation, workspaceId] + properties: + protocolVersion: { type: integer, enum: [1] } + operation: + type: string + enum: [list_files, search_text, read_file, write_file, preview_edit, edit_file, execute_command] + workspaceId: { type: string } + additionalProperties: true + + WorkspaceRequestStatus: + type: object + required: [requestId, state, workerId, workspaceId, queueWaitMs, cancelRequested] + properties: + requestId: { type: string } + state: { type: string, enum: [queued, admitted, completed, failed, cancelled] } + workerId: { type: string } + workspaceId: { type: string } + workspaceInstanceId: { type: string } + worktree: { type: string } + queuePosition: { type: integer, minimum: 1 } + queueWaitMs: { type: integer, minimum: 0 } + executionDeadlineAt: { type: string, format: date-time } + cancelRequested: { type: boolean } + result: { type: object, additionalProperties: true } + error: + type: object + required: [code, message] + properties: + code: { type: string } + message: { type: string } + FileRef: type: object properties: @@ -312,6 +346,77 @@ components: type: string paths: + /workspace-tools/capabilities: + get: + summary: Discover opt-in durable workspace request support + responses: + '200': + description: All replicas must support version 1 before a client enables durable submission. + content: + application/json: + schema: + type: object + properties: + durableWorkspaceRequests: { type: integer, enum: [0, 1] } + + /workspace-tools/requests: + post: + summary: Submit an idempotent durable workspace request + description: >- + Disconnect does not cancel accepted work. IDs and results are retained for 24 hours + from acceptance. Never resubmit after that retention window or fall back to synchronous + execution after durable acceptance. The execution budget begins after admission. + parameters: + - $ref: '#/components/parameters/ExpectedExecutionProfile' + - in: header + name: X-LibreChat-Workspace-Request-Id + required: true + schema: { type: string, pattern: '^[A-Za-z0-9_-]{16,128}$' } + - in: header + name: X-LibreChat-Workspace-Queue-Wait-Ms + schema: { type: integer, minimum: 1, maximum: 300000, default: 30000 } + requestBody: + required: true + content: + application/json: + schema: { $ref: '#/components/schemas/WorkspaceToolRequest' } + responses: + '202': + description: Accepted, or an identical request with this principal-scoped ID already exists. + content: + application/json: + schema: { $ref: '#/components/schemas/WorkspaceRequestStatus' } + '400': { $ref: '#/components/responses/BadRequest' } + '409': { $ref: '#/components/responses/Conflict' } + '429': { description: Admission capacity or execution rate limit reached } + '503': { description: Code environment unavailable } + + /workspace-tools/requests/{requestId}: + parameters: + - in: path + name: requestId + required: true + schema: { type: string, pattern: '^[A-Za-z0-9_-]{16,128}$' } + - $ref: '#/components/parameters/ExpectedExecutionProfile' + get: + summary: Read a retained workspace request and result without consuming it + responses: + '200': + description: Status scoped to the authenticated tenant and user. ASSIGNMENT_EXPIRED is not proof of non-execution. + content: + application/json: + schema: { $ref: '#/components/schemas/WorkspaceRequestStatus' } + '404': { description: Unknown, expired, or another principal's request } + delete: + summary: Request cancellation of the same logical call + responses: + '200': + description: Cancellation recorded, or terminal status returned. A committed result can win the race. + content: + application/json: + schema: { $ref: '#/components/schemas/WorkspaceRequestStatus' } + '404': { description: Unknown, expired, or another principal's request } + /hosted-apps: post: summary: Start or reassert a resident hosted app diff --git a/service/src/bridge/admission.ts b/service/src/bridge/admission.ts index 4002ed6d..18ab1f49 100644 --- a/service/src/bridge/admission.ts +++ b/service/src/bridge/admission.ts @@ -1,5 +1,22 @@ import type Redis from 'ioredis'; +/** Redis time and claim ownership gate every durable admission mutation. */ +export function durableAdmissionFence( + requestKey: string, + claimKey: string, + token: string, + rejected: number, +): string[] { + return [ + `if redis.call('GET', ${claimKey}) ~= ${token} then return ${rejected} end`, + `if redis.call('HGET', ${requestKey}, 'state') ~= 'queued' then return ${rejected} end`, + `if redis.call('HGET', ${requestKey}, 'cancelRequested') == '1' then return ${rejected} end`, + 'local admissionTime = redis.call(\'TIME\')', + 'local admissionNowMs = tonumber(admissionTime[1]) * 1000.0 + math.floor(tonumber(admissionTime[2]) / 1000)', + `if tonumber(redis.call('HGET', ${requestKey}, 'queueDeadlineAtMs')) <= admissionNowMs then return ${rejected} end`, + ]; +} + /** Bounded FIFO admission shared by API replicas. Entries expire after caller deadlines. */ export class BridgeAdmissionQueue { constructor( @@ -57,6 +74,44 @@ export class BridgeAdmissionQueue { ); } + async submit(args: { + workerId: string; id: string; deadlineAtMs: number; workspaceId?: string; + key: string; activeKey: string; fingerprint: string; record: string; retentionMs: number; + }): Promise<'accepted' | 'existing' | 'conflict' | 'full'> { + const result = Number(await this.redis.eval([ + 'local fingerprint = redis.call(\'HGET\', KEYS[5], \'fingerprint\')', + 'if fingerprint then', + ' if fingerprint == ARGV[6] then return 2 end', + ' return -1', + 'end', + 'local expired = redis.call(\'ZRANGEBYSCORE\', KEYS[2], \'-inf\', ARGV[2])', + 'for _, id in ipairs(expired) do', + ' redis.call(\'ZREM\', KEYS[1], id); redis.call(\'ZREM\', KEYS[2], id); redis.call(\'HDEL\', KEYS[4], id)', + 'end', + 'if redis.call(\'ZCARD\', KEYS[1]) >= tonumber(ARGV[4]) then return 0 end', + 'local sequence = redis.call(\'INCR\', KEYS[3])', + 'redis.call(\'ZADD\', KEYS[1], sequence, ARGV[1])', + 'redis.call(\'ZADD\', KEYS[2], ARGV[3], ARGV[1])', + 'if ARGV[5] ~= \'\' then redis.call(\'HSET\', KEYS[4], ARGV[1], ARGV[5]) end', + 'local latest = redis.call(\'ZREVRANGE\', KEYS[2], 0, 0, \'WITHSCORES\')', + 'for i = 1, 4 do redis.call(\'PEXPIREAT\', KEYS[i], tonumber(latest[2]) + 30000) end', + 'redis.call(\'HSET\', KEYS[5], \'record\', ARGV[7], \'fingerprint\', ARGV[6], \'state\', \'queued\', \'queueDeadlineAtMs\', ARGV[3])', + 'redis.call(\'PEXPIRE\', KEYS[5], ARGV[8])', + 'redis.call(\'ZADD\', KEYS[6], ARGV[2], KEYS[5])', + 'return 1', + ].join('\n'), 6, ...this.keys(args.workerId), args.key, args.activeKey, + args.id, Date.now(), args.deadlineAtMs, this.capacity, args.workspaceId ?? '', + args.fingerprint, args.record, args.retentionMs)); + if (result === 2) return 'existing'; + if (result === -1) return 'conflict'; + return result === 1 ? 'accepted' : 'full'; + } + + async position(workerId: string, id: string): Promise { + const rank = await this.redis.zrank(this.keys(workerId)[0], id); + return rank == null ? undefined : rank + 1; + } + async isHead(workerId: string, id: string): Promise { const [order, deadlines, , workspaces] = this.keys(workerId); return ( diff --git a/service/src/bridge/slots.ts b/service/src/bridge/slots.ts index 81f60ebe..f748054b 100644 --- a/service/src/bridge/slots.ts +++ b/service/src/bridge/slots.ts @@ -1,4 +1,5 @@ import type Redis from 'ioredis'; +import { durableAdmissionFence } from './admission'; /** Hard bound keeps every atomic scheduling scan constant-sized. */ export const MAX_WORKSPACE_LEASE_SLOTS = 8; @@ -37,6 +38,8 @@ export class BridgeWorkspaceSlots { workspaceId: string; capacity: number; expiresAtMs: number; + refreshOwned?: boolean; + guard?: { key: string; claimKey: string; token: string }; }): Promise { if ( !Number.isSafeInteger(args.capacity) || @@ -48,9 +51,12 @@ export class BridgeWorkspaceSlots { ) { throw new Error('Invalid workspace slot reservation'); } + const keys = this.keys(args.workerId); + if (args.guard != null) keys.push(args.guard.key, args.guard.claimKey); const result = Number( await this.redis.eval( [ + ...(args.guard == null ? [] : durableAdmissionFence('KEYS[9]', 'KEYS[10]', 'ARGV[8]', -3)), "if redis.call('GET', KEYS[4]) ~= ARGV[1] then return -2 end", "if (redis.call('GET', KEYS[8]) or '1') ~= ARGV[4] then return -2 end", // Parent of a linked-worktree lane key; nil for a checkout key. @@ -83,7 +89,15 @@ export class BridgeWorkspaceSlots { " redis.call('HDEL', KEYS[1], 'a:' .. slot, 'i:' .. slot, 'w:' .. slot, 'e:' .. slot)", ' occupied = false', ' else', - ' if entry[1] == ARGV[2] then return slot end', + ' if entry[1] == ARGV[2] then', + ' if ARGV[7] == \'1\' then', + ' redis.call(\'HSET\', KEYS[1], \'e:\' .. slot, ARGV[5])', + ' for _, key in ipairs({KEYS[1], KEYS[2], KEYS[3]}) do', + ' if redis.call(\'PTTL\', key) < tonumber(ARGV[5]) - tonumber(ARGV[6]) then redis.call(\'PEXPIREAT\', key, ARGV[5]) end', + ' end', + ' end', + ' return slot', + ' end', ' busy[entry[3]] = true', ' local busyParent = parentOf(entry[3])', ' if busyParent then busyChildren[busyParent] = true end', @@ -125,14 +139,16 @@ export class BridgeWorkspaceSlots { "redis.call('SET', KEYS[3], ARGV[1], 'PXAT', latest)", 'return free', ].join('\n'), - 8, - ...this.keys(args.workerId), + keys.length, + ...keys, args.incarnationId, args.assignmentId, args.workspaceId, args.capacity, args.expiresAtMs, Date.now(), + args.refreshOwned === true ? '1' : '0', + args.guard?.token ?? '', ), ); if (result === -2) diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index f64e748a..542c5903 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -21,8 +21,9 @@ import { workspaceIsolationParent, } from '../../../packages/code/src/protocol'; import type { BridgeWorkerBinding } from './pairing'; -import { BridgeAdmissionQueue } from './admission'; +import { BridgeAdmissionQueue, durableAdmissionFence } from './admission'; import { BridgeWorkspaceSlots } from './slots'; +import type { StoredWorkspaceRequest } from '../workspace-tools/requests'; const PREFIX = 'codeapi:bridge:v1'; const POLL_INTERVAL_MS = 100; @@ -90,6 +91,22 @@ interface StoredAssignment extends CodeBridgeAssignment { leaseTokenHash: string; workerIdentityId?: string; workspaceFence?: string; + durableRequestKey?: string; +} + +export type DurableWorkspaceAssignment = StoredAssignment; + +export interface DurableWorkspaceGuard { + key: string; + claimKey: string; + token: string; +} + +export interface DurableWorkspaceOutcome { + state: 'completed' | 'failed' | 'cancelled'; + result?: WorkspaceToolResult; + error?: { code: string; message: string }; + settlement?: CodeBridgeWorkspaceSettlement; } type AssignmentOwnership = Pick< @@ -104,6 +121,7 @@ type AssignmentOwnership = Pick< | 'workerIdentityId' | 'expiresAt' | 'runtimeSessionId' + | 'durableRequestKey' >; const NATIVE_WORKSPACE_FENCE_PREFIX = 'native-workspace:'; @@ -148,6 +166,10 @@ export interface BridgeWorkerStatus { capabilities?: BridgeWorkerRegistration['capabilities']; } +function isWorkspaceMutation(request: WorkspaceToolRequest): boolean { + return request.operation === 'write_file' || request.operation === 'edit_file' || request.operation === 'execute_command'; +} + function supportsWorkspaceTool( registration: RegisteredBridgeWorker, request: WorkspaceToolRequest, @@ -381,6 +403,12 @@ function assignmentTtlSeconds(deadlineAtMs: number): number { return Math.max(1, Math.ceil((deadlineAtMs - Date.now()) / 1000) + 30); } +function assignmentOutcomeTtlSeconds(assignment: AssignmentOwnership): number { + return assignment.durableRequestKey == null + ? assignmentTtlSeconds(Date.parse(assignment.expiresAt)) + : 24 * 60 * 60; +} + async function delay(ms: number, signal?: AbortSignal): Promise { if (signal?.aborted === true) return; await new Promise((resolve) => { @@ -775,6 +803,127 @@ export class RedisBridgeStore { } } + async prepareDurableWorkspaceTool(args: { + workerId: string; tenantId: string; requireTenantBinding: boolean; + request: WorkspaceToolRequest; executionTimeoutMs: number; queueWaitMs: number; + }): Promise { + if (!isWorkspaceToolRequest(args.request) || + !Number.isSafeInteger(args.executionTimeoutMs) || args.executionTimeoutMs < 1 || args.executionTimeoutMs > 305_000 || + !Number.isSafeInteger(args.queueWaitMs) || args.queueWaitMs < 1 || args.queueWaitMs > 300_000) { + throw new BridgeStoreError('ASSIGNMENT_INVALID', 'Invalid durable workspace request'); + } + const current = await boundedCommand(this.dispatchableRegistration(args.workerId), this.redisCommandTimeoutMs, 'Durable worker registration'); + if (current == null) throw new BridgeStoreError('WORKER_OFFLINE', 'Code environment is offline'); + const registration = current.registration; + if ((args.requireTenantBinding && registration.binding == null) || + (registration.binding != null && registration.binding.tenantId !== args.tenantId)) { + throw new BridgeStoreError('WORKER_UNAUTHORIZED', 'Code environment is not authorized for this tenant'); + } + if ((registration.capabilities.workspaceLeaseSlots ?? 1) > this.maxWorkspaceLeaseSlots || + !supportsWorkspaceTool(registration, args.request)) { + throw new BridgeStoreError('WORKER_MISMATCH', 'Code environment does not support the requested workspace operation'); + } + return registration; + } + + async advanceDurableWorkspaceTool( + record: StoredWorkspaceRequest, + guard: DurableWorkspaceGuard, + ): Promise { + if (record.state === 'queued') { + if (record.cancelRequested === true) return { state: 'cancelled' }; + if (Date.now() >= record.queueDeadlineAtMs) return { + state: 'failed', error: { code: 'WORKSPACE_QUEUE_TIMEOUT', message: 'Workspace admission deadline expired. The operation was not started.' }, + }; + const registration = await this.prepareDurableWorkspaceTool({ + ...record, queueWaitMs: Math.max(1, record.queueDeadlineAtMs - Date.now()), + }); + if (registration.incarnationId !== record.registration.incarnationId || + registration.identityId !== record.registration.identityId || + registration.binding?.tenantId !== record.registration.binding?.tenantId) { + throw new BridgeStoreError('WORKER_FENCED', 'Code environment changed while the request was waiting'); + } + const admission = new BridgeAdmissionQueue(this.redis); + const workspace = workspaceAdmissionId(record.request.workspaceId, record.request.workspaceInstanceId, record.request.worktree); + const capacity = registration.capabilities.workspaceLeaseSlots ?? 1; + if (capacity === 1 && !await admission.isHead(record.workerId, record.id)) return; + const expiresAtMs = Date.now() + record.executionTimeoutMs; + const ttlSeconds = assignmentTtlSeconds(expiresAtMs); + let slot: number | undefined; + if (capacity > 1) { + slot = await new BridgeWorkspaceSlots(this.redis).reserve({ + workerId: record.workerId, incarnationId: registration.incarnationId, + assignmentId: record.id, workspaceId: workspace, capacity, + expiresAtMs: Date.now() + ttlSeconds * 1000, refreshOwned: true, guard, + }); + if (slot === undefined) return; + } else if (!await this.acquireLock(record.workerId, record.id, registration.incarnationId, ttlSeconds, true, guard)) return; + if (Date.now() >= record.queueDeadlineAtMs) return { + state: 'failed', error: { code: 'WORKSPACE_QUEUE_TIMEOUT', message: 'Workspace admission deadline expired. The operation was not started.' }, + }; + const leaseToken = randomBytes(32).toString('base64url'); + const assignment: StoredAssignment = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + assignmentId: record.id, workerId: record.workerId, + incarnationId: registration.incarnationId, + generation: await this.redis.incr(generationKey(record.workerId)), + leaseToken, leaseTokenHash: tokenHash(leaseToken), + workerIdentityId: registration.identityId, + workspaceFence: `${NATIVE_WORKSPACE_FENCE_PREFIX}${workspace}`, + workspaceLeaseSlot: slot, executionKind: 'workspace_tool', durableRequestKey: record.key, + expiresAt: new Date(Date.now() + record.executionTimeoutMs).toISOString(), + request: record.request, + }; + const ready = await this.dispatchableRegistration(record.workerId); + if (ready == null) throw new BridgeStoreError('WORKER_OFFLINE', 'Code environment is temporarily unavailable'); + if (ready.registration.incarnationId !== registration.incarnationId || + ready.registration.identityId !== registration.identityId || + ready.registration.binding?.tenantId !== registration.binding?.tenantId || + !supportsWorkspaceTool(ready.registration, record.request)) { + throw new BridgeStoreError('WORKER_FENCED', 'Code environment changed before admission'); + } + await this.enqueueForActiveIncarnation(assignment, assignmentTtlSeconds(Date.parse(assignment.expiresAt)), ready.readyToken, guard); + return; + } + const assignment = record.assignment; + if (assignment == null) throw new Error('Durable admitted request has no assignment'); + let settlement: CodeBridgeWorkspaceSettlement; + try { + const raw = await boundedCommand(this.redis.get(settlementKey(record.id)), this.redisCommandTimeoutMs, 'Durable settlement lookup'); + if (raw == null && record.cancelRequested !== true && Date.now() < Date.parse(assignment.expiresAt)) return; + if (raw != null) settlement = JSON.parse(raw) as CodeBridgeWorkspaceSettlement; + else { + const controller = new AbortController(); + if (record.cancelRequested === true) controller.abort(); + settlement = await this.waitForSettlement(assignment, Date.parse(assignment.expiresAt), controller.signal) as unknown as CodeBridgeWorkspaceSettlement; + } + } catch (error) { + if (!(error instanceof BridgeStoreError) || error.code !== 'ASSIGNMENT_EXPIRED') throw error; + if (record.cancelRequested === true && !isWorkspaceMutation(record.request)) return { state: 'cancelled' }; + return { state: 'failed', error: { code: error.code, message: 'Assignment ended without a confirmed result. Do not replay this operation.' } }; + } + if (settlement.status === 'fulfilled') { + if (!isWorkspaceToolResult(record.request, settlement.result, record.registration.capabilities.workspaceTools)) { + return { state: 'failed', error: { code: 'RESULT_INVALID', message: 'Code environment returned an invalid workspace result. Do not replay this operation.' } }; + } + return { state: 'completed', result: settlement.result, settlement }; + } + return { + state: record.cancelRequested === true ? 'cancelled' : 'failed', + error: { code: settlement.errorCode ?? 'WORKSPACE_TOOL_REJECTED', message: settlement.error }, settlement, + }; + } + + async finishDurableWorkspaceTool(record: StoredWorkspaceRequest): Promise { + if (record.assignment != null) { + if (record.settlement != null) await this.commitPendingWorkspace(record.assignment, record.settlement, record.key); + await this.cleanupDispatch(record.workerId, record.id, record.assignment); + } else { + await this.cleanupUnassignedSlot(record.workerId, record.registration.incarnationId, record.id); + } + await new BridgeAdmissionQueue(this.redis).leave(record.workerId, record.id); + } + async dispatchWorkspaceTool(args: { workerId: string; tenantId?: string; @@ -1414,6 +1563,7 @@ export class RedisBridgeStore { leaseTokenHash: _leaseTokenHash, workerIdentityId: _workerIdentityId, workspaceFence: _workspaceFence, + durableRequestKey: _durableRequestKey, ...wireAssignment } = assignment; return { @@ -1706,7 +1856,7 @@ export class RedisBridgeStore { 'Bridge assignment has expired', ); } - const ttlSeconds = assignmentTtlSeconds(Date.parse(assignment.expiresAt)); + const ttlSeconds = assignmentOutcomeTtlSeconds(assignment); const settlementKeys = [ assignmentKey(assignmentId), settlementKey(assignmentId), @@ -1985,7 +2135,7 @@ export class RedisBridgeStore { raw.epoch, assignmentId, JSON.stringify(settlement), - assignmentTtlSeconds(Date.parse(receipt.expiresAt)), + assignmentOutcomeTtlSeconds(receipt), quarantine ? '1' : '0', ), signal, @@ -2130,14 +2280,12 @@ export class RedisBridgeStore { const cancelledMutation = signal.aborted && (assignment.executionKind === 'workspace_programmatic' || - (workspaceRequest != null && - (workspaceRequest.operation === 'write_file' || - workspaceRequest.operation === 'edit_file' || - workspaceRequest.operation === 'execute_command'))); - if (cancelledMutation) { + (workspaceRequest != null && isWorkspaceMutation(workspaceRequest))); + const cancelledDurableWorkspace = signal.aborted && assignment.durableRequestKey != null && workspaceRequest != null; + if (cancelledMutation || cancelledDurableWorkspace) { try { // Keep the acknowledged assignment available long enough for the - // worker to terminate its process tree and commit a clean rejection. + // worker to stop its operation and commit a clean rejection. // Closing it first makes that rejection impossible to acknowledge and // leaves the worker's durable mutation guard armed. await this.cancel(assignment.assignmentId, assignment); @@ -2237,17 +2385,28 @@ export class RedisBridgeStore { assignment: StoredAssignment, ttlSeconds: number, readyToken?: string, + guard?: DurableWorkspaceGuard, ): Promise { const script = [ + 'local keyCount = tonumber(ARGV[10])', + ...(guard == null ? [] : [ + ...durableAdmissionFence('KEYS[keyCount + 1]', 'KEYS[keyCount + 2]', 'ARGV[11]', -2), + ...(assignment.workspaceLeaseSlot === undefined + ? ['if redis.call(\'GET\', KEYS[keyCount + 3]) ~= ARGV[4] or redis.call(\'GET\', KEYS[4]) ~= ARGV[1] then return -2 end'] + : [ + `if redis.call('HGET', KEYS[keyCount + 3], 'a:${assignment.workspaceLeaseSlot}') ~= ARGV[4] then return -2 end`, + `if redis.call('HGET', KEYS[keyCount + 3], 'i:${assignment.workspaceLeaseSlot}') ~= ARGV[1] then return -2 end`, + ]), + ]), "if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end", 'if ARGV[7] ~= "" and redis.call(\'GET\', KEYS[6]) ~= ARGV[7] then return 0 end', - "if #KEYS >= 7 and redis.call('EXISTS', KEYS[7]) == 1 then return -1 end", + 'if keyCount >= 7 and redis.call(\'EXISTS\', KEYS[7]) == 1 then return -1 end', // A lane refuses a quarantined checkout. A checkout reaches enqueue only once no // lane beneath it holds a slot, so any lane fence still indexed here is stuck. "if ARGV[9] == 'lane' and redis.call('EXISTS', KEYS[10]) == 1 then return -1 end", "if ARGV[9] == 'checkout' then", - " for i = 11, #KEYS do if redis.call('EXISTS', KEYS[i]) == 1 then return -1 end end", - " for i = 11, #KEYS do redis.call('SREM', KEYS[10], KEYS[i]) end", + ' for i = 11, keyCount do if redis.call(\'EXISTS\', KEYS[i]) == 1 then return -1 end end', + ' for i = 11, keyCount do redis.call(\'SREM\', KEYS[10], KEYS[i]) end', 'end', 'redis.call(\'SET\', KEYS[2], ARGV[2], \"EX\", ARGV[3])', "redis.call('RPUSH', KEYS[3], ARGV[4])", @@ -2256,15 +2415,18 @@ export class RedisBridgeStore { ? ['redis.call(\'SET\', KEYS[4], ARGV[1], \"PX\", ARGV[5])'] : []), 'redis.call(\'SET\', KEYS[5], "1", \"PXAT\", ARGV[6])', - "if #KEYS >= 7 then redis.call('SET', KEYS[7], ARGV[4]) end", + 'if keyCount >= 7 then redis.call(\'SET\', KEYS[7], ARGV[4]) end', "if ARGV[9] == 'lane' then redis.call('SADD', KEYS[11], KEYS[7]) end", - 'if #KEYS >= 9 then', + 'if keyCount >= 9 then', " local epoch = redis.call('GET', KEYS[9])", - " if type(epoch) ~= 'string' then epoch = '0'; redis.call('SET', KEYS[9], epoch, 'EX', ARGV[3]) end", - " if redis.call('PTTL', KEYS[9]) < tonumber(ARGV[3]) * 1000 then redis.call('EXPIRE', KEYS[9], ARGV[3]) end", + ' if type(epoch) ~= \'string\' then epoch = \'0\'; redis.call(\'SET\', KEYS[9], epoch, \'EX\', ARGV[12]) end', + ' if redis.call(\'PTTL\', KEYS[9]) < tonumber(ARGV[12]) * 1000 then redis.call(\'EXPIRE\', KEYS[9], ARGV[12]) end', " redis.call('HSET', KEYS[8], 'metadata', ARGV[8], 'epoch', epoch)", - " redis.call('EXPIRE', KEYS[8], ARGV[3])", + ' redis.call(\'EXPIRE\', KEYS[8], ARGV[12])', 'end', + ...(guard == null ? [] : [ + 'redis.call(\'HSET\', KEYS[keyCount + 1], \'state\', \'admitted\', \'assignment\', ARGV[2], \'admittedAtMs\', admissionNowMs)', + ]), 'return 1', ].join('\n'); const keys = [ @@ -2297,6 +2459,7 @@ export class RedisBridgeStore { leaseTokenHash: assignment.leaseTokenHash, workerIdentityId: assignment.workerIdentityId, expiresAt: assignment.expiresAt, + durableRequestKey: assignment.durableRequestKey, }; let fenceScope: '' | 'lane' | 'checkout' = ''; if (assignment.workspaceLeaseSlot !== undefined) { @@ -2322,6 +2485,10 @@ export class RedisBridgeStore { fenceScope = 'checkout'; } } + const keyCount = keys.length; + if (guard != null) keys.push(guard.key, guard.claimKey, + assignment.workspaceLeaseSlot === undefined ? lockKey(assignment.workerId) + : `${workerKey(assignment.workerId)}:workspace-slots`); const result = await this.redis.eval( script, keys.length, @@ -2335,6 +2502,9 @@ export class RedisBridgeStore { readyToken ?? '', JSON.stringify(receipt), fenceScope, + keyCount, + guard?.token ?? '', + guard == null ? ttlSeconds : assignmentOutcomeTtlSeconds(assignment), ); if (Number(result) === -1) { throw new BridgeStoreError( @@ -2350,21 +2520,31 @@ export class RedisBridgeStore { assignmentId: string, incarnationId: string, ttlSeconds: number, + reuseOwned = false, + guard?: DurableWorkspaceGuard, ): Promise { const script = [ + ...(guard == null ? [] : [ + ...durableAdmissionFence('KEYS[3]', 'KEYS[4]', 'ARGV[5]', 0), + 'if redis.call(\'GET\', KEYS[5]) ~= ARGV[2] then return 0 end', + ]), + 'if ARGV[4] == \'1\' and redis.call(\'GET\', KEYS[1]) == ARGV[1] and redis.call(\'GET\', KEYS[2]) == ARGV[2] then redis.call(\'PEXPIRE\', KEYS[1], ARGV[3]); redis.call(\'PEXPIRE\', KEYS[2], ARGV[3]); return 1 end', 'if redis.call(\'EXISTS\', KEYS[1]) == 1 then return 0 end', 'redis.call(\'SET\', KEYS[1], ARGV[1], \"PX\", ARGV[3])', 'redis.call(\'SET\', KEYS[2], ARGV[2], \"PX\", ARGV[3])', 'return 1', ].join('\n'); + const keys = [lockKey(workerId), lockIncarnationKey(workerId)]; + if (guard != null) keys.push(guard.key, guard.claimKey, workerIncarnationKey(workerId)); const result = await this.redis.eval( script, - 2, - lockKey(workerId), - lockIncarnationKey(workerId), + keys.length, + ...keys, assignmentId, incarnationId, String(ttlSeconds * 1000), + reuseOwned ? '1' : '0', + guard?.token ?? '', ); return Number(result) === 1; } @@ -2412,6 +2592,7 @@ export class RedisBridgeStore { private async commitPendingWorkspace( assignment: StoredAssignment, settlement: AnyCodeBridgeSettlement, + durableKey?: string, ): Promise { if (assignment.workspaceLeaseSlot !== undefined) { const committed = Number( @@ -2450,8 +2631,17 @@ export class RedisBridgeStore { } const runtimeSessionId = assignmentWorkspace(assignment)!; const script = [ + ...(durableKey == null ? [] : [ + 'if redis.call(\'EXISTS\', KEYS[2]) == 1 then return 1 end', + ]), "if redis.call('GET', KEYS[1]) == ARGV[1] then", - " return redis.call('DEL', KEYS[1])", + ...(durableKey == null + ? [' return redis.call(\'DEL\', KEYS[1])'] + : [ + ' redis.call(\'DEL\', KEYS[1])', + ' redis.call(\'SET\', KEYS[2], \'1\', \'PX\', 86400000)', + ' return 1', + ]), 'end', 'return 0', ].join('\n'); @@ -2459,8 +2649,9 @@ export class RedisBridgeStore { await boundedCommand( this.redis.eval( script, - 1, + durableKey == null ? 1 : 2, workspaceQuarantineKey(assignment.workerId, runtimeSessionId), + ...(durableKey == null ? [] : [`${durableKey}:committed:${assignment.assignmentId}`]), assignment.assignmentId, ), // Once settlement wins, caller cancellation must not prevent its diff --git a/service/src/workspace-tools/coordination.ts b/service/src/workspace-tools/coordination.ts new file mode 100644 index 00000000..0bcd5e91 --- /dev/null +++ b/service/src/workspace-tools/coordination.ts @@ -0,0 +1,21 @@ +import { createHash } from 'crypto'; + +export interface WorkspaceRequestCoordinationPolicy { + bridgeEnabled: boolean; + backend: string; + executionProfile: string; + authMode: string; + configuredWorkerId: string; + dynamicWorkers: boolean; + maxWorkspaceLeaseSlots: number; + maxCommandTimeoutMs: number; +} + +/** Only replicas with the same bridge policy may reconcile accepted work. */ +export function workspaceRequestCoordinationScope(policy: WorkspaceRequestCoordinationPolicy): string | undefined { + if (!policy.bridgeEnabled) return undefined; + return createHash('sha256').update(JSON.stringify([ + policy.backend, policy.executionProfile, policy.authMode, policy.configuredWorkerId, + policy.dynamicWorkers, policy.maxWorkspaceLeaseSlots, policy.maxCommandTimeoutMs, + ])).digest('hex'); +} diff --git a/service/src/workspace-tools/index.ts b/service/src/workspace-tools/index.ts index 6f105f16..3c7caee2 100644 --- a/service/src/workspace-tools/index.ts +++ b/service/src/workspace-tools/index.ts @@ -4,12 +4,59 @@ import { bridgeStore } from '../bridge'; import { env } from '../config'; import { executionLimiter } from '../middleware/limits'; import { createWorkspaceToolsRouter } from './router'; +import { RedisWorkspaceRequests } from './requests'; +import { connection } from '../queue'; +import logger from '../logger'; +import { checkServiceStartUp, checkServiceShutDown } from '../lifecycle'; +import { withSpan } from '../telemetry'; +import { isBridgeEnabled } from '../bridge/enabled'; +import { workspaceRequestCoordinationScope } from './coordination'; const router = Router(); -router.use('/workspace-tools/execute', executionLimiter); +const coordinationScope = workspaceRequestCoordinationScope({ + bridgeEnabled: isBridgeEnabled(), backend: env.SANDBOX_BACKEND, + executionProfile: env.EXECUTION_PROFILE, authMode: env.BRIDGE_AUTH_MODE, + configuredWorkerId: env.BRIDGE_WORKER_ID, dynamicWorkers: env.BRIDGE_DYNAMIC_WORKERS, + maxWorkspaceLeaseSlots: env.BRIDGE_MAX_WORKSPACE_LEASE_SLOTS, maxCommandTimeoutMs: env.JOB_TIMEOUT, +}); +const requests = coordinationScope === undefined ? undefined : new RedisWorkspaceRequests(connection, bridgeStore, record => { + const attributes = { + 'codeapi.request.id': record.id, + 'codeapi.admission.state': record.state, + 'codeapi.worker.id': record.workerId, + 'codeapi.workspace.id': record.request.workspaceId, + 'codeapi.workspace.instance_id': record.request.workspaceInstanceId ?? '', + 'codeapi.workspace.worktree': record.request.worktree ?? '', + 'codeapi.admission.wait_ms': Math.max(0, (record.admittedAtMs ?? Math.min(record.finishedAtMs ?? Date.now(), record.queueDeadlineAtMs)) - record.createdAtMs), + 'codeapi.admission.error_code': record.error?.code ?? '', + }; + logger.info('Durable workspace admission transition', attributes); + void withSpan('codeapi.workspace.admission.transition', attributes, () => undefined).catch(error => { + logger.warn('Admission transition telemetry failed', { error }); + }); +}, coordinationScope); +if (requests != null) { + const coordinator = requests; + let reconciling = false; + let unavailable = false; + const reconciliation = setInterval(() => { + if (reconciling || checkServiceStartUp() || checkServiceShutDown()) return; + reconciling = true; + void coordinator.reconcile().then(() => { unavailable = false; }).catch(error => { + if (!unavailable) logger.warn('Durable workspace reconciliation unavailable', { error }); + unavailable = true; + }).finally(() => { reconciling = false; }); + }, 250); + reconciliation.unref(); +} +router.use(['/workspace-tools/execute', '/workspace-tools/requests'], (req, res, next) => { + if (req.method === 'POST') executionLimiter(req, res, next); + else next(); +}); router.use( createWorkspaceToolsRouter({ store: bridgeStore, + requests, backend: env.SANDBOX_BACKEND, configuredWorkerId: env.BRIDGE_WORKER_ID, dynamicWorkers: env.BRIDGE_DYNAMIC_WORKERS, diff --git a/service/src/workspace-tools/outcome.ts b/service/src/workspace-tools/outcome.ts index d965d515..e8dc100f 100644 --- a/service/src/workspace-tools/outcome.ts +++ b/service/src/workspace-tools/outcome.ts @@ -25,7 +25,7 @@ const earlyErrorCodes: Record = { 500: 'INTERNAL_ERROR', }; -export function getWorkspaceToolOutcome(res: Response): WorkspaceToolOutcome { +export function getWorkspaceToolOutcome(res: Response, route = '/workspace-tools/execute'): WorkspaceToolOutcome { const existing = outcomes.get(res); if (existing != null) return existing; const startedAt = performance.now(); @@ -39,7 +39,7 @@ export function getWorkspaceToolOutcome(res: Response): WorkspaceToolOutcome { outcomes.delete(res); const finished = res.writableFinished; logger.log(finished && res.statusCode < 400 ? 'info' : 'warn', 'Workspace tool request completed', { - route: '/workspace-tools/execute', + route, operation: outcome.operation, workerId: outcome.workerId, status: finished ? res.statusCode : undefined, diff --git a/service/src/workspace-tools/requests-router.test.ts b/service/src/workspace-tools/requests-router.test.ts new file mode 100644 index 00000000..856a7a38 --- /dev/null +++ b/service/src/workspace-tools/requests-router.test.ts @@ -0,0 +1,96 @@ +import { createServer } from 'node:http'; +import type { Server } from 'node:http'; +import { afterEach, expect, test } from 'bun:test'; +import express, { json } from 'express'; +import RedisMock from 'ioredis-mock'; +import type Redis from 'ioredis'; +import { applyPrincipal } from '../auth/principal'; +import { RedisBridgeStore } from '../bridge/store'; +import { createWorkspaceToolsRouter } from './router'; +import { RedisWorkspaceRequests } from './requests'; + +let server: Server | undefined; +afterEach(() => { server?.close(); server = undefined; }); +const body = { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }; +const requestId = 'http-request-000001'; + +async function setup(durable = true, rejectSynchronous = false): Promise { + const redis = new RedisMock() as unknown as Redis; + const bridge = new RedisBridgeStore(redis); + await bridge.register({ protocolVersion: 1, workerId: 'worker', incarnationId: 'incarnation-00000001', capabilities: { + sandboxProfile: 'native-srt', statefulWorkspace: false, runtimes: [], + workspaceTools: { protocolVersion: 1, operations: ['read_file'], workspaces: [{ id: 'primary' }] }, + } }); + const app = express(); + app.use(json()); + app.use((req, _res, next) => { + if (req.header('X-Test-User') !== 'unauthenticated') applyPrincipal(req, { + tenantId: 'tenant', userId: req.header('X-Test-User') ?? 'user', principalSource: 'local', codeWorkerId: 'worker', + }); + next(); + }); + app.use(createWorkspaceToolsRouter({ + store: rejectSynchronous ? { async dispatchWorkspaceTool(): Promise { throw new Error('Durable URL dispatched synchronously'); } } : bridge, requests: durable ? new RedisWorkspaceRequests(redis, bridge) : undefined, + backend: 'remote-bridge', configuredWorkerId: 'worker', dynamicWorkers: false, + })); + server = createServer(app); + await new Promise(resolve => server!.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Missing listener'); + return `http://127.0.0.1:${address.port}`; +} + +function post(url: string, id = requestId, value = body): Promise { + return fetch(`${url}/workspace-tools/requests`, { method: 'POST', headers: { + 'Content-Type': 'application/json', 'X-LibreChat-Workspace-Request-Id': id, + }, body: JSON.stringify(value) }); +} + +test.each([false, true])('capability discovery explicitly distinguishes durable support (%s)', async durable => { + const url = await setup(durable); + expect(await (await fetch(`${url}/workspace-tools/capabilities`)).json()).toEqual({ durableWorkspaceRequests: durable ? 1 : 0 }); +}); + +test('submission, duplicate lookup and cancellation use the same principal-scoped handle without exposing leases', async () => { + const url = await setup(); + const accepted = await post(url); + expect(accepted.status).toBe(202); + expect(await accepted.json()).toMatchObject({ requestId, state: 'queued', queuePosition: 1 }); + const duplicate = await post(url); + expect(duplicate.status).toBe(202); + const status = await (await fetch(`${url}/workspace-tools/requests/${requestId}`)).json(); + expect(status).toMatchObject({ requestId, state: 'queued' }); + expect(status).not.toHaveProperty('assignment'); + expect(status).not.toHaveProperty('leaseToken'); + expect((await fetch(`${url}/workspace-tools/requests/${requestId}`, { headers: { 'X-Test-User': 'another' } })).status).toBe(404); + const cancelled = await fetch(`${url}/workspace-tools/requests/${requestId}`, { method: 'DELETE' }); + expect(await cancelled.json()).toMatchObject({ requestId, cancelRequested: true }); +}); + +test('missing IDs and conflicting requests fail without dispatching', async () => { + const url = await setup(); + expect((await post(url, '')).status).toBe(400); + expect((await post(url)).status).toBe(202); + const conflict = await post(url, requestId, { ...body, path: 'different' }); + expect(conflict.status).toBe(409); + expect(await conflict.json()).toMatchObject({ code: 'REQUEST_CONFLICT' }); +}); + +test('unauthenticated request status and capability discovery are rejected', async () => { + const url = await setup(); + for (const path of ['capabilities', `requests/${requestId}`]) { + expect((await fetch(`${url}/workspace-tools/${path}`, { headers: { 'X-Test-User': 'unauthenticated' } })).status).toBe(401); + } +}); + +test.each(['/workspace-tools/requests/', '/workspace-tools/REQUESTS'])('durable route aliases remain idempotent (%s)', async path => { + const url = await setup(true, true); + for (let attempt = 0; attempt < 2; attempt++) { + const response = await fetch(`${url}${path}`, { method: 'POST', headers: { + 'Content-Type': 'application/json', 'X-LibreChat-Workspace-Request-Id': requestId, + }, body: JSON.stringify(body) }); + expect(response.status).toBe(202); + expect(await response.json()).toMatchObject({ requestId, state: 'queued', queuePosition: 1 }); + } + expect(await (await fetch(`${url}/workspace-tools/requests/${requestId}`)).json()).toMatchObject({ state: 'queued' }); +}); diff --git a/service/src/workspace-tools/requests.test.ts b/service/src/workspace-tools/requests.test.ts new file mode 100644 index 00000000..a0f7d99f --- /dev/null +++ b/service/src/workspace-tools/requests.test.ts @@ -0,0 +1,671 @@ +import { afterEach, expect, spyOn, test } from 'bun:test'; +import { createHash } from 'node:crypto'; +import RedisMock from 'ioredis-mock'; +import Redis from 'ioredis'; +import { RedisBridgeStore } from '../bridge/store'; +import { BridgeAdmissionQueue } from '../bridge/admission'; +import type { CodeBridgeAssignment } from '../bridge/store'; +import type { WorkspaceToolRequest } from '../../../packages/code/src/protocol'; +import { RedisWorkspaceRequests } from './requests'; +import { workspaceRequestCoordinationScope } from './coordination'; +import type { WorkspaceRequestCoordinationPolicy } from './coordination'; + +const testRedisUrl = process.env.BRIDGE_TEST_REDIS_URL; +const redis = testRedisUrl !== undefined && testRedisUrl.length > 0 + ? new Redis(testRedisUrl) + : new RedisMock() as unknown as Redis; +const bridge = new RedisBridgeStore(redis, 600, 1000, 2); +const owner = { tenantId: 'tenant', userId: 'user' }; +const workerId = 'durable-worker'; +const incarnationId = 'incarnation-00000001'; +const requestId = 'request-00000000001'; +const activeKey = 'codeapi:workspace-requests:v1:active'; +const queueKey = `codeapi:bridge:v1:worker:${workerId}:admission`; + +// Only a disposable test Redis may be supplied. +afterEach(async () => { await redis.flushall(); }); + +async function setup(slots = 1): Promise { + const generation = await bridge.register({ protocolVersion: 1, workerId, incarnationId, + capabilities: { + sandboxProfile: 'native-srt', statefulWorkspace: false, runtimes: ['bash'], + ...(slots > 1 ? { workspaceLeaseSlots: slots, requiresReadyConfirmation: true } : {}), + workspaceTools: { protocolVersion: 1, operations: ['read_file', 'execute_command', 'list_files', 'search_text', 'preview_edit'], + workspaces: [{ id: 'primary' }, { id: 'independent' }] }, + }, + }); + if (slots > 1) await bridge.confirmReady(workerId, incarnationId, generation); + return new RedisWorkspaceRequests(redis, bridge); +} + +function submit(requests: RedisWorkspaceRequests, id = requestId, workspaceId = 'primary'): ReturnType { + return requests.submit({ owner, requestId: id, workerId, requireTenantBinding: false, + request: { protocolVersion: 1, operation: 'read_file', workspaceId, path: 'README.md' }, + queueWaitMs: 300_000, executionTimeoutMs: 30_000, + }); +} + +async function tick(requests: RedisWorkspaceRequests): Promise { + const keys = await redis.zrange(activeKey, 0, -1); + for (const key of keys) await redis.zadd(activeKey, 0, key); + await requests.reconcile(); +} + +async function settle(assignment: CodeBridgeAssignment, fulfilled = true): Promise { + await bridge.acknowledgeLease(workerId, incarnationId, assignment.assignmentId, + assignment.generation, assignment.leaseToken); + await bridge.settle(workerId, assignment.assignmentId, { + protocolVersion: 1, incarnationId, generation: assignment.generation, + leaseToken: assignment.leaseToken, + ...(fulfilled ? { status: 'fulfilled' as const, result: { protocolVersion: 1, operation: 'read_file', workspaceId: (assignment.request as { workspaceId: string }).workspaceId, path: 'README.md', content: 'hello', startLine: 1, endLine: 1, truncated: false } } + : { status: 'rejected' as const, error: 'stopped' }), + }); +} + +async function mutateRecord(update: (record: Record) => void): Promise { + const key = (await redis.zrange(activeKey, 0, -1))[0]; + const record = JSON.parse((await redis.hget(key, 'record'))!); + update(record); + await redis.hset(key, 'record', JSON.stringify(record)); +} + +test('submission is idempotent across replicas, scoped by principal, and conflicts do not move FIFO position', async () => { + const requests = await setup(); + await submit(requests); + await submit(requests, 'request-00000000002'); + const replica = new RedisWorkspaceRequests(redis, bridge); + expect((await submit(replica)).queuePosition).toBe(1); + expect(await redis.zcard(queueKey)).toBe(2); + await expect(submit(replica, requestId, 'independent')).rejects.toThrow('different work'); + expect(await replica.get({ ...owner, userId: 'someone-else' }, requestId)).toBeUndefined(); + expect(await replica.cancel({ ...owner, tenantId: 'someone-else' }, requestId)).toBeUndefined(); +}); + +test('restart before dispatch preserves the queue and starts a full execution budget after sixty seconds waiting', async () => { + const requests = await setup(); + await submit(requests); + const key = (await redis.zrange(activeKey, 0, -1))[0]; + await mutateRecord(record => { + record.createdAtMs = Number(record.createdAtMs) - 60_000; + record.queueDeadlineAtMs = Number(record.queueDeadlineAtMs) - 60_000; + }); + const record = JSON.parse((await redis.hget(key, 'record'))!); + await redis.hset(key, 'queueDeadlineAtMs', record.queueDeadlineAtMs); + const restarted = new RedisWorkspaceRequests(redis, new RedisBridgeStore(redis)); + await tick(restarted); + const assignment = await bridge.lease(workerId, incarnationId, 0); + expect(assignment).toBeDefined(); + expect(assignment!.remainingMs).toBeGreaterThan(29_000); + // Mock Redis TIME has subsecond clock skew. Execution still receives its full budget. + expect((await restarted.get(owner, requestId))?.queueWaitMs).toBeGreaterThan(59_000); + await settle(assignment!); + await tick(restarted); + expect(await restarted.get(owner, requestId)).toMatchObject({ state: 'completed', result: { content: 'hello' } }); +}); + +test.each([false, true])('restart around admission acknowledgement never enqueues another assignment (ack=%s)', async acknowledged => { + const requests = await setup(); + await submit(requests); + await tick(requests); + const assignment = await bridge.lease(workerId, incarnationId, 0); + if (acknowledged) await bridge.acknowledgeLease(workerId, incarnationId, assignment!.assignmentId, + assignment!.generation, assignment!.leaseToken); + const restarted = new RedisWorkspaceRequests(redis, new RedisBridgeStore(redis)); + await tick(restarted); + await submit(restarted); + expect((await bridge.lease(workerId, incarnationId, 0))?.assignmentId).toBe(assignment!.assignmentId); + expect(await redis.llen(`codeapi:bridge:v1:worker:${workerId}:incarnation:${incarnationId}:assignments`)).toBe(0); + await settle(assignment!); + await tick(restarted); + expect((await restarted.get(owner, requestId))?.state).toBe('completed'); +}); + +test('a lost enqueue acknowledgement is reconciled rather than replayed', async () => { + const requests = await setup(); + await submit(requests); + const original = redis.eval.bind(redis); + const failure = spyOn(redis, 'eval').mockImplementation((async (...args: Parameters) => { + const result = await original(...args); + if (String(args[0]).includes('\'state\', \'admitted\', \'assignment\'')) throw new Error('lost acknowledgement'); + return result; + }) as Redis['eval']); + try { await expect(tick(requests)).rejects.toThrow('lost acknowledgement'); } finally { failure.mockRestore(); } + const assignment = await bridge.lease(workerId, incarnationId, 0); + expect(assignment).toBeDefined(); + await tick(new RedisWorkspaceRequests(redis, bridge)); + expect((await requests.get(owner, requestId))?.state).toBe('admitted'); + await settle(assignment!); + await tick(requests); + expect((await requests.get(owner, requestId))?.state).toBe('completed'); +}); + +test('cancel before admission is durable and does not dispatch', async () => { + const requests = await setup(); + await submit(requests); + await requests.cancel(owner, requestId); + await tick(new RedisWorkspaceRequests(redis, bridge)); + expect((await requests.get(owner, requestId))?.state).toBe('cancelled'); + expect(await bridge.lease(workerId, incarnationId, 0)).toBeUndefined(); + expect(await redis.zcard(queueKey)).toBe(0); +}); + +test('cancel racing a persisted fulfillment returns the winning result without consuming it', async () => { + const requests = await setup(); + await submit(requests); await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0))!; + await settle(assignment); + await requests.cancel(owner, requestId); + await tick(new RedisWorkspaceRequests(redis, bridge)); + const statuses = await Promise.all([requests.get(owner, requestId), requests.get(owner, requestId), requests.cancel(owner, requestId)]); + for (const status of statuses) expect(status).toMatchObject({ state: 'completed', result: { content: 'hello' } }); + expect(await bridge.lease(workerId, incarnationId, 0)).toBeUndefined(); +}); + +test('restart after result persistence repeats only idempotent cleanup', async () => { + const requests = await setup(); + await submit(requests); await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0))!; + await settle(assignment); + const cleanup = spyOn(bridge, 'finishDurableWorkspaceTool').mockRejectedValue(new Error('process stopped')); + try { await expect(tick(requests)).rejects.toThrow('process stopped'); } finally { cleanup.mockRestore(); } + expect((await requests.get(owner, requestId))?.state).toBe('completed'); + await tick(new RedisWorkspaceRequests(redis, bridge)); + expect(await redis.zcard(activeKey)).toBe(0); + await submit(requests); + expect(await bridge.lease(workerId, incarnationId, 0)).toBeUndefined(); +}); + +test('pending work rejects a replacement worker incarnation and never transfers to it', async () => { + const requests = await setup(); + await submit(requests); + await bridge.register({ protocolVersion: 1, workerId, incarnationId: 'replacement-00001', + capabilities: { sandboxProfile: 'native-srt', statefulWorkspace: false, runtimes: [], + workspaceTools: { protocolVersion: 1, operations: ['read_file'], workspaces: [{ id: 'primary' }] } }, + }); + await tick(requests); + expect(await requests.get(owner, requestId)).toMatchObject({ state: 'failed', error: { code: 'WORKER_FENCED' } }); +}); + +test('durable requests preserve negotiated independent-root concurrency', async () => { + const requests = await setup(2); + await submit(requests); await submit(requests, 'request-00000000002', 'independent'); + await tick(requests); await tick(requests); + const first = await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0); + const second = await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 1); + expect(first).toBeDefined(); expect(second).toBeDefined(); + await settle(first!); await settle(second!); await tick(requests); + expect((await requests.get(owner, requestId))?.state).toBe('completed'); + expect((await requests.get(owner, 'request-00000000002'))?.state).toBe('completed'); +}); + +test.each([1, 2])('restart after capacity reservation refreshes its own reservation without duplicating work (slots=%s)', async slots => { + const requests = await setup(slots); + await submit(requests); + const generation = spyOn(redis, 'incr').mockRejectedValue(new Error('process stopped before enqueue')); + try { await expect(tick(requests)).rejects.toThrow('process stopped before enqueue'); } finally { generation.mockRestore(); } + expect((await requests.get(owner, requestId))?.state).toBe('queued'); + await redis.pexpire(`codeapi:bridge:v1:worker:${workerId}:lock`, 5000); + await tick(new RedisWorkspaceRequests(redis, bridge)); + const assignment = await bridge.lease(workerId, incarnationId, 0, undefined, undefined, slots === 1 ? undefined : 0); + expect(assignment).toBeDefined(); + expect(await redis.pttl(`codeapi:bridge:v1:worker:${workerId}:lock`)).toBeGreaterThan(30_000); +}); + +test.each([1, 2])('a stale coordinator cannot acquire capacity or dispatch (slots=%s)', async slots => { + const requests = await setup(slots); await submit(requests); + const key = (await redis.zrange(activeKey, 0, -1))[0]; + const record = JSON.parse((await redis.hget(key, 'record'))!); + await redis.set(`${key}:claim`, 'new-owner', 'PX', 10000); + await bridge.advanceDurableWorkspaceTool(record, { key, claimKey: `${key}:claim`, token: 'old-owner' }); + expect(await bridge.lease(workerId, incarnationId, 0)).toBeUndefined(); + expect((await requests.get(owner, requestId))?.state).toBe('queued'); + expect(await redis.get(`codeapi:bridge:v1:worker:${workerId}:lock`)).toBeNull(); + expect(await redis.hlen(`codeapi:bridge:v1:worker:${workerId}:workspace-slots`)).toBe(0); +}); + +test('a queued deadline expires as definitely unstarted work', async () => { + const requests = await setup(); await submit(requests); + const key = (await redis.zrange(activeKey, 0, -1))[0]; + await mutateRecord(record => { record.queueDeadlineAtMs = Date.now() - 1; }); + await redis.hset(key, 'queueDeadlineAtMs', Date.now() - 1); + await tick(requests); + expect(await requests.get(owner, requestId)).toMatchObject({ state: 'failed', error: { code: 'WORKSPACE_QUEUE_TIMEOUT' } }); + expect(await bridge.lease(workerId, incarnationId, 0)).toBeUndefined(); +}); + +test('an expired acknowledged command is unknown, remains fenced, and is not re-enqueued on restart', async () => { + const requests = await setup(); await submit(requests); await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0))!; + await bridge.acknowledgeLease(workerId, incarnationId, assignment.assignmentId, assignment.generation, assignment.leaseToken); + const key = (await redis.zrange(activeKey, 0, -1))[0]; + const stored = JSON.parse((await redis.hget(key, 'assignment'))!); + stored.expiresAt = new Date(Date.now() - 1).toISOString(); + await redis.hset(key, 'assignment', JSON.stringify(stored)); + await tick(new RedisWorkspaceRequests(redis, bridge)); + expect(await requests.get(owner, requestId)).toMatchObject({ state: 'failed', error: { code: 'ASSIGNMENT_EXPIRED' } }); + await submit(requests); + expect(await redis.llen(`codeapi:bridge:v1:worker:${workerId}:incarnation:${incarnationId}:assignments`)).toBe(0); + expect(await redis.get(`codeapi:bridge:v1:worker:${workerId}:workspace:${createHash('sha256').update('native-workspace:primary').digest('hex')}:quarantined`)).toBe(assignment.assignmentId); +}); + +test('cancel during command execution waits for clean rejection and does not replay', async () => { + const requests = await setup(); + await requests.submit({ owner, requestId, workerId, requireTenantBinding: false, + request: { protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'sleep 30' }, + queueWaitMs: 300_000, executionTimeoutMs: 35_000, + }); + await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0))!; + await bridge.acknowledgeLease(workerId, incarnationId, assignment.assignmentId, assignment.generation, assignment.leaseToken); + await requests.cancel(owner, requestId); + const cancelling = tick(requests); + const deadline = Date.now() + 1000; + while (!await bridge.cancelled(workerId, incarnationId, assignment.assignmentId)) { + if (Date.now() >= deadline) throw new Error('Worker never received cancellation'); + await new Promise(resolve => setTimeout(resolve, 5)); + } + await settle(assignment, false); + await cancelling; + expect((await requests.get(owner, requestId))?.state).toBe('cancelled'); +}); + +test('restart between workspace commit and cleanup preserves the result and releases the same assignment', async () => { + const requests = await setup(); await submit(requests); await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0))!; await settle(assignment); + const original = redis.eval.bind(redis); + let failed = false; + const failure = spyOn(redis, 'eval').mockImplementation((async (...args: Parameters) => { + if (!failed && String(args[0]).includes('local queued = redis.call(\'LREM\'')) { + failed = true; throw new Error('process stopped after commit'); + } + return original(...args); + }) as Redis['eval']); + try { await expect(tick(requests)).rejects.toThrow('process stopped after commit'); } finally { failure.mockRestore(); } + await tick(new RedisWorkspaceRequests(redis, bridge)); + expect((await requests.get(owner, requestId))?.state).toBe('completed'); + expect(await redis.zcard(activeKey)).toBe(0); +}); + +test('simultaneous submissions from replicas keep one request identity and one FIFO entry', async () => { + const requests = await setup(); + const replica = new RedisWorkspaceRequests(redis, bridge); + const statuses = await Promise.all([submit(requests), submit(replica)]); + for (const status of statuses) expect(status).toMatchObject({ requestId, state: 'queued', queuePosition: 1 }); + expect(await redis.zcard(queueKey)).toBe(1); + expect(await redis.zcard(activeKey)).toBe(1); + await tick(requests); + expect(await bridge.lease(workerId, incarnationId, 0)).toBeDefined(); +}); + +test.each([false, true])('temporary registration loss preserves FIFO until the original worker reconnects (replacement=%s)', async replacement => { + const requests = await setup(); + await submit(requests); await submit(requests, 'request-00000000002'); + // Registration/readiness TTL expiry, without losing accepted work. + const worker = `codeapi:bridge:v1:worker:${workerId}`; + await redis.del(worker, `${worker}:incarnation`, `${worker}:ready`); + const restarted = new RedisWorkspaceRequests(redis, bridge); + await tick(restarted); + expect(await restarted.get(owner, requestId)).toMatchObject({ state: 'queued', queuePosition: 1 }); + expect(await redis.zcard(activeKey)).toBe(2); + if (replacement) { + await bridge.register({ protocolVersion: 1, workerId, incarnationId: 'replacement-00001', + capabilities: { sandboxProfile: 'native-srt', statefulWorkspace: false, runtimes: [], + workspaceTools: { protocolVersion: 1, operations: ['read_file'], workspaces: [{ id: 'primary' }] } }, + }); + await tick(restarted); + expect(await restarted.get(owner, requestId)).toMatchObject({ state: 'failed', error: { code: 'WORKER_FENCED' } }); + expect(await bridge.lease(workerId, 'replacement-00001', 0)).toBeUndefined(); + } else { + await setup(); await tick(restarted); + const assignment = (await bridge.lease(workerId, incarnationId, 0))!; + expect(assignment).toBeDefined(); + await settle(assignment); await tick(restarted); + expect((await restarted.get(owner, requestId))?.state).toBe('completed'); + } +}); + +test('an offline worker cannot extend the queue deadline', async () => { + const requests = await setup(); await submit(requests); + const worker = `codeapi:bridge:v1:worker:${workerId}`; + await redis.del(worker, `${worker}:incarnation`, `${worker}:ready`); + await tick(requests); + expect((await requests.get(owner, requestId))?.state).toBe('queued'); + await mutateRecord(record => { record.queueDeadlineAtMs = Date.now() - 1; }); + await tick(requests); + expect(await requests.get(owner, requestId)).toMatchObject({ state: 'failed', error: { code: 'WORKSPACE_QUEUE_TIMEOUT' } }); + expect(await redis.zcard(activeKey)).toBe(0); +}); + +test('durable reset epochs outlive their receipts and reject late quarantine after reset', async () => { + const requests = await setup(2); await submit(requests); await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0))!; + await settle(assignment); await tick(requests); + const receipt = `codeapi:bridge:v1:assignment:${assignment.assignmentId}:workspace-fence-owner`; + const fence = `codeapi:bridge:v1:worker:${workerId}:workspace:${createHash('sha256').update('native-workspace:primary').digest('hex')}:quarantined`; + expect(await redis.pttl(`${fence}:epoch`)).toBeGreaterThanOrEqual(await redis.pttl(receipt) - 50); + expect(await redis.pttl(`${fence}:epoch`)).toBeGreaterThan(86_000_000); + // Simulate execution-ownership TTL expiry. The durable receipt and epoch survive. + await redis.del(`codeapi:bridge:v1:worker:${workerId}:workspace-slots`, + `codeapi:bridge:v1:worker:${workerId}:lock`, `codeapi:bridge:v1:worker:${workerId}:lock:incarnation`); + await bridge.resetWorkspace(workerId, incarnationId, 'native-workspace:primary'); + await submit(requests, 'request-00000000002'); await tick(requests); + const next = (await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0))!; + await expect(bridge.settle(workerId, assignment.assignmentId, { + protocolVersion: 1, incarnationId, generation: assignment.generation, leaseToken: assignment.leaseToken, + status: 'rejected', error: 'delayed local cleanup failure', + }, undefined, undefined, true)).rejects.toMatchObject({ code: 'ASSIGNMENT_FENCED' }); + expect(await redis.get(fence)).toBe(next.assignmentId); + await settle(next); await tick(requests); + expect((await requests.get(owner, 'request-00000000002'))?.state).toBe('completed'); +}); + +const coordinationPolicy: WorkspaceRequestCoordinationPolicy = { + bridgeEnabled: true, backend: 'remote-bridge', executionProfile: 'default', authMode: 'paired', + configuredWorkerId: '', dynamicWorkers: true, maxWorkspaceLeaseSlots: 2, maxCommandTimeoutMs: 30_000, +}; + +test('disabled bridges do not participate and each scheduling policy has an isolated scope', () => { + expect(workspaceRequestCoordinationScope({ ...coordinationPolicy, bridgeEnabled: false })).toBeUndefined(); + const scope = workspaceRequestCoordinationScope(coordinationPolicy); + expect(workspaceRequestCoordinationScope({ ...coordinationPolicy })).toBe(scope); + const variants: Partial[] = [ + { maxWorkspaceLeaseSlots: 1 }, { backend: 'http' }, { executionProfile: 'stateful' }, + { authMode: 'static' }, { configuredWorkerId: 'another' }, { dynamicWorkers: false }, + { maxCommandTimeoutMs: 60_000 }, + ]; + for (const variant of variants) expect(workspaceRequestCoordinationScope({ ...coordinationPolicy, ...variant })).not.toBe(scope); +}); + +test('a different policy cannot claim or fail work accepted by a two-slot bridge API', async () => { + await setup(2); + const scope = workspaceRequestCoordinationScope(coordinationPolicy)!; + const requests = new RedisWorkspaceRequests(redis, bridge, undefined, scope); + await submit(requests); + const foreignScope = workspaceRequestCoordinationScope({ ...coordinationPolicy, maxWorkspaceLeaseSlots: 1 })!; + const foreign = new RedisWorkspaceRequests(redis, new RedisBridgeStore(redis), undefined, foreignScope); + await foreign.reconcile(); + expect((await requests.get(owner, requestId))?.state).toBe('queued'); + expect(await foreign.get(owner, requestId)).toBeUndefined(); + const restarted = new RedisWorkspaceRequests(redis, new RedisBridgeStore(redis, 600, 1000, 2), undefined, scope); + await restarted.reconcile(); + const assignment = await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0); + expect(assignment).toBeDefined(); +}); + +test('a late FIFO response cannot relock a cancelled request after the coordinator timed out', async () => { + const requests = await setup(); await submit(requests); + let release!: () => void; + let entered!: () => void; + let finished!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const started = new Promise(resolve => { entered = resolve; }); + const drained = new Promise(resolve => { finished = resolve; }); + const originalHead = BridgeAdmissionQueue.prototype.isHead; + const originalAdvance = bridge.advanceDurableWorkspaceTool.bind(bridge); + const head = spyOn(BridgeAdmissionQueue.prototype, 'isHead').mockImplementation(async (worker, id) => { + const result = await originalHead.call(new BridgeAdmissionQueue(redis), worker, id); + entered(); await gate; return result; + }); + const advance = spyOn(bridge, 'advanceDurableWorkspaceTool').mockImplementation(async (...args) => { + try { return await originalAdvance(...args); } finally { finished(); } + }); + const pending = tick(requests); + void pending.catch(() => undefined); + try { + await started; + await expect(pending).rejects.toThrow('Durable workspace transition timed out'); + head.mockRestore(); advance.mockRestore(); + const key = (await redis.zrange(activeKey, 0, -1))[0]; + const claimDeadline = Date.now() + 1000; + while (await redis.get(`${key}:claim`) != null) { + if (Date.now() >= claimDeadline) throw new Error('Timed-out coordinator did not release its claim'); + await new Promise(resolve => setTimeout(resolve, 5)); + } + await requests.cancel(owner, requestId); + await tick(new RedisWorkspaceRequests(redis, bridge)); + expect((await requests.get(owner, requestId))?.state).toBe('cancelled'); + expect(await redis.zcard(activeKey)).toBe(0); + release(); await drained; + expect(await redis.get(`codeapi:bridge:v1:worker:${workerId}:lock`)).toBeNull(); + await submit(requests, 'request-00000000002'); await tick(requests); + expect(await bridge.lease(workerId, incarnationId, 0)).toBeDefined(); + } finally { release(); head.mockRestore(); advance.mockRestore(); await drained; } +}, 15_000); + +test.each([1, 2])('Redis capacity writes delayed past cancellation cannot strand a reservation (slots=%s)', async slots => { + const requests = await setup(slots); await submit(requests); + let release!: () => void; + let entered!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const started = new Promise(resolve => { entered = resolve; }); + const original = redis.eval.bind(redis); + let held = false; + const delay = spyOn(redis, 'eval').mockImplementation((async (...args: Parameters) => { + const script = String(args[0]); + if (!held && script.includes('local admissionTime') && !script.includes('local keyCount')) { + held = true; entered(); await gate; + } + return original(...args); + }) as Redis['eval']); + const pending = tick(requests); + void pending.catch(() => undefined); + try { + await started; + const key = (await redis.zrange(activeKey, 0, -1))[0]; + // Expire this claim while its Redis command is still undelivered. + await redis.del(`${key}:claim`); + await requests.cancel(owner, requestId); + await tick(new RedisWorkspaceRequests(redis, bridge)); + expect((await requests.get(owner, requestId))?.state).toBe('cancelled'); + release(); await pending; + expect(await redis.get(`codeapi:bridge:v1:worker:${workerId}:lock`)).toBeNull(); + expect(await redis.hlen(`codeapi:bridge:v1:worker:${workerId}:workspace-slots`)).toBe(0); + expect(await redis.zcard(activeKey)).toBe(0); + } finally { release(); delay.mockRestore(); await pending.catch(() => undefined); } +}); + +test.each([1, 2])('a durable enqueue refuses lost capacity ownership (slots=%s)', async slots => { + const requests = await setup(slots); await submit(requests); + const original = redis.eval.bind(redis); + const loss = spyOn(redis, 'eval').mockImplementation((async (...args: Parameters) => { + if (String(args[0]).includes('local keyCount')) { + await redis.del(`codeapi:bridge:v1:worker:${workerId}:lock`, + `codeapi:bridge:v1:worker:${workerId}:workspace-slots`); + } + return original(...args); + }) as Redis['eval']); + try { await tick(requests); } finally { loss.mockRestore(); } + expect((await requests.get(owner, requestId))?.state).toBe('queued'); + expect(await bridge.lease(workerId, incarnationId, 0, undefined, undefined, slots === 1 ? undefined : 0)).toBeUndefined(); + await tick(requests); + expect(await bridge.lease(workerId, incarnationId, 0, undefined, undefined, slots === 1 ? undefined : 0)).toBeDefined(); +}); + +test.each([1, 2])('capacity acquisition rejects terminal, cancelled, expired, and unclaimed requests (slots=%s)', async slots => { + for (const denial of ['terminal', 'cancelled', 'expired', 'unclaimed']) { + await redis.flushall(); + const requests = await setup(slots); await submit(requests); + const key = (await redis.zrange(activeKey, 0, -1))[0]; + const record = JSON.parse((await redis.hget(key, 'record'))!); + const token = 'current-coordinator'; + await redis.set(`${key}:claim`, token, 'PX', 10000); + if (denial === 'terminal') await redis.hset(key, 'state', 'cancelled'); + if (denial === 'cancelled') await redis.hset(key, 'cancelRequested', '1'); + if (denial === 'expired') await redis.hset(key, 'queueDeadlineAtMs', Date.now() - 2000); + if (denial === 'unclaimed') await redis.del(`${key}:claim`); + await bridge.advanceDurableWorkspaceTool(record, { key, claimKey: `${key}:claim`, token }); + expect(await redis.get(`codeapi:bridge:v1:worker:${workerId}:lock`)).toBeNull(); + expect(await redis.hlen(`codeapi:bridge:v1:worker:${workerId}:workspace-slots`)).toBe(0); + expect(await bridge.lease(workerId, incarnationId, 0, undefined, undefined, slots === 1 ? undefined : 0)).toBeUndefined(); + } +}); + +test.each([false, true])('quarantine retains the winning durable outcome across two-minute recovery (settled=%s)', async settled => { + const requests = await setup(2); + await requests.submit({ owner, requestId, workerId, requireTenantBinding: false, + request: { protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, + queueWaitMs: 300_000, executionTimeoutMs: 30_000, + }); + await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0))!; + await bridge.acknowledgeLease(workerId, incarnationId, assignment.assignmentId, assignment.generation, assignment.leaseToken); + const firstError = 'Command stopped before local cleanup'; + const quarantineError = 'Workspace cleanup failed before settlement'; + const envelope = { protocolVersion: 1 as const, incarnationId, + generation: assignment.generation, leaseToken: assignment.leaseToken, + status: 'rejected' as const, + }; + if (settled) await bridge.settle(workerId, assignment.assignmentId, { ...envelope, error: firstError }); + await bridge.settle(workerId, assignment.assignmentId, { ...envelope, error: quarantineError }, undefined, undefined, true); + const resultKey = `codeapi:bridge:v1:assignment:${assignment.assignmentId}:settlement`; + const receiptKey = `codeapi:bridge:v1:assignment:${assignment.assignmentId}:workspace-fence-owner`; + const receipt = JSON.parse((await redis.hget(receiptKey, 'metadata'))!); + expect(receipt.durableRequestKey).toBe((await redis.zrange(activeKey, 0, -1))[0]); + const remaining = await redis.pttl(resultKey); + expect(remaining).toBeGreaterThan(86_000_000); + expect(await redis.get(`codeapi:bridge:v1:assignment:${assignment.assignmentId}`)).toBeNull(); + // Advance retained storage and the coordinator clock without a two-minute sleep. + await redis.pexpire(resultKey, Math.max(0, remaining - 120_000)); + const clock = spyOn(Date, 'now').mockReturnValue(Date.now() + 120_000); + const restarted = new RedisWorkspaceRequests(redis, new RedisBridgeStore(redis, 600, 1000, 2)); + try { + await tick(restarted); + expect(await restarted.get(owner, requestId)).toMatchObject({ + state: 'failed', error: { code: 'WORKSPACE_TOOL_REJECTED', message: settled ? firstError : quarantineError }, + }); + expect(await redis.pttl(resultKey)).toBeGreaterThan(85_000_000); + expect(await redis.zcard(activeKey)).toBe(0); + await submit(restarted, 'request-00000000002'); await tick(restarted); + expect(await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0)).toBeUndefined(); + expect(await restarted.get(owner, 'request-00000000002')).toMatchObject({ + state: 'failed', error: { code: 'WORKSPACE_QUARANTINED' }, + }); + } finally { clock.mockRestore(); } +}); + +test('queued durable requests retain FIFO workspace metadata across same-incarnation slot changes', async () => { + const requests = await setup(); + await submit(requests); + await submit(requests, 'request-00000000002'); + await submit(requests, 'request-00000000003', 'independent'); + await submit(requests); + await setup(2); + await tick(requests); await tick(requests); + const first = (await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0))!; + const independent = (await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 1))!; + expect(first).toBeDefined(); expect(independent).toBeDefined(); + expect(first.request).toMatchObject({ workspaceId: 'primary' }); + expect(independent.request).toMatchObject({ workspaceId: 'independent' }); + expect(await requests.get(owner, 'request-00000000002')).toMatchObject({ state: 'queued', queuePosition: 2 }); + await settle(first); await settle(independent); await tick(requests); + await bridge.confirmWorkspaceCleanup(workerId, first.assignmentId, { + protocolVersion: 1, incarnationId, generation: first.generation, leaseToken: first.leaseToken, + status: 'rejected', error: 'local cleanup confirmed', + }); + await tick(requests); + const next = (await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0))!; + expect(next).toBeDefined(); + await settle(next); await tick(requests); + expect((await requests.get(owner, 'request-00000000002'))?.state).toBe('completed'); +}); + +const readOnlyRequests: WorkspaceToolRequest[] = [ + { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, + { protocolVersion: 1, operation: 'list_files', workspaceId: 'primary' }, + { protocolVersion: 1, operation: 'search_text', workspaceId: 'primary', query: 'needle' }, + { protocolVersion: 1, operation: 'preview_edit', workspaceId: 'primary', path: 'README.md', oldText: 'old', newText: 'new' }, +]; +const readCancellationCases = readOnlyRequests.flatMap(request => [1, 2].flatMap(slots => + ['unleased', 'claimed', 'acknowledged'].map(phase => ({ request, slots, phase })), +)); + +test.each(readCancellationCases)('durable read cancellation notifies the worker before closing (%j)', async ({ request, slots, phase }) => { + const requests = await setup(slots); + await requests.submit({ owner, requestId, workerId, requireTenantBinding: false, + request, queueWaitMs: 300_000, executionTimeoutMs: 30_000, + }); + await tick(requests); + const slot = slots === 1 ? undefined : 0; + let assignment = phase === 'unleased' ? undefined : await bridge.lease(workerId, incarnationId, 0, undefined, undefined, slot); + if (phase === 'acknowledged') await bridge.acknowledgeLease(workerId, incarnationId, assignment!.assignmentId, + assignment!.generation, assignment!.leaseToken); + const key = (await redis.zrange(activeKey, 0, -1))[0]; + const stored = JSON.parse((await redis.hget(key, 'assignment'))!); + await requests.cancel(owner, requestId); + const cancelling = tick(new RedisWorkspaceRequests(redis, bridge)); + const marker = `codeapi:bridge:v1:assignment:${stored.assignmentId}:cancelled`; + const limit = Date.now() + 1000; + try { + while (await redis.get(marker) !== '1') { + if (Date.now() >= limit) throw new Error('Read-only worker was never notified'); + await new Promise(resolve => setTimeout(resolve, 5)); + } + expect((await requests.get(owner, requestId))?.state).toBe('admitted'); + assignment ??= await bridge.lease(workerId, incarnationId, 0, undefined, undefined, slot); + expect(assignment).toBeDefined(); + expect(await bridge.cancelled(workerId, incarnationId, assignment!.assignmentId)).toBe(true); + await settle(assignment!, false); + await cancelling; + expect(await requests.get(owner, requestId)).toMatchObject({ state: 'cancelled', cancelRequested: true }); + expect(await redis.zcard(activeKey)).toBe(0); + } finally { await cancelling; } +}); + +test.each([1, 2])('a durable read cancellation remains cancelled without worker settlement (slots=%s)', async slots => { + const requests = await setup(slots); await submit(requests); await tick(requests); + const key = (await redis.zrange(activeKey, 0, -1))[0]; + const stored = JSON.parse((await redis.hget(key, 'assignment'))!); + await requests.cancel(owner, requestId); + await tick(new RedisWorkspaceRequests(redis, bridge)); + expect(await redis.get(`codeapi:bridge:v1:assignment:${stored.assignmentId}:cancelled`)).toBe('1'); + expect(await requests.get(owner, requestId)).toMatchObject({ state: 'cancelled', cancelRequested: true }); + expect((await requests.get(owner, requestId))?.error).toBeUndefined(); + expect(await bridge.lease(workerId, incarnationId, 0, undefined, undefined, slots === 1 ? undefined : 0)).toBeUndefined(); + expect(await redis.zcard(activeKey)).toBe(0); + await submit(requests, 'request-00000000002'); await tick(requests); + expect(await bridge.lease(workerId, incarnationId, 0, undefined, undefined, slots === 1 ? undefined : 0)).toBeDefined(); +}, 10_000); + +test('fulfillment racing durable read cancellation keeps the winning result', async () => { + const requests = await setup(); await submit(requests); await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0))!; + await requests.cancel(owner, requestId); + const cancelling = tick(requests); + const marker = `codeapi:bridge:v1:assignment:${assignment.assignmentId}:cancelled`; + const limit = Date.now() + 1000; + try { + while (await redis.get(marker) !== '1') { + if (Date.now() >= limit) throw new Error('Read-only worker was never notified'); + await new Promise(resolve => setTimeout(resolve, 5)); + } + await settle(assignment); + await cancelling; + expect(await requests.get(owner, requestId)).toMatchObject({ state: 'completed', result: { content: 'hello' } }); + } finally { await cancelling; } +}); + +test('queued workspace metadata also permits a same-incarnation return to serial admission', async () => { + const requests = await setup(2); await submit(requests); await setup(); await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0))!; + expect(assignment).toBeDefined(); + expect(assignment.workspaceLeaseSlot).toBeUndefined(); + await settle(assignment); await tick(requests); + expect((await requests.get(owner, requestId))?.state).toBe('completed'); +}); + +test('unconfirmed durable mutation cancellation still reports an unknown outcome and retains its fence', async () => { + const requests = await setup(); + await requests.submit({ owner, requestId, workerId, requireTenantBinding: false, + request: { protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'sleep 30' }, + queueWaitMs: 300_000, executionTimeoutMs: 35_000, + }); + await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0))!; + await bridge.acknowledgeLease(workerId, incarnationId, assignment.assignmentId, assignment.generation, assignment.leaseToken); + await requests.cancel(owner, requestId); await tick(requests); + expect(await requests.get(owner, requestId)).toMatchObject({ + state: 'failed', cancelRequested: true, error: { code: 'ASSIGNMENT_EXPIRED' }, + }); + const fence = `codeapi:bridge:v1:worker:${workerId}:workspace:${createHash('sha256').update('native-workspace:primary').digest('hex')}:quarantined`; + expect(await redis.get(fence)).toBe(assignment.assignmentId); +}, 10_000); diff --git a/service/src/workspace-tools/requests.ts b/service/src/workspace-tools/requests.ts new file mode 100644 index 00000000..2988c7cc --- /dev/null +++ b/service/src/workspace-tools/requests.ts @@ -0,0 +1,251 @@ +import { createHash, randomBytes } from 'crypto'; +import type Redis from 'ioredis'; +import type { WorkspaceToolRequest, WorkspaceToolResult } from '../../../packages/code/src/protocol'; +import type { RedisBridgeStore, RegisteredBridgeWorker, DurableWorkspaceAssignment, CodeBridgeWorkspaceSettlement } from '../bridge/store'; +import { BridgeAdmissionQueue } from '../bridge/admission'; +import { BridgeStoreError, workspaceAdmissionId } from '../bridge/store'; + +const PREFIX = 'codeapi:workspace-requests:v1'; +const ACTIVE = `${PREFIX}:active`; +const RETENTION_MS = 24 * 60 * 60_000; +const CLAIM_MS = 10_000; +export const WORKSPACE_REQUEST_ID_PATTERN = /^[A-Za-z0-9_-]{16,128}$/; + +export class WorkspaceRequestConflict extends Error {} + +export interface WorkspaceRequestOwner { + tenantId: string; + userId: string; +} + +export interface StoredWorkspaceRequest { + id: string; + key: string; + fingerprint: string; + workerId: string; + tenantId: string; + requireTenantBinding: boolean; + request: WorkspaceToolRequest; + registration: RegisteredBridgeWorker; + createdAtMs: number; + queueDeadlineAtMs: number; + executionTimeoutMs: number; + assignment?: DurableWorkspaceAssignment; + settlement?: CodeBridgeWorkspaceSettlement; + admittedAtMs?: number; + finishedAtMs?: number; + state: 'queued' | 'admitted' | 'completed' | 'failed' | 'cancelled'; + cancelRequested?: boolean; + result?: WorkspaceToolResult; + error?: { code: string; message: string }; +} + +export interface WorkspaceRequestStatus { + requestId: string; + state: StoredWorkspaceRequest['state']; + workerId: string; + workspaceId: string; + workspaceInstanceId?: string; + worktree?: string; + queuePosition?: number; + queueWaitMs: number; + executionDeadlineAt?: string; + cancelRequested: boolean; + result?: WorkspaceToolResult; + error?: StoredWorkspaceRequest['error']; +} + +function canonical(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`; + if (value !== null && typeof value === 'object') { + const record = value as Record; + return `{${Object.keys(record).filter(key => record[key] !== undefined).sort() + .map(key => `${JSON.stringify(key)}:${canonical(record[key])}`).join(',')}}`; + } + return JSON.stringify(value); +} + +async function deadline(operation: Promise): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([operation, new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error('Durable workspace transition timed out')), 8_000); + timer.unref(); + })]); + } finally { clearTimeout(timer); } +} + +function requestKey(owner: WorkspaceRequestOwner, requestId: string, scope: string): string { + return `${PREFIX}:${createHash('sha256').update(canonical([scope, owner.tenantId, owner.userId, requestId])).digest('hex')}`; +} + +/** Redis owns requests; API replicas only advance bounded, fenced transitions. */ +export class RedisWorkspaceRequests { + private readonly activeKey: string; + constructor( + private readonly redis: Redis, + private readonly bridge: RedisBridgeStore, + private readonly transition?: (record: StoredWorkspaceRequest) => void, + private readonly scope = '', + ) { + this.activeKey = scope.length === 0 ? ACTIVE : `${PREFIX}:${scope}:active`; + } + + async submit(args: { + owner: WorkspaceRequestOwner; + requestId: string; + workerId: string; + requireTenantBinding: boolean; + request: WorkspaceToolRequest; + queueWaitMs: number; + executionTimeoutMs: number; + }): Promise { + if (!WORKSPACE_REQUEST_ID_PATTERN.test(args.requestId)) { + throw new BridgeStoreError('ASSIGNMENT_INVALID', 'Invalid durable workspace request ID'); + } + const key = requestKey(args.owner, args.requestId, this.scope); + const fingerprint = createHash('sha256').update(canonical({ + workerId: args.workerId, request: args.request, + queueWaitMs: args.queueWaitMs, executionTimeoutMs: args.executionTimeoutMs, + requireTenantBinding: args.requireTenantBinding, + })).digest('hex'); + const existing = await this.read(key); + if (existing != null) { + if (existing.fingerprint !== fingerprint) throw new WorkspaceRequestConflict('Request ID was already used for different work'); + return this.status(existing, args.requestId); + } + const registration = await this.bridge.prepareDurableWorkspaceTool({ + ...args, tenantId: args.owner.tenantId, + }); + const now = Date.now(); + const record: StoredWorkspaceRequest = { + id: `durable-${randomBytes(18).toString('base64url')}`, key, fingerprint, + workerId: args.workerId, tenantId: args.owner.tenantId, + requireTenantBinding: args.requireTenantBinding, + request: args.request, registration, createdAtMs: now, + queueDeadlineAtMs: now + args.queueWaitMs, + executionTimeoutMs: args.executionTimeoutMs, state: 'queued', + }; + const queue = new BridgeAdmissionQueue(this.redis); + const accepted = await queue.submit({ + workerId: record.workerId, id: record.id, deadlineAtMs: record.queueDeadlineAtMs, + workspaceId: workspaceAdmissionId(record.request.workspaceId, record.request.workspaceInstanceId, record.request.worktree), + key, activeKey: this.activeKey, fingerprint, record: JSON.stringify(record), retentionMs: RETENTION_MS, + }); + if (accepted === 'conflict') throw new WorkspaceRequestConflict('Request ID was already used for different work'); + if (accepted === 'full') throw new BridgeStoreError('WORKER_QUEUE_FULL', 'Bridge worker pending request limit reached'); + const submitted = (await this.read(key))!; + if (accepted === 'accepted') this.transition?.(submitted); + return this.status(submitted, args.requestId); + } + + async get(owner: WorkspaceRequestOwner, requestId: string): Promise { + if (!WORKSPACE_REQUEST_ID_PATTERN.test(requestId)) return undefined; + const record = await this.read(requestKey(owner, requestId, this.scope)); + return record == null ? undefined : this.status(record, requestId); + } + + async cancel(owner: WorkspaceRequestOwner, requestId: string): Promise { + if (!WORKSPACE_REQUEST_ID_PATTERN.test(requestId)) return undefined; + const key = requestKey(owner, requestId, this.scope); + await this.redis.eval([ + 'local state = redis.call(\'HGET\', KEYS[1], \'state\')', + 'if state ~= \'queued\' and state ~= \'admitted\' then return 0 end', + 'redis.call(\'HSET\', KEYS[1], \'cancelRequested\', \'1\')', + 'redis.call(\'ZADD\', KEYS[2], ARGV[1], KEYS[1])', + 'return 1', + ].join('\n'), 2, key, this.activeKey, Date.now()); + return this.get(owner, requestId); + } + + private async read(key: string): Promise { + const fields: Partial> = await this.redis.hgetall(key); + if (fields.record == null) return undefined; + return { + ...JSON.parse(fields.record) as StoredWorkspaceRequest, + ...(fields.outcome == null ? {} : JSON.parse(fields.outcome)), + state: fields.state as StoredWorkspaceRequest['state'], + assignment: fields.assignment == null ? undefined : JSON.parse(fields.assignment), + admittedAtMs: fields.admittedAtMs == null ? undefined : Number(fields.admittedAtMs), + cancelRequested: fields.cancelRequested === '1', + }; + } + + private async status(record: StoredWorkspaceRequest, requestId: string): Promise { + const rank = record.state === 'queued' + ? await new BridgeAdmissionQueue(this.redis).position(record.workerId, record.id) + : undefined; + return { + requestId, state: record.state, workerId: record.workerId, + workspaceId: record.request.workspaceId, + workspaceInstanceId: record.request.workspaceInstanceId, worktree: record.request.worktree, + queuePosition: rank, queueWaitMs: Math.max(0, (record.admittedAtMs ?? Math.min(record.finishedAtMs ?? Date.now(), record.queueDeadlineAtMs)) - record.createdAtMs), + executionDeadlineAt: record.assignment?.expiresAt, + cancelRequested: record.cancelRequested === true, + result: record.result, error: record.error, + }; + } + + async reconcile(): Promise { + const keys = await deadline(this.redis.zrangebyscore(this.activeKey, '-inf', Date.now(), 'LIMIT', 0, 32)); + const results = await Promise.allSettled(keys.map(key => deadline(this.advance(key)))); + const failure = results.find(result => result.status === 'rejected'); + if (failure?.status === 'rejected') throw failure.reason; + } + + private async advance(key: string): Promise { + const claimKey = `${key}:claim`; + const token = randomBytes(18).toString('hex'); + const claimed = await this.redis.eval([ + 'if not redis.call(\'SET\', KEYS[1], ARGV[1], \'PX\', ARGV[2], \'NX\') then return 0 end', + 'redis.call(\'ZADD\', KEYS[2], ARGV[3], KEYS[3])', + 'return 1', + ].join('\n'), 3, claimKey, this.activeKey, key, token, CLAIM_MS, Date.now() + CLAIM_MS); + if (Number(claimed) !== 1) return; + try { + let record = await this.read(key); + if (record == null) { await this.redis.zrem(this.activeKey, key); return; } + if (record.state === 'queued' || record.state === 'admitted') { + const previousState = record.state; + try { + const outcome = await deadline(this.bridge.advanceDurableWorkspaceTool(record, { key, claimKey, token })); + if (outcome != null) { + await this.redis.eval([ + 'if redis.call(\'GET\', KEYS[2]) ~= ARGV[1] then return 0 end', + 'local state = redis.call(\'HGET\', KEYS[1], \'state\')', + 'if state ~= \'queued\' and state ~= \'admitted\' then return 0 end', + 'redis.call(\'HSET\', KEYS[1], \'state\', ARGV[3], \'outcome\', ARGV[2])', + 'return 1', + ].join('\n'), 2, key, claimKey, token, JSON.stringify({ ...outcome, finishedAtMs: Date.now() }), outcome.state); + } + } catch (error) { + // Infrastructure uncertainty is reconciled, never retried as a new command. + if (!(error instanceof BridgeStoreError)) throw error; + if (error.code === 'WORKER_OFFLINE') { + // The original worker may reconnect within the remaining admission budget. + await this.redis.zadd(this.activeKey, Date.now() + 250, key); + return; + } + const current = await this.read(key); + if (current?.state === 'admitted') throw error; + await this.redis.eval([ + 'if redis.call(\'GET\', KEYS[2]) ~= ARGV[1] then return 0 end', + 'if redis.call(\'HGET\', KEYS[1], \'state\') ~= \'queued\' then return 0 end', + 'redis.call(\'HSET\', KEYS[1], \'state\', \'failed\', \'outcome\', ARGV[2])', + 'return 1', + ].join('\n'), 2, key, claimKey, token, JSON.stringify({ state: 'failed', finishedAtMs: Date.now(), error: { code: error.code, message: error.message } })); + } + record = (await this.read(key))!; + if (record.state !== previousState) this.transition?.(record); + } + if (record.state !== 'queued' && record.state !== 'admitted') { + await deadline(this.bridge.finishDurableWorkspaceTool(record)); + await this.redis.zrem(this.activeKey, key); + } else { + await this.redis.zadd(this.activeKey, Date.now() + 250, key); + } + } finally { + await this.redis.eval('if redis.call(\'GET\', KEYS[1]) == ARGV[1] then return redis.call(\'DEL\', KEYS[1]) end return 0', 1, claimKey, token); + } + } +} diff --git a/service/src/workspace-tools/router.ts b/service/src/workspace-tools/router.ts index 78dbd1e7..d4d65d14 100644 --- a/service/src/workspace-tools/router.ts +++ b/service/src/workspace-tools/router.ts @@ -19,6 +19,8 @@ import { resolveBridgeWorkerSelection, } from '../bridge/selection'; import { principalWorkspaceInstanceId } from '../bridge/workspace-instance'; +import { WorkspaceRequestConflict } from './requests'; +import type { RedisWorkspaceRequests } from './requests'; const MAX_WORKSPACE_QUEUE_WAIT_MS = 5 * 60_000; const DEFAULT_WORKSPACE_QUEUE_WAIT_MS = 30_000; @@ -26,6 +28,7 @@ const WORKSPACE_QUEUE_WAIT_HEADER = 'X-LibreChat-Workspace-Queue-Wait-Ms'; interface WorkspaceToolsRouterOptions { store: Pick; + requests?: RedisWorkspaceRequests; backend: 'http' | 'lambda-microvm' | 'remote-bridge'; configuredWorkerId: string; dynamicWorkers: boolean; @@ -67,161 +70,200 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) } const router = Router(); - router.post( - '/workspace-tools/execute', - asyncRoute(async (req, res) => { - const outcome = getWorkspaceToolOutcome(res); - const principal = getPrincipalOrReject(req, res); - if (!principal) { - outcome.errorCode = 'UNAUTHENTICATED'; - return; - } - if ((options.isShuttingDown ?? checkServiceShutDown)()) { - outcome.errorCode = 'SERVICE_SHUTTING_DOWN'; - res.status(503).json({ error: 'Service is shutting down' }); - return; - } - if (!isWorkspaceToolRequest(req.body)) { - outcome.errorCode = 'INVALID_WORKSPACE_TOOL_REQUEST'; - res.status(400).json({ - error: 'Invalid workspace tool request', - }); - return; - } - const advertisedQueueWait = req.header(WORKSPACE_QUEUE_WAIT_HEADER); - if (advertisedQueueWait !== undefined && !/^[1-9]\d*$/.test(advertisedQueueWait)) { - outcome.errorCode = 'INVALID_WORKSPACE_QUEUE_WAIT'; - res.status(400).json({ error: 'Invalid workspace queue wait', code: 'INVALID_WORKSPACE_QUEUE_WAIT' }); + const execute = (durable: boolean): RequestHandler => asyncRoute(async (req, res) => { + const outcome = getWorkspaceToolOutcome(res, req.path); + const principal = getPrincipalOrReject(req, res); + if (!principal) { + outcome.errorCode = 'UNAUTHENTICATED'; + return; + } + if ((options.isShuttingDown ?? checkServiceShutDown)()) { + outcome.errorCode = 'SERVICE_SHUTTING_DOWN'; + res.status(503).json({ error: 'Service is shutting down' }); + return; + } + if (!isWorkspaceToolRequest(req.body)) { + outcome.errorCode = 'INVALID_WORKSPACE_TOOL_REQUEST'; + res.status(400).json({ + error: 'Invalid workspace tool request', + }); + return; + } + const advertisedQueueWait = req.header(WORKSPACE_QUEUE_WAIT_HEADER); + if (advertisedQueueWait !== undefined && !/^[1-9]\d*$/.test(advertisedQueueWait)) { + outcome.errorCode = 'INVALID_WORKSPACE_QUEUE_WAIT'; + res.status(400).json({ error: 'Invalid workspace queue wait', code: 'INVALID_WORKSPACE_QUEUE_WAIT' }); + return; + } + const requestedQueueWaitMs = advertisedQueueWait === undefined + ? DEFAULT_WORKSPACE_QUEUE_WAIT_MS + : Number(advertisedQueueWait); + if (!Number.isSafeInteger(requestedQueueWaitMs) || requestedQueueWaitMs > MAX_WORKSPACE_QUEUE_WAIT_MS) { + outcome.errorCode = 'INVALID_WORKSPACE_QUEUE_WAIT'; + res.status(400).json({ error: 'Invalid workspace queue wait', code: 'INVALID_WORKSPACE_QUEUE_WAIT' }); + return; + } + const queueBudgetMs = Math.min(requestedQueueWaitMs, queueCeilingMs); + outcome.operation = req.body.operation; + const principalRequest: WorkspaceToolRequest = req.body.workspaceInstanceId == null + ? req.body + : { + ...req.body, + workspaceInstanceId: principalWorkspaceInstanceId({ + instanceId: req.body.workspaceInstanceId, + tenantId: principal.tenantId, + principalId: principal.userId, + }), + }; + const request: WorkspaceToolRequest = principalRequest.operation === 'execute_command' + ? { ...principalRequest, timeoutMs: Math.min( + principalRequest.timeoutMs ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, + options.timeoutMs ?? Number.MAX_SAFE_INTEGER, + ) } + : principalRequest; + const executionBudgetMs = request.operation === 'execute_command' + ? request.timeoutMs! + 5_000 + : Math.min(options.timeoutMs ?? 30_000, 30_000); + outcome.deadlineBudgetMs = queueBudgetMs + executionBudgetMs; + + let selection: { workerId: string; explicit: boolean } | undefined; + try { + selection = resolveBridgeWorkerSelection({ + backend: options.backend, + configuredWorkerId: options.configuredWorkerId, + dynamicWorkers: options.dynamicWorkers, + requestedWorkerId: req.header(CODEAPI_BRIDGE_WORKER_HEADER), + trustedWorkerId: principal.codeWorkerId, + }); + } catch (error) { + if (error instanceof BridgeWorkerSelectionError) { + outcome.errorCode = 'WORKER_SELECTION_REJECTED'; + res.status(error.status).json({ error: error.message }); return; } - const requestedQueueWaitMs = advertisedQueueWait === undefined - ? DEFAULT_WORKSPACE_QUEUE_WAIT_MS - : Number(advertisedQueueWait); - if (!Number.isSafeInteger(requestedQueueWaitMs) || requestedQueueWaitMs > MAX_WORKSPACE_QUEUE_WAIT_MS) { - outcome.errorCode = 'INVALID_WORKSPACE_QUEUE_WAIT'; - res.status(400).json({ error: 'Invalid workspace queue wait', code: 'INVALID_WORKSPACE_QUEUE_WAIT' }); + throw error; + } + if (selection == null) { + outcome.errorCode = 'WORKSPACE_BACKEND_UNAVAILABLE'; + res.status(503).json({ + error: 'Workspace tools require the remote-bridge backend', + }); + return; + } + outcome.workerId = selection.workerId; + + if (durable) { + if (options.requests == null) { + res.status(404).json({ error: 'Durable workspace requests are unavailable' }); return; } - const queueBudgetMs = Math.min(requestedQueueWaitMs, queueCeilingMs); - outcome.operation = req.body.operation; - const principalRequest: WorkspaceToolRequest = req.body.workspaceInstanceId == null - ? req.body - : { - ...req.body, - workspaceInstanceId: principalWorkspaceInstanceId({ - instanceId: req.body.workspaceInstanceId, - tenantId: principal.tenantId, - principalId: principal.userId, - }), - }; - const request: WorkspaceToolRequest = principalRequest.operation === 'execute_command' - ? { ...principalRequest, timeoutMs: Math.min( - principalRequest.timeoutMs ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, - options.timeoutMs ?? Number.MAX_SAFE_INTEGER, - ) } - : principalRequest; - const executionBudgetMs = request.operation === 'execute_command' - ? request.timeoutMs! + 5_000 - : Math.min(options.timeoutMs ?? 30_000, 30_000); - outcome.deadlineBudgetMs = queueBudgetMs + executionBudgetMs; - - let selection: { workerId: string; explicit: boolean } | undefined; try { - selection = resolveBridgeWorkerSelection({ - backend: options.backend, - configuredWorkerId: options.configuredWorkerId, - dynamicWorkers: options.dynamicWorkers, - requestedWorkerId: req.header(CODEAPI_BRIDGE_WORKER_HEADER), - trustedWorkerId: principal.codeWorkerId, + const status = await options.requests.submit({ + owner: principal, requestId: req.header('X-LibreChat-Workspace-Request-Id') ?? '', + workerId: selection.workerId, + requireTenantBinding: selection.explicit && (options.dynamicWorkers || selection.workerId !== options.configuredWorkerId), + request, queueWaitMs: queueBudgetMs, executionTimeoutMs: executionBudgetMs, }); + res.status(202).json(status); } catch (error) { - if (error instanceof BridgeWorkerSelectionError) { - outcome.errorCode = 'WORKER_SELECTION_REJECTED'; - res.status(error.status).json({ error: error.message }); - return; - } - throw error; - } - if (selection == null) { - outcome.errorCode = 'WORKSPACE_BACKEND_UNAVAILABLE'; - res.status(503).json({ - error: 'Workspace tools require the remote-bridge backend', - }); - return; + if (error instanceof WorkspaceRequestConflict) { + res.status(409).json({ error: error.message, code: 'REQUEST_CONFLICT' }); + } else if (error instanceof BridgeStoreError) { + res.status(bridgeStoreStatus(error)).json({ error: error.message, code: error.code }); + } else throw error; } - outcome.workerId = selection.workerId; + return; + } - const controller = new AbortController(); - const abort = (): void => controller.abort(); - req.once('aborted', abort); - const abortClosedResponse = (): void => { - if (!res.writableEnded) abort(); - }; - res.once('close', abortClosedResponse); - try { - outcome.dispatchPending = true; - const dispatchStartedAt = performance.now(); - const settlement = await options.store.dispatchWorkspaceTool({ - workerId: selection.workerId, - tenantId: principal.tenantId, - requireTenantBinding: + const controller = new AbortController(); + const abort = (): void => controller.abort(); + req.once('aborted', abort); + const abortClosedResponse = (): void => { + if (!res.writableEnded) abort(); + }; + res.once('close', abortClosedResponse); + try { + outcome.dispatchPending = true; + const dispatchStartedAt = performance.now(); + const settlement = await options.store.dispatchWorkspaceTool({ + workerId: selection.workerId, + tenantId: principal.tenantId, + requireTenantBinding: selection.explicit && (options.dynamicWorkers || selection.workerId !== options.configuredWorkerId), - request, - deadlineAtMs: Date.now() + queueBudgetMs, - executionTimeoutMs: executionBudgetMs, - signal: controller.signal, - }).finally(() => { - outcome.dispatchDurationMs = Math.round(performance.now() - dispatchStartedAt); - }); - if (settlement.status === 'rejected') { - outcome.errorCode = settlement.errorCode ?? 'WORKSPACE_TOOL_REJECTED'; - let status = 422; - if ( - settlement.errorCode === 'SEARCH_TIMEOUT' || + request, + deadlineAtMs: Date.now() + queueBudgetMs, + executionTimeoutMs: executionBudgetMs, + signal: controller.signal, + }).finally(() => { + outcome.dispatchDurationMs = Math.round(performance.now() - dispatchStartedAt); + }); + if (settlement.status === 'rejected') { + outcome.errorCode = settlement.errorCode ?? 'WORKSPACE_TOOL_REJECTED'; + let status = 422; + if ( + settlement.errorCode === 'SEARCH_TIMEOUT' || settlement.errorCode === 'LIST_TIMEOUT' || settlement.errorCode === 'COMMAND_TIMEOUT' - ) { - status = 504; - } - if ( - settlement.errorCode === 'SEARCH_UNAVAILABLE' || + ) { + status = 504; + } + if ( + settlement.errorCode === 'SEARCH_UNAVAILABLE' || settlement.errorCode === 'LIST_UNAVAILABLE' || settlement.errorCode === 'COMMAND_UNAVAILABLE' - ) { - status = 503; - } - if (settlement.errorCode === 'WRITE_DISABLED') status = 403; - if (settlement.errorCode === 'COMMAND_DISABLED') status = 403; - if (settlement.errorCode === 'WRITE_LIMIT_EXCEEDED') status = 413; - if (settlement.errorCode === 'WRITE_UNAVAILABLE') status = 503; - if (settlement.errorCode === 'EDIT_CONFLICT') status = 409; - res.status(status).json({ - error: settlement.error, - code: settlement.errorCode ?? 'WORKSPACE_TOOL_REJECTED', - }); - return; - } - res.status(200).json(settlement.result); - } catch (error) { - if (error instanceof BridgeStoreError) { - outcome.errorCode = error.code; - if (error.code === 'WORKSPACE_QUEUE_TIMEOUT') res.setHeader('Retry-After', '1'); - res.status(bridgeStoreStatus(error)).json({ - error: error.message, - code: error.code, - }); - return; + ) { + status = 503; } - outcome.errorCode = 'INTERNAL_ERROR'; - throw error; - } finally { - outcome.dispatchPending = false; - outcome.flush(); - req.removeListener('aborted', abort); - res.removeListener('close', abortClosedResponse); + if (settlement.errorCode === 'WRITE_DISABLED') status = 403; + if (settlement.errorCode === 'COMMAND_DISABLED') status = 403; + if (settlement.errorCode === 'WRITE_LIMIT_EXCEEDED') status = 413; + if (settlement.errorCode === 'WRITE_UNAVAILABLE') status = 503; + if (settlement.errorCode === 'EDIT_CONFLICT') status = 409; + res.status(status).json({ + error: settlement.error, + code: settlement.errorCode ?? 'WORKSPACE_TOOL_REJECTED', + }); + return; + } + res.status(200).json(settlement.result); + } catch (error) { + if (error instanceof BridgeStoreError) { + outcome.errorCode = error.code; + if (error.code === 'WORKSPACE_QUEUE_TIMEOUT') res.setHeader('Retry-After', '1'); + res.status(bridgeStoreStatus(error)).json({ + error: error.message, + code: error.code, + }); + return; } - }), - ); + outcome.errorCode = 'INTERNAL_ERROR'; + throw error; + } finally { + outcome.dispatchPending = false; + outcome.flush(); + req.removeListener('aborted', abort); + res.removeListener('close', abortClosedResponse); + } + }); + router.post('/workspace-tools/execute', execute(false)); + router.post('/workspace-tools/requests', execute(true)); + + router.get('/workspace-tools/capabilities', asyncRoute(async (req, res) => { + if (!getPrincipalOrReject(req, res)) return; + res.json({ durableWorkspaceRequests: options.requests != null ? 1 : 0 }); + })); + + const requestStatus = (cancel: boolean): RequestHandler => asyncRoute(async (req, res) => { + const principal = getPrincipalOrReject(req, res); + if (!principal) return; + const status = cancel + ? await options.requests?.cancel(principal, req.params.requestId) + : await options.requests?.get(principal, req.params.requestId); + if (status == null) { res.status(404).json({ error: 'Workspace request not found' }); return; } + res.json(status); + }); + router.get('/workspace-tools/requests/:requestId', requestStatus(false)); + router.delete('/workspace-tools/requests/:requestId', requestStatus(true)); return router; } From 192f61cd36785a885980274d1792dd30efc296bb Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 11:33:52 -0400 Subject: [PATCH 30/48] =?UTF-8?q?=F0=9F=93=96=20feat:=20Stream=20Bounded?= =?UTF-8?q?=20Workspace=20File=20Reads=20(#303)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 📖 feat: Stream Bounded Workspace File Reads * 🧹 fix: Settle Interrupted Reads Before Pending I/O Cleanup * ⏱️ fix: Track Workspace Read Interruption Through Cleanup * 🧪 test: Queue Descriptor I/O Consistently on Node 20 --------- Co-authored-by: Lia --- packages/code/README.md | 7 +- packages/code/WORKSPACE-READS.md | 32 ++ packages/code/src/root-access.ts | 10 +- packages/code/src/workspace-read-io.test.ts | 147 ++++++ packages/code/src/workspace-read.test.ts | 488 ++++++++++++++++++++ packages/code/src/workspace.ts | 295 ++++++++++-- 6 files changed, 928 insertions(+), 51 deletions(-) create mode 100644 packages/code/WORKSPACE-READS.md create mode 100644 packages/code/src/workspace-read-io.test.ts create mode 100644 packages/code/src/workspace-read.test.ts diff --git a/packages/code/README.md b/packages/code/README.md index 636cd1a8..d348fe88 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -729,9 +729,10 @@ native SRT command backend. This operator switch controls availability; LibreChat tool approval hooks remain the user-facing allow/deny boundary for each invocation. -Reads reject absolute paths, traversal, escaping symlinks, non-regular files, -and files larger than 1 MiB. The opened file is checked against its canonical -in-workspace inode before it is read. Text search uses `rg` only to enumerate a +Reads reject absolute paths, traversal, escaping symlinks, and non-regular files. +The opened file is checked against its canonical in-workspace inode before it is +read. Ordinary reads stream bounded line windows even from files larger than 1 MiB; +see the [read contract](WORKSPACE-READS.md). Text search uses `rg` only to enumerate a bounded set of ignored-aware candidates with configuration and symlink following disabled. It then opens and verifies each candidate through the same confined 1 MiB read boundary before matching locally. File listing invokes `rg` without diff --git a/packages/code/WORKSPACE-READS.md b/packages/code/WORKSPACE-READS.md new file mode 100644 index 00000000..f3edbe58 --- /dev/null +++ b/packages/code/WORKSPACE-READS.md @@ -0,0 +1,32 @@ +# Workspace file read contract + +Ordinary `read_file` streams complete LF-delimited line windows through the verified +file descriptor. It defaults to 200 lines, accepts at most 500, and returns at most +1 MiB of UTF-8 content, including inter-line separators, regardless of file size. +CR in CRLF is preserved; the final LF does not add a phantom line. An empty file +has one empty line. A start beyond EOF returns an empty, non-truncated window. + +Byte-limited windows stop before the first line that cannot fit and return +`nextStartLine = endLine + 1`. If the first requested line exceeds 1 MiB, the read +fails with `READ_LIMIT_EXCEEDED`, without a repeating continuation. Scanning and +collection check cancellation and share a 10-second deadline. Far-away starts +can fail with `READ_LIMIT_EXCEEDED`; use an earlier start or `search_text`. +Cancellation and timeout settle the request without waiting for queued I/O. +Node retains an interrupted read's descriptor and chunk until that I/O drains; +file and held-root cleanup are initiated immediately and finish asynchronously. +This releases the request lane, but does not cancel kernel I/O. + +Memory is bounded by 64 KiB chunks and the returned window. Reads stop at the +opened file's initial size, so concurrent growth cannot extend the scan. Truncation +ends the scan at observed EOF; pathname replacement never switches the open +handle. In-place writes can change observed content. Pagination is not a snapshot +across requests and assumes the file stays unchanged. + +UTF-8 and BOM-marked UTF-16LE/BE are decoded incrementally. One leading BOM is +removed; malformed sequences use replacement characters, as before. Binary bytes +are decoded as text, including NUL, not classified or rejected. Neither a successful +window nor continuation validates unread content. + +Request/result shapes, protocol version, and capabilities are unchanged. Search, +preview, edit, and write limits remain unchanged. Digest-checked repository +instruction snapshots retain their separate byte-bounded, newline-preserving path. diff --git a/packages/code/src/root-access.ts b/packages/code/src/root-access.ts index 5bdddca9..2f049bcb 100644 --- a/packages/code/src/root-access.ts +++ b/packages/code/src/root-access.ts @@ -378,6 +378,12 @@ export class WorkspaceRootAccess { } const context = new AsyncLocalStorage(); +const deferredCleanup = new WeakSet(); +/** Interrupted I/O must not hold a request open while Node drains its handles. */ +export function deferWorkspaceRootCleanup(): void { + const access = context.getStore(); + if (access) deferredCleanup.add(access); +} /** Whether filesystem adapters in this call are anchored to a held root descriptor. */ export const holdsWorkspaceRoot = (): boolean => context.getStore() != null; export async function withWorkspaceRoot( @@ -390,7 +396,9 @@ export async function withWorkspaceRoot( try { return await context.run(access, action); } finally { - await access.close(); + const closing = access.close(); + if (deferredCleanup.delete(access)) void closing.catch(() => {}); + else await closing; } } diff --git a/packages/code/src/workspace-read-io.test.ts b/packages/code/src/workspace-read-io.test.ts new file mode 100644 index 00000000..2fdd2b90 --- /dev/null +++ b/packages/code/src/workspace-read-io.test.ts @@ -0,0 +1,147 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import * as fs from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); + +// Isolate libuv saturation from the test runner and sibling tests. +const queuedRead = ` +import assert from 'node:assert/strict'; +import * as fs from 'node:fs'; +import * as fsp from 'node:fs/promises'; +import { execFileSync } from 'node:child_process'; +import { getEventListeners } from 'node:events'; +import { join } from 'node:path'; +import { mock } from 'node:test'; +import { LocalWorkspaceTools } from ${JSON.stringify( + new URL('./workspace.js', import.meta.url).href +)}; +import { captureWorkspaceRootIdentity } from ${JSON.stringify( + new URL('./root-identity.js', import.meta.url).href +)}; +import { WorkspaceRootAccess } from ${JSON.stringify( + new URL('./root-access.js', import.meta.url).href +)}; +const [root, cause, held] = process.argv.slice(1); +const tools = await LocalWorkspaceTools.create({ workspaces: [{ + id: 'primary', root, + ...(held === 'true' ? { identity: await captureWorkspaceRootIdentity(root) } : {}), +}] }); +const fifo = join(root, 'pool'); +execFileSync('mkfifo', [fifo]); +const pipe = fs.openSync(fifo, fs.constants.O_RDWR); +const probe = await fsp.open(join(root, 'file'), 'r'); +const prototype = Object.getPrototypeOf(probe); +await probe.close(); +const originalRead = prototype.read; +const controller = new AbortController(); +let entered, blocker, pendingIo, physicalFd, readSettled = false; +const started = new Promise(resolve => { entered = resolve; }); +const closing = []; +const descriptors = []; +let elapsed = 0; +const now = performance.now(); +mock.method(performance, 'now', () => now + elapsed); +function trackClose(handle) { + const originalClose = handle.close; + mock.method(handle, 'close', function (...args) { + descriptors.push(this.fd); + const promise = originalClose.apply(this, args); + closing.push(promise); + return promise; + }); +} +const originalOpen = WorkspaceRootAccess.open; +mock.method(WorkspaceRootAccess, 'open', async (...args) => { + const access = await originalOpen(...args); + trackClose(access.handle); + return access; +}); +mock.method(prototype, 'read', function (...args) { + physicalFd = this.fd; + trackClose(this); + // This actual pipe read occupies the sole libuv worker until explicitly drained. + blocker = new Promise((resolve, reject) => fs.read(pipe, Buffer.alloc(1), 0, 1, null, + error => error ? reject(error) : resolve())); + // Invoke Node's original FileHandle.read, including its active-I/O references. + pendingIo = originalRead.apply(this, args).finally(() => { readSettled = true; }); + if (cause === 'deadline') elapsed = 10_000; + if (cause === 'early-deadline') elapsed = 9_999.75; + entered(); + return pendingIo; +}); +let released = false; +function drain() { + if (!released) { released = true; fs.writeSync(pipe, Buffer.from('x')); } +} +const wallStart = Date.now(); +const pending = tools.execute({ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'file' }, controller.signal); +const rejected = assert.rejects(pending, error => error.code === (cause === 'abort' ? 'EXECUTION_ABORTED' : 'READ_LIMIT_EXCEEDED')); +await started; +if (cause === 'abort') controller.abort(); +// Always drain, even if a regression prevents bounded settlement. +const fallback = setTimeout(drain, 2_000); +try { + await rejected; + assert.equal(released, false, 'request settled before pool was drained'); + if (cause === 'early-deadline') assert.ok(performance.now() < now + 10_000, 'timer settled before the monotonic deadline'); + assert.equal(readSettled, false, 'real descriptor read remained pending'); + assert.ok(fs.fstatSync(physicalFd).isFile(), 'Node still owns the physical descriptor'); + assert.equal(closing.length, held === 'true' ? 2 : 1, 'file and held-root closes initiated'); + assert.deepEqual(getEventListeners(controller.signal, 'abort'), []); + console.log(JSON.stringify({ cause, held: held === 'true', settledBeforeDrain: true, readSettled, elapsedMs: Date.now() - wallStart })); +} finally { + clearTimeout(fallback); + drain(); + await blocker; + await pendingIo; + await Promise.all(closing); + for (const fd of descriptors) assert.throws(() => fs.fstatSync(fd), { code: 'EBADF' }); + mock.restoreAll(); + fs.closeSync(pipe); +} +`; + +for (const cause of ['abort', 'deadline', 'early-deadline'] as const) { + for (const held of [false, true]) { + test(`${cause} settles before real queued I/O drains (${ + held ? 'held' : 'legacy' + } root)`, async t => { + if (!['linux', 'darwin'].includes(process.platform)) + return t.skip('requires POSIX FIFO and descriptor access'); + const root = await fs.realpath( + await fs.mkdtemp(join(tmpdir(), 'workspace-read-io-')) + ); + t.after(() => fs.rm(root, { recursive: true, force: true })); + await fs.writeFile(join(root, 'file'), 'line\n'.repeat(300_000)); + const { stdout } = await execFileAsync( + process.execPath, + [ + '--input-type=module', + '--eval', + queuedRead, + root, + cause, + String(held), + ], + { + // Node 20 can bypass the pool for regular files via io_uring. + env: { + ...process.env, + UV_THREADPOOL_SIZE: '1', + UV_USE_IO_URING: '0', + }, + timeout: 10_000, + } + ); + const observed = JSON.parse(stdout); + assert.equal(observed.settledBeforeDrain, true); + assert.equal(observed.readSettled, false); + t.diagnostic(stdout.trim()); + }); + } +} diff --git a/packages/code/src/workspace-read.test.ts b/packages/code/src/workspace-read.test.ts new file mode 100644 index 00000000..1c6a7f96 --- /dev/null +++ b/packages/code/src/workspace-read.test.ts @@ -0,0 +1,488 @@ +import assert from 'node:assert/strict'; +import * as fs from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import type { TestContext } from 'node:test'; +import type { FileHandle } from 'node:fs/promises'; +import { LocalWorkspaceTools, WorkspaceToolError } from './workspace.js'; +import { WorkspaceRootAccess } from './root-access.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; +import { readRepositoryInstructions } from './instructions.js'; +import { + BRIDGE_WORKSPACE_READ_MAX_BYTES as MAX_BYTES, + isWorkspaceToolResult, +} from './protocol.js'; +import type { WorkspaceReadFileRequest } from './protocol.js'; + +async function fixture(t: TestContext, held = false) { + const parent = await fs.realpath( + await fs.mkdtemp(join(tmpdir(), 'workspace-read-')) + ); + t.after(() => fs.rm(parent, { recursive: true, force: true })); + const root = join(parent, 'root'); + await fs.mkdir(root); + const tools = await LocalWorkspaceTools.create({ + workspaces: [ + { + id: 'primary', + root, + writable: true, + ...(held + ? { identity: await captureWorkspaceRootIdentity(root) } + : {}), + }, + ], + repositoryInstructions: true, + }); + const read = async ( + startLine?: number, + maxLines?: number, + signal?: AbortSignal, + path = 'file' + ) => { + const request: WorkspaceReadFileRequest = { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path, + ...(startLine !== undefined ? { startLine } : {}), + ...(maxLines !== undefined ? { maxLines } : {}), + }; + const result = await tools.execute(request, signal); + assert.ok( + isWorkspaceToolResult(request, result), + 'current protocol accepts the window' + ); + assert.equal(result.operation, 'read_file'); + return result; + }; + const probe = await fs.open(join(root, 'file'), 'w+'); + const prototype = Object.getPrototypeOf(probe) as FileHandle; + await probe.close(); + return { parent, root, tools, read, prototype }; +} + +function hasCode(code: string) { + return (error: unknown) => + error instanceof WorkspaceToolError && error.code === code; +} + +test('small windows from large files use bounded descriptor reads, not readFile', async t => { + const { root, read, prototype } = await fixture(t, true); + await fs.writeFile( + join(root, 'file'), + 'first\nsecond\n' + 'tail\n'.repeat(300_000) + ); + const originalRead = prototype.read; + let bytesRead = 0; + t.mock.method(prototype, 'readFile', () => assert.fail('whole-file read')); + t.mock.method( + prototype, + 'read', + async function ( + this: FileHandle, + ...args: Parameters + ) { + const result = await originalRead.apply(this, args); + bytesRead += result.bytesRead; + return result; + } + ); + const result = await read(1, 2); + assert.equal(result.content, 'first\nsecond'); + assert.equal(result.nextStartLine, 3); + assert.ok(bytesRead <= 64 * 1024 + 3); +}); + +test('sequential line and byte pagination reconstructs large content exactly', async t => { + const { root, read } = await fixture(t); + const lines = Array.from( + { length: 1_201 }, + (_, i) => `${i}: ${'é🦊'.repeat(300)}` + ); + await fs.writeFile(join(root, 'file'), '\ufeff' + lines.join('\n') + '\n'); + const collected: string[] = []; + let start = 1; + for (;;) { + const result = await read(start, 500); + assert.ok(Buffer.byteLength(result.content) <= MAX_BYTES); + collected.push(...result.content.split('\n')); + assert.equal(result.endLine, collected.length); + if (!result.truncated) break; + assert.ok(result.nextStartLine! > start); + start = result.nextStartLine!; + } + assert.deepEqual(collected, lines); + const defaults = await read(); + assert.equal(defaults.endLine, 200); + assert.equal(defaults.nextStartLine, 201); +}); + +test('byte ceilings include separators and never split or repeat an oversized line', async t => { + const { root, read } = await fixture(t); + const exact = 'a'.repeat(MAX_BYTES); + await fs.writeFile( + join(root, 'file'), + exact + '\nx\n' + 'z'.repeat(MAX_BYTES + 1) + ); + const first = await read(1, 500); + assert.equal(first.content, exact); + assert.equal(first.nextStartLine, 2); + const second = await read(2, 500); + assert.equal(second.content, 'x'); + assert.equal(second.nextStartLine, 3); + await assert.rejects(read(3), error => { + assert.ok(hasCode('READ_LIMIT_EXCEEDED')(error)); + assert.match( + (error as Error).message, + /line 3.*later startLine.*search_text/ + ); + assert.ok((error as Error).message.length < 300); + return true; + }); + await fs.writeFile( + join(root, 'file'), + 'a'.repeat(MAX_BYTES - 2) + '\nb\nc' + ); + const separators = await read(1, 500); + assert.equal(Buffer.byteLength(separators.content), MAX_BYTES); + assert.equal(separators.endLine, 2); + assert.equal(separators.nextStartLine, 3); +}); + +test('EOF, empty lines, CRLF, BOM-only files, and starts beyond EOF retain their contract', async t => { + const { root, read } = await fixture(t); + for (const [source, expected] of [ + ['', ['']], + ['\ufeff', ['']], + ['\n', ['']], + ['\n\n', ['', '']], + ['a\n', ['a']], + ['a\r\nb\r\n', ['a\r', 'b\r']], + ['a\nb', ['a', 'b']], + ['\ufeff\ufeffa', ['\ufeffa']], + ] as const) { + await fs.writeFile(join(root, 'file'), source); + for (let line = 1; line <= expected.length; line++) { + const result = await read(line, 1); + assert.equal(result.content, expected[line - 1]); + assert.equal(result.endLine, line); + assert.equal(result.truncated, line < expected.length); + } + const past = await read(expected.length + 1); + assert.equal(past.content, ''); + assert.equal(past.endLine, expected.length); + assert.equal(past.truncated, false); + } +}); + +test('streaming decoding handles short reads, split UTF-8, CRLF, and UTF-16 surrogates', async t => { + const { root, read, prototype } = await fixture(t); + const originalRead = prototype.read; + t.mock.method( + prototype, + 'read', + async function ( + this: FileHandle, + buffer: Buffer, + offset: number, + length: number, + position: number + ) { + return originalRead.call(this, { + buffer, + offset, + length: Math.min(length, 1), + position, + }); + } + ); + const text = 'é🦊\r\n\ufeffsecond\n'; + const little = Buffer.from(text, 'utf16le'); + const big = Buffer.from(little).swap16(); + for (const source of [ + Buffer.from('\ufeff' + text), + Buffer.concat([Buffer.from([0xff, 0xfe]), little]), + Buffer.concat([Buffer.from([0xfe, 0xff]), big]), + ]) { + await fs.writeFile(join(root, 'file'), source); + const result = await read(); + assert.equal(result.content, 'é🦊\r\n\ufeffsecond'); + assert.equal(result.endLine, 2); + assert.equal(result.truncated, false); + } +}); + +test('UTF-8 replacement decoding and binary bytes are bounded text, not file validation', async t => { + const { root, read } = await fixture(t); + await fs.writeFile( + join(root, 'file'), + Buffer.from([0x61, 0x0a, 0xff, 0x00, 0xe2, 0x82]) + ); + const first = await read(1, 1); + assert.equal(first.content, 'a'); + assert.equal(first.truncated, true); + const second = await read(2); + assert.equal(second.content, '\ufffd\0\ufffd'); + await fs.writeFile(join(root, 'file'), Buffer.alloc(MAX_BYTES / 2, 0xff)); + await assert.rejects(read(), hasCode('READ_LIMIT_EXCEEDED')); +}); + +test('scanning skips huge earlier lines without accumulating them', async t => { + const { root, read } = await fixture(t); + await fs.writeFile( + join(root, 'file'), + 'x'.repeat(4 * MAX_BYTES) + '\nrequested\n' + ); + assert.equal((await read(2, 1)).content, 'requested'); +}); + +test('cancellation during scanning and collection closes every read handle', async t => { + for (const startLine of [1, 300_000]) { + const { root, read, prototype } = await fixture(t); + await fs.writeFile(join(root, 'file'), 'line\n'.repeat(300_000)); + const controller = new AbortController(); + const originalRead = prototype.read; + const handles = new Set(); + let calls = 0; + const mock = t.mock.method( + prototype, + 'read', + async function ( + this: FileHandle, + ...args: Parameters + ) { + handles.add(this); + const result = await originalRead.apply(this, args); + if (++calls === 2) controller.abort(); + return result; + } + ); + await assert.rejects( + read(startLine, 500, controller.signal), + hasCode('EXECUTION_ABORTED') + ); + assert.ok(handles.size > 0); + for (const handle of handles) assert.equal(handle.fd, -1); + mock.mock.restore(); + } +}); + +test('far-away starts hit the scan deadline and close the descriptor', async t => { + const { root, read, prototype } = await fixture(t); + await fs.writeFile(join(root, 'file'), 'line\n'.repeat(300_000)); + const originalRead = prototype.read; + const now = performance.now(); + let elapsed = 0; + let handle: FileHandle | undefined; + t.mock.method(performance, 'now', () => now + elapsed); + t.mock.method( + prototype, + 'read', + async function ( + this: FileHandle, + ...args: Parameters + ) { + handle = this; + const result = await originalRead.apply(this, args); + elapsed += 6_000; + return result; + } + ); + await assert.rejects(read(300_000), hasCode('READ_LIMIT_EXCEEDED')); + assert.equal(handle?.fd, -1); +}); + +test('growth is capped at the opened extent and truncation produces a valid EOF window', async t => { + for (const change of ['grow', 'truncate'] as const) { + const { root, read, prototype } = await fixture(t); + const path = join(root, 'file'); + await fs.writeFile(path, 'first\nsecond\n'); + const originalRead = prototype.read; + let calls = 0; + const mock = t.mock.method( + prototype, + 'read', + async function ( + this: FileHandle, + ...args: Parameters + ) { + const result = await originalRead.apply(this, args); + if (++calls === 1) { + if (change === 'grow') + await fs.appendFile(path, 'appended\n'); + else await fs.truncate(path, 3); + } + return result; + } + ); + const result = await read(); + assert.equal( + result.content, + change === 'grow' ? 'first\nsecond' : 'fir' + ); + assert.equal(result.truncated, false); + mock.mock.restore(); + } +}); + +test('file and root replacement cannot redirect a held streaming read; descriptors close', async t => { + const { parent, root, read, prototype } = await fixture(t, true); + const path = join(root, 'file'); + await fs.writeFile(path, 'original\n' + 'inside\n'.repeat(200_000)); + await fs.writeFile(join(parent, 'secret'), 'outside secret'); + const originalRead = prototype.read; + const originalOpen = WorkspaceRootAccess.open; + let rootHandle: FileHandle | undefined; + let fileHandle: FileHandle | undefined; + t.mock.method( + WorkspaceRootAccess, + 'open', + async (...args: Parameters) => { + const access = await originalOpen(...args); + rootHandle = access.handle; + return access; + } + ); + let calls = 0; + t.mock.method( + prototype, + 'read', + async function ( + this: FileHandle, + ...args: Parameters + ) { + fileHandle = this; + const result = await originalRead.apply(this, args); + if (++calls === 1) { + await fs.rename(path, join(root, 'old-file')); + await fs.symlink(join(parent, 'secret'), path); + await fs.rename(root, join(parent, 'old-root')); + await fs.mkdir(root); + await fs.writeFile(path, 'replacement'); + } + return result; + } + ); + const result = await read(2, 500); + assert.equal(result.content, Array(500).fill('inside').join('\n')); + assert.equal(result.nextStartLine, 502); + assert.equal(fileHandle?.fd, -1); + assert.equal(rootHandle?.fd, -1); + await assert.rejects(read(), hasCode('REGISTRATION_INVALID')); +}); + +test('oversized lines and I/O failures close file and held-root descriptors', async t => { + const { root, read, prototype } = await fixture(t, true); + const originalRead = prototype.read; + const originalOpen = WorkspaceRootAccess.open; + let fileHandle: FileHandle | undefined; + let rootHandle: FileHandle | undefined; + t.mock.method( + WorkspaceRootAccess, + 'open', + async (...args: Parameters) => { + const access = await originalOpen(...args); + rootHandle = access.handle; + return access; + } + ); + t.mock.method( + prototype, + 'read', + async function ( + this: FileHandle, + ...args: Parameters + ) { + fileHandle = this; + return originalRead.apply(this, args); + } + ); + await fs.writeFile(join(root, 'file'), 'x'.repeat(MAX_BYTES + 1)); + await assert.rejects(read(), hasCode('READ_LIMIT_EXCEEDED')); + assert.equal(fileHandle?.fd, -1); + assert.equal(rootHandle?.fd, -1); + t.mock.method(prototype, 'read', async function (this: FileHandle) { + fileHandle = this; + throw Object.assign(new Error('read failed'), { code: 'EIO' }); + }); + await assert.rejects(read(), hasCode('INVALID_PATH')); + assert.equal(fileHandle?.fd, -1); + assert.equal(rootHandle?.fd, -1); +}); + +test('search, preview, edit, and digest-checked instruction limits remain separate', async t => { + const { root, tools } = await fixture(t, true); + await fs.writeFile(join(root, 'file'), 'needle\n'.repeat(200_000)); + const search = await tools.execute({ + protocolVersion: 1, + operation: 'search_text', + workspaceId: 'primary', + path: 'file', + query: 'needle', + }); + assert.equal(search.operation, 'search_text'); + assert.deepEqual(search.matches, []); + for (const operation of ['preview_edit', 'edit_file'] as const) { + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation, + workspaceId: 'primary', + path: 'file', + oldText: 'needle', + newText: 'changed', + }), + hasCode( + operation === 'preview_edit' + ? 'READ_LIMIT_EXCEEDED' + : 'WRITE_LIMIT_EXCEEDED' + ) + ); + } + await fs.writeFile( + join(root, 'AGENTS.md'), + 'instructions\n'.repeat(200_000) + ); + const snapshot = await readRepositoryInstructions(root); + assert.ok(snapshot); + const request: WorkspaceReadFileRequest = { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'AGENTS.md', + instructionSha256: snapshot.descriptor.sha256, + }; + const result = await tools.execute(request); + assert.ok(isWorkspaceToolResult(request, result)); + assert.equal(result.operation, 'read_file'); + assert.equal(result.content, snapshot.content); + assert.equal(result.truncated, true); + assert.equal(result.nextStartLine, undefined); + await fs.writeFile(join(root, 'AGENTS.md'), 'changed'); + await assert.rejects(tools.execute(request), hasCode('INVALID_PATH')); +}); + +test('byte-budget pagination reconstructs every complete line', async t => { + const { root, read } = await fixture(t); + const lines = Array.from( + { length: 1_201 }, + (_, i) => `${i}: ${'é🦊'.repeat(400)}` + ); + await fs.writeFile(join(root, 'file'), lines.join('\n') + '\n'); + const collected: string[] = []; + let startLine = 1; + for (;;) { + const result = await read(startLine, 500); + const selected = result.content.split('\n'); + assert.ok(Buffer.byteLength(result.content) <= MAX_BYTES); + if (result.truncated) + assert.ok(selected.length < 500, 'bytes forced continuation'); + collected.push(...selected); + if (!result.truncated) break; + assert.ok(result.nextStartLine! > startLine); + startLine = result.nextStartLine!; + } + assert.deepEqual(collected, lines); +}); diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index e48365c8..746c7249 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -1,6 +1,6 @@ import { createHash, randomBytes } from 'node:crypto'; import { constants } from 'node:fs'; -import { holdsWorkspaceRoot, link, lstat, mkdir, open, realpath, rename, stat, unlink, spawn, withWorkspaceRoot, WorkspaceRootAccessError } from './root-access.js'; +import { deferWorkspaceRootCleanup, holdsWorkspaceRoot, link, lstat, mkdir, open, realpath, rename, stat, unlink, spawn, withWorkspaceRoot, WorkspaceRootAccessError } from './root-access.js'; import { basename, dirname, isAbsolute, relative, resolve, sep } from 'node:path'; import type { FileHandle } from 'node:fs/promises'; @@ -133,6 +133,8 @@ const MAX_SEARCH_CANDIDATE_BYTES = 1024 * 1024; const MAX_SEARCH_CANDIDATES = 20_000; const SEARCH_TIMEOUT_MS = 10_000; const LIST_TIMEOUT_MS = 10_000; +const READ_TIMEOUT_MS = 10_000; +const READ_CHUNK_BYTES = 64 * 1024; const READ_OPERATIONS = [ 'read_file', @@ -273,10 +275,18 @@ function resolveWorkspacePath(root: string, requestedPath: string): string { return candidate; } -async function readConfinedFileBuffer( +interface WorkspaceReadControl { + signal?: AbortSignal; + deadline: number; + interrupted: boolean; +} + +async function withConfinedFile( root: string, requestedPath: string, -): Promise { + read: (handle: FileHandle, size: number) => Promise, + control?: WorkspaceReadControl, +): Promise { const candidate = resolveWorkspacePath(root, requestedPath); let handle: FileHandle | undefined; try { @@ -297,7 +307,31 @@ async function readConfinedFileBuffer( ) { throw new Error('Invalid workspace path'); } - if (openedFile.size > BRIDGE_WORKSPACE_READ_MAX_BYTES) { + return await read(handle, openedFile.size); + } catch (error) { + if (error instanceof WorkspaceToolError) throw error; + if (isMissingEntry(error)) { + throw await classifyMissingWorkspacePath(root, candidate); + } + throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + } finally { + if (handle) { + const closing = handle.close(); + if (control?.interrupted) { + // Node retains the descriptor and buffer until outstanding I/O drains. + deferWorkspaceRootCleanup(); + void closing.catch(() => {}); + } else await closing; + } + } +} + +async function readConfinedFileBuffer( + root: string, + requestedPath: string, +): Promise { + return withConfinedFile(root, requestedPath, async (handle, size) => { + if (size > BRIDGE_WORKSPACE_READ_MAX_BYTES) { throw new WorkspaceToolError( 'Workspace file exceeds read limit', 'READ_LIMIT_EXCEEDED', @@ -322,31 +356,222 @@ async function readConfinedFileBuffer( ); } return buffer.subarray(0, bytesRead); - } catch (error) { - if (error instanceof WorkspaceToolError) throw error; - if (isMissingEntry(error)) { - throw await classifyMissingWorkspacePath(root, candidate); - } - throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + }); +} + +function interruptRead( + control: WorkspaceReadControl, + message: string, + code: 'EXECUTION_ABORTED' | 'READ_LIMIT_EXCEEDED', +): WorkspaceToolError { + control.interrupted = true; + return new WorkspaceToolError(message, code); +} + +function checkReadDeadline(control: WorkspaceReadControl): void { + if (control.signal?.aborted) { + throw interruptRead( + control, + 'Workspace tool execution aborted', + 'EXECUTION_ABORTED', + ); + } + if (performance.now() >= control.deadline) { + throw interruptRead( + control, + 'Workspace read exceeded its scan time limit; request an earlier startLine or use search_text to locate content', + 'READ_LIMIT_EXCEEDED', + ); + } +} + +async function readWorkspaceChunk( + handle: FileHandle, + buffer: Buffer, + length: number, + position: number, + control: WorkspaceReadControl, +): Promise { + checkReadDeadline(control); + const { signal, deadline } = control; + let timer: ReturnType | undefined; + let abort: (() => void) | undefined; + try { + return await Promise.race([ + handle + .read(buffer, 0, length, position) + .then((result) => result.bytesRead), + new Promise((_, reject) => { + abort = () => + reject( + interruptRead( + control, + 'Workspace tool execution aborted', + 'EXECUTION_ABORTED', + ), + ); + signal?.addEventListener('abort', abort, { once: true }); + if (signal?.aborted) abort(); + timer = setTimeout( + () => + reject( + interruptRead( + control, + 'Workspace read exceeded its scan time limit; retry with an earlier startLine', + 'READ_LIMIT_EXCEEDED', + ), + ), + Math.max(0, deadline - performance.now()), + ); + }), + ]); } finally { - await handle?.close(); + if (timer !== undefined) clearTimeout(timer); + if (abort) signal?.removeEventListener('abort', abort); } } async function readConfinedFile( root: string, - requestedPath: string, -): Promise { - const decoded = decodeWorkspaceText( - await readConfinedFileBuffer(root, requestedPath), - ); - if (Buffer.byteLength(decoded, 'utf8') > BRIDGE_WORKSPACE_READ_MAX_BYTES) { - throw new WorkspaceToolError( - 'Workspace file exceeds read limit', - 'READ_LIMIT_EXCEEDED', + request: WorkspaceReadFileRequest, + signal?: AbortSignal, +): Promise { + const startLine = request.startLine ?? 1; + const maxLines = request.maxLines ?? 200; + const control: WorkspaceReadControl = { + signal, + deadline: performance.now() + READ_TIMEOUT_MS, + interrupted: false, + }; + const read = async ( + handle: FileHandle, + size: number, + ): Promise => { + const buffer = Buffer.allocUnsafe(READ_CHUNK_BYTES); + // Fix the read extent at admission so an appending writer cannot extend the scan. + let position = 0; + const header = Buffer.alloc(3); + let headerBytes = 0; + while (headerBytes < header.length && position < size) { + const bytes = await readWorkspaceChunk( + handle, + buffer, + Math.min(header.length - headerBytes, size - position), + position, + control, + ); + checkReadDeadline(control); + if (bytes === 0) break; + buffer.copy(header, headerBytes, 0, bytes); + headerBytes += bytes; + position += bytes; + } + const utf16le = + headerBytes >= 2 && header[0] === 0xff && header[1] === 0xfe; + const utf16be = + headerBytes >= 2 && header[0] === 0xfe && header[1] === 0xff; + const bomBytes = + utf16le || utf16be + ? 2 + : headerBytes === 3 && + header[0] === 0xef && + header[1] === 0xbb && + header[2] === 0xbf + ? 3 + : 0; + // Preserve ordinary reads' replacement decoding and BOM-marked UTF-16 support. + const decoder = new TextDecoder( + utf16le ? 'utf-16le' : utf16be ? 'utf-16be' : 'utf-8', + { ignoreBOM: true }, ); - } - return decoded; + const lines: string[] = []; + let fragments: string[] = []; + let line = 1; + let lineBytes = 0; + let returnedBytes = 0; + let hasText = false; + let truncated = false; + const consume = (text: string): void => { + let offset = 0; + while (offset < text.length) { + checkReadDeadline(control); + const newline = text.indexOf('\n', offset); + const end = newline < 0 ? text.length : newline; + if (line >= startLine) { + if (lines.length === maxLines) { + truncated = true; + return; + } + const fragment = text.slice(offset, end); + const bytes = Buffer.byteLength(fragment, 'utf8'); + if ( + returnedBytes + (lines.length > 0 ? 1 : 0) + lineBytes + bytes > + BRIDGE_WORKSPACE_READ_MAX_BYTES + ) { + if (lines.length === 0) { + throw new WorkspaceToolError( + `Workspace file exceeds read limit: line ${line} cannot fit within ${BRIDGE_WORKSPACE_READ_MAX_BYTES} UTF-8 bytes; request a later startLine or use search_text`, + 'READ_LIMIT_EXCEEDED', + ); + } + truncated = true; + return; + } + if (fragment.length > 0) fragments.push(fragment); + lineBytes += bytes; + } + hasText ||= end > offset; + if (newline < 0) return; + if (line >= startLine) { + lines.push(fragments.join('')); + returnedBytes += (lines.length > 1 ? 1 : 0) + lineBytes; + fragments = []; + lineBytes = 0; + } + line += 1; + hasText = false; + offset = newline + 1; + } + }; + consume( + decoder.decode(header.subarray(bomBytes, headerBytes), { + stream: true, + }), + ); + while (!truncated && position < size) { + const bytes = await readWorkspaceChunk( + handle, + buffer, + Math.min(buffer.length, size - position), + position, + control, + ); + checkReadDeadline(control); + if (bytes === 0) break; + position += bytes; + consume(decoder.decode(buffer.subarray(0, bytes), { stream: true })); + } + if (!truncated) { + consume(decoder.decode()); + if (!truncated && line >= startLine && (hasText || line === 1)) { + lines.push(fragments.join('')); + } + } + checkReadDeadline(control); + const endLine = startLine + lines.length - 1; + return { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'read_file', + workspaceId: request.workspaceId, + path: request.path, + content: lines.join('\n'), + startLine, + endLine, + truncated, + ...(truncated ? { nextStartLine: endLine + 1 } : {}), + }; + }; + return withConfinedFile(root, request.path, read, control); } interface WorkspaceRoot { @@ -1831,31 +2056,7 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { ) { throw new WorkspaceToolError('Invalid workspace read', 'INVALID_REQUEST'); } - const content = await readConfinedFile(root, request.path); - if (signal?.aborted) { - throw new WorkspaceToolError( - 'Workspace tool execution aborted', - 'EXECUTION_ABORTED', - ); - } - const lines = content.endsWith('\n') - ? content.slice(0, -1).split('\n') - : content.split('\n'); - const selected = lines.slice(startLine - 1, startLine - 1 + maxLines); - const endLine = startLine + selected.length - 1; - const truncated = endLine < lines.length; - - return { - protocolVersion: BRIDGE_PROTOCOL_VERSION, - operation: 'read_file', - workspaceId: request.workspaceId, - path: request.path, - content: selected.join('\n'), - startLine, - endLine, - truncated, - ...(truncated ? { nextStartLine: endLine + 1 } : {}), - }; + return readConfinedFile(root, request, signal); } } From e829f7554c05bfc16c51a698b4a8216b8c027e5c Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 11:41:29 -0400 Subject: [PATCH 31/48] =?UTF-8?q?=F0=9F=97=82=EF=B8=8F=20fix:=20Guard=20La?= =?UTF-8?q?ne=20Worktree=20Registration=20(#304)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🗂️ fix: Guard Lane Worktree Registration * test: Cover Linked Lane Worktree Registration Denials --------- Co-authored-by: Lia --- docs/adr/0010-worktree-metadata-admission.md | 101 ++++++++++++++++++ .../src/linked-worktree-git-guard.test.ts | 13 +++ .../code/src/linked-worktree-git-guard.ts | 6 ++ tests/worktree-metadata-phases.mjs | 23 ++++ 4 files changed, 143 insertions(+) create mode 100644 docs/adr/0010-worktree-metadata-admission.md create mode 100644 tests/worktree-metadata-phases.mjs diff --git a/docs/adr/0010-worktree-metadata-admission.md b/docs/adr/0010-worktree-metadata-admission.md new file mode 100644 index 00000000..4976adc3 --- /dev/null +++ b/docs/adr/0010-worktree-metadata-admission.md @@ -0,0 +1,101 @@ +# Target-Lane Worktree Lifecycle Admission + +Status: proposed. No scheduling or sandbox policy changes are enabled by this PR. + +## Failure + +A shell call to `git worktree add` currently uses the checkout isolation key. +`BridgeWorkspaceSlots.reserve` waits for every child lane; the worker's lane +family guard and native sandbox enforce the same exclusion. Repeated short +admission timeouts discard a waiting checkout request before lanes drain. +The reviewer in chat e2fedf18 (LibreChat PR #16722) could not create its lane. +Durable admission preserves FIFO, but does not remove the checkout-wide barrier. + +## Decision + +Add explicit `create_worktree` and `remove_worktree` workspace operations. +Never infer a weaker scope by parsing an arbitrary shell command. +Each lifecycle request reserves the target lane's ordinary isolation key, +including checkout exclusion and existing quarantine fencing. Sibling lanes +remain independent. Advertise support only when the service and worker agree. +Old clients keep using checkout-wide shell calls. + +Git metadata mutation gets a separate worker-owned mutex, scoped to the common +Git directory and held only while changing a worktree registration. The target +reservation covers the whole operation. It is not the metadata mutex. + +```text +create(name, frozenCommit) + target-lane reservation + validate root identity, target absence, commit and branch policy + metadata mutex → git worktree add --no-checkout --detach → unlock + target lane → git checkout frozenCommit → verify pointers and root identity + publish ready result, then release reservation + +remove(name) + target-lane reservation, require no mutation quarantine + verify pointers and identity, clean tracked/untracked state, no nested mounts + metadata mutex → rename lane to private tombstone + → git worktree remove original-path (now absent) → unlock + delete tombstone outside mutex; keep branch and reachable objects + persist outcome, then release reservation +``` + +A lifecycle journal outside untrusted roots records `reserved`, `registered`, +`ready`, `retiring`, `detached` and `removed`, with assignment owner and captured +identities. Worker recovery reconciles journal entries, never reruns uncertain +shell commands. Creation is not published until checkout completes. Failed +checkout retains the target fence until reconciled. Rename-before-detach must +roll back only while registration still points to the captured original target; +otherwise preserve the tombstone and require reconciliation. Never remove dirty +worktrees, force-remove, prune all registrations, or delete branches as recovery. + +## Boundaries + +- `service/src/bridge/slots.ts`: reserve the named target, not a new global slot + limit. A waiting root still prevents newer sibling admissions. A metadata + operation consumes an existing worker slot; fully occupied workers still wait. +- `packages/code/src/worker.ts`: treat lifecycle operations as mutations and + derive target-lane guards before execution. Keep assignment/identity fencing. +- `packages/code/src/linked-worktrees.ts`: target ownership includes absent + lanes during creation and excludes local retirement of that lane. +- `packages/code/src/linked-worktree-git-guard.ts`: ordinary lanes reject + registration-changing `git worktree` commands. Do not grant write access to + `.git/worktrees` or sibling checkouts to make shell operations work. +- Native sandbox: bulk checkout runs only in the target's admitted directory. + Run no repo hooks or credential helpers in the trusted metadata phase. + +## Alternatives + +1. Longer checkout waits: smallest fix and required for old servers, but still + drains all lanes and serializes bulk checkout/deletion. +2. Allow `git worktree` from arbitrary lane shells: rejected. A shell can remove + a busy sibling or repair foreign metadata; mutexing Git does not confine it. +3. Explicit lifecycle operations: recommended. More protocol/recovery surface, + but target ownership and metadata lifetime are independently testable. + +## Compatibility and rollout + +Deploy the worker journal/guard first, then service capability and matching +client tools. Never send target-scoped lifecycle work to an older worker. +Disable new submissions and drain lifecycle journals before rollback. Root +shell operations and destructive object maintenance remain checkout-exclusive. +Concurrent Git object/ref access still uses Git's own atomic locks; automatic +maintenance stays disabled in lanes. + +A read-only GitHub compare tool supplies merge-base and ancestry for public +repos without any worker call. This helps review orient while admission is busy, +but does not substitute for reading the frozen source or running tests. + +## Blocking verification + +- A long sibling lane remains running while another target is created/removed. +- Same-target/root exclusion and older-root FIFO barriers in Redis and worker. +- Short mutex lifetime verified while bulk checkout/deletion is deliberately gated. +- Cancellation/restart at each journal transition and lost acknowledgements. +- Dirty/untracked, symlink substitution, nested mount, quarantine and foreign-root denial. +- Lane Git guard rejects add/remove/move/repair/prune, including global options. +- Old/new service, worker and client combinations fail closed. + +`tests/worktree-metadata-phases.mjs` probes the local Git mechanism only. It does +not prove service scheduling, worker recovery, or live sandbox containment. diff --git a/packages/code/src/linked-worktree-git-guard.test.ts b/packages/code/src/linked-worktree-git-guard.test.ts index 96f16b4a..96386418 100644 --- a/packages/code/src/linked-worktree-git-guard.test.ts +++ b/packages/code/src/linked-worktree-git-guard.test.ts @@ -274,3 +274,16 @@ test('Git LFS fetch and pull cannot request pruning in a lane', async t => { assert.equal((await run(['cat-file', '-t', object])).stdout.trim(), 'blob'); } }); + +test('lane guard rejects registration changes but permits read-only worktree listing', async t => { + const parent = await mkdtemp(join(tmpdir(), 'lane-registration-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const bin = join(parent, 'bin'); + await mkdir(bin); await writeLinkedWorktreeGitGuard(bin); + await execFileAsync('/usr/bin/git', ['init', '-q', '-b', 'main'], { cwd: parent }); + const run = (args: string[]) => execFileAsync(join(bin, 'git'), args, { cwd: parent }); + assert.match((await run(['worktree', 'list', '--porcelain'])).stdout, /worktree/); + for (const operation of ['add', 'remove', 'move', 'repair', 'prune', 'lock', 'unlock']) { + await assert.rejects(run(['-C', parent, 'worktree', operation, '.worktrees/other']), /require checkout admission/); + } +}); diff --git a/packages/code/src/linked-worktree-git-guard.ts b/packages/code/src/linked-worktree-git-guard.ts index 227bfd3f..cee4a7f0 100644 --- a/packages/code/src/linked-worktree-git-guard.ts +++ b/packages/code/src/linked-worktree-git-guard.ts @@ -72,6 +72,12 @@ export async function writeLinkedWorktreeGitGuard(directory: string): Promise&2', + ' return 1', + ' fi ;;', + ' prune|gc|repack|prune-packed|maintenance|multi-pack-index|for-each-repo)', ' echo "git: run storage maintenance from the checkout, not a linked worktree lane" >&2', ' return 1 ;;', diff --git a/tests/worktree-metadata-phases.mjs b/tests/worktree-metadata-phases.mjs new file mode 100644 index 00000000..089251b9 --- /dev/null +++ b/tests/worktree-metadata-phases.mjs @@ -0,0 +1,23 @@ +import assert from 'node:assert/strict'; +import { mkdtemp,writeFile,mkdir,rename,rm } from 'node:fs/promises'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +const exec=promisify(execFile),root=await mkdtemp(join(tmpdir(),'metadata-phases-')); +try { + const git=(args,cwd=root)=>exec('git',args,{cwd,env:{...process.env,GIT_CONFIG_NOSYSTEM:'1'}}); + await git(['init','-q','-b','main']); await writeFile(join(root,'file'),'tracked\n'); await git(['add','file']); + await git(['-c','user.name=test','-c','user.email=test@example.test','-c','commit.gpgsign=false','commit','-qm','seed']); + const commit=(await git(['rev-parse','HEAD'])).stdout.trim(); await mkdir(join(root,'.worktrees')); + const lane=join(root,'.worktrees','review'); + await git(['worktree','add','--no-checkout','--detach',lane,commit]); + await git(['-c','core.hooksPath=/dev/null','checkout','--detach',commit],lane); + assert.equal((await git(['status','--porcelain'],lane)).stdout,''); + const tombstone=join(root,'.retired-review'); await rename(lane,tombstone); + await git(['worktree','remove',lane]); + assert.doesNotMatch((await git(['worktree','list','--porcelain'])).stdout,/review/); + // Registration removal did not delete the detached bulk directory. + await rm(tombstone,{recursive:true}); + console.log('Split-phase registration/checkout and rename/detach/delete: passed'); +} finally { await rm(root,{recursive:true,force:true}); } From 829992a4d837da67737edcdc3c5b57eaaa5c3b3d Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Sun, 4 Oct 2026 12:41:15 -0400 Subject: [PATCH 32/48] =?UTF-8?q?=F0=9F=9B=A1=EF=B8=8F=20fix:=20Harden=20S?= =?UTF-8?q?andbox=20Startup=20and=20Internal=20Transports=20(#305)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Lia --- api/src/nsjail.test.ts | 16 +++ api/src/nsjail.ts | 10 ++ docs/SECURITY-MAINTENANCE.md | 63 +++++++++++ launcher/src/main.rs | 46 ++++++-- service/bun.lock | 102 +++++++++++------- service/package.json | 10 +- service/src/egress-gateway-client.test.ts | 3 + service/src/egress-gateway-client.ts | 4 + service/src/egress-gateway.test.ts | 37 +++++++ service/src/egress-gateway.ts | 11 +- service/src/hosted-app/microvm-runtime.ts | 3 + service/src/internal-redirect-policy.test.ts | 56 ++++++++++ service/src/minio-compat.test.ts | 39 +++++++ service/src/runtime-session/checkpoint.ts | 4 + service/src/runtime-session/files.ts | 1 + service/src/sandbox-backend/http.test.ts | 24 ++++- service/src/sandbox-backend/http.ts | 1 + service/src/sandbox-backend/lambda-microvm.ts | 2 + service/src/service/programmatic-router.ts | 6 +- service/src/service/replay-state.ts | 2 + service/src/service/router.ts | 7 +- test-sandbox.sh | 24 ++++- 22 files changed, 400 insertions(+), 71 deletions(-) create mode 100644 docs/SECURITY-MAINTENANCE.md create mode 100644 service/src/internal-redirect-policy.test.ts create mode 100644 service/src/minio-compat.test.ts diff --git a/api/src/nsjail.test.ts b/api/src/nsjail.test.ts index ee6fbbc8..95de6251 100644 --- a/api/src/nsjail.test.ts +++ b/api/src/nsjail.test.ts @@ -374,6 +374,22 @@ describe('NsJail seccomp policy', () => { expect(errnoSocketRule).not.toContain('AF_VSOCK'); }); + test('blocks priority-inheritance futex operations without denying ordinary wait/wake', () => { + const policy = seccompPolicy(); + expect(policy).toContain('#define FUTEX_CMD_MASK 0x7f'); + const rule = policy.split('ERRNO(1)')[1]?.split('\n') + .find(line => line.includes('futex(uaddr, op)')); + expect(rule).toBeDefined(); + for (const [operation, command] of [ + ['LOCK_PI', 6], ['UNLOCK_PI', 7], ['TRYLOCK_PI', 8], + ['WAIT_REQUEUE_PI', 11], ['CMP_REQUEUE_PI', 12], ['LOCK_PI2', 13], + ] as const) { + expect(policy).toContain(`#define FUTEX_${operation} ${command}`); + expect(rule).toContain(`(op & FUTEX_CMD_MASK) == FUTEX_${operation}`); + } + expect(rule).not.toMatch(/== FUTEX_(WAIT|WAKE)(?:\s|$)/); + }); + test('rejects Copy Fail and Dirty Frag socket entry points in the sandbox', () => { const policy = seccompPolicy(); const errnoBlock = policy.split('ERRNO(1)')[1] ?? ''; diff --git a/api/src/nsjail.ts b/api/src/nsjail.ts index a9f02931..3a5de4ad 100644 --- a/api/src/nsjail.ts +++ b/api/src/nsjail.ts @@ -104,6 +104,13 @@ const SECCOMP_POLICY = [ '#define AF_VSOCK 40', '#define CLONE_NAMESPACE_FLAGS 0x7e020000', '#define KVM_IOCTL_MAGIC 0xAE00', + '#define FUTEX_CMD_MASK 0x7f', + '#define FUTEX_LOCK_PI 6', + '#define FUTEX_UNLOCK_PI 7', + '#define FUTEX_TRYLOCK_PI 8', + '#define FUTEX_WAIT_REQUEUE_PI 11', + '#define FUTEX_CMP_REQUEUE_PI 12', + '#define FUTEX_LOCK_PI2 13', 'POLICY sandbox {', ' KILL {', ' ptrace, memfd_create, personality, userfaultfd,', @@ -141,6 +148,9 @@ const SECCOMP_POLICY = [ ' },', ' ERRNO(1) {', ' io_uring_setup, io_uring_enter, io_uring_register, sched_setaffinity, vmsplice,', + /* GhostLock (CVE-2026-43499): deny PI operations, including private and + * realtime variants, without blocking ordinary runtime futex wait/wake. */ + ' futex(uaddr, op) { (op & FUTEX_CMD_MASK) == FUTEX_LOCK_PI || (op & FUTEX_CMD_MASK) == FUTEX_UNLOCK_PI || (op & FUTEX_CMD_MASK) == FUTEX_TRYLOCK_PI || (op & FUTEX_CMD_MASK) == FUTEX_WAIT_REQUEUE_PI || (op & FUTEX_CMD_MASK) == FUTEX_CMP_REQUEUE_PI || (op & FUTEX_CMD_MASK) == FUTEX_LOCK_PI2 },', ' clone(flags) { (flags & CLONE_NAMESPACE_FLAGS) != 0 },', /* Block signals to PID 1 of the sandbox PID namespace (the NsJail * monitor). With clone_newpid the user can't reach other tenants — but diff --git a/docs/SECURITY-MAINTENANCE.md b/docs/SECURITY-MAINTENANCE.md new file mode 100644 index 00000000..24c1ee01 --- /dev/null +++ b/docs/SECURITY-MAINTENANCE.md @@ -0,0 +1,63 @@ +# Sandbox security maintenance + +The default boundary is NsJail inside a libkrun guest. Direct NsJail mode +shares the host kernel and is for trusted development, not hostile workloads. + +## Verify deployed patches + +Repository dependencies do not establish production kernel patch status. +Inventory each runner node and deployed runner image: + +- Host OS and running kernel release (`uname -r` on the node). +- Container runtime version and runner image digest. +- Installed `libkrun` and `libkrunfw` packages + (`rpm -q libkrun libkrunfw` inside the runner image). +- Bundled guest kernel release, from an operator-controlled guest inspection. +- Guest-visible nested virtualization features and exposed virtual devices. + +Match kernel packages against distro security advisories, including backports. +Patch both the host kernel and bundled guest kernel. Rebuild and replace runner +images after guest-kernel updates; reboot or use the vendor-supported livepatch +procedure for host fixes. A container rebuild does not update host KVM. + +Priority 2026 checks: + +| Boundary | Advisory | +|---|---| +| Guest kernel; host kernel in direct mode | Bad Epoll, CVE-2026-46242 | +| Guest and host kernels | GhostLock, CVE-2026-43499 | +| x86 host KVM | CVE-2026-46113, CVE-2026-53359, CVE-2026-64561 | +| ARM host KVM, when virtual ITS is exposed | CVE-2026-46316 | + +PI-futex filtering reduces the GhostLock surface but does not replace patches. +Do not disable ordinary futex or epoll operations: language runtimes use them. +Do not expose nested virtualization to untrusted guests. Confirm the installed +VMM's behavior rather than assuming current upstream defaults match its build. + +The October 2026 Vercel KVM report has no disclosed root cause or affected-version +matrix at the time of this change. These controls are not a verified fix for it. + +## Preserve host-side confinement + +- Run sandbox runners on isolated nodes, separate from credential-bearing + control-plane workloads, with least-privilege node IAM. +- Keep hardened startup, signed manifests, egress controls, network policies, + and disabled runner service-account token mounting. +- The launcher refuses to boot if its host-side seccomp filter cannot install. +- Internal authenticated HTTP requests reject redirects. Configure their final + endpoints directly; do not depend on HTTP-to-HTTPS redirects. + +## Dependency checks + +Run `bun audit --cwd api` and `bun audit --cwd service` against the lockfiles. +Distinguish development dependencies from production paths and establish the +advisory's prerequisites before claiming an application exploit. Do not force +incompatible transitive major upgrades solely to obtain a clean audit. + +Remaining service audit findings at this change: + +- `stream-json` (GHSA-528h-pc64-c93x): MinIO 8 uses its JSON-lines parser, + not the affected path filters. Keep MinIO's supported major dependency; + reassess if notifications or path filters become reachable. +- `braces` (GHSA-vfj7-8cjw-p6xm): development-only dependency with no published + fix. Do not feed untrusted patterns to build tooling; track an upstream fix. diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 27f08f92..84e48347 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -570,6 +570,16 @@ fn guest_cmdline_problem(env: &[(String, String)], args: &[String]) -> Option, + enter: impl FnOnce() -> i32, +) -> Result { + let allowed = filter_result?; + eprintln!("[launcher] VMM seccomp filter applied ({allowed} syscalls allowed, ioctl restricted to KVM+terminal)"); + eprintln!("[launcher] Starting microVM..."); + Ok(enter()) +} + fn main() { let vcpus: u8 = env::var("LAUNCHER_VCPUS") .ok() @@ -701,16 +711,13 @@ fn main() { ffi::check("set_rlimits", ffi::krun_set_rlimits(ctx, rlimit_ptrs.as_ptr())); eprintln!("[launcher] Guest RLIMIT_NOFILE target: {nofile_target}"); - match seccomp::apply_vmm_filter() { - Ok(n) => eprintln!("[launcher] VMM seccomp filter applied ({n} syscalls allowed, ioctl restricted to KVM+terminal)"), - Err(e) => { - eprintln!("[launcher] WARNING: Failed to apply VMM seccomp: {e}"); + let ret = match enter_microvm(seccomp::apply_vmm_filter(), || ffi::krun_start_enter(ctx)) { + Ok(ret) => ret, + Err(error) => { + eprintln!("[launcher] ERROR: refusing to start microVM without VMM seccomp: {error}"); + process::exit(1); } - } - - eprintln!("[launcher] Starting microVM..."); - - let ret = ffi::krun_start_enter(ctx); + }; eprintln!("[launcher] krun_start_enter returned {ret} (should not return on success)"); process::exit(1); } @@ -719,7 +726,7 @@ fn main() { #[cfg(test)] mod tests { use super::{ - desired_nofile_soft_limit, guest_cmdline_problem, guest_nofile_rlimit, + desired_nofile_soft_limit, enter_microvm, guest_cmdline_problem, guest_nofile_rlimit, is_allowed_guest_env_key, GUEST_CMDLINE_LIMIT, LIBKRUN_CMDLINE_RESERVE, }; @@ -738,6 +745,25 @@ mod tests { .collect() } + #[test] + fn microvm_does_not_start_when_seccomp_installation_fails() { + let result = enter_microvm(Err("filter installation denied".into()), || { + panic!("microVM entry must not run without seccomp"); + }); + assert_eq!(result, Err("filter installation denied".into())); + } + + #[test] + fn microvm_starts_after_seccomp_installation_succeeds() { + let mut entered = false; + let result = enter_microvm(Ok(80), || { + entered = true; + -1 + }); + assert!(entered); + assert_eq!(result, Ok(-1)); + } + #[test] fn guest_cmdline_accepts_single_line_printable_env() { let env = env(&[ diff --git a/service/bun.lock b/service/bun.lock index 0a0dea4a..9aacef7e 100644 --- a/service/bun.lock +++ b/service/bun.lock @@ -12,14 +12,14 @@ "@opentelemetry/exporter-trace-otlp-proto": "0.219.0", "@opentelemetry/resources": "2.8.0", "@opentelemetry/sdk-trace-base": "2.8.0", - "axios": "^1.8.4", + "axios": "^1.20.0", "bullmq": "^5.13.1", "busboy": "^1.6.0", "dotenv": "^17.4.2", - "express": "^4.19.2", + "express": "^4.22.3", "express-rate-limit": "^7.4.1", "ioredis": "^5.4.1", - "minio": "^8.0.5", + "minio": "^8.0.7", "nanoid": "^5.1.16", "prom-client": "^15.1.3", "rate-limit-redis": "^4.2.0", @@ -52,6 +52,8 @@ }, "overrides": { "decode-uri-component": "0.5.0", + "qs": "6.16.0", + "uuid": "11.1.1", }, "packages": { "@aws-sdk/checksums": ["@aws-sdk/checksums@3.1000.19", "", { "dependencies": { "@aws-sdk/core": "^3.976.0", "@aws-sdk/types": "^3.974.2", "@smithy/core": "^3.29.4", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-Hc4N100RdkuWshKBnhPzmpdftfi9mCLz+OHFELHM1QIgMH4QRUUWyWgfiebta/YX2Bd62wTcm3EqAP8TeXv0gA=="], @@ -142,6 +144,8 @@ "@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@0.2.12", "", { "dependencies": { "@emnapi/core": "^1.4.3", "@emnapi/runtime": "^1.4.3", "@tybys/wasm-util": "^0.10.0" } }, "sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ=="], + "@nodable/entities": ["@nodable/entities@3.1.0", "", {}, "sha512-LsS/DjHr+uDM647Gru/cA8+J3a3HfhttwCKLyuoyN7yXTFCxKBtSgMQBvrW9yNPa2/zDRUNuGPaH5QUFoa4arQ=="], + "@nodelib/fs.scandir": ["@nodelib/fs.scandir@2.1.5", "", { "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" } }, "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g=="], "@nodelib/fs.stat": ["@nodelib/fs.stat@2.0.5", "", {}, "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A=="], @@ -312,8 +316,6 @@ "@unrs/resolver-binding-win32-x64-msvc": ["@unrs/resolver-binding-win32-x64-msvc@1.11.1", "", { "os": "win32", "cpu": "x64" }, "sha512-lrW200hZdbfRtztbygyaq/6jP6AKE8qQN2KvPcJ+x7wiD038YtnYtZ82IMNJ69GJibV7bwL3y9FgK+5w/pYt6g=="], - "@zxing/text-encoding": ["@zxing/text-encoding@0.9.0", "", {}, "sha512-U/4aVJ2mxI0aDNI8Uq0wEhMgY+u4CNtEb0om3+y3+niDAsoTCOB33UF0sxpzqzdqXLqmvc+vZyAt4O8pPdfkwA=="], - "accepts": ["accepts@1.3.8", "", { "dependencies": { "mime-types": "~2.1.34", "negotiator": "0.6.3" } }, "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw=="], "acorn": ["acorn@8.15.0", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg=="], @@ -322,12 +324,16 @@ "acorn-walk": ["acorn-walk@8.3.4", "", { "dependencies": { "acorn": "^8.11.0" } }, "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g=="], - "ajv": ["ajv@6.12.6", "", { "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", "json-schema-traverse": "^0.4.1", "uri-js": "^4.2.2" } }, "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g=="], + "agent-base": ["agent-base@6.0.2", "", { "dependencies": { "debug": "4" } }, "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ=="], + + "ajv": ["ajv@6.14.0", "", { "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", "json-schema-traverse": "^0.4.1", "uri-js": "^4.2.2" } }, "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw=="], "ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], "ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], + "anynum": ["anynum@1.0.1", "", {}, "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A=="], + "arg": ["arg@4.1.3", "", {}, "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA=="], "argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], @@ -358,7 +364,7 @@ "available-typed-arrays": ["available-typed-arrays@1.0.7", "", { "dependencies": { "possible-typed-array-names": "^1.0.0" } }, "sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ=="], - "axios": ["axios@1.13.2", "", { "dependencies": { "follow-redirects": "^1.15.6", "form-data": "^4.0.4", "proxy-from-env": "^1.1.0" } }, "sha512-VPk9ebNqPcy5lRGuSlKx752IlDatOjT9paPlm8A7yOuW2Fbvp4X3JznJtT4f0GzGLLiWE9W8onz51SqLYwzGaA=="], + "axios": ["axios@1.20.0", "", { "dependencies": { "follow-redirects": "^1.16.0", "form-data": "^4.0.6", "https-proxy-agent": "^5.0.1", "proxy-from-env": "^2.1.0" } }, "sha512-r8aOh8j9cGKpgQAqpzrUHnSIc6a59Y3Xf/cv8sy1DrHCkZHzQGEuoq1tARk6qSyDdtQGSDgpb9kFlruzPvrgwg=="], "balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], @@ -366,11 +372,11 @@ "block-stream2": ["block-stream2@2.1.0", "", { "dependencies": { "readable-stream": "^3.4.0" } }, "sha512-suhjmLI57Ewpmq00qaygS8UgEq2ly2PCItenIyhMqVjo4t4pGzqMvfgJuX8iWTeSDdfSSqS6j38fL4ToNL7Pfg=="], - "body-parser": ["body-parser@1.20.4", "", { "dependencies": { "bytes": "~3.1.2", "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", "destroy": "~1.2.0", "http-errors": "~2.0.1", "iconv-lite": "~0.4.24", "on-finished": "~2.4.1", "qs": "~6.14.0", "raw-body": "~2.5.3", "type-is": "~1.6.18", "unpipe": "~1.0.0" } }, "sha512-ZTgYYLMOXY9qKU/57FAo8F+HA2dGX7bqGc71txDRC1rS4frdFI5R7NhluHxH6M0YItAP0sHB4uqAOcYKxO6uGA=="], + "body-parser": ["body-parser@1.20.6", "", { "dependencies": { "bytes": "~3.1.2", "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", "destroy": "~1.2.0", "http-errors": "~2.0.1", "iconv-lite": "~0.4.24", "on-finished": "~2.4.1", "qs": "~6.15.1", "raw-body": "~2.5.3", "type-is": "~1.6.18", "unpipe": "~1.0.0" } }, "sha512-p5tAzS57i5MV9fZFDj9LeIiTZEufbSe2eDozP+ElheSUq1m74CRq1jI4mYNDdVs9vQztXFLuk/Gd6BWTdwRJ5g=="], "bowser": ["bowser@2.14.1", "", {}, "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg=="], - "brace-expansion": ["brace-expansion@1.1.12", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg=="], + "brace-expansion": ["brace-expansion@1.1.21", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-9zeA+KLZNNzglF2TPKRQEDyx6Yby7daAkuy8MiPzpXPsYDWi/DRM8jmwUDxokQjYqBpv5DgPiwD4h4ZZSy1Ujw=="], "braces": ["braces@3.0.3", "", { "dependencies": { "fill-range": "^7.1.1" } }, "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA=="], @@ -456,7 +462,7 @@ "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], - "diff": ["diff@4.0.2", "", {}, "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A=="], + "diff": ["diff@4.0.4", "", {}, "sha512-X07nttJQkwkfKfvTPG/KSnE2OMdcUCao6+eXF3wmnIQRn2aPAHH3VxDbDOdegkd6JbPsXqShpvEOHfAT+nCNwQ=="], "dir-glob": ["dir-glob@3.0.1", "", { "dependencies": { "path-type": "^4.0.0" } }, "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA=="], @@ -520,7 +526,7 @@ "eventemitter3": ["eventemitter3@5.0.1", "", {}, "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA=="], - "express": ["express@4.22.1", "", { "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", "body-parser": "~1.20.3", "content-disposition": "~0.5.4", "content-type": "~1.0.4", "cookie": "~0.7.1", "cookie-signature": "~1.0.6", "debug": "2.6.9", "depd": "2.0.0", "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "etag": "~1.8.1", "finalhandler": "~1.3.1", "fresh": "~0.5.2", "http-errors": "~2.0.0", "merge-descriptors": "1.0.3", "methods": "~1.1.2", "on-finished": "~2.4.1", "parseurl": "~1.3.3", "path-to-regexp": "~0.1.12", "proxy-addr": "~2.0.7", "qs": "~6.14.0", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", "send": "~0.19.0", "serve-static": "~1.16.2", "setprototypeof": "1.2.0", "statuses": "~2.0.1", "type-is": "~1.6.18", "utils-merge": "1.0.1", "vary": "~1.1.2" } }, "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g=="], + "express": ["express@4.22.3", "", { "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", "body-parser": "~1.20.5", "content-disposition": "~0.5.4", "content-type": "~1.0.4", "cookie": "~0.7.1", "cookie-signature": "~1.0.6", "debug": "2.6.9", "depd": "2.0.0", "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "etag": "~1.8.1", "finalhandler": "~1.3.1", "fresh": "~0.5.2", "http-errors": "~2.0.0", "merge-descriptors": "1.0.3", "methods": "~1.1.2", "on-finished": "~2.4.1", "parseurl": "~1.3.3", "path-to-regexp": "~0.1.13", "proxy-addr": "~2.0.7", "qs": "~6.16.0", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", "send": "~0.19.0", "serve-static": "~1.16.2", "setprototypeof": "1.2.0", "statuses": "~2.0.1", "type-is": "~1.6.18", "utils-merge": "1.0.1", "vary": "~1.1.2" } }, "sha512-Bdcs4+3qlpVlx2NRn6fgX2Ue2/gGRaPeawebgclM0ERSCqDpA+owF1fdPwjJUTAJWMTuAaxjDf+hzb0/4eKvvw=="], "express-rate-limit": ["express-rate-limit@7.5.1", "", { "peerDependencies": { "express": ">= 4.11" } }, "sha512-7iN8iPMDzOMHPUYllBEsQdWVB6fPDMPqwjBaFrgr4Jgr/+okjvzAy+UHlYYL/Vs0OsOrMkwS6PJDkFlJwoxUnw=="], @@ -532,7 +538,9 @@ "fast-levenshtein": ["fast-levenshtein@2.0.6", "", {}, "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw=="], - "fast-xml-parser": ["fast-xml-parser@4.5.3", "", { "dependencies": { "strnum": "^1.1.1" }, "bin": { "fxparser": "src/cli/cli.js" } }, "sha512-RKihhV+SHsIUGXObeVy9AXiBbFwkVk7Syp8XgwN5U3JV416+Gwp/GO9i0JYKmikykgz/UHRrrV4ROuZEo/T0ig=="], + "fast-xml-builder": ["fast-xml-builder@1.3.1", "", { "dependencies": { "path-expression-matcher": "^1.6.2", "xml-naming": "^0.3.0" } }, "sha512-pIM/1n3ntFXKYrUZwW7QCK0gAW7XY+wzj1YMIV3tLDvPj/V+zTGJK5e3/4WJfwj0qWw2ElNXiTixda/R+3YSug=="], + + "fast-xml-parser": ["fast-xml-parser@5.11.2", "", { "dependencies": { "@nodable/entities": "^3.0.1", "fast-xml-builder": "^1.2.0", "is-unsafe": "^2.0.0", "path-expression-matcher": "^1.6.2", "strnum": "^2.4.2", "xml-naming": "^0.3.0" }, "bin": { "fxparser": "src/cli/cli.js" } }, "sha512-R9iDuNrQYeQut46cn2r2wHKn4HYzVDvDm5J1wW+koZewykv0yuO2HChTYeZtrULyHIDM9cj9TUXloCsueCQUog=="], "fastq": ["fastq@1.19.1", "", { "dependencies": { "reusify": "^1.0.4" } }, "sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ=="], @@ -556,15 +564,15 @@ "flat-cache": ["flat-cache@3.2.0", "", { "dependencies": { "flatted": "^3.2.9", "keyv": "^4.5.3", "rimraf": "^3.0.2" } }, "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw=="], - "flatted": ["flatted@3.3.3", "", {}, "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg=="], + "flatted": ["flatted@3.4.2", "", {}, "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA=="], "fn.name": ["fn.name@1.1.0", "", {}, "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw=="], - "follow-redirects": ["follow-redirects@1.15.11", "", {}, "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ=="], + "follow-redirects": ["follow-redirects@1.16.0", "", { "peerDependencies": { "debug": "*" }, "optionalPeers": ["debug"] }, "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw=="], "for-each": ["for-each@0.3.5", "", { "dependencies": { "is-callable": "^1.2.7" } }, "sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg=="], - "form-data": ["form-data@4.0.5", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "hasown": "^2.0.2", "mime-types": "^2.1.12" } }, "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w=="], + "form-data": ["form-data@4.0.6", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "hasown": "^2.0.4", "mime-types": "^2.1.35" } }, "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ=="], "forwarded": ["forwarded@0.2.0", "", {}, "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="], @@ -624,6 +632,8 @@ "http-errors": ["http-errors@2.0.1", "", { "dependencies": { "depd": "~2.0.0", "inherits": "~2.0.4", "setprototypeof": "~1.2.0", "statuses": "~2.0.2", "toidentifier": "~1.0.1" } }, "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ=="], + "https-proxy-agent": ["https-proxy-agent@5.0.1", "", { "dependencies": { "agent-base": "6", "debug": "4" } }, "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA=="], + "iconv-lite": ["iconv-lite@0.4.24", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3" } }, "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA=="], "ignore": ["ignore@7.0.5", "", {}, "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg=="], @@ -644,8 +654,6 @@ "ipaddr.js": ["ipaddr.js@2.3.0", "", {}, "sha512-Zv/pA+ciVFbCSBBjGfaKUya/CcGmUHzTydLMaTwrUUEM2DIEO3iZvueGxmacvmN50fGpGVKeTXpb2LcYQxeVdg=="], - "is-arguments": ["is-arguments@1.2.0", "", { "dependencies": { "call-bound": "^1.0.2", "has-tostringtag": "^1.0.2" } }, "sha512-7bVbi0huj/wrIAOzb8U1aszg9kdi3KN/CyU19CTI7tAoZYEZoL9yCDXpbXN+uPsuWnP02cyug1gleqq+TU+YCA=="], - "is-array-buffer": ["is-array-buffer@3.0.5", "", { "dependencies": { "call-bind": "^1.0.8", "call-bound": "^1.0.3", "get-intrinsic": "^1.2.6" } }, "sha512-DDfANUiiG2wC1qawP66qlTugJeL5HyzMpfr8lLK+jMQirGzNod0B12cFB/9q838Ru27sBwfw78/rdoU7RERz6A=="], "is-async-function": ["is-async-function@2.1.1", "", { "dependencies": { "async-function": "^1.0.0", "call-bound": "^1.0.3", "get-proto": "^1.0.1", "has-tostringtag": "^1.0.2", "safe-regex-test": "^1.1.0" } }, "sha512-9dgM/cZBnNvjzaMYHVoxxfPj2QXt22Ev7SuuPrs+xav0ukGB0S6d4ydZdEiM48kLx5kDV+QBPrpVnFyefL8kkQ=="], @@ -702,6 +710,8 @@ "is-typed-array": ["is-typed-array@1.1.15", "", { "dependencies": { "which-typed-array": "^1.1.16" } }, "sha512-p3EcsicXjit7SaskXHs1hA91QxgTw46Fv6EFKKGS5DRFLD8yKnohjF3hxoju94b/OcMZoQukzpPpBE9uLVKzgQ=="], + "is-unsafe": ["is-unsafe@2.0.2", "", {}, "sha512-HgbIHPBH0KHHCcjLfGsCvhtPTVxjaAZlXjwdz7/GQC40SjSe4sfQsar8J5VFo8JOSbarkpV0OLG95bbaNd9aAQ=="], + "is-weakmap": ["is-weakmap@2.0.2", "", {}, "sha512-K5pXYOm9wqY1RgjpL3YTkF39tni1XajUIkawTLUo9EZEVUFga5gSQJF8nNS7ZwJQ02y+1YCNYcMh+HIf1ZqE+w=="], "is-weakref": ["is-weakref@1.1.1", "", { "dependencies": { "call-bound": "^1.0.3" } }, "sha512-6i9mGWSlqzNMEqpCp93KwRS1uUOodk2OJ6b+sq7ZPDSy2WuI5NFIxp/254TytR8ftefexkWn5xNiHUNpPOfSew=="], @@ -712,7 +722,7 @@ "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], - "js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], + "js-yaml": ["js-yaml@4.3.2", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA=="], "json-buffer": ["json-buffer@3.0.1", "", {}, "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="], @@ -732,7 +742,7 @@ "locate-path": ["locate-path@6.0.0", "", { "dependencies": { "p-locate": "^5.0.0" } }, "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw=="], - "lodash": ["lodash@4.17.21", "", {}, "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="], + "lodash": ["lodash@4.18.0", "", {}, "sha512-l1mfj2atMqndAHI3ls7XqPxEjV2J9ZkcNyHpoZA3r2T1LLwDB69jgkMWh71YKwhBbK0G2f4WSn05ahmQXVxupA=="], "lodash.defaults": ["lodash.defaults@4.2.0", "", {}, "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ=="], @@ -766,11 +776,11 @@ "mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], - "minimatch": ["minimatch@3.1.2", "", { "dependencies": { "brace-expansion": "^1.1.7" } }, "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw=="], + "minimatch": ["minimatch@3.1.4", "", { "dependencies": { "brace-expansion": "^1.1.7" } }, "sha512-twmL+S8+7yIsE9wsqgzU3E8/LumN3M3QELrBZ20OdmQ9jB2JvW5oZtBEmft84k/Gs5CG9mqtWc6Y9vW+JEzGxw=="], "minimist": ["minimist@1.2.8", "", {}, "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="], - "minio": ["minio@8.0.6", "", { "dependencies": { "async": "^3.2.4", "block-stream2": "^2.1.0", "browser-or-node": "^2.1.1", "buffer-crc32": "^1.0.0", "eventemitter3": "^5.0.1", "fast-xml-parser": "^4.4.1", "ipaddr.js": "^2.0.1", "lodash": "^4.17.21", "mime-types": "^2.1.35", "query-string": "^7.1.3", "stream-json": "^1.8.0", "through2": "^4.0.2", "web-encoding": "^1.1.5", "xml2js": "^0.5.0 || ^0.6.2" } }, "sha512-sOeh2/b/XprRmEtYsnNRFtOqNRTPDvYtMWh+spWlfsuCV/+IdxNeKVUMKLqI7b5Dr07ZqCPuaRGU/rB9pZYVdQ=="], + "minio": ["minio@8.0.7", "", { "dependencies": { "async": "^3.2.4", "block-stream2": "^2.1.0", "browser-or-node": "^2.1.1", "buffer-crc32": "^1.0.0", "eventemitter3": "^5.0.1", "fast-xml-parser": "^5.3.4", "ipaddr.js": "^2.0.1", "lodash": "^4.17.21", "mime-types": "^2.1.35", "query-string": "^7.1.3", "stream-json": "^1.8.0", "through2": "^4.0.2", "xml2js": "^0.5.0 || ^0.6.2" } }, "sha512-E737MgufW8CeQAsTAtnEMrxZ9scMSf29kkhZoXzDTKj/Jszzo2SfeZUH9wbDQH2Rsq6TCtl/yQL0+XdVKZansQ=="], "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], @@ -822,17 +832,19 @@ "path-exists": ["path-exists@4.0.0", "", {}, "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w=="], + "path-expression-matcher": ["path-expression-matcher@1.6.2", "", {}, "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ=="], + "path-is-absolute": ["path-is-absolute@1.0.1", "", {}, "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg=="], "path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="], "path-parse": ["path-parse@1.0.7", "", {}, "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw=="], - "path-to-regexp": ["path-to-regexp@0.1.12", "", {}, "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ=="], + "path-to-regexp": ["path-to-regexp@0.1.13", "", {}, "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA=="], "path-type": ["path-type@4.0.0", "", {}, "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw=="], - "picomatch": ["picomatch@4.0.3", "", {}, "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q=="], + "picomatch": ["picomatch@4.0.4", "", {}, "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A=="], "possible-typed-array-names": ["possible-typed-array-names@1.1.0", "", {}, "sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg=="], @@ -842,11 +854,11 @@ "proxy-addr": ["proxy-addr@2.0.7", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg=="], - "proxy-from-env": ["proxy-from-env@1.1.0", "", {}, "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="], + "proxy-from-env": ["proxy-from-env@2.1.0", "", {}, "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA=="], "punycode": ["punycode@2.3.1", "", {}, "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="], - "qs": ["qs@6.14.0", "", { "dependencies": { "side-channel": "^1.1.0" } }, "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w=="], + "qs": ["qs@6.16.0", "", { "dependencies": { "es-define-property": "^1.0.1", "side-channel": "^1.1.1" } }, "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA=="], "query-string": ["query-string@7.1.3", "", { "dependencies": { "decode-uri-component": "^0.2.2", "filter-obj": "^1.1.0", "split-on-first": "^1.0.0", "strict-uri-encode": "^2.0.0" } }, "sha512-hh2WYhq4fi8+b+/2Kg9CEge4fDPvHS534aOOvOZeQ3+Vf2mCFsaFBYj0i+iXcAq6I9Vzp5fjMFBlONvayDC1qg=="], @@ -880,7 +892,7 @@ "rimraf": ["rimraf@3.0.2", "", { "dependencies": { "glob": "^7.1.3" }, "bin": { "rimraf": "bin.js" } }, "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA=="], - "rollup": ["rollup@2.79.2", "", { "optionalDependencies": { "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-fS6iqSPZDs3dr/y7Od6y5nha8dW1YnbgtsyotCVvoFGKbERG++CVRFv1meyGDE1SNItQA8BrnCw7ScdAhRJ3XQ=="], + "rollup": ["rollup@2.80.0", "", { "optionalDependencies": { "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-cIFJOD1DESzpjOBl763Kp1AH7UE/0fcdHe6rZXUdQ9c50uvgigvW97u3IcSeBwOkgqL/PXPBktBCh0KEu5L8XQ=="], "rollup-plugin-copy": ["rollup-plugin-copy@3.5.0", "", { "dependencies": { "@types/fs-extra": "^8.0.1", "colorette": "^1.1.0", "fs-extra": "^8.1.0", "globby": "10.0.1", "is-plain-object": "^3.0.0" } }, "sha512-wI8D5dvYovRMx/YYKtUNt3Yxaw4ORC9xo6Gt9t22kveWz1enG9QrhVlagzwrxSC455xD1dHMKhIJkbsQ7d48BA=="], @@ -920,9 +932,9 @@ "shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="], - "side-channel": ["side-channel@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3", "side-channel-list": "^1.0.0", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw=="], + "side-channel": ["side-channel@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.4", "side-channel-list": "^1.0.1", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ=="], - "side-channel-list": ["side-channel-list@1.0.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3" } }, "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA=="], + "side-channel-list": ["side-channel-list@1.0.1", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.4" } }, "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w=="], "side-channel-map": ["side-channel-map@1.0.1", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3" } }, "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA=="], @@ -968,7 +980,7 @@ "strip-json-comments": ["strip-json-comments@3.1.1", "", {}, "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig=="], - "strnum": ["strnum@1.1.2", "", {}, "sha512-vrN+B7DBIoTTZjnPNewwhx6cBA/H+IS7rfW68n7XxC1y7uoiGQBxaKzqucGUgavX15dJgiGztLJ8vxuEzwqBdA=="], + "strnum": ["strnum@2.4.2", "", { "dependencies": { "anynum": "^1.0.1" } }, "sha512-rDG3Ah4TV0k1hWvLSzkZtMmLN9+eS+h3knq4MP6A42Y3Yh5qGNnOUs1jJkoSr8FG5dsL28c7KgkIBzSEykqtuw=="], "supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], @@ -984,7 +996,7 @@ "tinyglobby": ["tinyglobby@0.2.15", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.3" } }, "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ=="], - "tmp": ["tmp@0.2.5", "", {}, "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow=="], + "tmp": ["tmp@0.2.7", "", {}, "sha512-e0votIpp4Uo2AJYSzVHV6xCcawuiez3DzqDAbrTc3YxBkplN6e+dM13ZeIcZnDg/QpSuU2zfZ3rzwY8ukEnaXw=="], "to-regex-range": ["to-regex-range@5.0.1", "", { "dependencies": { "is-number": "^7.0.0" } }, "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ=="], @@ -1028,20 +1040,16 @@ "uri-js": ["uri-js@4.4.1", "", { "dependencies": { "punycode": "^2.1.0" } }, "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg=="], - "util": ["util@0.12.5", "", { "dependencies": { "inherits": "^2.0.3", "is-arguments": "^1.0.4", "is-generator-function": "^1.0.7", "is-typed-array": "^1.1.3", "which-typed-array": "^1.1.2" } }, "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA=="], - "util-deprecate": ["util-deprecate@1.0.2", "", {}, "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="], "utils-merge": ["utils-merge@1.0.1", "", {}, "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA=="], - "uuid": ["uuid@11.1.0", "", { "bin": { "uuid": "dist/esm/bin/uuid" } }, "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A=="], + "uuid": ["uuid@11.1.1", "", { "bin": { "uuid": "dist/esm/bin/uuid" } }, "sha512-vIYxrBCC/N/K+Js3qSN88go7kIfNPssr/hHCesKCQNAjmgvYS2oqr69kIufEG+O4+PfezOH4EbIeHCfFov8ZgQ=="], "v8-compile-cache-lib": ["v8-compile-cache-lib@3.0.1", "", {}, "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg=="], "vary": ["vary@1.1.2", "", {}, "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg=="], - "web-encoding": ["web-encoding@1.1.5", "", { "dependencies": { "util": "^0.12.3" }, "optionalDependencies": { "@zxing/text-encoding": "0.9.0" } }, "sha512-HYLeVCdJ0+lBYV2FvNZmv3HJ2Nt0QYXqZojk3d9FJOLkwnuhzM9tmamh8d7HPM8QqjKH8DeHkFTx+CFlWpZZDA=="], - "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], "which-boxed-primitive": ["which-boxed-primitive@1.1.1", "", { "dependencies": { "is-bigint": "^1.1.0", "is-boolean-object": "^1.2.1", "is-number-object": "^1.1.1", "is-string": "^1.1.1", "is-symbol": "^1.1.1" } }, "sha512-TbX3mj8n0odCBFVlY8AxkqcHASw3L60jIuF8jFP78az3C2YhmGvqbHBpAjTRH2/xqYunrJ9g1jSyjCjpoWzIAA=="], @@ -1060,6 +1068,8 @@ "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], + "xml-naming": ["xml-naming@0.3.0", "", {}, "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ=="], + "xml2js": ["xml2js@0.6.2", "", { "dependencies": { "sax": ">=0.6.0", "xmlbuilder": "~11.0.0" } }, "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA=="], "xmlbuilder": ["xmlbuilder@11.0.1", "", {}, "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA=="], @@ -1088,11 +1098,11 @@ "@eslint/eslintrc/ignore": ["ignore@5.3.2", "", {}, "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g=="], - "@types/minimatch/minimatch": ["minimatch@9.0.5", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow=="], + "@types/minimatch/minimatch": ["minimatch@9.0.7", "", { "dependencies": { "brace-expansion": "^5.0.2" } }, "sha512-MOwgjc8tfrpn5QQEvjijjmDVtMw2oL88ugTevzxQnzRLm6l3fVEF2gzU0kYeYYKD8C66+IdGX6peJ4MyUlUnPg=="], "@types/serve-static/@types/send": ["@types/send@0.17.6", "", { "dependencies": { "@types/mime": "^1", "@types/node": "*" } }, "sha512-Uqt8rPBE8SY0RK8JB1EzVOIZ32uqy8HwdxCnoCOsYrvnswqmFZ/k+9Ikidlk/ImhsdvBsloHbAlewb2IEBV/Og=="], - "@typescript-eslint/typescript-estree/minimatch": ["minimatch@9.0.5", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow=="], + "@typescript-eslint/typescript-estree/minimatch": ["minimatch@9.0.7", "", { "dependencies": { "brace-expansion": "^5.0.2" } }, "sha512-MOwgjc8tfrpn5QQEvjijjmDVtMw2oL88ugTevzxQnzRLm6l3fVEF2gzU0kYeYYKD8C66+IdGX6peJ4MyUlUnPg=="], "@typescript-eslint/typescript-estree/semver": ["semver@7.7.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q=="], @@ -1122,13 +1132,17 @@ "finalhandler/debug": ["debug@2.6.9", "", { "dependencies": { "ms": "2.0.0" } }, "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA=="], + "form-data/hasown": ["hasown@2.0.4", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A=="], + "globby/ignore": ["ignore@5.3.2", "", {}, "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g=="], + "internal-slot/side-channel": ["side-channel@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3", "side-channel-list": "^1.0.0", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw=="], + "ioredis-mock/semver": ["semver@7.7.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q=="], "is-bun-module/semver": ["semver@7.7.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q=="], - "micromatch/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], + "micromatch/picomatch": ["picomatch@2.3.2", "", {}, "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA=="], "proxy-addr/ipaddr.js": ["ipaddr.js@1.9.1", "", {}, "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="], @@ -1146,9 +1160,9 @@ "@aws-sdk/middleware-sdk-s3/@aws-sdk/signature-v4-multi-region/@smithy/signature-v4": ["@smithy/signature-v4@5.6.9", "", { "dependencies": { "@smithy/core": "^3.29.8", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-g5rnEii/mkT0mjVJmlsaOfyNBtHNTecD9Lo4NP8D5HzMUEnZNpz7/FbvBCjNcV4vteHFAxOGiLUYNxPkDZZAPw=="], - "@types/minimatch/minimatch/brace-expansion": ["brace-expansion@2.0.2", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ=="], + "@types/minimatch/minimatch/brace-expansion": ["brace-expansion@5.0.12", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ=="], - "@typescript-eslint/typescript-estree/minimatch/brace-expansion": ["brace-expansion@2.0.2", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ=="], + "@typescript-eslint/typescript-estree/minimatch/brace-expansion": ["brace-expansion@5.0.12", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ=="], "body-parser/debug/ms": ["ms@2.0.0", "", {}, "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="], @@ -1158,12 +1172,18 @@ "finalhandler/debug/ms": ["ms@2.0.0", "", {}, "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="], + "internal-slot/side-channel/side-channel-list": ["side-channel-list@1.0.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3" } }, "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA=="], + "rollup-plugin-sourcemaps/@rollup/pluginutils/@types/estree": ["@types/estree@0.0.39", "", {}, "sha512-EYNwp3bU+98cpU4lAWYYL7Zz+2gryWH1qbdDTidVd6hkiR6weksdbMadyXKXNPEkQFhXM+hVO9ZygomHXp+AIw=="], "rollup-plugin-sourcemaps/@rollup/pluginutils/estree-walker": ["estree-walker@1.0.1", "", {}, "sha512-1fMXF3YP4pZZVozF8j/ZLfvnR8NSIljt56UhbZ5PeeDmmGHpgpdwQt7ITlGvYaQukCvuBRMLEiKiYC+oeIg4cg=="], - "rollup-plugin-sourcemaps/@rollup/pluginutils/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], + "rollup-plugin-sourcemaps/@rollup/pluginutils/picomatch": ["picomatch@2.3.2", "", {}, "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA=="], "send/debug/ms": ["ms@2.0.0", "", {}, "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="], + + "@types/minimatch/minimatch/brace-expansion/balanced-match": ["balanced-match@4.0.4", "", {}, "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA=="], + + "@typescript-eslint/typescript-estree/minimatch/brace-expansion/balanced-match": ["balanced-match@4.0.4", "", {}, "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA=="], } } diff --git a/service/package.json b/service/package.json index c5a49a82..a3679c8f 100644 --- a/service/package.json +++ b/service/package.json @@ -33,14 +33,14 @@ "@opentelemetry/exporter-trace-otlp-proto": "0.219.0", "@opentelemetry/resources": "2.8.0", "@opentelemetry/sdk-trace-base": "2.8.0", - "axios": "^1.8.4", + "axios": "^1.20.0", "bullmq": "^5.13.1", "busboy": "^1.6.0", "dotenv": "^17.4.2", - "express": "^4.19.2", + "express": "^4.22.3", "express-rate-limit": "^7.4.1", "ioredis": "^5.4.1", - "minio": "^8.0.5", + "minio": "^8.0.7", "nanoid": "^5.1.16", "prom-client": "^15.1.3", "rate-limit-redis": "^4.2.0", @@ -74,6 +74,8 @@ "typescript": "^5.5.4" }, "overrides": { - "decode-uri-component": "0.5.0" + "decode-uri-component": "0.5.0", + "uuid": "11.1.1", + "qs": "6.16.0" } } diff --git a/service/src/egress-gateway-client.test.ts b/service/src/egress-gateway-client.test.ts index c3d7ab9c..ef9e3fb7 100644 --- a/service/src/egress-gateway-client.test.ts +++ b/service/src/egress-gateway-client.test.ts @@ -21,6 +21,7 @@ type AxiosPostCall = { headers?: Record; signal?: AbortSignal; timeout?: number; + maxRedirects?: number; }; }; @@ -109,6 +110,7 @@ describe('egress gateway client', () => { allowedToolNames: ['query_clickhouse'], }, { signal: controller.signal }); + expect(calls.map(call => call.config.maxRedirects)).toEqual([0, 0, 0]); expect(calls.map(call => call.config.timeout)).toEqual([12_345, 12_345, 12_345]); expect(calls.map(call => call.config.signal)).toEqual([ controller.signal, @@ -147,6 +149,7 @@ describe('egress gateway client', () => { await revokeGatewayEgressGrant({ grantId: 'grant_123', reason: 'completed' }); await revokeGatewayEgressGrant({ grantId: 'grant_123', reason: 'failed', timeoutMs: 321 }); + expect(calls.map(call => call.config.maxRedirects)).toEqual([0, 0]); expect(calls.map(call => call.config.timeout)).toEqual([987, 321]); }); diff --git a/service/src/egress-gateway-client.ts b/service/src/egress-gateway-client.ts index ebcef4ad..67b7dca6 100644 --- a/service/src/egress-gateway-client.ts +++ b/service/src/egress-gateway-client.ts @@ -48,6 +48,7 @@ export async function createGatewayEgressGrant(args: { gatewayUrl('/internal/egress-grants'), body, { + maxRedirects: 0, headers: injectTraceHeaders(gatewayHeaders({ 'Content-Type': 'application/json' }, isSynthetic)), signal, timeout: env.EGRESS_GATEWAY_REQUEST_TIMEOUT_MS, @@ -73,6 +74,7 @@ export async function restoreGatewaySandboxResult { }); describe('egress gateway routes', () => { + test('rejects file-server redirects before forwarding internal credentials', async () => { + let redirected = 0; + let contacted = 0; + const target = Bun.serve({ + port: 0, + fetch() { + redirected++; + return new Response('unexpected'); + }, + }); + const upstream = Bun.serve({ + port: 0, + fetch(req) { + contacted++; + expect(req.headers.get(INTERNAL_SERVICE_TOKEN_HEADER)).toBe(INTERNAL_TOKEN); + return new Response(null, { + status: 307, + headers: { Location: `http://127.0.0.1:${target.port}/stolen` }, + }); + }, + }); + try { + globalThis.fetch = originalFetch; + env.EGRESS_GATEWAY_FILE_SERVER_URL = `http://127.0.0.1:${upstream.port}`; + const response = await gatewayFetch( + `/sessions/${sessionHandle({ dir: 'read', sessionId: 'sess_input' })}/objects/${objectHandle({})}`, + { headers: grantHeader() }, + ); + expect(response.status).toBe(500); + expect(contacted).toBe(1); + expect(redirected).toBe(0); + } finally { + upstream.stop(true); + target.stop(true); + } + }); + test('protects internal grant create, restore, and revoke routes', async () => { const createBody = JSON.stringify({ payload: payload(), claims: executionClaims() }); const unauthorized = await gatewayFetch('/internal/egress-grants', { diff --git a/service/src/egress-gateway.ts b/service/src/egress-gateway.ts index 873db173..5a36ebc5 100644 --- a/service/src/egress-gateway.ts +++ b/service/src/egress-gateway.ts @@ -544,7 +544,7 @@ app.get('/sessions/:sessionHandle/objects', async (req, res) => { `/sessions/${encodeURIComponent(sessionId)}/objects`, '?detail=normalized', ), - { headers: injectTraceHeaders(internalServiceHeaders({ Accept: 'application/json' })) }, + { redirect: 'error', headers: injectTraceHeaders(internalServiceHeaders({ Accept: 'application/json' })) }, ); if (!upstream.ok) { return pipeFetchResponse(upstream, res); @@ -638,7 +638,7 @@ app.post('/input-manifest', express.json({ limit: '4mb' }), async (req, res) => controller.signal.throwIfAborted(); const upstream = await fetch(forwardUrl(env.EGRESS_GATEWAY_FILE_SERVER_URL, `/sessions/${encodeURIComponent(sessionId)}/objects/${encodeURIComponent(objectId)}/metadata`), - { headers: injectTraceHeaders(internalServiceHeaders()), signal: controller.signal }); + { redirect: 'error', headers: injectTraceHeaders(internalServiceHeaders()), signal: controller.signal }); if (!upstream.ok) { await upstream.body?.cancel(); // Recheck failures individually through the existing retry/classification path. @@ -668,7 +668,7 @@ app.get('/sessions/:sessionHandle/objects/:objectHandle/metadata', async (req, r await recordEgressRead(grant); const upstream = await fetch(forwardUrl(env.EGRESS_GATEWAY_FILE_SERVER_URL, `/sessions/${encodeURIComponent(sessionId)}/objects/${encodeURIComponent(object.id)}/metadata`), - { headers: injectTraceHeaders(internalServiceHeaders()) }); + { redirect: 'error', headers: injectTraceHeaders(internalServiceHeaders()) }); if (!upstream.ok) return pipeFetchResponse(upstream, res); return res.json(inputMetadata(await upstream.json(), grant, sessionId, object.id)); } catch (error) { @@ -694,7 +694,7 @@ app.get('/sessions/:sessionHandle/objects/:objectHandle', async (req, res) => { env.EGRESS_GATEWAY_FILE_SERVER_URL, `/sessions/${encodeURIComponent(sessionId)}/objects/${encodeURIComponent(object.id)}`, ), - { headers: injectTraceHeaders(internalServiceHeaders(expectedVersion ? { 'X-CodeAPI-Input-Version': expectedVersion } : {})) }, + { redirect: 'error', headers: injectTraceHeaders(internalServiceHeaders(expectedVersion ? { 'X-CodeAPI-Input-Version': expectedVersion } : {})) }, ); const headerOverrides = isOpaqueObjectContentDisposition( upstream.headers.get('content-disposition'), @@ -750,6 +750,7 @@ app.put('/sessions/:sessionHandle/objects/:fileId', async (req, res) => { ), { method: 'PUT', + redirect: 'error', headers, body: req as unknown as BodyInit, duplex: 'half', @@ -832,6 +833,7 @@ app.post('/tool-call', async (req, res) => { await recordEgressToolCall(callback.grant_id, executionId); const upstream = await fetch(forwardUrl(env.EGRESS_GATEWAY_TOOL_CALL_SERVER_URL, '/tool-call'), { method: 'POST', + redirect: 'error', headers: { ...injectTraceHeaders(), 'Content-Type': req.header('content-type') ?? 'application/json', @@ -847,6 +849,7 @@ app.post('/tool-call', async (req, res) => { await recordEgressToolCall(callback.grant_id, executionId); const upstream = await fetch(forwardUrl(env.EGRESS_GATEWAY_TOOL_CALL_SERVER_URL, '/tool-call'), { method: 'POST', + redirect: 'error', headers: { ...injectTraceHeaders(), 'Content-Type': req.header('content-type') ?? 'application/json', diff --git a/service/src/hosted-app/microvm-runtime.ts b/service/src/hosted-app/microvm-runtime.ts index 8b594f37..f0cb69e9 100644 --- a/service/src/hosted-app/microvm-runtime.ts +++ b/service/src/hosted-app/microvm-runtime.ts @@ -377,6 +377,7 @@ export class HostedAppMicrovmRuntime { token = await this.mintToken(vm.microvmId, this.config.controlPort, callerSignal); } const response = await this.deps.fetch(`${endpoint}/api/v2/health`, { + redirect: 'error', headers: { [token.headerName]: token.token, ...microvmPortHeaders(this.config.controlPort), @@ -414,6 +415,7 @@ export class HostedAppMicrovmRuntime { `${normalizeHostedAppMicrovmEndpoint(vm.endpoint ?? '')}/api/v2/hosted-app/start`, { method: 'POST', + redirect: 'error', headers: { [token.headerName]: token.token, ...microvmPortHeaders(this.config.controlPort), @@ -454,6 +456,7 @@ export class HostedAppMicrovmRuntime { const response = await this.deps.fetch( `${normalizeHostedAppMicrovmEndpoint(vm.endpoint ?? '')}/api/v2/hosted-app/status`, { + redirect: 'error', headers: { [token.headerName]: token.token, ...microvmPortHeaders(this.config.controlPort), diff --git a/service/src/internal-redirect-policy.test.ts b/service/src/internal-redirect-policy.test.ts new file mode 100644 index 00000000..577b468c --- /dev/null +++ b/service/src/internal-redirect-policy.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as ts from 'typescript'; + +const transports: Record = { + 'egress-gateway-client.ts': 4, + 'egress-gateway.ts': 7, + 'hosted-app/microvm-runtime.ts': 3, + 'runtime-session/checkpoint.ts': 4, + 'runtime-session/files.ts': 1, + 'sandbox-backend/http.ts': 1, + 'sandbox-backend/lambda-microvm.ts': 2, + 'service/programmatic-router.ts': 3, + 'service/replay-state.ts': 2, + 'service/router.ts': 5, +}; + +// Every request to these internal endpoints must stay on its original origin. +describe('internal transport redirect policy', () => { + for (const [file, count] of Object.entries(transports)) { + test(`${file} rejects redirects on every internal request`, () => { + const source = ts.createSourceFile(file, + fs.readFileSync(path.join(__dirname, file), 'utf8'), ts.ScriptTarget.Latest, true); + let requests = 0; + function visit(node: ts.Node): void { + if (ts.isCallExpression(node)) { + const name = node.expression.getText(source); + const fetchCall = name === 'fetch' || name === 'this.deps.fetch'; + const axiosCall = name === 'axios' || /^axios\.(get|post|delete)$/.test(name); + if (fetchCall || axiosCall) { + requests++; + let index = 1; + if (name === 'axios') index = 0; + if (name === 'axios.post') index = 2; + let options = node.arguments[index]; + while (options && (ts.isAsExpression(options) || ts.isTypeAssertionExpression(options))) { + options = options.expression; + } + expect(options && ts.isObjectLiteralExpression(options)).toBe(true); + if (!options || !ts.isObjectLiteralExpression(options)) return; + const property = options.properties.find(prop => ts.isPropertyAssignment(prop) + && prop.name.getText(source) === (fetchCall ? 'redirect' : 'maxRedirects')); + expect(property && ts.isPropertyAssignment(property)).toBe(true); + if (property && ts.isPropertyAssignment(property)) { + expect(property.initializer.getText(source)).toBe(fetchCall ? '\'error\'' : '0'); + } + } + } + ts.forEachChild(node, visit); + } + visit(source); + expect(requests).toBe(count); + }); + } +}); diff --git a/service/src/minio-compat.test.ts b/service/src/minio-compat.test.ts new file mode 100644 index 00000000..255c6e00 --- /dev/null +++ b/service/src/minio-compat.test.ts @@ -0,0 +1,39 @@ +import { expect, test } from 'bun:test'; +import { Client } from 'minio'; + +test('patched MinIO preserves S3 object-list XML decoding', async () => { + const server = Bun.serve({ + port: 0, + fetch(req) { + const url = new URL(req.url); + expect(url.pathname).toBe('/testbucket'); + expect(url.searchParams.get('list-type')).toBe('2'); + return new Response(` + + testbucketinputs/1 + 1000false + inputs/a&b.csv2026-01-01T00:00:00.000Z + "test-etag"12STANDARD + +`, { headers: { 'Content-Type': 'application/xml' } }); + }, + }); + try { + const client = new Client({ + endPoint: '127.0.0.1', + port: server.port, + useSSL: false, + region: 'us-east-1', + accessKey: 'test-access-key', + secretKey: 'test-secret-key', + }); + const objects = []; + for await (const object of client.listObjectsV2('testbucket', 'inputs/', true)) { + objects.push(object); + } + expect(objects).toHaveLength(1); + expect(objects[0]).toMatchObject({ name: 'inputs/a&b.csv', size: 12, etag: 'test-etag' }); + } finally { + server.stop(true); + } +}); diff --git a/service/src/runtime-session/checkpoint.ts b/service/src/runtime-session/checkpoint.ts index 3562cd17..e49d9290 100644 --- a/service/src/runtime-session/checkpoint.ts +++ b/service/src/runtime-session/checkpoint.ts @@ -94,6 +94,7 @@ export async function pullCheckpoint( ? AbortSignal.any([args.signal, transferDeadlineSignal]) : transferDeadlineSignal; const response = await axios.get(`${args.endpointBase}/api/v2/session/checkpoint`, { + maxRedirects: 0, headers: { [token.headerName]: token.token, ...microvmPortHeaders(config.port), @@ -125,6 +126,7 @@ export async function pushRestore( ): Promise { const token = await args.mintToken(); await axios.post(`${args.endpointBase}/api/v2/session/restore`, fs.createReadStream(data.path), { + maxRedirects: 0, headers: { [token.headerName]: token.token, ...microvmPortHeaders(config.port), @@ -153,6 +155,7 @@ export async function probeInputs( `${args.endpointBase}/api/v2/session/inputs/probe`, { refs }, { + maxRedirects: 0, headers: { [token.headerName]: token.token, ...microvmPortHeaders(config.port), @@ -197,6 +200,7 @@ export async function pushInputs( * unlink the archive first and the stream then emits an unhandled ENOENT. */ const requestData = typeof data === 'function' ? data() : data; await axios.post(`${args.endpointBase}/api/v2/session/inputs`, requestData, { + maxRedirects: 0, headers: { [token.headerName]: token.token, ...microvmPortHeaders(config.port), diff --git a/service/src/runtime-session/files.ts b/service/src/runtime-session/files.ts index f17998dd..dd53508d 100644 --- a/service/src/runtime-session/files.ts +++ b/service/src/runtime-session/files.ts @@ -202,6 +202,7 @@ async function fetchFileObjectToPath( const url = `${baseUrl}/sessions/${encodeURIComponent(ref.storage_session_id)}/objects/${encodeURIComponent(ref.id)}`; try { const response = await axios.get(url, { + maxRedirects: 0, headers: internalServiceHeaders(), responseType: 'stream', timeout: opts.timeoutMs, diff --git a/service/src/sandbox-backend/http.test.ts b/service/src/sandbox-backend/http.test.ts index d3370646..403772be 100644 --- a/service/src/sandbox-backend/http.test.ts +++ b/service/src/sandbox-backend/http.test.ts @@ -14,7 +14,7 @@ type CapturedRequest = { let server: ReturnType; let captured: CapturedRequest[] = []; -let nextResponse: { status: number; body: unknown; delayMs?: number } = { status: 200, body: {} }; +let nextResponse: { status: number; body: unknown; delayMs?: number; location?: string } = { status: 200, body: {} }; const savedEndpoint = env.SANDBOX_ENDPOINT; @@ -33,7 +33,7 @@ beforeAll(() => { } return new Response(JSON.stringify(nextResponse.body), { status: nextResponse.status, - headers: { 'Content-Type': 'application/json' }, + headers: { 'Content-Type': 'application/json', ...(nextResponse.location ? { Location: nextResponse.location } : {}) }, }); }, }); @@ -104,6 +104,26 @@ describe('HttpSandboxBackend', () => { expect(result).toEqual(responseBody); }); + test('rejects redirects without sending the signed execution to another endpoint', async () => { + let redirected = 0; + const target = Bun.serve({ + port: 0, + fetch() { + redirected++; + return Response.json({ session_id: 'unexpected' }); + }, + }); + try { + for (const status of [302, 307, 308]) { + nextResponse = { status, body: {}, location: `http://127.0.0.1:${target.port}/stolen` }; + await expect(new HttpSandboxBackend().execute(request(), context())).rejects.toThrow(); + } + expect(redirected).toBe(0); + } finally { + target.stop(true); + } + }); + test('does not mutate the signed request body', async () => { const req = request(); const before = JSON.stringify(req.body); diff --git a/service/src/sandbox-backend/http.ts b/service/src/sandbox-backend/http.ts index ce5f5c0a..1ebdfcbe 100644 --- a/service/src/sandbox-backend/http.ts +++ b/service/src/sandbox-backend/http.ts @@ -20,6 +20,7 @@ export class HttpSandboxBackend implements SandboxBackend { `${env.SANDBOX_ENDPOINT}/${Jobs.execute}`, req.body, { + maxRedirects: 0, headers: injectTraceHeaders(req.headers), signal: ctx.signal, } diff --git a/service/src/sandbox-backend/lambda-microvm.ts b/service/src/sandbox-backend/lambda-microvm.ts index d947c76e..e4ccf9a5 100644 --- a/service/src/sandbox-backend/lambda-microvm.ts +++ b/service/src/sandbox-backend/lambda-microvm.ts @@ -1437,6 +1437,7 @@ export class LambdaMicrovmSandboxBackend implements SandboxBackend { `${base}/api/v2/${Jobs.execute}`, req.body, { + maxRedirects: 0, headers: { ...injectTraceHeaders(req.headers), [token.headerName]: token.token, @@ -1644,6 +1645,7 @@ export class LambdaMicrovmSandboxBackend implements SandboxBackend { private async assertHealthy(base: string, token: string, ctx: SandboxExecuteContext): Promise { try { const response = await axios.get(`${base}/api/v2/health`, { + maxRedirects: 0, headers: { 'X-aws-proxy-auth': token, ...microvmPortHeaders(this.config.port) }, timeout: this.config.healthTimeoutMs, signal: ctx.signal, diff --git a/service/src/service/programmatic-router.ts b/service/src/service/programmatic-router.ts index 1e2a367c..86b1cdbb 100644 --- a/service/src/service/programmatic-router.ts +++ b/service/src/service/programmatic-router.ts @@ -319,7 +319,7 @@ function waitForExecutionState( () => axios.get( `${env.TOOL_CALL_SERVER_URL}/sessions/${id}/pending`, - { headers: internalServiceHeaders() }, + { maxRedirects: 0, headers: internalServiceHeaders() }, ), 'Get pending tool calls', ); @@ -1598,7 +1598,7 @@ async function handleBlocking( error_message: r.error_message, })), }, - { headers: internalServiceHeaders() }, + { maxRedirects: 0, headers: internalServiceHeaders() }, ), 'Submit tool results', ); @@ -1783,7 +1783,7 @@ async function handleBlocking( timeout, tools, }, - { headers: internalServiceHeaders() }, + { maxRedirects: 0, headers: internalServiceHeaders() }, ), 'Create Tool Call Server session', ); diff --git a/service/src/service/replay-state.ts b/service/src/service/replay-state.ts index 003ce6b5..9f33586c 100644 --- a/service/src/service/replay-state.ts +++ b/service/src/service/replay-state.ts @@ -800,6 +800,7 @@ export async function cleanupStaleExecutions(): Promise { if (state.mode === 'blocking') { await axios.delete(`${env.TOOL_CALL_SERVER_URL}/sessions/${state.execution_id}`, { + maxRedirects: 0, headers: internalServiceHeaders(), }) .catch(() => {}); @@ -835,6 +836,7 @@ export async function cleanupExecution(execution_id: string, mode: 'blocking' | if (mode === 'blocking') { ops.push( axios.delete(`${env.TOOL_CALL_SERVER_URL}/sessions/${execution_id}`, { + maxRedirects: 0, headers: internalServiceHeaders(), }).catch(() => {}), ); diff --git a/service/src/service/router.ts b/service/src/service/router.ts index 43d0ff4b..a977182e 100644 --- a/service/src/service/router.ts +++ b/service/src/service/router.ts @@ -71,6 +71,7 @@ async function putFileToFileServer( ): Promise { const response = await fetch(url, { method: 'PUT', + redirect: 'error', headers, body: Readable.toWeb(file) as unknown as ReadableStream, signal, @@ -416,6 +417,7 @@ router.get('/download/:session_id/:fileId', downloadLimiter, sessionAuth, async const response = await axios({ method: 'get', url: `${env.FILE_SERVER_URL}/sessions/${session_id}/objects/${fileId}`, + maxRedirects: 0, headers: internalServiceHeaders(), responseType: 'stream' }); @@ -909,6 +911,7 @@ router.get('/files/:session_id', fetchLimiter, sessionAuth, async (req: t.Authen try { const response = await axios.get(`${env.FILE_SERVER_URL}/sessions/${session_id}/objects`, { params: { detail }, + maxRedirects: 0, headers: internalServiceHeaders({ 'Accept': 'application/json' }) }); @@ -941,7 +944,7 @@ router.get('/sessions/:session_id/objects/:fileId', fetchLimiter, sessionAuth, a try { const response = await axios.get( `${env.FILE_SERVER_URL}/sessions/${session_id}/objects/${fileId}/metadata`, - { headers: internalServiceHeaders({ Accept: 'application/json' }) }, + { maxRedirects: 0, headers: internalServiceHeaders({ Accept: 'application/json' }) }, ); return res.status(200).json(response.data); @@ -972,7 +975,7 @@ const deleteSessionObject = async (req: t.AuthenticatedRequest, res: Response) = try { const response = await axios.delete( `${env.FILE_SERVER_URL}/sessions/${session_id}/objects/${fileId}`, - { headers: internalServiceHeaders() } + { maxRedirects: 0, headers: internalServiceHeaders() } ); await connection.del(`upload:${req.sessionKey}${session_id}${fileId}`); diff --git a/test-sandbox.sh b/test-sandbox.sh index 27c8a877..cf6db2cd 100755 --- a/test-sandbox.sh +++ b/test-sandbox.sh @@ -286,15 +286,28 @@ expect_socket_blocked("AF_RXRPC", getattr(socket, "AF_RXRPC", 33), socket.SOCK_D expect_socket_blocked("AF_ALG", getattr(socket, "AF_ALG", 38), socket.SOCK_SEQPACKET, 0) syscalls = { - "x86_64": {"clone": 56, "clone3": 435, "vmsplice": 278}, - "amd64": {"clone": 56, "clone3": 435, "vmsplice": 278}, - "aarch64": {"clone": 220, "clone3": 435, "vmsplice": 75}, - "arm64": {"clone": 220, "clone3": 435, "vmsplice": 75}, + "x86_64": {"clone": 56, "clone3": 435, "vmsplice": 278, "futex": 202}, + "amd64": {"clone": 56, "clone3": 435, "vmsplice": 278, "futex": 202}, + "aarch64": {"clone": 220, "clone3": 435, "vmsplice": 75, "futex": 98}, + "arm64": {"clone": 220, "clone3": 435, "vmsplice": 75, "futex": 98}, } arch = platform.machine().lower() if arch not in syscalls: raise SystemExit(f"unsupported arch for syscall smoke test: {arch}") +# PI operations must be denied before the kernel sees even invalid addresses. +for command in (6, 7, 8, 11, 12, 13): + for flags in (0, 128, 256, 384): + ctypes.set_errno(0) + rc = libc.syscall(syscalls[arch]["futex"], 0, command | flags, 0, 0, 0, 0) + if rc != -1 or ctypes.get_errno() != errno.EPERM: + raise SystemExit(f"PI futex operation was not denied: {command | flags}") +word = ctypes.c_int(0) +rc = libc.syscall(syscalls[arch]["futex"], ctypes.byref(word), 1 | 128, 1, 0, 0, 0) +if rc != 0: + raise SystemExit("ordinary FUTEX_WAKE_PRIVATE was blocked") +print("futex_pi=blocked:ordinary_wake=ok") + CLONE_NEWUSER = 0x10000000 CLONE_NEWNET = 0x40000000 @@ -359,7 +372,8 @@ PY && [[ "$stdout" == *"AF_ALG=errno:1"* ]] \ && [[ "$stdout" == *"clone_namespace=rc:-1:errno:1"* ]] \ && [[ "$stdout" == *"clone3=rc:-1:errno:38"* ]] \ - && [[ "$stdout" == *"vmsplice=rc:-1:errno:1"* ]]; then + && [[ "$stdout" == *"vmsplice=rc:-1:errno:1"* ]] \ + && [[ "$stdout" == *"futex_pi=blocked:ordinary_wake=ok"* ]]; then log_success "Kernel attack-surface syscalls blocked" return 0 else From e68fae8f73ef98bdc507a1882c52f7a1345ebe10 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Sun, 4 Oct 2026 21:23:44 -0400 Subject: [PATCH 33/48] =?UTF-8?q?=F0=9F=90=B3=20ci:=20Publish=20Code=20API?= =?UTF-8?q?=20Images=20to=20GHCR=20(#307)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐳 ci: Publish Code API Images to GHCR Build one linux/amd64 image per Compose build on pushes to main that touch an image input, tag it sha-, and move the main tags together once every image has built. Pull requests build without pushing. Each image keeps a registry build cache so the KVM sandbox runner does not recompile its runtime packages on every push. README documents the tags and a Compose override for pulling instead of building. * 🏷️ fix: Build Every Main Commit and Promote the Newest Complete One Give each main commit its own concurrency group so a queued run can no longer be replaced and lose its sha tags. The promote job now serializes, resolves the newest main commit that has every image, retries each retag, and finishes a partial move on the next run. Report each pushed digest in the job summary and document per-image digest pinning. Drop the disk cleanup step: a cold KVM runner build peaked at about 17 GB of builder state with over 95 GB free. * 🔒 fix: Write Commit Tags Once and Never Move main Backwards Skip building any image whose sha tag for the commit already exists, so a re-run that fails part-way cannot mix two builds under one commit. Promotion retries tag inspections and refuses to move an image's main to an older commit than it names, so a transient registry error can delay a promotion but not reverse one. Note that the macOS override keeps building its own sandbox target. * 🧯 fix: Fail Closed When the Registry Cannot Answer Read image revisions through one script that tells a missing tag (or one this token may not read, and so may not write) apart from a registry that does not answer. The write-once check and the promotion now fail on the latter instead of treating it as a missing tag, so neither can rebuild over an existing tag or move main backwards. Restore the disk cleanup on the KVM runner leg, since the hosted runner only guarantees 14 GB free. --- .github/scripts/image-revision.sh | 35 +++ .github/workflows/images.yml | 350 ++++++++++++++++++++++++++++++ README.md | 84 +++++++ 3 files changed, 469 insertions(+) create mode 100755 .github/scripts/image-revision.sh create mode 100644 .github/workflows/images.yml diff --git a/.github/scripts/image-revision.sh b/.github/scripts/image-revision.sh new file mode 100755 index 00000000..ae4c88b4 --- /dev/null +++ b/.github/scripts/image-revision.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Prints the org.opencontainers.image.revision label of an image reference. +# Used by .github/workflows/images.yml, which must tell a missing tag apart +# from a registry that cannot answer. +# +# Exit 0 the revision was printed. +# Exit 3 the tag does not exist, or this token may not read it (a token that +# may not read a repository may not write to it either). +# Exit 1 the registry did not answer after retries, or the image has no +# revision label. +set -uo pipefail + +ref=${1:?usage: image-revision.sh REF} +attempts=${IMAGE_REVISION_ATTEMPTS:-3} +err=$(mktemp) +trap 'rm -f "$err"' EXIT + +for attempt in $(seq 1 "$attempts"); do + if json=$(docker buildx imagetools inspect "$ref" --format '{{ json .Image }}' 2>"$err"); then + if jq -er 'if has("config") then . else .["linux/amd64"] end + | .config.Labels["org.opencontainers.image.revision"]' <<<"$json"; then + exit 0 + fi + echo "$ref has no revision label" >&2 + exit 1 + fi + if grep -qiE ': not found$|denied|\b40[134]\b' "$err"; then + exit 3 + fi + cat "$err" >&2 + if [ "$attempt" -lt "$attempts" ]; then + sleep $(( attempt * 5 )) + fi +done +exit 1 diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml new file mode 100644 index 00000000..7c626767 --- /dev/null +++ b/.github/workflows/images.yml @@ -0,0 +1,350 @@ +# Builds the Code API container images and publishes them to GHCR, so a host +# can pull a tested image instead of compiling one in place. Like ci.yml this +# file is inert inside the monorepo — GitHub only runs workflows from the repo +# root — and becomes a root workflow in the published repo. +# +# Entry points: +# +# * push to main — build every image, push `sha-` tags, and refresh +# each image's registry build cache. The promote job then points the +# `main` tags at the newest main commit that has all images. +# * pull_request — build every image without pushing, reading the cache that +# main wrote, so Dockerfile breakage surfaces before merge. +# * workflow_dispatch — on main, the same as a push; on any other branch, a +# build-only run. +# +# The image list lives in the plan job and feeds both the build matrix and the +# promotion, so the two cannot drift apart. README.md ("Prebuilt images") +# documents the tags and how a Compose host switches to them. +name: Images + +on: + push: + branches: [main] + # Everything the Dockerfiles below COPY from the build context, plus this + # workflow. Every run builds every image, so a commit that has tags has + # them for all images; unchanged images are cache hits. + paths: &image-inputs + - .dockerignore + - .github/scripts/image-revision.sh + - .github/workflows/images.yml + - api/** + - docker/** + - javascript-packages.txt + - launcher/** + - packages/code/src/** + - service/** + - shared/** + pull_request: + paths: *image-inputs + workflow_dispatch: + +permissions: + contents: read + +concurrency: + # Each main commit gets its own group and is never cancelled, so no commit's + # build can be dropped by a later queued run (GitHub keeps only one pending + # run per group). Other refs share one group per ref and cancel superseded + # runs. + group: images-${{ github.ref == 'refs/heads/main' && github.sha || github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +env: + NAMESPACE: ghcr.io/librechat-ai + +jobs: + plan: + name: Plan + runs-on: ubuntu-24.04 + outputs: + images: ${{ steps.images.outputs.images }} + publish: ${{ github.repository == 'LibreChat-AI/code-interpreter' && github.ref == 'refs/heads/main' }} + steps: + - name: List images + id: images + # One image per distinct Compose build in docker-compose.yaml. The + # sandbox runner's two Compose targets (KVM_ENABLED=true/false) are + # published as two images. The Lambda MicroVM targets are not built + # here; they are not used by Compose hosts. + run: | + images=$(jq -c . <<'JSON' + [ + {"image": "code-interpreter-api", "dockerfile": "service/Dockerfile", "target": "api", + "description": "Code API HTTP server (Compose service api)", "timeout": 30}, + {"image": "code-interpreter-worker", "dockerfile": "service/Dockerfile", "target": "worker", + "description": "Code API job worker (Compose service service-worker)", "timeout": 30}, + {"image": "code-interpreter-file-server", "dockerfile": "service/Dockerfile", "target": "production", + "description": "Code API file server (Compose service file_server)", "timeout": 30}, + {"image": "code-interpreter-egress-gateway", "dockerfile": "service/Dockerfile.egress-gateway", "target": "production", + "description": "Sandbox egress gateway (Compose service egress_gateway)", "timeout": 30}, + {"image": "code-interpreter-tool-call-server", "dockerfile": "service/Dockerfile.tool-call-server", "target": "production", + "description": "Tool call server (Compose service tool_call_server)", "timeout": 30}, + {"image": "code-interpreter-sandbox-runner", "dockerfile": "api/Dockerfile", "target": "sandbox-runner-true", + "description": "libkrun microVM sandbox runner with the guest rootfs and runtime packages baked in (Compose service sandbox-runner, KVM_ENABLED=true)", + "timeout": 60, "free_disk": true}, + {"image": "code-interpreter-sandbox-runner-direct", "dockerfile": "api/Dockerfile", "target": "sandbox-runner-false", + "description": "Directory-root sandbox runner; runtime packages are mounted from the host (Compose service sandbox-runner, KVM_ENABLED=false)", + "timeout": 30} + ] + JSON + ) + echo "images=$images" >> "$GITHUB_OUTPUT" + + build: + name: ${{ matrix.image }} + needs: plan + runs-on: ubuntu-24.04 + timeout-minutes: ${{ matrix.timeout }} + permissions: + contents: read + # Pushes happen only when needs.plan.outputs.publish is true (main). + # Pull request runs read the build cache and never push; GitHub caps a + # fork's token at read access regardless. + packages: write + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.images) }} + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + + - name: Free runner disk + # A cold KVM runner build peaks at roughly 17 GB of builder state. The + # hosted runner had about 87 GB free when measured, but GitHub only + # guarantees 14 GB, so this leg removes unused preinstalled SDKs + # (about 28 GB, 1.5 minutes) rather than depend on the surplus. + if: matrix.free_disk + run: | + sudo rm -rf \ + /opt/ghc \ + /opt/hostedtoolcache/CodeQL \ + /usr/local/.ghcup \ + /usr/local/lib/android \ + /usr/local/share/powershell \ + /usr/share/dotnet \ + /usr/share/swift + docker image prune --all --force >/dev/null + df -h / + + - name: Sample free disk + run: | + nohup bash -c 'while :; do df -BM --output=avail / | tail -n1 | tr -dc 0-9; echo; sleep 10; done' \ + >"$RUNNER_TEMP/disk-avail-mib.log" 2>/dev/null & + + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + - name: Log in to GHCR + # Pull request runs log in only to read the cache, and only from this + # repository; fork runs read public packages anonymously. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Check for an existing tag + # `sha-` tags are written once. A re-run (or a dispatch on the same + # commit) builds only the images whose tag is missing, so a failed + # re-run cannot leave the commit's set mixed across two builds. + id: existing + if: needs.plan.outputs.publish == 'true' + env: + REF: ${{ env.NAMESPACE }}/${{ matrix.image }}:sha-${{ github.sha }} + # A registry that cannot answer fails the job rather than risk + # rebuilding over a tag that may exist. + run: | + status=0 + revision=$(.github/scripts/image-revision.sh "$REF") || status=$? + case $status in + 0) + if [ "$revision" != "$GITHUB_SHA" ]; then + echo "::error::$REF records revision $revision" + exit 1 + fi + echo "$REF already exists; not rebuilding it." + echo "exists=true" >> "$GITHUB_OUTPUT" + ;; + 3) + echo "exists=false" >> "$GITHUB_OUTPUT" + ;; + *) + echo "::error::Could not read $REF from the registry." + exit 1 + ;; + esac + + - name: Image metadata + id: meta + if: steps.existing.outputs.exists != 'true' + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 + env: + DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index + with: + images: ${{ env.NAMESPACE }}/${{ matrix.image }} + tags: type=sha,format=long + labels: | + org.opencontainers.image.title=${{ matrix.image }} + org.opencontainers.image.description=${{ matrix.description }} + annotations: | + org.opencontainers.image.title=${{ matrix.image }} + org.opencontainers.image.description=${{ matrix.description }} + + - name: Build${{ needs.plan.outputs.publish == 'true' && ' and push' || '' }} + id: build + if: steps.existing.outputs.exists != 'true' + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + file: ${{ matrix.dockerfile }} + target: ${{ matrix.target }} + platforms: linux/amd64 + push: ${{ needs.plan.outputs.publish == 'true' }} + # Without a push, keep the result in the builder only: nothing is + # loaded into the runner's Docker engine, which would need a second + # copy of the multi-gigabyte sandbox image on disk. + outputs: ${{ needs.plan.outputs.publish != 'true' && 'type=cacheonly' || '' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + annotations: ${{ steps.meta.outputs.annotations }} + # A registry cache next to each image, written only from main. + # mode=max keeps the intermediate stages (the KVM runner's package + # stage above all), which the GitHub Actions cache could not hold: + # its 10 GB repository limit would evict them between pushes. + cache-from: type=registry,ref=${{ env.NAMESPACE }}/${{ matrix.image }}:buildcache + cache-to: ${{ needs.plan.outputs.publish == 'true' && format('type=registry,ref={0}/{1}:buildcache,mode=max,ignore-error=true', env.NAMESPACE, matrix.image) || '' }} + + - name: Summarize + if: always() + env: + IMAGE_NAME: ${{ matrix.image }} + PUSHED: ${{ needs.plan.outputs.publish == 'true' && steps.build.outcome == 'success' }} + EXISTED: ${{ steps.existing.outputs.exists == 'true' }} + DIGEST: ${{ steps.build.outputs.digest }} + run: | + lowest=$(sort -n "$RUNNER_TEMP/disk-avail-mib.log" 2>/dev/null | head -n1) + { + echo "### $IMAGE_NAME" + if [ "$EXISTED" = true ]; then + echo "\`$NAMESPACE/$IMAGE_NAME:sha-$GITHUB_SHA\` was already published; left unchanged." + echo + fi + if [ "$PUSHED" = true ]; then + echo "Pushed \`$NAMESPACE/$IMAGE_NAME:sha-$GITHUB_SHA\`, pinnable as \`$NAMESPACE/$IMAGE_NAME@$DIGEST\`." + echo + fi + echo "Lowest free space on / while building: ${lowest:-unknown} MiB" + echo '```' + df -h / + docker buildx du 2>/dev/null | tail -n1 || true + echo '```' + } | tee -a "$GITHUB_STEP_SUMMARY" + + promote: + name: Move main tags + needs: [plan, build] + if: needs.plan.outputs.publish == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + # Promotions run one at a time. Each resolves its target when it runs + # instead of trusting its own commit, so a stale re-run, or a queued + # promotion that GitHub replaces, cannot leave `main` behind or move it + # backwards. + concurrency: + group: images-promote + cancel-in-progress: false + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + sparse-checkout: .github/scripts + + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Point main at the newest complete commit + # Target: the newest of the last 100 commits on main for which every + # image has a `sha-` tag. No image's `main` moves to an older commit + # than the one it names, and a registry that cannot answer fails the + # promotion instead of being read as a missing tag, so an error can + # delay a promotion but never reverse one. A registry cannot move + # several repositories' tags in one transaction, so a failure part-way + # leaves some `main` tags behind; the next promotion, or a re-run of + # this job, finishes the move. Deployments should pin `sha-` tags + # rather than `main`. + env: + GH_TOKEN: ${{ github.token }} + IMAGES: ${{ needs.plan.outputs.images }} + run: | + set -euo pipefail + + # Prints the revision a tag records, or nothing if the tag does not + # exist; exits the step if the registry cannot answer. + revision_of() { + local revision status=0 + revision=$(.github/scripts/image-revision.sh "$1") || status=$? + case $status in + 0) echo "$revision" ;; + 3) ;; + *) echo "::error::Could not read $1 from the registry." >&2; exit 1 ;; + esac + } + + complete() { + local image revision + for image in "${images[@]}"; do + revision=$(revision_of "$NAMESPACE/$image:sha-$1") || exit 1 + [ "$revision" = "$1" ] || return 1 + done + } + + retag() { + local attempt + for attempt in 1 2 3; do + docker buildx imagetools create --tag "$1:main" "$1:sha-$2" && return 0 + sleep $(( attempt * 10 )) + done + return 1 + } + + mapfile -t images < <(jq -r '.[].image' <<<"$IMAGES") + mapfile -t commits < <(gh api "repos/$GITHUB_REPOSITORY/commits?sha=main&per_page=100" --jq '.[].sha') + declare -A position + for i in "${!commits[@]}"; do position[${commits[$i]}]=$i; done + + target='' + for sha in "${commits[@]}"; do + if complete "$sha"; then + target=$sha + break + fi + done + if [ -z "$target" ]; then + echo "::error::None of the last ${#commits[@]} commits on main has all images." + exit 1 + fi + + for image in "${images[@]}"; do + ref="$NAMESPACE/$image" + current=$(revision_of "$ref:main") || exit 1 + if [ "$current" = "$target" ]; then + echo "$ref:main already at sha-$target" | tee -a "$GITHUB_STEP_SUMMARY" + continue + fi + if [ -n "$current" ] && [ -n "${position[$current]:-}" ] && + [ "${position[$current]}" -lt "${position[$target]}" ]; then + echo "::warning::$ref:main names $current, newer than $target; leaving it." + continue + fi + retag "$ref" "$target" + echo "$ref:main -> sha-$target" | tee -a "$GITHUB_STEP_SUMMARY" + done diff --git a/README.md b/README.md index b746e3c0..db4038c4 100644 --- a/README.md +++ b/README.md @@ -128,6 +128,90 @@ Versions are `vMAJOR.MINOR.PATCH`, with `-rcN` release candidates published as pre-releases. See [docs/RELEASING.md](docs/RELEASING.md) for how releases are cut. +## Prebuilt images + +The [Images workflow](.github/workflows/images.yml) builds one `linux/amd64` +image per Compose build and publishes it to GHCR whenever a push to `main` +changes an image input: + +| Image (`ghcr.io/librechat-ai/…`) | Dockerfile (target) | Compose service | +|---|---|---| +| `code-interpreter-api` | `service/Dockerfile` (`api`) | `api` | +| `code-interpreter-worker` | `service/Dockerfile` (`worker`) | `service-worker` | +| `code-interpreter-file-server` | `service/Dockerfile` (`production`) | `file_server` | +| `code-interpreter-egress-gateway` | `service/Dockerfile.egress-gateway` (`production`) | `egress_gateway` | +| `code-interpreter-tool-call-server` | `service/Dockerfile.tool-call-server` (`production`) | `tool_call_server` | +| `code-interpreter-sandbox-runner` | `api/Dockerfile` (`sandbox-runner-true`) | `sandbox-runner`, `KVM_ENABLED=true` (default) | +| `code-interpreter-sandbox-runner-direct` | `api/Dockerfile` (`sandbox-runner-false`) | `sandbox-runner`, `KVM_ENABLED=false` | + +Tags: + +- `sha-` for every commit on `main` that changes an image + input. Commits that change none (docs, Helm, tests) get no tags, so take the + SHA from a successful Images run or the package page. +- `main`, pointed at the newest commit on `main` that has all seven images. + The seven tags move one after another, so pin `sha-` tags for deployments + and treat `main` as a convenience. + +There is no `latest`. `sha-` tags are written once: re-running the workflow +for a commit builds only the images whose tag is missing. For exact bytes, +replace a service's +`image:` below with `ghcr.io/librechat-ai/@sha256:`; each image +has its own digest, listed in the summary of the run that pushed it. + +Pulling a public package needs no login. A private package needs +`docker login ghcr.io` with a token that has `read:packages`; GHCR creates new +packages as private until an organization owner changes their visibility. + +To switch a Compose host from building to pulling, save this override as +`docker-compose.images.yml` (`!reset` needs Docker Compose 2.24.4 or later): + +```yaml +services: + api: + build: !reset null + image: ghcr.io/librechat-ai/code-interpreter-api:${CODEAPI_IMAGE_TAG:?set CODEAPI_IMAGE_TAG} + pull_policy: missing + service-worker: + build: !reset null + image: ghcr.io/librechat-ai/code-interpreter-worker:${CODEAPI_IMAGE_TAG:?set CODEAPI_IMAGE_TAG} + pull_policy: missing + file_server: + build: !reset null + image: ghcr.io/librechat-ai/code-interpreter-file-server:${CODEAPI_IMAGE_TAG:?set CODEAPI_IMAGE_TAG} + pull_policy: missing + egress_gateway: + build: !reset null + image: ghcr.io/librechat-ai/code-interpreter-egress-gateway:${CODEAPI_IMAGE_TAG:?set CODEAPI_IMAGE_TAG} + pull_policy: missing + tool_call_server: + build: !reset null + image: ghcr.io/librechat-ai/code-interpreter-tool-call-server:${CODEAPI_IMAGE_TAG:?set CODEAPI_IMAGE_TAG} + pull_policy: missing + sandbox-runner: + build: !reset null + # With KVM_ENABLED=false use code-interpreter-sandbox-runner-direct, which + # still reads runtime packages from SANDBOX_PACKAGES_PATH. + image: ghcr.io/librechat-ai/code-interpreter-sandbox-runner:${CODEAPI_IMAGE_TAG:?set CODEAPI_IMAGE_TAG} + pull_policy: missing +``` + +Then pin a built commit and start the stack, listing any host override after +it. Keep the checkout at the same commit as the images, so the Compose +configuration matches what the images expect. `docker-compose.mac.yml` runs +the unpublished `sandbox-build` target with its own entrypoint, so on macOS +leave `sandbox-runner` out of this override and keep building it: + +```bash +export CODEAPI_IMAGE_TAG=sha- +docker compose -f docker-compose.yaml -f docker-compose.images.yml pull +docker compose -f docker-compose.yaml -f docker-compose.images.yml up -d +``` + +`pull_policy: missing` never re-pulls a tag the host already has, which suits +an immutable `sha-` tag. To track `main` instead, set `CODEAPI_IMAGE_TAG=main` +and run `pull` before every `up`. + ## Local Development Copy `.env.example` to `.env` and set `CODEAPI_BRIDGE_TOKEN` to a private value From 4b2b8e159225df3cb894781a003d843069503e12 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 5 Oct 2026 09:14:48 -0400 Subject: [PATCH 34/48] fix: block sandbox descriptor passing into unix socket gc (#309) --- .github/workflows/ci.yml | 28 ++++++ api/src/nsjail.test.ts | 14 +++ api/src/nsjail.ts | 7 ++ docs/SECURITY-MAINTENANCE.md | 46 +++++++++ tests/nsjail_unix_gc.py | 176 +++++++++++++++++++++++++++++++++++ 5 files changed, 271 insertions(+) create mode 100644 tests/nsjail_unix_gc.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ee14372..37f0fe2c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -138,6 +138,34 @@ jobs: oven/bun:1.3.14-debian \ bun test src/cleanup.integration.test.ts + nsjail-ipc-tests: + name: NsJail IPC Filter (${{ matrix.runner }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + runner: [ubuntu-latest, ubuntu-24.04-arm] + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.14 + - name: Render native sandbox policy + working-directory: api + run: | + bun ci + bun scripts/dump-seccomp-policy.ts > "$RUNNER_TEMP/nsjail.policy" + - name: Build pinned NsJail and Kafel + run: docker build --target nsjail-builder -t nsjail-ipc-test -f api/Dockerfile . + - name: Verify descriptor denial and runtime IPC with the installed filter + run: | + docker run --rm --security-opt seccomp=unconfined \ + --mount "type=bind,source=$RUNNER_TEMP/nsjail.policy,target=/policy,readonly" \ + --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ + nsjail-ipc-test \ + python3 /tests/nsjail_unix_gc.py /policy /nsjail/kafel/libkafel.so + service-unit-tests: name: Service Unit Tests runs-on: ubuntu-latest diff --git a/api/src/nsjail.test.ts b/api/src/nsjail.test.ts index 95de6251..eb7100f8 100644 --- a/api/src/nsjail.test.ts +++ b/api/src/nsjail.test.ts @@ -390,6 +390,20 @@ describe('NsJail seccomp policy', () => { expect(rule).not.toMatch(/== FUTEX_(WAIT|WAKE)(?:\s|$)/); }); + test('blocks descriptor-passing sends while preserving ordinary Unix socket IPC', () => { + const policy = seccompPolicy(); + const errnoBlock = policy.split('ERRNO(1) {')[1]?.split(' }')[0] ?? ''; + for (const name of ['sendmsg', 'sendmmsg', 'io_uring_setup', 'io_uring_enter', 'io_uring_register']) { + expect(errnoBlock).toMatch(new RegExp(`\\b${name}\\b[,\\s]`)); + } + // AF_UNIX/socketpair are needed by the tool-call channel and runtime IPC. + expect(policy).not.toContain('AF_UNIX'); + for (const name of ['socketpair', 'read', 'write', 'sendto', 'recvfrom']) { + expect(policy).not.toMatch(new RegExp(`\\b${name}\\b`)); + } + expect(policy).toContain('USE sandbox DEFAULT ALLOW'); + }); + test('rejects Copy Fail and Dirty Frag socket entry points in the sandbox', () => { const policy = seccompPolicy(); const errnoBlock = policy.split('ERRNO(1)')[1] ?? ''; diff --git a/api/src/nsjail.ts b/api/src/nsjail.ts index 3a5de4ad..ce1353d7 100644 --- a/api/src/nsjail.ts +++ b/api/src/nsjail.ts @@ -148,6 +148,13 @@ const SECCOMP_POLICY = [ ' },', ' ERRNO(1) {', ' io_uring_setup, io_uring_enter, io_uring_register, sched_setaffinity, vmsplice,', + /* CVE-2026-80521: AF_UNIX GC is reachable through SCM_RIGHTS descriptor + * passing. Seccomp cannot inspect the msghdr/mmsghdr control data, so + * refuse both send APIs outright, including on inherited sockets. + * Keep AF_UNIX/socketpair and read/write/sendto for /tmp/tcs.sock and + * ordinary multiprocessing IPC. io_uring above must remain blocked: + * its sendmsg operations bypass syscall-level filtering. */ + ' sendmsg, sendmmsg,', /* GhostLock (CVE-2026-43499): deny PI operations, including private and * realtime variants, without blocking ordinary runtime futex wait/wake. */ ' futex(uaddr, op) { (op & FUTEX_CMD_MASK) == FUTEX_LOCK_PI || (op & FUTEX_CMD_MASK) == FUTEX_UNLOCK_PI || (op & FUTEX_CMD_MASK) == FUTEX_TRYLOCK_PI || (op & FUTEX_CMD_MASK) == FUTEX_WAIT_REQUEUE_PI || (op & FUTEX_CMD_MASK) == FUTEX_CMP_REQUEUE_PI || (op & FUTEX_CMD_MASK) == FUTEX_LOCK_PI2 },', diff --git a/docs/SECURITY-MAINTENANCE.md b/docs/SECURITY-MAINTENANCE.md index 24c1ee01..c8494a90 100644 --- a/docs/SECURITY-MAINTENANCE.md +++ b/docs/SECURITY-MAINTENANCE.md @@ -26,6 +26,7 @@ Priority 2026 checks: |---|---| | Guest kernel; host kernel in direct mode | Bad Epoll, CVE-2026-46242 | | Guest and host kernels | GhostLock, CVE-2026-43499 | +| Guest kernel; host kernel in direct mode | AF_UNIX garbage collection, CVE-2026-80521 | | x86 host KVM | CVE-2026-46113, CVE-2026-53359, CVE-2026-64561 | | ARM host KVM, when virtual ITS is exposed | CVE-2026-46316 | @@ -37,6 +38,51 @@ VMM's behavior rather than assuming current upstream defaults match its build. The October 2026 Vercel KVM report has no disclosed root cause or affected-version matrix at the time of this change. These controls are not a verified fix for it. +## Unix-socket descriptor passing + +The NsJail policy returns EPERM for `sendmsg` and `sendmmsg` to prevent +`SCM_RIGHTS` descriptor passing into the AF_UNIX garbage collector affected by +[CVE-2026-80521](https://www.cve.org/CVERecord?id=CVE-2026-80521). +Seccomp cannot inspect ancillary data behind a userspace pointer, so both calls +are denied even for messages without descriptors and for inherited sockets. +Keep `io_uring_setup`, `io_uring_enter`, and `io_uring_register` denied: ring +operations can submit sends without passing through these syscall filters. + +Unlike the UDF sandbox's socket-family restriction, this sandbox retains +AF_UNIX and `socketpair` for `/tmp/tcs.sock` and ordinary IPC using reads, +writes, and `sendto`. Python multiprocessing pipes, queues, and pools remain +supported; transferring socket/file handles through `multiprocessing.reduction` +or similar descriptor-passing APIs returns EPERM. Do not restore `sendmsg` or +`sendmmsg` to make those APIs work without reassessing the kernel boundary. + +This mitigation is separate from PI-futex filtering. Verify vendor patches for +the deployed guest and node kernels; a policy update does not establish patch +status. Rebuild and replace sandbox runner images to deploy the policy change. + +The native `NsJail IPC Filter` CI jobs compile the rendered policy with the +image's pinned Kafel on amd64 and arm64, install it in a Linux process, and +check descriptor sends, inherited sockets, Unix HTTP, and Python fork/spawn +queues and pools. This validates the filter and IPC compatibility, not a +production kernel's patch status or the full microVM deployment. + +### Planned transport hardening + +Replace the job-visible Unix HTTP socket with two dedicated per-job pipes to +a trusted broker. Keep them separate from stdin/stdout/stderr, which carry user +input and execution output. Limit message sizes, outstanding requests, and +timeouts; retain the existing job authorization and tool-call budgets at the +broker. Update each language's tool-call client to use framed messages, and +explicitly preserve only these pipe FDs through NsJail and `spec-guard`. +The current guard closes all FDs above 2; a redesign must preserve its protection +against unrelated inherited descriptors. + +Then deny `socket` and `socketpair` in the job policy and retain the send and +io_uring denials. Validate language/runtime compatibility, including Python's +multiprocessing resource tracker and descriptor-sharing features, before making +that stricter policy the default. The broker retains network access outside the +job filter; it must not expose raw socket handles to the job. This is follow-up +work, not part of the immediate descriptor-passing mitigation. + ## Preserve host-side confinement - Run sandbox runners on isolated nodes, separate from credential-bearing diff --git a/tests/nsjail_unix_gc.py b/tests/nsjail_unix_gc.py new file mode 100644 index 00000000..84cec7b5 --- /dev/null +++ b/tests/nsjail_unix_gc.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Exercise the rendered policy with the pinned Kafel on Linux. + +Usage: python3 tests/nsjail_unix_gc.py POLICY /nsjail/kafel/libkafel.so +Run in the nsjail-builder image. No exploit or cyclic descriptor graph is used. +The filter stays installed until this test process and its children exit. +""" +import array +import ctypes as c +import errno +import http.client +import multiprocessing as mp +import os +import socket +import struct +import sys +import tempfile +import threading + + +class SockFilter(c.Structure): + _fields_ = [("code", c.c_ushort), ("jt", c.c_ubyte), + ("jf", c.c_ubyte), ("k", c.c_uint)] + + +class SockFprog(c.Structure): + _fields_ = [("len", c.c_ushort), ("filter", c.POINTER(SockFilter))] + + +class Iovec(c.Structure): + _fields_ = [("base", c.c_void_p), ("length", c.c_size_t)] + + +class Msghdr(c.Structure): + _fields_ = [("name", c.c_void_p), ("namelen", c.c_uint), + ("iov", c.POINTER(Iovec)), ("iovlen", c.c_size_t), + ("control", c.c_void_p), ("controllen", c.c_size_t), + ("flags", c.c_int)] + + +class Mmsghdr(c.Structure): + _fields_ = [("header", Msghdr), ("length", c.c_uint)] + + +def expect_eperm(call): + try: + call() + except OSError as error: + assert error.errno == errno.EPERM, error + else: + raise AssertionError("descriptor-passing send was allowed") + + +def square(value): + return value * value + + +def queue_child(queue): + queue.put("queue works") + + +def test_sends(libc, pair): + left, right = pair + # A regular-file descriptor avoids creating any socket reference cycle, + # even if a regressed policy allows the send and the assertion fails. + descriptor = os.open("/dev/null", os.O_RDONLY) + rights = [(socket.SOL_SOCKET, socket.SCM_RIGHTS, + array.array("i", [descriptor]))] + expect_eperm(lambda: left.sendmsg([b"x"], rights)) + expect_eperm(lambda: left.sendmsg([b"x"])) + # Valid batch: seccomp must deny even messages without ancillary data. + data = c.create_string_buffer(b"x") + iov = Iovec(c.cast(data, c.c_void_p), 1) + message = Mmsghdr() + message.header.iov = c.pointer(iov) + message.header.iovlen = 1 + c.set_errno(0) + assert libc.sendmmsg(left.fileno(), c.byref(message), 1, 0) == -1 + assert c.get_errno() == errno.EPERM + control = c.create_string_buffer(socket.CMSG_SPACE(c.sizeof(c.c_int))) + header = struct.pack("@Nii", socket.CMSG_LEN(c.sizeof(c.c_int)), + socket.SOL_SOCKET, socket.SCM_RIGHTS) + control[:len(header)] = header + offset = socket.CMSG_LEN(0) + control[offset:offset + c.sizeof(c.c_int)] = struct.pack("@i", descriptor) + message.header.control = c.cast(control, c.c_void_p) + message.header.controllen = socket.CMSG_SPACE(c.sizeof(c.c_int)) + assert libc.sendmmsg(left.fileno(), c.byref(message), 1, 0) == -1 + assert c.get_errno() == errno.EPERM + os.close(descriptor) + left.sendall(b"ordinary IPC") + assert right.recv(12) == b"ordinary IPC" + left.close() + right.close() + + +def test_unix_http(): + with tempfile.TemporaryDirectory() as directory: + pathname = os.path.join(directory, "tcs.sock") + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as listener: + listener.bind(pathname) + listener.listen(1) + listener.settimeout(5) + errors = [] + + def serve(): + try: + with listener.accept()[0] as peer: + peer.settimeout(5) + request = b"" + while b"\r\n\r\n" not in request: + chunk = peer.recv(4096) + assert chunk, "client disconnected before HTTP headers" + request += chunk + assert request.startswith(b"POST /tool-call HTTP/1.1") + peer.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: 11\r\n" + b"Connection: close\r\n\r\n{\"ok\":true}") + except Exception as error: + errors.append(error) + + thread = threading.Thread(target=serve) + thread.start() + connection = http.client.HTTPConnection("localhost", timeout=5) + connection.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + connection.sock.settimeout(5) + connection.sock.connect(pathname) + try: + connection.request("POST", "/tool-call", body=b"{}") + response = connection.getresponse() + assert response.status == 200 + assert response.read() == b'{"ok":true}' + finally: + connection.close() + thread.join(5) + assert not thread.is_alive() + assert not errors, errors + + +def main(): + # Open before installation to verify the rule also covers inherited FDs. + inherited = socket.socketpair() + libc = c.CDLL(None, use_errno=True) + libc.sendmmsg.argtypes = [c.c_int, c.POINTER(Mmsghdr), c.c_uint, c.c_int] + kafel = c.CDLL(sys.argv[2]) + kafel.kafel_compile_string.argtypes = [c.c_char_p, c.POINTER(SockFprog)] + program = SockFprog() + with open(sys.argv[1], "rb") as source: + assert kafel.kafel_compile_string(source.read(), c.byref(program)) == 0 + # PR_SET_NO_NEW_PRIVS, followed by PR_SET_SECCOMP / SECCOMP_MODE_FILTER. + assert libc.prctl(38, 1, 0, 0, 0) == 0 + assert libc.prctl(22, 2, c.byref(program), 0, 0) == 0, c.get_errno() + # These numbers are shared on x86_64 and arm64. No ring may bypass sends. + for syscall in (425, 426, 427): + assert libc.syscall(syscall, 0, 0, 0, 0, 0, 0) == -1 + assert c.get_errno() == errno.EPERM + test_sends(libc, inherited) + for kind in (socket.SOCK_STREAM, socket.SOCK_DGRAM): + test_sends(libc, socket.socketpair(socket.AF_UNIX, kind)) + test_unix_http() + for method in ("fork", "spawn"): + context = mp.get_context(method) + queue = context.Queue() + process = context.Process(target=queue_child, args=(queue,)) + process.start() + assert queue.get(timeout=10) == "queue works" + process.join(10) + assert process.exitcode == 0 + queue.close() + queue.join_thread() + with context.Pool(2) as pool: + assert pool.map(square, [2, 3, 4]) == [4, 9, 16] + print("PASS: descriptor sends denied; Unix IPC, HTTP, queues and pools work") + + +if __name__ == "__main__": + main() From 2318ba0d27517310586dcc4692dc53fc9d375e5c Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 5 Oct 2026 11:40:19 -0400 Subject: [PATCH 35/48] feat: replace tool-call sockets with anonymous pipes (#310) * feat: replace job-visible tool-call sockets with anonymous pipes * test: Observe Broker Cancellation Through TCP Peer * fix: Align Bun 1.4.2 and Verify Packaged Pipe Runtime * chore: Drop Node 20 Support * fix: Bound Pipe Failure Paths and Package Launcher Runtime --- .github/workflows/ci.yml | 46 +- api/Dockerfile | 22 +- api/package.json | 2 +- api/src/api/v2.ts | 4 - api/src/entrypoint.sh | 10 +- api/src/index.ts | 4 - api/src/job.ts | 6 +- api/src/nsjail.test.ts | 25 +- api/src/nsjail.ts | 55 +- api/src/session-inputs.test.ts | 1 + api/src/spec-guard.c | 28 +- api/src/tool-call-pipe-bridge.c | 123 +++ api/src/tool-call-pipe-broker.test.ts | 158 +++ api/src/tool-call-pipe-broker.ts | 165 ++++ api/src/tool-call-socket-process.test.ts | 119 --- api/src/tool-call-socket-process.ts | 191 ---- .../tool-call-socket-proxy-runtime.test.ts | 905 ------------------ api/src/tool-call-socket-proxy.test.ts | 448 --------- api/src/tool-call-socket-proxy.ts | 669 ------------- build-packages.sh | 4 +- docker/Dockerfile.worker-sandbox | 17 +- docker/package-init.sh | 2 +- docs/SECURITY-MAINTENANCE.md | 125 ++- docs/adr/0011-tool-call-pipes.md | 61 ++ docs/lambda-microvm/README.md | 2 +- docs/openshell/README.md | 2 +- docs/remote-bridge/worker-runbook.md | 2 +- helm/codeapi/values.yaml | 2 +- launcher/Dockerfile | 15 +- packages/code/README.md | 3 + packages/code/package-lock.json | 2 +- packages/code/package.json | 2 +- service/scripts/dump-pipe-preamble.ts | 6 + service/src/config.spec.ts | 6 +- service/src/config.ts | 4 +- service/src/egress-gateway.ts | 4 +- service/src/preamble.test.ts | 79 +- service/src/preamble.ts | 188 ++-- tests/nsjail_unix_gc.py | 57 +- tests/pipe_runtime_image.cjs | 61 ++ tests/specguard_fd_isolation.py | 289 ------ tests/specguard_tool_call_pipes.py | 45 + tests/tool_call_pipes.py | 171 ++++ tests/tool_call_runner.cjs | 127 +++ 44 files changed, 1321 insertions(+), 2936 deletions(-) create mode 100644 api/src/tool-call-pipe-bridge.c create mode 100644 api/src/tool-call-pipe-broker.test.ts create mode 100644 api/src/tool-call-pipe-broker.ts delete mode 100644 api/src/tool-call-socket-process.test.ts delete mode 100644 api/src/tool-call-socket-process.ts delete mode 100644 api/src/tool-call-socket-proxy-runtime.test.ts delete mode 100644 api/src/tool-call-socket-proxy.test.ts delete mode 100644 api/src/tool-call-socket-proxy.ts create mode 100644 docs/adr/0011-tool-call-pipes.md create mode 100644 service/scripts/dump-pipe-preamble.ts create mode 100644 tests/pipe_runtime_image.cjs delete mode 100644 tests/specguard_fd_isolation.py create mode 100644 tests/specguard_tool_call_pipes.py create mode 100644 tests/tool_call_pipes.py create mode 100644 tests/tool_call_runner.cjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 37f0fe2c..58f68b7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,6 +59,7 @@ jobs: run: | docker buildx build --check -f api/Dockerfile . docker buildx build --check -f docker/Dockerfile.worker-sandbox . + docker buildx build --check -f launcher/Dockerfile . launcher-unit-tests: name: Launcher Unit Tests @@ -89,15 +90,15 @@ jobs: - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.3.14 + bun-version: 1.4.2 - name: Cache Bun packages uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 with: path: ~/.bun/install/cache - key: ${{ runner.os }}-codeapi-api-bun-1.3.14-${{ hashFiles('api/bun.lock') }} + key: ${{ runner.os }}-codeapi-api-bun-1.4.2-${{ hashFiles('api/bun.lock') }} restore-keys: | - ${{ runner.os }}-codeapi-api-bun-1.3.14- + ${{ runner.os }}-codeapi-api-bun-1.4.2- - name: Install dependencies run: bun ci @@ -135,7 +136,7 @@ jobs: --workdir /work/api \ --env SANDBOX_CLEANUP_TMPFS_TEST=1 \ --env SANDBOX_LOG_LEVEL=error \ - oven/bun:1.3.14-debian \ + oven/bun:1.4.2-debian \ bun test src/cleanup.integration.test.ts nsjail-ipc-tests: @@ -150,21 +151,40 @@ jobs: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.3.14 + bun-version: 1.4.2 - name: Render native sandbox policy working-directory: api run: | bun ci bun scripts/dump-seccomp-policy.ts > "$RUNNER_TEMP/nsjail.policy" + - name: Build pipe broker and generate Python client + run: | + bun run --cwd api build + bun ci --cwd service + bun service/scripts/dump-pipe-preamble.ts > "$RUNNER_TEMP/pipe-client.py" - name: Build pinned NsJail and Kafel - run: docker build --target nsjail-builder -t nsjail-ipc-test -f api/Dockerfile . + run: docker build --target nsjail-ipc-test -t nsjail-ipc-test -f api/Dockerfile . + - name: Verify standalone launcher guest pipe runtime + run: | + docker build --target sandbox-build -t launcher-guest-test -f launcher/Dockerfile . + docker run --rm --security-opt seccomp=unconfined \ + --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ + --entrypoint /usr/local/bin/node \ + launcher-guest-test /tests/pipe_runtime_image.cjs - name: Verify descriptor denial and runtime IPC with the installed filter run: | docker run --rm --security-opt seccomp=unconfined \ --mount "type=bind,source=$RUNNER_TEMP/nsjail.policy,target=/policy,readonly" \ --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ nsjail-ipc-test \ - python3 /tests/nsjail_unix_gc.py /policy /nsjail/kafel/libkafel.so + bash -c 'python3 /tests/nsjail_unix_gc.py /policy /nsjail/kafel/libkafel.so && python3 /tests/specguard_tool_call_pipes.py && gcc -O2 -static -Wall -Wextra -Werror -DSPEC_GUARD_TEST_DISABLE_CLOSE_RANGE -o /tmp/guard-fallback /tmp/spec-guard.c && SPEC_GUARD_BINARY=/tmp/guard-fallback python3 /tests/specguard_tool_call_pipes.py' + docker run --rm --security-opt seccomp=unconfined \ + --mount "type=bind,source=$RUNNER_TEMP/nsjail.policy,target=/policy,readonly" \ + --mount "type=bind,source=$RUNNER_TEMP/pipe-client.py,target=/client.py,readonly" \ + --mount "type=bind,source=$GITHUB_WORKSPACE/api/.build/tool-call-pipe-broker.cjs,target=/broker.cjs,readonly" \ + --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ + nsjail-ipc-test \ + python3 /tests/tool_call_pipes.py /policy /broker.cjs /client.py service-unit-tests: name: Service Unit Tests @@ -177,15 +197,15 @@ jobs: - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.3.14 + bun-version: 1.4.2 - name: Cache Bun packages uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 with: path: ~/.bun/install/cache - key: ${{ runner.os }}-codeapi-service-bun-1.3.14-${{ hashFiles('service/bun.lock') }} + key: ${{ runner.os }}-codeapi-service-bun-1.4.2-${{ hashFiles('service/bun.lock') }} restore-keys: | - ${{ runner.os }}-codeapi-service-bun-1.3.14- + ${{ runner.os }}-codeapi-service-bun-1.4.2- - name: Install dependencies run: bun ci @@ -212,7 +232,7 @@ jobs: strategy: fail-fast: false matrix: - node-version: ['20.11.0', '22.21.0', '24.16.0'] + node-version: ['22.21.0', '24.16.0'] defaults: run: working-directory: packages/code @@ -301,7 +321,7 @@ jobs: - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.3.14 + bun-version: 1.4.2 - name: Install service dependencies working-directory: service @@ -373,7 +393,7 @@ jobs: docker buildx inspect --bootstrap - name: Build Lambda MicroVM runner image - # The target pins Bun 1.3.14 and installs api/bun.lock frozen. Building + # The target pins Bun 1.4.2 and installs api/bun.lock frozen. Building # natively catches architecture-specific package and Docker layer # failures without retaining a second multi-gigabyte image copy. run: | diff --git a/api/Dockerfile b/api/Dockerfile index 66db9e3d..0ee343a7 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -29,11 +29,18 @@ RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \ RUN make -j$(nproc) COPY api/src/spec-guard.c /tmp/spec-guard.c +COPY api/src/tool-call-pipe-bridge.c /tmp/tool-call-pipe-bridge.c COPY docker/rootfs-setup.c /tmp/rootfs-setup.c -RUN gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ +RUN gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/tool-call-pipe-bridge /tmp/tool-call-pipe-bridge.c \ + && gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ && gcc -O2 -static -o /usr/local/bin/sandbox-rootfs-setup /tmp/rootfs-setup.c \ && chmod 0111 /usr/local/bin/spec-guard /usr/local/bin/sandbox-rootfs-setup +# Native policy/transport verification without building the full runner. +FROM nsjail-builder AS nsjail-ipc-test +COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node +COPY --from=oven/bun:1.4.2-debian /usr/local/bin/bun /usr/local/bin/bun + # ============================================================================ # Stage 1b: Build language runtime packages (only consumed by sandbox-runner-baked) # @@ -46,7 +53,7 @@ FROM buildpack-deps:bookworm AS package-builder ENV DEBIAN_FRONTEND=noninteractive ENV PYTHON_VERSION=3.14.4 -ENV BUN_VERSION=1.3.14 +ENV BUN_VERSION=1.4.2 COPY docker/apt-install.sh /usr/local/bin/apt-install RUN chmod +x /usr/local/bin/apt-install @@ -68,7 +75,7 @@ RUN chmod +x /package-init.sh && FORCE_REBUILD=true /package-init.sh # ============================================================================ # Stage 2: Build sandbox rootfs # ============================================================================ -FROM oven/bun:1.3.14-debian AS sandbox-build +FROM oven/bun:1.4.2-debian AS sandbox-build ENV DEBIAN_FRONTEND=noninteractive @@ -110,14 +117,11 @@ RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen COPY --from=nsjail-builder /nsjail/nsjail /usr/sbin/nsjail COPY --from=nsjail-builder /usr/local/bin/spec-guard /usr/local/bin/spec-guard +COPY --from=nsjail-builder /usr/local/bin/tool-call-pipe-bridge /usr/local/bin/tool-call-pipe-bridge RUN chmod +x /usr/sbin/nsjail -# The tool-call socket proxy must run under Node, not Bun. Bun's node:http -# compat layer never fires `'connection'` events and Bun.serve's idleTimeout -# does not close silent unix-socket connections — so our DoS protections -# (maxConnections, idleSocketTimeoutMs, accept-time tracking) silently no-op -# under Bun and a `for i in range(N): socket.connect` storm pins FDs in the -# proxy until the OS limit. Pulling just the Node binary from a slim image +# The per-invocation broker starts Node after restore, so its OpenSSL process +# state is not captured in the MicroVM snapshot. Pulling the Node binary # is ~30 MB and avoids inheriting npm/node_modules from a full apt install. COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node diff --git a/api/package.json b/api/package.json index de62fb55..a27fd780 100644 --- a/api/package.json +++ b/api/package.json @@ -5,7 +5,7 @@ "main": "src/index.ts", "scripts": { "dev": "bun run --watch src/index.ts", - "build": "bun build ./src/index.ts --minify --outdir .build --target bun --external '@opentelemetry/*' && bun build ./src/tool-call-socket-proxy.ts --target=node --format=cjs --outfile=.build/tool-call-socket-proxy.cjs", + "build": "bun build ./src/index.ts --minify --outdir .build --target bun --external '@opentelemetry/*' && bun build ./src/tool-call-pipe-broker.ts --target=node --format=cjs --outfile=.build/tool-call-pipe-broker.cjs", "start": "bun run .build/index.js", "test": "bun test" }, diff --git a/api/src/api/v2.ts b/api/src/api/v2.ts index 3ace08cf..cba8e80a 100644 --- a/api/src/api/v2.ts +++ b/api/src/api/v2.ts @@ -26,7 +26,6 @@ import { parseSessionBindingFromHeader, } from '../session-workspace'; import { streamSessionCheckpoint, restoreSessionCheckpoint } from '../session-checkpoint'; -import { ensureToolCallSocketProxyReady } from '../tool-call-socket-process'; import { SESSION_INPUT_CACHE_MAX_OBJECTS, hasCachedInput, @@ -579,9 +578,6 @@ router.post('/execute', express.json({ limit: config.execute_body_limit }), asyn } try { - if (toolCallSocketEnabled) { - await ensureToolCallSocketProxyReady(); - } await withSpan('codeapi.sandbox.prime', { 'codeapi.language': job.runtime.language, }, () => job!.prime()); diff --git a/api/src/entrypoint.sh b/api/src/entrypoint.sh index 4fce5ba4..d670a359 100755 --- a/api/src/entrypoint.sh +++ b/api/src/entrypoint.sh @@ -123,8 +123,8 @@ fi ALLOWED_PORT="${SANDBOX_ALLOWED_LOCAL_NETWORK_PORT:-0}" # NsJail runs sandboxed processes as inside UID 65534 (nobody), mapped to a -# per-job outside UID by the sandbox API. The proxy socket remains mounted -# into each jail, so it does not need per-UID ownership. +# per-job outside UID by the sandbox API. Authorized tool-call invocations +# receive two anonymous pipe ends; no socket path is mounted into a jail. SANDBOX_UID=65534 # Extract API port from bind address (default 2000) @@ -136,11 +136,11 @@ if [ "$ALLOWED_PORT" -gt 0 ] 2>/dev/null; then exit 1 fi - echo "Tool-call socket forwarding enabled for UID $SANDBOX_UID (port $ALLOWED_PORT)" + echo "Tool-call pipe forwarding enabled for UID $SANDBOX_UID (port $ALLOWED_PORT)" # Do not start Node here. Lambda MicroVM image creation snapshots this # already-running container, and the official Node binary embeds OpenSSL. - # The API starts and awaits the narrow Unix-socket proxy only after a - # post-restore /execute is authorized for tool calls. + # The API starts a per-invocation pipe broker only after a post-restore + # /execute is authorized for tool calls. fi # Package permissions are finalized by package-init when the PVC is populated. diff --git a/api/src/index.ts b/api/src/index.ts index cfbc936c..00d10e9d 100644 --- a/api/src/index.ts +++ b/api/src/index.ts @@ -7,7 +7,6 @@ import { initializeSandboxWorkspaceIsolation, startWorkspaceReaper } from './wor import { httpMetricsMiddleware, metricsHandler } from './metrics'; import { positiveInt, shutdownTelemetry, traceHttpRequest } from './telemetry'; import { startWarmupCommand } from './warmup'; -import { stopToolCallSocketProxy } from './tool-call-socket-process'; import { hostedAppSupervisor, validateHostedAppStartup } from './hosted-app'; import v2Router from './api/v2'; import lifecycleRouter, { LIFECYCLE_HOOK_BASE_PATH } from './api/lifecycle'; @@ -116,9 +115,6 @@ async function main(): Promise { await hostedAppSupervisor.shutdown().catch((err) => { logger.warn({ err }, 'Hosted-app process shutdown failed'); }); - await stopToolCallSocketProxy().catch((err) => { - logger.warn({ err }, 'Tool-call socket proxy shutdown failed'); - }); try { await shutdownTelemetry(); } catch (err) { diff --git a/api/src/job.ts b/api/src/job.ts index 5c2fddef..bfa99b63 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -701,7 +701,7 @@ export class Job { memory_limits: { run: number; compile: number }; extra_env_vars?: Record; egressGrantToken?: string; - toolCallSocketEnabled: boolean; + toolCallPipesEnabled: boolean; isSynthetic: boolean; outputSessionId: string; @@ -786,7 +786,7 @@ export class Job { this.memory_limits = opts.memory_limits; this.extra_env_vars = opts.extra_env_vars; this.egressGrantToken = opts.egress_grant; - this.toolCallSocketEnabled = opts.tool_call_socket_enabled === true; + this.toolCallPipesEnabled = opts.tool_call_socket_enabled === true; this.isSynthetic = opts.is_synthetic === true; } @@ -1644,7 +1644,7 @@ export class Job { stdin, extraPkgdirs, identity: this.sandboxIdentity(), - enableToolCallSocket: this.toolCallSocketEnabled && script === 'run', + enableToolCallPipes: this.toolCallPipesEnabled && script === 'run', suppressSuccessLogs: this.isSynthetic, }); } diff --git a/api/src/nsjail.test.ts b/api/src/nsjail.test.ts index eb7100f8..73dc3863 100644 --- a/api/src/nsjail.test.ts +++ b/api/src/nsjail.test.ts @@ -91,7 +91,7 @@ describe('NsJail args', () => { expect(valueAfter(args, '--config')).toBe('/tmp/nsjail-job-xyz.cfg'); }); - test('does not export TOOL_CALL_SOCKET into the jail (preamble references the literal path)', () => { + test('does not export a legacy tool-call socket path into the jail', () => { const args = buildArgs({ logPath: '/tmp/nsjail-test.log', pkgdir: '/pkgs/python/3.14.4', @@ -109,7 +109,7 @@ describe('NsJail args', () => { } }); - test('binds the tool-call socket only for jobs that explicitly request it', () => { + test('passes tool-call pipes only for jobs that explicitly request the capability', () => { const originalAllowedPort = config.allowed_local_network_port; config.allowed_local_network_port = 3190; try { @@ -126,8 +126,12 @@ describe('NsJail args', () => { const withoutSocket = buildArgs(baseOpts); expect(hasArgPair(withoutSocket, '-B', '/tmp/tcs.sock:/tmp/tcs.sock')).toBe(false); - const withSocket = buildArgs({ ...baseOpts, enableToolCallSocket: true }); - expect(hasArgPair(withSocket, '-B', '/tmp/tcs.sock:/tmp/tcs.sock')).toBe(true); + const withSocket = buildArgs({ ...baseOpts, enableToolCallPipes: true }); + expect(hasArgPair(withSocket, '-B', '/tmp/tcs.sock:/tmp/tcs.sock')).toBe(false); + expect(hasArgPair(withSocket, '--pass_fd', '3')).toBe(true); + expect(hasArgPair(withSocket, '--pass_fd', '4')).toBe(true); + expect(withSocket).toContain('--tool-call-pipes'); + expect(withoutSocket).not.toContain('--pass_fd'); } finally { config.allowed_local_network_port = originalAllowedPort; } @@ -371,7 +375,7 @@ describe('NsJail seccomp policy', () => { const errnoSocketRule = errnoBlock.split('\n').find(line => line.includes('socket(domain)')); expect(errnoSocketRule).toBeDefined(); - expect(errnoSocketRule).not.toContain('AF_VSOCK'); + expect(errnoSocketRule).toContain('domain != AF_VSOCK'); }); test('blocks priority-inheritance futex operations without denying ordinary wait/wake', () => { @@ -396,9 +400,9 @@ describe('NsJail seccomp policy', () => { for (const name of ['sendmsg', 'sendmmsg', 'io_uring_setup', 'io_uring_enter', 'io_uring_register']) { expect(errnoBlock).toMatch(new RegExp(`\\b${name}\\b[,\\s]`)); } - // AF_UNIX/socketpair are needed by the tool-call channel and runtime IPC. - expect(policy).not.toContain('AF_UNIX'); - for (const name of ['socketpair', 'read', 'write', 'sendto', 'recvfrom']) { + // Restrict socketpairs to runtime stream IPC; tool calls use real pipes. + expect(policy).toContain('socketpair(domain, type, protocol) { domain != AF_UNIX || (type & 0xf) != SOCK_STREAM || protocol != 0 }'); + for (const name of ['read', 'write', 'sendto', 'recvfrom']) { expect(policy).not.toMatch(new RegExp(`\\b${name}\\b`)); } expect(policy).toContain('USE sandbox DEFAULT ALLOW'); @@ -409,10 +413,7 @@ describe('NsJail seccomp policy', () => { const errnoBlock = policy.split('ERRNO(1)')[1] ?? ''; const socketRule = errnoBlock.split('\n').find(line => line.includes('socket(domain)')); expect(socketRule).toBeDefined(); - for (const family of ['AF_ALG', 'AF_RXRPC', 'AF_INET', 'AF_INET6']) { - expect(socketRule).toContain(family); - } - expect(socketRule).not.toContain('AF_VSOCK'); + expect(socketRule).toContain('domain != AF_VSOCK'); }); test('KILLs the defense-in-depth batch from the audit', () => { diff --git a/api/src/nsjail.ts b/api/src/nsjail.ts index ce1353d7..51b8700e 100644 --- a/api/src/nsjail.ts +++ b/api/src/nsjail.ts @@ -95,6 +95,8 @@ const archSpecificLowPrioritySyscalls = process.arch === 'arm64' const SECCOMP_POLICY = [ ...syscallDefines, + '#define AF_UNIX 1', + '#define SOCK_STREAM 1', '#define AF_INET 2', '#define AF_INET6 10', '#define AF_NETLINK 16', @@ -151,8 +153,9 @@ const SECCOMP_POLICY = [ /* CVE-2026-80521: AF_UNIX GC is reachable through SCM_RIGHTS descriptor * passing. Seccomp cannot inspect the msghdr/mmsghdr control data, so * refuse both send APIs outright, including on inherited sockets. - * Keep AF_UNIX/socketpair and read/write/sendto for /tmp/tcs.sock and - * ordinary multiprocessing IPC. io_uring above must remain blocked: + * The job receives only anonymous tool-call pipes. Restricted Unix + * stream socketpairs remain for asyncio and subprocess/runtime IPC. + * io_uring above must remain blocked: * its sendmsg operations bypass syscall-level filtering. */ ' sendmsg, sendmmsg,', /* GhostLock (CVE-2026-43499): deny PI operations, including private and @@ -181,10 +184,11 @@ const SECCOMP_POLICY = [ * runtimes the sandbox supports. */ ' pidfd_open(pid) { pid == 1 },', ' pidfd_send_signal,', - /* Block direct network and kernel-control socket domains from sandboxed - * code. AF_ALG covers the Linux kernel crypto API used by Copy Fail, and - * AF_RXRPC covers the RxRPC family used by Dirty Frag. */ - ' socket(domain) { domain == AF_INET || domain == AF_INET6 || domain == AF_NETLINK || domain == AF_KEY || domain == AF_RXRPC || domain == AF_ALG }', + /* Deny every socket() family. VSOCK keeps its stronger KILL action above. + * Only anonymous Unix stream socketpairs are allowed for runtime IPC; + * sendmsg/sendmmsg remain denied, so no SCM_RIGHTS graph can be built. */ + ' socket(domain) { domain != AF_VSOCK },', + ' socketpair(domain, type, protocol) { domain != AF_UNIX || (type & 0xf) != SOCK_STREAM || protocol != 0 }', ' }', '}', 'USE sandbox DEFAULT ALLOW', @@ -242,7 +246,7 @@ interface ExecuteOptions { stdin?: string; extraPkgdirs?: string[]; identity: SandboxJobIdentity; - enableToolCallSocket?: boolean; + enableToolCallPipes?: boolean; suppressSuccessLogs?: boolean; } @@ -258,7 +262,7 @@ export async function execute(opts: ExecuteOptions, setupGate: NsJailSetupGate = stdin, extraPkgdirs, identity, - enableToolCallSocket, + enableToolCallPipes, suppressSuccessLogs, } = opts; const logId = nanoid(); @@ -277,7 +281,7 @@ export async function execute(opts: ExecuteOptions, setupGate: NsJailSetupGate = command, extraPkgdirs, identity, - enableToolCallSocket, + enableToolCallPipes, }); const startTime = Date.now(); @@ -399,11 +403,20 @@ export async function execute(opts: ExecuteOptions, setupGate: NsJailSetupGate = const childDiedSignal = new AbortController(); try { ({ value: proc, markerSeen, pollError } = await setupGate.runSetup(logPath, () => { - const child = spawn(config.nsjail_path, nsjailArgs, { + const pipeMode = enableToolCallPipes === true; + if (pipeMode && (config.allowed_local_network_port <= 0 || !process.env.SANDBOX_FORWARD_TARGET?.trim())) { + throw new Error('tool-call pipe broker is not configured'); + } + const child = spawn(pipeMode ? (process.env.TCS_PIPE_BRIDGE || '/usr/local/bin/tool-call-pipe-bridge') : config.nsjail_path, pipeMode ? [ + '--broker', process.env.TCS_NODE_BINARY || 'node', + process.env.TCS_PIPE_BROKER_BUNDLE ?? path.resolve(__dirname, '..', '.build', 'tool-call-pipe-broker.cjs'), + config.nsjail_path, ...nsjailArgs, + ] : nsjailArgs, { /* stdin always 'pipe' so we have a writable handle to .end() even * when no input is supplied. The child sees EOF on stdin * immediately, equivalent to the previous Bun.spawn 'ignore'. */ stdio: ['pipe', 'pipe', 'pipe'], + env: { ...process.env, TCS_REQUEST_TIMEOUT_MS: String(timeout) }, }) as ChildProcessWithoutNullStreams; /* Attach the 'error' listener SYNCHRONOUSLY before any await. The * setup gate then polls the log file for ~tens of ms; during that @@ -654,7 +667,7 @@ interface BuildArgsOptions { command: string[]; extraPkgdirs?: string[]; identity: SandboxJobIdentity; - enableToolCallSocket?: boolean; + enableToolCallPipes?: boolean; } export function buildArgs(opts: BuildArgsOptions): string[] { @@ -668,7 +681,7 @@ export function buildArgs(opts: BuildArgsOptions): string[] { command, extraPkgdirs, identity, - enableToolCallSocket, + enableToolCallPipes, } = opts; const timeoutSecs = Math.max(1, Math.ceil(timeout / 1000)); @@ -721,25 +734,17 @@ export function buildArgs(opts: BuildArgsOptions): string[] { args.push('--cgroup_mem_max', String(memoryLimit)); } - if (config.allowed_local_network_port > 0 && enableToolCallSocket === true) { - const socketPath = '/tmp/tcs.sock'; - args.push('-B', `${socketPath}:${socketPath}`); + if (enableToolCallPipes === true) { + args.push('--pass_fd', '3', '--pass_fd', '4'); } for (const [key, value] of Object.entries(envVars)) { args.push('-E', `${key}=${value}`); } - /* TOOL_CALL_SOCKET is intentionally NOT exported: the path is fixed at - * /tmp/tcs.sock and the blocking-mode runtime preamble references it - * as a literal. - * Skipping the env entry shrinks the surface user code can introspect - * (`os.environ`). The per-job /tmp/tcs.sock bind-mount above is what - * actually makes the path connectable, and ordinary execute/replay jobs - * do not receive that mount. */ - - args.push('--'); - args.push('/usr/local/bin/spec-guard', ...command); + args.push('--', '/usr/local/bin/spec-guard'); + if (enableToolCallPipes === true) args.push('--tool-call-pipes'); + args.push(...command); return args; } diff --git a/api/src/session-inputs.test.ts b/api/src/session-inputs.test.ts index 639ccadd..6efdb610 100644 --- a/api/src/session-inputs.test.ts +++ b/api/src/session-inputs.test.ts @@ -151,6 +151,7 @@ describe('pushed input cache', () => { const readOnly = cachedInputResponse((await openCachedInput('s1', 'ro'))!); expect(readOnly.headers.get('x-read-only')).toBe('true'); + await readOnly.text(); // Consume the response so its FileHandle is closed. }); test('an opened cache hit remains readable when concurrent pruning unlinks its path', async () => { diff --git a/api/src/spec-guard.c b/api/src/spec-guard.c index 31f153f9..4ce56de4 100644 --- a/api/src/spec-guard.c +++ b/api/src/spec-guard.c @@ -34,8 +34,10 @@ # endif #endif +static int first_untrusted_fd = 3; + static void close_fd_loop(long start, long end) { - if (start < 3) start = 3; + if (start < first_untrusted_fd) start = first_untrusted_fd; if (end > INT_MAX) end = INT_MAX; if (start > end) return; for (long fd = start; fd <= end; fd++) { @@ -54,7 +56,7 @@ static bool close_proc_fds(void) { char *end = NULL; long fd = strtol(entry->d_name, &end, 10); if (end == entry->d_name || *end != '\0') continue; - if (fd < 3 || fd > (long)INT_MAX) continue; + if (fd < first_untrusted_fd || fd > (long)INT_MAX) continue; if ((int)fd == dir_fd) continue; /* don't close our iterator */ close((int)fd); } @@ -62,7 +64,8 @@ static bool close_proc_fds(void) { return true; } -/* Close every file descriptor >= 3 (preserve stdin/stdout/stderr) before +/* Close every unrelated descriptor (preserve stdin/stdout/stderr and, + * only with --tool-call-pipes, the validated pipe endpoints 3/4) before * the user command runs. The bug this prevents: * * (1) sandbox API / proxy / NsJail allocate FDs in the runner; @@ -86,7 +89,7 @@ static bool close_proc_fds(void) { */ static void close_inherited_fds(void) { #if defined(__NR_close_range) && !defined(SPEC_GUARD_TEST_DISABLE_CLOSE_RANGE) - long rc = syscall(__NR_close_range, (unsigned int)3, ~0U, 0u); + long rc = syscall(__NR_close_range, (unsigned int)first_untrusted_fd, ~0U, 0u); if (rc == 0) return; /* ENOSYS on pre-5.9 kernels, EINVAL on some odd builds. Fall through. */ #endif @@ -123,9 +126,24 @@ int main(int argc, char *argv[]) { umask(0077); + int command = 1; + if (strcmp(argv[1], "--tool-call-pipes") == 0) { + struct stat writer, reader; + int write_flags = fcntl(3, F_GETFL), read_flags = fcntl(4, F_GETFL); + if (argc < 3 || fstat(3, &writer) != 0 || fstat(4, &reader) != 0 + || !S_ISFIFO(writer.st_mode) || !S_ISFIFO(reader.st_mode) + || write_flags < 0 || read_flags < 0 + || (write_flags & O_ACCMODE) != O_WRONLY + || (read_flags & O_ACCMODE) != O_RDONLY) { + fprintf(stderr, "invalid tool-call pipes\n"); + return 1; + } + first_untrusted_fd = 5; + command = 2; + } close_inherited_fds(); - execvp(argv[1], &argv[1]); + execvp(argv[command], &argv[command]); perror("exec"); return 1; } diff --git a/api/src/tool-call-pipe-bridge.c b/api/src/tool-call-pipe-bridge.c new file mode 100644 index 00000000..a3e9f084 --- /dev/null +++ b/api/src/tool-call-pipe-bridge.c @@ -0,0 +1,123 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef __linux__ +#include +#endif + +/* Node/Bun extra stdio uses socketpairs. Keep that trusted-side socket in + * this relay; only two anonymous pipe ends ever reach NsJail/the job. + * FD 3 in the job writes requests; FD 4 reads responses. */ +struct flow { + int source, destination; + unsigned char bytes[65536]; + size_t start, length; +}; + +static int nonblock(int fd) { + int flags = fcntl(fd, F_GETFL); + return flags < 0 ? -1 : fcntl(fd, F_SETFL, flags | O_NONBLOCK); +} + +static int transfer(struct flow *flow, short readable, short writable) { + if (writable && flow->length) { + ssize_t n = write(flow->destination, flow->bytes + flow->start, flow->length); + if (n > 0) { + flow->start += (size_t)n; + flow->length -= (size_t)n; + if (!flow->length) flow->start = 0; + } else if (n < 0 && errno != EINTR && errno != EAGAIN) return -1; + } + if (flow->source >= 0 && readable && !flow->length) { + ssize_t n = read(flow->source, flow->bytes, sizeof(flow->bytes)); + if (n > 0) flow->length = (size_t)n; + else if (n == 0) flow->source = -1; + else if (errno != EINTR && errno != EAGAIN) return -1; + } + return 0; +} + +static int parent_death(pid_t parent) { +#ifdef __linux__ + if (prctl(PR_SET_PDEATHSIG, SIGKILL) != 0) return -1; +#endif + return getppid() == parent ? 0 : -1; +} + +int main(int argc, char **argv) { + if (argc < 2) return 125; + pid_t broker = getppid(); + if (parent_death(broker) != 0) return 125; + /* The API also enters here to exec Node with a parent-death signal. + * Without this, an API crash could orphan a still-running broker. */ + if (strcmp(argv[1], "--broker") == 0) { + if (argc < 3) return 125; + execvp(argv[2], &argv[2]); + perror("tool-call broker exec"); + return 125; + } + int ipc = fcntl(3, F_DUPFD_CLOEXEC, 5); + int requests[2], responses[2]; + if (ipc < 0 || pipe(requests) != 0 || pipe(responses) != 0) return 125; + pid_t relay = getpid(); + pid_t child = fork(); + if (child < 0) return 125; + if (child == 0) { + if (parent_death(relay) != 0 || dup2(requests[1], 3) < 0 + || dup2(responses[0], 4) < 0) _exit(125); + int fds[] = {ipc, requests[0], requests[1], responses[0], responses[1]}; + for (unsigned i = 0; i < sizeof(fds) / sizeof(fds[0]); i++) { + if (fds[i] > 4) close(fds[i]); + } + execvp(argv[1], &argv[1]); + perror("tool-call pipe exec"); + _exit(125); + } + close(3); + close(requests[1]); + close(responses[0]); + signal(SIGPIPE, SIG_IGN); + if (nonblock(ipc) < 0 || nonblock(requests[0]) < 0 || nonblock(responses[1]) < 0) { + kill(child, SIGKILL); + waitpid(child, NULL, 0); + return 125; + } + struct flow request = {.source = requests[0], .destination = ipc}; + struct flow response = {.source = ipc, .destination = responses[1]}; + int status = 125 << 8; + for (;;) { + pid_t done = waitpid(child, &status, WNOHANG); + if (done == child) break; + if (done < 0 && errno != EINTR) return 125; + struct pollfd pollfds[] = { + {.fd = ipc, .events = (response.length || response.source < 0 ? 0 : POLLIN) | (request.length ? POLLOUT : 0)}, + {.fd = request.source, .events = request.length ? 0 : POLLIN}, + {.fd = responses[1], .events = response.length ? POLLOUT : 0}, + }; + int result = poll(pollfds, 3, 100); + if (result < 0 && errno == EINTR) continue; + if (result < 0 || (pollfds[0].revents & (POLLERR | POLLNVAL)) + || (pollfds[1].revents & (POLLERR | POLLNVAL)) + || (pollfds[2].revents & (POLLERR | POLLHUP | POLLNVAL)) + || transfer(&request, pollfds[1].revents & (POLLIN | POLLHUP), pollfds[0].revents & POLLOUT) < 0 + || transfer(&response, pollfds[0].revents & (POLLIN | POLLHUP), pollfds[2].revents & POLLOUT) < 0 || response.source < 0) { + /* EOF/error terminates the entire invocation, including an + * abandoned tool call. Never leave an orphaned NsJail monitor. */ + if (waitpid(child, &status, WNOHANG) == child) break; + kill(child, SIGKILL); + while (waitpid(child, &status, 0) < 0 && errno == EINTR) {} + break; + } + } + close(ipc); + close(requests[0]); + close(responses[1]); + return WIFEXITED(status) ? WEXITSTATUS(status) : 128 + WTERMSIG(status); +} diff --git a/api/src/tool-call-pipe-broker.test.ts b/api/src/tool-call-pipe-broker.test.ts new file mode 100644 index 00000000..a55c6c33 --- /dev/null +++ b/api/src/tool-call-pipe-broker.test.ts @@ -0,0 +1,158 @@ +import { afterEach, describe, expect, test, spyOn } from 'bun:test'; +import * as http from 'node:http'; +import * as net from 'node:net'; +import { once } from 'node:events'; +import { MAX_FRAME_BYTES, serveToolCallPipe } from './tool-call-pipe-broker'; + +const cleanups: (() => void)[] = []; +afterEach(() => { for (const close of cleanups.splice(0).reverse()) close(); }); + +async function setup(handler: http.RequestListener, options = {}, target?: string): Promise<{ + client: net.Socket; replies: () => Promise; send: (value: unknown) => void; +}> { + const upstream = http.createServer(handler); + upstream.listen(0, '127.0.0.1'); + await once(upstream, 'listening'); + cleanups.push(() => { upstream.closeAllConnections(); upstream.close(); }); + const server = net.createServer(channel => { + cleanups.push(serveToolCallPipe(channel, target ?? `http://127.0.0.1:${(upstream.address() as net.AddressInfo).port}`, options)); + }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + cleanups.push(() => server.close()); + const client = net.createConnection((server.address() as net.AddressInfo).port, '127.0.0.1'); + await once(client, 'connect'); + cleanups.push(() => client.destroy()); + let buffered = Buffer.alloc(0); + const queued: any[] = []; + const waiters: ((response: any) => void)[] = []; + client.on('data', bytes => { + buffered = Buffer.concat([buffered, typeof bytes === 'string' ? Buffer.from(bytes) : bytes]); + while (buffered.length >= 4 && buffered.length >= 4 + buffered.readUInt32BE()) { + const size = buffered.readUInt32BE(); + const value = JSON.parse(buffered.subarray(4, 4 + size).toString()); + buffered = buffered.subarray(4 + size); + const waiter = waiters.shift(); + if (waiter) waiter(value); else queued.push(value); + } + }); + return { client, replies: () => queued.length ? Promise.resolve(queued.shift()) : new Promise(resolve => waiters.push(resolve)), + send: value => { const frame = Buffer.from(JSON.stringify(value)); const prefix = Buffer.alloc(4); prefix.writeUInt32BE(frame.length); client.write(Buffer.concat([prefix, frame])); } }; +} + +const claims = { 'x-execution-id': 'exec', 'x-callback-token': 'token', 'x-tool-call-id': 'call' }; +const frame = (id: number) => ({ id, headers: claims, body: '{"tool_name":"echo","input":{}}' }); + +describe('per-invocation tool-call pipe broker', () => { + test('forwards only its fixed route and whitelisted claims, preserving JSON errors', async () => { + let observed = false; + const harness = await setup((req, res) => { + observed = true; + expect(req.url).toBe('/tool-call'); + expect(req.method).toBe('POST'); + expect(req.headers['x-callback-token']).toBe('token'); + expect(req.headers['x-forged']).toBeUndefined(); + req.resume(); + res.writeHead(404).end('{"success":false,"error":"Session not found"}'); + }); + harness.send({ ...frame(7), path: '/admin', headers: { ...claims, 'x-forged': 'secret' } }); + expect(await harness.replies()).toEqual({ id: 7, status: 404, body: '{"success":false,"error":"Session not found"}' }); + expect(observed).toBe(true); + }); + + test('isolates identical frame IDs across invocation channels', async () => { + const first = await setup((_req, res) => res.end('first')); + const second = await setup((_req, res) => res.end('second')); + first.send(frame(1)); + second.send(frame(1)); + const replies = await Promise.all([first.replies(), second.replies()]); + expect(replies.map(reply => reply.body)).toEqual(['first', 'second']); + }); + + test('rejects missing claims before opening an upstream connection', async () => { + let count = 0; + const harness = await setup((_req, res) => { count++; res.end(); }); + harness.send({ ...frame(1), headers: { ...claims, 'x-callback-token': ', , ' } }); + expect((await harness.replies()).status).toBe(400); + expect(count).toBe(0); + }); + + test('returns redirects without following them', async () => { + let count = 0; + const harness = await setup((_req, res) => { count++; res.writeHead(302, { Location: '/credential-sink' }).end('redirect'); }); + harness.send(frame(1)); + expect((await harness.replies()).status).toBe(302); + expect(count).toBe(1); + }); + + test('bounds outstanding upstream calls and aborts them when the channel closes', async () => { + let upstreamClosed = false; + let received: () => void; + const started = new Promise(resolve => { received = resolve; }); + // Observe the actual TCP peer: Bun 1.3.14's HTTP IncomingMessage socket + // does not emit close when an unfinished response loses its connection. + const upstream = net.createServer(socket => { + socket.once('close', () => { upstreamClosed = true; }); + socket.once('data', () => received()); + socket.resume(); + }); + upstream.listen(0, '127.0.0.1'); + await once(upstream, 'listening'); + cleanups.push(() => upstream.close()); + const harness = await setup(() => {}, { maxActiveRequests: 1 }, + `http://127.0.0.1:${(upstream.address() as net.AddressInfo).port}`); + harness.send(frame(1)); + await started; + harness.send(frame(2)); + expect((await harness.replies()).status).toBe(429); + harness.client.destroy(); + for (let n = 0; n < 100 && !upstreamClosed; n++) await Bun.sleep(5); + expect(upstreamClosed).toBe(true); + }); + + test.each(['invalid claims', 'invalid body', 'active limit'])('closes a drained channel when %s exhausts the frame budget', async kind => { + const clock = spyOn(Date, 'now').mockReturnValue(1_000); + try { + let count = 0; + const harness = await setup(req => { count++; req.resume(); }, { maxActiveRequests: 1 }); + const ended = once(harness.client, 'close'); + for (let id = 1; id <= 65; id++) { + harness.send(kind === 'invalid claims' ? { ...frame(id), headers: {} } + : kind === 'invalid body' ? { ...frame(id), body: null } : frame(id)); + } + await ended; // setup drains replies continuously, so backpressure cannot hide the bypass. + expect(count).toBeLessThanOrEqual(kind === 'active limit' ? 1 : 0); + } finally { clock.mockRestore(); } + }); + + test('enforces an absolute deadline on a stalled upstream', async () => { + const harness = await setup(req => req.resume(), { requestTimeoutMs: 30 }); + harness.send(frame(1)); + expect((await harness.replies()).status).toBe(504); + }); + + test('rejects oversized upstream responses without buffering an unbounded body', async () => { + const harness = await setup((_req, res) => res.end(Buffer.alloc(MAX_FRAME_BYTES + 1, 'x'))); + harness.send(frame(1)); + const response = await harness.replies(); + expect(response.status).toBe(502); + expect(response.body).toContain('too large'); + }); + + test('closes an oversized frame before any upstream request', async () => { + let count = 0; + const harness = await setup((_req, res) => { count++; res.end(); }); + const ended = once(harness.client, 'close'); + const prefix = Buffer.alloc(4); prefix.writeUInt32BE(MAX_FRAME_BYTES + 1); + harness.client.write(prefix); + await ended; + expect(count).toBe(0); + }); + + test('bounds a partial-header slow writer', async () => { + const harness = await setup((_req, res) => res.end(), { frameTimeoutMs: 30 }); + const ended = once(harness.client, 'close'); + harness.client.write(Buffer.from([0])); + await ended; + }); +}); diff --git a/api/src/tool-call-pipe-broker.ts b/api/src/tool-call-pipe-broker.ts new file mode 100644 index 00000000..4fb923cc --- /dev/null +++ b/api/src/tool-call-pipe-broker.ts @@ -0,0 +1,165 @@ +import { spawn } from 'node:child_process'; +import type { Duplex } from 'node:stream'; +import * as http from 'node:http'; +import * as https from 'node:https'; + +export const MAX_FRAME_BYTES = 1024 * 1024; +const HEADER_NAMES = ['x-execution-id', 'x-callback-token', 'x-tool-call-id'] as const; + +/** One broker/channel per authorized invocation. The target and route come + * from trusted runner configuration; the job supplies only PTC claims/body. + * The upstream still authenticates those claims and enforces tool budgets. */ +export function serveToolCallPipe(channel: Duplex, rawTarget: string, options: { + requestTimeoutMs?: number; frameTimeoutMs?: number; maxActiveRequests?: number; + onFailure?: () => void; +} = {}): () => void { + const target = new URL(rawTarget.includes('://') ? rawTarget : `http://${rawTarget}`); + if (!['http:', 'https:'].includes(target.protocol) || target.username || target.password) { + throw new Error('invalid tool-call broker target'); + } + const transport = target.protocol === 'https:' ? https : http; + const requests = new Map(); + const timeout = options.requestTimeoutMs ?? 300_000; + const frameTimeout = options.frameTimeoutMs ?? 5_000; + const maxActive = options.maxActiveRequests ?? 16; + let closed = false; + let frameTimer: ReturnType | undefined; + let drainTimer: ReturnType | undefined; + let header = Buffer.alloc(4); + let offset = 0; + let frame: Buffer | undefined; + let tokens = 64; + let lastRefill = Date.now(); + const close = (): void => { + if (closed) return; + closed = true; + clearTimeout(frameTimer); + clearTimeout(drainTimer); + for (const request of requests.values()) request.destroy(); + requests.clear(); + channel.destroy(); + }; + const fail = (): void => { close(); options.onFailure?.(); }; + const reply = (id: number, status: number, body: string): void => { + if (closed) return; + let bytes = Buffer.from(JSON.stringify({ id, status, body })); + if (bytes.length > MAX_FRAME_BYTES) { + bytes = Buffer.from(JSON.stringify({ id, status: 502, body: JSON.stringify({ success: false, error: 'tool-call response too large' }) })); + } + const prefix = Buffer.alloc(4); + prefix.writeUInt32BE(bytes.length); + if (!channel.write(Buffer.concat([prefix, bytes]))) { + channel.pause(); + drainTimer ??= setTimeout(fail, frameTimeout); + } + }; + const reject = (id: number, status: number, error: string): void => reply(id, status, JSON.stringify({ success: false, error })); + const accept = (bytes: Buffer): void => { + // Charge every complete frame before JSON/claim validation, including + // rejected concurrency requests. Exhaustion closes the capability rather + // than generating an unlimited stream of cheap 429/400 replies. + const now = Date.now(); + tokens = Math.min(64, tokens + Math.max(0, now - lastRefill) * 0.02); + lastRefill = now; + if (tokens < 1) { fail(); return; } + tokens -= 1; + let parsed: { id?: unknown; headers?: unknown; body?: unknown }; + try { parsed = JSON.parse(bytes.toString('utf8')); } catch { fail(); return; } + if (!parsed || typeof parsed !== 'object' || !Number.isSafeInteger(parsed.id) + || (parsed.id as number) < 1 || requests.has(parsed.id as number)) { fail(); return; } + const id = parsed.id as number; + if (!parsed.headers || typeof parsed.headers !== 'object' || Array.isArray(parsed.headers) || typeof parsed.body !== 'string') { + reject(id, 400, 'invalid tool-call frame'); return; + } + const headers: http.OutgoingHttpHeaders = { 'content-type': 'application/json', connection: 'close' }; + for (const name of HEADER_NAMES) { + const value = (parsed.headers as Record)[name]; + if (typeof value !== 'string' || !value.replace(/[,\s]/g, '') || value.length > 8192 || /[^\x20-\x7e]/.test(value)) { + reject(id, 400, 'missing or invalid tool-call claims'); return; + } + headers[name] = value; + } + if (requests.size >= maxActive) { reject(id, 429, 'tool-call request budget exceeded'); return; } + const body = Buffer.from(parsed.body); + headers['content-length'] = body.length; + let deadline: ReturnType; + let finished = false; + const finish = (status: number, response: string): void => { + if (finished) return; + finished = true; + clearTimeout(deadline); + requests.delete(id); + reply(id, status, response); + }; + const upstream = transport.request({ protocol: target.protocol, hostname: target.hostname, + port: target.port || undefined, method: 'POST', path: '/tool-call', headers, agent: false }, response => { + const chunks: Buffer[] = []; + let size = 0; + response.on('data', (chunk: Buffer) => { + size += chunk.length; + if (size > MAX_FRAME_BYTES) { + finish(502, JSON.stringify({ success: false, error: 'tool-call response too large' })); + upstream.destroy(); + } else chunks.push(chunk); + }); + response.on('end', () => finish(response.statusCode ?? 502, Buffer.concat(chunks).toString('utf8'))); + response.on('error', () => finish(502, JSON.stringify({ success: false, error: 'tool-call upstream disconnected' }))); + response.on('aborted', () => finish(502, JSON.stringify({ success: false, error: 'tool-call upstream disconnected' }))); + }); + requests.set(id, upstream); + deadline = setTimeout(() => { + finish(504, JSON.stringify({ success: false, error: 'tool-call upstream timeout' })); + upstream.destroy(); + }, timeout); + upstream.on('error', () => finish(502, JSON.stringify({ success: false, error: 'tool-call upstream unavailable' }))); + upstream.end(body); + }; + channel.on('data', (chunk: Buffer) => { + let cursor = 0; + while (cursor < chunk.length && !closed) { + frameTimer ??= setTimeout(fail, frameTimeout); + const buffer = frame ?? header; + const count = Math.min(buffer.length - offset, chunk.length - cursor); + chunk.copy(buffer, offset, cursor, cursor + count); + offset += count; + cursor += count; + if (offset !== buffer.length) continue; + offset = 0; + if (!frame) { + const length = header.readUInt32BE(); + if (length < 1 || length > MAX_FRAME_BYTES) { fail(); break; } + frame = Buffer.allocUnsafe(length); + } else { + const complete = frame; + frame = undefined; + clearTimeout(frameTimer); + frameTimer = undefined; + accept(complete); + } + } + }); + channel.on('drain', () => { clearTimeout(drainTimer); drainTimer = undefined; channel.resume(); }); + channel.on('error', fail); + channel.on('end', fail); + channel.on('close', close); + return close; +} + +// Node starts only after restore, on an authenticated job's execution path. +// Never create a persistent listener or snapshot Node's TLS process state. +if (require.main === module) { + const target = process.env.SANDBOX_FORWARD_TARGET?.trim(); + if (!target || !process.argv[2]) throw new Error('tool-call pipe broker is not configured'); + const child = spawn(process.env.TCS_PIPE_BRIDGE || '/usr/local/bin/tool-call-pipe-bridge', process.argv.slice(2), { + stdio: [0, 1, 2, 'pipe'], + }); + const channel = child.stdio[3] as Duplex; + const stop = serveToolCallPipe(channel, target, { + requestTimeoutMs: Math.min(600_000, Math.max(5_000, Number(process.env.TCS_REQUEST_TIMEOUT_MS) || 300_000)), + onFailure: () => child.kill('SIGKILL'), + }); + child.on('error', () => { stop(); process.exitCode = 125; }); + child.on('exit', (code, signal) => { stop(); process.exitCode = code ?? (signal === 'SIGKILL' ? 137 : 125); }); + process.once('SIGTERM', () => { stop(); child.kill('SIGKILL'); }); + process.once('SIGINT', () => { stop(); child.kill('SIGKILL'); }); +} diff --git a/api/src/tool-call-socket-process.test.ts b/api/src/tool-call-socket-process.test.ts deleted file mode 100644 index 3efe2a4b..00000000 --- a/api/src/tool-call-socket-process.test.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { afterEach, describe, expect, test } from 'bun:test'; -import * as fsp from 'fs/promises'; -import * as os from 'os'; -import * as path from 'path'; -import { config } from './config'; -import { - ensureToolCallSocketProxyReady, - stopToolCallSocketProxy, -} from './tool-call-socket-process'; - -const originalPort = config.allowed_local_network_port; -const originalEnv = { - target: process.env.SANDBOX_FORWARD_TARGET, - socket: process.env.TCS_SOCKET, - bundle: process.env.TCS_PROXY_BUNDLE, - node: process.env.TCS_NODE_BINARY, - marker: process.env.TCS_TEST_MARKER, -}; -let tempDir: string | undefined; - -function restoreEnv(name: string, value: string | undefined): void { - if (value === undefined) delete process.env[name]; - else process.env[name] = value; -} - -afterEach(async () => { - await stopToolCallSocketProxy(); - config.allowed_local_network_port = originalPort; - restoreEnv('SANDBOX_FORWARD_TARGET', originalEnv.target); - restoreEnv('TCS_SOCKET', originalEnv.socket); - restoreEnv('TCS_PROXY_BUNDLE', originalEnv.bundle); - restoreEnv('TCS_NODE_BINARY', originalEnv.node); - restoreEnv('TCS_TEST_MARKER', originalEnv.marker); - if (tempDir) await fsp.rm(tempDir, { recursive: true, force: true }); - tempDir = undefined; -}); - -describe('lazy tool-call socket process', () => { - test('starts once on demand, awaits readiness, and stops cleanly', async () => { - tempDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'lazy-tcs-')); - const socketPath = path.join(tempDir, 'proxy.sock'); - const markerPath = path.join(tempDir, 'launches.txt'); - const bundlePath = path.join(tempDir, 'fake-proxy.cjs'); - await fsp.writeFile(bundlePath, ` - const fs = require('fs'); - const net = require('net'); - try { fs.unlinkSync(process.env.TCS_SOCKET); } catch {} - fs.appendFileSync(process.env.TCS_TEST_MARKER, 'launch\\n'); - const server = net.createServer(socket => socket.end()); - server.listen(process.env.TCS_SOCKET); - const stop = () => server.close(() => process.exit(0)); - process.on('SIGTERM', stop); - process.on('SIGINT', stop); - `); - - config.allowed_local_network_port = 443; - process.env.SANDBOX_FORWARD_TARGET = 'https://gateway.example'; - process.env.TCS_SOCKET = socketPath; - process.env.TCS_PROXY_BUNDLE = bundlePath; - process.env.TCS_TEST_MARKER = markerPath; - - await Promise.all([ - ensureToolCallSocketProxyReady(), - ensureToolCallSocketProxyReady(), - ensureToolCallSocketProxyReady(), - ]); - - expect((await fsp.lstat(socketPath)).isSocket()).toBe(true); - expect((await fsp.readFile(markerPath, 'utf8')).trim().split('\n')).toHaveLength(1); - - /* A live child can lose its socket pathname (for example after an external - * cleanup). A previously-resolved launch promise must not mask that loss. */ - await fsp.unlink(socketPath); - await Promise.all([ - ensureToolCallSocketProxyReady(), - ensureToolCallSocketProxyReady(), - ensureToolCallSocketProxyReady(), - ]); - expect((await fsp.lstat(socketPath)).isSocket()).toBe(true); - expect((await fsp.readFile(markerPath, 'utf8')).trim().split('\n')).toHaveLength(2); - - await stopToolCallSocketProxy(); - expect(await fsp.lstat(socketPath).catch(() => null)).toBeNull(); - }); - - test('fails closed when forwarding is not configured', async () => { - config.allowed_local_network_port = 0; - process.env.SANDBOX_FORWARD_TARGET = ''; - await expect(ensureToolCallSocketProxyReady()).rejects.toThrow('not configured'); - }); - - test('reports a spawn failure without crashing and permits a later retry', async () => { - tempDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'lazy-tcs-spawn-')); - const socketPath = path.join(tempDir, 'proxy.sock'); - const bundlePath = path.join(tempDir, 'fake-proxy.cjs'); - await fsp.writeFile(bundlePath, ` - const fs = require('fs'); - const net = require('net'); - try { fs.unlinkSync(process.env.TCS_SOCKET); } catch {} - const server = net.createServer(socket => socket.end()); - server.listen(process.env.TCS_SOCKET); - process.on('SIGTERM', () => server.close(() => process.exit(0))); - `); - - config.allowed_local_network_port = 443; - process.env.SANDBOX_FORWARD_TARGET = 'https://gateway.example'; - process.env.TCS_SOCKET = socketPath; - process.env.TCS_PROXY_BUNDLE = bundlePath; - process.env.TCS_NODE_BINARY = path.join(tempDir, 'missing-node'); - - await expect(ensureToolCallSocketProxyReady()).rejects.toThrow( - 'tool-call socket proxy failed to spawn', - ); - - process.env.TCS_NODE_BINARY = process.execPath; - await ensureToolCallSocketProxyReady(); - expect((await fsp.lstat(socketPath)).isSocket()).toBe(true); - }); -}); diff --git a/api/src/tool-call-socket-process.ts b/api/src/tool-call-socket-process.ts deleted file mode 100644 index 4694c07f..00000000 --- a/api/src/tool-call-socket-process.ts +++ /dev/null @@ -1,191 +0,0 @@ -import { spawn, type ChildProcess } from 'child_process'; -import * as net from 'net'; -import * as path from 'path'; -import { logger } from './logger'; -import { config } from './config'; - -const START_TIMEOUT_MS = 10_000; -const STOP_TIMEOUT_MS = 3_000; - -let child: ChildProcess | undefined; -let starting: Promise | undefined; - -function proxyBundlePath(): string { - return process.env.TCS_PROXY_BUNDLE - ?? path.resolve(__dirname, '..', '.build', 'tool-call-socket-proxy.cjs'); -} - -function proxySocketPath(): string { - return process.env.TCS_SOCKET || '/tmp/tcs.sock'; -} - -function proxyOwner(): { uid: string; gid: string } { - const runningAsRoot = typeof process.getuid === 'function' && process.getuid() === 0; - return { - uid: String(runningAsRoot ? 65534 : (process.getuid?.() ?? 65534)), - gid: String(runningAsRoot ? 65534 : (process.getgid?.() ?? 65534)), - }; -} - -function socketAcceptsConnections(socketPath: string): Promise { - return new Promise((resolve) => { - const socket = net.createConnection(socketPath); - let settled = false; - const finish = (ready: boolean): void => { - if (settled) return; - settled = true; - socket.destroy(); - resolve(ready); - }; - socket.once('connect', () => finish(true)); - socket.once('error', () => finish(false)); - socket.setTimeout(100, () => finish(false)); - }); -} - -async function waitUntilReady( - started: ChildProcess, - socketPath: string, - timeoutMs: number, -): Promise { - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - if (started.exitCode !== null || started.signalCode !== null) { - throw new Error( - `tool-call socket proxy exited before readiness` - + ` (code=${String(started.exitCode)}, signal=${String(started.signalCode)})`, - ); - } - if (await socketAcceptsConnections(socketPath)) return; - await new Promise(resolve => setTimeout(resolve, 25)); - } - throw new Error(`tool-call socket proxy did not become ready within ${timeoutMs}ms`); -} - -async function launch(): Promise { - const rawTarget = process.env.SANDBOX_FORWARD_TARGET?.trim() ?? ''; - if (config.allowed_local_network_port <= 0 || !rawTarget) { - throw new Error('tool-call socket proxy is not configured'); - } - - const socketPath = proxySocketPath(); - const owner = proxyOwner(); - const started = spawn(process.env.TCS_NODE_BINARY || 'node', [proxyBundlePath()], { - env: { - ...process.env, - TCS_SOCKET: socketPath, - TCS_SOCKET_UID: owner.uid, - TCS_SOCKET_GID: owner.gid, - SANDBOX_FORWARD_TARGET: rawTarget, - }, - stdio: ['ignore', 'pipe', 'pipe'], - }); - child = started; - let rejectSpawnFailure: (error: Error) => void; - const spawnFailure = new Promise((_resolve, reject) => { - rejectSpawnFailure = reject; - }); - /* A failed spawn emits `error`; it does not reliably set exitCode or - * signalCode. Without an immediate listener Node treats ENOENT/EACCES as an - * uncaught event and crashes the runner instead of failing this execution. */ - started.on('error', (error) => { - if (child === started) child = undefined; - const launchError = new Error( - `tool-call socket proxy failed to spawn: ${error.message}`, - { cause: error }, - ); - logger.error({ error }, 'tool-call socket proxy spawn failed'); - rejectSpawnFailure(launchError); - }); - started.stdout?.on('data', (chunk: Buffer) => { - logger.debug({ proxy: chunk.toString().trim() }, 'tool-call socket proxy'); - }); - started.stderr?.on('data', (chunk: Buffer) => { - logger.warn({ proxy: chunk.toString().trim() }, 'tool-call socket proxy stderr'); - }); - started.once('exit', (code, signal) => { - if (child === started) { - child = undefined; - } - if (code !== 0 && signal !== 'SIGTERM') { - logger.error({ code, signal }, 'tool-call socket proxy exited unexpectedly'); - } - }); - - try { - await Promise.race([ - waitUntilReady(started, socketPath, START_TIMEOUT_MS), - spawnFailure, - ]); - logger.info( - { socketPath, target: rawTarget }, - 'Tool-call socket proxy started after MicroVM restore', - ); - } catch (error) { - started.kill('SIGKILL'); - if (child === started) child = undefined; - throw error; - } -} - -/** - * Starts Node only when an authenticated execute actually receives the - * tool-call socket capability. Lambda MicroVM image creation snapshots the - * already-running container; starting Node in entrypoint would therefore clone - * its embedded OpenSSL process state into every VM. Lazy initialization keeps - * Node entirely on the post-restore side of that boundary. - */ -async function ensureToolCallSocketProxyReadyOnce(): Promise { - const socketPath = proxySocketPath(); - const running = child; - if (running && running.exitCode === null && running.signalCode === null) { - if (await socketAcceptsConnections(socketPath)) return; - running.kill('SIGKILL'); - if (child === running) child = undefined; - } - await launch(); -} - -export async function ensureToolCallSocketProxyReady(): Promise { - /* Serialize the whole probe/relaunch transaction, not only spawn readiness. - * Otherwise concurrent recovery callers can both await a failed socket probe, - * then each kill/relaunch through the shared `child` slot. Publishing this - * promise before the first probe await makes every follower join one attempt. */ - if (starting) { - await starting; - return; - } - const attempt = ensureToolCallSocketProxyReadyOnce(); - starting = attempt; - try { - await attempt; - } finally { - /* A successful launch must not leave a resolved promise cached forever: - * if the socket later disappears while the process is still alive, the - * next caller has to kill/relaunch rather than return false readiness. */ - if (starting === attempt) starting = undefined; - } -} - -export async function stopToolCallSocketProxy(): Promise { - const running = child; - child = undefined; - starting = undefined; - if (!running || running.exitCode !== null || running.signalCode !== null) return; - - await new Promise((resolve) => { - let settled = false; - const finish = (): void => { - if (settled) return; - settled = true; - clearTimeout(timer); - resolve(); - }; - const timer = setTimeout(() => { - running.kill('SIGKILL'); - finish(); - }, STOP_TIMEOUT_MS); - running.once('exit', finish); - running.kill('SIGTERM'); - }); -} diff --git a/api/src/tool-call-socket-proxy-runtime.test.ts b/api/src/tool-call-socket-proxy-runtime.test.ts deleted file mode 100644 index 496f017b..00000000 --- a/api/src/tool-call-socket-proxy-runtime.test.ts +++ /dev/null @@ -1,905 +0,0 @@ -/** - * Runtime regression tests for the tool-call socket proxy. - * - * The tests in `tool-call-socket-proxy.test.ts` instantiate the proxy - * in-process via `startToolCallSocketProxy()` under bun:test. That is fine - * for verifying request-handler logic, but it CANNOT catch the audit DoS - * because Bun's node:http compat layer never fires `'connection'` events - * and Bun.serve's idleTimeout doesn't close silent unix-socket connects — - * so `activeConnections()`, `maxConnections`, and `idleSocketTimeoutMs` - * silently no-op when the proxy is hosted by Bun. The in-process tests - * pass vacuously (counters stay 0) while the production runtime is - * unprotected. - * - * In production the proxy runs under Node (see api/Dockerfile + the - * entrypoint.sh edit). These tests spawn the BUILT proxy bundle as a - * Node subprocess so the runtime under test matches production exactly. - * - * Skipped automatically when: - * - `node` isn't on PATH (CI without Node) - * - `.build/tool-call-socket-proxy.cjs` doesn't exist (test invoked - * before `bun run build`) - * - we're on Windows (unix sockets in temp dirs hit AppData EACCES - * quirks; production is Linux) - */ -import { afterEach, beforeAll, beforeEach, describe, expect, test } from 'bun:test'; -import { spawn, spawnSync, type ChildProcess } from 'node:child_process'; -import * as fs from 'node:fs'; -import * as fsp from 'node:fs/promises'; -import * as http from 'node:http'; -import * as net from 'node:net'; -import * as os from 'node:os'; -import * as path from 'node:path'; - -const PROXY_BUNDLE = path.resolve(__dirname, '..', '.build', 'tool-call-socket-proxy.cjs'); -const NODE_AVAILABLE = (() => { - try { return spawnSync('node', ['--version'], { stdio: 'ignore' }).status === 0; } - catch { return false; } -})(); -const SHOULD_RUN = NODE_AVAILABLE && fs.existsSync(PROXY_BUNDLE) && process.platform !== 'win32'; - -const describeIfRuntime = SHOULD_RUN ? describe : describe.skip; - -interface UpstreamHandle { - server: http.Server; - url: string; - calls: () => number; - close: () => Promise; -} - -interface ProxyHandle { - socketPath: string; - proc: ChildProcess; - stop: () => Promise; -} - -let workdir: string; - -beforeAll(() => { - if (!NODE_AVAILABLE) { - console.warn('[tcs-proxy-runtime] skipping: `node` not on PATH'); - } else if (!fs.existsSync(PROXY_BUNDLE)) { - console.warn(`[tcs-proxy-runtime] skipping: ${PROXY_BUNDLE} missing — run \`bun run build\` first`); - } else if (process.platform === 'win32') { - console.warn('[tcs-proxy-runtime] skipping: Windows (unix-socket EACCES quirk); production is Linux'); - } -}); - -beforeEach(async () => { - workdir = await fsp.mkdtemp(path.join(os.tmpdir(), 'tcs-proxy-runtime-')); -}); - -afterEach(async () => { - if (workdir) await fsp.rm(workdir, { recursive: true, force: true }); -}); - -async function startUpstream(handler?: http.RequestListener): Promise { - let calls = 0; - const server = http.createServer((req, res) => { - calls++; - if (handler) return handler(req, res); - let total = 0; - req.on('data', chunk => { total += chunk.length; }); - req.on('end', () => { - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ ok: true, received: total })); - }); - }); - await new Promise((resolve, reject) => { - server.once('error', reject); - server.listen(0, '127.0.0.1', resolve); - }); - const addr = server.address(); - if (!addr || typeof addr === 'string') throw new Error('upstream listen failed'); - return { - server, - url: `http://127.0.0.1:${addr.port}`, - calls: () => calls, - close: () => new Promise(r => server.close(() => r())), - }; -} - -async function spawnProxy(opts: { upstreamUrl: string; env?: Record }): Promise { - const socketPath = path.join(workdir, `tcs-${Math.random().toString(36).slice(2)}.sock`); - const proc = spawn('node', [PROXY_BUNDLE], { - env: { - ...process.env, - TCS_SOCKET: socketPath, - SANDBOX_FORWARD_TARGET: opts.upstreamUrl, - ...(opts.env || {}), - }, - stdio: ['ignore', 'pipe', 'pipe'], - }); - - /* Wait for the proxy to be connectable. fs.existsSync on a unix socket - * is unreliable on some platforms; try-connect-then-disconnect is the - * authoritative readiness probe. */ - let ready = false; - for (let i = 0; i < 50; i++) { - try { - const probe = net.createConnection(socketPath); - await new Promise((res, rej) => { - probe.once('connect', () => { probe.destroy(); res(); }); - probe.once('error', rej); - }); - ready = true; - break; - } catch { /* not yet */ } - await new Promise(r => setTimeout(r, 100)); - } - if (!ready) { - proc.kill('SIGKILL'); - throw new Error('proxy never became connectable'); - } - - return { - socketPath, - proc, - stop: async () => { - proc.kill('SIGTERM'); - await new Promise(r => { - const killTimer = setTimeout(() => { proc.kill('SIGKILL'); r(); }, 1000); - proc.once('exit', () => { - clearTimeout(killTimer); - r(); - }); - }); - }, - }; -} - -function rawConnect(socketPath: string): Promise { - return new Promise((resolve, reject) => { - const sock = net.createConnection(socketPath); - sock.once('connect', () => resolve(sock)); - sock.once('error', reject); - }); -} - -function slowHeaderConnect( - socketPath: string, - dripEveryMs: number, -): Promise<{ socket: net.Socket; close: () => void }> { - return new Promise((resolve, reject) => { - const sock = net.createConnection(socketPath); - let drip: ReturnType | undefined; - const clearDrip = (): void => { - if (drip) clearInterval(drip); - drip = undefined; - }; - const onInitialError = (error: Error): void => { - clearDrip(); - reject(error); - }; - - sock.once('error', onInitialError); - sock.once('connect', () => { - sock.off('error', onInitialError); - sock.on('error', () => { /* connection was destroyed by the proxy */ }); - sock.once('close', clearDrip); - sock.write('POST /tool-call HTTP/1.1\r\nHost: x\r\nX-Drip: '); - drip = setInterval(() => { - if (sock.destroyed) { - clearDrip(); - return; - } - try { sock.write('x'); } catch { clearDrip(); } - }, dripEveryMs); - resolve({ - socket: sock, - close: () => { - clearDrip(); - sock.destroy(); - }, - }); - }); - }); -} - -/* The proxy's PTC-presence filter rejects /tool-call requests that lack - * the SDK-supplied headers (canonical 404 — see proxy source for why). - * Tests that want to exercise the forwarding path supply matching shape - * here. Cryptographic validation is the upstream's job; presence is the - * proxy's. */ -const PTC_HEADERS = { - 'X-Execution-ID': 'test-exec-id', - 'X-Tool-Call-ID': 'test-call-001', - 'X-Callback-Token': 'test-callback-token', -} as const; - -function postToolCall( - socketPath: string, - body: string, - extraHeaders: Record = {}, -): Promise<{ status: number; body: string }> { - return new Promise((resolve, reject) => { - const req = http.request( - { - socketPath, - method: 'POST', - path: '/tool-call', - headers: { - 'Content-Type': 'application/json', - 'Content-Length': Buffer.byteLength(body), - ...PTC_HEADERS, - ...extraHeaders, - }, - }, - res => { - const chunks: Buffer[] = []; - res.on('data', c => chunks.push(c)); - res.on('end', () => resolve({ status: res.statusCode ?? 0, body: Buffer.concat(chunks).toString() })); - }, - ); - req.on('error', reject); - req.write(body); - req.end(); - }); -} - -describeIfRuntime('tool-call-socket-proxy under Node runtime (production parity)', () => { - test('forwards a normal POST /tool-call', async () => { - const upstream = await startUpstream(); - const proxy = await spawnProxy({ upstreamUrl: upstream.url }); - try { - const res = await postToolCall(proxy.socketPath, '{"x":1}'); - expect(res.status).toBe(200); - expect(JSON.parse(res.body)).toEqual({ ok: true, received: 7 }); - expect(upstream.calls()).toBe(1); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('SILENT-CONNECTION DoS: idle accept-then-do-nothing connections close before they pin FDs', async () => { - /* This is the audit's actual attack: - * for i in range(N): socket.connect('/tmp/tcs.sock') - * Open many connections without writing anything. Without the Node - * runtime + working `'connection'` events + working socket.setTimeout, - * each accepted FD sits in the proxy until the OS limit. With them - * all working, every FD must be reclaimed within a small multiple of - * idleSocketTimeoutMs. */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ - upstreamUrl: upstream.url, - env: { /* defaults: idle 2s, max 64. Test against the production defaults. */ }, - }); - try { - const sockets: net.Socket[] = []; - for (let i = 0; i < 32; i++) { - try { sockets.push(await rawConnect(proxy.socketPath)); } - catch { /* over-cap rejection is fine — also "didn't pin a FD" */ } - } - /* Wait for idle timer (default 2000ms) + slack. */ - const start = Date.now(); - const closeTimes = await Promise.all(sockets.map(s => new Promise(resolve => { - if (s.destroyed) { resolve(0); return; } - s.once('close', () => resolve(Date.now() - start)); - setTimeout(() => resolve(-1), 8000); - }))); - const closed = closeTimes.filter(t => t >= 0).length; - const stuck = closeTimes.filter(t => t < 0).length; - expect(stuck).toBe(0); - /* All idle FDs must be reclaimed within a small multiple of the - * default idleSocketTimeoutMs (2s). Slack is generous. */ - const maxClose = Math.max(...closeTimes); - expect(maxClose).toBeLessThan(7000); - expect(closed).toBeGreaterThan(0); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('legitimate POST still succeeds after a silent-connection storm clears', async () => { - /* Regression guard for the actual user-facing failure: while - * malicious code spams idle connections, real tool calls from - * sibling sandboxes must still succeed. With the default cap (64) - * and idleSocketTimeoutMs=2s, we open the storm, wait ~2.5s for - * the proxy to reclaim those FDs, then verify a real POST works. */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ upstreamUrl: upstream.url }); - const stormSockets: net.Socket[] = []; - try { - for (let i = 0; i < 32; i++) { - try { stormSockets.push(await rawConnect(proxy.socketPath)); } - catch { /* over-cap rejection is fine */ } - } - /* Let the proxy's idleSocketTimeoutMs (default 2s) reclaim FDs. */ - await new Promise(r => setTimeout(r, 2500)); - const res = await postToolCall(proxy.socketPath, '{"during":"storm"}'); - expect(res.status).toBe(200); - expect(JSON.parse(res.body)).toEqual({ ok: true, received: 18 }); - } finally { - stormSockets.forEach(s => { try { s.destroy(); } catch { /* ignore */ } }); - await proxy.stop(); - await upstream.close(); - } - }); - - test('CIRCUMVENTION: slow incomplete headers cannot starve all connection slots', async () => { - /* Header-drip clients are more subtle than silent sockets: they send - * bytes under the idle timeout, so socket.setTimeout never fires. The - * proxy must still reclaim each accepted socket on an absolute - * accept-to-headers deadline before all maxConnections slots can be held. */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ - upstreamUrl: upstream.url, - env: { - TCS_MAX_CONNECTIONS: '4', - TCS_HEADER_TIMEOUT_MS: '500', - TCS_CONNECTION_RATE_BURST: '32', - TCS_CONNECTION_RATE_REFILL_PER_SEC: '200', - }, - }); - const drips: Array<{ socket: net.Socket; close: () => void }> = []; - - try { - for (let i = 0; i < 4; i++) { - drips.push(await slowHeaderConnect(proxy.socketPath, 100)); - } - - await new Promise(r => setTimeout(r, 900)); - - const res = await postToolCall(proxy.socketPath, '{"x":1}'); - expect(res.status).toBe(200); - expect(JSON.parse(res.body)).toEqual({ ok: true, received: 7 }); - expect(upstream.calls()).toBe(1); - } finally { - drips.forEach(drip => drip.close()); - await proxy.stop(); - await upstream.close(); - } - }, { timeout: 10_000 }); - - test('CIRCUMVENTION: HTTP request smuggling (Content-Length + Transfer-Encoding) does not reach upstream', async () => { - /* The classic CL.TE / TE.CL smuggling primitive: send both headers - * with conflicting interpretations. If the proxy and upstream - * disagree on body boundary, the attacker can prepend bytes to a - * synthesized "second request" that bypasses the path filter and - * could reach a different upstream route. The defense per RFC 7230 - * §3.3.3 is to reject the request rather than try to reconcile. - * Whether node:http rejects in its own parser or our isSmugglingShaped - * gate fires first, the security property is the same: upstream - * must never see the smuggled bytes. */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ upstreamUrl: upstream.url }); - try { - const sock = await rawConnect(proxy.socketPath); - const smuggled = - 'POST /tool-call HTTP/1.1\r\n' + - 'Host: x\r\n' + - 'Content-Length: 13\r\n' + - 'Transfer-Encoding: chunked\r\n' + - 'Connection: close\r\n' + - '\r\n' + - '0\r\n\r\n' + - 'POST /admin HTTP/1.1\r\nHost: x\r\nContent-Length: 0\r\n\r\n'; - sock.write(smuggled); - await new Promise(resolve => { - sock.once('close', () => resolve()); - setTimeout(resolve, 3000); - }); - /* Critical security property: upstream MUST NOT have been called. */ - expect(upstream.calls()).toBe(0); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('CIRCUMVENTION: hop-by-hop headers (Connection, Upgrade, TE, Trailer, Transfer-Encoding) do not reach upstream', async () => { - /* RFC 7230 §6.1: a proxy MUST NOT forward hop-by-hop headers. If we - * leak them through, an upstream that treats Upgrade specially could - * be coerced into an unexpected protocol switch, and Transfer- - * Encoding pollution feeds smuggling chains at the next hop. */ - let receivedHeaders: http.IncomingHttpHeaders = {}; - const upstream = await startUpstream((req, res) => { - receivedHeaders = req.headers; - let total = 0; - req.on('data', c => { total += c.length; }); - req.on('end', () => { - res.writeHead(200, { 'Content-Type': 'application/json', 'Connection': 'keep-alive' }); - res.end(JSON.stringify({ ok: true, received: total })); - }); - }); - const proxy = await spawnProxy({ upstreamUrl: upstream.url }); - try { - const body = '{"x":1}'; - const reply = await new Promise<{ status: number; body: string; headers: http.IncomingHttpHeaders }>((resolve, reject) => { - const req = http.request({ - socketPath: proxy.socketPath, - method: 'POST', - path: '/tool-call', - headers: { - 'Content-Type': 'application/json', - 'Content-Length': Buffer.byteLength(body), - /* PTC presence required to forward (proxy filter). */ - ...PTC_HEADERS, - /* Hop-by-hop headers we expect the proxy to strip: */ - 'Connection': 'keep-alive, Upgrade-Insecure-Requests', - 'Keep-Alive': 'timeout=300, max=1000', - 'Upgrade': 'websocket', - 'TE': 'trailers', - 'Trailer': 'X-Foo', - 'Proxy-Authorization': 'Basic c3RlYWw=', - 'Proxy-Connection': 'close', - 'Host': 'attacker-controlled.example', - }, - }, res => { - const chunks: Buffer[] = []; - res.on('data', c => chunks.push(c)); - res.on('end', () => resolve({ - status: res.statusCode ?? 0, - body: Buffer.concat(chunks).toString(), - headers: res.headers, - })); - }); - req.on('error', reject); - req.write(body); - req.end(); - }); - expect(reply.status).toBe(200); - /* Upstream must not have seen the attacker's hop-by-hop values. - * The Connection header is special: node:http always sets one - * automatically (we tell it `connection: close`), so what we - * verify there is "the proxy's choice, not the attacker's". */ - expect(receivedHeaders.connection).toBe('close'); - expect(receivedHeaders.connection).not.toContain('Upgrade'); - for (const stripped of [ - 'keep-alive', 'upgrade', 'te', 'trailer', - 'transfer-encoding', 'proxy-authorization', 'proxy-connection', - ]) { - expect(receivedHeaders[stripped]).toBeUndefined(); - } - /* Host header must be the upstream's, not the attacker's value. */ - expect(receivedHeaders.host).not.toBe('attacker-controlled.example'); - /* And the upstream's hop-by-hop response headers must not reach - * the sandbox client (Connection: keep-alive from upstream gets - * stripped, our Connection: close wins). */ - expect(reply.headers.connection).toBe('close'); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('CIRCUMVENTION: slow-loris body upload is bounded by requestBodyTimeoutMs (~5s default)', async () => { - /* Drip body bytes one at a time, each within the per-byte idle - * window. Without an absolute server.requestTimeout, this would - * stretch up to activeRequestTimeoutMs (default 30s) and let an - * attacker monopolize a slot. With server.requestTimeout = - * requestBodyTimeoutMs, the proxy must close us within ~5s. */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ upstreamUrl: upstream.url }); - try { - const sock = await rawConnect(proxy.socketPath); - sock.write( - 'POST /tool-call HTTP/1.1\r\n' + - 'Host: x\r\n' + - 'Content-Type: application/json\r\n' + - 'Content-Length: 100\r\n' + - 'Connection: close\r\n' + - /* PTC headers required so the proxy forwards (presence-only check; - * lets us reach the bodyUploadDeadline path the test exercises). */ - 'X-Execution-ID: test-exec-id\r\n' + - 'X-Tool-Call-ID: test-call-001\r\n' + - 'X-Callback-Token: test-callback-token\r\n' + - '\r\n', - ); - /* Drip 1 byte every 800ms. After 5s we've sent ~6 bytes — far short - * of the declared 100 — and the proxy must have closed us. */ - const start = Date.now(); - const dripInterval = setInterval(() => { - if (sock.destroyed) { clearInterval(dripInterval); return; } - try { sock.write('x'); } catch { clearInterval(dripInterval); } - }, 800); - const closedAt = await new Promise(resolve => { - sock.once('close', () => resolve(Date.now() - start)); - setTimeout(() => resolve(-1), 15000); - }); - clearInterval(dripInterval); - expect(closedAt).toBeGreaterThanOrEqual(0); - /* Bounded close — proves slow-loris is mitigated. Generous upper - * bound to absorb scheduler jitter; the point is "<<30s". */ - expect(closedAt).toBeLessThan(12000); - expect(upstream.calls()).toBe(0); - } finally { - await proxy.stop(); - await upstream.close(); - } - }, { timeout: 15_000 }); - - test('CIRCUMVENTION: HTTP/2 binary framing on a /tmp/tcs.sock connection fails clean (no crash)', async () => { - /* If the proxy crashed on unexpected protocol bytes, an attacker - * could brick it. Send raw HTTP/2 connection-preface bytes; the - * node:http parser must close the socket with clientError without - * affecting subsequent legitimate clients. */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ upstreamUrl: upstream.url }); - try { - /* HTTP/2 connection preface (24 bytes) per RFC 7540. */ - const h2Preface = Buffer.from('PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n'); - for (let i = 0; i < 5; i++) { - const sock = await rawConnect(proxy.socketPath); - sock.write(h2Preface); - await new Promise(r => sock.once('close', () => r())); - } - /* Proxy must still serve a normal POST. */ - const res = await postToolCall(proxy.socketPath, '{"x":1}'); - expect(res.status).toBe(200); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('CIRCUMVENTION: oversize request body is rejected with 413 before reaching upstream', async () => { - /* maxBodyBytes (default 1 MiB) bounds memory pressure. The - * Content-Length precheck closes the loophole where streaming the - * body to upstream would let the first chunks of an honestly- - * declared oversize request leak through. */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ - upstreamUrl: upstream.url, - env: { /* default maxBodyBytes */ }, - }); - try { - const big = 'A'.repeat(2 * 1024 * 1024); // 2 MiB body - const reply = await new Promise<{ status: number; body: string }>(resolve => { - const req = http.request({ - socketPath: proxy.socketPath, - method: 'POST', - path: '/tool-call', - headers: { - 'Content-Length': Buffer.byteLength(big), - 'Connection': 'close', - /* PTC presence so the proxy actually attempts to forward, - * letting the maxBodyBytes / Content-Length precheck fire. */ - ...PTC_HEADERS, - }, - }, res => { - const chunks: Buffer[] = []; - res.on('data', c => chunks.push(c)); - res.on('end', () => resolve({ - status: res.statusCode ?? 0, - body: Buffer.concat(chunks).toString(), - })); - }); - req.on('error', err => resolve({ status: 0, body: err.message })); - req.write(big); - req.end(); - }); - /* Either 413 (CL precheck or runtime byte counter) or transport - * error (proxy closed mid-stream). The security property is the - * same: NO bytes of the oversize body reach the upstream. */ - expect([413, 0]).toContain(reply.status); - expect(upstream.calls()).toBe(0); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('PTC presence filter: duplicate empty header lines do not bypass the gate', async () => { - /* Codex caught this: Node joins duplicate request headers with - * `, ` (per RFC 7230 §3.2.2). A naive `!headers[name]` check on - * `X-Execution-ID:` sent twice with empty values produces the - * truthy string `", "`, slips past the gate, and lets the - * attacker reach the upstream — whose structured auth-error - * response defeats the route-opacity goal. The proxy must - * normalise (strip commas + whitespace) before checking. */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ upstreamUrl: upstream.url }); - try { - const sock = await rawConnect(proxy.socketPath); - /* Each required PTC header sent TWICE with empty values. Without - * the fix, Node's join produces `, ` for each name and the - * proxy forwards. With the fix, the values normalise to '' and - * the proxy returns its canonical 404. */ - sock.write( - 'POST /tool-call HTTP/1.1\r\n' + - 'Host: x\r\n' + - 'Content-Length: 0\r\n' + - 'Connection: close\r\n' + - 'X-Execution-ID:\r\n' + - 'X-Execution-ID:\r\n' + - 'X-Tool-Call-ID:\r\n' + - 'X-Tool-Call-ID:\r\n' + - 'X-Callback-Token:\r\n' + - 'X-Callback-Token:\r\n' + - '\r\n', - ); - const reply = await new Promise(resolve => { - const chunks: Buffer[] = []; - sock.on('data', c => chunks.push(c)); - sock.on('close', () => resolve(Buffer.concat(chunks).toString())); - setTimeout(() => resolve(Buffer.concat(chunks).toString()), 3000); - }); - /* Critical: upstream MUST NOT have been called. Status MUST be - * 404. Body MUST be the canonical "not found". */ - expect(reply).toContain('404'); - expect(reply).toContain('not found'); - expect(reply).not.toMatch(/X-Request-ID|ETag/i); - expect(upstream.calls()).toBe(0); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('PTC presence filter: missing headers get canonical 404, never reach upstream', async () => { - /* The proxy filters on PTC-header presence before forwarding so a - * sandbox attacker probing /tool-call without SDK headers cannot - * fingerprint the route via the upstream's structured "missing PTC - * headers" response. The `bogus_headers` case proves cryptographic - * validation is still upstream's job — bogus values DO forward and - * upstream's structured error is preserved (so the SDK can parse it). */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ upstreamUrl: upstream.url }); - try { - // No PTC headers → proxy 404, upstream NOT called - const noHeaders = await new Promise<{ status: number; body: string; ct?: string }>(resolve => { - const r = http.request({ socketPath: proxy.socketPath, method: 'POST', path: '/tool-call' }, res => { - const chunks: Buffer[] = []; - res.on('data', c => chunks.push(c)); - res.on('end', () => resolve({ - status: res.statusCode ?? 0, - body: Buffer.concat(chunks).toString(), - ct: res.headers['content-type'] as string | undefined, - })); - }); - r.on('error', err => resolve({ status: 0, body: err.message })); - r.end(); - }); - expect(noHeaders.status).toBe(404); - expect(noHeaders.body).toBe('not found'); - expect(noHeaders.ct).toContain('text/plain'); - const callsAfterNoHeaders = upstream.calls(); - expect(callsAfterNoHeaders).toBe(0); - - // With PTC headers (any values) → proxy forwards - const withHeaders = await postToolCall(proxy.socketPath, '{"x":1}'); - expect(withHeaders.status).toBe(200); - expect(upstream.calls()).toBe(1); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('CIRCUMVENTION: pipelined requests on a single socket only target /tool-call and respect active-request budget', async () => { - /* HTTP/1.1 pipelining lets a client batch requests on one socket. - * Both pipelined requests still go through the proxy's path filter - * and active-request budget — they cannot reach a different upstream - * route. node:http may parse and dispatch the second request - * before the first response's Connection: close fires; that is fine - * as long as the second is ALSO subject to all the proxy's defenses. */ - let receivedPaths: string[] = []; - const upstream = await startUpstream((req, res) => { - receivedPaths.push(req.url ?? ''); - let n = 0; req.on('data', c => { n += c.length; }); - req.on('end', () => { - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ ok: true, received: n })); - }); - }); - const proxy = await spawnProxy({ upstreamUrl: upstream.url }); - try { - const sock = await rawConnect(proxy.socketPath); - /* Second request tries to hit /admin (a path the proxy's filter - * MUST reject). If pipelining could bypass the filter, we'd see - * /admin in receivedPaths. */ - /* PTC headers on the first request so it actually forwards; the - * second request deliberately omits them AND targets /admin to - * verify both the path filter AND the presence filter hold even - * across pipelined dispatch. */ - const pipelined = - 'POST /tool-call HTTP/1.1\r\nHost: x\r\nContent-Length: 7\r\n' + - 'X-Execution-ID: t\r\nX-Tool-Call-ID: t\r\nX-Callback-Token: t\r\n\r\n{"x":1}' + - 'POST /admin HTTP/1.1\r\nHost: x\r\nContent-Length: 7\r\n\r\n{"y":2}'; - sock.write(pipelined); - await new Promise(r => sock.once('close', () => r())); - /* Whatever node:http chose to dispatch, the path filter must hold: - * upstream must NEVER see /admin. */ - expect(receivedPaths.every(p => p === '/tool-call')).toBe(true); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('ACTIVE-FLOOD DoS: excess concurrent /tool-call requests get 429 and do not reach upstream', async () => { - /* Hold the upstream so requests pile up at the proxy's active cap. - * With maxActiveRequests=2, only 2 requests should reach upstream; - * the others should get 429 + Retry-After. */ - let pendingUpstreams = 0; - let release: (() => void) | undefined; - const releaseAll = new Promise(r => { release = r; }); - const upstream = await startUpstream(async (req, res) => { - pendingUpstreams++; - req.resume(); - await new Promise(r => req.on('end', () => r())); - await releaseAll; - res.writeHead(200).end('{"ok":true}'); - }); - - const proxy = await spawnProxy({ - upstreamUrl: upstream.url, - env: { - SANDBOX_MAX_CONCURRENT_JOBS: '2', - }, - }); - - try { - const inflight = Array.from({ length: 32 }, () => - postToolCall(proxy.socketPath, '{"x":1}').catch(err => ({ status: -1, body: String(err) })), - ); - /* Give the proxy time to admit the first batch and 429 the rest. */ - await new Promise(r => setTimeout(r, 500)); - release!(); - const results = await Promise.all(inflight); - const ok = results.filter(r => r.status === 200).length; - const tooMany = results.filter(r => r.status === 429).length; - /* Active cap default-derives from SANDBOX_MAX_CONCURRENT_JOBS but is - * floored at 16. Launch above the floor so this test actually exercises - * the 429 branch instead of merely proving that 16 requests succeed. */ - expect(ok).toBeGreaterThan(0); - expect(ok).toBeLessThanOrEqual(16); - expect(tooMany).toBeGreaterThan(0); - expect(upstream.calls()).toBeLessThanOrEqual(16); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('CIRCUMVENTION: SIGTERM during proxy init exits cleanly (no TypeError on undefined handle)', async () => { - /* Codex flagged that `shutdown()` could fall through to `handle.close()` - * when a signal arrives before `startToolCallSocketProxy()` resolves — - * which would crash with `TypeError: Cannot read properties of undefined`. - * Hold the socket path with a sentinel server so the real proxy's - * listen() blocks indefinitely (EADDRINUSE on a unix socket retries - * via Node's listen-with-existing-socket dance, but with another - * server actively listening the bind never resolves), then signal it. - * The proxy must exit code 0 (signal handler ran cleanly) — never - * exit on the SIGTERM signal directly (which would mean the handler - * never ran) and never exit non-zero (which would mean the handler - * ran but threw). */ - const socketPath = path.join(workdir, 'sigterm-init-race.sock'); - /* Pre-bind the socket path so the proxy's listen() hangs in EADDRINUSE - * retry loops, keeping `handle` undefined long enough to test the race. */ - const blocker = http.createServer(() => { /* noop */ }); - await new Promise(r => blocker.listen(socketPath, () => r())); - - const proc = spawn('node', [PROXY_BUNDLE], { - env: { - ...process.env, - TCS_SOCKET: socketPath, - SANDBOX_FORWARD_TARGET: 'http://127.0.0.1:1', - }, - stdio: ['ignore', 'pipe', 'pipe'], - }); - /* Wait long enough for Node to install the SIGTERM handler but - * NOT long enough for startToolCallSocketProxy to have resolved - * (it can't — the socket is bound by `blocker`). 100ms is well - * above the ~10ms handler-install window observed in probes. */ - await new Promise(r => setTimeout(r, 100)); - proc.kill('SIGTERM'); - - const exit = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>(resolve => { - const t = setTimeout(() => { - proc.kill('SIGKILL'); - resolve({ code: -1, signal: null }); - }, 3000); - proc.once('exit', (code, signal) => { - clearTimeout(t); - resolve({ code, signal }); - }); - }); - - /* Critical assertion: handler ran AND exited cleanly. Exit code 0 is - * the only acceptable outcome — anything else (signal:'SIGTERM' meaning - * default behavior, code 1 meaning thrown error, code -1 meaning hung - * past 3s) indicates the race window is broken. */ - expect(exit.code).toBe(0); - expect(exit.signal).toBeNull(); - - await new Promise(r => blocker.close(() => r())); - }); - - test('CONNECT-FLOOD DoS: connection-rate limiter drops fast-burst accepts that would pressure kernel slab', async () => { - /* This is the SPECIFIC audit scenario PR #1648 follow-up was opened - * for: 500 AF_UNIX connect()s in <50 ms. Without the rate limiter, - * every accept allocates kernel unix_sock + dentry + skb queue slab - * entries; the proxy's existing maxConnections only bounds the - * application-level activeSockets set, not the allocation rate. - * Observation from the audit: in-VM kernel slab pressure surfaces - * as EMFILE in subsequent loader open() calls even though the fd - * table itself sits at 0.1% utilization. - * - * Test verifies the wire-level effect: with burst=8, refill=0 (frozen - * for determinism), only the first 8 of 100 concurrent /tool-call - * requests reach the upstream. The remaining 92 hit the rate limiter - * and are destroyed before the HTTP request layer sees them — they - * surface as either ECONNRESET / 'socket hang up' or EPIPE on the - * client side. */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ - upstreamUrl: upstream.url, - env: { - /* burst=8 — much smaller than maxConnections so the RATE limit - * bites first. refill=1/sec is effectively frozen over the - * test's ~100ms window. */ - TCS_CONNECTION_RATE_BURST: '8', - TCS_CONNECTION_RATE_REFILL_PER_SEC: '1', - }, - }); - - try { - const results = await Promise.allSettled( - Array.from({ length: 100 }, () => - postToolCall(proxy.socketPath, '{"x":1}'), - ), - ); - - const ok = results.filter(r => r.status === 'fulfilled' && r.value.status === 200).length; - const dropped = results.length - ok; - - /* Burst=8 with a near-frozen refill — somewhere in [8, 12] should - * succeed (burst + occasional refill during test window). The - * key assertion is that the count is bounded WELL below 100. */ - expect(ok).toBeGreaterThanOrEqual(1); - expect(ok).toBeLessThanOrEqual(20); - expect(dropped).toBeGreaterThanOrEqual(80); - /* And critically: upstream sees only the rate-limit-allowed ones, - * NOT 100. This bounds kernel slab pressure: the proxy never - * creates the in-VM unix_sock entries for the dropped batch. */ - expect(upstream.calls()).toBeLessThanOrEqual(ok); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('CONNECT-FLOOD recovery: bucket refills so legitimate traffic resumes after a flood', async () => { - /* Follow-up to the test above: confirm the limiter doesn't permanently - * starve a connection. After draining the burst bucket, wait long - * enough for the refill to top it back up, then verify a new - * /tool-call succeeds. This guards against a bug where dropped - * connections might leak the rate-limit credit (e.g. if the bucket - * was decremented BEFORE the maxConnections check and never - * compensated when both gates were hit by the same connection). */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ - upstreamUrl: upstream.url, - env: { - TCS_CONNECTION_RATE_BURST: '4', - /* 20/sec refill — burst recovers in ~250ms. */ - TCS_CONNECTION_RATE_REFILL_PER_SEC: '20', - }, - }); - - try { - /* Drain the burst with a parallel flood. */ - const drain = await Promise.allSettled( - Array.from({ length: 30 }, () => postToolCall(proxy.socketPath, '{"x":1}')), - ); - const okDuringDrain = drain.filter(r => r.status === 'fulfilled' && r.value.status === 200).length; - expect(okDuringDrain).toBeGreaterThanOrEqual(1); - expect(okDuringDrain).toBeLessThanOrEqual(15); - - /* Wait for the bucket to refill past 1 token (50ms refills 1 token - * at 20/sec; 500ms gives us plenty of margin). */ - await new Promise(r => setTimeout(r, 500)); - - /* A single legitimate request after the wait must succeed. */ - const post = await postToolCall(proxy.socketPath, '{"x":2}'); - expect(post.status).toBe(200); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); -}); diff --git a/api/src/tool-call-socket-proxy.test.ts b/api/src/tool-call-socket-proxy.test.ts deleted file mode 100644 index c15cde60..00000000 --- a/api/src/tool-call-socket-proxy.test.ts +++ /dev/null @@ -1,448 +0,0 @@ -import { afterEach, describe, expect, test } from 'bun:test'; -import fs from 'node:fs'; -import http from 'node:http'; -import net from 'node:net'; -import os from 'node:os'; -import path from 'node:path'; -import { - createTokenBucket, - defaultConnectionRateBurst, - defaultConnectionRateRefillPerSec, - startToolCallSocketProxy, - type ToolCallSocketProxyHandle, -} from './tool-call-socket-proxy'; - -/* PTC-presence headers required by the proxy's filter so legitimate - * /tool-call requests forward instead of getting the canonical 404. The - * proxy only checks PRESENCE; cryptographic validation happens upstream. */ -const PTC_HEADERS = { - 'X-Execution-ID': 'test-exec-id', - 'X-Tool-Call-ID': 'test-call-001', - 'X-Callback-Token': 'test-callback-token', -} as const; - -const handles: ToolCallSocketProxyHandle[] = []; -const tempDirs: string[] = []; - -function makeSocketPath(): string { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tcs-proxy-test-')); - tempDirs.push(dir); - return path.join(dir, 'tcs.sock'); -} - -function wait(ms: number): Promise { - return new Promise(resolve => setTimeout(resolve, ms)); -} - -function createRawSocket(socketPath: string): Promise { - return new Promise((resolve, reject) => { - const socket = net.createConnection(socketPath); - socket.once('connect', () => resolve(socket)); - socket.once('error', reject); - }); -} - -async function startUpstream(): Promise<{ url: string; close: () => Promise }> { - const server = http.createServer((req, res) => { - if (req.method !== 'POST' || req.url !== '/tool-call') { - res.writeHead(404).end(); - return; - } - const chunks: Buffer[] = []; - req.on('data', chunk => chunks.push(Buffer.from(chunk))); - req.on('end', () => { - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ - success: true, - body: Buffer.concat(chunks).toString('utf8'), - })); - }); - }); - await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); - const address = server.address(); - if (address == null || typeof address === 'string') { - throw new Error('unexpected upstream address'); - } - return { - url: `http://127.0.0.1:${address.port}`, - close: () => new Promise(resolve => server.close(() => resolve())), - }; -} - -afterEach(async () => { - while (handles.length > 0) { - await handles.pop()!.close(); - } - for (const dir of tempDirs.splice(0)) { - fs.rmSync(dir, { recursive: true, force: true }); - } -}); - -describe('tool-call socket proxy', () => { - test('forwards valid tool-call requests with Connection: close', async () => { - const upstream = await startUpstream(); - const socketPath = makeSocketPath(); - const proxy = await startToolCallSocketProxy({ - socketPath, - rawTarget: upstream.url, - idleSocketTimeoutMs: 100, - log: { log() {}, warn() {}, error() {} }, - }); - handles.push(proxy); - - const response = await new Promise<{ status: number; body: string; closeHeader: unknown }>((resolve, reject) => { - const req = http.request({ - socketPath, - method: 'POST', - path: '/tool-call', - headers: { - 'Content-Type': 'application/json', - Connection: 'keep-alive', - ...PTC_HEADERS, - }, - }, res => { - const chunks: Buffer[] = []; - res.on('data', chunk => chunks.push(Buffer.from(chunk))); - res.on('end', () => resolve({ - status: res.statusCode ?? 0, - body: Buffer.concat(chunks).toString('utf8'), - closeHeader: res.headers.connection, - })); - }); - req.on('error', reject); - req.end('{"tool_name":"safe","input":{}}'); - }); - - expect(response.status).toBe(200); - expect(response.closeHeader).toBe('close'); - expect(JSON.parse(response.body).body).toBe('{"tool_name":"safe","input":{}}'); - await upstream.close(); - }); - - test('destroys idle raw connections so socket probes cannot pin FDs', async () => { - const upstream = await startUpstream(); - const socketPath = makeSocketPath(); - const proxy = await startToolCallSocketProxy({ - socketPath, - rawTarget: upstream.url, - idleSocketTimeoutMs: 50, - log: { log() {}, warn() {}, error() {} }, - }); - handles.push(proxy); - - const sockets = await Promise.all( - Array.from({ length: 8 }, () => createRawSocket(socketPath)), - ); - - await wait(250); - - expect(proxy.activeConnections()).toBe(0); - for (const socket of sockets) { - socket.destroy(); - } - await upstream.close(); - }); - - test('keeps socket connectable for per-job UID sandboxes', async () => { - const upstream = await startUpstream(); - const socketPath = makeSocketPath(); - const proxy = await startToolCallSocketProxy({ - socketPath, - rawTarget: upstream.url, - log: { log() {}, warn() {}, error() {} }, - }); - handles.push(proxy); - - expect(fs.statSync(socketPath).mode & 0o777).toBe(0o666); - await upstream.close(); - }); - - test('caps active connections and drops excess clients', async () => { - const upstream = await startUpstream(); - const socketPath = makeSocketPath(); - const proxy = await startToolCallSocketProxy({ - socketPath, - rawTarget: upstream.url, - maxConnections: 2, - idleSocketTimeoutMs: 500, - log: { log() {}, warn() {}, error() {} }, - }); - handles.push(proxy); - - const sockets = await Promise.allSettled( - Array.from({ length: 6 }, () => createRawSocket(socketPath)), - ); - - await wait(50); - - expect(sockets.filter(result => result.status === 'fulfilled').length).toBeGreaterThanOrEqual(2); - expect(proxy.activeConnections()).toBeLessThanOrEqual(2); - for (const result of sockets) { - if (result.status === 'fulfilled') result.value.destroy(); - } - await upstream.close(); - }); - - test('caps active tool-call requests separately from raw socket connections', async () => { - let releaseUpstream!: () => void; - let upstreamCalls = 0; - const upstreamReleased = new Promise(resolve => { - releaseUpstream = resolve; - }); - const server = http.createServer(async (req, res) => { - upstreamCalls += 1; - req.resume(); - await new Promise(resolve => req.on('end', resolve)); - await upstreamReleased; - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ success: true })); - }); - await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); - const address = server.address(); - if (address == null || typeof address === 'string') throw new Error('bad address'); - - const socketPath = makeSocketPath(); - const proxy = await startToolCallSocketProxy({ - socketPath, - rawTarget: `http://127.0.0.1:${address.port}`, - maxConnections: 8, - maxActiveRequests: 1, - log: { log() {}, warn() {}, error() {} }, - }); - handles.push(proxy); - - const first = new Promise((resolve, reject) => { - const req = http.request({ socketPath, method: 'POST', path: '/tool-call', headers: PTC_HEADERS }, res => { - res.on('end', () => resolve(res.statusCode ?? 0)); - res.resume(); - }); - req.on('error', reject); - req.end('{"tool_name":"slow","input":{}}'); - }); - - while (proxy.activeRequests() < 1) { - await wait(5); - } - - const second = await new Promise<{ status: number; body: string; retryAfter: unknown }>((resolve, reject) => { - const req = http.request({ socketPath, method: 'POST', path: '/tool-call', headers: PTC_HEADERS }, res => { - const chunks: Buffer[] = []; - res.on('data', chunk => chunks.push(Buffer.from(chunk))); - res.on('end', () => resolve({ - status: res.statusCode ?? 0, - body: Buffer.concat(chunks).toString('utf8'), - retryAfter: res.headers['retry-after'], - })); - }); - req.on('error', reject); - req.end('{"tool_name":"second","input":{}}'); - }); - - expect(second.status).toBe(429); - expect(second.retryAfter).toBe('1'); - expect(JSON.parse(second.body)).toEqual({ - success: false, - error: 'Too many concurrent tool-call requests', - }); - expect(upstreamCalls).toBe(1); - - releaseUpstream(); - expect(await first).toBe(200); - await new Promise(resolve => server.close(() => resolve())); - }); - - test('expires active tool-call requests instead of holding proxy FDs indefinitely', async () => { - let upstreamCalls = 0; - let resolveUpstreamSeen!: () => void; - const upstreamSeen = new Promise(resolve => { - resolveUpstreamSeen = resolve; - }); - const server = http.createServer((req, res) => { - upstreamCalls += 1; - req.resume(); - req.on('end', resolveUpstreamSeen); - // Intentionally never respond; the proxy must time out and release its - // active request/upstream budgets. - void res; - }); - await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); - const address = server.address(); - if (address == null || typeof address === 'string') throw new Error('bad address'); - - const socketPath = makeSocketPath(); - const proxy = await startToolCallSocketProxy({ - socketPath, - rawTarget: `http://127.0.0.1:${address.port}`, - maxActiveRequests: 1, - activeRequestTimeoutMs: 50, - log: { log() {}, warn() {}, error() {} }, - }); - handles.push(proxy); - - const client = await createRawSocket(socketPath); - const body = '{"tool_name":"slow","input":{}}'; - client.end([ - 'POST /tool-call HTTP/1.1', - 'Host: localhost', - 'Content-Type: application/json', - `Content-Length: ${Buffer.byteLength(body)}`, - 'Connection: close', - 'X-Execution-ID: t', - 'X-Tool-Call-ID: t', - 'X-Callback-Token: t', - '', - body, - ].join('\r\n')); - - await upstreamSeen; - expect(upstreamCalls).toBe(1); - expect(proxy.activeRequests()).toBe(1); - expect(proxy.activeUpstreams()).toBe(1); - - const deadline = Date.now() + 500; - while (proxy.activeUpstreams() > 0 && Date.now() < deadline) { - await wait(5); - } - - expect(proxy.activeRequests()).toBe(0); - expect(proxy.activeUpstreams()).toBe(0); - client.destroy(); - await new Promise(resolve => server.close(() => resolve())); - }); - - test('rejects oversized request bodies before proxying', async () => { - let upstreamCalls = 0; - const server = http.createServer((_req, res) => { - upstreamCalls += 1; - res.writeHead(200).end('unexpected'); - }); - await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); - const address = server.address(); - if (address == null || typeof address === 'string') throw new Error('bad address'); - - const socketPath = makeSocketPath(); - const proxy = await startToolCallSocketProxy({ - socketPath, - rawTarget: `http://127.0.0.1:${address.port}`, - maxBodyBytes: 4, - log: { log() {}, warn() {}, error() {} }, - }); - handles.push(proxy); - - const response = await new Promise<{ status: number; body: string }>((resolve, reject) => { - const req = http.request({ socketPath, method: 'POST', path: '/tool-call', headers: PTC_HEADERS }, res => { - const chunks: Buffer[] = []; - res.on('data', chunk => chunks.push(Buffer.from(chunk))); - res.on('end', () => resolve({ - status: res.statusCode ?? 0, - body: Buffer.concat(chunks).toString('utf8'), - })); - }); - req.on('error', reject); - req.end('too large'); - }); - - expect(response.status).toBe(413); - expect(response.body).toBe('request body too large'); - expect(upstreamCalls).toBe(0); - await new Promise(resolve => server.close(() => resolve())); - }); - - /* Integration tests for the connection-rate limiter live in - * tool-call-socket-proxy-runtime.test.ts. Bun's node:http compat layer - * does not fire `'connection'` events, so the in-process proxy here - * never invokes the rate-limiter code path under `bun test` — the - * checks would pass vacuously. The runtime tests spawn the proxy as a - * real Node subprocess, matching the production runtime. The pure - * createTokenBucket logic is fully unit-tested below and runs fine - * under either runtime. */ -}); - -describe('createTokenBucket', () => { - test('starts full at burst capacity', () => { - const b = createTokenBucket({ burst: 10, refillPerSec: 1 }); - expect(b.tokens()).toBeCloseTo(10); - }); - - test('returns true until the bucket is drained, then false', () => { - const b = createTokenBucket({ burst: 3, refillPerSec: 0, now: () => 0 }); - expect(b.tryConsume()).toBe(true); - expect(b.tryConsume()).toBe(true); - expect(b.tryConsume()).toBe(true); - expect(b.tryConsume()).toBe(false); - expect(b.tryConsume()).toBe(false); - }); - - test('refills at refillPerSec without exceeding the burst cap', () => { - let t = 1000; - const b = createTokenBucket({ burst: 10, refillPerSec: 100, now: () => t }); - /* Drain. */ - for (let i = 0; i < 10; i++) expect(b.tryConsume()).toBe(true); - expect(b.tryConsume()).toBe(false); - /* Advance 50ms — 100/sec * 0.050 = 5 tokens. */ - t += 50; - for (let i = 0; i < 5; i++) expect(b.tryConsume()).toBe(true); - expect(b.tryConsume()).toBe(false); - /* Advance way more than burst's worth — capacity caps at 10, not - * unbounded growth. */ - t += 10_000; - expect(b.tokens()).toBeCloseTo(10); - }); - - test('refillPerSec=0 freezes the bucket once drained (test-determinism guarantee)', () => { - const b = createTokenBucket({ burst: 2, refillPerSec: 0, now: () => 0 }); - expect(b.tryConsume()).toBe(true); - expect(b.tryConsume()).toBe(true); - /* No matter how much time we'd advance with a real clock, refill of - * 0/sec adds zero tokens. */ - expect(b.tryConsume()).toBe(false); - expect(b.tokens()).toBe(0); - }); -}); - -describe('connection-rate defaults scale with SANDBOX_MAX_CONCURRENT_JOBS', () => { - /* Codex P2 follow-up on PR #1652: the previous fixed 20-accept/sec - * refill would throttle legitimate sustained traffic on deployments - * where maxConcurrentJobs is scaled up (e.g. 64), because the proxy's - * other defaults (maxConnections, maxActiveRequests) already scale - * from SANDBOX_MAX_CONCURRENT_JOBS but the rate limiter did not. */ - const originalEnv = process.env.SANDBOX_MAX_CONCURRENT_JOBS; - - afterEach(() => { - if (originalEnv === undefined) { - delete process.env.SANDBOX_MAX_CONCURRENT_JOBS; - } else { - process.env.SANDBOX_MAX_CONCURRENT_JOBS = originalEnv; - } - }); - - test('unset env keeps the fixed 64/20 defaults', () => { - delete process.env.SANDBOX_MAX_CONCURRENT_JOBS; - expect(defaultConnectionRateBurst()).toBe(64); - expect(defaultConnectionRateRefillPerSec()).toBe(20); - }); - - test('small concurrency floors at the baseline (never tighter than the fixed defaults)', () => { - process.env.SANDBOX_MAX_CONCURRENT_JOBS = '8'; - /* 8 * 4 = 32, but floor at 64 baseline. */ - expect(defaultConnectionRateBurst()).toBe(64); - /* 8 * 2 = 16, but floor at 20 baseline. */ - expect(defaultConnectionRateRefillPerSec()).toBe(20); - }); - - test('mid concurrency scales burst proportionally', () => { - process.env.SANDBOX_MAX_CONCURRENT_JOBS = '32'; - /* 32 * 4 = 128 (above 64 floor, below 256 ceiling). */ - expect(defaultConnectionRateBurst()).toBe(128); - /* 32 * 2 = 64 (above 20 floor, below 200 ceiling). */ - expect(defaultConnectionRateRefillPerSec()).toBe(64); - }); - - test('high concurrency caps at the ceiling — bounded resource use even with extreme config', () => { - process.env.SANDBOX_MAX_CONCURRENT_JOBS = '256'; - /* 256 * 4 = 1024, capped at 256 ceiling. */ - expect(defaultConnectionRateBurst()).toBe(256); - /* 256 * 2 = 512, capped at 200 ceiling. */ - expect(defaultConnectionRateRefillPerSec()).toBe(200); - }); -}); diff --git a/api/src/tool-call-socket-proxy.ts b/api/src/tool-call-socket-proxy.ts deleted file mode 100644 index 78e689b7..00000000 --- a/api/src/tool-call-socket-proxy.ts +++ /dev/null @@ -1,669 +0,0 @@ -import fs from 'node:fs'; -import http from 'node:http'; -import https from 'node:https'; -import type net from 'node:net'; - -export interface ToolCallSocketProxyOptions { - socketPath: string; - rawTarget: string; - maxConnections?: number; - maxActiveRequests?: number; - idleSocketTimeoutMs?: number; - headerTimeoutMs?: number; - requestBodyTimeoutMs?: number; - activeRequestTimeoutMs?: number; - maxBodyBytes?: number; - listenBacklog?: number; - /** Token-bucket capacity for the connection-rate limiter. The bucket - * starts full and refills at `connectionRateRefillPerSec`. Burst of - * legitimate concurrent tool calls (up to maxActiveRequests=16 by - * default) easily fits; a 500-connect()-in-50ms flood drains the - * bucket and subsequent connections are dropped at the application - * layer instead of allocating kernel unix_sock slab entries. */ - connectionRateBurst?: number; - /** Sustained connection-acceptance rate in connections/second after the - * burst bucket is drained. The SDK opens one connection per tool call; - * even an aggressive agent rarely exceeds a few /sec. */ - connectionRateRefillPerSec?: number; - socketUid?: number; - socketGid?: number; - socketMode?: number; - log?: Pick; -} - -export interface ToolCallSocketProxyHandle { - server: http.Server; - close: () => Promise; - activeConnections: () => number; - activeRequests: () => number; - activeUpstreams: () => number; - /** For tests: current available tokens in the connection-rate bucket. */ - connectionRateTokens: () => number; - /** For tests + observability: cumulative count of connections dropped - * by the connection-rate limiter since proxy start. */ - connectionRateDropped: () => number; -} - -const DEFAULT_MAX_CONNECTIONS = 64; -const DEFAULT_MAX_ACTIVE_REQUESTS = 16; -const DEFAULT_IDLE_SOCKET_TIMEOUT_MS = 2_000; -const DEFAULT_REQUEST_BODY_TIMEOUT_MS = 5_000; -const DEFAULT_ACTIVE_REQUEST_TIMEOUT_MS = 30_000; -const DEFAULT_MAX_BODY_BYTES = 1_048_576; -const DEFAULT_LISTEN_BACKLOG = 16; -const MAX_DEFAULT_CONNECTIONS = 256; -const MAX_DEFAULT_ACTIVE_REQUESTS = 64; -const ACTIVE_REQUEST_TIMEOUT_GRACE_MS = 5_000; -const MAX_DEFAULT_ACTIVE_REQUEST_TIMEOUT_MS = 60_000; -const DEFAULT_CONNECTION_RATE_BURST = 64; -const DEFAULT_CONNECTION_RATE_REFILL_PER_SEC = 20; -const MAX_DEFAULT_CONNECTION_RATE_BURST = 256; -const MAX_DEFAULT_CONNECTION_RATE_REFILL_PER_SEC = 200; - -/** Monotonic-ish token bucket. Tokens refill continuously at a configured - * rate up to a burst cap. `tryConsume()` returns false instead of waiting - * when empty — callers drop the connection synchronously, which is the - * right behavior for a SOCK_STREAM accept handler (queuing in JS would - * still hold the kernel socket struct alive, defeating the point). */ -export interface TokenBucket { - tryConsume(): boolean; - tokens(): number; -} - -export function createTokenBucket(opts: { - burst: number; - refillPerSec: number; - now?: () => number; -}): TokenBucket { - const now = opts.now ?? Date.now; - const refillPerMs = opts.refillPerSec / 1000; - let tokens = opts.burst; - let last = now(); - - function refill(): void { - const t = now(); - const elapsed = t - last; - if (elapsed <= 0) return; - tokens = Math.min(opts.burst, tokens + elapsed * refillPerMs); - last = t; - } - - return { - tryConsume(): boolean { - refill(); - if (tokens >= 1) { - tokens -= 1; - return true; - } - return false; - }, - tokens(): number { - refill(); - return tokens; - }, - }; -} - -function normalizeTarget(rawTarget: string): URL { - if (!rawTarget) { - throw new Error('SANDBOX_FORWARD_TARGET is required'); - } - return new URL(rawTarget.includes('://') ? rawTarget : `http://${rawTarget}`); -} - -function destroySocket(socket: net.Socket): void { - socket.destroy(); -} - -/* RFC 7230 §6.1 hop-by-hop headers. A proxy MUST NOT forward these to the - * upstream — they describe the proxy<->client connection, not the request. - * `host` is rewritten separately to point at the upstream. Forwarding the - * client's `host` would let a malicious sandbox steer the upstream's - * routing if it ever virtual-hosts. */ -const HOP_BY_HOP_HEADERS: ReadonlySet = new Set([ - 'connection', - 'keep-alive', - 'proxy-authenticate', - 'proxy-authorization', - 'proxy-connection', - 'te', - 'trailer', - 'transfer-encoding', - 'upgrade', - 'host', -]); - -function buildForwardedHeaders( - reqHeaders: http.IncomingHttpHeaders, - upstreamHost: string, -): http.OutgoingHttpHeaders { - const out: http.OutgoingHttpHeaders = {}; - for (const [k, v] of Object.entries(reqHeaders)) { - if (HOP_BY_HOP_HEADERS.has(k.toLowerCase())) continue; - if (v == null) continue; - out[k] = v; - } - /* Set our own connection-management headers; the proxy never keep-alives - * to upstream because each request is one-shot. */ - out.host = upstreamHost; - out.connection = 'close'; - return out; -} - -/* True only when the header has at least one non-empty, non-whitespace - * character. Naive `!headers[name]` is bypassable: Node joins duplicate - * request headers with `, ` (per RFC 7230 §3.2.2), so two empty - * `X-Foo:` lines arrive as the truthy string `", "`. An attacker who - * sends each PTC header twice with empty values would slip past the - * presence filter and reach the upstream — defeating the route-opacity - * goal. Stripping commas + whitespace before the empty check closes - * that gap. */ -function hasPtcHeaderValue(value: string | string[] | undefined): boolean { - if (value == null) return false; - const joined = Array.isArray(value) ? value.join(',') : value; - /* Strip every comma (Node's separator for joined duplicates) and every - * whitespace character. If anything is left, the header carries real - * content. */ - return joined.replace(/[,\s]/g, '') !== ''; -} - -/* Request smuggling defense (CVE class). When a request carries BOTH - * Transfer-Encoding and Content-Length, an upstream may pick the - * different one than the proxy did, letting an attacker prepend bytes - * to a synthesized "second request" that bypasses the proxy's path - * filter. The fix per RFC 7230 §3.3.3 is to reject the request rather - * than try to reconcile. */ -function isSmugglingShaped(headers: http.IncomingHttpHeaders): boolean { - const hasCL = headers['content-length'] != null; - const hasTE = headers['transfer-encoding'] != null; - return hasCL && hasTE; -} - -function parsePositiveInt(raw: string | undefined): number | undefined { - if (raw == null) return undefined; - const n = Number(raw); - if (!Number.isFinite(n) || n < 1) return undefined; - return Math.floor(n); -} - -function defaultMaxConnections(): number { - const maxConcurrentJobs = parsePositiveInt(process.env.SANDBOX_MAX_CONCURRENT_JOBS); - if (maxConcurrentJobs == null) return DEFAULT_MAX_CONNECTIONS; - return Math.min(Math.max(maxConcurrentJobs * 4, DEFAULT_MAX_CONNECTIONS), MAX_DEFAULT_CONNECTIONS); -} - -function defaultMaxActiveRequests(): number { - const maxConcurrentJobs = parsePositiveInt(process.env.SANDBOX_MAX_CONCURRENT_JOBS); - if (maxConcurrentJobs == null) return DEFAULT_MAX_ACTIVE_REQUESTS; - return Math.min(Math.max(maxConcurrentJobs, DEFAULT_MAX_ACTIVE_REQUESTS), MAX_DEFAULT_ACTIVE_REQUESTS); -} - -function defaultActiveRequestTimeoutMs(): number { - const runTimeoutMs = parsePositiveInt(process.env.SANDBOX_RUN_TIMEOUT); - if (runTimeoutMs == null) return DEFAULT_ACTIVE_REQUEST_TIMEOUT_MS; - return Math.min( - Math.max(runTimeoutMs + ACTIVE_REQUEST_TIMEOUT_GRACE_MS, DEFAULT_REQUEST_BODY_TIMEOUT_MS), - MAX_DEFAULT_ACTIVE_REQUEST_TIMEOUT_MS, - ); -} - -/* Rate-limit defaults scale with SANDBOX_MAX_CONCURRENT_JOBS in the same - * shape as defaultMaxConnections / defaultMaxActiveRequests above. A - * deployment with maxConcurrentJobs=64 is sized to run ~64 tool calls - * in parallel; capping the connection-acceptance rate at the fixed 20/sec - * baseline would throttle legitimate sustained traffic from that - * workload — connections that would have been 429'd by maxActiveRequests - * instead get destroyed at the rate limiter, which is the wrong shape - * (clients see ECONNRESET instead of a structured Retry-After response). */ -export function defaultConnectionRateBurst(): number { - const maxConcurrentJobs = parsePositiveInt(process.env.SANDBOX_MAX_CONCURRENT_JOBS); - if (maxConcurrentJobs == null) return DEFAULT_CONNECTION_RATE_BURST; - return Math.min( - Math.max(maxConcurrentJobs * 4, DEFAULT_CONNECTION_RATE_BURST), - MAX_DEFAULT_CONNECTION_RATE_BURST, - ); -} - -export function defaultConnectionRateRefillPerSec(): number { - const maxConcurrentJobs = parsePositiveInt(process.env.SANDBOX_MAX_CONCURRENT_JOBS); - if (maxConcurrentJobs == null) return DEFAULT_CONNECTION_RATE_REFILL_PER_SEC; - /* 2x maxConcurrentJobs gives headroom over the steady-state rate - * (~1 connection per active job per second for typical tool-call - * workloads). Floor at 20/sec so small deployments don't get a - * tighter limit than the fixed baseline. */ - return Math.min( - Math.max(maxConcurrentJobs * 2, DEFAULT_CONNECTION_RATE_REFILL_PER_SEC), - MAX_DEFAULT_CONNECTION_RATE_REFILL_PER_SEC, - ); -} - -export async function startToolCallSocketProxy( - opts: ToolCallSocketProxyOptions, -): Promise { - const log = opts.log ?? console; - const socketPath = opts.socketPath; - const target = normalizeTarget(opts.rawTarget); - const transport = target.protocol === 'https:' ? https : http; - const maxConnections = opts.maxConnections ?? defaultMaxConnections(); - const maxActiveRequests = opts.maxActiveRequests ?? defaultMaxActiveRequests(); - const idleSocketTimeoutMs = opts.idleSocketTimeoutMs ?? DEFAULT_IDLE_SOCKET_TIMEOUT_MS; - const headerTimeoutMs = opts.headerTimeoutMs ?? idleSocketTimeoutMs; - const requestBodyTimeoutMs = opts.requestBodyTimeoutMs ?? DEFAULT_REQUEST_BODY_TIMEOUT_MS; - const activeRequestTimeoutMs = opts.activeRequestTimeoutMs ?? defaultActiveRequestTimeoutMs(); - const maxBodyBytes = opts.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES; - const listenBacklog = opts.listenBacklog ?? DEFAULT_LISTEN_BACKLOG; - const connectionRateBurst = opts.connectionRateBurst ?? defaultConnectionRateBurst(); - const connectionRateRefillPerSec = opts.connectionRateRefillPerSec ?? defaultConnectionRateRefillPerSec(); - // The socket is intentionally connectable by sandbox jobs even when - // SANDBOX_PER_JOB_UIDS maps each job to a distinct outside UID. Abuse is - // bounded by the proxy's connection caps/timeouts, not by inode ownership. - const socketMode = opts.socketMode ?? 0o666; - - const activeSockets = new Set(); - /* Absolute accept-to-headers deadlines cover slow-header clients. Node's - * headersTimeout and socket idle timers are not a short per-socket deadline - * for AF_UNIX header drip attacks, so track the accepted socket ourselves - * until node:http has parsed enough bytes to dispatch a request. */ - const headerReceiveDeadlines = new WeakMap>(); - let activeRequests = 0; - let activeUpstreams = 0; - - const clearHeaderReceiveDeadline = (socket: net.Socket): void => { - const deadline = headerReceiveDeadlines.get(socket); - if (deadline) clearTimeout(deadline); - headerReceiveDeadlines.delete(socket); - }; - - /* Connection-rate limiter — counters the audit pattern of 500 AF_UNIX - * connect()s in <50 ms. Without this, even though the proxy itself - * stays bounded (maxConnections destroys excess accepts), each accept - * still allocates a unix_sock + dentry + skb queue in the in-VM - * kernel slab. On a memory-tight microVM, 500+ allocations in a - * single tick pressures the slab allocator hard enough that - * subsequent legitimate openat() / mmap() calls (e.g. ld.so loading - * shared libraries) sporadically fail with EMFILE — even though the - * fd table itself is at <1% utilization. Capping the *rate* of - * accepts at the application layer bounds how fast new slab entries - * can be created, giving the kernel time to reclaim. */ - const connectionRateLimiter = createTokenBucket({ - burst: connectionRateBurst, - refillPerSec: connectionRateRefillPerSec, - }); - let connectionRateDroppedCount = 0; - - const server = http.createServer((req, res) => { - const socket = req.socket; - clearHeaderReceiveDeadline(socket); - socket.setTimeout(requestBodyTimeoutMs, () => destroySocket(socket)); - req.setTimeout(requestBodyTimeoutMs, () => destroySocket(socket)); - res.setHeader('Connection', 'close'); - - res.on('finish', () => { - socket.end(); - }); - - if (req.method !== 'POST' || req.url !== '/tool-call') { - req.resume(); - res.writeHead(404, { 'Content-Type': 'text/plain', Connection: 'close' }); - res.end('not found'); - return; - } - - /* PTC-header presence filter at the proxy boundary. Three reasons: - * - * (1) Route opacity. If a sandbox attacker probes /tool-call without - * the SDK-supplied PTC headers, the upstream's 404 leaks Express's - * framing (ETag/X-Request-ID/charset/content-length) which is - * visibly different from the proxy's own 404 for unknown paths. - * Filtering at the proxy means missing-header probes never reach - * the upstream and the response is byte-identical to /any/unknown. - * - * (2) Cheap rejection. Don't burn an active-request slot or open an - * upstream connection for a request that's going to be rejected. - * - * (3) Preserves legitimate 404s. The previous shape blanket-masked - * ALL upstream 404 responses, which broke "Session not found" and - * other structured upstream errors that the SDK's preamble parses - * as JSON (json.loads("not found") raises). With the filter - * here, only missing-header probes are masked; a genuine 404 with - * a JSON body for an authenticated request still reaches the - * SDK intact. - * - * The proxy only checks PRESENCE — actual cryptographic validation - * happens at the upstream. An attacker supplying garbage values for - * these headers gets forwarded and the upstream rejects with its - * structured response, but they've already revealed sandbox-side - * activity by sending non-empty headers, so route opacity isn't the - * goal there. */ - if ( - !hasPtcHeaderValue(req.headers['x-execution-id']) - || !hasPtcHeaderValue(req.headers['x-tool-call-id']) - || !hasPtcHeaderValue(req.headers['x-callback-token']) - ) { - req.resume(); - res.writeHead(404, { 'Content-Type': 'text/plain', Connection: 'close' }); - res.end('not found'); - return; - } - - /* Smuggling defense — block before we do anything else with the - * request so neither our active-request budget nor any upstream - * connection is consumed. */ - if (isSmugglingShaped(req.headers)) { - req.resume(); - res.writeHead(400, { 'Content-Type': 'text/plain', Connection: 'close' }); - res.end('ambiguous Content-Length and Transfer-Encoding'); - return; - } - - /* Content-Length precheck — if the client honestly declares a body - * larger than maxBodyBytes, reject BEFORE opening an upstream and - * BEFORE leaking any of those bytes through req.pipe(upstream). The - * runtime byte counter below is the second line of defense for - * chunked uploads and dishonest CL values, but this gate handles - * the common case cleanly. */ - const declaredCL = Number(req.headers['content-length'] ?? '0'); - if (Number.isFinite(declaredCL) && declaredCL > maxBodyBytes) { - req.resume(); - res.writeHead(413, { 'Content-Type': 'text/plain', Connection: 'close' }); - res.end('request body too large'); - return; - } - - if (activeRequests >= maxActiveRequests) { - req.resume(); - res.writeHead(429, { - 'Content-Type': 'application/json', - Connection: 'close', - 'Retry-After': '1', - }); - res.end(JSON.stringify({ - success: false, - error: 'Too many concurrent tool-call requests', - })); - return; - } - - activeRequests += 1; - let releasedActiveRequest = false; - const releaseActiveRequest = (): void => { - if (releasedActiveRequest) return; - releasedActiveRequest = true; - activeRequests = Math.max(0, activeRequests - 1); - }; - res.on('finish', releaseActiveRequest); - res.on('close', releaseActiveRequest); - - let bodyBytes = 0; - let rejected = false; - let upstreamClosed = false; - - /* Absolute body-upload deadline. socket.setTimeout / req.setTimeout - * are idle timers and reset on every byte, so a malicious client - * dripping bytes just under the idle threshold (slow-loris) bypasses - * them. Node's `server.requestTimeout` was meant to bound this but - * empirically does NOT fire mid-body for unix-socket clients in - * Node 22 (probed; drip kept flowing past 3x the configured value). - * This explicit setTimeout fires unconditionally `requestBodyTimeoutMs` - * after the request handler runs and is cleared once the body is - * fully received. */ - const bodyUploadDeadline = setTimeout(() => { - if (req.complete || rejected) return; - destroySocket(socket); - }, requestBodyTimeoutMs); - const clearBodyUploadDeadline = (): void => clearTimeout(bodyUploadDeadline); - req.on('end', clearBodyUploadDeadline); - req.on('aborted', clearBodyUploadDeadline); - res.on('finish', clearBodyUploadDeadline); - res.on('close', clearBodyUploadDeadline); - - req.on('end', () => { - socket.setTimeout(activeRequestTimeoutMs, () => destroySocket(socket)); - }); - - /* BUFFER then forward — do NOT open upstream or pipe bytes until the - * full body is received. Streaming would let a slow-loris that drips - * body bytes pin an UPSTREAM connection slot for the entire body- - * upload window, even though the proxy itself bounds its own socket - * lifetime. Tool-call payloads are small JSON (capped at maxBodyBytes, - * default 1 MiB), so buffering is cheap and the security property - * "upstream never sees a partial request" is much stronger. */ - const bodyChunks: Buffer[] = []; - let upstream: http.ClientRequest | undefined; - const abortUpstream = (): void => { - if (!upstream || upstreamClosed || res.writableEnded) return; - upstream.destroy(new Error('tool-call client disconnected')); - }; - req.on('aborted', abortUpstream); - res.on('close', abortUpstream); - socket.on('close', abortUpstream); - - req.on('data', chunk => { - if (rejected) return; - bodyBytes += chunk.length; - if (bodyBytes > maxBodyBytes) { - rejected = true; - res.writeHead(413, { 'Content-Type': 'text/plain', Connection: 'close' }); - res.end('request body too large'); - destroySocket(socket); - return; - } - bodyChunks.push(Buffer.from(chunk)); - }); - - req.on('end', () => { - if (rejected) return; - const body = bodyChunks.length === 1 ? bodyChunks[0] : Buffer.concat(bodyChunks); - - /* Strip ALL hop-by-hop headers before forwarding (RFC 7230 §6.1). - * Previously we only stripped `proxy-connection`, which left - * Upgrade, TE, Trailer, and friends as a vector for protocol - * confusion at the upstream. */ - const headers = buildForwardedHeaders(req.headers, target.host); - /* Override declared content-length with the actual buffered size - * — if the client lied, what we send is what we have. */ - headers['content-length'] = body.length; - - upstream = transport.request({ - protocol: target.protocol, - hostname: target.hostname, - port: target.port || undefined, - method: 'POST', - path: '/tool-call', - headers, - }, upstreamRes => { - if (rejected) { - upstreamRes.resume(); - return; - } - /* Strip hop-by-hop on the response path. Upstream's Connection / - * Keep-Alive / Upgrade headers describe the proxy <-> upstream - * link, not what we should tell the sandbox client. */ - const respHeaders: http.OutgoingHttpHeaders = {}; - for (const [k, v] of Object.entries(upstreamRes.headers)) { - if (HOP_BY_HOP_HEADERS.has(k.toLowerCase())) continue; - if (v == null) continue; - respHeaders[k] = v; - } - respHeaders.Connection = 'close'; - res.writeHead(upstreamRes.statusCode || 502, respHeaders); - upstreamRes.pipe(res); - }); - activeUpstreams += 1; - - upstream.on('close', () => { - upstreamClosed = true; - activeUpstreams = Math.max(0, activeUpstreams - 1); - releaseActiveRequest(); - }); - - upstream.setTimeout(activeRequestTimeoutMs, () => { - upstream?.destroy(new Error('tool-call upstream timeout')); - }); - - upstream.on('error', error => { - if (rejected) return; - log.error('tool-call socket proxy upstream error', error); - if (!res.headersSent) { - res.writeHead(502, { 'Content-Type': 'text/plain', Connection: 'close' }); - } - res.end('bad gateway'); - }); - - upstream.end(body); - }); - }); - - server.maxConnections = maxConnections; - server.headersTimeout = headerTimeoutMs; - server.keepAliveTimeout = 1; - /* Bound the entire request reception (first byte -> last body byte) to - * the body-upload window. Prevents slow-loris drip attacks: socket-idle - * timers reset on every byte, so a malicious client could otherwise - * stretch body upload to activeRequestTimeoutMs (default 30s) by - * sending one byte just under the idle threshold. */ - server.requestTimeout = requestBodyTimeoutMs; - server.timeout = activeRequestTimeoutMs; - - server.on('connection', socket => { - /* Rate-limit BEFORE the concurrency check. The concurrency check - * (`activeSockets.size`) tells us how many connections are alive - * RIGHT NOW; the rate limiter tells us how fast new ones are being - * created. The audit's flood passes the concurrency check trivially - * (most connections accepted, processed, and destroyed faster than - * the count grows) but creates thousands of socket structs in the - * process. Rate-limiting at connect-time is what bounds the kernel - * slab pressure that surfaces as EMFILE in subsequent loader open() - * calls — see createTokenBucket comment for the full mechanism. */ - if (!connectionRateLimiter.tryConsume()) { - connectionRateDroppedCount += 1; - destroySocket(socket); - return; - } - if (activeSockets.size >= maxConnections) { - log.warn('tool-call socket proxy connection limit reached; dropping connection'); - destroySocket(socket); - return; - } - - activeSockets.add(socket); - const headerReceiveDeadline = setTimeout(() => { - headerReceiveDeadlines.delete(socket); - destroySocket(socket); - }, headerTimeoutMs); - headerReceiveDeadlines.set(socket, headerReceiveDeadline); - - socket.setTimeout(idleSocketTimeoutMs, () => destroySocket(socket)); - socket.on('close', () => { - clearHeaderReceiveDeadline(socket); - activeSockets.delete(socket); - }); - socket.on('error', () => { - clearHeaderReceiveDeadline(socket); - activeSockets.delete(socket); - }); - }); - - server.on('clientError', (_error, socket) => { - destroySocket(socket as net.Socket); - }); - - try { - fs.unlinkSync(socketPath); - } catch { - // Missing stale socket is fine. - } - - await new Promise((resolve, reject) => { - const onError = (error: Error): void => { - server.off('listening', onListening); - reject(error); - }; - const onListening = (): void => { - server.off('error', onError); - if (opts.socketUid != null && opts.socketGid != null) { - fs.chownSync(socketPath, opts.socketUid, opts.socketGid); - } - fs.chmodSync(socketPath, socketMode); - log.log(`tool-call socket proxy listening on ${socketPath}`); - resolve(); - }; - server.once('error', onError); - server.once('listening', onListening); - server.listen(socketPath, listenBacklog); - }); - - return { - server, - close: async () => { - for (const socket of activeSockets) { - destroySocket(socket); - } - await new Promise((resolve) => { - server.close(() => { - try { - fs.unlinkSync(socketPath); - } catch { - // Socket was already removed. - } - resolve(); - }); - }); - }, - activeConnections: () => activeSockets.size, - activeRequests: () => activeRequests, - activeUpstreams: () => activeUpstreams, - connectionRateTokens: () => connectionRateLimiter.tokens(), - connectionRateDropped: () => connectionRateDroppedCount, - }; -} - -if (require.main === module) { - const socketPath = process.env.TCS_SOCKET || '/tmp/tcs.sock'; - const rawTarget = process.env.SANDBOX_FORWARD_TARGET || ''; - const socketUid = process.env.TCS_SOCKET_UID ? Number(process.env.TCS_SOCKET_UID) : undefined; - const socketGid = process.env.TCS_SOCKET_GID ? Number(process.env.TCS_SOCKET_GID) : undefined; - const maxConnections = parsePositiveInt(process.env.TCS_MAX_CONNECTIONS); - const headerTimeoutMs = parsePositiveInt(process.env.TCS_HEADER_TIMEOUT_MS); - /* Operator overrides for the connection-rate limiter — see - * createTokenBucket comment for the kernel-slab-pressure motivation. - * Defaults (burst=64, refill=20/sec) are sized for the SDK's tool-call - * pattern (one connection per call) plus generous headroom; raise only - * if a legitimate workload trips the dropped-connection counter. */ - const rateBurst = parsePositiveInt(process.env.TCS_CONNECTION_RATE_BURST); - const rateRefillPerSec = parsePositiveInt(process.env.TCS_CONNECTION_RATE_REFILL_PER_SEC); - let handle: ToolCallSocketProxyHandle | undefined; - let shuttingDown = false; - - const shutdown = (): void => { - if (shuttingDown) return; - shuttingDown = true; - if (!handle) { - /* Signal arrived before startToolCallSocketProxy() resolved. - * `process.exit` is terminal in Node, but the explicit `return` makes - * the contract local: no `handle.close()` on `undefined` even if a - * future runtime ever defers exit (atexit hook, async-cleanup mode). */ - process.exit(0); - return; - } - void handle.close().finally(() => process.exit(0)); - }; - - process.on('SIGTERM', shutdown); - process.on('SIGINT', shutdown); - - startToolCallSocketProxy({ - socketPath, rawTarget, socketUid, socketGid, - maxConnections, - headerTimeoutMs, - connectionRateBurst: rateBurst, - connectionRateRefillPerSec: rateRefillPerSec, - }) - .then(started => { - handle = started; - }) - .catch(error => { - console.error('tool-call socket proxy failed to start', error); - process.exit(1); - }); -} diff --git a/build-packages.sh b/build-packages.sh index 2ae04d02..67912ee5 100755 --- a/build-packages.sh +++ b/build-packages.sh @@ -10,7 +10,7 @@ # Environment Variables: # PYTHON_VERSION=3.14.4 # Python version to install # NODE_VERSION=24.15.0 # Node.js version to install -# BUN_VERSION=1.3.14 # Bun version to install +# BUN_VERSION=1.4.2 # Bun version to install # BASH_PACKAGE_VERSION=5.2.0 # Bash package registration version (semver-like x.y.z) # SKIP_PYTHON_PACKAGES=1 # Skip Python pip packages # SKIP_JS_PACKAGES=1 # Skip JavaScript npm packages for Node/Bun @@ -28,7 +28,7 @@ cd "$SCRIPT_DIR" PYTHON_VERSION="${PYTHON_VERSION:-3.14.4}" NODE_VERSION="${NODE_VERSION:-24.15.0}" -BUN_VERSION="${BUN_VERSION:-1.3.14}" +BUN_VERSION="${BUN_VERSION:-1.4.2}" PACKAGES_DIR="./data/pkgs" JS_PACKAGE_MANIFEST="${JS_PACKAGE_MANIFEST:-${SCRIPT_DIR}/javascript-packages.txt}" diff --git a/docker/Dockerfile.worker-sandbox b/docker/Dockerfile.worker-sandbox index 7156719c..5e66baed 100644 --- a/docker/Dockerfile.worker-sandbox +++ b/docker/Dockerfile.worker-sandbox @@ -42,8 +42,10 @@ RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \ RUN make -j$(nproc) COPY api/src/spec-guard.c /tmp/spec-guard.c +COPY api/src/tool-call-pipe-bridge.c /tmp/tool-call-pipe-bridge.c COPY docker/rootfs-setup.c /tmp/rootfs-setup.c -RUN gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ +RUN gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/tool-call-pipe-bridge /tmp/tool-call-pipe-bridge.c \ + && gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ && gcc -O2 -static -o /usr/local/bin/sandbox-rootfs-setup /tmp/rootfs-setup.c \ && chmod 0111 /usr/local/bin/spec-guard /usr/local/bin/sandbox-rootfs-setup @@ -58,7 +60,7 @@ FROM buildpack-deps:bookworm AS package-builder ENV DEBIAN_FRONTEND=noninteractive ENV PYTHON_VERSION=3.14.4 -ENV BUN_VERSION=1.3.14 +ENV BUN_VERSION=1.4.2 COPY docker/apt-install.sh /usr/local/bin/apt-install RUN chmod +x /usr/local/bin/apt-install @@ -80,7 +82,7 @@ RUN chmod +x /package-init.sh && FORCE_REBUILD=true /package-init.sh # ============================================================================ # Stage 2: Build worker # ============================================================================ -FROM oven/bun:1.3.14 AS worker-builder +FROM oven/bun:1.4.2 AS worker-builder WORKDIR /app COPY service/package.json service/bun.lock ./ RUN bun install --frozen-lockfile @@ -93,19 +95,20 @@ RUN bun build ./src/worker-server.ts --minify --outdir .build --target bun --ext # ============================================================================ # Stage 3: Build sandbox API # ============================================================================ -FROM oven/bun:1.3.14 AS sandbox-builder +FROM oven/bun:1.4.2 AS sandbox-builder WORKDIR /app COPY api/package.json ./ RUN bun install --frozen-lockfile COPY api/src ./src COPY shared /shared COPY api/tsconfig.json ./ -RUN bun build ./src/index.ts --minify --outdir .build --target bun --external '@opentelemetry/*' +RUN bun build ./src/index.ts --minify --outdir .build --target bun --external '@opentelemetry/*' \ + && bun build ./src/tool-call-pipe-broker.ts --target=node --format=cjs --outfile=.build/tool-call-pipe-broker.cjs # ============================================================================ # Stage 4: Build sandbox rootfs (full OS layer for the microVM guest) # ============================================================================ -FROM oven/bun:1.3.14-debian AS sandbox-rootfs +FROM oven/bun:1.4.2-debian AS sandbox-rootfs ENV DEBIAN_FRONTEND=noninteractive @@ -143,7 +146,9 @@ RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen COPY --from=nsjail-builder /nsjail/nsjail /usr/sbin/nsjail COPY --from=nsjail-builder /usr/local/bin/spec-guard /usr/local/bin/spec-guard +COPY --from=nsjail-builder /usr/local/bin/tool-call-pipe-bridge /usr/local/bin/tool-call-pipe-bridge RUN chmod +x /usr/sbin/nsjail +COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node WORKDIR /sandbox_api diff --git a/docker/package-init.sh b/docker/package-init.sh index 60855b95..1ffaaf3a 100644 --- a/docker/package-init.sh +++ b/docker/package-init.sh @@ -21,7 +21,7 @@ PYTHON_ALIAS="python${PYTHON_SITE_VERSION}" PYTHON_PACKAGE_INSTALLER="${PYTHON_PACKAGE_INSTALLER:-uv}" UV_VERSION="${UV_VERSION:-0.11.26}" NODE_VERSION="${NODE_VERSION:-24.15.0}" -BUN_VERSION="${BUN_VERSION:-1.3.14}" +BUN_VERSION="${BUN_VERSION:-1.4.2}" BASH_PACKAGE_VERSION="${BASH_PACKAGE_VERSION:-5.2.0}" INSTALL_FAILED=false JS_PACKAGE_MANIFEST="${JS_PACKAGE_MANIFEST:-${SCRIPT_DIR}/javascript-packages.txt}" diff --git a/docs/SECURITY-MAINTENANCE.md b/docs/SECURITY-MAINTENANCE.md index c8494a90..b8026383 100644 --- a/docs/SECURITY-MAINTENANCE.md +++ b/docs/SECURITY-MAINTENANCE.md @@ -48,40 +48,107 @@ are denied even for messages without descriptors and for inherited sockets. Keep `io_uring_setup`, `io_uring_enter`, and `io_uring_register` denied: ring operations can submit sends without passing through these syscall filters. -Unlike the UDF sandbox's socket-family restriction, this sandbox retains -AF_UNIX and `socketpair` for `/tmp/tcs.sock` and ordinary IPC using reads, -writes, and `sendto`. Python multiprocessing pipes, queues, and pools remain -supported; transferring socket/file handles through `multiprocessing.reduction` -or similar descriptor-passing APIs returns EPERM. Do not restore `sendmsg` or -`sendmmsg` to make those APIs work without reassessing the kernel boundary. +The job policy denies every `socket()` family (VSOCK retains its KILL action). +It allows only AF_UNIX SOCK_STREAM `socketpair` with protocol 0, including +CLOEXEC/NONBLOCK flags. Python asyncio and duplex multiprocessing pipes need +these anonymous socketpairs. Ordinary fork/spawn queues and pools are covered +by native tests. Named Unix listeners, `multiprocessing.Manager`, datagram or +seqpacket pairs, and descriptor-sharing APIs are unsupported. The send and +io_uring denials still prevent SCM_RIGHTS descriptor graphs on allowed pairs. This mitigation is separate from PI-futex filtering. Verify vendor patches for the deployed guest and node kernels; a policy update does not establish patch status. Rebuild and replace sandbox runner images to deploy the policy change. -The native `NsJail IPC Filter` CI jobs compile the rendered policy with the -image's pinned Kafel on amd64 and arm64, install it in a Linux process, and -check descriptor sends, inherited sockets, Unix HTTP, and Python fork/spawn -queues and pools. This validates the filter and IPC compatibility, not a -production kernel's patch status or the full microVM deployment. - -### Planned transport hardening - -Replace the job-visible Unix HTTP socket with two dedicated per-job pipes to -a trusted broker. Keep them separate from stdin/stdout/stderr, which carry user -input and execution output. Limit message sizes, outstanding requests, and -timeouts; retain the existing job authorization and tool-call budgets at the -broker. Update each language's tool-call client to use framed messages, and -explicitly preserve only these pipe FDs through NsJail and `spec-guard`. -The current guard closes all FDs above 2; a redesign must preserve its protection -against unrelated inherited descriptors. - -Then deny `socket` and `socketpair` in the job policy and retain the send and -io_uring denials. Validate language/runtime compatibility, including Python's -multiprocessing resource tracker and descriptor-sharing features, before making -that stricter policy the default. The broker retains network access outside the -job filter; it must not expose raw socket handles to the job. This is follow-up -work, not part of the immediate descriptor-passing mitigation. +## Tool-call pipe transport + +Authorized blocking Python invocations receive two anonymous pipe endpoints: +FD 3 writes requests and FD 4 reads responses. No tool-call socket path is +mounted in the jail. The signed, body-bound `tool_call_socket` boolean remains +the wire capability name for compatibility with manifest verification; it now +grants pipes. Ordinary executions, compile steps, and replay/Bash tool calls +receive no endpoints. The Python client makes both ends non-inheritable before +user code runs and refuses tool calls from forked children. + +A Node broker starts per invocation after MicroVM restore and forwards only +POST `/tool-call` to the configured `SANDBOX_FORWARD_TARGET`. It forwards only +the three PTC claim headers; upstream session/token/tool authorization remains +in force. It never follows redirects. Each channel bounds frames and upstream +responses to 1 MiB, active upstream requests to 16, admission to a 64-request +burst refilling at 20 requests/second, partial-frame and response-drain stalls +to 5 seconds, and upstream requests to the job timeout (5–600 seconds). +Malformed framing closes the invocation; disconnect/exit aborts upstream work. +The Python client correlates concurrent responses by frame ID. + +Node/Bun extra stdio descriptors are socketpairs. A small trusted C relay +converts the broker's channel into real anonymous pipes before execing NsJail. +NsJail explicitly passes only FDs 3/4; `spec-guard --tool-call-pipes` checks their +FIFO type and access direction, retains those two ends, and closes unrelated +FDs. Its ordinary mode still closes every descriptor above 2. API/broker/relay +parent-death handling kills the invocation if either supervisor disappears. +Each invocation adds a Node broker and C relay outside the job cgroup; size +runner CPU/memory and PID limits for the configured simultaneous-job count. + +The amd64/arm64 `NsJail IPC Filter` CI jobs compile the policy with pinned Kafel, +exercise socket/send denials plus asyncio/multiprocessing, check guard FD +cleanup, and run the actual broker/relay/NsJail/guard/generated Python client +with concurrent replies. These checks do not boot libkrun or establish deployed +kernel patch status. + +Every complete frame consumes the admission budget before JSON or claim +validation, including frames rejected by the active-request limit. Exhausting +that budget closes the channel and terminates the invocation instead of +producing an unlimited stream of rejection replies. Losing the response pipe's +reader also terminates the invocation, even when no reply is buffered. + +The standalone launcher guest, API runner and combined worker runner all +package Node, the broker bundle and the compiled pipe bridge. Native CI builds +the standalone launcher guest on both architectures and exercises a round trip +using that image's own binaries. + +### Local Mac/Docker canary + +`tests/tool_call_runner.cjs` exercises signed API authorization, the normal +NsJail mount/user/PID/network/IPC/UTS/cgroup namespaces, the generated Python +client, concurrent tool calls, descriptor cleanup, spawn/fork pipe IPC, ordinary +Python jobs and basic Bun JavaScript/TypeScript execution. Spawned workers can +import the preamble but cannot issue tool calls. Python 3.14's default forkserver +requires a named Unix listener and is denied; select an explicit spawn or fork +context. Semaphore-based pools/queues require `/dev/shm`, which the default jail +does not mount. The namespace-disabled native policy tests cover those APIs +when shared memory exists, not their availability in the default jail. + +With runtime packages built by `build-packages.sh` (or a compatible `/pkgs` +tree containing Python 3.14.4 and Bun 1.4.2), run from the repository root: + +```sh +bun service/scripts/dump-pipe-preamble.ts > /tmp/codeapi-pipe-client.py +docker build -f api/Dockerfile --target sandbox-build -t codeapi-pipe-canary . +docker run --rm --init \ + --cap-add SYS_ADMIN --cap-add SYS_CHROOT --cap-add SETUID \ + --cap-add SETGID --cap-add NET_ADMIN \ + --security-opt seccomp=./seccomp/nsjail.json \ + --mount "type=bind,source=$PWD/data/pkgs,target=/pkgs,readonly" \ + --mount "type=bind,source=/tmp/codeapi-pipe-client.py,target=/client.py,readonly" \ + --mount "type=bind,source=$PWD/tests,target=/tests,readonly" \ + --entrypoint /usr/local/bin/node \ + codeapi-pipe-canary /tests/tool_call_runner.cjs /client.py +``` + +The canary disables resource cgroup enforcement as the Mac compose override +does, while retaining the cgroup namespace and all remaining NsJail isolation. +It exercises Docker's Linux VM, not a libkrun guest. It does not establish +production kernel patch status or third-party language-package compatibility. + +### Coordinated rollout + +Deploy the service's new blocking preamble together with rebuilt runner images +containing the broker, relay and updated guard/policy. Drain old blocking jobs +and route new blocking requests only to matching images during rollout. Old +socket preambles fail on pipe-only runners; new pipe preambles fail immediately +on old runners. There is no TCP or Unix-socket fallback. Replay mode is unchanged. +Rollback must restore both service and runner versions; retain #309's send and +io_uring denials. See [ADR 0011](adr/0011-tool-call-pipes.md) for the design. ## Preserve host-side confinement diff --git a/docs/adr/0011-tool-call-pipes.md b/docs/adr/0011-tool-call-pipes.md new file mode 100644 index 00000000..69e0b0ee --- /dev/null +++ b/docs/adr/0011-tool-call-pipes.md @@ -0,0 +1,61 @@ +# 0011: Per-invocation anonymous pipes for blocking tool calls + +Status: Accepted + +## Context + +Blocking Python tool calls used an HTTP listener at `/tmp/tcs.sock` mounted into +selected NsJail jobs. CVE-2026-80521 motivated descriptor-passing denials in +#309. Keeping socket creation enabled nevertheless exposes named endpoints +and socket families to untrusted code. The runner is snapshotted before work: +Node and its TLS state must continue to start after MicroVM restore. + +The current stdio spawn APIs create socketpairs, not anonymous pipe endpoints. +The existing guard also intentionally removes unrelated inherited descriptors. +Python asyncio and duplex multiprocessing pipes use Unix stream socketpairs; +a blanket socketpair denial would break these ordinary workloads. + +## Decision + +Use one post-restore Node broker per authorized blocking invocation. Keep the +existing signed/body-bound `tool_call_socket` capability name on the wire, but +grant two pipe endpoints instead of a socket mount. Use length-prefixed JSON +frames with response IDs, a fixed upstream route, whitelisted claim headers, +and explicit size, rate, concurrency and deadline limits. Keep upstream +cryptographic/session/tool authorization authoritative. + +A trusted C relay holds the spawn API's socket on its own side and passes real +anonymous pipe ends at FDs 3/4 into NsJail. The guard preserves them only under +an explicit option after validating FIFO type and write/read access; all +unrelated inherited descriptors are closed. API/broker parent death, channel failure and +job exit terminate the relay and its invocation and abort outstanding upstream +requests. The Python client clears descriptor inheritance and refuses calls +from forked or spawned children. Spawned workers may re-import tool definitions +without receiving or using the pipe capability. stdout/stderr and user stdin remain separate. + +Deny all socket creation. Permit only protocol-0 Unix stream socketpairs for +runtime IPC, retaining the sendmsg/sendmmsg/io_uring denials. This preserves +asyncio and pipe-based multiprocessing using explicit spawn/fork contexts without permitting named socket access +or descriptor passing. No legacy TCP/socket fallback is provided. + +## Consequences + +New service preambles and runner images require coordinated deployment; +blocking jobs must be drained or version-routed during upgrades and rollbacks. +Bash/replay mode has no pipe grant. Unix listeners/managers, non-stream pairs, +and descriptor-sharing APIs are unsupported. Python 3.14 defaults to forkserver, +which requires a named Unix listener and is denied; use an explicit spawn or +fork context. Semaphore-based queues/pools also require `/dev/shm`, which the +normal jail currently does not mount. Each active invocation adds two +trusted processes outside its job cgroup; runner sizing must account for them. + +A shared post-restore broker could reduce process overhead, but would require +an additional channel admission/multiplexing mechanism. Separate brokers keep +request state, failure and lifetime tied directly to one invocation. Named +FIFOs avoid inherited-FD changes but add filesystem names and open/unlink races; +anonymous pipes require neither path discovery nor shared listener state. + +Native CI verifies both CPU architectures, actual FIFO descriptors, guard +cleanup, socket/send denials, runtime compatibility and concurrent generated +client calls through the compiled policy. Production guest/node patches and +full libkrun rollout remain operator checks. diff --git a/docs/lambda-microvm/README.md b/docs/lambda-microvm/README.md index 80e5e468..b8fbbcc5 100644 --- a/docs/lambda-microvm/README.md +++ b/docs/lambda-microvm/README.md @@ -511,7 +511,7 @@ appear in `api/src/config.ts`. | `EGRESS_GATEWAY_FILE_SERVER_URL` / `_TOOL_CALL_SERVER_URL` | Gateway only | Private upstream origins. Do not expose them to the runner. | | `FILE_SERVER_URL` | Worker only | Direct source used to fetch authorized by-reference inputs before pushing them into the runner cache. Do not bake it into the runner. | | `CODEAPI_EGRESS_LEDGER_REQUIRED=true` / `REDIS_*` | Gateway | Makes grant replay/revocation state fail closed in Redis. | -| `SANDBOX_ALLOWED_LOCAL_NETWORK_PORT` / `SANDBOX_FORWARD_TARGET` | Runner image | Configures the narrow tool-call socket proxy; it does not start or expose the socket by itself. The target host/port must match `EGRESS_GATEWAY_URL`. | +| `SANDBOX_ALLOWED_LOCAL_NETWORK_PORT` / `SANDBOX_FORWARD_TARGET` | Runner image | Configures the per-invocation tool-call pipe broker; Node starts only on an authorized post-restore execution. The target host/port must match `EGRESS_GATEWAY_URL`. | | `CODEAPI_EXECUTION_MANIFEST_PRIVATE_KEY` | Worker only | Signs the manifest after the gateway returns the scoped grant. | | `SANDBOX_EXECUTION_MANIFEST_PUBLIC_KEY` / `SANDBOX_REQUIRE_EGRESS_MANIFEST=true` | Runner image | Verifies and requires the signed per-execution manifest. | diff --git a/docs/openshell/README.md b/docs/openshell/README.md index a139ab7c..880225b0 100644 --- a/docs/openshell/README.md +++ b/docs/openshell/README.md @@ -143,7 +143,7 @@ full authorization and delivery contract before implementation. ## Run the lifecycle probe -Requirements: Node 20+, an OpenShell `0.1.2` CLI, a registered named evaluation +Requirements: Node 22.21+, an OpenShell `0.1.2` CLI, a registered named evaluation gateway using the matching release, and a digest-pinned shell/coreutils image pullable by that gateway. Install and configure these separately on dedicated infrastructure. This repository does not install or start a gateway. diff --git a/docs/remote-bridge/worker-runbook.md b/docs/remote-bridge/worker-runbook.md index 4026a755..99f19992 100644 --- a/docs/remote-bridge/worker-runbook.md +++ b/docs/remote-bridge/worker-runbook.md @@ -119,7 +119,7 @@ a separate machine. Install: -- Node.js 20.11 or newer (Node.js 24 is supported); +- Node.js 22.21 or newer (Node.js 24 is supported); - Git; - `bubblewrap`, `socat`, and `ripgrep` on Linux; - Bash 5.2 or newer and `jq` when Bash Programmatic Tool Calling is enabled; diff --git a/helm/codeapi/values.yaml b/helm/codeapi/values.yaml index 0a242639..3ce4cd09 100644 --- a/helm/codeapi/values.yaml +++ b/helm/codeapi/values.yaml @@ -310,7 +310,7 @@ workerSandbox: pullPolicy: IfNotPresent pythonVersion: "3.14.4" nodeVersion: "24.15.0" - bunVersion: "1.3.14" + bunVersion: "1.4.2" bashPackageVersion: "5.2.0" backoffLimit: 2 ttlSecondsAfterFinished: 3600 diff --git a/launcher/Dockerfile b/launcher/Dockerfile index b2b02386..4acd2094 100644 --- a/launcher/Dockerfile +++ b/launcher/Dockerfile @@ -28,17 +28,20 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \ - && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800 + && git checkout 66ad78dc361721512fbf9470488e584364c75489 \ + && git submodule update --init --recursive RUN make -j$(nproc) COPY api/src/spec-guard.c /tmp/spec-guard.c +COPY api/src/tool-call-pipe-bridge.c /tmp/tool-call-pipe-bridge.c COPY docker/rootfs-setup.c /tmp/rootfs-setup.c -RUN gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ +RUN gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/tool-call-pipe-bridge /tmp/tool-call-pipe-bridge.c \ + && gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ && gcc -O2 -static -o /usr/local/bin/sandbox-rootfs-setup /tmp/rootfs-setup.c \ && chmod 0111 /usr/local/bin/spec-guard /usr/local/bin/sandbox-rootfs-setup -FROM oven/bun:1.3.14-debian AS sandbox-build +FROM oven/bun:1.4.2-debian AS sandbox-build ENV DEBIAN_FRONTEND=noninteractive @@ -64,12 +67,14 @@ RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen COPY --from=nsjail-builder /nsjail/nsjail /usr/sbin/nsjail COPY --from=nsjail-builder /usr/local/bin/spec-guard /usr/local/bin/spec-guard +COPY --from=nsjail-builder /usr/local/bin/tool-call-pipe-bridge /usr/local/bin/tool-call-pipe-bridge +COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node RUN chmod +x /usr/sbin/nsjail WORKDIR /sandbox_api -COPY api/package.json ./ -RUN bun install +COPY api/package.json api/bun.lock ./ +RUN bun ci COPY api/tsconfig.json ./ COPY api/src ./src diff --git a/packages/code/README.md b/packages/code/README.md index d348fe88..0437ecfb 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -3,6 +3,9 @@ Provider-neutral protocol and worker CLI for attaching a stateful, sandboxed code environment to LibreChat Code API. +Requires Node.js 22.21 or newer. CI tests Node.js 22 and 24; Node.js 20 is +unsupported. + For a complete machine setup and operations guide, see the [self-hosted worker runbook](../../docs/remote-bridge/worker-runbook.md). diff --git a/packages/code/package-lock.json b/packages/code/package-lock.json index 426b950f..a27188a6 100644 --- a/packages/code/package-lock.json +++ b/packages/code/package-lock.json @@ -21,7 +21,7 @@ "typescript": "^5.5.4" }, "engines": { - "node": ">=20.11" + "node": ">=22.21" } }, "node_modules/@anthropic-ai/sandbox-runtime": { diff --git a/packages/code/package.json b/packages/code/package.json index d3cf714b..95eea841 100644 --- a/packages/code/package.json +++ b/packages/code/package.json @@ -65,7 +65,7 @@ "typescript": "^5.5.4" }, "engines": { - "node": ">=20.11" + "node": ">=22.21" }, "dependencies": { "@anthropic-ai/sandbox-runtime": "0.0.75", diff --git a/service/scripts/dump-pipe-preamble.ts b/service/scripts/dump-pipe-preamble.ts new file mode 100644 index 00000000..86ec11fa --- /dev/null +++ b/service/scripts/dump-pipe-preamble.ts @@ -0,0 +1,6 @@ +process.env.DOTENV_CONFIG_QUIET = 'true'; +const { generatePreamble } = await import('../src/preamble'); +process.stdout.write(generatePreamble({ + callbackUrl: 'http://unused.invalid', callbackToken: 'pipe-test-token', + executionId: 'pipe-test-execution', tools: [], +})); diff --git a/service/src/config.spec.ts b/service/src/config.spec.ts index 69b416d8..3c2ca9d2 100644 --- a/service/src/config.spec.ts +++ b/service/src/config.spec.ts @@ -67,15 +67,15 @@ describe('runtime version configuration', () => { }); }); - it('maps Bun JavaScript and TypeScript requests to Bun 1.3.14', () => { + it('maps Bun JavaScript and TypeScript requests to Bun 1.4.2', () => { expect(languageConfig[Languages.js]).toMatchObject({ language: 'bun-js', - version: '1.3.14', + version: '1.4.2', fileName: 'index.js', }); expect(languageConfig[Languages.ts]).toMatchObject({ language: 'bun-ts', - version: '1.3.14', + version: '1.4.2', fileName: 'main.ts', }); }); diff --git a/service/src/config.ts b/service/src/config.ts index 29ae854d..7ba8b9b4 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -10,10 +10,10 @@ import { export const languageConfig: Record = { [Languages.bash]: { language: 'bash', version: '5.2.0', fileName: 'script.sh' }, - [Languages.js]: { language: 'bun-js', version: '1.3.14', fileName: 'index.js' }, + [Languages.js]: { language: 'bun-js', version: '1.4.2', fileName: 'index.js' }, [Languages.node]: { language: 'node', version: '24.15.0', fileName: 'index.js' }, [Languages.py]: { language: 'python', version: '3.14.4', fileName: 'main.py' }, - [Languages.ts]: { language: 'bun-ts', version: '1.3.14', fileName: 'main.ts' }, + [Languages.ts]: { language: 'bun-ts', version: '1.4.2', fileName: 'main.ts' }, }; const languageAliases: Record = { diff --git a/service/src/egress-gateway.ts b/service/src/egress-gateway.ts index 5a36ebc5..2af379c1 100644 --- a/service/src/egress-gateway.ts +++ b/service/src/egress-gateway.ts @@ -785,8 +785,8 @@ app.post('/tool-call', async (req, res) => { if (!executionId || !callId || !opaqueCallbackToken) { /* Generic 404 so a sandbox attacker probing paths cannot tell that * `/tool-call` is the live route by reading the error body. The - * body and headers must MATCH the tool-call socket proxy's own 404 - * for unknown paths (see api/src/tool-call-socket-proxy.ts) byte- + * body and headers must match the generic unknown-route response + * for unknown paths byte- * for-byte — case included — otherwise the sandbox can still * fingerprint the real route by comparing response shapes. */ logger.warn( diff --git a/service/src/preamble.test.ts b/service/src/preamble.test.ts index 803e1839..b4293fa5 100644 --- a/service/src/preamble.test.ts +++ b/service/src/preamble.test.ts @@ -15,71 +15,16 @@ const baseConfig = { tools: [], }; -describe('generatePreamble — Unix-vs-TCP transport gate', () => { - /* Codex audit follow-up: a path-existence check (`os.path.exists`) on - * /tmp/tcs.sock was user-spoofable inside the sandbox — in - * legacy/no-proxy mode, user code could `open('/tmp/tcs.sock', 'w')` - * and force the next tool call into the unix-socket branch (where - * connect() then fails with ENOTSOCK) instead of falling back to TCP. - * The fix probes a real AF_UNIX connect() at preamble import time - * (before user code runs) and caches the verdict. These tests assert - * the rendered preamble keeps that property. */ - - test('renders a probe that performs an AF_UNIX connect, not a path-existence check', () => { - const preamble = generatePreamble(baseConfig); - expect(preamble).toContain('def _probe_tool_call_socket():'); - /* The probe must construct an AF_UNIX socket and call connect(). A - * file-existence check would not detect a regular file or stale - * inode planted by the user. */ - expect(preamble).toMatch(/AF_UNIX/); - expect(preamble).toMatch(/\.connect\(_TOOL_CALL_SOCKET\)/); - /* Regression guard against reintroducing the user-spoofable check. */ - expect(preamble).not.toMatch( - /if\s+os\.path\.exists\(_TOOL_CALL_SOCKET\)/, - ); - }); - - test('caches the probe verdict at module load (before user code can plant a spoof)', () => { +describe('generatePreamble — anonymous pipe transport', () => { + test('uses fixed FIFO descriptors and fails closed without a pipe-enabled runner', () => { const preamble = generatePreamble(baseConfig); - /* The probe call must appear at top level of the preamble, NOT - * inside _do_request. Otherwise a user could plant a regular file - * at the path between calls and flip the gate per-request. */ - const probeCallIdx = preamble.indexOf( - '_USE_TOOL_CALL_SOCKET = _probe_tool_call_socket()', - ); - expect(probeCallIdx).toBeGreaterThan(-1); - /* _do_request must consult the cached verdict, not re-probe. */ - const doReqMatch = preamble.match( - /def\s+_do_request[\s\S]*?(?=\ndef\s|\nclass\s|\Z)/, - ); - expect(doReqMatch).not.toBeNull(); - expect(doReqMatch![0]).toContain('_USE_TOOL_CALL_SOCKET'); - expect(doReqMatch![0]).not.toContain('_probe_tool_call_socket('); - }); - - test('keeps the TCP fallback path for legacy / no-proxy deployments', () => { - const preamble = generatePreamble(baseConfig); - /* The fallback must still construct the URL from _CALLBACK_URL and - * delegate to _tcp_request. Without this, runners without the - * proxy bind-mount would have no way to reach the orchestrator. */ - const doReqMatch = preamble.match( - /def\s+_do_request[\s\S]*?(?=\ndef\s|\nclass\s|\Z)/, - ); - expect(doReqMatch).not.toBeNull(); - expect(doReqMatch![0]).toContain('_CALLBACK_URL + path'); - expect(doReqMatch![0]).toContain('_tcp_request('); - }); - - test('the literal socket path stays inside the preamble, not in os.environ', () => { - const preamble = generatePreamble(baseConfig); - /* Hardening item from PR #1648: TOOL_CALL_SOCKET is no longer - * exported. The path must remain hardcoded so the preamble does - * not depend on env-var injection. */ - expect(preamble).toContain('_TOOL_CALL_SOCKET = "/tmp/tcs.sock"'); - expect(preamble).not.toMatch( - /os\.environ\.get\(['"]TOOL_CALL_SOCKET['"]/, - ); - expect(preamble).not.toMatch(/os\.environ\[['"]TOOL_CALL_SOCKET['"]\]/); + expect(preamble).toContain('_PIPE_WRITE_FD = 3'); + expect(preamble).toContain('_PIPE_READ_FD = 4'); + expect(preamble).toContain('_stat.S_ISFIFO(os.fstat(_fd).st_mode)'); + expect(preamble).toContain('blocking tool calls require a pipe-enabled runner'); + expect(preamble).not.toContain('_tcp_request'); + expect(preamble).not.toContain('/tmp/tcs.sock'); + expect(preamble).not.toContain('import socket'); }); }); @@ -131,14 +76,14 @@ describe('extractPendingFromStdout — input hash metadata', () => { }); }); -describe('createProgrammaticPayload — tool-call socket opt-in', () => { +describe('createProgrammaticPayload — tool-call pipe capability', () => { const req = { body: { code: 'print("ok")', }, } as Parameters[0]['req']; - test('requests the sandbox socket for blocking PTC only', () => { + test('requests the sandbox pipe capability for blocking PTC only', () => { const payload = createProgrammaticPayload({ req, session_id: 'session-blocking', @@ -152,7 +97,7 @@ describe('createProgrammaticPayload — tool-call socket opt-in', () => { expect(payload.tool_call_socket).toBe(true); }); - test('does not request the sandbox socket for replay PTC', () => { + test('does not request the sandbox pipe capability for replay PTC', () => { const payload = createProgrammaticPayload({ req, session_id: 'session-replay', diff --git a/service/src/preamble.ts b/service/src/preamble.ts index 94685c6c..4e8003c2 100644 --- a/service/src/preamble.ts +++ b/service/src/preamble.ts @@ -324,7 +324,7 @@ ${nameComment} _input = {${inputDict}} * Generates dual-mode stubs that work with or without await */ export function generatePreamble(config: PreambleConfig): string { - const { callbackUrl, callbackToken, executionId, tools } = config; + const { callbackToken, executionId, tools } = config; // Header with HTTP callback infrastructure (dual-mode version) let preamble = ` @@ -337,73 +337,137 @@ import json import sys import os import asyncio -import socket as _socket -import http.client as _http_client from typing import Any, Dict, List, Optional, Union -from urllib import request, error as urllib_error -_CALLBACK_URL = "${callbackUrl}" -_CALLBACK_TOKEN = "${callbackToken}" -_EXECUTION_ID = "${executionId}" +_CALLBACK_TOKEN = ${JSON.stringify(callbackToken)} +_EXECUTION_ID = ${JSON.stringify(executionId)} _TOOL_CALL_COUNTER = 0 _TOOL_CALL_LOCK = asyncio.Lock() -# The proxy bind-mounts this fixed path into every sandbox; the env var -# version was dropped to keep the path off os.environ introspection. -_TOOL_CALL_SOCKET = "/tmp/tcs.sock" - -class _UnixHTTPConnection(_http_client.HTTPConnection): - """HTTPConnection that tunnels through a Unix domain socket""" - def __init__(self, socket_path): - super().__init__("localhost") - self._socket_path = socket_path - def connect(self): - self.sock = _socket.socket(_socket.AF_UNIX, _socket.SOCK_STREAM) - self.sock.settimeout(self.timeout) - self.sock.connect(self._socket_path) - -def _unix_request(method, path, body=None, headers=None, timeout=300): - conn = _UnixHTTPConnection(_TOOL_CALL_SOCKET) - conn.timeout = timeout - conn.connect() - conn.request(method, path, body=body, headers=headers or {}) - resp = conn.getresponse() - data = resp.read() - conn.close() - return resp.status, data - -def _tcp_request(method, url, body=None, headers=None, timeout=300): - req = request.Request(url, data=body, headers=headers or {}, method=method) - with request.urlopen(req, timeout=timeout) as resp: - return resp.status, resp.read() - -def _probe_tool_call_socket(): - # One-shot probe: is the path a live AF_UNIX listener? A real - # connect() succeeds against the proxy; a regular file or stale - # node returns ENOTSOCK / ECONNREFUSED. Done at preamble import - # time, *before* user code runs, so a malicious user cannot spoof - # /tmp/tcs.sock to flip the gate (a previous os.path.exists check - # was user-spoofable in legacy/no-proxy mode). Each NsJail - # invocation gets a fresh /tmp tmpfs so the probe result cannot - # bleed across jobs. +import stat as _stat +import struct as _struct +import threading as _threading +import select as _select +import time as _time +from concurrent.futures import Future as _Future + +# These two anonymous pipe ends are granted only to an authorized invocation. +_PIPE_WRITE_FD = 3 +_PIPE_READ_FD = 4 +_PIPE_MAX_FRAME = 1024 * 1024 +_PIPE_OWNER_PID = os.getpid() +_PIPE_LOCK = _threading.Lock() +_PIPE_WRITE_LOCK = _threading.Lock() +_PIPE_PENDING = {} +_PIPE_SEQUENCE = 0 +_PIPE_READER = None +_PIPE_FAILURE = None + +# Spawned multiprocessing workers re-import this module without the granted +# descriptors. They may import tool definitions, but cannot use the capability. +if __name__ == "__mp_main__": + _PIPE_OWNER_PID = None +else: + for _fd in (_PIPE_WRITE_FD, _PIPE_READ_FD): + try: + if not _stat.S_ISFIFO(os.fstat(_fd).st_mode): + raise OSError("tool-call descriptor is not an anonymous pipe") + os.set_inheritable(_fd, False) + except OSError as _error: + raise RuntimeError("blocking tool calls require a pipe-enabled runner") from _error + os.set_blocking(_PIPE_WRITE_FD, False) + +def _pipe_fail(error): + global _PIPE_FAILURE + with _PIPE_LOCK: + _PIPE_FAILURE = error + pending = list(_PIPE_PENDING.values()) + _PIPE_PENDING.clear() + for future in pending: + if not future.done(): + future.set_exception(error) + +def _pipe_read_exact(size): + chunks = [] + remaining = size + while remaining: + chunk = os.read(_PIPE_READ_FD, remaining) + if not chunk: + raise EOFError("tool-call broker disconnected") + chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) + +def _pipe_read_responses(): try: - s = _socket.socket(_socket.AF_UNIX, _socket.SOCK_STREAM) - s.settimeout(0.5) - s.connect(_TOOL_CALL_SOCKET) - s.close() - return True - except OSError: - return False - -_USE_TOOL_CALL_SOCKET = _probe_tool_call_socket() + while True: + size = _struct.unpack("!I", _pipe_read_exact(4))[0] + if not 0 < size <= _PIPE_MAX_FRAME: + raise ValueError("invalid tool-call response size") + response = json.loads(_pipe_read_exact(size).decode("utf-8")) + if not isinstance(response, dict) or not isinstance(response.get("status"), int) or not isinstance(response.get("body"), str): + raise ValueError("invalid tool-call response") + with _PIPE_LOCK: + future = _PIPE_PENDING.pop(response.get("id"), None) + # A late response after a caller timeout has no pending waiter. + if future is not None: + future.set_result((response["status"], response["body"].encode("utf-8"))) + except Exception as error: + _pipe_fail(error) def _do_request(method, path, body=None, headers=None, timeout=300): - # Cached at import time (see _probe_tool_call_socket). User code - # cannot influence this decision because it has not yet executed. - if _USE_TOOL_CALL_SOCKET: - return _unix_request(method, path, body, headers, timeout) - url = _CALLBACK_URL + path - return _tcp_request(method, url, body, headers, timeout) + global _PIPE_READER, _PIPE_SEQUENCE + if os.getpid() != _PIPE_OWNER_PID: + raise RuntimeError("tool calls must run in the invocation's primary process") + if method != "POST" or path != "/tool-call": + raise ValueError("unsupported tool-call route") + deadline = _time.monotonic() + timeout + future = _Future() + with _PIPE_LOCK: + if _PIPE_FAILURE is not None: + raise _PIPE_FAILURE + if len(_PIPE_PENDING) >= 16: + raise RuntimeError("too many concurrent tool calls") + if _PIPE_READER is None: + _PIPE_READER = _threading.Thread(target=_pipe_read_responses, daemon=True) + _PIPE_READER.start() + _PIPE_SEQUENCE += 1 + sequence = _PIPE_SEQUENCE + _PIPE_PENDING[sequence] = future + try: + payload = json.dumps({ + "id": sequence, + "headers": {key.lower(): value for key, value in (headers or {}).items()}, + "body": (body or b"").decode("utf-8") + }).encode("utf-8") + if not 0 < len(payload) <= _PIPE_MAX_FRAME: + raise ValueError("tool-call request too large") + framed = _struct.pack("!I", len(payload)) + payload + # Serialize writes, but allow concurrent upstream requests and responses. + if not _PIPE_WRITE_LOCK.acquire(timeout=max(0, deadline - _time.monotonic())): + raise TimeoutError("tool-call write timeout") + try: + remaining = memoryview(framed) + while remaining: + wait = deadline - _time.monotonic() + if wait <= 0 or not _select.select([], [_PIPE_WRITE_FD], [], wait)[1]: + raise TimeoutError("tool-call write timeout") + try: + written = os.write(_PIPE_WRITE_FD, remaining[:4096]) + except BlockingIOError: + continue + remaining = remaining[written:] + except Exception as error: + # A partial frame cannot be reused for the next request. + _pipe_fail(error) + os.close(_PIPE_WRITE_FD) + raise + finally: + _PIPE_WRITE_LOCK.release() + return future.result(timeout=max(0, deadline - _time.monotonic())) + finally: + with _PIPE_LOCK: + _PIPE_PENDING.pop(sequence, None) class ToolExecutionError(Exception): """Raised when a tool call fails""" @@ -500,7 +564,7 @@ async def _execute_tool_internal_async(tool_name: str, tool_input: Dict[str, Any * History is delivered as a file (not an env var) so very large histories * are not bounded by the Linux ARG_MAX / MAX_ARG_STRLEN ceiling. * - * No socket bind mount, no Tool Call Server callback, no long-poll. + * No pipe capability, no Tool Call Server callback, no long-poll. */ export function generateReplayPreamble(config: ReplayPreambleConfig): string { const { executionId, tools } = config; @@ -519,7 +583,7 @@ import os import asyncio from typing import Any, Dict, List, Optional, Union -_EXECUTION_ID = "${executionId}" +_EXECUTION_ID = ${JSON.stringify(executionId)} _PTC_SENTINEL_START = "${scopedStart}" _PTC_SENTINEL_END = "${scopedEnd}" _PTC_HISTORY_PATH = os.environ.get("PTC_HISTORY_PATH") or "${PTC_HISTORY_SANDBOX_PATH}" diff --git a/tests/nsjail_unix_gc.py b/tests/nsjail_unix_gc.py index 84cec7b5..a576fcdb 100644 --- a/tests/nsjail_unix_gc.py +++ b/tests/nsjail_unix_gc.py @@ -8,14 +8,11 @@ import array import ctypes as c import errno -import http.client import multiprocessing as mp import os import socket import struct import sys -import tempfile -import threading class SockFilter(c.Structure): @@ -94,48 +91,6 @@ def test_sends(libc, pair): right.close() -def test_unix_http(): - with tempfile.TemporaryDirectory() as directory: - pathname = os.path.join(directory, "tcs.sock") - with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as listener: - listener.bind(pathname) - listener.listen(1) - listener.settimeout(5) - errors = [] - - def serve(): - try: - with listener.accept()[0] as peer: - peer.settimeout(5) - request = b"" - while b"\r\n\r\n" not in request: - chunk = peer.recv(4096) - assert chunk, "client disconnected before HTTP headers" - request += chunk - assert request.startswith(b"POST /tool-call HTTP/1.1") - peer.sendall(b"HTTP/1.1 200 OK\r\nContent-Length: 11\r\n" - b"Connection: close\r\n\r\n{\"ok\":true}") - except Exception as error: - errors.append(error) - - thread = threading.Thread(target=serve) - thread.start() - connection = http.client.HTTPConnection("localhost", timeout=5) - connection.sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - connection.sock.settimeout(5) - connection.sock.connect(pathname) - try: - connection.request("POST", "/tool-call", body=b"{}") - response = connection.getresponse() - assert response.status == 200 - assert response.read() == b'{"ok":true}' - finally: - connection.close() - thread.join(5) - assert not thread.is_alive() - assert not errors, errors - - def main(): # Open before installation to verify the rule also covers inherited FDs. inherited = socket.socketpair() @@ -154,9 +109,13 @@ def main(): assert libc.syscall(syscall, 0, 0, 0, 0, 0, 0) == -1 assert c.get_errno() == errno.EPERM test_sends(libc, inherited) - for kind in (socket.SOCK_STREAM, socket.SOCK_DGRAM): - test_sends(libc, socket.socketpair(socket.AF_UNIX, kind)) - test_unix_http() + test_sends(libc, socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM)) + for domain in (socket.AF_UNIX, socket.AF_INET, socket.AF_INET6, socket.AF_NETLINK): + expect_eperm(lambda: socket.socket(domain, socket.SOCK_STREAM)) + expect_eperm(lambda: socket.socketpair(socket.AF_UNIX, socket.SOCK_DGRAM)) + expect_eperm(lambda: socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM, 6)) + import asyncio + asyncio.run(asyncio.sleep(0)) for method in ("fork", "spawn"): context = mp.get_context(method) queue = context.Queue() @@ -169,7 +128,7 @@ def main(): queue.join_thread() with context.Pool(2) as pool: assert pool.map(square, [2, 3, 4]) == [4, 9, 16] - print("PASS: descriptor sends denied; Unix IPC, HTTP, queues and pools work") + print("PASS: descriptor sends denied; socket creation denied; stream IPC, asyncio, queues and pools work") if __name__ == "__main__": diff --git a/tests/pipe_runtime_image.cjs b/tests/pipe_runtime_image.cjs new file mode 100644 index 00000000..8b2d1008 --- /dev/null +++ b/tests/pipe_runtime_image.cjs @@ -0,0 +1,61 @@ +#!/usr/bin/env node +// Packaging regression: run in each guest rootfs. Uses the image's own broker, +// Node, bridge, NsJail and guard, with namespaces disabled for this smoke test. +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const http = require('node:http'); +const { spawn } = require('node:child_process'); +const { once } = require('node:events'); + +if (process.argv[2] === '--client') { + assert(fs.fstatSync(3).isFIFO()); + assert(fs.fstatSync(4).isFIFO()); + const frame = Buffer.from(JSON.stringify({ id: 1, headers: { + 'x-execution-id': 'image-smoke', 'x-callback-token': 'image-token', 'x-tool-call-id': 'image-call', + }, body: '{"tool_name":"echo","input":{}}' })); + const prefix = Buffer.alloc(4); prefix.writeUInt32BE(frame.length); + const bytes = Buffer.concat([prefix, frame]); + for (let offset = 0; offset < bytes.length;) offset += fs.writeSync(3, bytes, offset, bytes.length - offset); + function readExact(size) { + const bytes = Buffer.alloc(size); + for (let offset = 0; offset < size;) { + const count = fs.readSync(4, bytes, offset, size - offset); + assert(count > 0, 'broker response ended prematurely'); offset += count; + } + return bytes; + } + const reply = JSON.parse(readExact(readExact(4).readUInt32BE()).toString()); + assert.deepEqual(reply, { id: 1, status: 200, body: 'packaged runtime works' }); + console.log('PASS: image Node/broker/bridge/NsJail/guard pipe round trip'); +} else { + (async () => { + const upstream = http.createServer(async (req, res) => { + assert.equal(req.url, '/tool-call'); + assert.equal(req.headers['x-callback-token'], 'image-token'); + for await (const bytes of req) {} + res.end('packaged runtime works'); + }); + upstream.listen(0, '127.0.0.1'); await once(upstream, 'listening'); + let child; + let timer; + try { + child = spawn('/usr/local/bin/tool-call-pipe-bridge', [ + '--broker', '/usr/local/bin/node', '/sandbox_api/.build/tool-call-pipe-broker.cjs', + '/usr/sbin/nsjail', '--mode', 'e', '--disable_clone_newnet', '--disable_clone_newuser', + '--disable_clone_newns', '--disable_clone_newpid', '--disable_clone_newipc', + '--disable_clone_newuts', '--disable_clone_newcgroup', '--disable_proc', '--disable_rlimits', + '--pass_fd', '3', '--pass_fd', '4', '--', '/usr/local/bin/spec-guard', + '--tool-call-pipes', '/usr/local/bin/node', '/tests/pipe_runtime_image.cjs', '--client', + ], { stdio: ['ignore', 'inherit', 'inherit'], env: { + ...process.env, SANDBOX_FORWARD_TARGET: `http://127.0.0.1:${upstream.address().port}`, + }}); + timer = setTimeout(() => child.kill('SIGKILL'), 5000); + const [code, signal] = await once(child, 'exit'); + assert.equal(code, 0, `image pipe runtime failed (${signal})`); + } finally { + clearTimeout(timer); + if (child && child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); + upstream.closeAllConnections(); upstream.close(); + } + })().catch(error => { console.error(error); process.exitCode = 1; }); +} diff --git a/tests/specguard_fd_isolation.py b/tests/specguard_fd_isolation.py deleted file mode 100644 index d7aa79cf..00000000 --- a/tests/specguard_fd_isolation.py +++ /dev/null @@ -1,289 +0,0 @@ -""" -spec-guard FD isolation regression test. - -Reproduces the audit's "permanent sandbox brick" pattern: - - 1. A first /api/v2/execute opens N socket FDs against /tmp/tcs.sock - and never closes them (simulates malicious sandbox code). - 2. After the first job exits, the runner's per-job FD slot bookkeeping - is supposed to be back to baseline. - 3. A SECOND /api/v2/execute runs trivial Python and reports its - inherited FD count via /proc/self/fd. - -Without `spec-guard` closing inherited descriptors before `execvp()`, the -second job's user process starts with whatever FDs leaked through the -runner -> NsJail -> child chain. With the fix, the second job's Python -process sees only stdin/stdout/stderr (plus whatever it itself opens). - -The test asserts BOTH: - - The simulated leak in job 1 actually opens many FDs. - - The follow-up job 2 starts with a clean FD table (<= a small bound). - -Usage (local docker compose stack): - - SANDBOX_REQUIRE_EGRESS_MANIFEST=false CODEAPI_HARDENED_SANDBOX_MODE=false \\ - docker compose up -d - python3 services/codeapi/tests/specguard_fd_isolation.py -""" -from __future__ import annotations - -import json -import os -import sys -import urllib.error -import urllib.request - -SANDBOX_URL = os.environ.get("SANDBOX_URL", "http://127.0.0.1:2000") -REQUEST_TIMEOUT_S = float(os.environ.get("SPEC_GUARD_TEST_TIMEOUT_S", "30")) - -# After spec-guard runs and the user's Python interpreter starts, a tiny -# number of FDs are legitimately open: 0/1/2 plus whatever Python opened -# during startup (typically the dirfd it iterates /proc/self/fd through). -# Anything above this bound is leaked descriptors. -MAX_LEGITIMATE_INHERITED_FDS = 10 - - -def execute(language: str, code: str, timeout_ms: int = 15_000) -> dict: - body = { - "language": language, - "version": "*", - "files": [{"name": "main.py" if language == "python" else "main.sh", - "content": code}], - "stdin": "", - "args": [], - "run_timeout": timeout_ms, - } - data = json.dumps(body).encode("utf-8") - req = urllib.request.Request( - f"{SANDBOX_URL}/api/v2/execute", - data=data, - headers={"Content-Type": "application/json"}, - method="POST", - ) - with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT_S) as resp: - return json.loads(resp.read().decode("utf-8")) - - -STORM_TARGET = 500 -"""Open many more connections than the proxy's maxConnections cap (64 by -default). The kernel queues some (listen backlog) and accepts the rest as -slots free up via the proxy's idle timer. This matches the audit's -literal red-team script — `for i in range(500): connect(); leak`. Without -this volume, the test wouldn't exercise the steady-state-pressure path -the audit broke on. The storm job exits without closing any of them; the -proxy must reclaim accept-side FDs and the runner must remain serviceable.""" - -MIN_STORM_OPENED = 200 -"""Lower bound on what the storm job must successfully open before we -trust the test was meaningful. Without this assertion the test would -silently degrade — if the proxy ever started refusing all connections -at the kernel layer, the storm would open 0 sockets, the FD-poisoning -preconditions would never be exercised, and the regression would still -report PASS. 200 is comfortably above the production cap of 64 so the -kernel accept queue + proxy reaping cycle is exercised.""" - - -def fd_storm_then_report() -> None: - """First job: open many connections to /tmp/tcs.sock and never close them. - The connections live until the user process exits — but the proxy and - runner-side accounting may retain state. Spec-guard must not let any of - that leak into the NEXT job's process.""" - code = ( - "import os, socket, json\n" - # The runner bind-mounts the proxy socket at this fixed path; the - # TOOL_CALL_SOCKET env var was dropped to keep the path off - # `os.environ`, so the test references the literal path instead. - "s = '/tmp/tcs.sock'\n" - "leaked = []\n" - "errors = []\n" - # Append every socket to a list so CPython's refcount keeps the FD - # alive until process exit. Without this, each loop iteration - # rebinds `sk` and the previous socket is GC'd before the next - # connect() runs — the loop reports `opened == STORM_TARGET` but - # only ONE FD is ever actually held, so the FD-poisoning - # preconditions the test exists to exercise are never created. - f"for _ in range({STORM_TARGET}):\n" - " try:\n" - " sk = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)\n" - " sk.connect(s); leaked.append(sk)\n" - " except OSError as e:\n" - " errors.append(str(e)); break\n" - # Read /proc/self/fd to PROVE we're actually holding `opened` FDs - # concurrently, so the assertion downstream isn't satisfied by - # the loop counter alone. - "concurrent_fds = len([x for x in os.listdir('/proc/self/fd') if x.isdigit()])\n" - "print(json.dumps({'opened': len(leaked), 'concurrent_fds': concurrent_fds, " - "'target': " - + str(STORM_TARGET) + ", 'errors': errors[:3]}))\n" - ) - result = execute("python", code) - run = result.get("run") or {} - if run.get("code") != 0: - print(f"FAIL: storm job exited non-zero: {run!r}", file=sys.stderr) - sys.exit(1) - stdout = (run.get("stdout") or "").strip() - print(f"[storm-job] {stdout}") - last_line = stdout.splitlines()[-1] if stdout else "" - try: - payload = json.loads(last_line) - except json.JSONDecodeError: - print( - f"FAIL: storm job did not produce JSON status line; got {last_line!r}", - file=sys.stderr, - ) - sys.exit(1) - opened = int(payload.get("opened", 0)) - concurrent = int(payload.get("concurrent_fds", 0)) - if opened < MIN_STORM_OPENED: - print( - f"FAIL: storm job opened only {opened} sockets (required >= " - f"{MIN_STORM_OPENED}); the FD-poisoning preconditions were not " - f"exercised, so a downstream PASS would be vacuous. errors: " - f"{payload.get('errors')}", - file=sys.stderr, - ) - sys.exit(1) - # Defend against the "vacuous pass via socket reuse" mode — assert - # the storm process is actually HOLDING the FDs concurrently in - # /proc/self/fd, not just opening-then-closing in a loop. - if concurrent < MIN_STORM_OPENED: - print( - f"FAIL: storm job reported {opened} opens but /proc/self/fd " - f"shows only {concurrent} concurrent FDs — sockets are being " - f"GC'd between iterations and the test is vacuous.", - file=sys.stderr, - ) - sys.exit(1) - - -def report_inherited_fd_count() -> int: - """Second job: introspect /proc/self/fd. Spec-guard should have closed - everything but stdin/out/err before this Python invocation began.""" - code = ( - "import os, json\n" - "fds = sorted(int(x) for x in os.listdir('/proc/self/fd') if x.isdigit())\n" - "print(json.dumps({'fd_count': len(fds), 'fds': fds[:20]}))\n" - ) - result = execute("python", code) - run = result.get("run") or {} - if run.get("code") != 0: - print(f"FAIL: report job exited non-zero: {run!r}", file=sys.stderr) - sys.exit(1) - stdout = (run.get("stdout") or "").strip() - print(f"[report-job] {stdout}") - payload = json.loads(stdout.split("\n")[-1]) - return int(payload["fd_count"]) - - -def post_storm_recovery() -> int: - """The audit's specific failure: after a socket storm, subsequent - dynamic-binary executions (python/node/bash/bun) failed during - library-load with `Error 24` (EMFILE). Run a real exec in each - language and require code 0 — proves the dynamic loader is no - longer poisoned by inherited descriptors.""" - cases = [ - ("python", "import sys; print('python', sys.version_info[:3])"), - ("node", "console.log('node', process.version)"), - ("bash", "echo bash $(/bin/bash --version | head -1)"), - ] - failed = 0 - for lang, code in cases: - result = execute(lang, code) - run = result.get("run") or {} - out = (run.get("stdout") or "").strip() - err = (run.get("stderr") or "").strip() - ok = run.get("code") == 0 and out and not err - marker = "ok " if ok else "FAIL" - print(f" [{marker}] {lang:6} code={run.get('code')!r:>4} stdout={out!r:<50} stderr={err[:80]!r}") - if not ok: - failed += 1 - # Specifically guard against the audit's EMFILE-on-loader pattern - if "Error 24" in err or "Too many open files" in err: - print(f" ^ audit-mode regression: dynamic loader hit EMFILE") - failed += 1 - return failed - - -def ptc_route_opacity_check() -> bool: - """Probe the proxy at /tool-call (with no PTC headers) and at unknown - paths from inside the sandbox; both must produce BYTE-IDENTICAL wire - responses, otherwise an attacker can still fingerprint the real route - by reading the response framing (ETag, X-Request-ID, charset, etc.). - The original audit broke on this exact gap — the response BODY matched - but the upstream's Express-added headers leaked through the proxy.""" - code = ( - "import os, socket, json\n" - # Same fixed-path reasoning as fd_storm_then_report(). - "s = '/tmp/tcs.sock'\n" - "responses = {}\n" - "for path in ['/tool-call', '/randomroute']:\n" - " sk = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)\n" - " sk.connect(s)\n" - " sk.sendall(('POST ' + path + ' HTTP/1.1\\r\\nHost: x\\r\\n" - "Content-Length: 0\\r\\nConnection: close\\r\\n\\r\\n').encode())\n" - " buf = b''\n" - " while True:\n" - " chunk = sk.recv(4096)\n" - " if not chunk: break\n" - " buf += chunk\n" - " sk.close()\n" - " # Strip the volatile Date header before comparing\n" - " lines = buf.split(b'\\r\\n')\n" - " filtered = b'\\r\\n'.join(l for l in lines if not l.lower().startswith(b'date:'))\n" - " responses[path] = filtered.decode('utf-8', 'replace')\n" - "print(json.dumps(responses))\n" - ) - result = execute("python", code) - run = result.get("run") or {} - if run.get("code") != 0: - print(f"FAIL: opacity probe exited non-zero: {run!r}", file=sys.stderr) - return False - payload = json.loads((run.get("stdout") or "").splitlines()[-1]) - tc, rr = payload["/tool-call"], payload["/randomroute"] - if tc == rr: - print(f" [ok ] /tool-call response is byte-identical to /randomroute " - f"({len(tc)} bytes)") - return True - print(f" [FAIL] wire responses differ — sandbox can fingerprint the route:") - print(f" /tool-call: {tc!r}") - print(f" /randomroute: {rr!r}") - return False - - -def main() -> int: - print(f"=== spec-guard FD isolation regression — runner: {SANDBOX_URL} ===") - print(f"step 1: open {STORM_TARGET} socket FDs in a first job (audit's exact red-team script)") - fd_storm_then_report() - print("step 2: launch a second job and report its inherited FD count") - fd_count = report_inherited_fd_count() - print(f" -> second job sees {fd_count} FDs (bound: {MAX_LEGITIMATE_INHERITED_FDS})") - if fd_count > MAX_LEGITIMATE_INHERITED_FDS: - print( - f"FAIL: second job inherited {fd_count} FDs — " - f"spec-guard did NOT close inherited descriptors before execvp", - file=sys.stderr, - ) - return 1 - print("step 3: post-storm recovery for python / node / bash") - failed = post_storm_recovery() - if failed: - print(f"FAIL: {failed} language(s) regressed", file=sys.stderr) - return 1 - print("step 4: PTC route opacity (byte-identical 404 wire responses)") - if not ptc_route_opacity_check(): - return 1 - print("PASS: spec-guard isolated the second job's FD table, " - "all runtimes recovered post-storm, and the /tool-call route " - "is wire-indistinguishable from any unknown path") - return 0 - - -if __name__ == "__main__": - try: - sys.exit(main()) - except urllib.error.HTTPError as exc: - print(f"HTTPError {exc.code}: {exc.read()[:300]!r}", file=sys.stderr) - sys.exit(2) - except urllib.error.URLError as exc: - print(f"URLError: {exc}", file=sys.stderr) - sys.exit(2) diff --git a/tests/specguard_tool_call_pipes.py b/tests/specguard_tool_call_pipes.py new file mode 100644 index 00000000..549b1fa9 --- /dev/null +++ b/tests/specguard_tool_call_pipes.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Verify the native guard preserves only validated pipe capabilities.""" +import fcntl +import os + + +def run_guard(pipes, preserve, reversed_ends=False): + request = os.pipe() + response = os.pipe() + ordinary = os.open('/dev/null', os.O_RDWR) + original = [request[0 if reversed_ends else 1] if pipes else ordinary, + response[1 if reversed_ends else 0] if pipes else ordinary, ordinary] + sources = [fcntl.fcntl(fd, fcntl.F_DUPFD_CLOEXEC, 10) for fd in original] + child = os.fork() + if child == 0: + for source, destination in zip(sources, [3, 4, 9]): + os.dup2(source, destination, inheritable=True) + code = ''' +import os, stat +for fd in ([9] if PRESERVE else [3,4,9]): + try: + os.fstat(fd) + except OSError: + pass + else: + raise AssertionError("unrelated FD survived guard: " + str(fd)) +if PRESERVE: + assert stat.S_ISFIFO(os.fstat(3).st_mode) + assert stat.S_ISFIFO(os.fstat(4).st_mode) +'''.replace('PRESERVE', repr(preserve)) + args = [os.environ.get('SPEC_GUARD_BINARY', '/usr/local/bin/spec-guard')] + if preserve: + args.append('--tool-call-pipes') + os.execv(args[0], args + ['/usr/bin/python3', '-c', code]) + for fd in sources + list(request) + list(response) + [ordinary]: + os.close(fd) + _, status = os.waitpid(child, 0) + return os.waitstatus_to_exitcode(status) + + +assert run_guard(True, True) == 0 +assert run_guard(True, False) == 0 +assert run_guard(False, True) == 1 +assert run_guard(True, True, reversed_ends=True) == 1 +print('PASS: guard preserves only explicit FIFO endpoints and rejects regular files') diff --git a/tests/tool_call_pipes.py b/tests/tool_call_pipes.py new file mode 100644 index 00000000..c9388530 --- /dev/null +++ b/tests/tool_call_pipes.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""End-to-end broker -> anonymous pipes -> NsJail -> guard -> Python client. + +Runs in the native nsjail-ipc-test image with Docker's outer filter disabled. +Arguments: rendered policy, built Node broker, generated Python preamble. +""" +import ctypes +import http.server +import json +import os +import subprocess +import sys +import tempfile +import threading +import time + + +class Upstream(http.server.BaseHTTPRequestHandler): + calls = [] + + def log_message(self, *args): + pass + + def do_POST(self): + assert self.path == '/tool-call' + assert self.headers['X-Execution-ID'] == 'pipe-test-execution' + assert self.headers['X-Callback-Token'] == 'pipe-test-token' + data = json.loads(self.rfile.read(int(self.headers['Content-Length']))) + value = data['input']['n'] + self.calls.append(value) + time.sleep((8 - value) * 0.01) # Responses arrive out of order. + body = json.dumps({'success': True, 'result': value}).encode() + self.send_response(200) + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + + +def main(): + policy, broker, preamble = sys.argv[1:] + server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), Upstream) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + with tempfile.TemporaryDirectory() as directory: + program = os.path.join(directory, 'client.py') + with open(preamble) as source: + client = source.read() + client += ''' +import errno +import socket +import subprocess +assert _stat.S_ISFIFO(os.fstat(3).st_mode) +assert _stat.S_ISFIFO(os.fstat(4).st_mode) +try: + os.fstat(9) +except OSError: + pass +else: + raise AssertionError("unrelated inherited descriptor survived spec-guard") +for domain in (socket.AF_UNIX, socket.AF_INET, socket.AF_INET6): + try: + socket.socket(domain, socket.SOCK_STREAM) + except OSError as error: + assert error.errno == errno.EPERM + else: + raise AssertionError("socket creation was allowed") +async def check(): + results = await asyncio.gather(*[_execute_tool_internal_async("echo", {"n": n}) for n in range(8)]) + assert results == list(range(8)), results +asyncio.run(check()) +# The capability must not leak through ordinary subprocess exec. +assert subprocess.check_output([sys.executable, "-c", "import os; print(os.path.exists('/proc/self/fd/3'))"]).strip() == b'False' +print("PASS: real pipes, restricted filter, guard cleanup and concurrent Python tool calls") +''' + with open(program, 'w') as output: + output.write(client) + env = {**os.environ, 'SANDBOX_FORWARD_TARGET': f'http://127.0.0.1:{server.server_port}', + 'TCS_PIPE_BRIDGE': '/usr/local/bin/tool-call-pipe-bridge'} + command = ['/usr/local/bin/tool-call-pipe-bridge', '--broker', '/usr/local/bin/node', broker, '/nsjail/nsjail', '--mode', 'e', + '--disable_clone_newnet', '--disable_clone_newuser', '--disable_clone_newns', + '--disable_clone_newpid', '--disable_clone_newipc', '--disable_clone_newuts', + '--disable_clone_newcgroup', '--disable_proc', '--disable_rlimits', + '--pass_fd', '3', '--pass_fd', '4', + '--seccomp_policy', policy, '--', '/usr/local/bin/spec-guard', + '--tool-call-pipes', '/usr/bin/python3', program] + result = subprocess.run(command, env=env, capture_output=True, text=True, timeout=30) + assert result.returncode == 0, (result.returncode, result.stdout, result.stderr) + assert sorted(Upstream.calls) == list(range(8)), Upstream.calls + print(result.stdout.strip()) + # Losing the response reader must terminate even with no buffered reply. + with tempfile.TemporaryDirectory() as directory: + closer = os.path.join(directory, 'close-response.py') + with open(closer, 'w') as output: + output.write("import os,time; os.close(4); time.sleep(30)") + interrupted = subprocess.run(command[:-1] + [closer], env=env, + capture_output=True, text=True, timeout=2) + assert interrupted.returncode == 137, (interrupted.returncode, interrupted.stdout, interrupted.stderr) + print('PASS: closing the response reader promptly terminates the invocation') + # A guest that drains rejection replies must still lose the capability + # once malformed-claim frames exhaust the broker's admission budget. + with tempfile.TemporaryDirectory() as directory: + flooder = os.path.join(directory, 'rejected-frames.py') + with open(flooder, 'w') as output: + output.write("""import json,os,struct,threading,time +threading.Thread(target=lambda: [os.read(4, 4096) for _ in range(1000)], daemon=True).start() +frames = [] +for id in range(1, 258): + frame = json.dumps({'id': id, 'headers': {}, 'body': ''}).encode() + frames.append(struct.pack('!I', len(frame)) + frame) +data = b''.join(frames) +while data: + data = data[os.write(3, data):] +time.sleep(30) +""") + interrupted = subprocess.run(command[:-1] + [flooder], env=env, + capture_output=True, text=True, timeout=2) + assert interrupted.returncode == 137, (interrupted.returncode, interrupted.stdout, interrupted.stderr) + print('PASS: drained rejection-frame flood promptly terminates the invocation') + # A killed API/controller must not orphan its broker, relay or job. Become a + # subreaper so this test can reap both adopted descendants in Docker. + assert ctypes.CDLL(None).prctl(36, 1, 0, 0, 0) == 0 + with tempfile.TemporaryDirectory() as directory: + marker = os.path.join(directory, 'job.pid') + sleeper = os.path.join(directory, 'sleep.py') + with open(sleeper, 'w') as output: + output.write(f"import os,time; open({marker!r},'w').write(str(os.getpid())); time.sleep(300)") + command[-1] = sleeper + controller = os.path.join(directory, 'controller.js') + broker_marker = os.path.join(directory, 'broker.pid') + with open(controller, 'w') as output: + output.write("import {spawn} from 'node:child_process'; import fs from 'node:fs';" + + f"const args={json.dumps(command)};" + + "const child=spawn(args[0], args.slice(1), {stdio:['pipe','inherit','inherit']}); child.stdin.end();" + + f"fs.writeFileSync({json.dumps(broker_marker)},String(child.pid));" + + "setInterval(()=>{},1000);") + running = subprocess.Popen(['/usr/local/bin/bun', controller], env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + deadline = time.monotonic() + 5 + while not os.path.exists(marker) and time.monotonic() < deadline: + time.sleep(0.01) + assert os.path.exists(marker), 'job did not start' + with open(marker) as source: + job_pid = int(source.read()) + with open(f'/proc/{job_pid}/status') as source: + relay_pid = next(int(line.split()[1]) for line in source if line.startswith('PPid:')) + with open(broker_marker) as source: + broker_pid = int(source.read()) + running.kill() + running.communicate(timeout=5) + for pid in [job_pid, relay_pid, broker_pid]: + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + reaped, status = os.waitpid(pid, os.WNOHANG) + if reaped: + assert os.waitstatus_to_exitcode(status) == -9 + break + time.sleep(0.01) + else: + os.kill(pid, 9) + raise AssertionError('orphan survived broker death') + finally: + if running.poll() is None: + running.kill() + running.communicate(timeout=5) + print('PASS: API death kills and reaps broker, relay and invocation') + server.shutdown() + server.server_close() + + +if __name__ == '__main__': + main() diff --git a/tests/tool_call_runner.cjs b/tests/tool_call_runner.cjs new file mode 100644 index 00000000..d40e0721 --- /dev/null +++ b/tests/tool_call_runner.cjs @@ -0,0 +1,127 @@ +#!/usr/bin/env node +// Run inside the sandbox-build image with its Mac/Docker capabilities and a +// read-only /pkgs mount. Arguments: generated pipe preamble, Python version. +// Exercises the signed API route and the normal NsJail namespace/mount setup. +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const http = require('node:http'); +const { spawn } = require('node:child_process'); +const { once } = require('node:events'); +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); +function canonical(value) { + if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`; + if (value && typeof value === 'object') return `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonical(value[key])}`).join(',')}}`; + return JSON.stringify(value); +} +async function main() { + const [preamblePath, version = '3.14.4'] = process.argv.slice(2); + const calls = []; + const upstream = http.createServer(async (req, res) => { + try { + assert.equal(req.url, '/tool-call'); + assert.equal(req.headers['x-execution-id'], 'pipe-test-execution'); + assert.equal(req.headers['x-callback-token'], 'pipe-test-token'); + let bytes = ''; + for await (const part of req) bytes += part; + const { n } = JSON.parse(bytes).input; + calls.push(n); + await sleep((8 - n) * 10); + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ success: true, result: n })); + } catch (error) { res.writeHead(500).end(JSON.stringify({ success: false, error: String(error) })); } + }); + upstream.listen(0, '127.0.0.1'); + await once(upstream, 'listening'); + const port = upstream.address().port; + const { privateKey, publicKey } = crypto.generateKeyPairSync('ed25519'); + const api = spawn('/sandbox_api/entrypoint.sh', [], { cwd: '/sandbox_api', stdio: ['ignore', 'inherit', 'inherit'], env: { + ...process.env, PORT: '2000', SANDBOX_USE_CGROUPV2: 'false', + SANDBOX_REMOVE_UMOUNT_AFTER_STARTUP: 'false', SANDBOX_REQUIRE_EGRESS_MANIFEST: 'true', + SANDBOX_EXECUTION_MANIFEST_PUBLIC_KEY: publicKey.export({ type: 'spki', format: 'pem' }), + SANDBOX_ALLOWED_LOCAL_NETWORK_PORT: String(port), SANDBOX_FORWARD_TARGET: `http://127.0.0.1:${port}`, + SANDBOX_OUTPUT_MAX_SIZE: '65536', SANDBOX_LOG_LEVEL: 'info', + }}); + try { + for (let n = 0; n < 200; n++) { + if (api.exitCode !== null) throw new Error(`API exited: ${api.exitCode}`); + try { if ((await fetch('http://127.0.0.1:2000/')).ok) break; } catch {} + if (n === 199) throw new Error('API did not start'); + await sleep(50); + } + const runtimes = await (await fetch('http://127.0.0.1:2000/api/v2/runtimes')).json(); + assert(runtimes.some(runtime => runtime.language === 'python' && runtime.version === version), JSON.stringify(runtimes)); + const outerNamespaces = Object.fromEntries(['net', 'pid', 'mnt', 'user', 'ipc', 'uts', 'cgroup'].map(name => [name, fs.readlinkSync(`/proc/self/ns/${name}`)])); + const client = fs.readFileSync(preamblePath, 'utf8') + ` +import errno, socket, subprocess, sys, multiprocessing as mp +def _mp_echo(sender): + try: + _do_request('POST', '/tool-call', '{}') + except RuntimeError as error: + assert 'primary process' in str(error) + else: + raise AssertionError('child acquired tool-call capability') + sender.send('child IPC works') + sender.close() +if __name__ == '__main__': + assert sys.version.split()[0] == ${JSON.stringify(version)} + assert os.getuid() == 65534 + assert _stat.S_ISFIFO(os.fstat(3).st_mode) and _stat.S_ISFIFO(os.fstat(4).st_mode) + for _namespace, _outside in ${JSON.stringify(outerNamespaces)}.items(): + assert os.readlink('/proc/self/ns/' + _namespace) != _outside, _namespace + for domain in (socket.AF_UNIX, socket.AF_INET, socket.AF_INET6): + try: socket.socket(domain, socket.SOCK_STREAM) + except OSError as error: assert error.errno == errno.EPERM + else: raise AssertionError('socket creation allowed') + assert subprocess.check_output([sys.executable, '-c', "import os; print(os.path.exists('/proc/self/fd/3'))"]).strip() == b'False' + async def check(): + results = await asyncio.gather(*[_execute_tool_internal_async('echo', {'n': n}) for n in range(8)]) + assert results == list(range(8)), results + asyncio.run(check()) + # Pools need /dev/shm and forkserver needs a named AF_UNIX listener. + # Test the supported pipe-only spawn/fork multiprocessing paths. + for method in ('spawn', 'fork'): + ctx = mp.get_context(method) + receiver, sender = ctx.Pipe(duplex=False) + child = ctx.Process(target=_mp_echo, args=(sender,)) + child.start() + sender.close() + assert receiver.poll(5), method + assert receiver.recv() == 'child IPC works' + receiver.close() + child.join(5) + assert child.exitcode == 0, (method, child.exitcode) + print('PASS: packaged Python, signed API, NsJail namespaces, pipes, concurrent tools, subprocess and multiprocessing') +`; + async function execute(content, pipes, language = 'python', runtimeVersion = version) { + const body = { language, version: runtimeVersion, session_id: `pipe-canary-${crypto.randomUUID()}`, files: [{ name: language === 'bun-js' ? 'main.js' : language === 'bun-ts' ? 'main.ts' : 'main.py', content }], tool_call_socket: pipes, run_timeout: 30000 }; + const now = Math.floor(Date.now() / 1000); + const claims = { v: 1, exec_id: 'pipe-test-execution', tenant_id: 'canary', user_id: 'canary', session_key: 'canary:user:canary', input_files: [], read_sessions: [], output_session_id: body.session_id, max_upload_bytes: 52428800, max_output_files: 50, max_requests: 1000, iat: now, exp: now + 300, tool_call_socket: pipes, execute_body_sha256: crypto.createHash('sha256').update(canonical(body)).digest('base64url') }; + const payload = canonical(claims); + body.execution_manifest = `${Buffer.from(payload).toString('base64url')}.${crypto.sign(null, Buffer.from(payload), privateKey).toString('base64url')}`; + const response = await fetch('http://127.0.0.1:2000/api/v2/execute', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); + const result = await response.json(); + assert.equal(response.status, 200, JSON.stringify(result)); + assert.equal(result.run.code, 0, JSON.stringify(result)); + return result; + } + const result = await execute(client, true); + assert.equal(calls.length, 8); + assert.deepEqual([...calls].sort((a, b) => a - b), [0,1,2,3,4,5,6,7]); + assert(result.run.stdout.includes('PASS:'), JSON.stringify(result)); + console.log(result.run.stdout.trim()); + // Ordinary jobs still work and do not receive the tool-call descriptors. + const plain = await execute("import os; assert not os.path.exists('/proc/self/fd/3'); print('PASS: ordinary execution without pipe capability')", false); + console.log(plain.run.stdout.trim()); + for (const language of ['bun-js', 'bun-ts']) { + const js = await execute("console.log('PASS: Bun ' + Bun.version)", false, language, '1.4.2'); + assert(js.run.stdout.includes('PASS: Bun 1.4.2'), JSON.stringify(js)); + console.log(language + ': ' + js.run.stdout.trim()); + } + } finally { + api.kill('SIGTERM'); + await once(api, 'exit'); + upstream.closeAllConnections(); upstream.close(); + } +} +main().catch(error => { console.error(error); process.exitCode = 1; }); From 4401420df463cb195230a201404056cca1e10f81 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 5 Oct 2026 13:52:11 -0400 Subject: [PATCH 36/48] =?UTF-8?q?=F0=9F=A7=B5=20fix:=20Replace=20Runtime?= =?UTF-8?q?=20Socketpairs=20With=20Anonymous=20Pipes=20(#312)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: deny socketpairs with pipe-only Python runtime IPC * test: document and verify Bun socketpair spawn denial * fix: refresh mutable package-init images during runtime rollout * test: render runtime rollout chart without optional dependencies * fix: preserve offline package-init image policy --- .github/workflows/ci.yml | 7 ++- api/Dockerfile | 3 + api/src/nsjail.test.ts | 6 +- api/src/nsjail.ts | 12 ++-- build-packages.sh | 3 + docker/Dockerfile.package-init | 1 + docker/Dockerfile.worker-sandbox | 1 + docker/package-init.sh | 2 + .../_sandbox_multiprocessing.py | 47 +++++++++++++++ docker/python-runtime/_sandbox_pipe.py | 32 ++++++++++ docker/python-runtime/install.py | 41 +++++++++++++ docs/SECURITY-MAINTENANCE.md | 32 +++++++--- docs/adr/0011-tool-call-pipes.md | 11 ++-- docs/adr/0012-pipe-only-python-runtime.md | 58 +++++++++++++++++++ helm/codeapi/templates/package-init-job.yaml | 3 +- helm/codeapi/values.yaml | 3 +- tests/nsjail_unix_gc.py | 23 +++++++- tests/package_init_runtime_rollout.sh | 31 ++++++++++ tests/python_pipe_runtime.py | 47 +++++++++++++++ tests/tool_call_runner.cjs | 10 +++- 20 files changed, 340 insertions(+), 33 deletions(-) create mode 100644 docker/python-runtime/_sandbox_multiprocessing.py create mode 100644 docker/python-runtime/_sandbox_pipe.py create mode 100644 docker/python-runtime/install.py create mode 100644 docs/adr/0012-pipe-only-python-runtime.md create mode 100755 tests/package_init_runtime_rollout.sh create mode 100644 tests/python_pipe_runtime.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 58f68b7b..98ddb2ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,9 @@ jobs: - name: Block-root package delivery run: tests/block_root_package_delivery.sh + - name: Python runtime package-init rollout + run: tests/package_init_runtime_rollout.sh + - name: KVM guest resolver handoff run: tests/kvm_guest_dns.sh @@ -173,12 +176,12 @@ jobs: launcher-guest-test /tests/pipe_runtime_image.cjs - name: Verify descriptor denial and runtime IPC with the installed filter run: | - docker run --rm --security-opt seccomp=unconfined \ + docker run --rm --init --security-opt seccomp=unconfined \ --mount "type=bind,source=$RUNNER_TEMP/nsjail.policy,target=/policy,readonly" \ --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ nsjail-ipc-test \ bash -c 'python3 /tests/nsjail_unix_gc.py /policy /nsjail/kafel/libkafel.so && python3 /tests/specguard_tool_call_pipes.py && gcc -O2 -static -Wall -Wextra -Werror -DSPEC_GUARD_TEST_DISABLE_CLOSE_RANGE -o /tmp/guard-fallback /tmp/spec-guard.c && SPEC_GUARD_BINARY=/tmp/guard-fallback python3 /tests/specguard_tool_call_pipes.py' - docker run --rm --security-opt seccomp=unconfined \ + docker run --rm --init --security-opt seccomp=unconfined \ --mount "type=bind,source=$RUNNER_TEMP/nsjail.policy,target=/policy,readonly" \ --mount "type=bind,source=$RUNNER_TEMP/pipe-client.py,target=/client.py,readonly" \ --mount "type=bind,source=$GITHUB_WORKSPACE/api/.build/tool-call-pipe-broker.cjs,target=/broker.cjs,readonly" \ diff --git a/api/Dockerfile b/api/Dockerfile index 0ee343a7..f953f537 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -38,6 +38,8 @@ RUN gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/tool-call-pipe-bridg # Native policy/transport verification without building the full runner. FROM nsjail-builder AS nsjail-ipc-test +COPY docker/python-runtime /python-runtime +RUN python3 /python-runtime/install.py COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node COPY --from=oven/bun:1.4.2-debian /usr/local/bin/bun /usr/local/bin/bun @@ -68,6 +70,7 @@ RUN apt-install \ RUN mkdir -p /pkgs +COPY docker/python-runtime /python-runtime COPY docker/package-init.sh /package-init.sh COPY javascript-packages.txt /javascript-packages.txt RUN chmod +x /package-init.sh && FORCE_REBUILD=true /package-init.sh diff --git a/api/src/nsjail.test.ts b/api/src/nsjail.test.ts index 73dc3863..ab046b7f 100644 --- a/api/src/nsjail.test.ts +++ b/api/src/nsjail.test.ts @@ -394,14 +394,14 @@ describe('NsJail seccomp policy', () => { expect(rule).not.toMatch(/== FUTEX_(WAIT|WAKE)(?:\s|$)/); }); - test('blocks descriptor-passing sends while preserving ordinary Unix socket IPC', () => { + test('blocks descriptor-passing sends and all socketpair creation', () => { const policy = seccompPolicy(); const errnoBlock = policy.split('ERRNO(1) {')[1]?.split(' }')[0] ?? ''; for (const name of ['sendmsg', 'sendmmsg', 'io_uring_setup', 'io_uring_enter', 'io_uring_register']) { expect(errnoBlock).toMatch(new RegExp(`\\b${name}\\b[,\\s]`)); } - // Restrict socketpairs to runtime stream IPC; tool calls use real pipes. - expect(policy).toContain('socketpair(domain, type, protocol) { domain != AF_UNIX || (type & 0xf) != SOCK_STREAM || protocol != 0 }'); + // The patched runtime and tool calls use anonymous pipes. + expect(policy).toContain(' socketpair'); for (const name of ['read', 'write', 'sendto', 'recvfrom']) { expect(policy).not.toMatch(new RegExp(`\\b${name}\\b`)); } diff --git a/api/src/nsjail.ts b/api/src/nsjail.ts index 51b8700e..5c39884f 100644 --- a/api/src/nsjail.ts +++ b/api/src/nsjail.ts @@ -95,8 +95,6 @@ const archSpecificLowPrioritySyscalls = process.arch === 'arm64' const SECCOMP_POLICY = [ ...syscallDefines, - '#define AF_UNIX 1', - '#define SOCK_STREAM 1', '#define AF_INET 2', '#define AF_INET6 10', '#define AF_NETLINK 16', @@ -153,8 +151,8 @@ const SECCOMP_POLICY = [ /* CVE-2026-80521: AF_UNIX GC is reachable through SCM_RIGHTS descriptor * passing. Seccomp cannot inspect the msghdr/mmsghdr control data, so * refuse both send APIs outright, including on inherited sockets. - * The job receives only anonymous tool-call pipes. Restricted Unix - * stream socketpairs remain for asyncio and subprocess/runtime IPC. + * The job receives only anonymous tool-call pipes. Anonymous + * pipes replace socketpairs in the packaged Python runtime. * io_uring above must remain blocked: * its sendmsg operations bypass syscall-level filtering. */ ' sendmsg, sendmmsg,', @@ -185,10 +183,10 @@ const SECCOMP_POLICY = [ ' pidfd_open(pid) { pid == 1 },', ' pidfd_send_signal,', /* Deny every socket() family. VSOCK keeps its stronger KILL action above. - * Only anonymous Unix stream socketpairs are allowed for runtime IPC; - * sendmsg/sendmmsg remain denied, so no SCM_RIGHTS graph can be built. */ + * Deny socketpairs as well; packaged runtime IPC uses anonymous pipes. + * Keep sendmsg/sendmmsg denied for inherited descriptors too. */ ' socket(domain) { domain != AF_VSOCK },', - ' socketpair(domain, type, protocol) { domain != AF_UNIX || (type & 0xf) != SOCK_STREAM || protocol != 0 }', + ' socketpair', ' }', '}', 'USE sandbox DEFAULT ALLOW', diff --git a/build-packages.sh b/build-packages.sh index 67912ee5..1d635297 100755 --- a/build-packages.sh +++ b/build-packages.sh @@ -118,6 +118,9 @@ install_python() { rm -rf /tmp/Python-${PYTHON_VERSION}* " + docker cp "$SCRIPT_DIR/docker/python-runtime" "$CONTAINER_NAME:/tmp/python-runtime" + docker exec "$CONTAINER_NAME" "${pkg_dest}/bin/python3" /tmp/python-runtime/install.py + docker exec "$CONTAINER_NAME" bash -c "cat > ${pkg_dest}/pkg-info.json << 'EOF' { \"language\": \"python\", diff --git a/docker/Dockerfile.package-init b/docker/Dockerfile.package-init index c94cb8af..745346f8 100644 --- a/docker/Dockerfile.package-init +++ b/docker/Dockerfile.package-init @@ -23,6 +23,7 @@ RUN apt-install \ RUN mkdir -p /pkgs +COPY docker/python-runtime /python-runtime COPY docker/package-init.sh /package-init.sh COPY javascript-packages.txt /javascript-packages.txt RUN chmod +x /package-init.sh diff --git a/docker/Dockerfile.worker-sandbox b/docker/Dockerfile.worker-sandbox index 5e66baed..0117132d 100644 --- a/docker/Dockerfile.worker-sandbox +++ b/docker/Dockerfile.worker-sandbox @@ -75,6 +75,7 @@ RUN apt-install \ RUN mkdir -p /pkgs +COPY docker/python-runtime /python-runtime COPY docker/package-init.sh /package-init.sh COPY javascript-packages.txt /javascript-packages.txt RUN chmod +x /package-init.sh && FORCE_REBUILD=true /package-init.sh diff --git a/docker/package-init.sh b/docker/package-init.sh index 1ffaaf3a..4b96305f 100644 --- a/docker/package-init.sh +++ b/docker/package-init.sh @@ -87,6 +87,7 @@ echo "==============================================" echo "" packages_ready() { + [ -f "/pkgs/python/${PYTHON_VERSION}/lib/python${PYTHON_SITE_VERSION}/.sandbox-pipe-runtime-v1" ] && [ -f "/pkgs/python/${PYTHON_VERSION}/.package-installed" ] && [ -d "/pkgs/python/${PYTHON_VERSION}/lib/python${PYTHON_SITE_VERSION}/site-packages/PIL" ] && [ -d "/pkgs/python/${PYTHON_VERSION}/lib/python${PYTHON_SITE_VERSION}/site-packages/markitdown" ] && @@ -138,6 +139,7 @@ cd "Python-${PYTHON_VERSION}" ./configure --prefix="$PKG_DEST" --enable-optimizations 2>/dev/null make -j$(nproc) make install +"${PKG_DEST}/bin/python3" "${SCRIPT_DIR}/python-runtime/install.py" cd /tmp rm -rf /tmp/Python-${PYTHON_VERSION}* diff --git a/docker/python-runtime/_sandbox_multiprocessing.py b/docker/python-runtime/_sandbox_multiprocessing.py new file mode 100644 index 00000000..62fc2362 --- /dev/null +++ b/docker/python-runtime/_sandbox_multiprocessing.py @@ -0,0 +1,47 @@ +"""Duplex multiprocessing connections backed by two anonymous pipes.""" +import os +from multiprocessing.connection import Connection +from multiprocessing.context import reduction + + +class DuplexConnection(Connection): + def __init__(self, reader, writer): + super().__init__(reader) + self._writer = writer + + def _send(self, data): + remaining = memoryview(data) + while remaining: + written = os.write(self._writer, remaining) + remaining = remaining[written:] + + def _close(self): + try: + os.close(self._handle) + finally: + os.close(self._writer) + + +def rebuild(reader, writer): + return DuplexConnection(reader.detach(), writer.detach()) + + +def reduce_connection(connection): + connection._check_closed() + return rebuild, (reduction.DupFd(connection.fileno()), + reduction.DupFd(connection._writer)) + + +reduction.register(DuplexConnection, reduce_connection) + + +def duplex_pipe(): + first_reader, first_writer = os.pipe() + try: + second_reader, second_writer = os.pipe() + except BaseException: + os.close(first_reader) + os.close(first_writer) + raise + return (DuplexConnection(first_reader, second_writer), + DuplexConnection(second_reader, first_writer)) diff --git a/docker/python-runtime/_sandbox_pipe.py b/docker/python-runtime/_sandbox_pipe.py new file mode 100644 index 00000000..645dbc2a --- /dev/null +++ b/docker/python-runtime/_sandbox_pipe.py @@ -0,0 +1,32 @@ +"""Pipe-only Linux runtime IPC; seccomp still enforces socket denial.""" +import os + + +class WakeupPipe: + def __init__(self, fd): + self._fd = fd + + def __del__(self): + self.close() + + def fileno(self): + return self._fd + + def setblocking(self, blocking): + os.set_blocking(self._fd, blocking) + + def recv(self, size): + return os.read(self._fd, size) + + def send(self, data): + return os.write(self._fd, data) + + def close(self): + fd, self._fd = self._fd, -1 + if fd >= 0: + os.close(fd) + + +def wakeup_pipe(): + reader, writer = os.pipe2(os.O_CLOEXEC | os.O_NONBLOCK) + return WakeupPipe(reader), WakeupPipe(writer) diff --git a/docker/python-runtime/install.py b/docker/python-runtime/install.py new file mode 100644 index 00000000..61194cbb --- /dev/null +++ b/docker/python-runtime/install.py @@ -0,0 +1,41 @@ +"""Apply checked stdlib edits to the interpreter running this installer.""" +from pathlib import Path +import shutil +import sysconfig + +root = Path(sysconfig.get_path('stdlib')) +source = Path(__file__).resolve().parent +edits = [ + ('asyncio/selector_events.py', + ' self._ssock, self._csock = socket.socketpair()', + ' from _sandbox_pipe import wakeup_pipe\n self._ssock, self._csock = wakeup_pipe()'), + ('multiprocessing/connection.py', + ''' if duplex: + s1, s2 = socket.socketpair() + s1.setblocking(True) + s2.setblocking(True) + c1 = Connection(s1.detach()) + c2 = Connection(s2.detach())''', + ''' if duplex: + from _sandbox_multiprocessing import duplex_pipe + return duplex_pipe()'''), +] +prepared = [] +for name, old, new in edits: + path = root / name + text = path.read_text() + if text.count(new) == 1: + continue + if text.count(old) != 1: + raise RuntimeError(f'unsupported Python runtime layout: {path}') + prepared.append((path, text.replace(old, new))) +for name in ('_sandbox_pipe.py', '_sandbox_multiprocessing.py'): + shutil.copyfile(source / name, root / name) + for cached in (root / '__pycache__').glob(Path(name).stem + '.*.pyc'): + cached.unlink() +for path, text in prepared: + path.write_text(text) + # Avoid reusing timestamp-based bytecode after editing installed sources. + for cached in (path.parent / '__pycache__').glob(path.stem + '.*.pyc'): + cached.unlink() +(root / '.sandbox-pipe-runtime-v1').write_text('1\n') diff --git a/docs/SECURITY-MAINTENANCE.md b/docs/SECURITY-MAINTENANCE.md index b8026383..f33e509f 100644 --- a/docs/SECURITY-MAINTENANCE.md +++ b/docs/SECURITY-MAINTENANCE.md @@ -49,12 +49,13 @@ Keep `io_uring_setup`, `io_uring_enter`, and `io_uring_register` denied: ring operations can submit sends without passing through these syscall filters. The job policy denies every `socket()` family (VSOCK retains its KILL action). -It allows only AF_UNIX SOCK_STREAM `socketpair` with protocol 0, including -CLOEXEC/NONBLOCK flags. Python asyncio and duplex multiprocessing pipes need -these anonymous socketpairs. Ordinary fork/spawn queues and pools are covered -by native tests. Named Unix listeners, `multiprocessing.Manager`, datagram or -seqpacket pairs, and descriptor-sharing APIs are unsupported. The send and -io_uring denials still prevent SCM_RIGHTS descriptor graphs on allowed pairs. +It also denies every `socketpair()`. The packaged Python runtime replaces +asyncio's wakeup socketpair with a nonblocking anonymous pipe and duplex +multiprocessing socketpairs with two anonymous pipes. Native tests cover +thread wakeups, signals, cancellation, subprocesses, duplex spawn/fork IPC, +queues and pools. Named Unix listeners, `multiprocessing.Manager`, forkserver, +and descriptor-sharing APIs remain unsupported. Keep the send and io_uring +denials for inherited descriptors and defense in depth. This mitigation is separate from PI-futex filtering. Verify vendor patches for the deployed guest and node kernels; a policy update does not establish patch @@ -143,7 +144,11 @@ production kernel patch status or third-party language-package compatibility. ### Coordinated rollout Deploy the service's new blocking preamble together with rebuilt runner images -containing the broker, relay and updated guard/policy. Drain old blocking jobs +containing the broker, relay and updated guard/policy. Rebuild Python packages +with the pipe-runtime patch before enabling socketpair denial. Package init +checks the `.sandbox-pipe-runtime-v1` marker and rebuilds unpatched packages. +Drain or version-route execution while replacing packages; rollback must pair +the policy with its compatible runtime packages. Drain old blocking jobs and route new blocking requests only to matching images during rollout. Old socket preambles fail on pipe-only runners; new pipe preambles fail immediately on old runners. There is no TCP or Unix-socket fallback. Replay mode is unchanged. @@ -174,3 +179,16 @@ Remaining service audit findings at this change: reassess if notifications or path filters become reachable. - `braces` (GHSA-vfj7-8cjw-p6xm): development-only dependency with no published fix. Do not feed untrusted patterns to build tooling; track an upstream fix. + +Bun 1.4.2 subprocess spawning with piped stdio also uses socketpairs and is +denied by this policy. Basic JS/TS execution remains supported. Replacing Bun +subprocess IPC requires a separate runtime change; do not re-enable socketpairs +to provide that compatibility. + +For PVC-backed Helm deployments, pin the package-init image to the same immutable +release as the runner. The chart refreshes the mutable `latest` tag with `Always`, +including upgrades that reuse an old `IfNotPresent` value, so a cached old init +image cannot skip the runtime-marker check. Publish the patched init image +before upgrading the runner; refresh does not make an unpublished image current. +An explicit `Never` policy is preserved for locally loaded kind/minikube images; +rebuild and reload the patched package-init image before a local upgrade. diff --git a/docs/adr/0011-tool-call-pipes.md b/docs/adr/0011-tool-call-pipes.md index 69e0b0ee..c4b53f96 100644 --- a/docs/adr/0011-tool-call-pipes.md +++ b/docs/adr/0011-tool-call-pipes.md @@ -12,8 +12,8 @@ Node and its TLS state must continue to start after MicroVM restore. The current stdio spawn APIs create socketpairs, not anonymous pipe endpoints. The existing guard also intentionally removes unrelated inherited descriptors. -Python asyncio and duplex multiprocessing pipes use Unix stream socketpairs; -a blanket socketpair denial would break these ordinary workloads. +Unmodified Python asyncio and duplex multiprocessing pipes use Unix stream +socketpairs. ADR 0012 replaces those runtime internals with anonymous pipes. ## Decision @@ -33,10 +33,9 @@ requests. The Python client clears descriptor inheritance and refuses calls from forked or spawned children. Spawned workers may re-import tool definitions without receiving or using the pipe capability. stdout/stderr and user stdin remain separate. -Deny all socket creation. Permit only protocol-0 Unix stream socketpairs for -runtime IPC, retaining the sendmsg/sendmmsg/io_uring denials. This preserves -asyncio and pipe-based multiprocessing using explicit spawn/fork contexts without permitting named socket access -or descriptor passing. No legacy TCP/socket fallback is provided. +Deny all socket and socketpair creation, retaining sendmsg/sendmmsg/io_uring +denials. The patched Python runtime preserves asyncio and duplex multiprocessing +using explicit spawn/fork contexts. No TCP/socket fallback is provided. ## Consequences diff --git a/docs/adr/0012-pipe-only-python-runtime.md b/docs/adr/0012-pipe-only-python-runtime.md new file mode 100644 index 00000000..ac1b844b --- /dev/null +++ b/docs/adr/0012-pipe-only-python-runtime.md @@ -0,0 +1,58 @@ +# 0012: Deny socketpairs with pipe-only Python runtime IPC + +Status: Proposed + +## Context + +ADR 0011 removed the named tool-call socket but retained Unix stream socketpairs +for Python runtime compatibility. Removing that exception reduces the kernel +surface available to untrusted code. A seccomp-only change breaks standard +asyncio initialization and duplex multiprocessing connections. + +## Decision + +Deny every socketpair in the job filter. Patch the packaged Linux Python stdlib +at build time, including the package-init and local package-builder paths: + +- Replace asyncio's self-socket with nonblocking CLOEXEC anonymous pipe endpoints. + Preserve thread wakeups, signal wakeup FDs, read callbacks and loop cleanup. +- Replace duplex multiprocessing connections with two anonymous pipes. Preserve + Connection framing, readable fileno/poll, close/EOF, and both descriptor ends + through the existing spawn reduction mechanism. +- Validate each stdlib edit before writing any edits. Unknown source layouts fail + the package build. Mark successful installation so package init cannot reuse an + unpatched package solely because its Python version matches. + +The trusted broker/relay may continue using sockets outside the jail. Untrusted +code receives only anonymous pipe capabilities. No socketpair emulation is +installed globally: direct socket APIs remain denied by seccomp. + +## Consequences + +Rebuild Python packages and deploy them with the new runner policy. Existing +unpatched packages are incompatible with the policy; drain or version-route +jobs during replacement. Snapshot/baked runners need rebuilt runtime contents. +The marker is build bookkeeping, not an authorization boundary. + +Other runtimes or third-party event loops that require socketpairs remain denied; +the patch only covers the standard Python runtime. Named listeners, forkserver, +managers and post-start descriptor sharing remain unsupported. Queues/pools +still require the jail to mount /dev/shm, independent of this change. + +Native regression tests exercise the compiled policy on both CPU architectures, +thread completion, signals, cancellation, async subprocesses, duplex spawn/fork +IPC and EOF, queues/pools, and generated concurrent tool calls. A local signed +runner canary additionally checks packaged Python and ordinary asyncio jobs. + +Bun 1.4.2 subprocess spawning with piped stdio also uses socketpairs and is +denied by this policy. Basic JS/TS execution remains supported. Replacing Bun +subprocess IPC requires a separate runtime change; do not re-enable socketpairs +to provide that compatibility. + +For PVC-backed Helm deployments, pin the package-init image to the same immutable +release as the runner. The chart refreshes the mutable `latest` tag with `Always`, +including upgrades that reuse an old `IfNotPresent` value, so a cached old init +image cannot skip the runtime-marker check. Publish the patched init image +before upgrading the runner; refresh does not make an unpublished image current. +An explicit `Never` policy is preserved for locally loaded kind/minikube images; +rebuild and reload the patched package-init image before a local upgrade. diff --git a/helm/codeapi/templates/package-init-job.yaml b/helm/codeapi/templates/package-init-job.yaml index 8d927503..baf81b17 100644 --- a/helm/codeapi/templates/package-init-job.yaml +++ b/helm/codeapi/templates/package-init-job.yaml @@ -40,7 +40,8 @@ spec: containers: - name: package-init image: "{{ .Values.workerSandbox.packages.initJob.image.repository }}:{{ .Values.workerSandbox.packages.initJob.image.tag }}" - imagePullPolicy: {{ .Values.workerSandbox.packages.initJob.image.pullPolicy }} + {{/* Refresh cached latest images; preserve explicit offline Never. */}} + imagePullPolicy: {{ if and (eq (.Values.workerSandbox.packages.initJob.image.tag | toString) "latest") (ne .Values.workerSandbox.packages.initJob.image.pullPolicy "Never") }}Always{{ else }}{{ .Values.workerSandbox.packages.initJob.image.pullPolicy }}{{ end }} env: - name: PYTHON_VERSION value: {{ .Values.workerSandbox.packages.initJob.pythonVersion | quote }} diff --git a/helm/codeapi/values.yaml b/helm/codeapi/values.yaml index 3ce4cd09..4858fd71 100644 --- a/helm/codeapi/values.yaml +++ b/helm/codeapi/values.yaml @@ -306,8 +306,9 @@ workerSandbox: forceRebuild: false image: repository: codeapi-package-init + # Pin this tag to the same immutable release as the runner in production. tag: latest - pullPolicy: IfNotPresent + pullPolicy: Always pythonVersion: "3.14.4" nodeVersion: "24.15.0" bunVersion: "1.4.2" diff --git a/tests/nsjail_unix_gc.py b/tests/nsjail_unix_gc.py index a576fcdb..08c7cf8d 100644 --- a/tests/nsjail_unix_gc.py +++ b/tests/nsjail_unix_gc.py @@ -109,15 +109,32 @@ def main(): assert libc.syscall(syscall, 0, 0, 0, 0, 0, 0) == -1 assert c.get_errno() == errno.EPERM test_sends(libc, inherited) - test_sends(libc, socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM)) + expect_eperm(lambda: socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM)) for domain in (socket.AF_UNIX, socket.AF_INET, socket.AF_INET6, socket.AF_NETLINK): expect_eperm(lambda: socket.socket(domain, socket.SOCK_STREAM)) expect_eperm(lambda: socket.socketpair(socket.AF_UNIX, socket.SOCK_DGRAM)) expect_eperm(lambda: socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM, 6)) import asyncio - asyncio.run(asyncio.sleep(0)) + from python_pipe_runtime import check_asyncio, duplex_child + asyncio.run(check_asyncio()) for method in ("fork", "spawn"): context = mp.get_context(method) + left, right = context.Pipe() + process = context.Process(target=duplex_child, args=(right,)) + process.start() + right.close() + left.send_bytes(b'x' * 100_000) + assert left.poll(10) + assert left.recv_bytes() == b'x' * 100_000 + process.join(10) + assert process.exitcode == 0 + try: + left.recv_bytes() + except EOFError: + pass + else: + raise AssertionError('duplex peer EOF missing') + left.close() queue = context.Queue() process = context.Process(target=queue_child, args=(queue,)) process.start() @@ -128,7 +145,7 @@ def main(): queue.join_thread() with context.Pool(2) as pool: assert pool.map(square, [2, 3, 4]) == [4, 9, 16] - print("PASS: descriptor sends denied; socket creation denied; stream IPC, asyncio, queues and pools work") + print("PASS: descriptor sends denied; socket creation denied; socketpairs denied; pipe-based asyncio, duplex IPC, queues and pools work") if __name__ == "__main__": diff --git a/tests/package_init_runtime_rollout.sh b/tests/package_init_runtime_rollout.sh new file mode 100755 index 00000000..3bb8cf6c --- /dev/null +++ b/tests/package_init_runtime_rollout.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TMP_DIR="$(mktemp -d)" +trap 'rm -r "$TMP_DIR"' EXIT +mkdir "$TMP_DIR/chart" +cp "$ROOT/helm/codeapi/values.yaml" "$TMP_DIR/chart/values.yaml" +cp -R "$ROOT/helm/codeapi/templates" "$TMP_DIR/chart/templates" +awk '/^dependencies:/{exit} {print}' "$ROOT/helm/codeapi/Chart.yaml" > "$TMP_DIR/chart/Chart.yaml" +render() { + helm template runtime-rollout "$TMP_DIR/chart" \ + --show-only templates/package-init-job.yaml \ + --set executionManifest.privateKey=test \ + --set executionManifest.publicKey=test \ + --set workerSandbox.enabled=true \ + --set workerSandbox.packages.source=pvc "$@" +} +# Default mutable image must refresh on every pre-install/pre-upgrade hook. +render | grep -q 'imagePullPolicy: Always' +# Reused historical values must not bypass the runtime migration. +render --set workerSandbox.packages.initJob.image.pullPolicy=IfNotPresent \ + | grep -q 'imagePullPolicy: Always' +# Immutable release images retain their explicitly configured cache policy. +render --set workerSandbox.packages.initJob.image.tag=release-123 \ + --set workerSandbox.packages.initJob.image.pullPolicy=IfNotPresent \ + | grep -q 'imagePullPolicy: IfNotPresent' +# Local kind/minikube builds are deliberately loaded without a registry. +render -f "$ROOT/helm/codeapi/values-local.yaml" | grep -q 'imagePullPolicy: Never' +render --set workerSandbox.packages.initJob.image.pullPolicy=Never \ + | grep -q 'imagePullPolicy: Never' +echo 'PASS: package-init refreshes cached latest and preserves offline images' diff --git a/tests/python_pipe_runtime.py b/tests/python_pipe_runtime.py new file mode 100644 index 00000000..52a48095 --- /dev/null +++ b/tests/python_pipe_runtime.py @@ -0,0 +1,47 @@ +"""Runtime transition checks also run under the installed seccomp filter.""" +import asyncio +import os +import signal +import stat +import sys + + +def duplex_child(connection): + try: + assert stat.S_ISFIFO(os.fstat(connection.fileno()).st_mode) + connection.send_bytes(connection.recv_bytes()) + finally: + connection.close() + + +async def check_asyncio(): + loop = asyncio.get_running_loop() + assert stat.S_ISFIFO(os.fstat(loop._ssock.fileno()).st_mode) + assert stat.S_ISFIFO(os.fstat(loop._csock.fileno()).st_mode) + assert not os.get_inheritable(loop._ssock.fileno()) + assert not os.get_inheritable(loop._csock.fileno()) + # Executor completion crosses from a worker thread into the event loop. + values = await asyncio.wait_for(asyncio.gather(*[ + asyncio.to_thread(lambda n=n: n * n) for n in range(64) + ]), 10) + assert values == [n * n for n in range(64)] + awakened = asyncio.Event() + loop.add_signal_handler(signal.SIGUSR1, awakened.set) + try: + os.kill(os.getpid(), signal.SIGUSR1) + await asyncio.wait_for(awakened.wait(), 2) + finally: + loop.remove_signal_handler(signal.SIGUSR1) + process = await asyncio.create_subprocess_exec( + sys.executable, '-c', 'print("child works")', + stdout=asyncio.subprocess.PIPE) + output, _ = await asyncio.wait_for(process.communicate(), 10) + assert output == b'child works\n' and process.returncode == 0 + pending = asyncio.create_task(asyncio.sleep(30)) + pending.cancel() + try: + await pending + except asyncio.CancelledError: + pass + else: + raise AssertionError('cancellation lost') diff --git a/tests/tool_call_runner.cjs b/tests/tool_call_runner.cjs index d40e0721..4aab25e4 100644 --- a/tests/tool_call_runner.cjs +++ b/tests/tool_call_runner.cjs @@ -73,6 +73,9 @@ if __name__ == '__main__': try: socket.socket(domain, socket.SOCK_STREAM) except OSError as error: assert error.errno == errno.EPERM else: raise AssertionError('socket creation allowed') + try: socket.socketpair() + except OSError as error: assert error.errno == errno.EPERM + else: raise AssertionError('socketpair allowed') assert subprocess.check_output([sys.executable, '-c', "import os; print(os.path.exists('/proc/self/fd/3'))"]).strip() == b'False' async def check(): results = await asyncio.gather(*[_execute_tool_internal_async('echo', {'n': n}) for n in range(8)]) @@ -82,7 +85,8 @@ if __name__ == '__main__': # Test the supported pipe-only spawn/fork multiprocessing paths. for method in ('spawn', 'fork'): ctx = mp.get_context(method) - receiver, sender = ctx.Pipe(duplex=False) + receiver, sender = ctx.Pipe() + assert _stat.S_ISFIFO(os.fstat(receiver.fileno()).st_mode) child = ctx.Process(target=_mp_echo, args=(sender,)) child.start() sender.close() @@ -111,10 +115,10 @@ if __name__ == '__main__': assert(result.run.stdout.includes('PASS:'), JSON.stringify(result)); console.log(result.run.stdout.trim()); // Ordinary jobs still work and do not receive the tool-call descriptors. - const plain = await execute("import os; assert not os.path.exists('/proc/self/fd/3'); print('PASS: ordinary execution without pipe capability')", false); + const plain = await execute("import os, asyncio; assert not os.path.exists('/proc/self/fd/3'); asyncio.run(asyncio.to_thread(lambda: 1)); print('PASS: ordinary asyncio execution without pipe capability')", false); console.log(plain.run.stdout.trim()); for (const language of ['bun-js', 'bun-ts']) { - const js = await execute("console.log('PASS: Bun ' + Bun.version)", false, language, '1.4.2'); + const js = await execute("let denied = false; try { Bun.spawn([process.execPath, '-e', '0'], {stdout:'pipe', stderr:'pipe'}); } catch (error) { if (error.code !== 'EPERM' || error.syscall !== 'socketpair') throw error; denied = true; } if (!denied) throw new Error('socketpair spawn allowed'); console.log('PASS: Bun ' + Bun.version + ', socketpair-based spawn denied')", false, language, '1.4.2'); assert(js.run.stdout.includes('PASS: Bun 1.4.2'), JSON.stringify(js)); console.log(language + ': ' + js.run.stdout.trim()); } From 26d1ab8db0f040d714b9f7abbfa902f59474e107 Mon Sep 17 00:00:00 2001 From: Marco Beretta <81851188+berry-13@users.noreply.github.com> Date: Mon, 5 Oct 2026 23:47:30 +0200 Subject: [PATCH 37/48] =?UTF-8?q?=F0=9F=8C=BF=20feat:=20Report=20Lane=20Gi?= =?UTF-8?q?t=20Branch=20And=20Head=20On=20Command=20Results=20(#311)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Report Lane Git Branch And Head On Command Results The worker reads the checked-out branch and HEAD of the lane a command ran in and attaches them to execute_command results as laneGit. Code API validates the value and forwards it only when CODEAPI_BRIDGE_LANE_GIT is enabled and the worker advertised lane_git. * fix: Address Review Findings on Lane Git Reporting Compare a retried settlement with the stored one before replica-local lane Git sanitization, and skip the Git probe until lane_git is negotiated at registration. * fix: Bound Lane Git Probe and Pass Flag Through Compose Cap the post-command Git probe to the budget the command left, reserving settlement time, and pass CODEAPI_BRIDGE_LANE_GIT to the api and service-worker compose services. * fix: Harden Lane Git Probe Bounds and Reported Values Bound root resolution with the probe limit, enable the probe from the worker's active registration, require HEAD to peel to a commit, and reject Unicode control and format characters in branch names. * fix: Trust Only Probed Lane Git and Use the Worker Deadline Assign the lane Git worker on the normal startup path, budget the probe from the worker's assignment deadline, and drop any laneGit supplied by the command executor. * fix: Reuse Settlement Reads and Preserve Branch Whitespace Fold lane Git sanitization into the settle path's existing reads instead of a separate pre-pass, and strip only Git's trailing newline from probe output. * fix: Report Null for Non-UTF-8 Branch Names Read the lane branch as bytes and decode it strictly, so a ref name that is not valid UTF-8 is reported as null instead of a lossy copy. * fix: Revalidate Source Git Metadata Before Probing Probe a source or conversation checkout only while its .git is its own plain directory, so a command that rewrites .git to point at another repository cannot make the host-side probe report that repository's branch and head. * fix: Reject Linked Refs Before Probing Source Git Walk refs/ and require every entry to be a plain file or directory before the host-side probe runs, so a ref symlinked to another checkout cannot be followed. * fix: Guard Linked Lane Refs and Report Full Branch Names Check a linked lane's metadata HEAD and the shared refs before probing, read the branch from the full symbolic ref instead of --short, and avoid Dirent.parentPath so the refs walk works on Node 20.11. * fix: Stop the Git Refs Walk When the Probe Aborts Pass the probe signal into the source and linked worktree metadata checks and check it between directory operations, so an abandoned walk does not keep running after the command result is returned. * fix: Report Lane Git Only From a Stable Branch Read Read the branch before and after the head and omit laneGit when it changed in between, so a concurrent checkout cannot pair one branch with another branch's commit. * fix: Own Object Storage and Pin the Head Before Probing Require a link-free refs tree and object store with no alternates or reftable before the host-side probe, ignore replacement refs, and read the head from the branch ref the first read named. * fix: Disable Transports and Lazy Fetch in the Lane Git Probe Run the probe with no allowed protocols and lazy fetch disabled, so a rewritten .git/config cannot make a missing object trigger a fetch or a transport command on the host. --- docker-compose.yaml | 2 + packages/code/src/cli.ts | 26 +- packages/code/src/environment.ts | 4 +- packages/code/src/lane-git.test.ts | 830 +++++++++++++++++++++++ packages/code/src/lane-git.ts | 297 ++++++++ packages/code/src/projects.ts | 14 +- packages/code/src/protocol.ts | 61 ++ packages/code/src/worker.ts | 28 + packages/code/src/workspace.ts | 2 + packages/code/src/worktree-retirement.ts | 19 +- service/src/bridge/index.ts | 1 + service/src/bridge/lane-git.test.ts | 251 +++++++ service/src/bridge/lane-git.ts | 36 + service/src/bridge/router.test.ts | 49 ++ service/src/bridge/router.ts | 3 + service/src/bridge/store.ts | 64 +- service/src/config.ts | 5 + tests/compose-bridge-config.cjs | 3 + 18 files changed, 1681 insertions(+), 14 deletions(-) create mode 100644 packages/code/src/lane-git.test.ts create mode 100644 packages/code/src/lane-git.ts create mode 100644 service/src/bridge/lane-git.test.ts create mode 100644 service/src/bridge/lane-git.ts diff --git a/docker-compose.yaml b/docker-compose.yaml index 1d7d28cf..db517b4c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -14,6 +14,7 @@ services: - CODEAPI_BRIDGE_AUTH_MODE=${CODEAPI_BRIDGE_AUTH_MODE:-paired} - CODEAPI_BRIDGE_DYNAMIC_WORKERS=${CODEAPI_BRIDGE_DYNAMIC_WORKERS:-true} - CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS=${CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS:-1} + - CODEAPI_BRIDGE_LANE_GIT=${CODEAPI_BRIDGE_LANE_GIT:-false} - CODEAPI_BRIDGE_WORKER_ID=${CODEAPI_BRIDGE_WORKER_ID:-} - CODEAPI_BRIDGE_RECOVERY_SERVER_ID=${CODEAPI_BRIDGE_RECOVERY_SERVER_ID:-} - CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS=${CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS:-0} @@ -69,6 +70,7 @@ services: - CODEAPI_BRIDGE_AUTH_MODE=${CODEAPI_BRIDGE_AUTH_MODE:-paired} - CODEAPI_BRIDGE_DYNAMIC_WORKERS=${CODEAPI_BRIDGE_DYNAMIC_WORKERS:-true} - CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS=${CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS:-1} + - CODEAPI_BRIDGE_LANE_GIT=${CODEAPI_BRIDGE_LANE_GIT:-false} - CODEAPI_BRIDGE_WORKER_ID=${CODEAPI_BRIDGE_WORKER_ID:-} - CODEAPI_BRIDGE_RECOVERY_SERVER_ID=${CODEAPI_BRIDGE_RECOVERY_SERVER_ID:-} - CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS=${CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS:-0} diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 6b450b64..85ce947e 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -41,7 +41,8 @@ import { RuntimeWorkspaceCommandSandbox } from './workspace-runtime.js'; import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; import { NativeWorkspaceCommandPool } from './native-pool.js'; import { GitWorktreeWorkspaceTools, internalWorkspaceId } from './workspace-instances.js'; -import { LINKED_WORKTREE_DIRECTORY, LinkedWorktreeWorkspaceTools } from './linked-worktrees.js'; +import { LaneGitWorkspaceTools, ownsGitMetadata, ownsLinkedWorktreeMetadata } from './lane-git.js'; +import { LINKED_WORKTREE_DIRECTORY, LinkedWorktreeWorkspaceTools, verifyLinkedWorktree } from './linked-worktrees.js'; import { GitWorktreeManager } from './worktrees.js'; import { WorktreeRetirementScheduler, @@ -1312,6 +1313,28 @@ async function run( }); workspaceTools = linkedWorktreeTools; } + // Probe Git only while the active registration advertises lane_git, which the worker decides. + let laneGitWorker: BridgeWorker | undefined; + if (workspaceTools?.capabilities.operations.includes('execute_command')) { + workspaceTools = new LaneGitWorkspaceTools({ + delegate: workspaceTools, + isEnabled: () => laneGitWorker?.commandResultFeatureActive('lane_git') === true, + async resolveRoot(request, signal) { + const source = roots.find((root) => root.id === request.workspaceId); + if (!source) return undefined; + if (request.workspaceInstanceId != null) { + const root = await conversationWorktrees?.plannedRoot(request.workspaceId, request.workspaceInstanceId); + return root != null && (await ownsGitMetadata(root, signal)) ? root : undefined; + } + if (request.worktree != null) { + signal?.throwIfAborted(); + const lane = await verifyLinkedWorktree(source.root, request.worktree, source.identity); + return (await ownsLinkedWorktreeMetadata(lane.commonGitDir, request.worktree, signal)) ? lane.root : undefined; + } + return (await ownsGitMetadata(source.root, signal)) ? source.root : undefined; + }, + }); + } const retirementSources = roots.filter((root) => root.writable); const debugLogs = process.env.LIBRECHAT_CODE_LOG_LEVEL?.trim().toLowerCase() === 'debug'; @@ -1633,6 +1656,7 @@ async function run( ); }, }); + laneGitWorker = worker; if (runtimeSessionId !== undefined) { await worker.refreshCredential(controller.signal); await worker.register(controller.signal); diff --git a/packages/code/src/environment.ts b/packages/code/src/environment.ts index 576a149f..c527729f 100644 --- a/packages/code/src/environment.ts +++ b/packages/code/src/environment.ts @@ -237,12 +237,13 @@ export class EnvironmentWorkspaceTools implements WorkspaceToolExecutor { async execute( request: WorkspaceToolRequest, signal?: AbortSignal, + context?: { deadlineAtMs?: number }, ): Promise { if ( request.operation !== 'execute_command' || !request.environmentAction ) { - return this.delegate.execute(request, signal); + return this.delegate.execute(request, signal, context); } const environment = this.environments.get(request.workspaceId); const action = environment?.definition.actions?.find( @@ -271,6 +272,7 @@ export class EnvironmentWorkspaceTools implements WorkspaceToolExecutor { cwd: '.', }, signal, + context, ); } } diff --git a/packages/code/src/lane-git.test.ts b/packages/code/src/lane-git.test.ts new file mode 100644 index 00000000..27b400b5 --- /dev/null +++ b/packages/code/src/lane-git.test.ts @@ -0,0 +1,830 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { mkdir, mkdtemp, readFile, realpath, rename, rm, symlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { promisify } from 'node:util'; + +import { LaneGitWorkspaceTools, ownsGitMetadata, ownsLinkedWorktreeMetadata, readLaneGit, snapshotLaneGit } from './lane-git.js'; +import { isValidBridgeWorkspaceToolCapabilities, isWorkspaceLaneGit, isWorkspaceToolResult } from './protocol.js'; +import { BridgeWorker } from './worker.js'; + +import type { TestContext } from 'node:test'; +import type { WorkspaceExecuteCommandRequest, WorkspaceToolRequest } from './protocol.js'; + +const exec = promisify(execFile); +const incarnationId = 'incarnation-00000001'; +const identity = ['-c', 'user.name=Test', '-c', 'user.email=test@example.com']; + +async function scratch(t: TestContext): Promise { + const root = await realpath(await mkdtemp(join(tmpdir(), 'lane-git-'))); + t.after(() => rm(root, { recursive: true, force: true })); + return root; +} + +async function repo(root: string, name: string, commit = true): Promise { + const directory = join(root, name); + await mkdir(directory, { recursive: true }); + await exec('git', ['init', '--initial-branch=main', directory]); + if (commit) await exec('git', ['-C', directory, ...identity, 'commit', '--allow-empty', '-m', 'initial']); + return directory; +} + +async function sha(directory: string): Promise { + return (await exec('git', ['-C', directory, 'rev-parse', 'HEAD'])).stdout.trim(); +} + +test('isolated worktree lane reports its own branch and head', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const lane = join(root, 'lane'); + await exec('git', ['-C', source, 'worktree', 'add', '-b', 'librechat/conversation-abcd1234-0123', lane]); + assert.deepEqual(await readLaneGit(lane), { branch: 'librechat/conversation-abcd1234-0123', head: await sha(lane) }); + assert.equal((await readLaneGit(source))?.branch, 'main'); +}); + +test('source checkout reports whatever is checked out', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, 'checkout', '-b', 'feature/pr-header']); + assert.deepEqual(await readLaneGit(source), { branch: 'feature/pr-header', head: await sha(source) }); +}); + +test('a branch renamed by the agent shows up on the next read', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + assert.equal((await readLaneGit(source))?.branch, 'main'); + await exec('git', ['-C', source, 'branch', '-m', 'renamed-by-agent']); + assert.equal((await readLaneGit(source))?.branch, 'renamed-by-agent'); +}); + +test('detached HEAD reports a null branch and the commit', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, 'checkout', '--detach']); + assert.deepEqual(await readLaneGit(source), { branch: null, head: await sha(source) }); +}); + +test('a repository without commits reports the branch and a null head', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source', false); + assert.deepEqual(await readLaneGit(source), { branch: 'main', head: null }); +}); + +test('a branch name beyond the bound reports null and never forwards it', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const long = ['a'.repeat(100), 'b'.repeat(100), 'c'.repeat(100)].join('/'); + await exec('git', ['-C', source, 'checkout', '-b', long]); + assert.deepEqual(await readLaneGit(source), { branch: null, head: await sha(source) }); +}); + +test('a directory that is not a repository reports nothing', async (t) => { + const root = await scratch(t); + assert.equal(await readLaneGit(root), undefined); +}); + +test('lane Git state never carries paths, remotes or credentials', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, 'remote', 'add', 'origin', 'https://user:secret@github.com/example/app.git']); + const state = await readLaneGit(source); + assert.deepEqual(Object.keys(state ?? {}).sort(), ['branch', 'head']); + const wire = JSON.stringify(state); + assert.equal(wire.includes(root), false); + assert.equal(wire.includes('secret'), false); + assert.equal(wire.includes('github.com'), false); +}); + +test('wire validator accepts only bounded branch and head', () => { + const head = 'a'.repeat(40); + assert.equal(isWorkspaceLaneGit({ branch: 'main', head }), true); + assert.equal(isWorkspaceLaneGit({ branch: null, head: null }), true); + assert.equal(isWorkspaceLaneGit({ branch: 'main', head: 'b'.repeat(64) }), true); + for (const bad of [ + { branch: 'x'.repeat(257), head }, + { branch: 'bad\nname', head }, + { branch: '', head }, + { branch: 'main', head: 'A'.repeat(40) }, + { branch: 'main', head: 'a'.repeat(41) }, + { branch: 'main' }, + { branch: 'main', head, path: '/srv/repo' }, + { branch: undefined, head }, + null, + [], + ]) { + assert.equal(isWorkspaceLaneGit(bad), false, JSON.stringify(bad)); + } +}); + +const commandRequest: WorkspaceExecuteCommandRequest = { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command: 'git checkout -b agent-branch', +}; +const commandResult = { + protocolVersion: 1 as const, + operation: 'execute_command' as const, + workspaceId: 'primary', + exitCode: 0, + stdout: '', + stderr: '', + truncated: false, + timedOut: false, +}; + +function delegate(onRun?: () => Promise) { + return { + capabilities: { + protocolVersion: 1 as const, + operations: ['read_file' as const, 'execute_command' as const], + workspaces: [{ id: 'primary' }], + }, + async execute(request: WorkspaceToolRequest) { + await onRun?.(); + return request.operation === 'execute_command' + ? commandResult + : { protocolVersion: 1 as const, operation: 'read_file' as const, workspaceId: 'primary', path: 'a', content: '', startLine: 1, endLine: 1, truncated: false }; + }, + }; +} + +test('a finished command refreshes the lane state, including a branch the command created', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const tools = new LaneGitWorkspaceTools({ + delegate: delegate(async () => { + await exec('git', ['-C', source, 'checkout', '-b', 'agent-branch']); + }), + resolveRoot: async () => source, + }); + const result = await tools.execute(commandRequest); + assert.deepEqual('laneGit' in result && result.laneGit, { branch: 'agent-branch', head: await sha(source) }); + assert.equal(isWorkspaceToolResult(commandRequest, result), true); +}); + +test('file operations do not read Git', async () => { + let resolved = 0; + const tools = new LaneGitWorkspaceTools({ + delegate: delegate(), + resolveRoot: async () => { + resolved++; + return undefined; + }, + }); + const result = await tools.execute({ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'a' }); + assert.equal('laneGit' in result, false); + assert.equal(resolved, 0); +}); + +test('an unreadable lane leaves the command result untouched', async (t) => { + const root = await scratch(t); + for (const resolveRoot of [async () => root, async () => undefined, async () => Promise.reject(new Error('gone'))]) { + const result = await new LaneGitWorkspaceTools({ delegate: delegate(), resolveRoot }).execute(commandRequest); + assert.deepEqual(result, commandResult); + } +}); + +test('the feature is advertised only when commands are', () => { + const withCommands = new LaneGitWorkspaceTools({ delegate: delegate(), resolveRoot: async () => undefined }); + assert.deepEqual(withCommands.capabilities.commandResultFeatures, ['lane_git']); + assert.equal(isValidBridgeWorkspaceToolCapabilities(withCommands.capabilities), true); + const readOnly = new LaneGitWorkspaceTools({ + delegate: { ...delegate(), capabilities: { protocolVersion: 1, operations: ['read_file'], workspaces: [{ id: 'primary' }] } }, + resolveRoot: async () => undefined, + }); + assert.equal(readOnly.capabilities.commandResultFeatures, undefined); + assert.equal( + isValidBridgeWorkspaceToolCapabilities({ ...withCommands.capabilities, commandResultFeatures: ['other'] }), + false, + ); +}); + +test('the probe is skipped until lane_git is negotiated, then runs', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + let negotiated = false; + let resolved = 0; + const tools = new LaneGitWorkspaceTools({ + delegate: delegate(), + isEnabled: () => negotiated, + resolveRoot: async () => { + resolved++; + return source; + }, + }); + const before = await tools.execute(commandRequest); + assert.equal('laneGit' in before, false); + assert.equal(resolved, 0); + negotiated = true; + const after = await tools.execute(commandRequest); + assert.equal(resolved, 1); + assert.deepEqual('laneGit' in after && after.laneGit, { branch: 'main', head: await sha(source) }); +}); + +test('the probe is skipped when the command left too little of its settlement budget', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const timed = { ...commandRequest, timeoutMs: 1_000 }; + // Budget is timeoutMs plus a 5 s grace (6 s here), minus a 1 s settlement reserve; the clock jumps while the command runs. + for (const [elapsedMs, probed] of [ + [100, true], + [4_000, true], + [4_800, false], + [7_000, false], + ] as const) { + let clock = 0; + let resolved = 0; + const tools = new LaneGitWorkspaceTools({ + delegate: delegate(async () => { + clock += elapsedMs; + }), + now: () => clock, + resolveRoot: async () => { + resolved++; + return source; + }, + }); + const result = await tools.execute(timed); + assert.equal(resolved, probed ? 1 : 0, `elapsed ${elapsedMs}`); + assert.equal('laneGit' in result, probed, `elapsed ${elapsedMs}`); + } +}); + +test('the probe budget follows the worker deadline, not a rebuilt timeout', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + // A full 31 s budget by timeout alone, but the worker has only 1.5 s left before it aborts. + for (const [remainingMs, probed] of [ + [10_000, true], + [1_500, true], + [1_200, false], + [500, false], + ] as const) { + let resolved = 0; + const tools = new LaneGitWorkspaceTools({ + delegate: delegate(), + now: () => 1_000, + resolveRoot: async () => { + resolved++; + return source; + }, + }); + await tools.execute(commandRequest, undefined, { deadlineAtMs: 1_000 + remainingMs }); + assert.equal(resolved, probed ? 1 : 0, `remaining ${remainingMs}`); + } +}); + +test('a slow probe is cut short and never fails the command', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const tools = new LaneGitWorkspaceTools({ + delegate: delegate(), + probeTimeoutMs: 1, + resolveRoot: async (_request, signal) => { + await new Promise((resolve) => setTimeout(resolve, 50)); + signal?.throwIfAborted(); + return source; + }, + }); + assert.deepEqual(await tools.execute(commandRequest), commandResult); +}); + +test('a HEAD that names a blob is reported as a null head, not as a commit', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const id = (await exec('bash', ['-c', `echo hello | git -C '${source}' hash-object -w --stdin`])).stdout.trim(); + await writeFile(join(source, '.git', 'HEAD'), `${id}\n`); + assert.deepEqual(await readLaneGit(source), { branch: null, head: null }); +}); + +test('branch names with Unicode control or format characters are not reported', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, 'checkout', '-b', 'feature/\u202eabc']); + assert.deepEqual(await readLaneGit(source), { branch: null, head: await sha(source) }); + for (const name of ['a\u0085b', 'a\u202eb', 'a\u200bb', 'a\u2028b']) { + assert.equal(isWorkspaceLaneGit({ branch: name, head: null }), false, JSON.stringify(name)); + } + assert.equal(isWorkspaceLaneGit({ branch: 'feature/caf\u00e9-\u65e5\u672c', head: null }), true); +}); + +test('a resolver that ignores its signal cannot hold the command result past the probe limit', async () => { + const tools = new LaneGitWorkspaceTools({ + delegate: delegate(), + probeTimeoutMs: 50, + resolveRoot: () => new Promise(() => {}), + }); + const started = Date.now(); + const result = await Promise.race([ + tools.execute(commandRequest), + new Promise<'hung'>((resolve) => setTimeout(() => resolve('hung'), 2_000)), + ]); + assert.notEqual(result, 'hung'); + assert.deepEqual(result, commandResult); + assert.ok(Date.now() - started < 1_000); +}); + +test('the worker reports lane_git active only when the registration that stuck advertised it', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + for (const secondFails of [false, true]) { + let registrations = 0; + const tools = new LaneGitWorkspaceTools({ delegate: delegate(), resolveRoot: async () => source }); + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: false, sandboxProfile: 'anthropic-srt', runtimes: [], workspaceTools: tools.capabilities }, + workspaceTools: tools, + workspaceMutationQuarantine: quarantine(), + fetchImpl: async () => { + registrations++; + if (secondFails && registrations === 2) return new Response('{}', { status: 500 }); + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: ['read_file', 'execute_command'], + supportedWorkspaceCommandResultFeatures: ['lane_git'], + }); + }, + }); + await worker.register(); + assert.equal(worker.commandResultFeatureActive('lane_git'), !secondFails, `secondFails=${secondFails}`); + } +}); + +test('a branch name with leading or trailing Unicode whitespace is reported as it is', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, 'checkout', '-b', '\u00a0main\u00a0']); + assert.deepEqual(await readLaneGit(source), { branch: '\u00a0main\u00a0', head: await sha(source) }); +}); + +test('a branch name that is not valid UTF-8 is reported as null, not as a lossy copy', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await writeFile(join(source, '.git', 'HEAD'), Buffer.concat([Buffer.from('ref: refs/heads/bad'), Buffer.from([0xff, 0x0a])])); + const state = await readLaneGit(source); + assert.equal(state?.branch, null); + assert.equal(JSON.stringify(state).includes('\ufffd'), false); +}); + +test('a valid non-ASCII branch name survives the byte level read', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, 'checkout', '-b', 'feature/caf\u00e9-\u65e5\u672c']); + assert.equal((await readLaneGit(source))?.branch, 'feature/caf\u00e9-\u65e5\u672c'); +}); + +test('an own Git directory is accepted, and a commit made there keeps it accepted', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + assert.equal(await ownsGitMetadata(source), true); + await exec('git', ['-C', source, ...identity, 'commit', '--allow-empty', '-m', 'second']); + await exec('git', ['-C', source, 'pack-refs', '--all']); + assert.equal(await ownsGitMetadata(source), true); +}); + +test('a .git rewritten to point at another repository is not probed', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const victim = await repo(root, 'victim'); + await exec('git', ['-C', victim, 'checkout', '-b', 'victim-secret-branch']); + await rm(join(source, '.git'), { recursive: true }); + await writeFile(join(source, '.git'), `gitdir: ${join(victim, '.git')}\n`); + assert.equal(await ownsGitMetadata(source), false); + const tools = new LaneGitWorkspaceTools({ + delegate: delegate(), + resolveRoot: async () => ((await ownsGitMetadata(source)) ? source : undefined), + }); + const result = await tools.execute(commandRequest); + assert.equal('laneGit' in result, false); + assert.equal(JSON.stringify(result).includes('victim-secret-branch'), false); +}); + +test('Git storage redirected by a symlink, commondir or a swapped directory is not probed', async (t) => { + const root = await scratch(t); + const victim = await repo(root, 'victim'); + const cases: Array<[string, (source: string) => Promise]> = [ + ['.git symlink', async (s) => { await rm(join(s, '.git'), { recursive: true }); await symlink(join(victim, '.git'), join(s, '.git')); }], + ['commondir', async (s) => { await writeFile(join(s, '.git', 'commondir'), `${join(victim, '.git')}\n`); }], + ['HEAD symlink', async (s) => { await rm(join(s, '.git', 'HEAD')); await symlink(join(victim, '.git', 'HEAD'), join(s, '.git', 'HEAD')); }], + ['refs symlink', async (s) => { await rm(join(s, '.git', 'refs'), { recursive: true }); await symlink(join(victim, '.git', 'refs'), join(s, '.git', 'refs')); }], + ['packed-refs symlink', async (s) => { await symlink(join(victim, '.git', 'HEAD'), join(s, '.git', 'packed-refs')); }], + ['no .git', async (s) => { await rename(join(s, '.git'), join(root, `moved-${Math.random()}`)); }], + ]; + for (const [name, tamper] of cases) { + const source = await repo(root, `source-${name.replace(/\W/g, '')}`); + await tamper(source); + assert.equal(await ownsGitMetadata(source), false, name); + } +}); + +test('a nested ref symlinked to another checkout is not probed', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const firstCommit = await sha(source); + await exec('git', ['-C', source, ...identity, 'commit', '--allow-empty', '-m', 'second']); + // A clone shares history, so the commit its branch sits on also exists in the source's objects. + const other = join(root, 'other'); + await exec('git', ['clone', '-q', source, other]); + await exec('git', ['-C', other, 'checkout', '-q', '-b', 'other-wip', firstCommit]); + await symlink(join(other, '.git', 'refs', 'heads', 'other-wip'), join(source, '.git', 'refs', 'heads', 'leak')); + await writeFile(join(source, '.git', 'HEAD'), 'ref: refs/heads/leak\n'); + assert.equal(await ownsGitMetadata(source), false); + const tools = new LaneGitWorkspaceTools({ + delegate: delegate(), + resolveRoot: async () => ((await ownsGitMetadata(source)) ? source : undefined), + }); + const result = await tools.execute(commandRequest); + assert.equal('laneGit' in result, false); +}); + +test('a nested branch ref that is a plain file inside .git is still probed', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, 'checkout', '-b', 'feature/deep/name']); + assert.equal(await ownsGitMetadata(source), true); + assert.equal((await readLaneGit(source))?.branch, 'feature/deep/name'); +}); + +test('a branch that shares its short name with a tag is reported by its real name', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, 'checkout', '-b', 'foo']); + await exec('git', ['-C', source, 'tag', 'foo']); + assert.equal((await readLaneGit(source))?.branch, 'foo'); +}); + +test('a HEAD that points outside refs/heads reports a null branch', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, 'update-ref', 'refs/remotes/origin/main', 'HEAD']); + await exec('git', ['-C', source, 'symbolic-ref', 'HEAD', 'refs/remotes/origin/main']); + assert.equal((await readLaneGit(source))?.branch, null); +}); + +test('a linked lane whose HEAD names a ref symlinked at another checkout is not probed', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const firstCommit = await sha(source); + await exec('git', ['-C', source, ...identity, 'commit', '--allow-empty', '-m', 'second']); + const other = join(root, 'other'); + await exec('git', ['clone', '-q', source, other]); + await exec('git', ['-C', other, 'checkout', '-q', '-b', 'other-wip', firstCommit]); + const lane = join(root, 'lane'); + await exec('git', ['-C', source, 'worktree', 'add', '-b', 'lane-branch', lane]); + const common = join(source, '.git'); + assert.equal(await ownsLinkedWorktreeMetadata(common, 'lane'), true); + await symlink(join(other, '.git', 'refs', 'heads', 'other-wip'), join(common, 'refs', 'heads', 'leak')); + await writeFile(join(common, 'worktrees', 'lane', 'HEAD'), 'ref: refs/heads/leak\n'); + assert.equal(await ownsLinkedWorktreeMetadata(common, 'lane'), false); +}); + +test('a linked lane with a symlinked metadata HEAD is not probed', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const lane = join(root, 'lane'); + await exec('git', ['-C', source, 'worktree', 'add', '-b', 'lane-branch', lane]); + const head = join(source, '.git', 'worktrees', 'lane', 'HEAD'); + await rm(head); + await symlink(join(source, '.git', 'HEAD'), head); + assert.equal(await ownsLinkedWorktreeMetadata(join(source, '.git'), 'lane'), false); +}); + +test('the refs walk stops as soon as the probe signal aborts', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + for (let i = 0; i < 30; i++) await mkdir(join(source, '.git', 'refs', 'heads', `dir${i}`), { recursive: true }); + let checks = 0; + const aborting = { + aborted: false, + throwIfAborted() { + if (++checks >= 3) throw new Error('aborted'); + }, + } as unknown as AbortSignal; + assert.equal(await ownsGitMetadata(source, aborting), false); + assert.equal(checks, 3); + const linked = join(root, 'lane'); + await exec('git', ['-C', source, 'worktree', 'add', '-b', 'lane-branch', linked]); + checks = 0; + assert.equal(await ownsLinkedWorktreeMetadata(join(source, '.git'), 'lane', aborting), false); + assert.equal(checks, 3); +}); + +test('the head is read from the branch ref the first read named, and only a stable branch is reported', async () => { + const head = 'a'.repeat(40); + const sequence = (values: Array) => { + let index = 0; + return async () => values[index++]; + }; + const refsRead: string[] = []; + const commit = (answer: string | null | undefined) => async (ref: string) => { + refsRead.push(ref); + return answer; + }; + // Stable branch: head comes from that branch's own ref, never from HEAD. + assert.deepEqual(await snapshotLaneGit(sequence(['refs/heads/a', 'refs/heads/a']), commit(head)), { branch: 'a', head }); + assert.deepEqual(refsRead, ['refs/heads/a']); + // A checkout away during the read is rejected. + assert.equal(await snapshotLaneGit(sequence(['refs/heads/a', 'refs/heads/b']), commit(head)), undefined); + assert.equal(await snapshotLaneGit(sequence(['refs/heads/a', null]), commit(head)), undefined); + assert.equal(await snapshotLaneGit(sequence([null, 'refs/heads/a']), commit(head)), undefined); + // Detached: HEAD is read twice and must agree, so a move between the reads is rejected. + refsRead.length = 0; + let calls = 0; + const moving = async (ref: string) => { + refsRead.push(ref); + return calls++ === 0 ? head : 'b'.repeat(40); + }; + assert.equal(await snapshotLaneGit(sequence([null, null]), moving), undefined); + assert.deepEqual(refsRead, ['HEAD', 'HEAD']); + assert.deepEqual(await snapshotLaneGit(sequence([null, null]), commit(head)), { branch: null, head }); + // HEAD on a non-branch ref reports no branch but still a head. + assert.deepEqual(await snapshotLaneGit(sequence(['refs/remotes/o/m', 'refs/remotes/o/m']), commit(head)), { branch: null, head }); + // Unreadable values omit the field. + assert.equal(await snapshotLaneGit(sequence(['refs/heads/a', 'refs/heads/a']), commit(undefined)), undefined); +}); + +test('a replace ref cannot turn a blob into a reported commit', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const commit = await sha(source); + const blob = (await exec('bash', ['-c', `echo hello | git -C '${source}' hash-object -w --stdin`])).stdout.trim(); + await mkdir(join(source, '.git', 'refs', 'replace'), { recursive: true }); + await writeFile(join(source, '.git', 'refs', 'replace', blob), `${commit}\n`); + await writeFile(join(source, '.git', 'refs', 'heads', 'main'), `${blob}\n`); + assert.deepEqual(await readLaneGit(source), { branch: 'main', head: null }); +}); + +test('redirected object storage is not probed, for source and linked checks alike', async (t) => { + const root = await scratch(t); + const victim = await repo(root, 'victim'); + const cases: Array<[string, (source: string) => Promise]> = [ + ['alternates', async (s) => { + await mkdir(join(s, '.git', 'objects', 'info'), { recursive: true }); + await writeFile(join(s, '.git', 'objects', 'info', 'alternates'), `${join(victim, '.git', 'objects')}\n`); + }], + ['http-alternates', async (s) => { + await mkdir(join(s, '.git', 'objects', 'info'), { recursive: true }); + await writeFile(join(s, '.git', 'objects', 'info', 'http-alternates'), 'https://example.invalid/\n'); + }], + ['objects symlink', async (s) => { + await rename(join(s, '.git', 'objects'), join(root, `moved-${Math.random()}`)); + await symlink(join(victim, '.git', 'objects'), join(s, '.git', 'objects')); + }], + ['fanout symlink', async (s) => { + await symlink(join(victim, '.git', 'objects', 'info'), join(s, '.git', 'objects', 'ab')); + }], + ['pack entry symlink', async (s) => { + await mkdir(join(s, '.git', 'objects', 'pack'), { recursive: true }); + await symlink(join(victim, '.git', 'HEAD'), join(s, '.git', 'objects', 'pack', 'pack-x.pack')); + }], + ['reftable redirect', async (s) => { + await mkdir(join(s, '.git', 'reftable'), { recursive: true }); + }], + ]; + for (const [name, tamper] of cases) { + const source = await repo(root, `src-${name.replace(/\W/g, '')}`); + await tamper(source); + assert.equal(await ownsGitMetadata(source), false, `source ${name}`); + } + const common = join(root, 'common'); + await exec('git', ['init', '-q', '--initial-branch=main', common]); + await exec('git', ['-C', common, ...identity, 'commit', '--allow-empty', '-m', 'i']); + await exec('git', ['-C', common, 'worktree', 'add', '-b', 'lane', join(root, 'lane')]); + assert.equal(await ownsLinkedWorktreeMetadata(join(common, '.git'), 'lane'), true); + await mkdir(join(common, '.git', 'objects', 'info'), { recursive: true }); + await writeFile(join(common, '.git', 'objects', 'info', 'alternates'), `${join(victim, '.git', 'objects')}\n`); + assert.equal(await ownsLinkedWorktreeMetadata(join(common, '.git'), 'lane'), false); +}); + +test('a repository with packed objects and refs is still probed', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, ...identity, 'commit', '--allow-empty', '-m', 'second']); + await exec('git', ['-C', source, 'gc', '-q']); + assert.equal(await ownsGitMetadata(source), true); + assert.equal((await readLaneGit(source))?.head, await sha(source)); +}); + +test('a promisor remote cannot make the probe fetch or run a command on the host', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const victim = await repo(root, 'victim'); + await exec('git', ['-C', victim, ...identity, 'commit', '--allow-empty', '-m', 'secret']); + const secret = await sha(victim); + const marker = join(root, 'ran-on-host'); + const config = [ + '[remote "origin"]', + `\turl = ${victim}`, + '\tpromisor = true', + '\tpartialclonefilter = blob:none', + `\tuploadpack = touch ${marker}; git-upload-pack`, + '[extensions]', + '\tpartialclone = origin', + '[core]', + '\trepositoryformatversion = 1', + '[protocol]', + '\tallow = always', + '[protocol "file"]', + '\tallow = always', + '', + ].join('\n'); + await writeFile(join(source, '.git', 'config'), config, { flag: 'a' }); + await writeFile(join(source, '.git', 'refs', 'heads', 'main'), `${secret}\n`); + const state = await readLaneGit(source); + assert.notEqual(state?.head, secret); + await assert.rejects(readFile(marker), { code: 'ENOENT' }); +}); + +test('a linked worktree .git file is not claimed by the source checkout check', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const lane = join(root, 'lane'); + await exec('git', ['-C', source, 'worktree', 'add', '-b', 'lane-branch', lane]); + assert.equal(await ownsGitMetadata(lane), false); + assert.equal((await readLaneGit(lane))?.branch, 'lane-branch'); +}); + +test('a laneGit supplied by the delegate is never forwarded, only the probe value is', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const forged = { branch: 'forged', head: 'f'.repeat(40) }; + const forging = { + ...delegate(), + async execute() { + return { ...commandResult, laneGit: forged }; + }, + }; + const unreadable = await new LaneGitWorkspaceTools({ delegate: forging, resolveRoot: async () => undefined }).execute(commandRequest); + assert.equal('laneGit' in unreadable, false); + const probed = await new LaneGitWorkspaceTools({ delegate: forging, resolveRoot: async () => source }).execute(commandRequest); + assert.deepEqual('laneGit' in probed && probed.laneGit, { branch: 'main', head: await sha(source) }); +}); + +test('the worker passes its real assignment deadline to the executor', async () => { + const seen: Array = []; + const base = delegate(); + const spying = { + ...base, + async execute(request: WorkspaceToolRequest, signal?: AbortSignal, context?: { deadlineAtMs?: number }) { + seen.push(context?.deadlineAtMs); + return base.execute(request); + }, + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: false, sandboxProfile: 'anthropic-srt', runtimes: [], workspaceTools: spying.capabilities }, + workspaceTools: spying, + workspaceMutationQuarantine: quarantine(), + fetchImpl: async () => + Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: ['read_file', 'execute_command'], + }), + }); + await worker.register(); + const before = Date.now(); + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-1', + workerId: 'vm-1', + incarnationId, + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + remainingMs: 2_000, + executionKind: 'workspace_tool', + workspaceId: 'primary', + request: commandRequest, + }); + assert.equal(seen.length, 1); + const deadline = seen[0] as number; + assert.ok(deadline >= before + 1_500 && deadline <= Date.now() + 2_000, `deadline ${deadline - before}`); +}); + +// Compatibility: negotiation with old and new Code API servers. + +function quarantine() { + return { async assertAvailable() {}, async arm() {}, async clear() {}, async quarantine() {} }; +} + +async function run(t: TestContext, supported: boolean) { + const root = await scratch(t); + const source = await repo(root, 'source'); + const tools = new LaneGitWorkspaceTools({ delegate: delegate(), resolveRoot: async () => source }); + const registrations: Array<{ commandResultFeatures?: string[] }> = []; + const settlements: Array<{ result?: Record }> = []; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: false, sandboxProfile: 'anthropic-srt', runtimes: [], workspaceTools: tools.capabilities }, + workspaceTools: tools, + workspaceMutationQuarantine: quarantine(), + fetchImpl: async (input, init) => { + const url = String(input); + const body = init?.body ? JSON.parse(String(init.body)) : {}; + if (url.endsWith('/bridge/workers/register')) { + registrations.push(body.capabilities.workspaceTools ?? {}); + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: ['read_file', 'execute_command'], + ...(supported ? { supportedWorkspaceCommandResultFeatures: ['lane_git'] } : {}), + }); + } + if (url.endsWith('/settle')) settlements.push(body); + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + await worker.register(); + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-1', + workerId: 'vm-1', + incarnationId, + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + workspaceId: 'primary', + request: commandRequest, + }); + return { registrations, settlements }; +} + +test('a Code API without the feature still pairs, registers without it, and never receives laneGit', async (t) => { + const { registrations, settlements } = await run(t, false); + assert.ok(registrations.length > 0); + for (const registration of registrations) assert.equal('commandResultFeatures' in registration, false); + assert.equal(settlements.length, 1); + assert.equal(settlements[0]?.result?.exitCode, 0); + assert.equal('laneGit' in (settlements[0]?.result ?? {}), false); +}); + +test('a Code API that acknowledges the feature registers it and receives laneGit', async (t) => { + const { registrations, settlements } = await run(t, true); + assert.deepEqual(registrations.at(-1)?.commandResultFeatures, ['lane_git']); + assert.deepEqual(Object.keys((settlements[0]?.result?.laneGit as object) ?? {}).sort(), ['branch', 'head']); + assert.equal((settlements[0]?.result?.laneGit as { branch: string }).branch, 'main'); +}); + +test('a worker without the field still pairs and reports status', async () => { + const executor = { + capabilities: { protocolVersion: 1 as const, operations: ['read_file' as const], workspaces: [{ id: 'primary' }] }, + async execute(): Promise { + throw new Error('not executed'); + }, + }; + const registrations: unknown[] = []; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: false, sandboxProfile: 'anthropic-srt', runtimes: [], workspaceTools: executor.capabilities }, + workspaceTools: executor, + fetchImpl: async (_input, init) => { + registrations.push(JSON.parse(String(init?.body)).capabilities.workspaceTools); + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: ['read_file'], + }); + }, + }); + const registration = await worker.register(); + assert.equal(registration.workerId, 'vm-1'); + assert.ok(registrations.length > 0); + for (const advertised of registrations) { + assert.equal(JSON.stringify(advertised).includes('commandResultFeatures'), false); + assert.equal(JSON.stringify(advertised).includes('laneGit'), false); + } +}); diff --git a/packages/code/src/lane-git.ts b/packages/code/src/lane-git.ts new file mode 100644 index 00000000..f7aa3122 --- /dev/null +++ b/packages/code/src/lane-git.ts @@ -0,0 +1,297 @@ +import { lstat, opendir, realpath } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, boundedBranch, boundedHead } from './protocol.js'; +import { gitBytes } from './worktree-retirement.js'; + +import type { WorkspaceLaneGit, WorkspaceToolRequest, WorkspaceToolResult } from './protocol.js'; +import type { WorkspaceToolExecutor } from './workspace.js'; + +/** Longest the advisory probe may run, however much budget is left. */ +const LANE_GIT_PROBE_MAX_MS = 1_000; +/** Settlement budget the probe never touches, for validation and the settle request. */ +const LANE_GIT_SETTLEMENT_RESERVE_MS = 1_000; +/** Below this, a probe is not worth starting. */ +const LANE_GIT_PROBE_MIN_MS = 250; +/** Code API gives a command its own timeout plus this much before it expires the assignment. */ +const COMMAND_EXECUTION_GRACE_MS = 5_000; +const LANE_GIT_TIMEOUT_MS = 5_000; +const LANE_GIT_OUTPUT_LIMIT = 4096; + +/** + * A read-only probe must never reach the network or run a transport command, whatever the checkout's + * own config says. A command can rewrite `.git/config` to mark a remote as a promisor and point a ref + * at a missing object, which makes a plain object lookup fetch lazily and run `ext::`, `ssh` or + * `uploadpack` programs on the host. An empty `GIT_ALLOW_PROTOCOL` disallows every transport and + * overrides repository configuration; `GIT_NO_LAZY_FETCH` covers Git 2.44 and later directly. + */ +const LANE_GIT_PROBE_ENV = { GIT_ALLOW_PROTOCOL: '', GIT_NO_LAZY_FETCH: '1' }; + +/** Exit codes that mean "no value" rather than "could not read": detached HEAD, unborn branch. */ +const DETACHED_EXIT = 1; +const UNBORN_EXIT = 128; + +async function read( + root: string, + args: string[], + emptyExit: number, + signal?: AbortSignal, + timeoutMs: number = LANE_GIT_TIMEOUT_MS, +): Promise { + try { + const bytes = await gitBytes(root, args, signal, timeoutMs, LANE_GIT_OUTPUT_LIMIT, LANE_GIT_PROBE_ENV); + // Git permits ref names that are not UTF-8. Decoding them leniently would report a different, + // valid-looking name, so anything that is not strictly valid UTF-8 is reported as null. + let text: string; + try { + text = new TextDecoder('utf-8', { fatal: true }).decode(bytes); + } catch { + return null; + } + // Drop only Git's own trailing line terminator: a ref name may legally begin or end with + // other whitespace, and trimming it would report a different branch. + return text.replace(/\r?\n$/, ''); + } catch (error) { + signal?.throwIfAborted(); + return (error as { code?: unknown }).code === emptyExit ? null : undefined; + } +} + +/** + * The branch and head commit checked out in a lane root, or undefined when Git + * could not be read (timeout, missing binary, aborted). Detached HEAD and a + * repository without commits are ordinary states and report null. Values that + * fail the bounds are reported as null, never forwarded. Nothing else from the + * repository is read or returned. + */ +export async function readLaneGit( + root: string, + signal?: AbortSignal, + timeoutMs: number = LANE_GIT_TIMEOUT_MS, +): Promise { + return await snapshotLaneGit( + () => read(root, ['symbolic-ref', '--quiet', 'HEAD'], DETACHED_EXIT, signal, timeoutMs), + // `--no-replace-objects`: a `refs/replace` entry must not turn a blob into a reported commit. + (ref) => read(root, ['--no-replace-objects', 'rev-parse', '--verify', `${ref}^{commit}`], UNBORN_EXIT, signal, timeoutMs), + ); +} + +/** + * Two Git processes cannot see one atomic snapshot, so a checkout between them could pair one + * branch with another branch's commit, a tuple that was never checked out. The head is therefore + * read from the branch ref the first read named, not from `HEAD`, and the branch is read again + * afterwards: a checkout away during the read omits the field. A detached HEAD has no ref to pin, + * so it is read twice and must agree. A change that happens and reverts within one probe is not + * detectable without a lock, and is not worth one for an advisory field. + */ +export async function snapshotLaneGit( + readBranch: () => Promise, + readCommit: (ref: string) => Promise, +): Promise { + const before = await readBranch(); + if (before === undefined) return undefined; + const head = await readCommit(before ?? 'HEAD'); + const after = await readBranch(); + if (head === undefined || after === undefined || before !== after) return undefined; + if (before === null && (await readCommit('HEAD')) !== head) return undefined; + // `--short` shortens ambiguously (`heads/foo` when a tag `foo` exists). Take the full ref and strip + // exactly `refs/heads/`; a HEAD pointing anywhere else is not a branch. + const name = before?.startsWith('refs/heads/') ? before.slice('refs/heads/'.length) : null; + return { branch: boundedBranch(name), head: boundedHead(head) }; +} + +async function absent(path: string): Promise { + try { + await lstat(path); + return false; + } catch (error) { + return (error as NodeJS.ErrnoException).code === 'ENOENT'; + } +} + +async function plain(path: string, kind: 'file' | 'directory'): Promise { + try { + const status = await lstat(path); + return !status.isSymbolicLink() && (kind === 'file' ? status.isFile() : status.isDirectory()); + } catch { + return false; + } +} + +/** Entries a checkout may have before the probe gives up on proving they are all plain. */ +const LANE_GIT_REFS_MAX_ENTRIES = 20_000; +const LANE_GIT_OBJECTS_MAX_ENTRIES = 100_000; + +/** Whether everything beneath a directory is a plain file or directory: no link can redirect a read. */ +async function treeIsLinkFree( + refs: string, + signal?: AbortSignal, + maxEntries = LANE_GIT_REFS_MAX_ENTRIES, +): Promise { + const pending = [refs]; + let seen = 0; + while (pending.length > 0) { + // Stop between directory operations: the probe timeout returns the command result, but only an + // abort observed here keeps an abandoned walk from running on in the background. + signal?.throwIfAborted(); + const current = pending.pop()!; + const directory = await opendir(current); + for await (const entry of directory) { + signal?.throwIfAborted(); + if (++seen > maxEntries) return false; + if (entry.isSymbolicLink()) return false; + // `Dirent.parentPath` needs Node 20.12; the package supports 20.11. + if (entry.isDirectory()) pending.push(join(current, entry.name)); + else if (!entry.isFile()) return false; + } + } + return true; +} + +/** + * The shared part of both ownership checks: the refs and object storage of a Git directory hold + * no link, no `alternates` redirect and no `reftable` backend, so the host-side reads cannot be + * steered at another repository's refs or objects. + */ +async function storageIsOwn(gitDir: string, signal?: AbortSignal): Promise { + const objects = join(gitDir, 'objects'); + return ( + (await plain(join(gitDir, 'refs'), 'directory')) && + (await treeIsLinkFree(join(gitDir, 'refs'), signal)) && + (await plain(objects, 'directory')) && + (await absent(join(objects, 'info', 'alternates'))) && + (await absent(join(objects, 'info', 'http-alternates'))) && + (await treeIsLinkFree(objects, signal, LANE_GIT_OBJECTS_MAX_ENTRIES)) && + (await absent(join(gitDir, 'reftable'))) && + ((await absent(join(gitDir, 'packed-refs'))) || (await plain(join(gitDir, 'packed-refs'), 'file'))) + ); +} + +/** + * Whether `/.git` is the checkout's own Git directory, so the host-side probe cannot be + * steered at another repository. A command can rewrite its own `.git` after it ran: replace it + * with a `gitdir:` file, add a `commondir` redirect, or link `HEAD`, `packed-refs`, `refs` or any + * ref beneath it to another checkout, and the unsandboxed probe would then report that checkout's + * branch and head. Every entry under `refs/` is therefore checked, up to a bound. + * The check runs after the command has finished, so it does not cover a process the command left + * running in the background. + * Anything else, including a `.git` file (a linked worktree or submodule as a source root), is + * not probed: the field is simply omitted. Linked worktree lanes are verified separately. + */ +export async function ownsGitMetadata(root: string, signal?: AbortSignal): Promise { + const dotGit = join(root, '.git'); + try { + if (!(await plain(dotGit, 'directory')) || (await realpath(dotGit)) !== join(await realpath(root), '.git')) { + return false; + } + return ( + (await plain(join(dotGit, 'HEAD'), 'file')) && + (await absent(join(dotGit, 'commondir'))) && + (await storageIsOwn(dotGit, signal)) + ); + } catch { + return false; + } +} + +/** + * The linked worktree counterpart of `ownsGitMetadata`. `verifyLinkedWorktree` proves the lane's + * `.git` pointer chain, but the probe also reads the lane's own `HEAD` and the refs it names from + * the shared common directory, and a command can write to both. Without this, a ref symlinked at + * another checkout and named by the lane's `HEAD` would be followed by the host-side probe. + */ +export async function ownsLinkedWorktreeMetadata(commonGitDir: string, name: string, signal?: AbortSignal): Promise { + const metadata = join(commonGitDir, 'worktrees', name); + try { + if (!(await plain(join(metadata, 'HEAD'), 'file'))) return false; + if (!(await absent(join(metadata, 'refs'))) && !(await treeIsLinkFree(join(metadata, 'refs'), signal))) return false; + return await storageIsOwn(commonGitDir, signal); + } catch { + return false; + } +} + +export interface LaneGitWorkspaceToolsOptions { + delegate: WorkspaceToolExecutor; + /** The lane root a request ran in (isolated worktree, linked worktree or source checkout), or undefined. */ + resolveRoot: (request: WorkspaceToolRequest, signal?: AbortSignal) => Promise; + /** Whether Code API negotiated `lane_git`. Git is not probed while this is false. Defaults to true. */ + isEnabled?: () => boolean; + /** Longest the probe may run. Defaults to 1 s; it is also cut to fit the command's remaining budget. */ + probeTimeoutMs?: number; + /** Clock, for tests. */ + now?: () => number; +} + +/** + * Attaches `laneGit` to every finished `execute_command` result, so a branch the + * agent created or switched is visible as soon as the command returns. Git is + * read once per command, never on a timer. The field is stripped by the worker + * unless Code API acknowledged the `lane_git` feature at registration. + */ +export class LaneGitWorkspaceTools implements WorkspaceToolExecutor { + readonly mutationFailuresAreAtomic?: true; + readonly capabilities: WorkspaceToolExecutor['capabilities']; + + constructor(private readonly options: LaneGitWorkspaceToolsOptions) { + this.mutationFailuresAreAtomic = options.delegate.mutationFailuresAreAtomic; + const base = options.delegate.capabilities; + this.capabilities = base.operations.includes('execute_command') + ? { ...base, commandResultFeatures: ['lane_git'] } + : base; + } + + async execute( + request: WorkspaceToolRequest, + signal?: AbortSignal, + context?: { deadlineAtMs?: number }, + ): Promise { + const now = this.options.now ?? Date.now; + const startedAt = now(); + const delegated = await this.options.delegate.execute(request, signal, context); + if (request.operation !== 'execute_command' || delegated.operation !== 'execute_command') return delegated; + // Only the value this wrapper reads is trusted. A sandbox executor must not be able to supply + // its own branch and head, whether or not the probe below produces anything. + const { laneGit: _untrusted, ...result } = delegated; + if (this.options.isEnabled?.() === false) return result; + // The probe is advisory, so it only runs in budget the command left over, and never into the + // part reserved for settlement. The worker's real deadline accounts for time already spent on + // lease transport and credential refresh; without it, assume the full timeout plus the grace. + const deadlineAtMs = + context?.deadlineAtMs ?? + startedAt + (request.timeoutMs ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS) + COMMAND_EXECUTION_GRACE_MS; + const probeMs = Math.min( + this.options.probeTimeoutMs ?? LANE_GIT_PROBE_MAX_MS, + deadlineAtMs - now() - LANE_GIT_SETTLEMENT_RESERVE_MS, + ); + if (probeMs < Math.min(LANE_GIT_PROBE_MIN_MS, this.options.probeTimeoutMs ?? LANE_GIT_PROBE_MIN_MS)) return result; + // The resolver may do filesystem work that cannot observe a signal, so the race, not the + // signal, is what bounds the wait. The signal still stops Git when the limit hits first. + const controller = new AbortController(); + const probeSignal = AbortSignal.any([...(signal ? [signal] : []), controller.signal]); + let timer: NodeJS.Timeout | undefined; + try { + const probe = (async (): Promise => { + try { + const root = await this.options.resolveRoot(request, probeSignal); + return root == null ? undefined : await readLaneGit(root, probeSignal, probeMs); + } catch { + return undefined; + } + })(); + const limit = new Promise((resolve) => { + timer = setTimeout(() => { + controller.abort(); + resolve(undefined); + }, probeMs); + }); + const laneGit = await Promise.race([probe, limit]); + return laneGit ? { ...result, laneGit } : result; + } catch { + // Git state is advisory; a failed read must never fail or delay the command's own result. + return result; + } finally { + clearTimeout(timer); + } + } +} diff --git a/packages/code/src/projects.ts b/packages/code/src/projects.ts index 1e2fcbd3..e91694f3 100644 --- a/packages/code/src/projects.ts +++ b/packages/code/src/projects.ts @@ -4,6 +4,8 @@ import { lstat, opendir, realpath } from 'node:fs/promises'; import { isAbsolute, relative, resolve, sep } from 'node:path'; import { promisify } from 'node:util'; +import { boundedBranch, boundedHead } from './protocol.js'; + const exec = promisify(execFile); const skipped = new Set(['node_modules', 'vendor']); @@ -221,16 +223,8 @@ export async function discoverProjects( ); const path = rel.split(sep).join('/') || '.'; const normalizedRemote = remote ? projectRemote(remote) : null; - const validBranch = - branch && - branch.length <= 256 && - !/[\x00-\x1f\x7f]/.test(branch) - ? branch - : null; - const validHead = - head && /^[a-f0-9]{40}(?:[a-f0-9]{24})?$/.test(head) - ? head - : null; + const validBranch = boundedBranch(branch); + const validHead = boundedHead(head); if ( (remote !== null && normalizedRemote === null) || (branch !== null && validBranch === null) || diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index 216784a4..1451986d 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -340,7 +340,50 @@ const WORKSPACE_EDIT_MATCH_STRATEGIES = new Set([ 'whitespace-normalized', 'indentation-flexible', ]); + +/** A branch name reportable to LibreChat: 1 to 256 chars, no control characters. */ +export function boundedBranch(value: unknown): string | null { + return typeof value === 'string' && + value.length > 0 && + value.length <= 256 && + // Git forbids only ASCII controls in ref names; the header must also never show Unicode + // controls, format characters (bidi overrides, zero-width) or line and paragraph separators. + !/[\x00-\x1f\x7f\p{Cc}\p{Cf}\p{Zl}\p{Zp}]/u.test(value) + ? value + : null; +} + +/** A commit id reportable to LibreChat: 40 (SHA-1) or 64 (SHA-256) lowercase hex chars. */ +export function boundedHead(value: unknown): string | null { + return typeof value === 'string' && /^[a-f0-9]{40}(?:[a-f0-9]{24})?$/.test(value) + ? value + : null; +} + export type WorkspaceListFileFeature = 'after_path'; +/** `lane_git`: `execute_command` results may carry `laneGit`. Sent only when Code API echoes it. */ +export type WorkspaceCommandResultFeature = 'lane_git'; + +/** Git state of the lane a command ran in. Never carries paths, remotes or repository content. */ +export interface WorkspaceLaneGit { + /** Checked-out branch, or null when HEAD is detached or the name is not reportable. */ + branch: string | null; + /** HEAD commit (40 or 64 lowercase hex), or null before the first commit or when unreportable. */ + head: string | null; +} + +export function isWorkspaceLaneGit(value: unknown): value is WorkspaceLaneGit { + if (typeof value !== 'object' || value === null || Array.isArray(value)) return false; + const state = value as Record; + const keys = Object.keys(state); + return ( + keys.length === 2 && + keys.includes('branch') && + keys.includes('head') && + (state.branch === null || (state.branch !== undefined && boundedBranch(state.branch) === state.branch)) && + (state.head === null || (state.head !== undefined && boundedHead(state.head) === state.head)) + ); +} export type WorkspaceProgrammaticLanguage = 'bash'; export interface BridgeWorkspaceDescriptor { @@ -374,6 +417,8 @@ export interface BridgeWorkspaceToolCapabilities { editFileFeatures?: WorkspaceEditFileFeature[]; /** Omitted by workers that cannot continue a bounded file listing. */ listFileFeatures?: WorkspaceListFileFeature[]; + /** Omitted by workers that do not report lane Git state on command results. */ + commandResultFeatures?: WorkspaceCommandResultFeature[]; /** Languages that can execute PTC replay inside a selected workspace. */ programmaticLanguages?: WorkspaceProgrammaticLanguage[]; } @@ -625,6 +670,8 @@ export interface WorkspaceExecuteCommandResult { stderr: string; truncated: boolean; timedOut: boolean; + /** Present only when `commandResultFeatures` negotiated `lane_git` and Git was readable. */ + laneGit?: WorkspaceLaneGit; } export type WorkspaceToolRequest = @@ -789,6 +836,7 @@ const WORKSPACE_COMMAND_RESULT_KEYS = new Set([ 'stderr', 'truncated', 'timedOut', + 'laneGit', ]); const WORKSPACE_SEARCH_MATCH_KEYS = new Set(['path', 'line', 'column', 'text']); @@ -836,6 +884,8 @@ export interface BridgeWorkerRegistrationResponse { supportedWorkspaceInstanceTypes?: ['git_worktree']; /** Scheduling scopes this Code API can admit as independent lanes. */ supportedWorkspaceScopes?: ['git_linked_worktree']; + /** Command result fields this Code API accepts in settlements. */ + supportedWorkspaceCommandResultFeatures?: WorkspaceCommandResultFeature[]; /** Added workspace tool error codes this Code API accepts in settlements. */ supportedWorkspaceToolErrorCodes?: WorkspaceToolErrorCode[]; } @@ -1717,6 +1767,7 @@ export function isWorkspaceToolResult( /^SIG[A-Z0-9]+$/.test(result.signal))) && typeof result.truncated === 'boolean' && typeof result.timedOut === 'boolean' && + (result.laneGit === undefined || isWorkspaceLaneGit(result.laneGit)) && (result.exitCode === null ? result.timedOut === true || result.signal !== undefined : result.timedOut === false && result.signal === undefined) @@ -1839,6 +1890,16 @@ export function isValidBridgeWorkspaceToolCapabilities( return false; } + if ( + capabilities.commandResultFeatures !== undefined && + (!Array.isArray(capabilities.commandResultFeatures) || + capabilities.commandResultFeatures.length !== 1 || + !capabilities.operations.includes('execute_command') || + capabilities.commandResultFeatures[0] !== 'lane_git') + ) { + return false; + } + if ( capabilities.programmaticLanguages !== undefined && (!Array.isArray(capabilities.programmaticLanguages) || diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 17a9a6e1..49b96eb1 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -18,6 +18,7 @@ import { signBridgeRequest } from './identity.js'; import { isWorkspaceToolRequest, WorkspaceToolError } from './workspace.js'; import type { + WorkspaceCommandResultFeature, BridgeAssignment, BridgeLeaseResponse, BridgeSandboxRequest, @@ -218,6 +219,11 @@ function workspaceCapabilitiesMatch( (feature, index) => feature === executor.listFileFeatures?.[index], ) ?? executor.listFileFeatures == null) && + advertised.commandResultFeatures?.length === executor.commandResultFeatures?.length && + (advertised.commandResultFeatures?.every( + (feature, index) => feature === executor.commandResultFeatures?.[index], + ) ?? + executor.commandResultFeatures == null) && advertised.programmaticLanguages?.length === executor.programmaticLanguages?.length && (advertised.programmaticLanguages?.every( @@ -314,6 +320,7 @@ function registrationCompatibleCapabilities( editFileModes: _editFileModes, editFileFeatures: _editFileFeatures, listFileFeatures: _listFileFeatures, + commandResultFeatures: _commandResultFeatures, programmaticLanguages: _programmaticLanguages, ...compatibleWorkspaceTools } = workspaceTools; @@ -409,6 +416,9 @@ function supportedWorkspaceCapabilities( const listFileFeatures = desired.listFileFeatures?.filter((feature) => registration.supportedWorkspaceListFileFeatures?.includes(feature), ); + const commandResultFeatures = desired.commandResultFeatures?.filter((feature) => + registration.supportedWorkspaceCommandResultFeatures?.includes(feature), + ); const programmaticLanguages = desired.programmaticLanguages?.filter( (language) => registration.supportedWorkspaceProgrammaticLanguages?.includes(language), @@ -418,6 +428,7 @@ function supportedWorkspaceCapabilities( editFileModes: _editFileModes, editFileFeatures: _editFileFeatures, listFileFeatures: _listFileFeatures, + commandResultFeatures: _commandResultFeatures, programmaticLanguages: _programmaticLanguages, ...compatibleDesired } = desired; @@ -439,6 +450,9 @@ function supportedWorkspaceCapabilities( ...(operations.includes('list_files') && listFileFeatures?.length ? { listFileFeatures } : {}), + ...(operations.includes('execute_command') && commandResultFeatures?.length + ? { commandResultFeatures } + : {}), ...(operations.includes('execute_command') && programmaticLanguages?.length ? { programmaticLanguages } @@ -468,6 +482,11 @@ export class BridgeWorker { private instructionMetadataSupported = true; /** Added error codes the current Code API registration accepts in settlements. */ private settlementErrorCodes: ReadonlySet = new Set(); + /** Whether the registration that is actually active advertised this command result feature. */ + commandResultFeatureActive(feature: WorkspaceCommandResultFeature): boolean { + return this.activeCapabilities.workspaceTools?.commandResultFeatures?.includes(feature) === true; + } + private registrationTtlMs = DEFAULT_REGISTRATION_TTL_MS; private lastRegisteredAtMs = 0; private maintenanceOnly = false; @@ -1854,6 +1873,7 @@ export class BridgeWorker { payload = await this.options.workspaceTools.execute( workspaceRequest, executionController.signal, + { deadlineAtMs: localDeadlineAtMs }, ); if ( workspaceRequest.operation === 'list_files' && @@ -1864,6 +1884,14 @@ export class BridgeWorker { payload; payload = compatiblePayload; } + if ( + workspaceRequest.operation === 'execute_command' && + !advertised.commandResultFeatures?.includes('lane_git') && + 'laneGit' in payload + ) { + const { laneGit: _laneGit, ...compatiblePayload } = payload; + payload = compatiblePayload; + } workspaceMutationApplied = isMutation; if (isMutation && !isWorkspaceToolResult(workspaceRequest, payload)) { throw new BridgeProtocolError( diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index 746c7249..d740d48a 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -100,6 +100,8 @@ export interface WorkspaceToolExecutor { execute( request: WorkspaceToolRequest, signal?: AbortSignal, + /** Local epoch ms after which the worker aborts this execution. Advisory work must finish well before it. */ + context?: { deadlineAtMs?: number }, ): Promise; } diff --git a/packages/code/src/worktree-retirement.ts b/packages/code/src/worktree-retirement.ts index 39a7b60c..2ca8b78f 100644 --- a/packages/code/src/worktree-retirement.ts +++ b/packages/code/src/worktree-retirement.ts @@ -158,7 +158,7 @@ function gitEnvironment(): NodeJS.ProcessEnv { }; } -async function git( +export async function git( cwd: string, args: string[], signal?: AbortSignal, @@ -173,6 +173,23 @@ async function git( return stdout; } +/** Same invocation as `git`, but the raw stdout bytes, for callers that must not accept lossy decoding. */ +export async function gitBytes( + cwd: string, + args: string[], + signal?: AbortSignal, + timeout = GIT_TIMEOUT_MS, + maxBuffer = GIT_OUTPUT_LIMIT, + extraEnvironment: Record = {}, +): Promise { + const { stdout } = await execFileAsync( + 'git', + ['--no-optional-locks', '-C', cwd, '-c', 'core.fsmonitor=false', ...args], + { encoding: 'buffer', env: { ...gitEnvironment(), ...extraEnvironment }, maxBuffer, signal, timeout }, + ); + return stdout; +} + /** The commit a ref names, or undefined when it does not resolve. */ async function resolveCommit(cwd: string, ref: string, signal?: AbortSignal): Promise { try { diff --git a/service/src/bridge/index.ts b/service/src/bridge/index.ts index b04de55f..f7ba03b4 100644 --- a/service/src/bridge/index.ts +++ b/service/src/bridge/index.ts @@ -10,6 +10,7 @@ export const bridgeStore = new RedisBridgeStore( undefined, undefined, env.BRIDGE_MAX_WORKSPACE_LEASE_SLOTS, + env.BRIDGE_LANE_GIT, ); export const bridgePairings = new RedisBridgePairingStore( connection, diff --git a/service/src/bridge/lane-git.test.ts b/service/src/bridge/lane-git.test.ts new file mode 100644 index 00000000..604f2070 --- /dev/null +++ b/service/src/bridge/lane-git.test.ts @@ -0,0 +1,251 @@ +import { afterEach, describe, expect, spyOn, test } from 'bun:test'; +import RedisMock from 'ioredis-mock'; + +import type Redis from 'ioredis'; + +import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; +import { laneGitPolicy } from './lane-git'; +import { RedisBridgeStore } from './store'; + +const redis = new RedisMock() as unknown as Redis; +const incarnationId = 'incarnation-00000001'; +const head = 'a'.repeat(40); + +afterEach(async () => { + await redis.flushall(); +}); + +function capabilities(advertise: boolean) { + return { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['read_file' as const, 'execute_command' as const], + workspaces: [{ id: 'primary' }], + ...(advertise ? { commandResultFeatures: ['lane_git' as const] } : {}), + }, + }; +} + +const request = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'execute_command' as const, + workspaceId: 'primary', + command: 'git checkout -b agent', +}; + +function commandResult(extra: Record = {}) { + return { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'execute_command' as const, + workspaceId: 'primary', + exitCode: 0, + stdout: '', + stderr: '', + truncated: false, + timedOut: false, + ...extra, + }; +} + +/** Registers a worker, runs one command through the store, and returns what the caller receives. */ +async function run(opts: { enabled: boolean; advertise: boolean; workerResult: Record }) { + const store = new RedisBridgeStore(redis, undefined, undefined, 1, opts.enabled); + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'lane-worker', + incarnationId, + capabilities: capabilities(opts.advertise), + }); + const completion = store.dispatchWorkspaceTool({ + workerId: 'lane-worker', + tenantId: 'tenant-1', + request, + deadlineAtMs: Date.now() + 5_000, + signal: new AbortController().signal, + }); + const assignment = await store.lease('lane-worker', incarnationId, 1_000); + await store.settle('lane-worker', assignment?.assignmentId ?? '', { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: assignment?.leaseToken ?? '', + incarnationId, + status: 'fulfilled', + result: opts.workerResult as never, + }); + // Read the durable copy before the dispatcher cleans it up; it must exist, or the + // "not stored" assertions below would pass vacuously. + const stored = await redis.get(`codeapi:bridge:v1:assignment:${assignment?.assignmentId}:settlement`); + expect(stored).not.toBeNull(); + const settlement = await completion; + return { settlement, stored: stored as string }; +} + +describe('laneGit on command settlements', () => { + test('a valid laneGit from an advertising worker reaches the caller unchanged', async () => { + const laneGit = { branch: 'librechat/conversation-abcd1234-0123', head }; + const { settlement } = await run({ enabled: true, advertise: true, workerResult: commandResult({ laneGit }) }); + expect(settlement.status).toBe('fulfilled'); + expect((settlement as { result: { laneGit?: unknown } }).result.laneGit).toEqual(laneGit); + }); + + test('null branch and head (detached, no commit) are valid and preserved', async () => { + const laneGit = { branch: null, head: null }; + const { settlement } = await run({ enabled: true, advertise: true, workerResult: commandResult({ laneGit }) }); + expect((settlement as { result: { laneGit?: unknown } }).result.laneGit).toEqual(laneGit); + }); + + test('a 64 character head is accepted', async () => { + const laneGit = { branch: 'main', head: 'b'.repeat(64) }; + const { settlement } = await run({ enabled: true, advertise: true, workerResult: commandResult({ laneGit }) }); + expect((settlement as { result: { laneGit?: unknown } }).result.laneGit).toEqual(laneGit); + }); + + const invalid: Array<[string, unknown]> = [ + ['extra keys', { branch: 'main', head, path: '/srv/repo' }], + ['an over-long branch', { branch: 'x'.repeat(257), head }], + ['an empty branch', { branch: '', head }], + ['a control character in the branch', { branch: 'bad\nname', head }], + ['a bidi override in the branch', { branch: 'feature/\u202eabc', head }], + ['a zero-width character in the branch', { branch: 'fix\u200bme', head }], + ['uppercase hex', { branch: 'main', head: 'A'.repeat(40) }], + ['a short head', { branch: 'main', head: 'a'.repeat(39) }], + ['non-hex head', { branch: 'main', head: 'g'.repeat(40) }], + ['a missing head', { branch: 'main' }], + ['a non-object', 'main'], + ['an array', [null, null]], + ]; + for (const [name, laneGit] of invalid) { + test(`laneGit with ${name} is dropped and the command still completes`, async () => { + const { settlement, stored } = await run({ enabled: true, advertise: true, workerResult: commandResult({ laneGit }) }); + expect(settlement.status).toBe('fulfilled'); + const result = (settlement as unknown as { result: Record }).result; + expect('laneGit' in result).toBe(false); + expect(result.exitCode).toBe(0); + expect(stored).not.toContain('laneGit'); + expect(stored).not.toContain('/srv/repo'); + }); + } + + test('laneGit from a worker that did not advertise lane_git is stripped', async () => { + const { settlement, stored } = await run({ + enabled: true, + advertise: false, + workerResult: commandResult({ laneGit: { branch: 'main', head } }), + }); + expect(settlement.status).toBe('fulfilled'); + expect('laneGit' in (settlement as { result: object }).result).toBe(false); + expect(stored).not.toContain('laneGit'); + }); + + test('laneGit is stripped when the deployment setting is off, even from an advertising worker', async () => { + const { settlement, stored } = await run({ + enabled: false, + advertise: true, + workerResult: commandResult({ laneGit: { branch: 'main', head } }), + }); + expect(settlement.status).toBe('fulfilled'); + expect('laneGit' in (settlement as { result: object }).result).toBe(false); + expect(stored).not.toContain('laneGit'); + }); + + test('an old worker without the field still registers and runs commands, with the setting on or off', async () => { + for (const enabled of [false, true]) { + const { settlement } = await run({ enabled, advertise: false, workerResult: commandResult() }); + expect(settlement.status).toBe('fulfilled'); + expect((settlement as { result: { exitCode: number } }).result.exitCode).toBe(0); + await redis.flushall(); + } + }); + + test('the policy never reports the value, only a fixed reason', () => { + const result = commandResult({ laneGit: { branch: 'secret-branch', head, path: '/srv/repo' } }); + const outcome = laneGitPolicy({ result, enabled: true, advertised: true }); + expect(outcome.dropped).toBe('invalid'); + expect(JSON.stringify(outcome.dropped)).not.toContain('secret-branch'); + }); +}); + +describe('retried settlements across replicas that disagree on the setting', () => { + // A replica commits a settlement and its response is lost; the worker retries the same body + // against another replica mid-rollout. The retry must be recognized as the same settlement. + for (const [first, second] of [ + [true, false], + [false, true], + ] as const) { + test(`commit with the setting ${first ? 'on' : 'off'}, retry on a replica with it ${second ? 'on' : 'off'}`, async () => { + const a = new RedisBridgeStore(redis, undefined, undefined, 1, first); + const b = new RedisBridgeStore(redis, undefined, undefined, 1, second); + await a.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'lane-worker', + incarnationId, + capabilities: capabilities(true), + }); + const completion = a.dispatchWorkspaceTool({ + workerId: 'lane-worker', + tenantId: 'tenant-1', + request, + deadlineAtMs: Date.now() + 5_000, + signal: new AbortController().signal, + }); + completion.catch(() => undefined); + const assignment = await a.lease('lane-worker', incarnationId, 1_000); + const settlement = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: assignment?.leaseToken ?? '', + incarnationId, + status: 'fulfilled' as const, + result: commandResult({ laneGit: { branch: 'main', head } }) as never, + }; + await a.settle('lane-worker', assignment?.assignmentId ?? '', settlement); + await expect(b.settle('lane-worker', assignment?.assignmentId ?? '', settlement)).resolves.toBeUndefined(); + await completion; + }); + } +}); + +describe('Redis reads while settling', () => { + async function readsDuring(withLaneGit: boolean): Promise { + const store = new RedisBridgeStore(redis, undefined, undefined, 1, true); + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'lane-worker', + incarnationId, + capabilities: capabilities(true), + }); + const completion = store.dispatchWorkspaceTool({ + workerId: 'lane-worker', + tenantId: 'tenant-1', + request, + deadlineAtMs: Date.now() + 5_000, + signal: new AbortController().signal, + }); + completion.catch(() => undefined); + const assignment = await store.lease('lane-worker', incarnationId, 1_000); + const spy = spyOn(redis, 'get'); + await store.settle('lane-worker', assignment?.assignmentId ?? '', { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: assignment?.leaseToken ?? '', + incarnationId, + status: 'fulfilled', + result: commandResult(withLaneGit ? { laneGit: { branch: 'main', head } } : {}) as never, + }); + const calls = spy.mock.calls.length; + spy.mockRestore(); + await completion; + return calls; + } + + test('a settlement with laneGit costs no more Redis reads than one without', async () => { + const without = await readsDuring(false); + await redis.flushall(); + const withIt = await readsDuring(true); + expect(without).toBeGreaterThan(0); + expect(withIt).toBe(without); + }); +}); diff --git a/service/src/bridge/lane-git.ts b/service/src/bridge/lane-git.ts new file mode 100644 index 00000000..69ddf7b9 --- /dev/null +++ b/service/src/bridge/lane-git.ts @@ -0,0 +1,36 @@ +import { isWorkspaceLaneGit } from '../../../packages/code/src/protocol'; + +export type LaneGitDropReason = 'disabled' | 'not_advertised' | 'invalid'; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** Whether a fulfilled command result carries a `laneGit` key at all. */ +export function hasLaneGit(result: unknown): boolean { + return isRecord(result) && 'laneGit' in result; +} + +/** + * Decide what happens to the `laneGit` on an execute_command result. It is kept, + * unchanged, only when the deployment enables the feature, the worker advertised + * `lane_git`, and the value is exactly `{ branch, head }` within bounds. Anything + * else is dropped; the command result itself is never failed over it. The reason + * is a fixed label, never the value. + */ +export function laneGitPolicy(args: { + result: Record; + enabled: boolean; + advertised: boolean; +}): { result: Record; dropped?: LaneGitDropReason } { + const reason: LaneGitDropReason | undefined = !args.enabled + ? 'disabled' + : !args.advertised + ? 'not_advertised' + : isWorkspaceLaneGit(args.result.laneGit) + ? undefined + : 'invalid'; + if (reason === undefined) return { result: args.result }; + const { laneGit: _laneGit, ...rest } = args.result; + return { result: rest, dropped: reason }; +} diff --git a/service/src/bridge/router.test.ts b/service/src/bridge/router.test.ts index 360414a0..5e76876e 100644 --- a/service/src/bridge/router.test.ts +++ b/service/src/bridge/router.test.ts @@ -167,6 +167,55 @@ describe('paired bridge HTTP API', () => { }); }); + for (const laneGit of [false, true]) { + test(`registration ${laneGit ? 'offers' : 'omits'} lane_git when the setting is ${laneGit ? 'on' : 'off'}`, async () => { + const app = express(); + app.use(json()); + app.use( + '/v1/bridge', + createBridgeRouter({ + store: new RedisBridgeStore(redis, undefined, undefined, 1, laneGit), + pairings: new RedisBridgePairingStore(redis), + authMode: 'static', + adminToken: 'strong-administrator-bootstrap-token', + configuredWorkerId: 'vm-1', + }), + ); + server = createServer(app); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Expected TCP listener'); + const response = await fetch(`http://127.0.0.1:${address.port}/v1/bridge/workers/register`, { + method: 'POST', + headers: { + Authorization: 'Bearer strong-administrator-bootstrap-token', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['read_file', 'execute_command'], + workspaces: [{ id: 'primary' }], + ...(laneGit ? { commandResultFeatures: ['lane_git'] } : {}), + }, + }, + }), + }); + expect(response.status).toBe(200); + const body = (await response.json()) as Record; + expect(body.supportedWorkspaceListFileFeatures).toEqual(['after_path']); + if (laneGit) expect(body.supportedWorkspaceCommandResultFeatures).toEqual(['lane_git']); + else expect('supportedWorkspaceCommandResultFeatures' in body).toBe(false); + }); + } + test('rejects a malformed optional binding for a configured worker', async () => { const app = express(); app.use(json()); diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 0bc20a1e..123f8c75 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -538,6 +538,9 @@ router.post( supportedWorkspaceEditFileModes: ['single', 'batch'], supportedWorkspaceEditFileFeatures: [...WORKSPACE_EDIT_FILE_FEATURES], supportedWorkspaceListFileFeatures: ['after_path'], + ...(options.store.laneGitEnabled + ? { supportedWorkspaceCommandResultFeatures: ['lane_git'] } + : {}), supportedWorkspaceProgrammaticLanguages: ['bash'], supportedWorkspaceInstanceTypes: ['git_worktree'], supportedWorkspaceScopes: ['git_linked_worktree'], diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 542c5903..0ca74c45 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -21,6 +21,8 @@ import { workspaceIsolationParent, } from '../../../packages/code/src/protocol'; import type { BridgeWorkerBinding } from './pairing'; +import logger from '../logger'; +import { hasLaneGit, laneGitPolicy } from './lane-git'; import { BridgeAdmissionQueue, durableAdmissionFence } from './admission'; import { BridgeWorkspaceSlots } from './slots'; import type { StoredWorkspaceRequest } from '../workspace-tools/requests'; @@ -383,6 +385,12 @@ function assignmentKey(assignmentId: string): string { return `${PREFIX}:assignment:${assignmentId}`; } +function withoutLaneGit(settlement: AnyCodeBridgeSettlement): AnyCodeBridgeSettlement { + if (settlement.status !== 'fulfilled') return settlement; + const { laneGit: _laneGit, ...result } = settlement.result as unknown as Record; + return { ...settlement, result } as AnyCodeBridgeSettlement; +} + function settlementKey(assignmentId: string): string { return `${PREFIX}:assignment:${assignmentId}:settlement`; } @@ -472,6 +480,8 @@ export class RedisBridgeStore { private readonly workerTtlSeconds = DEFAULT_WORKER_TTL_SECONDS, private readonly redisCommandTimeoutMs = DEFAULT_REDIS_COMMAND_TIMEOUT_MS, private readonly maxWorkspaceLeaseSlots = 1, + /** Accept `laneGit` on command results and offer `lane_git` at registration. Off by default. */ + public readonly laneGitEnabled = false, ) { if ( !Number.isSafeInteger(maxWorkspaceLeaseSlots) || @@ -1791,12 +1801,25 @@ export class RedisBridgeStore { ); return; } - const serializedSettlement = JSON.stringify(settlement); + let serializedSettlement = JSON.stringify(settlement); const existingSettlement = await this.leaseCommand( this.redis.get(settlementKey(assignmentId)), signal, 'Bridge settlement existing read', ); + if ( + existingSettlement != null && + existingSettlement !== serializedSettlement && + hasLaneGit(settlement.status === 'fulfilled' ? settlement.result : undefined) + ) { + // A retry of a committed settlement must stay recognizable whichever way this replica's + // lane Git setting differs from the replica that committed it. + const stripped = withoutLaneGit(settlement); + if (existingSettlement === JSON.stringify(stripped)) { + settlement = stripped; + serializedSettlement = existingSettlement; + } + } if ( existingSettlement != null && existingSettlement !== serializedSettlement @@ -1856,6 +1879,13 @@ export class RedisBridgeStore { 'Bridge assignment has expired', ); } + // Uses the assignment and registration already read above: no extra round trips on the + // settlement path, which has only a short grace after the command ran. + const sanitized = this.sanitizeLaneGit(assignmentId, settlement, assignment, registration); + if (sanitized !== settlement) { + settlement = sanitized; + serializedSettlement = JSON.stringify(settlement); + } const ttlSeconds = assignmentOutcomeTtlSeconds(assignment); const settlementKeys = [ assignmentKey(assignmentId), @@ -1964,6 +1994,38 @@ export class RedisBridgeStore { } } + /** + * Validate the optional `laneGit` of a fulfilled execute_command settlement before it is + * stored or forwarded. An invalid, unexpected or unadvertised value is dropped, never an + * error: the command's own result must still reach the caller. Pure over records the caller + * already read, so it adds no Redis round trips. + */ + private sanitizeLaneGit( + assignmentId: string, + settlement: AnyCodeBridgeSettlement, + assignment: StoredAssignment, + registration: RegisteredBridgeWorker | undefined, + ): AnyCodeBridgeSettlement { + if ( + settlement.status !== 'fulfilled' || + !hasLaneGit(settlement.result) || + assignment.executionKind !== 'workspace_tool' || + (assignment.request as WorkspaceToolRequest).operation !== 'execute_command' + ) { + return settlement; + } + const { dropped } = laneGitPolicy({ + result: settlement.result as unknown as Record, + enabled: this.laneGitEnabled, + advertised: + registration?.capabilities.workspaceTools?.commandResultFeatures?.includes('lane_git') === true, + }); + if (dropped === undefined) return settlement; + // Reason only: never the branch, the head or anything else from the result. + logger.debug('Dropped lane Git from a command settlement', { reason: dropped, assignmentId }); + return withoutLaneGit(settlement); + } + async cancelled( workerId: string, incarnationId: string, diff --git a/service/src/config.ts b/service/src/config.ts index 7ba8b9b4..b3e397b8 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -420,6 +420,11 @@ export const env = { BRIDGE_MAX_WORKSPACE_LEASE_SLOTS: Number( process.env.CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS ?? 1, ), + /** + * Offer `lane_git` to workers and accept `laneGit` (`{ branch, head }`) on command results. + * Off by default: enable only after the LibreChat talking to this service accepts the field. + */ + BRIDGE_LANE_GIT: process.env.CODEAPI_BRIDGE_LANE_GIT === 'true', /** Outbound worker selected by the remote-bridge backend. */ BRIDGE_WORKER_ID: process.env.CODEAPI_BRIDGE_WORKER_ID ?? '', /** Static compatibility auth or short-lived proof-of-possession credentials. */ diff --git a/tests/compose-bridge-config.cjs b/tests/compose-bridge-config.cjs index 8b3abd1d..bafea2fe 100644 --- a/tests/compose-bridge-config.cjs +++ b/tests/compose-bridge-config.cjs @@ -15,6 +15,7 @@ function render(overrides) { CODEAPI_BRIDGE_WORKER_ID: '', CODEAPI_BRIDGE_TOKEN: '', CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS: '', + CODEAPI_BRIDGE_LANE_GIT: '', CODEAPI_BRIDGE_RECOVERY_SERVER_ID: '', CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS: '', CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS: '', @@ -34,6 +35,7 @@ for (const overrides of [ CODEAPI_BRIDGE_DYNAMIC_WORKERS: 'false', CODEAPI_BRIDGE_WORKER_ID: 'test-worker', CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS: '4', + CODEAPI_BRIDGE_LANE_GIT: 'true', CODEAPI_BRIDGE_RECOVERY_SERVER_ID: 'https://code.example.test', CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS: '86400', CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS: '90', @@ -49,6 +51,7 @@ for (const overrides of [ assert.equal(env.CODEAPI_BRIDGE_AUTH_MODE, 'paired'); assert.equal(env.CODEAPI_BRIDGE_TOKEN, token); assert.equal(env.CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS, overrides.CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS ?? '1'); + assert.equal(env.CODEAPI_BRIDGE_LANE_GIT, overrides.CODEAPI_BRIDGE_LANE_GIT ?? 'false'); assert.equal(env.CODEAPI_BRIDGE_DYNAMIC_WORKERS, overrides.CODEAPI_BRIDGE_DYNAMIC_WORKERS ?? 'true'); assert.equal(env.CODEAPI_BRIDGE_WORKER_ID, overrides.CODEAPI_BRIDGE_WORKER_ID ?? ''); assert.equal(env.CODEAPI_BRIDGE_RECOVERY_SERVER_ID, overrides.CODEAPI_BRIDGE_RECOVERY_SERVER_ID ?? ''); From 68c9f931b67fea8d8d96eb64b869dc6d70d04466 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Mon, 5 Oct 2026 18:02:38 -0400 Subject: [PATCH 38/48] fix: Relay Loopback Resolvers Into KVM Guests (#314) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🧭 fix: Relay Loopback Resolvers Into KVM Guests The KVM launcher forwards the runner's /etc/resolv.conf into the libkrun guest. Under Docker Compose that names the embedded resolver 127.0.0.11, but the TSI guest kernel keeps loopback traffic on its own loopback device, so every service-name lookup in the guest failed. Generated-file uploads and session marker reads to egress_gateway were dropped with ConnectionRefused or FailedToOpenSocket, while connections by IP worked. When the runner has a loopback nameserver, relay it over UDP and TCP port 53 from the runner's own IPv4 address with socat and hand the guest that address instead. Further loopback nameservers are dropped, routable resolvers such as Kubernetes cluster DNS are forwarded unchanged, and startup fails if the relay cannot listen or the runner has no routable address. * 🧭 fix: Bound the Guest DNS Relay and Relay IPv6 Loopback Give both relays an idle timeout and cap concurrent TCP children. socat 1.8 fails every UDP-RECVFROM child given max-children, so the UDP relay relies on its short idle timeout. Treat a `::1` nameserver as loopback too, relaying it over IPv6 from the same runner address instead of forwarding an address the guest would only answer locally. * 🧭 fix: Recognize Every Loopback Resolver Spelling Classify nameservers as loopback by value rather than by the canonical `::1` string: zero-padded or compressed IPv6 loopback and IPv4-mapped 127.0.0.0/8 are relayed too, forwarding over IPv6 to the address exactly as written. Routable IPv6 resolvers stay untouched. * 🧭 fix: Parse Nameserver Addresses Before Loopback Detection Replace string matching with real parsing. IPv6 addresses are expanded to their eight groups (compressed zeros, dotted IPv4 tails), and IPv4 addresses are read in the inet_aton(3) forms glibc accepts in resolv.conf. Then ::1 and IPv4-mapped or plain 127.0.0.0/8 are relayed in any spelling. Mapped addresses are relayed over IPv4 to their canonical form. Spellings of the same resolver collapse into one relayed entry, and addresses glibc rejects are forwarded unchanged. --- README.md | 11 +++ launcher/entrypoint.sh | 167 ++++++++++++++++++++++++++++++++++++++++- tests/kvm_guest_dns.sh | 115 +++++++++++++++++++++++++--- 3 files changed, 282 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index db4038c4..94ad3b9c 100644 --- a/README.md +++ b/README.md @@ -245,6 +245,17 @@ read-only root disk does not need modification at boot. Rebuild the runner image to pick up this layout change. A missing resolver handoff fails startup rather than leaving the guest with an unrelated public DNS server. +The guest kernel keeps loopback traffic on its own loopback device instead of +proxying it through TSI, so it cannot reach a loopback resolver such as Docker's +embedded `127.0.0.11`. The launcher entrypoint relays the first loopback +nameserver (`127.0.0.0/8`, `::1` or IPv4-mapped `127.0.0.0/8`, in any spelling +glibc accepts) over UDP and TCP port 53 from the runner's +own IPv4 address with `socat`, and forwards that address to the guest in its +place; further loopback nameservers are dropped, and startup fails if the relay +cannot listen. Relay children exit when idle, and the TCP relay caps its +concurrent children. Routable nameservers, including Kubernetes cluster DNS, are forwarded +unchanged. + libkrun delivers the guest environment on the kernel command line, which only carries single-line printable ASCII and is capped at 2048 bytes by the guest kernel. The launcher entrypoint therefore forwards only the `nameserver`, diff --git a/launcher/entrypoint.sh b/launcher/entrypoint.sh index 304858e4..47fba9e6 100644 --- a/launcher/entrypoint.sh +++ b/launcher/entrypoint.sh @@ -12,15 +12,129 @@ set -e # joined by a separator that api/src/guest-dns.sh expands back into lines. RESOLV_FIELD_SEPARATOR='|' +# The guest kernel keeps loopback traffic on its own loopback device instead of +# proxying it through TSI, so a loopback resolver such as Docker's embedded +# 127.0.0.11 never answers inside the guest. Relay it from this container's own +# address, which the guest reaches through TSI, and hand the guest that address. +# The relay forks per datagram and per connection, so bound how long an idle +# child may live, and how many TCP children may run at once. socat 1.8 fails +# every UDP-RECVFROM child given max-children, so the UDP relay relies on its +# short idle timeout. Only the runner network reaches the relay: NsJail jobs in +# the guest run in their own network namespace. +RESOLVER_RELAY_READY_ATTEMPTS=50 +RESOLVER_RELAY_MAX_CHILDREN=64 +RESOLVER_RELAY_UDP_IDLE_SECONDS=5 +RESOLVER_RELAY_TCP_IDLE_SECONDS=30 + +# One inet_aton(3) part: hex with 0x, octal with a leading 0, else decimal. +inet_aton_part() { + case "$1" in + 0x*) [[ "$1" =~ ^0x[0-9a-f]+$ ]] && echo $((16#${1#0x})) ;; + 0*) [[ "$1" =~ ^0[0-7]*$ ]] && echo $((8#$1)) ;; + *) [[ "$1" =~ ^[0-9]+$ ]] && echo $((10#$1)) ;; + esac +} + +# Numeric value of an IPv4 address in the inet_aton(3) forms glibc accepts in +# resolv.conf: one to four parts, the last filling the remaining bytes. +ipv4_value() { + local parts part value=0 index last + IFS=. read -ra parts <<< "$1" + [[ "$1" != *. ]] && (( ${#parts[@]} >= 1 && ${#parts[@]} <= 4 )) || return 1 + last=$(( ${#parts[@]} - 1 )) + for ((index = 0; index < last; index++)); do + part="$(inet_aton_part "${parts[index]}")" || return 1 + (( part <= 255 )) || return 1 + value=$(( value | part << (24 - 8 * index) )) + done + part="$(inet_aton_part "${parts[last]}")" || return 1 + (( part < 1 << (32 - 8 * last) )) || return 1 + echo $(( value | part )) +} + +# The eight 16-bit groups of an IPv6 address in decimal, accepting compressed +# zeros and a dotted IPv4 tail. +ipv6_groups() { + local address="$1" head tail octets octet group groups=() tail_groups=() fill + if [[ "$address" =~ ^(.*:)([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)$ ]]; then + address="${BASH_REMATCH[1]}" + IFS=. read -ra octets <<< "${BASH_REMATCH[2]}" + for octet in "${octets[@]}"; do + [[ "$octet" =~ ^(0|[1-9][0-9]{0,2})$ ]] && (( octet <= 255 )) || return 1 + done + address+="$(printf '%x:%x' $(( octets[0] << 8 | octets[1] )) $(( octets[2] << 8 | octets[3] )))" + fi + [[ "$address" =~ ^[0-9a-f:]+$ && "$address" != :[!:]* && "$address" != *[!:]: && "$address" != *:::* ]] || return 1 + if [[ "$address" == *::* ]]; then + head="${address%%::*}" + tail="${address#*::}" + [[ "$tail" != *::* ]] || return 1 + [ -z "$head" ] || IFS=: read -ra groups <<< "$head" + [ -z "$tail" ] || IFS=: read -ra tail_groups <<< "$tail" + fill=$(( 8 - ${#groups[@]} - ${#tail_groups[@]} )) + (( fill >= 1 )) || return 1 + for ((; fill > 0; fill--)); do groups+=(0); done + groups+=("${tail_groups[@]}") + else + IFS=: read -ra groups <<< "$address" + fi + (( ${#groups[@]} == 8 )) || return 1 + for group in "${groups[@]}"; do + [[ "$group" =~ ^[0-9a-f]{1,4}$ ]] || return 1 + printf '%d ' $((16#$group)) + done +} + +# Canonical form of a loopback nameserver in any spelling glibc accepts: IPv4 +# 127.0.0.0/8, IPv6 ::1, or IPv4-mapped 127.0.0.0/8 (relayed as IPv4). Fails for +# every other address. +canonical_loopback() { + local address="${1,,}" text groups value + address="${address%%\%*}" + if [[ "$address" == *:* ]]; then + text="$(ipv6_groups "$address")" || return 1 + read -ra groups <<< "$text" + if [ "${groups[*]}" = '0 0 0 0 0 0 0 1' ]; then + echo ::1 + return 0 + fi + [ "${groups[*]:0:6}" = '0 0 0 0 0 65535' ] || return 1 + value=$(( groups[6] << 16 | groups[7] )) + else + value="$(ipv4_value "$address")" || return 1 + fi + (( value >> 24 == 127 )) || return 1 + printf '%d.%d.%d.%d' $(( value >> 24 )) $(( value >> 16 & 255 )) $(( value >> 8 & 255 )) $(( value & 255 )) +} + +loopback_nameserver() { + local line words + while IFS= read -r line || [ -n "$line" ]; do + read -ra words <<< "${line%$'\r'}" + if [ "${words[0]:-}" = nameserver ] && canonical_loopback "${words[1]:-}"; then + return 0 + fi + done < "$1" +} + +runner_address() { + getent ahostsv4 "$(cat /proc/sys/kernel/hostname)" | awk '$1 !~ /^127\./ { print $1; exit }' +} + encode_resolv_conf() { + local loopback="${1:-}" relay="${2:-}" local LC_ALL=C - local line words encoded='' + local line words canonical encoded='' while IFS= read -r line || [ -n "$line" ]; do line="${line%$'\r'}" if [[ ! "$line" =~ ^[[:space:]]*(nameserver|search|domain|options|sortlist)[[:space:]] ]]; then continue fi read -ra words <<< "$line" + if [ "${words[0]}" = nameserver ] && canonical="$(canonical_loopback "${words[1]}")"; then + [ "$canonical" = "$loopback" ] || continue + words[1]="$relay" + fi line="${words[*]}" if [[ "$line" == *[!' '-'~']* || "$line" == *[\"$RESOLV_FIELD_SEPARATOR]* ]]; then echo "ERROR: runner /etc/resolv.conf line cannot cross the kernel command line: $line" >&2 @@ -31,13 +145,62 @@ encode_resolv_conf() { printf '%s' "$encoded" } -SANDBOX_RESOLV_CONF="$(encode_resolv_conf < /etc/resolv.conf)" +proc_net_address() { + local a b c d + IFS=. read -r a b c d <<< "$1" + printf '%02X%02X%02X%02X:0035' "$d" "$c" "$b" "$a" +} + +start_resolver_relay() { + local nameserver="$1" address="$2" local_address udp_pid tcp_pid attempt + local udp_target="UDP4-SENDTO:$nameserver:53" tcp_target="TCP4:$nameserver:53" + if [ "$nameserver" = ::1 ]; then + udp_target='UDP6-SENDTO:[::1]:53' + tcp_target='TCP6:[::1]:53' + fi + socat -T "$RESOLVER_RELAY_UDP_IDLE_SECONDS" \ + "UDP4-RECVFROM:53,bind=$address,fork" "$udp_target" & + udp_pid=$! + socat -T "$RESOLVER_RELAY_TCP_IDLE_SECONDS" \ + "TCP4-LISTEN:53,bind=$address,reuseaddr,fork,max-children=$RESOLVER_RELAY_MAX_CHILDREN" "$tcp_target" & + tcp_pid=$! + local_address="$(proc_net_address "$address")" + for ((attempt = 0; attempt < RESOLVER_RELAY_READY_ATTEMPTS; attempt++)); do + if grep -q " $local_address " /proc/net/udp && grep -q " $local_address 00000000:0000 0A " /proc/net/tcp; then + echo "Relaying guest DNS from $address:53 to loopback resolver $nameserver" + return 0 + fi + if ! kill -0 "$udp_pid" "$tcp_pid" 2>/dev/null; then + break + fi + sleep 0.1 + done + kill "$udp_pid" "$tcp_pid" 2>/dev/null || true + echo "ERROR: guest DNS relay could not listen on $address:53 for loopback resolver $nameserver" >&2 + return 1 +} + +LOOPBACK_NAMESERVER="$(loopback_nameserver /etc/resolv.conf)" +RESOLVER_RELAY_ADDRESS='' +if [ -n "$LOOPBACK_NAMESERVER" ]; then + RESOLVER_RELAY_ADDRESS="$(runner_address)" + if [ -z "$RESOLVER_RELAY_ADDRESS" ]; then + echo "ERROR: runner has no non-loopback IPv4 address to relay loopback resolver $LOOPBACK_NAMESERVER" >&2 + exit 1 + fi +fi + +SANDBOX_RESOLV_CONF="$(encode_resolv_conf "$LOOPBACK_NAMESERVER" "$RESOLVER_RELAY_ADDRESS" < /etc/resolv.conf)" export SANDBOX_RESOLV_CONF if [[ "$RESOLV_FIELD_SEPARATOR$SANDBOX_RESOLV_CONF" != *"${RESOLV_FIELD_SEPARATOR}nameserver "[!\#]* ]]; then echo 'ERROR: runner /etc/resolv.conf has no nameserver' >&2 exit 1 fi +if [ -n "$LOOPBACK_NAMESERVER" ]; then + start_resolver_relay "$LOOPBACK_NAMESERVER" "$RESOLVER_RELAY_ADDRESS" +fi + if [ "${LAUNCHER_FILTER_VSOCK_ENOTCONN:-true}" = "true" ]; then # libkrun can emit this benign TSI/vsock teardown line after the guest has # already closed its side of the socket. It contains the word "error", so diff --git a/tests/kvm_guest_dns.sh b/tests/kvm_guest_dns.sh index 0e038a12..4ea99afb 100755 --- a/tests/kvm_guest_dns.sh +++ b/tests/kvm_guest_dns.sh @@ -80,31 +80,60 @@ printf '%s\n' "$SANDBOX_RESOLV_CONF" > "$TEST_RESOLVER_OUTPUT" STUB cat > "$TEST_DIR/bin/getent" <<'STUB' #!/usr/bin/env bash -printf '192.0.2.99 stale-address\n' +[[ -z "${TEST_RUNNER_ADDRESS-192.0.2.99}" ]] || printf '%s stale-address\n' "${TEST_RUNNER_ADDRESS-192.0.2.99}" STUB -chmod +x "$TEST_DIR/bin/launcher" "$TEST_DIR/bin/getent" +# The relay stub records its endpoints; readiness comes from the fake /proc/net. +cat > "$TEST_DIR/bin/socat" <<'STUB' +#!/usr/bin/env bash +printf '%s\n' "$*" >> "$TEST_SOCAT_CALLS" +STUB +chmod +x "$TEST_DIR/bin/launcher" "$TEST_DIR/bin/getent" "$TEST_DIR/bin/socat" +# 192.0.2.99:53 bound for UDP and listening for TCP, in /proc/net byte order. +mkdir "$TEST_DIR/proc-net-bound" "$TEST_DIR/proc-net-unbound" +printf ' sl local_address rem_address st\n 0: 630200C0:0035 00000000:0000 07 00000000:00000000\n' > "$TEST_DIR/proc-net-bound/udp" +printf ' sl local_address rem_address st\n 0: 630200C0:0035 00000000:0000 0A 00000000:00000000\n' > "$TEST_DIR/proc-net-bound/tcp" +printf ' sl local_address rem_address st\n' | tee "$TEST_DIR/proc-net-unbound/udp" > "$TEST_DIR/proc-net-unbound/tcp" # libkrun appends every guest environment entry to the kernel command line, # which linux-loader limits to single-line printable ASCII. Docker's generated # resolv.conf is multi-line with comments, so the entrypoint must forward only # the resolver directives, joined by the separator guest-dns.sh expands. run_entrypoint() { - local resolv_conf="$1" + local resolv_conf="$1" proc_net="${2:-$TEST_DIR/proc-net-bound}" sed -e "s|/usr/local/bin/launcher|$TEST_DIR/bin/launcher|g" \ -e "s|/etc/resolv.conf|$resolv_conf|g" \ + -e "s|/proc/net/|$proc_net/|g" \ + -e 's|RESOLVER_RELAY_READY_ATTEMPTS=50|RESOLVER_RELAY_READY_ATTEMPTS=3|' \ "$ROOT/launcher/entrypoint.sh" > "$TEST_DIR/entrypoint.sh" - rm -f "$TEST_DIR/forwarded" + rm -f "$TEST_DIR/forwarded" "$TEST_DIR/socat-calls" PATH="$TEST_DIR/bin:$PATH" \ EGRESS_GATEWAY_URL='https://egress_gateway:3190/base' \ FILE_SERVER_URL='http://[::1]:3000/base' \ SANDBOX_FORWARD_TARGET='tool_call_server:3033' \ LAUNCHER_FILTER_VSOCK_ENOTCONN=false \ TEST_RESOLVER_OUTPUT="$TEST_DIR/forwarded" \ + TEST_SOCAT_CALLS="$TEST_DIR/socat-calls" \ bash "$TEST_DIR/entrypoint.sh" } # Independent reference for the expected handoff: directives only, whitespace -# collapsed, joined by the separator. +# collapsed, joined by the separator. The first loopback nameserver becomes the +# relay address; other loopback nameservers are unreachable and dropped. encoded_reference() { - LC_ALL=C awk '/^[[:space:]]*(nameserver|search|domain|options|sortlist)[[:space:]]/ { sub(/\r$/, ""); $1 = $1; print }' "$1" | paste -sd '|' - + LC_ALL=C awk -v relay=192.0.2.99 '/^[[:space:]]*(nameserver|search|domain|options|sortlist)[[:space:]]/ { + sub(/\r$/, ""); $1 = $1 + address = tolower($2) + if ($1 == "nameserver" && (address ~ /^127\./ || address ~ /^[0:]*:0*1$/ || address ~ /^::ffff:127\./)) { if (loop == "") loop = $2; if ($2 != loop) next; $2 = relay } + print + }' "$1" | paste -sd '|' - +} +# The relays start in the background, so wait for both before comparing. Both +# bound idle children; the TCP listener also caps concurrent children. +assert_relayed() { + local udp_target="$1" tcp_target="$2" attempt + for ((attempt = 0; attempt < 50; attempt++)); do + [[ -e "$TEST_DIR/socat-calls" && "$(wc -l < "$TEST_DIR/socat-calls")" -ge 2 ]] && break + sleep 0.1 + done + [[ "$(LC_ALL=C sort "$TEST_DIR/socat-calls")" == "$(printf -- '-T 30 TCP4-LISTEN:53,bind=192.0.2.99,reuseaddr,fork,max-children=64 %s\n-T 5 UDP4-RECVFROM:53,bind=192.0.2.99,fork %s' "$tcp_target" "$udp_target")" ]] } cat > "$TEST_DIR/docker-resolv.conf" <<'RESOLV' # Generated by Docker Engine. @@ -119,24 +148,92 @@ options ndots:0 # Overrides: [] # Option ndots from: internal RESOLV +# The guest keeps loopback traffic local, so Docker's embedded resolver is +# relayed from the runner address and the guest gets that address instead. run_entrypoint "$TEST_DIR/docker-resolv.conf" -[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 127.0.0.11|options ndots:0' ]] +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99|options ndots:0' ]] [[ "$(cat "$TEST_DIR/forwarded")" == "$(encoded_reference "$TEST_DIR/docker-resolv.conf")" ]] +assert_relayed UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 [[ "$(wc -l < "$TEST_DIR/forwarded")" == 1 ]] if LC_ALL=C grep -q '[^ -~]' "$TEST_DIR/forwarded"; then echo 'forwarded resolver contains bytes the kernel command line rejects' >&2; exit 1 fi -# The guest restores exactly the directives Docker supplied. +# The guest restores the Docker directives with the relayed nameserver. rm -rf "$TEST_DIR/guest/run/codeapi-resolver" SANDBOX_RESOLV_CONF="$(cat "$TEST_DIR/forwarded")" configure_guest_dns "$TEST_DIR/guest" -printf 'nameserver 127.0.0.11\noptions ndots:0\n' > "$TEST_DIR/expected" +printf 'nameserver 192.0.2.99\noptions ndots:0\n' > "$TEST_DIR/expected" cmp "$TEST_DIR/expected" "$TEST_DIR/guest/etc/resolv.conf" +# Only one loopback resolver can be relayed; other loopback entries would time +# out in the guest, while routable nameservers keep their order. +printf 'nameserver 127.0.0.53\r\nnameserver 10.0.0.2\nnameserver\t127.0.0.11\nsearch example.internal\n' > "$TEST_DIR/multi-loopback-resolv.conf" +run_entrypoint "$TEST_DIR/multi-loopback-resolv.conf" +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99|nameserver 10.0.0.2|search example.internal' ]] +[[ "$(cat "$TEST_DIR/forwarded")" == "$(encoded_reference "$TEST_DIR/multi-loopback-resolv.conf")" ]] +assert_relayed UDP4-SENDTO:127.0.0.53:53 TCP4:127.0.0.53:53 + +# An IPv6 loopback resolver is just as unreachable from the guest; the IPv4 +# relay forwards to it over IPv6. +printf 'nameserver ::1\noptions ndots:0\n' > "$TEST_DIR/ipv6-loopback-resolv.conf" +run_entrypoint "$TEST_DIR/ipv6-loopback-resolv.conf" +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99|options ndots:0' ]] +[[ "$(cat "$TEST_DIR/forwarded")" == "$(encoded_reference "$TEST_DIR/ipv6-loopback-resolv.conf")" ]] +assert_relayed 'UDP6-SENDTO:[::1]:53' 'TCP6:[::1]:53' +printf 'nameserver 127.0.0.11\nnameserver ::1\n' > "$TEST_DIR/mixed-loopback-resolv.conf" +run_entrypoint "$TEST_DIR/mixed-loopback-resolv.conf" +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99' ]] +assert_relayed UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +# Every spelling glibc accepts for a loopback resolver is relayed to its +# canonical address: compressed or zero-padded ::1, IPv4-mapped 127/8 in any +# IPv6 form, and inet_aton(3) IPv4 forms. +while read -r spelling udp_target tcp_target; do + printf 'nameserver %s\nnameserver 10.0.0.2\n' "$spelling" > "$TEST_DIR/spelled-loopback-resolv.conf" + run_entrypoint "$TEST_DIR/spelled-loopback-resolv.conf" + [[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99|nameserver 10.0.0.2' ]] + assert_relayed "$udp_target" "$tcp_target" +done <<'SPELLINGS' +0:0:0:0:0:0:0:1 UDP6-SENDTO:[::1]:53 TCP6:[::1]:53 +0000::0001 UDP6-SENDTO:[::1]:53 TCP6:[::1]:53 +::1%lo UDP6-SENDTO:[::1]:53 TCP6:[::1]:53 +::FFFF:127.0.0.11 UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +0:0:0:0:0:ffff:127.0.0.11 UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +::ffff:7f00:b UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +127.1 UDP4-SENDTO:127.0.0.1:53 TCP4:127.0.0.1:53 +2130706443 UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +0x7f.0.0.0xb UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +0177.0.0.013 UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +SPELLINGS +# Spellings of the same loopback resolver collapse into one relayed entry. +printf 'nameserver 127.0.0.11\nnameserver ::ffff:7f00:b\nnameserver 127.0.0.53\n' > "$TEST_DIR/same-loopback-resolv.conf" +run_entrypoint "$TEST_DIR/same-loopback-resolv.conf" +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99|nameserver 192.0.2.99' ]] +assert_relayed UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +# Routable resolvers and addresses glibc rejects are forwarded unchanged, without +# a relay: 0127 is octal 87, and ::7f00:1 is not IPv4-mapped. +printf 'nameserver 2001:db8::1\nnameserver fd00::11\nnameserver ::ffff:10.0.0.2\nnameserver ::7f00:1\nnameserver 0127.0.0.1\nnameserver 127.0.0.256\nnameserver 1:2:3:4:5:6:7:8:\nnameserver 127..1\n' > "$TEST_DIR/routable-resolv.conf" +run_entrypoint "$TEST_DIR/routable-resolv.conf" +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 2001:db8::1|nameserver fd00::11|nameserver ::ffff:10.0.0.2|nameserver ::7f00:1|nameserver 0127.0.0.1|nameserver 127.0.0.256|nameserver 1:2:3:4:5:6:7:8:|nameserver 127..1' ]] +[[ ! -e "$TEST_DIR/socat-calls" ]] + +# A relay that never listens, or a runner without a routable address, fails +# startup instead of booting a guest that cannot resolve service names. +if run_entrypoint "$TEST_DIR/docker-resolv.conf" "$TEST_DIR/proc-net-unbound" 2> "$TEST_DIR/entrypoint-error"; then + echo 'started launcher without a listening resolver relay' >&2; exit 1 +fi +grep -q 'guest DNS relay could not listen on 192.0.2.99:53' "$TEST_DIR/entrypoint-error" +[[ ! -e "$TEST_DIR/forwarded" ]] +if TEST_RUNNER_ADDRESS='' run_entrypoint "$TEST_DIR/docker-resolv.conf" 2> "$TEST_DIR/entrypoint-error"; then + echo 'started launcher without a resolver relay address' >&2; exit 1 +fi +grep -q 'no non-loopback IPv4 address to relay loopback resolver 127.0.0.11' "$TEST_DIR/entrypoint-error" +[[ ! -e "$TEST_DIR/forwarded" && ! -e "$TEST_DIR/socat-calls" ]] + # Kubernetes resolvers: tabs, CRLF, trailing spaces, and unknown keywords are # normalized away; search domains and options survive intact. printf 'nameserver\t10.96.0.10 \r\nsearch tenant.svc.cluster.local svc.cluster.local cluster.local\n; resolver comment\nlookup file bind\noptions ndots:5\n' > "$TEST_DIR/k8s-resolv.conf" # leak-check:allow run_entrypoint "$TEST_DIR/k8s-resolv.conf" [[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 10.96.0.10|search tenant.svc.cluster.local svc.cluster.local cluster.local|options ndots:5' ]] # leak-check:allow +[[ ! -e "$TEST_DIR/socat-calls" ]] # The runner's own resolver must round-trip through the same reference. run_entrypoint /etc/resolv.conf From a62d69e05a4541ac9a20722ac7ff3fcfecb6ed24 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 6 Oct 2026 01:21:55 -0400 Subject: [PATCH 39/48] =?UTF-8?q?=F0=9F=94=92=20Harden=20Runner=20Supervis?= =?UTF-8?q?ors=20with=20Anonymous=20Pipes=20(#315)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Harden runner supervisors with anonymous pipes and inherited seccomp * Load hosted-app netfilter modules in native CI and retain diagnostics * Keep relay idle on buffered pipe EOF and assert owned descriptor cleanup * Preserve supervisor DNS resolution without descriptor-send syscalls --- .github/workflows/ci.yml | 36 ++++++ api/Dockerfile | 16 ++- api/src/anonymous-pipes.c | 34 ++++++ api/src/entrypoint.sh | 4 +- api/src/hosted-app.ts | 26 ++++- api/src/nsjail.ts | 3 +- api/src/pipe-process.test.ts | 89 +++++++++++++++ api/src/pipe-process.ts | 116 ++++++++++++++++++++ api/src/session-checkpoint.ts | 2 +- api/src/supervisor-policy.c | 60 ++++++++++ api/src/tool-call-pipe-bridge.c | 40 ++++--- api/src/tool-call-pipe-broker.ts | 19 +++- docker/Dockerfile.worker-sandbox | 15 ++- docker/nsjail-anonymous-control.patch | 82 ++++++++++++++ docs/adr/0013-supervisor-anonymous-pipes.md | 65 +++++++++++ launcher/Dockerfile | 15 ++- tests/pipe_relay_backpressure.py | 55 ++++++++++ tests/supervisor_dns.py | 49 +++++++++ tests/supervisor_hosted_network.ts | 42 +++++++ tests/supervisor_policy.py | 51 +++++++++ tests/tool_call_runner.cjs | 2 +- 21 files changed, 784 insertions(+), 37 deletions(-) create mode 100644 api/src/anonymous-pipes.c create mode 100644 api/src/pipe-process.test.ts create mode 100644 api/src/pipe-process.ts create mode 100644 api/src/supervisor-policy.c create mode 100644 docker/nsjail-anonymous-control.patch create mode 100644 docs/adr/0013-supervisor-anonymous-pipes.md create mode 100644 tests/pipe_relay_backpressure.py create mode 100644 tests/supervisor_dns.py create mode 100644 tests/supervisor_hosted_network.ts create mode 100644 tests/supervisor_policy.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98ddb2ff..50804e27 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,6 +106,16 @@ jobs: - name: Install dependencies run: bun ci + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + with: + node-version: 22 + + - name: Compile anonymous-pipe addon + run: | + mkdir -p .build + NODE_HEADERS="$(dirname "$(dirname "$(command -v node)")")/include/node" + gcc -shared -fPIC -O2 -Wall -Wextra -Werror -I"$NODE_HEADERS" -o .build/anonymous-pipes.node src/anonymous-pipes.c + - name: Build API run: bun run build @@ -174,6 +184,32 @@ jobs: --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ --entrypoint /usr/local/bin/node \ launcher-guest-test /tests/pipe_runtime_image.cjs + - name: Load legacy netfilter modules for hosted-app isolation + run: sudo modprobe -a ip_tables ip6_tables iptable_filter ip6table_filter xt_conntrack xt_owner ipt_REJECT ip6t_REJECT + - name: Verify supervisor filter and anonymous stdio + run: | + docker run --rm --init --security-opt seccomp=unconfined \ + --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ + nsjail-ipc-test python3 /tests/supervisor_policy.py + docker run --rm --init --security-opt seccomp=unconfined \ + --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ + nsjail-ipc-test python3 /tests/pipe_relay_backpressure.py + docker run --rm --init --dns 127.0.0.1 --security-opt seccomp=unconfined \ + --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ + nsjail-ipc-test python3 /tests/supervisor_dns.py + docker run --rm --init --security-opt seccomp=unconfined \ + --mount "type=bind,source=$GITHUB_WORKSPACE,target=/work,readonly" \ + --entrypoint /usr/local/bin/sandbox-supervisor-policy \ + nsjail-ipc-test bun test /work/api/src/pipe-process.test.ts + docker run --rm --init --security-opt seccomp=unconfined \ + --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ + --entrypoint /usr/local/bin/sandbox-supervisor-policy \ + launcher-guest-test node /tests/pipe_runtime_image.cjs + docker run --rm --init --cap-add NET_ADMIN --security-opt seccomp=unconfined \ + --mount "type=bind,source=$GITHUB_WORKSPACE,target=/work,readonly" \ + --entrypoint /usr/local/bin/sandbox-supervisor-policy \ + launcher-guest-test bun /work/tests/supervisor_hosted_network.ts + - name: Verify descriptor denial and runtime IPC with the installed filter run: | docker run --rm --init --security-opt seccomp=unconfined \ diff --git a/api/Dockerfile b/api/Dockerfile index f953f537..d630c547 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -26,18 +26,27 @@ RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \ && git checkout 66ad78dc361721512fbf9470488e584364c75489 \ && git submodule update --init --recursive -RUN make -j$(nproc) +COPY docker/nsjail-anonymous-control.patch /tmp/nsjail-anonymous-control.patch +# Network-isolated jobs need no libnl transport; removing it avoids sendmsg +# in the supervisor and fails closed for interface/traffic-rule configuration. +RUN git apply /tmp/nsjail-anonymous-control.patch && make -j$(nproc) NL3_EXISTS=no COPY api/src/spec-guard.c /tmp/spec-guard.c COPY api/src/tool-call-pipe-bridge.c /tmp/tool-call-pipe-bridge.c +COPY api/src/anonymous-pipes.c /tmp/anonymous-pipes.c +COPY api/src/supervisor-policy.c /tmp/supervisor-policy.c +COPY --from=node:22-slim /usr/local/include/node /usr/local/include/node COPY docker/rootfs-setup.c /tmp/rootfs-setup.c -RUN gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/tool-call-pipe-bridge /tmp/tool-call-pipe-bridge.c \ +RUN gcc -shared -fPIC -O2 -Wall -Wextra -Werror -I/usr/local/include/node -o /usr/local/lib/anonymous-pipes.node /tmp/anonymous-pipes.c \ + && gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/sandbox-supervisor-policy /tmp/supervisor-policy.c \ + && gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/tool-call-pipe-bridge /tmp/tool-call-pipe-bridge.c \ && gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ && gcc -O2 -static -o /usr/local/bin/sandbox-rootfs-setup /tmp/rootfs-setup.c \ && chmod 0111 /usr/local/bin/spec-guard /usr/local/bin/sandbox-rootfs-setup # Native policy/transport verification without building the full runner. FROM nsjail-builder AS nsjail-ipc-test +ENV SANDBOX_PIPE_ADDON=/usr/local/lib/anonymous-pipes.node COPY docker/python-runtime /python-runtime RUN python3 /python-runtime/install.py COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node @@ -121,6 +130,9 @@ RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen COPY --from=nsjail-builder /nsjail/nsjail /usr/sbin/nsjail COPY --from=nsjail-builder /usr/local/bin/spec-guard /usr/local/bin/spec-guard COPY --from=nsjail-builder /usr/local/bin/tool-call-pipe-bridge /usr/local/bin/tool-call-pipe-bridge +COPY --from=nsjail-builder /usr/local/bin/sandbox-supervisor-policy /usr/local/bin/sandbox-supervisor-policy +COPY --from=nsjail-builder /usr/local/lib/anonymous-pipes.node /usr/local/lib/anonymous-pipes.node +ENV SANDBOX_PIPE_ADDON=/usr/local/lib/anonymous-pipes.node RUN chmod +x /usr/sbin/nsjail # The per-invocation broker starts Node after restore, so its OpenSSL process diff --git a/api/src/anonymous-pipes.c b/api/src/anonymous-pipes.c new file mode 100644 index 00000000..524c3444 --- /dev/null +++ b/api/src/anonymous-pipes.c @@ -0,0 +1,34 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include + +static napi_value create_pipe(napi_env env, napi_callback_info info) { + (void)info; + int fds[2]; + if (pipe2(fds, O_CLOEXEC) != 0) { + napi_throw_error(env, NULL, strerror(errno)); + return NULL; + } + napi_value array, reader, writer; + if (napi_create_array_with_length(env, 2, &array) != napi_ok + || napi_create_int32(env, fds[0], &reader) != napi_ok + || napi_create_int32(env, fds[1], &writer) != napi_ok + || napi_set_element(env, array, 0, reader) != napi_ok + || napi_set_element(env, array, 1, writer) != napi_ok) { + close(fds[0]); close(fds[1]); + napi_throw_error(env, NULL, "cannot return anonymous pipe"); + return NULL; + } + return array; +} + +static napi_value initialize(napi_env env, napi_value exports) { + napi_value fn; + if (napi_create_function(env, "createPipe", NAPI_AUTO_LENGTH, create_pipe, NULL, &fn) != napi_ok + || napi_set_named_property(env, exports, "createPipe", fn) != napi_ok) return NULL; + return exports; +} +NAPI_MODULE(NODE_GYP_MODULE_NAME, initialize) diff --git a/api/src/entrypoint.sh b/api/src/entrypoint.sh index d670a359..0f93981b 100755 --- a/api/src/entrypoint.sh +++ b/api/src/entrypoint.sh @@ -178,7 +178,7 @@ if [ "$SANDBOX_USE_CGROUPV2" = "true" ]; then NSJAIL_CGROUP_ARGS=(--use_cgroupv2) fi -if timeout 10 "${NSJAIL_PATH:-/usr/sbin/nsjail}" --config "${NSJAIL_CONFIG:-/sandbox_api/config/sandbox.cfg}" \ +if timeout 10 /usr/local/bin/sandbox-supervisor-policy "${NSJAIL_PATH:-/usr/sbin/nsjail}" --config "${NSJAIL_CONFIG:-/sandbox_api/config/sandbox.cfg}" \ "${NSJAIL_CGROUP_ARGS[@]}" --log "$SMOKE_LOG" \ --user "65534:${SMOKE_OUTSIDE_UID}:1" --group "65534:${SMOKE_OUTSIDE_GID}:1" \ -s /usr/bin:/bin -s /usr/lib:/lib -s /usr/lib64:/lib64 \ @@ -201,4 +201,4 @@ rm -f "$SMOKE_STDERR" rm -rf "$SMOKE_DIR" echo "Starting sandbox API server..." -exec bun run /sandbox_api/.build/index.js +exec /usr/local/bin/sandbox-supervisor-policy bun run /sandbox_api/.build/index.js diff --git a/api/src/hosted-app.ts b/api/src/hosted-app.ts index 64a5aab8..0ce57370 100644 --- a/api/src/hosted-app.ts +++ b/api/src/hosted-app.ts @@ -1,10 +1,10 @@ -import { execFile, spawn, type ChildProcess, type SpawnOptions } from 'node:child_process'; +import { spawnPipeProcess as spawn } from './pipe-process'; +import type { ChildProcess, SpawnOptions } from 'node:child_process'; import { createHash } from 'node:crypto'; import * as fsp from 'node:fs/promises'; import * as net from 'node:net'; import * as path from 'node:path'; import type { Readable } from 'node:stream'; -import { promisify } from 'node:util'; import { config } from './config'; import { aggregateBashExtras, filterExtraEnvVars } from './job'; import { logger } from './logger'; @@ -12,7 +12,22 @@ import { getLatestRuntimeMatchingLanguageVersion, getRuntimes, type Runtime } fr import { getBoundSessionWorkspace, type SessionWorkspace } from './session-workspace'; import { ValidationError, validateFilePath } from './validation'; -const execFileAsync = promisify(execFile); +// Capture bounded diagnostics through an anonymous pipe; execFile's runtime +// capture would create socketpairs prohibited by the supervisor policy. +function execFileAsync(binary: string, args: string[]): Promise { + return new Promise((resolve, reject) => { + const child = spawn(binary, args, { stdio: ['ignore', 'ignore', 'pipe'] }); + let diagnostics = Buffer.alloc(0); + child.stderr.on('data', (chunk: Buffer) => { + diagnostics = Buffer.concat([diagnostics, chunk]).subarray(-8192); + }); + child.once('error', reject); + child.once('close', (code, signal) => { + if (code === 0) resolve(); + else reject(new Error(`${binary} failed (${signal ?? code}): ${diagnostics.toString().trim()}`)); + }); + }); +} const APP_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/; const REVISION_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; const ENV_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; @@ -342,7 +357,10 @@ async function commandSucceeds(binary: string, args: string[]): Promise * packets for accepted inbound connections remain allowed by conntrack. */ export async function installHostedAppNetworkGuard(uid: number): Promise { - for (const binary of ['/usr/sbin/iptables', '/usr/sbin/ip6tables']) { + // nft's netlink transport needs sendmsg, which the supervisor denies. The + // packaged legacy backend uses getsockopt/setsockopt instead. Failure to + // install either family remains fatal before admitting any hosted app. + for (const binary of ['/usr/sbin/iptables-legacy', '/usr/sbin/ip6tables-legacy']) { if (!(await commandSucceeds(binary, ['-w', '5', '-L', HOSTED_APP_EGRESS_CHAIN]))) { await execFileAsync(binary, ['-w', '5', '-N', HOSTED_APP_EGRESS_CHAIN]); } diff --git a/api/src/nsjail.ts b/api/src/nsjail.ts index 5c39884f..23c048a6 100644 --- a/api/src/nsjail.ts +++ b/api/src/nsjail.ts @@ -1,6 +1,7 @@ +import { spawnPipeProcess as spawn } from './pipe-process'; import * as fs from 'fs'; import * as path from 'path'; -import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'; +import type { ChildProcessWithoutNullStreams } from 'node:child_process'; import { nanoid } from 'nanoid'; import { config } from './config'; import { logger } from './logger'; diff --git a/api/src/pipe-process.test.ts b/api/src/pipe-process.test.ts new file mode 100644 index 00000000..608002c8 --- /dev/null +++ b/api/src/pipe-process.test.ts @@ -0,0 +1,89 @@ +import { describe, test, expect } from 'bun:test'; +import { once } from 'node:events'; +import { pipeline } from 'node:stream/promises'; +import { Readable } from 'node:stream'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { spawnPipeProcess } from './pipe-process'; + +async function collect(stream: Readable): Promise { + const chunks: Buffer[] = []; + for await (const chunk of stream) chunks.push(Buffer.from(chunk)); + return Buffer.concat(chunks); +} + +describe('anonymous process pipes', () => { + test('large concurrent streams preserve backpressure and output before close', async () => { + const bytes = Buffer.alloc(2 * 1024 * 1024, 'p'); + await Promise.all(Array.from({ length: 8 }, async () => { + const child = spawnPipeProcess('/bin/sh', ['-c', 'cat; printf stderr >&2']); + const closed = once(child, 'close'); + const output = collect(child.stdout); + const errors = collect(child.stderr); + await pipeline(Readable.from([bytes]), child.stdin); + const [code] = await closed; + expect(code).toBe(0); + expect(await output).toEqual(bytes); + expect((await errors).toString()).toBe('stderr'); + })); + }, 15000); + + test.skipIf(process.platform !== 'linux')('Linux children receive FIFO stdin, stdout and stderr', async () => { + const child = spawnPipeProcess('/bin/sh', ['-c', 'test -p /proc/self/fd/0 && test -p /proc/self/fd/1 && test -p /proc/self/fd/2']); + child.stdin.end(); + const [code] = await once(child, 'close'); + if (process.platform === 'linux') expect(code).toBe(0); + }); + + test('cancellation releases a blocked stdin writer', async () => { + const child = spawnPipeProcess('/bin/sleep', ['30']); + const closed = once(child, 'close'); + const writing = pipeline(Readable.from([Buffer.alloc(2 * 1024 * 1024)]), child.stdin); + // Observe immediately: cancellation may reject before the exit listener. + const rejected = writing.then(() => false, () => true); + await new Promise(resolve => setTimeout(resolve, 20)); + child.kill('SIGKILL'); + await closed; + expect(await rejected).toBe(true); + expect(child.stdin.destroyed).toBe(true); + }, 5000); + + test('spawn failures and killed children release every pipe', async () => { + const ownedPipes = new Set(); + const addon = process.platform === 'linux' + ? require(process.env.SANDBOX_PIPE_ADDON || path.resolve(__dirname, '..', '.build', 'anonymous-pipes.node')) : undefined; + const create = addon?.createPipe; + if (addon) addon.createPipe = () => { + const fds = create(); + for (const fd of fds) ownedPipes.add(fs.readlinkSync(`/proc/self/fd/${fd}`)); + return fds; + }; + try { + for (let n = 0; n < 12; n++) { + const missing = spawnPipeProcess('/missing-codeapi-executable'); + const missingClosed = new Promise(resolve => missing.once('close', () => resolve())); + const error = await new Promise(resolve => missing.once('error', resolve)); + expect(error.code).toBe('ENOENT'); + await missingClosed; + const child = spawnPipeProcess('/bin/sleep', ['30']); + const closed = once(child, 'close'); + child.kill('SIGKILL'); + await closed; + expect(child.stdin.destroyed).toBe(true); + expect(child.stdout.destroyed).toBe(true); + expect(child.stderr.destroyed).toBe(true); + } + } finally { + if (addon) addon.createPipe = create; + } + if (process.platform === 'linux') { + // Runtime watcher/pid descriptors can appear asynchronously. Assert the + // exact pipe inodes we allocated, including duplicates held by FileSink. + const survivors = fs.readdirSync('/proc/self/fd').flatMap(fd => { + try { const inode = fs.readlinkSync(`/proc/self/fd/${fd}`); return ownedPipes.has(inode) ? [inode] : []; } + catch { return []; } + }); + expect(survivors).toEqual([]); + } + }, 10000); +}); diff --git a/api/src/pipe-process.ts b/api/src/pipe-process.ts new file mode 100644 index 00000000..a4bc6b84 --- /dev/null +++ b/api/src/pipe-process.ts @@ -0,0 +1,116 @@ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as net from 'node:net'; +import { spawn, type SpawnOptions, type ChildProcessWithoutNullStreams } from 'node:child_process'; +import { Readable, Writable } from 'node:stream'; + +let native: { createPipe(): [number, number] } | undefined; +function endpoints(): [number, number] { + native ??= require(process.env.SANDBOX_PIPE_ADDON || path.resolve(__dirname, '..', '.build', 'anonymous-pipes.node')); + return native!.createPipe(); +} +function close(fd: number): void { try { fs.closeSync(fd); } catch (error) { if ((error as NodeJS.ErrnoException).code !== 'EBADF') throw error; } } +function readable(fd: number): Readable { + if (typeof Bun === 'undefined') return new net.Socket({ fd, readable: true, writable: false }); + const stream = Readable.fromWeb(Bun.file(fd).stream() as never); + stream.once('close', () => close(fd)); + return stream; +} +function writable(fd: number): Writable { + if (typeof Bun === 'undefined') return new net.Socket({ fd, readable: false, writable: true }); + const sink = Bun.file(fd).writer(); + return new Writable({ + write(chunk, _encoding, callback) { + try { sink.write(chunk); Promise.resolve(sink.flush()).then(() => callback(), callback); } + catch (error) { callback(error as Error); } + }, + final(callback) { + try { Promise.resolve(sink.end()).then(() => callback(), callback); } + catch (error) { callback(error as Error); } + }, + destroy(error, callback) { + // FileSink duplicates the supplied FD; end also releases that duplicate + // on spawn failure/cancellation, when _final was never reached. + try { void Promise.resolve(sink.end()).catch(() => {}); } catch {} + close(fd); callback(error); + }, + }); +} +/** Linux child stdio uses pipe2 endpoints, never the runtime's socketpairs. + * Non-Linux development keeps its native spawn implementation. */ +export function spawnPipeProcess(command: string, args: readonly string[] = [], options: SpawnOptions = {}): ChildProcessWithoutNullStreams { + if (process.platform !== 'linux') return spawn(command, args, options) as ChildProcessWithoutNullStreams; + const requested = options.stdio ?? ['pipe', 'pipe', 'pipe']; + const modes = typeof requested === 'string' ? [requested, requested, requested] : requested; + if (!Array.isArray(modes) || modes.slice(3).some(mode => typeof mode !== 'number')) throw new Error('unsupported anonymous-pipe process stdio'); + const childFds: number[] = []; + const parentFds: Array<[number, number]> = []; + const streams: Array = [null, null, null]; + let child: ReturnType | undefined; + try { + const stdio = Array.from({ length: Math.max(3, modes.length) }, (_, index) => index).map(index => { + const mode = modes[index] ?? 'pipe'; + if (mode !== 'pipe') { + if (mode !== 'ignore' && mode !== 'inherit' && typeof mode !== 'number') throw new Error('unsupported anonymous-pipe process stdio'); + return mode; + } + const [reader, writer] = endpoints(); + const parentFd = index === 0 ? writer : reader; + const childFd = index === 0 ? reader : writer; + childFds.push(childFd); + parentFds.push([index, parentFd]); + return childFd; + }); + child = spawn(command, args, { ...options, stdio }); + for (const [index, fd] of parentFds) streams[index] = index === 0 ? writable(fd) : readable(fd); + Object.defineProperties(child, { + stdin: { value: streams[0] }, stdout: { value: streams[1] }, stderr: { value: streams[2] }, + }); + // Integer stdio descriptors are not owned by ChildProcess. Reproduce its + // close-after-output semantics and release stdin when the child exits. + const outputClosed = Promise.all(streams.slice(1).map(stream => stream + ? new Promise(resolve => stream.once('close', resolve)) : undefined)); + const emit = child.emit; + child.emit = function (event: string | symbol, ...values: unknown[]): boolean { + if (event === 'close') { + void outputClosed.then(() => emit.call(this, event, ...values)); + return true; + } + return emit.call(this, event, ...values); + }; + child.once('exit', () => { + streams[0]?.destroy(); + // Like native spawn, drain output that has no consumer after exit. + for (const stream of streams.slice(1)) { + if (stream && stream.listenerCount('data') === 0 && stream.listenerCount('readable') === 0) (stream as Readable).resume(); + } + }); + child.once('error', () => { for (const stream of streams) stream?.destroy(); }); + return child as ChildProcessWithoutNullStreams; + } catch (error) { + child?.kill('SIGKILL'); + for (const [index, fd] of parentFds) { if (streams[index]) streams[index]!.destroy(); else close(fd); } + throw error; + } finally { for (const fd of childFds) close(fd); } +} + +export function toolCallPipes(): { child: [number, number]; reader: Readable; writer: Writable; closeChild(): void } { + const [requestReader, requestWriter] = endpoints(); + let responseReader: number, responseWriter: number; + try { [responseReader, responseWriter] = endpoints(); } + catch (error) { close(requestReader); close(requestWriter); throw error; } + let reader: Readable | undefined; + let writer: Writable | undefined; + try { + reader = readable(requestReader); + writer = writable(responseWriter); + let childClosed = false; + return { child: [requestWriter, responseReader], reader, writer, + closeChild() { if (!childClosed) { childClosed = true; close(requestWriter); close(responseReader); } } }; + } catch (error) { + if (reader) reader.destroy(); else close(requestReader); + if (writer) writer.destroy(); else close(responseWriter); + close(requestWriter); close(responseReader); + throw error; + } +} diff --git a/api/src/session-checkpoint.ts b/api/src/session-checkpoint.ts index 8ddd7390..dc642619 100644 --- a/api/src/session-checkpoint.ts +++ b/api/src/session-checkpoint.ts @@ -1,4 +1,4 @@ -import { spawn } from 'child_process'; +import { spawnPipeProcess as spawn } from './pipe-process'; import * as fsp from 'fs/promises'; import * as os from 'os'; import * as path from 'path'; diff --git a/api/src/supervisor-policy.c b/api/src/supervisor-policy.c new file mode 100644 index 00000000..b2091927 --- /dev/null +++ b/api/src/supervisor-policy.c @@ -0,0 +1,60 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__x86_64__) +#define NATIVE_ARCH AUDIT_ARCH_X86_64 +#elif defined(__aarch64__) +#define NATIVE_ARCH AUDIT_ARCH_AARCH64 +#else +#error Unsupported supervisor architecture +#endif +#define DENY(call) BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_##call, 0, 1), BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | EPERM) +int main(int argc, char **argv) { + if (argc < 2) return 125; + struct sock_filter instructions[] = { + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, arch)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, NATIVE_ARCH, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), +#ifdef __x86_64__ + BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0x40000000, 0, 1), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), +#endif + DENY(socketpair), DENY(sendmsg), DENY(sendmmsg), + DENY(io_uring_setup), DENY(io_uring_enter), DENY(io_uring_register), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_socket, 0, 3), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[0])), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, AF_UNIX, 0, 1), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | EPERM), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), + }; + struct sock_fprog program = { .len = sizeof(instructions) / sizeof(instructions[0]), .filter = instructions }; + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0 + || prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &program) != 0) { + perror("sandbox supervisor policy"); return 125; + } + /* glibc's resolver batches queries with sendmmsg. Use sequential sendto + * queries so hostname-based API/broker traffic needs no policy exception. */ + const char *existing = getenv("RES_OPTIONS"); + char *resolver_options = NULL; + if (asprintf(&resolver_options, "%s%ssingle-request", existing ? existing : "", + existing && *existing ? " " : "") < 0) { + perror("sandbox supervisor DNS options"); return 125; + } + if (setenv("RES_OPTIONS", resolver_options, 1) != 0) { + perror("sandbox supervisor DNS options"); free(resolver_options); return 125; + } + free(resolver_options); + execvp(argv[1], &argv[1]); + perror("sandbox supervisor exec"); return 125; +} diff --git a/api/src/tool-call-pipe-bridge.c b/api/src/tool-call-pipe-bridge.c index a3e9f084..af8c2af0 100644 --- a/api/src/tool-call-pipe-bridge.c +++ b/api/src/tool-call-pipe-bridge.c @@ -12,9 +12,9 @@ #include #endif -/* Node/Bun extra stdio uses socketpairs. Keep that trusted-side socket in - * this relay; only two anonymous pipe ends ever reach NsJail/the job. - * FD 3 in the job writes requests; FD 4 reads responses. */ +/* Both broker-facing and job-facing transports are anonymous pipes. + * FD 3 writes requests; FD 4 reads responses. The relay retains bounded + * buffering and detects lost readers even with no queued response. */ struct flow { int source, destination; unsigned char bytes[65536]; @@ -63,50 +63,57 @@ int main(int argc, char **argv) { perror("tool-call broker exec"); return 125; } - int ipc = fcntl(3, F_DUPFD_CLOEXEC, 5); + if (strcmp(argv[1], "--job-pipes") != 0 || argc < 3) return 125; + int request_upstream = fcntl(3, F_DUPFD_CLOEXEC, 5); + int response_upstream = fcntl(4, F_DUPFD_CLOEXEC, 5); + if (request_upstream < 0 || response_upstream < 0) return 125; + close(3); close(4); int requests[2], responses[2]; - if (ipc < 0 || pipe(requests) != 0 || pipe(responses) != 0) return 125; + if (pipe2(requests, O_CLOEXEC) != 0 || pipe2(responses, O_CLOEXEC) != 0) return 125; pid_t relay = getpid(); pid_t child = fork(); if (child < 0) return 125; if (child == 0) { if (parent_death(relay) != 0 || dup2(requests[1], 3) < 0 || dup2(responses[0], 4) < 0) _exit(125); - int fds[] = {ipc, requests[0], requests[1], responses[0], responses[1]}; + int fds[] = {request_upstream, response_upstream, requests[0], requests[1], responses[0], responses[1]}; for (unsigned i = 0; i < sizeof(fds) / sizeof(fds[0]); i++) { if (fds[i] > 4) close(fds[i]); } - execvp(argv[1], &argv[1]); + execvp(argv[2], &argv[2]); perror("tool-call pipe exec"); _exit(125); } - close(3); close(requests[1]); close(responses[0]); signal(SIGPIPE, SIG_IGN); - if (nonblock(ipc) < 0 || nonblock(requests[0]) < 0 || nonblock(responses[1]) < 0) { + if (nonblock(request_upstream) < 0 || nonblock(response_upstream) < 0 || nonblock(requests[0]) < 0 || nonblock(responses[1]) < 0) { kill(child, SIGKILL); waitpid(child, NULL, 0); return 125; } - struct flow request = {.source = requests[0], .destination = ipc}; - struct flow response = {.source = ipc, .destination = responses[1]}; + struct flow request = {.source = requests[0], .destination = request_upstream}; + struct flow response = {.source = response_upstream, .destination = responses[1]}; int status = 125 << 8; for (;;) { pid_t done = waitpid(child, &status, WNOHANG); if (done == child) break; if (done < 0 && errno != EINTR) return 125; struct pollfd pollfds[] = { - {.fd = ipc, .events = (response.length || response.source < 0 ? 0 : POLLIN) | (request.length ? POLLOUT : 0)}, - {.fd = request.source, .events = request.length ? 0 : POLLIN}, + /* HUP is reported even with events=0. Omit buffered sources + * until their destinations drain, or a closed writer spins poll. */ + {.fd = response.length ? -1 : response_upstream, .events = POLLIN}, + {.fd = request.length ? -1 : request.source, .events = POLLIN}, {.fd = responses[1], .events = response.length ? POLLOUT : 0}, + {.fd = request_upstream, .events = request.length ? POLLOUT : 0}, }; - int result = poll(pollfds, 3, 100); + int result = poll(pollfds, 4, 100); if (result < 0 && errno == EINTR) continue; if (result < 0 || (pollfds[0].revents & (POLLERR | POLLNVAL)) || (pollfds[1].revents & (POLLERR | POLLNVAL)) || (pollfds[2].revents & (POLLERR | POLLHUP | POLLNVAL)) - || transfer(&request, pollfds[1].revents & (POLLIN | POLLHUP), pollfds[0].revents & POLLOUT) < 0 + || (pollfds[3].revents & (POLLERR | POLLHUP | POLLNVAL)) + || transfer(&request, pollfds[1].revents & (POLLIN | POLLHUP), pollfds[3].revents & POLLOUT) < 0 || transfer(&response, pollfds[0].revents & (POLLIN | POLLHUP), pollfds[2].revents & POLLOUT) < 0 || response.source < 0) { /* EOF/error terminates the entire invocation, including an * abandoned tool call. Never leave an orphaned NsJail monitor. */ @@ -116,7 +123,8 @@ int main(int argc, char **argv) { break; } } - close(ipc); + close(request_upstream); + close(response_upstream); close(requests[0]); close(responses[1]); return WIFEXITED(status) ? WEXITSTATUS(status) : 128 + WTERMSIG(status); diff --git a/api/src/tool-call-pipe-broker.ts b/api/src/tool-call-pipe-broker.ts index 4fb923cc..d81524c2 100644 --- a/api/src/tool-call-pipe-broker.ts +++ b/api/src/tool-call-pipe-broker.ts @@ -1,5 +1,5 @@ -import { spawn } from 'node:child_process'; -import type { Duplex } from 'node:stream'; +import { spawnPipeProcess as spawn, toolCallPipes } from './pipe-process'; +import { Duplex, Readable, Writable } from 'node:stream'; import * as http from 'node:http'; import * as https from 'node:https'; @@ -150,10 +150,17 @@ export function serveToolCallPipe(channel: Duplex, rawTarget: string, options: { if (require.main === module) { const target = process.env.SANDBOX_FORWARD_TARGET?.trim(); if (!target || !process.argv[2]) throw new Error('tool-call pipe broker is not configured'); - const child = spawn(process.env.TCS_PIPE_BRIDGE || '/usr/local/bin/tool-call-pipe-bridge', process.argv.slice(2), { - stdio: [0, 1, 2, 'pipe'], - }); - const channel = child.stdio[3] as Duplex; + const pipes = toolCallPipes(); + let child: ReturnType; + try { + child = spawn(process.env.TCS_PIPE_BRIDGE || '/usr/local/bin/tool-call-pipe-bridge', ['--job-pipes', ...process.argv.slice(2)], { + stdio: [0, 1, 2, ...pipes.child], + }); + } catch (error) { + pipes.reader.destroy(); pipes.writer.destroy(); + throw error; + } finally { pipes.closeChild(); } + const channel = Duplex.fromWeb({ readable: Readable.toWeb(pipes.reader), writable: Writable.toWeb(pipes.writer) }); const stop = serveToolCallPipe(channel, target, { requestTimeoutMs: Math.min(600_000, Math.max(5_000, Number(process.env.TCS_REQUEST_TIMEOUT_MS) || 300_000)), onFailure: () => child.kill('SIGKILL'), diff --git a/docker/Dockerfile.worker-sandbox b/docker/Dockerfile.worker-sandbox index 0117132d..673dd42f 100644 --- a/docker/Dockerfile.worker-sandbox +++ b/docker/Dockerfile.worker-sandbox @@ -39,12 +39,20 @@ RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \ && git checkout 66ad78dc361721512fbf9470488e584364c75489 \ && git submodule update --init --recursive -RUN make -j$(nproc) +COPY docker/nsjail-anonymous-control.patch /tmp/nsjail-anonymous-control.patch +# Network-isolated jobs need no libnl transport; removing it avoids sendmsg +# in the supervisor and fails closed for interface/traffic-rule configuration. +RUN git apply /tmp/nsjail-anonymous-control.patch && make -j$(nproc) NL3_EXISTS=no COPY api/src/spec-guard.c /tmp/spec-guard.c COPY api/src/tool-call-pipe-bridge.c /tmp/tool-call-pipe-bridge.c +COPY api/src/anonymous-pipes.c /tmp/anonymous-pipes.c +COPY api/src/supervisor-policy.c /tmp/supervisor-policy.c +COPY --from=node:22-slim /usr/local/include/node /usr/local/include/node COPY docker/rootfs-setup.c /tmp/rootfs-setup.c -RUN gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/tool-call-pipe-bridge /tmp/tool-call-pipe-bridge.c \ +RUN gcc -shared -fPIC -O2 -Wall -Wextra -Werror -I/usr/local/include/node -o /usr/local/lib/anonymous-pipes.node /tmp/anonymous-pipes.c \ + && gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/sandbox-supervisor-policy /tmp/supervisor-policy.c \ + && gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/tool-call-pipe-bridge /tmp/tool-call-pipe-bridge.c \ && gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ && gcc -O2 -static -o /usr/local/bin/sandbox-rootfs-setup /tmp/rootfs-setup.c \ && chmod 0111 /usr/local/bin/spec-guard /usr/local/bin/sandbox-rootfs-setup @@ -148,6 +156,9 @@ RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen COPY --from=nsjail-builder /nsjail/nsjail /usr/sbin/nsjail COPY --from=nsjail-builder /usr/local/bin/spec-guard /usr/local/bin/spec-guard COPY --from=nsjail-builder /usr/local/bin/tool-call-pipe-bridge /usr/local/bin/tool-call-pipe-bridge +COPY --from=nsjail-builder /usr/local/bin/sandbox-supervisor-policy /usr/local/bin/sandbox-supervisor-policy +COPY --from=nsjail-builder /usr/local/lib/anonymous-pipes.node /usr/local/lib/anonymous-pipes.node +ENV SANDBOX_PIPE_ADDON=/usr/local/lib/anonymous-pipes.node RUN chmod +x /usr/sbin/nsjail COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node diff --git a/docker/nsjail-anonymous-control.patch b/docker/nsjail-anonymous-control.patch new file mode 100644 index 00000000..bfce44bf --- /dev/null +++ b/docker/nsjail-anonymous-control.patch @@ -0,0 +1,82 @@ +--- a/subproc.cc ++++ b/subproc.cc +@@ -510,6 +510,12 @@ + } + + pid_t runChild(nsj_t* nsj, int netfd, int fd_in, int fd_out, int fd_err) { ++ // This runner uses anonymous control pipes. Descriptor-passing user-mode ++ // networking and seccomp notification transports are intentionally unsupported. ++ if (nsj->njc.seccomp_unotify() || nsj->njc.has_user_net()) { ++ LOG_E("Descriptor-passing NsJail modes are disabled by runner hardening"); ++ return -1; ++ } + if (!net::limitConns(nsj, netfd)) { + return 0; + } +@@ -535,26 +541,34 @@ + LOG_D("Creating new process with clone flags:%s and exit_signal:SIGCHLD", + cloneFlagsToStr(flags).c_str()); + +- int sv[2]; +- if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0, sv) == -1) { +- PLOG_E("socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC) failed"); ++ int ready[2], errors[2]; ++ if (pipe2(ready, O_CLOEXEC) == -1) { ++ PLOG_E("pipe2(ready) failed"); + return -1; + } +- int child_fd = sv[0]; +- int parent_fd = sv[1]; ++ if (pipe2(errors, O_CLOEXEC) == -1) { ++ close(ready[0]); close(ready[1]); ++ PLOG_E("pipe2(errors) failed"); ++ return -1; ++ } ++ int child_fd = ready[0]; ++ int parent_fd = ready[1]; + + pid_t pid = cloneProc(flags, SIGCHLD); + if (pid == 0) { + close(parent_fd); ++ close(errors[0]); + newProc(nsj, netfd, fd_in, fd_out, fd_err, child_fd); +- util::writeToFd(child_fd, &kSubprocErrorChar, sizeof(kSubprocErrorChar)); ++ util::writeToFd(errors[1], &kSubprocErrorChar, sizeof(kSubprocErrorChar)); + LOG_F("Launching child process failed"); + } + close(child_fd); ++ close(errors[1]); + if (pid == -1) { + auto saved_errno = errno; + PLOG_W("clone(flags=%s) failed", cloneFlagsToStr(flags).c_str()); + close(parent_fd); ++ close(errors[0]); + errno = saved_errno; + return pid; + } +@@ -562,20 +576,22 @@ + + if (!initParent(nsj, pid, parent_fd)) { + close(parent_fd); ++ close(errors[0]); + return -1; + } ++ close(parent_fd); + + char rcvChar; +- while (util::readFromFd(parent_fd, &rcvChar, sizeof(rcvChar)) == sizeof(rcvChar)) { ++ while (util::readFromFd(errors[0], &rcvChar, sizeof(rcvChar)) == sizeof(rcvChar)) { + if (rcvChar == kSubprocErrorChar) { + LOG_W("Received error message from the child process before it has been " + "executed"); +- close(parent_fd); ++ close(errors[0]); + return -1; + } + } + +- close(parent_fd); ++ close(errors[0]); + return pid; + } + diff --git a/docs/adr/0013-supervisor-anonymous-pipes.md b/docs/adr/0013-supervisor-anonymous-pipes.md new file mode 100644 index 00000000..0a905455 --- /dev/null +++ b/docs/adr/0013-supervisor-anonymous-pipes.md @@ -0,0 +1,65 @@ +# 0013: Remove Unix sockets from runner supervisors + +Status: Proposed + +## Context + +ADRs 0011 and 0012 deny sockets in untrusted jobs. The trusted API, broker, +relay, and NsJail monitor still process job-controlled data. Node/Bun's piped +subprocess stdio and NsJail's startup handshake create Unix socketpairs outside +the job filter. Supervisor hardening must replace those transports before it +can deny the same kernel entry points. + +## Decision + +Install an inherited seccomp filter before starting the API. Deny AF_UNIX +socket creation, every socketpair, sendmsg/sendmmsg (including inherited +sockets), and all io_uring entry points. Validate the native audit architecture +and reject x32 syscall encodings. TCP remains available for API requests, +hosted previews, and the broker's fixed upstream. This is a targeted deny filter, +not a complete supervisor syscall allowlist. Startup fails if installation fails. Force sequential resolver queries with +`RES_OPTIONS=single-request` (preserving existing options), so glibc DNS uses +sendto instead of its blocked sendmmsg batching. + +A small N-API addon creates CLOEXEC anonymous pipes. Linux subprocess launches +pass integer pipe ends to Node/Bun rather than requesting runtime-created pipes. +The adapter preserves output draining, backpressure, child close ordering, and +cleanup on exit, failed spawn, and cancellation. macOS development retains native +spawn; Linux container tests on the Mac verify the real filter and pipe adapter. + +The tool-call broker uses separate request/response pipes to its relay. The relay +retains bounded buffers, parent-death signals, and immediate termination when a +response reader disappears, even without a pending response. Buffered sources +are omitted from poll until drained so a closed writer cannot cause a CPU spin. +Untrusted jobs now +receive anonymous pipes for stdin/stdout/stderr as well as tool-call FDs 3/4. + +Patch the pinned NsJail build to replace its supervisor/child handshake with +separate readiness and error pipes. Reject optional seccomp notification and +user-mode networking modes that require descriptor passing. Build without libnl: +network-isolated jobs do not need interface movement or traffic-rule programming, +and this avoids a gratuitous netlink enumeration during namespace setup. +Unsupported interface/traffic-rule requests fail closed. + +Hosted-app egress uses the packaged iptables-legacy and ip6tables-legacy binaries, +whose transport does not require sendmsg. Both chains must install before any +app is admitted. IPv4/IPv6 legacy netfilter support is consequently required on +the runner kernel; missing support rejects hosted-app admission. + +## Consequences and validation + +All three runner image paths package the matching addon, policy binary, broker, +relay, and patched NsJail. Deploy rebuilt runner images and regenerate snapshots; +these internal processes and binaries are an atomic image upgrade. The external +service protocol and previously patched Python runtime are unchanged. + +The filter also applies to hosted apps and their descendants. Runtimes and +third-party libraries that require Unix sockets/socketpairs remain unsupported. +The trusted API retains its existing privileges and other syscalls; reducing +those privileges requires another containment boundary. Kernel patching remains +necessary. + +CI runs native filter, anonymous stdio, cleanup/backpressure, packaged broker, +and hosted-app egress/preview tests on amd64 and arm64. A local signed runner +canary additionally exercises normal namespaces with Python 3.14.4 asyncio, +spawn/fork multiprocessing, concurrent tool calls, and Bun 1.4.2 JS/TS execution. diff --git a/launcher/Dockerfile b/launcher/Dockerfile index 4acd2094..bad5cd2f 100644 --- a/launcher/Dockerfile +++ b/launcher/Dockerfile @@ -31,12 +31,20 @@ RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \ && git checkout 66ad78dc361721512fbf9470488e584364c75489 \ && git submodule update --init --recursive -RUN make -j$(nproc) +COPY docker/nsjail-anonymous-control.patch /tmp/nsjail-anonymous-control.patch +# Network-isolated jobs need no libnl transport; removing it avoids sendmsg +# in the supervisor and fails closed for interface/traffic-rule configuration. +RUN git apply /tmp/nsjail-anonymous-control.patch && make -j$(nproc) NL3_EXISTS=no COPY api/src/spec-guard.c /tmp/spec-guard.c COPY api/src/tool-call-pipe-bridge.c /tmp/tool-call-pipe-bridge.c +COPY api/src/anonymous-pipes.c /tmp/anonymous-pipes.c +COPY api/src/supervisor-policy.c /tmp/supervisor-policy.c +COPY --from=node:22-slim /usr/local/include/node /usr/local/include/node COPY docker/rootfs-setup.c /tmp/rootfs-setup.c -RUN gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/tool-call-pipe-bridge /tmp/tool-call-pipe-bridge.c \ +RUN gcc -shared -fPIC -O2 -Wall -Wextra -Werror -I/usr/local/include/node -o /usr/local/lib/anonymous-pipes.node /tmp/anonymous-pipes.c \ + && gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/sandbox-supervisor-policy /tmp/supervisor-policy.c \ + && gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/tool-call-pipe-bridge /tmp/tool-call-pipe-bridge.c \ && gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ && gcc -O2 -static -o /usr/local/bin/sandbox-rootfs-setup /tmp/rootfs-setup.c \ && chmod 0111 /usr/local/bin/spec-guard /usr/local/bin/sandbox-rootfs-setup @@ -68,6 +76,9 @@ RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen COPY --from=nsjail-builder /nsjail/nsjail /usr/sbin/nsjail COPY --from=nsjail-builder /usr/local/bin/spec-guard /usr/local/bin/spec-guard COPY --from=nsjail-builder /usr/local/bin/tool-call-pipe-bridge /usr/local/bin/tool-call-pipe-bridge +COPY --from=nsjail-builder /usr/local/bin/sandbox-supervisor-policy /usr/local/bin/sandbox-supervisor-policy +COPY --from=nsjail-builder /usr/local/lib/anonymous-pipes.node /usr/local/lib/anonymous-pipes.node +ENV SANDBOX_PIPE_ADDON=/usr/local/lib/anonymous-pipes.node COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node RUN chmod +x /usr/sbin/nsjail diff --git a/tests/pipe_relay_backpressure.py b/tests/pipe_relay_backpressure.py new file mode 100644 index 00000000..fb18803c --- /dev/null +++ b/tests/pipe_relay_backpressure.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""A closed request writer must not spin the trusted relay under backpressure.""" +import os +import signal +import subprocess +import sys +import tempfile +import time + +request_reader, request_writer = os.pipe() +response_reader, response_writer = os.pipe() +os.set_blocking(request_writer, False) +try: + while True: + os.write(request_writer, b'x' * 4096) +except BlockingIOError: + pass # The upstream deliberately does not drain requests. + +with tempfile.TemporaryDirectory() as directory: + marker = os.path.join(directory, 'ready') + job = os.path.join(directory, 'job.py') + with open(job, 'w') as output: + output.write(f"import os,time; os.write(3,b'x'*4096); os.close(3); open({marker!r},'w').close(); time.sleep(30)") + bootstrap = (f"import os; os.dup2({request_writer},3); os.dup2({response_reader},4); " + f"os.execv('/usr/local/bin/tool-call-pipe-bridge'," + f"['tool-call-pipe-bridge','--job-pipes',{sys.executable!r},{job!r}])") + relay = subprocess.Popen([sys.executable, '-c', bootstrap], + pass_fds=[request_writer, response_reader]) + try: + deadline = time.monotonic() + 5 + while not os.path.exists(marker) and time.monotonic() < deadline: + assert relay.poll() is None + time.sleep(0.01) + assert os.path.exists(marker), 'job did not close its request writer' + def cpu_seconds(): + with open(f'/proc/{relay.pid}/stat') as source: + fields = source.read().rsplit(')', 1)[1].split() + return (int(fields[11]) + int(fields[12])) / os.sysconf('SC_CLK_TCK') + before = cpu_seconds() + time.sleep(0.5) + used = cpu_seconds() - before + assert used < 0.1, f'relay spun on pipe HUP under backpressure: {used:.3f}s CPU' + assert relay.poll() is None + # Closing the broker response writer still terminates the invocation. + os.close(response_writer) + response_writer = -1 + assert relay.wait(timeout=2) == 137 + print('PASS: blocked relay stays idle after request EOF and terminates on broker EOF') + finally: + if relay.poll() is None: + relay.send_signal(signal.SIGKILL) + relay.wait(timeout=2) +for fd in [request_reader, request_writer, response_reader, response_writer]: + if fd >= 0: + os.close(fd) diff --git a/tests/supervisor_dns.py b/tests/supervisor_dns.py new file mode 100644 index 00000000..f2a9a291 --- /dev/null +++ b/tests/supervisor_dns.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Run with Docker --dns 127.0.0.1; no external DNS/network dependency.""" +import json +import os +import socket +import struct +import subprocess +import threading + +server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +server.bind(('127.0.0.1', 53)) +queries = [] +def serve(): + while True: + packet, peer = server.recvfrom(4096) + offset = 12 + labels = [] + while packet[offset]: + length = packet[offset] + labels.append(packet[offset + 1:offset + length + 1]) + offset += length + 1 + offset += 1 + kind, cls = struct.unpack('!HH', packet[offset:offset + 4]) + offset += 4 + queries.append((b'.'.join(labels), kind)) + address = socket.inet_pton(socket.AF_INET6 if kind == 28 else socket.AF_INET, + '::1' if kind == 28 else '127.0.0.1') + reply = packet[:2] + struct.pack('!HHHHH', 0x8180, 1, 1, 0, 0) + packet[12:offset] + reply += b'\xc0\x0c' + struct.pack('!HHIH', kind, cls, 60, len(address)) + address + server.sendto(reply, peer) +threading.Thread(target=serve, daemon=True).start() +program = """ +const assert=require('node:assert/strict'); +assert(process.env.RES_OPTIONS.includes('timeout:1')); +assert(process.env.RES_OPTIONS.includes('single-request')); +require('node:dns').lookup('supervisor-test.invalid.',{all:true},(error,addresses)=>{ + if(error) throw error; + assert(addresses.length>0); + assert(addresses.every(a=>a.address==='127.0.0.1'||a.address==='::1')); + console.log('PASS: hostname resolution under supervisor filter'); +}); +""" +for runtime in ['/usr/local/bin/node', '/usr/local/bin/bun']: + result = subprocess.run(['/usr/local/bin/sandbox-supervisor-policy', runtime, '-e', program], + env={**os.environ, 'RES_OPTIONS': 'timeout:1 attempts:1'}, + capture_output=True, text=True, timeout=5) + assert result.returncode == 0, (runtime, result.stdout, result.stderr) + print(os.path.basename(runtime) + ': ' + result.stdout.strip()) +assert queries and all(name == b'supervisor-test.invalid' for name, _ in queries), queries diff --git a/tests/supervisor_hosted_network.ts b/tests/supervisor_hosted_network.ts new file mode 100644 index 00000000..e581576b --- /dev/null +++ b/tests/supervisor_hosted_network.ts @@ -0,0 +1,42 @@ +// Run as root with NET_ADMIN in the packaged guest, under supervisor-policy. +import { installHostedAppNetworkGuard } from '/work/api/src/hosted-app'; +import { spawnPipeProcess } from '/work/api/src/pipe-process'; +import { once } from 'node:events'; +import * as net from 'node:net'; +import assert from 'node:assert/strict'; + +const uid = 200123; +await installHostedAppNetworkGuard(uid); +const upstream = net.createServer(); +upstream.listen(0, '127.0.0.1'); +await once(upstream, 'listening'); +const port = (upstream.address() as net.AddressInfo).port; +const child = spawnPipeProcess(process.execPath, ['-e', ` +import net from 'node:net'; +import {once} from 'node:events'; +import assert from 'node:assert/strict'; +process.setgid(${uid}); process.setuid(${uid}); +const outbound = net.connect(${port}, '127.0.0.1'); +await assert.rejects(once(outbound, 'connect')); +outbound.destroy(); +const server = net.createServer(client => client.end('preview reply')); +server.listen(0, '127.0.0.1'); await once(server, 'listening'); +console.log(server.address().port); +await once(server, 'connection'); server.close(); +`]); +child.stdin.end(); +child.stderr.pipe(process.stderr); +const closed = once(child, 'close'); +let timer: ReturnType | undefined; +try { + timer = setTimeout(() => child.kill('SIGKILL'), 5000); + const [bytes] = await once(child.stdout, 'data'); + const preview = net.connect(Number(bytes.toString().trim()), '127.0.0.1'); + let reply = ''; + for await (const chunk of preview) reply += chunk.toString(); + assert.equal(reply, 'preview reply'); + assert.equal((await closed)[0], 0); + console.log('PASS: hosted-app egress is blocked and inbound preview replies work under supervisor policy'); +} finally { + clearTimeout(timer); child.kill('SIGKILL'); upstream.close(); +} diff --git a/tests/supervisor_policy.py b/tests/supervisor_policy.py new file mode 100644 index 00000000..d0fa0472 --- /dev/null +++ b/tests/supervisor_policy.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Exercise the installed supervisor filter, including inherited Unix sockets.""" +import ctypes +import errno +import os +import platform +import socket +import subprocess +import sys + +POLICY = '/usr/local/bin/sandbox-supervisor-policy' +if len(sys.argv) > 1 and sys.argv[1] == '--filtered': + def denied(action): + try: + action() + except OSError as error: + assert error.errno == errno.EPERM, error + else: + raise AssertionError('supervisor syscall allowed') + denied(lambda: socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)) + denied(socket.socketpair) + inherited = socket.socket(fileno=int(sys.argv[2])) + denied(lambda: inherited.sendmsg([b'x'], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, b'\0' * 4)])) + libc = ctypes.CDLL(None, use_errno=True) + calls = {'x86_64': [307, 425, 426, 427], 'aarch64': [269, 425, 426, 427]}[platform.machine()] + for number in calls: # sendmmsg and all io_uring entry points + assert libc.syscall(number, -1, 0, 0, 0, 0, 0) == -1 + assert ctypes.get_errno() == errno.EPERM + # TCP remains usable for the API listener and broker upstream. + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.bind(('127.0.0.1', 0)); listener.listen() + client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + client.connect(listener.getsockname()) + peer, _ = listener.accept() + client.sendall(b'tcp'); assert peer.recv(3) == b'tcp' + print('PASS: supervisor denies Unix sockets, inherited descriptor sends and io_uring; TCP works') +else: + left, right = socket.socketpair() + result = subprocess.run([POLICY, sys.executable, __file__, '--filtered', str(left.fileno())], + pass_fds=[left.fileno()], timeout=5) + assert result.returncode == 0, result.returncode + # The pipe-based NsJail handshake must handle child setup/exec failure, + # as well as the successful normal path tested by the runner canary. + command = [POLICY, '/nsjail/nsjail', '--mode', 'o', '--disable_clone_newnet', + '--disable_clone_newuser', '--disable_clone_newns', '--disable_clone_newpid', + '--disable_clone_newipc', '--disable_clone_newuts', '--disable_clone_newcgroup', + '--disable_proc', '--disable_rlimits', '--', '/missing-codeapi-executable'] + result = subprocess.run(command, capture_output=True, timeout=5) + assert result.returncode != 0 + assert b'Launching child process failed' in result.stderr, result.stderr + print('PASS: anonymous NsJail startup handshake reports child exec failure') diff --git a/tests/tool_call_runner.cjs b/tests/tool_call_runner.cjs index 4aab25e4..61ddcb9e 100644 --- a/tests/tool_call_runner.cjs +++ b/tests/tool_call_runner.cjs @@ -66,7 +66,7 @@ def _mp_echo(sender): if __name__ == '__main__': assert sys.version.split()[0] == ${JSON.stringify(version)} assert os.getuid() == 65534 - assert _stat.S_ISFIFO(os.fstat(3).st_mode) and _stat.S_ISFIFO(os.fstat(4).st_mode) + assert all(_stat.S_ISFIFO(os.fstat(fd).st_mode) for fd in range(5)) for _namespace, _outside in ${JSON.stringify(outerNamespaces)}.items(): assert os.readlink('/proc/self/ns/' + _namespace) != _outside, _namespace for domain in (socket.AF_UNIX, socket.AF_INET, socket.AF_INET6): From c854ab5f37c656e88e6e4ad8b834cc8d17659151 Mon Sep 17 00:00:00 2001 From: Danny Avila Date: Tue, 6 Oct 2026 08:00:03 -0400 Subject: [PATCH 40/48] test: check parent-side nsjail startup failure (#318) --- tests/supervisor_policy.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/supervisor_policy.py b/tests/supervisor_policy.py index d0fa0472..caab12c0 100644 --- a/tests/supervisor_policy.py +++ b/tests/supervisor_policy.py @@ -47,5 +47,7 @@ def denied(action): '--disable_proc', '--disable_rlimits', '--', '/missing-codeapi-executable'] result = subprocess.run(command, capture_output=True, timeout=5) assert result.returncode != 0 - assert b'Launching child process failed' in result.stderr, result.stderr + # The parent can kill the failed child before its final log is flushed. + # Its error-pipe diagnostic proves the startup handshake reported failure. + assert b'Received error message from the child process before it has been executed' in result.stderr, result.stderr print('PASS: anonymous NsJail startup handshake reports child exec failure') From 58a023da4c47452f5e40114934679d0348c39064 Mon Sep 17 00:00:00 2001 From: Dustin Healy <54083382+dustinhealy@users.noreply.github.com> Date: Fri, 16 Oct 2026 23:33:06 -0700 Subject: [PATCH 41/48] fix: resolve packaged pipe addon inside guests (#336) --- .github/workflows/ci.yml | 35 ++++++++++++ api/src/entrypoint.sh | 4 ++ tests/sandbox_pipe_addon_image.ts | 92 +++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100644 tests/sandbox_pipe_addon_image.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50804e27..f0191f0a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -184,6 +184,14 @@ jobs: --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ --entrypoint /usr/local/bin/node \ launcher-guest-test /tests/pipe_runtime_image.cjs + - name: Execute through the packaged guest API without build-stage ENV + run: | + for addon_path in unset empty override; do + docker run --rm --privileged --network none \ + --mount "type=bind,source=$GITHUB_WORKSPACE,target=/work,readonly" \ + --entrypoint bun launcher-guest-test \ + /work/tests/sandbox_pipe_addon_image.ts "$addon_path" + done - name: Load legacy netfilter modules for hosted-app isolation run: sudo modprobe -a ip_tables ip6_tables iptable_filter ip6table_filter xt_conntrack xt_owner ipt_REJECT ip6t_REJECT - name: Verify supervisor filter and anonymous stdio @@ -225,6 +233,33 @@ jobs: nsjail-ipc-test \ python3 /tests/tool_call_pipes.py /policy /broker.cjs /client.py + guest-api-image-tests: + name: Guest API Execution (${{ matrix.dockerfile }}) + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - dockerfile: api/Dockerfile + target: sandbox-build + - dockerfile: docker/Dockerfile.worker-sandbox + target: sandbox-rootfs + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - name: Build the production guest rootfs + run: docker build --target "${{ matrix.target }}" -t guest-api-test -f "${{ matrix.dockerfile }}" . + - name: Execute through the packaged guest API without build-stage ENV + # Test the roots copied into the direct/KVM runners, removing the + # Docker ENV metadata that does not survive that COPY/VM boundary. + run: | + for addon_path in unset empty override; do + docker run --rm --privileged --network none \ + --mount "type=bind,source=$GITHUB_WORKSPACE,target=/work,readonly" \ + --entrypoint bun guest-api-test \ + /work/tests/sandbox_pipe_addon_image.ts "$addon_path" + done + service-unit-tests: name: Service Unit Tests runs-on: ubuntu-latest diff --git a/api/src/entrypoint.sh b/api/src/entrypoint.sh index 0f93981b..a3e9af3b 100755 --- a/api/src/entrypoint.sh +++ b/api/src/entrypoint.sh @@ -3,6 +3,10 @@ set -e echo "Starting NsJail sandbox API..." +# Rootfs COPY carries the addon file, but not the build image's Docker ENV. +# Establish its installed path inside the guest, after the VM/chroot boundary. +export SANDBOX_PIPE_ADDON="${SANDBOX_PIPE_ADDON:-/usr/local/lib/anonymous-pipes.node}" + # Raise the RLIMIT_NOFILE limits so per-job nsjail can set its own soft # limit (SANDBOX_MAX_OPEN_FILES) without EPERM. The AWS Lambda MicroVM base # image ships a 1024 hard cap, below the sandbox default of 2048; nsjail diff --git a/tests/sandbox_pipe_addon_image.ts b/tests/sandbox_pipe_addon_image.ts new file mode 100644 index 00000000..82193ead --- /dev/null +++ b/tests/sandbox_pipe_addon_image.ts @@ -0,0 +1,92 @@ +// Run in a disposable guest rootfs image with --privileged --network none. +// Exercise the installed entrypoint and bundled API, not the IPC test stage: +// Docker ENV metadata is absent when the rootfs is copied into a runner/VM. +import assert from 'node:assert/strict'; +import { generateKeyPairSync } from 'node:crypto'; +import { mkdirSync, writeFileSync, copyFileSync, unlinkSync } from 'node:fs'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { + EXECUTION_MANIFEST_HEADER, + executionManifestBodySha256, + signExecutionManifestWithPrivateKey, +} from '../api/src/execution-manifest'; + +const mode = process.argv[2] ?? 'unset'; +assert(['unset', 'empty', 'override'].includes(mode)); +const env = { ...process.env }; +delete env.SANDBOX_PIPE_ADDON; +if (mode === 'empty') env.SANDBOX_PIPE_ADDON = ''; +if (mode === 'override') { + env.SANDBOX_PIPE_ADDON = '/opt/override-pipes.node'; + copyFileSync('/usr/local/lib/anonymous-pipes.node', env.SANDBOX_PIPE_ADDON); + unlinkSync('/usr/local/lib/anonymous-pipes.node'); +} + +// A tiny runtime uses the image's shell, avoiding the unrelated package build. +const runtime = '/pkgs/pipe-smoke/1.0.0'; +mkdirSync(runtime, { recursive: true }); +writeFileSync(`${runtime}/pkg-info.json`, JSON.stringify({ language: 'pipe-smoke', version: '1.0.0' })); +writeFileSync(`${runtime}/.package-installed`, ''); +writeFileSync(`${runtime}/run`, '#!/bin/sh\nexec /bin/sh "$@"\n', { mode: 0o755 }); + +const { privateKey, publicKey } = generateKeyPairSync('ed25519'); +const server = spawn('/sandbox_api/entrypoint.sh', [], { + env: { + ...env, + // Hosted Docker runners do not delegate cgroups. NsJail namespaces, + // per-job identities, seccomp and the supervisor filter remain enabled. + SANDBOX_USE_CGROUPV2: 'false', + SANDBOX_REQUIRE_EGRESS_MANIFEST: 'true', + SANDBOX_EXECUTION_MANIFEST_PUBLIC_KEY: publicKey.export({ type: 'spki', format: 'pem' }).toString(), + SANDBOX_LOG_LEVEL: 'error', + }, + stdio: ['ignore', 'inherit', 'inherit'], +}); +const exited = once(server, 'exit'); +const base = 'http://127.0.0.1:2000/api/v2'; +try { + let ready = false; + for (let attempt = 0; attempt < 100; attempt++) { + assert.equal(server.exitCode, null, 'guest entrypoint exited before API readiness'); + ready = await fetch(`${base}/health`, { signal: AbortSignal.timeout(1000) }) + .then(response => response.ok).catch(() => false); + if (ready) break; + await Bun.sleep(100); + } + assert(ready, 'guest API did not become ready'); + const body = { + session_id: 'pipe-addon-smoke', + language: 'pipe-smoke', + version: '1.0.0', + stdin: 'packaged anonymous pipes work\n', + files: [{ name: 'main.sh', content: 'IFS= read -r line; printf "%s\\n" "$line"; printf "pipe stderr\\n" >&2' }], + }; + const request = (manifest?: string) => fetch(`${base}/execute`, { + method: 'POST', + headers: { 'content-type': 'application/json', ...(manifest ? { [EXECUTION_MANIFEST_HEADER]: manifest } : {}) }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(15000), + }); + assert.equal((await request()).status, 401, 'unsigned execution must be rejected'); + const now = Math.floor(Date.now() / 1000); + const manifest = signExecutionManifestWithPrivateKey({ + v: 1, exec_id: 'pipe-smoke', tenant_id: 'test', user_id: 'test', session_key: 'test', + input_files: [], read_sessions: [], output_session_id: body.session_id, + max_upload_bytes: 1024, max_output_files: 1, max_requests: 10, + iat: now, exp: now + 60, tool_call_socket: false, + execute_body_sha256: executionManifestBodySha256(body), + }, privateKey.export({ type: 'pkcs8', format: 'pem' }).toString()); + const response = await request(manifest); + const result = await response.json(); + assert.equal(response.status, 200, JSON.stringify(result)); + assert.equal(result.run.code, 0, JSON.stringify(result)); + assert.equal(result.run.stdout, body.stdin); + assert.equal(result.run.stderr, 'pipe stderr\n'); + console.log(`PASS: packaged guest execution with ${mode} addon path`); +} finally { + server.kill('SIGTERM'); + const timer = setTimeout(() => server.kill('SIGKILL'), 5000); + await exited; + clearTimeout(timer); +} From 1d3955c4bcdf1ee7d06848eb3db6c18ba6ff914b Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Wed, 21 Oct 2026 16:43:32 +0200 Subject: [PATCH 42/48] =?UTF-8?q?=F0=9F=A7=AD=20feat:=20Add=20Guided=20Wor?= =?UTF-8?q?ker=20Setup=20And=20Saved=20Profiles=20(#339)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🧭 feat: Add Guided Worker Setup And Saved Profiles * 🔒 fix: Keep Pairing Codes Out Of Terminal History * 🔒 fix: Isolate Secret Prompt Editing State --------- Co-authored-by: Lia --- packages/code/src/cli.ts | 36 +- packages/code/src/pairing.test.ts | 58 +++ packages/code/src/pairing.ts | 39 +- packages/code/src/setup-cli.test.ts | 266 ++++++++++++ packages/code/src/setup-cli.ts | 253 +++++++++++ packages/code/src/setup.test.ts | 334 ++++++++++++++ packages/code/src/setup.ts | 254 +++++++++++ packages/code/src/storage.ts | 39 +- packages/code/src/worker-profile.test.ts | 529 +++++++++++++++++++++++ packages/code/src/worker-profile.ts | 200 +++++++++ 10 files changed, 1995 insertions(+), 13 deletions(-) create mode 100644 packages/code/src/setup-cli.test.ts create mode 100644 packages/code/src/setup-cli.ts create mode 100644 packages/code/src/setup.test.ts create mode 100644 packages/code/src/setup.ts create mode 100644 packages/code/src/worker-profile.test.ts create mode 100644 packages/code/src/worker-profile.ts diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 85ce947e..510d67e4 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -5,6 +5,10 @@ import { readdir, realpath, stat } from 'node:fs/promises'; import { basename, dirname, join, resolve, relative, isAbsolute, sep } from 'node:path'; import { pairBridgeWorker } from './pairing.js'; +import { runSetupCommand } from './setup-cli.js'; +import { profileRunConfiguration } from './setup.js'; +import { assertWorkerProfileIsolation } from './worker-profile.js'; +import type { WorkerProfile } from './worker-profile.js'; import { discoverProjects } from './projects.js'; import { loadProjectRoots, projectRootArguments } from './project-roots.js'; import { @@ -358,6 +362,7 @@ async function relay(): Promise { async function run( runtimeSessionId?: string, args: string[] = [], + savedProfile?: { path: string; config: WorkerProfile }, ): Promise { const projectArgs = projectRootArguments(args); if ( @@ -422,7 +427,7 @@ async function run( } const configuredWorkerId = process.env.LIBRECHAT_CODE_WORKER_ID?.trim(); const configuredIdentityPath = - process.env.LIBRECHAT_CODE_IDENTITY_FILE?.trim(); + savedProfile?.config.identityPath ?? process.env.LIBRECHAT_CODE_IDENTITY_FILE?.trim(); const configuredToken = process.env.LIBRECHAT_CODE_WORKER_TOKEN?.trim(); const identityPath = configuredIdentityPath ?? @@ -645,6 +650,9 @@ async function run( throw new Error('Invalid workspace registration'); } } + if (savedProfile) { + await assertWorkerProfileIsolation(savedProfile.path, identityPath!, canonicalWorkerDirectory); + } const mutationQuarantinePath = (allowWorkspaceWrites || allowWorkspaceCommands) && canonicalWorkerDirectory @@ -1092,6 +1100,7 @@ async function run( commandPolicy, protectedPaths: [ identityPath, + savedProfile?.path, ...(environments.some(environment => environment.definition.setup?.reuse) ? [preparationDirectory] : []), ...snapshotStores.map(store => store.store), ...environments.map(environment => environment.path), @@ -1766,6 +1775,26 @@ async function clearMutationQuarantine(args: string[]): Promise { async function main(): Promise { const args = process.argv.slice(2); + if (args.length === 1 && (args[0] === '--help' || args[0] === 'help')) { + process.stdout.write(`Usage: librechat-code + + setup Guided pairing and private profile setup + run [--profile ] Start a saved profile or existing worker configuration + pair --worker-id Existing one-time pairing flow + projects --root Inspect local Git projects + reset-workspace Existing runtime maintenance + clear-workspace-quarantine Existing workspace maintenance + +Run librechat-code setup --help for setup options. +Legacy LIBRECHAT_CODE_* configuration takes precedence over the default profile. +Explicit profiles cannot be mixed with legacy settings or run options. +`); + return; + } + if (args[0] === 'setup') { + await runSetupCommand(args.slice(1)); + return; + } if (args[0] === 'prune-environment-storage') { const paths: string[] = []; for (let i = 1; i < args.length; i++) { @@ -1823,7 +1852,10 @@ async function main(): Promise { if (args[0] && args[0] !== 'run') { throw new Error(`Unknown command: ${args[0]}`); } - await run(undefined, args.slice(1)); + const configuration = await profileRunConfiguration(args.slice(1)); + await run(undefined, configuration.args, configuration.profile && configuration.profilePath + ? { path: configuration.profilePath, config: configuration.profile } + : undefined); } main().catch((error: Error) => { process.stderr.write(`librechat-code: ${error.message}\n`); diff --git a/packages/code/src/pairing.test.ts b/packages/code/src/pairing.test.ts index 2b564394..a7dd6bd0 100644 --- a/packages/code/src/pairing.test.ts +++ b/packages/code/src/pairing.test.ts @@ -34,3 +34,61 @@ test('pairing binds a generated worker key to a single-use code', async () => { assert.match(paired.publicKey, /BEGIN PUBLIC KEY/); assert.match(paired.privateKey, /BEGIN PRIVATE KEY/); }); + +test('pairing propagates cancellation and never follows a code-bearing redirect', async () => { + const controller = new AbortController(); + const paired = await pairBridgeWorker({ + codeApiUrl: 'https://code.example/v1', + workerId: 'vm-1', + code: 'one-time-code', + signal: controller.signal, + fetchImpl: async (_input, init) => { + assert.equal(init?.redirect, 'error'); + assert.equal(init?.signal, controller.signal); + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + credential: 'issued-short-lived-credential-value', + expiresAt: new Date(Date.now() + 300_000).toISOString(), + }); + }, + }); + assert.equal(paired.workerId, 'vm-1'); +}); + +for (const payload of [ + null, + [], + 'not a response', + { protocolVersion: 2 }, + { protocolVersion: 1, workerId: 'another-worker' }, +]) { + test(`pairing rejects malformed response ${JSON.stringify(payload)}`, async () => { + await assert.rejects( + pairBridgeWorker({ + codeApiUrl: 'https://code.example/v1', + workerId: 'vm-1', + code: 'one-time-code', + fetchImpl: async () => Response.json(payload), + }), + /invalid/, + ); + }); +} + +test('non-JSON pairing failures have a bounded diagnosis without echoing response contents', async () => { + await assert.rejects( + pairBridgeWorker({ + codeApiUrl: 'https://code.example/v1', + workerId: 'vm-1', + code: 'one-time-code', + fetchImpl: async () => + new Response('private upstream response', { status: 502 }), + }), + error => { + assert.match(String(error), /invalid pairing response/); + assert.ok(!String(error).includes('private upstream response')); + return true; + }, + ); +}); diff --git a/packages/code/src/pairing.ts b/packages/code/src/pairing.ts index ed33fb18..d1504e82 100644 --- a/packages/code/src/pairing.ts +++ b/packages/code/src/pairing.ts @@ -1,8 +1,5 @@ import { createBridgeIdentity } from './identity.js'; -import { - BRIDGE_PROTOCOL_VERSION, - BridgeProtocolError, -} from './protocol.js'; +import { BRIDGE_PROTOCOL_VERSION, BridgeProtocolError } from './protocol.js'; import type { BridgeWorkerCredentialResponse } from './protocol.js'; @@ -11,6 +8,7 @@ export interface PairBridgeWorkerOptions { workerId: string; code: string; fetchImpl?: typeof fetch; + signal?: AbortSignal; } export interface PairedBridgeWorkerIdentity @@ -24,8 +22,8 @@ function normalizedBaseUrl(value: string): string { return value.replace(/\/+$/, ''); } -function errorMessage(value: object): string | undefined { - if ('error' in value && typeof value.error === 'string') return value.error; +function errorMessage(value: Record): string | undefined { + if (typeof value.error === 'string') return value.error; return undefined; } @@ -38,6 +36,8 @@ export async function pairBridgeWorker( `${codeApiUrl}/bridge/pairings/redeem`, { method: 'POST', + redirect: 'error', + signal: options.signal, headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ protocolVersion: BRIDGE_PROTOCOL_VERSION, @@ -47,10 +47,29 @@ export async function pairBridgeWorker( }), }, ); - const payload = (await response.json()) as object; + let payload: unknown; + try { + payload = await response.json(); + } catch { + throw new BridgeProtocolError( + 'Code API returned an invalid pairing response', + response.status, + ); + } + if ( + typeof payload !== 'object' || + payload == null || + Array.isArray(payload) + ) { + throw new BridgeProtocolError( + 'Code API returned an invalid pairing response', + response.status, + ); + } if (!response.ok) { throw new BridgeProtocolError( - errorMessage(payload) ?? `Bridge pairing failed with HTTP ${response.status}`, + errorMessage(payload as Record) ?? + `Bridge pairing failed with HTTP ${response.status}`, response.status, ); } @@ -62,7 +81,9 @@ export async function pairBridgeWorker( credential.credential.length < 32 || !Number.isFinite(Date.parse(credential.expiresAt)) ) { - throw new BridgeProtocolError('Code API returned an invalid worker credential'); + throw new BridgeProtocolError( + 'Code API returned an invalid worker credential', + ); } return { ...credential, diff --git a/packages/code/src/setup-cli.test.ts b/packages/code/src/setup-cli.test.ts new file mode 100644 index 00000000..1bef6e14 --- /dev/null +++ b/packages/code/src/setup-cli.test.ts @@ -0,0 +1,266 @@ +import assert from 'node:assert/strict'; +import { spawn, spawnSync } from 'node:child_process'; +import { once } from 'node:events'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { createServer } from 'node:http'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import test from 'node:test'; +import type { TestContext } from 'node:test'; + +const cli = fileURLToPath(new URL('./cli.js', import.meta.url)); +const code = 'secret-one-time-code-123456789'; + +async function fixture(t: TestContext) { + const home = await mkdtemp(join(tmpdir(), 'setup-cli-home-')); + t.after(() => rm(home, { recursive: true, force: true })); + const env = { + ...Object.fromEntries( + Object.entries(process.env).filter( + ([key]) => !key.startsWith('LIBRECHAT_CODE_'), + ), + ), + HOME: home, + }; + const profilePath = join( + home, + '.config', + 'librechat', + 'code', + 'profiles', + 'default.json', + ); + return { home, env, profilePath }; +} + +async function execute(args: string[], env: NodeJS.ProcessEnv, stdin = '') { + const child = spawn(process.execPath, [cli, ...args], { env, stdio: 'pipe' }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', chunk => { + stdout += String(chunk); + }); + child.stderr.on('data', chunk => { + stderr += String(chunk); + }); + child.stdin.end(stdin); + const [status] = await once(child, 'exit', { + signal: AbortSignal.timeout(10_000), + }); + return { status, stdout, stderr }; +} + +test('CLI help explains the pairing prerequisite and setup does not silently read piped prompts', async t => { + const { env } = await fixture(t); + const help = spawnSync(process.execPath, [cli, 'setup', '--help'], { + env, + encoding: 'utf8', + }); + assert.equal(help.status, 0, help.stderr); + assert.match(help.stdout, /Settings > Code Environments/); + assert.match(help.stdout, /does not\nsign in to LibreChat/); + const guided = await execute(['setup'], env, code); + assert.notEqual(guided.status, 0); + assert.match(guided.stderr, /requires a terminal/); + assert.ok(!guided.stdout.includes(code)); + assert.ok(!guided.stderr.includes(code)); +}); + +test( + 'CLI setup followed by plain run registers the saved identity with read-only access', + { timeout: 15_000 }, + async t => { + const { env, profilePath } = await fixture(t); + let paired = false; + let registered!: (body: Record) => void; + const registration = new Promise>(resolve => { + registered = resolve; + }); + const server = createServer(async (request, response) => { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + const body = JSON.parse(Buffer.concat(chunks).toString('utf8') || '{}'); + response.setHeader('Content-Type', 'application/json'); + if (request.url === '/v1/bridge/pairings/redeem') { + assert.equal(body.workerId, 'guided-vm'); + assert.equal(body.code, code); + assert.match(body.publicKey, /BEGIN PUBLIC KEY/); + assert.equal(body.privateKey, undefined); + paired = true; + response.end( + JSON.stringify({ + protocolVersion: 1, + workerId: body.workerId, + credential: 'short-lived-worker-credential-value', + expiresAt: '2099-01-01T00:00:00Z', + }), + ); + return; + } + if (request.url === '/v1/bridge/workers/register') { + assert.equal(paired, true); + assert.match(request.headers.authorization ?? '', /^Bridge /); + assert.ok(request.headers['x-librechat-code-signature']); + registered(body); + response.end( + JSON.stringify({ + protocolVersion: 1, + workerId: body.workerId, + incarnationId: body.incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + }), + ); + return; + } + response.end(JSON.stringify({ protocolVersion: 1, serverElapsedMs: 0 })); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(() => { + server.closeAllConnections(); + server.close(); + }); + const address = server.address(); + assert.ok(address && typeof address !== 'string'); + const result = await execute( + [ + 'setup', + '--non-interactive', + '--code-stdin', + '--url', + `http://127.0.0.1:${address.port}/v1`, + '--worker-id', + 'guided-vm', + '--default-workspace', + ], + env, + `${code}\n`, + ); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /librechat-code run\n/); + assert.match(result.stdout, /still needs to start/); + assert.ok(!result.stdout.includes(code)); + assert.ok(!result.stderr.includes(code)); + const profile = JSON.parse(await readFile(profilePath, 'utf8')); + assert.equal(profile.allowWorkspaceWrites, false); + assert.equal(profile.allowWorkspaceCommands, false); + const worker = spawn(process.execPath, [cli, 'run'], { + env, + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stderr = ''; + worker.stderr.on('data', chunk => { + stderr += String(chunk); + }); + const exited = once(worker, 'exit'); + t.after(() => worker.kill('SIGKILL')); + const body = await Promise.race([ + registration, + exited.then(() => { + throw new Error(`Worker exited before registration: ${stderr}`); + }), + ]); + worker.kill('SIGTERM'); + await exited; + assert.equal(body.workerId, 'guided-vm'); + const operations = body.capabilities.workspaceTools.operations as string[]; + assert.ok(operations.includes('read_file')); + assert.ok(!operations.includes('write_file')); + assert.ok(!operations.includes('execute_command')); + assert.ok(!JSON.stringify(body).includes(profile.identityPath)); + assert.ok(!JSON.stringify(body).includes(profilePath)); + }, +); + +test('CLI rejects missing automation inputs without consuming stdin or pairing', async t => { + const { env } = await fixture(t); + const result = await execute(['setup', '--non-interactive'], env, code); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /requires --url, --worker-id, --code-stdin/); + assert.ok(!result.stderr.includes(code)); +}); + +test('CLI refuses unsafe URLs before disclosing the code', async t => { + const { env } = await fixture(t); + const result = await execute( + [ + 'setup', + '--non-interactive', + '--code-stdin', + '--url', + 'http://external.example/v1', + '--worker-id', + 'vm', + '--default-workspace', + ], + env, + code, + ); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /requires HTTPS/); + assert.ok(!result.stdout.includes(code)); + assert.ok(!result.stderr.includes(code)); +}); + +for (const [name, secretEditing] of [ + ['history', ''], + ['kill beginning', '\x15\x19'], + ['kill end', '\x01\x0b\x19'], + ['kill word', '\x17\x19'], +]) + test( + `terminal ${name} never recalls a hidden pairing code at confirmation`, + { timeout: 10_000 }, + async t => { + const { env } = await fixture(t); + const secret = 'SyntheticPairingSecret0123456789'; + // Real readline terminal editing over pipes avoids an OS-specific PTY dependency. + const bootstrap = ` + Object.defineProperty(process.stdin, 'isTTY', { value: true }); + Object.defineProperty(process.stdout, 'isTTY', { value: true }); + process.argv = [process.execPath, ${JSON.stringify(cli)}, 'setup', '--url', + 'https://code.example/v1', '--worker-id', 'history-vm', '--default-workspace']; + await import(${JSON.stringify(new URL('./cli.js', import.meta.url).href)}); + `; + const child = spawn( + process.execPath, + ['--input-type=module', '--eval', bootstrap], + { env, stdio: 'pipe' }, + ); + t.after(() => child.kill('SIGKILL')); + let transcript = ''; + let stderr = ''; + let step = 0; + const inputs = [ + { prompt: 'Allow file writes', input: '\n' }, + { prompt: 'Allow sandboxed commands', input: '\n' }, + { + prompt: 'One-time pairing code (hidden): ', + input: `${secret}${secretEditing}\n`, + }, + // History and yank must not reveal secret editing state before redemption. + { + prompt: 'Pair this machine and save these choices?', + input: '\x1b[A\x10\x19\x15n\n', + }, + ]; + child.stderr.on('data', chunk => { + stderr += String(chunk); + }); + child.stdout.on('data', chunk => { + transcript += String(chunk); + if (step < inputs.length && transcript.includes(inputs[step].prompt)) { + child.stdin.write(inputs[step++].input); + } + }); + const [status] = await once(child, 'exit'); + assert.equal(step, inputs.length, stderr); + assert.equal(status, 0, stderr); + assert.ok( + !transcript.includes(secret), + 'hidden pairing code was recalled visibly from history', + ); + assert.match(transcript, /No pairing was redeemed/); + }, + ); diff --git a/packages/code/src/setup-cli.ts b/packages/code/src/setup-cli.ts new file mode 100644 index 00000000..877a5c17 --- /dev/null +++ b/packages/code/src/setup-cli.ts @@ -0,0 +1,253 @@ +import { createInterface } from 'node:readline/promises'; +import { Writable } from 'node:stream'; +import { resolve } from 'node:path'; + +import { + setupCodeApiUrl, + setupWorker, + setupWorkspaceDirectory, +} from './setup.js'; +import { defaultWorkerProfilePath } from './worker-profile.js'; +import { isValidBridgeWorkerId } from './protocol.js'; + +export const SETUP_HELP = `Usage: librechat-code setup [options] + +Create a private worker profile using a one-time pairing from LibreChat. +Open Settings > Code Environments in LibreChat to add the machine first. +Use its Code API URL, worker ID, and pairing code. This command does not +sign in to LibreChat or bypass its enrollment policy. + + --url Code API URL, including its /v1 prefix + --worker-id Worker ID supplied by LibreChat + --worker-dir Expose one existing project directory + --default-workspace Create a private workspace when the worker starts + --allow-workspace-writes Explicitly enable file writes (default: off) + --allow-workspace-commands Explicitly enable sandboxed commands (default: off) + --profile Profile destination (default: profiles/default.json) + --identity Separate private identity destination + --non-interactive Require explicit inputs instead of prompting + --code-stdin Read the pairing code from stdin, never an argument + --help Show this help + +Existing profiles and identities are never replaced. Choose another profile +and identity for a second machine. Setup does not install OS dependencies, +start a background service, or change the restricted command policy. +`; + +const VALUE_FLAGS = new Set([ + '--url', + '--worker-id', + '--worker-dir', + '--profile', + '--identity', +]); +const BOOLEAN_FLAGS = new Set([ + '--default-workspace', + '--allow-workspace-writes', + '--allow-workspace-commands', + '--non-interactive', + '--code-stdin', +]); + +export function parseSetupArguments( + args: string[], +): Map { + const options = new Map(); + for (let i = 0; i < args.length; i++) { + const [flag, ...inline] = args[i].split('='); + if ( + (!VALUE_FLAGS.has(flag) && !BOOLEAN_FLAGS.has(flag)) || + options.has(flag) + ) { + throw new Error( + 'Unknown or repeated setup option; use librechat-code setup --help', + ); + } + if (BOOLEAN_FLAGS.has(flag)) { + if (inline.length) + throw new Error( + 'Setup switches do not accept a value; use setup --help', + ); + options.set(flag, true); + } else { + const value = inline.length ? inline.join('=') : args[++i]; + if ( + !value?.trim() || + value.startsWith('--') || + /[\x00-\x1f\x7f]/.test(value) + ) { + throw new Error( + 'A setup option is missing its value; use setup --help', + ); + } + options.set(flag, value); + } + } + if (options.has('--worker-dir') && options.has('--default-workspace')) { + throw new Error('Choose --worker-dir or --default-workspace, not both'); + } + if ( + options.has('--non-interactive') && + (!options.has('--url') || + !options.has('--worker-id') || + !options.has('--code-stdin') || + (!options.has('--worker-dir') && !options.has('--default-workspace'))) + ) { + throw new Error( + 'Non-interactive setup requires --url, --worker-id, --code-stdin, and --worker-dir or --default-workspace', + ); + } + if (options.has('--code-stdin') && !options.has('--non-interactive')) { + throw new Error('--code-stdin requires --non-interactive'); + } + return options; +} + +async function pairingCodeFromStdin(signal: AbortSignal): Promise { + let code = ''; + const abort = () => process.stdin.destroy(new Error('Setup cancelled')); + signal.addEventListener('abort', abort, { once: true }); + try { + signal.throwIfAborted(); + for await (const chunk of process.stdin) { + code += String(chunk); + if (Buffer.byteLength(code) > 1024) + throw new Error('Pairing code input is too large'); + signal.throwIfAborted(); + } + return code.trim(); + } finally { + signal.removeEventListener('abort', abort); + } +} + +export async function runSetupCommand(args: string[]): Promise { + if (args.length === 1 && args[0] === '--help') { + process.stdout.write(SETUP_HELP); + return; + } + const options = parseSetupArguments(args); + const interactive = !options.has('--non-interactive'); + if (interactive && (!process.stdin.isTTY || !process.stdout.isTTY)) { + throw new Error( + 'Guided setup requires a terminal. For automation use --non-interactive and --code-stdin; see setup --help', + ); + } + const controller = new AbortController(); + const abort = () => controller.abort(new Error('Setup cancelled')); + process.once('SIGINT', abort); + let muted = false; + const output = new Writable({ + write(chunk, _encoding, callback) { + if (!muted) process.stdout.write(chunk); + callback(); + }, + }); + const ask = async (message: string, secret = false): Promise => { + controller.signal.throwIfAborted(); + muted = secret; + // Readline retains more than history (including a kill ring). Never reuse prompt state. + const terminal = createInterface({ + input: process.stdin, + output, + terminal: true, + historySize: 0, + }); + terminal.on('SIGINT', abort); + terminal.once('close', abort); + process.stdout.write(message); + try { + return ( + await terminal.question('', { signal: controller.signal }) + ).trim(); + } finally { + terminal.off('close', abort); + terminal.off('SIGINT', abort); + terminal.close(); + muted = false; + if (secret) process.stdout.write('\n'); + } + }; + const yesNo = async (message: string): Promise => { + for (;;) { + const value = (await ask(`${message} [y/N]: `)).toLowerCase(); + if (value === '' || value === 'n' || value === 'no') return false; + if (value === 'y' || value === 'yes') return true; + process.stdout.write('Enter yes or no.\n'); + } + }; + try { + if (interactive) + process.stdout.write( + 'Connect a machine using the pairing supplied by LibreChat.\nOnly selected folders and permissions will be exposed.\n\n', + ); + const codeApiUrl = setupCodeApiUrl( + String( + options.get('--url') ?? (await ask('Code API URL (including /v1): ')), + ), + ); + const workerId = String( + options.get('--worker-id') ?? (await ask('Worker ID from LibreChat: ')), + ); + if (!isValidBridgeWorkerId(workerId)) + throw new Error('Enter the worker ID supplied by LibreChat'); + let workspaceDirectory = options.get('--worker-dir') as string | undefined; + if ( + interactive && + !workspaceDirectory && + !options.has('--default-workspace') + ) { + workspaceDirectory = + (await ask( + 'Project directory (Enter for a new private workspace): ', + )) || undefined; + } + if (workspaceDirectory != null) + workspaceDirectory = await setupWorkspaceDirectory(workspaceDirectory); + const allowWorkspaceWrites = + options.has('--allow-workspace-writes') || + (interactive && + (await yesNo('Allow file writes within this workspace?'))); + const allowWorkspaceCommands = + options.has('--allow-workspace-commands') || + (interactive && + (await yesNo( + 'Allow sandboxed commands within this workspace? Commands can modify workspace files', + ))); + const profilePath = resolve( + String(options.get('--profile') ?? defaultWorkerProfilePath()), + ); + const code = interactive + ? await ask('One-time pairing code (hidden): ', true) + : await pairingCodeFromStdin(controller.signal); + process.stdout.write( + `\nCode API: ${codeApiUrl}\nWorker: ${workerId}\nWorkspace: ${workspaceDirectory ?? 'new private workspace'}\nFile writes: ${allowWorkspaceWrites ? 'enabled' : 'disabled'}\nCommands: ${allowWorkspaceCommands ? 'enabled, restricted sandbox policy' : 'disabled'}\nProfile: ${profilePath}\n`, + ); + if ( + interactive && + !(await yesNo('Pair this machine and save these choices?')) + ) { + process.stdout.write('Setup cancelled. No pairing was redeemed.\n'); + return; + } + await setupWorker({ + codeApiUrl, + workerId, + code, + profilePath, + identityPath: options.get('--identity') as string | undefined, + workspaceDirectory, + allowWorkspaceWrites, + allowWorkspaceCommands, + signal: controller.signal, + }); + const defaultProfile = profilePath === resolve(defaultWorkerProfilePath()); + const quotedProfile = `'${profilePath.replace(/'/g, "'\\''")}'`; + process.stdout.write( + `\nSetup saved. Start the worker with:\n librechat-code run${defaultProfile ? '' : ` --profile ${quotedProfile}`}\n\nPairing is complete. The worker still needs to start, prepare its sandbox,\nand register before LibreChat can report it ready.\n`, + ); + } finally { + process.off('SIGINT', abort); + output.destroy(); + } +} diff --git a/packages/code/src/setup.test.ts b/packages/code/src/setup.test.ts new file mode 100644 index 00000000..0241f083 --- /dev/null +++ b/packages/code/src/setup.test.ts @@ -0,0 +1,334 @@ +import assert from 'node:assert/strict'; +import { + mkdir, + mkdtemp, + readFile, + readdir, + rm, + writeFile, +} from 'node:fs/promises'; +import { homedir, tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { createBridgeIdentity } from './identity.js'; +import { loadBridgeIdentity } from './storage.js'; +import { loadWorkerProfile, saveWorkerProfile } from './worker-profile.js'; +import { + profileRunConfiguration, + setupCodeApiUrl, + setupWorker, +} from './setup.js'; +import { parseSetupArguments } from './setup-cli.js'; +import type { SetupWorkerOptions } from './setup.js'; +import type { PairBridgeWorkerOptions } from './pairing.js'; + +const code = 'one-time-code-value-123456789'; + +async function fixture(t: test.TestContext) { + const root = await mkdtemp(join(tmpdir(), 'worker-setup-')); + t.after(() => rm(root, { recursive: true, force: true })); + const options: SetupWorkerOptions = { + codeApiUrl: 'https://code.example/v1/', + workerId: 'setup-worker', + code, + profilePath: join(root, 'profiles', 'default.json'), + identityPath: join(root, 'identity.json'), + allowWorkspaceWrites: false, + allowWorkspaceCommands: false, + }; + const pair = async (input: PairBridgeWorkerOptions) => ({ + protocolVersion: 1 as const, + workerId: input.workerId, + codeApiUrl: input.codeApiUrl, + ...createBridgeIdentity(), + credential: 'issued-short-lived-credential-value', + expiresAt: '2099-01-01T00:00:00Z', + }); + return { root, options, pair }; +} + +test('setup saves separate identity and least-privilege profile after pairing', async t => { + const { options, pair } = await fixture(t); + let calls = 0; + const saved = await setupWorker(options, { + pair: async input => { + calls++; + assert.equal(input.codeApiUrl, 'https://code.example/v1'); + assert.equal(input.code, code); + assert.ok(input.signal); + return pair(input); + }, + }); + assert.equal(calls, 1); + assert.deepEqual(await loadWorkerProfile(saved.profilePath), saved.profile); + assert.deepEqual(saved.profile.workspace, { type: 'private' }); + assert.equal(saved.profile.allowWorkspaceCommands, false); + assert.equal(saved.profile.allowWorkspaceWrites, false); + assert.equal( + (await loadBridgeIdentity(saved.profile.identityPath)).workerId, + 'setup-worker', + ); + const profileSource = await readFile(saved.profilePath, 'utf8'); + assert.ok(!profileSource.includes(code)); + assert.ok(!profileSource.includes('PRIVATE KEY')); + assert.ok(!profileSource.includes('issued-short-lived')); +}); + +test('setup canonicalizes an explicitly selected project and preflights command dependencies', async t => { + const { root, options, pair } = await fixture(t); + const project = join(root, 'project with spaces'); + await mkdir(project); + let checked = false; + const saved = await setupWorker( + { ...options, workspaceDirectory: project, allowWorkspaceCommands: true }, + { + checkCommands: async () => { + checked = true; + }, + pair: async input => { + assert.equal(checked, true); + return pair(input); + }, + }, + ); + assert.deepEqual(saved.profile.workspace, { + type: 'directory', + path: project, + }); + assert.equal(saved.profile.allowWorkspaceCommands, true); + assert.equal(saved.profile.allowWorkspaceWrites, false); +}); + +test('missing command prerequisites do not spend pairing code or create storage', async t => { + const { root, options, pair } = await fixture(t); + await assert.rejects( + setupWorker( + { ...options, allowWorkspaceCommands: true }, + { + checkCommands: async () => { + throw new Error('missing sandbox dependency'); + }, + pair: async input => { + assert.fail('must not pair'); + return pair(input); + }, + }, + ), + /missing sandbox dependency/, + ); + assert.deepEqual(await readdir(root), []); +}); + +test('denied or expired pairing releases only the new empty reservations', async t => { + const { root, options } = await fixture(t); + await assert.rejects( + setupWorker(options, { + pair: async () => { + throw new Error('Pairing code is invalid or expired'); + }, + }), + /invalid or expired/, + ); + assert.deepEqual(await readdir(root), ['profiles']); + assert.deepEqual(await readdir(join(root, 'profiles')), []); +}); + +test('existing identity is never replaced and code is not redeemed', async t => { + const { options } = await fixture(t); + await writeFile(options.identityPath!, 'keep this identity', { mode: 0o600 }); + await assert.rejects( + setupWorker(options, { + pair: async () => { + assert.fail('must not pair'); + }, + }), + /already exists/, + ); + assert.equal( + await readFile(options.identityPath!, 'utf8'), + 'keep this identity', + ); + assert.equal(await loadWorkerProfile(options.profilePath!), undefined); +}); + +test('profile write failure preserves the paired identity for manual recovery', async t => { + const { options, pair } = await fixture(t); + await assert.rejects( + setupWorker(options, { + pair: async input => { + await rm(options.profilePath!); + await mkdir(options.profilePath!); + return pair(input); + }, + }), + /Machine paired and identity saved.*profile could not be saved/, + ); + assert.equal( + (await loadBridgeIdentity(options.identityPath!)).workerId, + options.workerId, + ); +}); + +test('setup rejects exposed control files even with read-only permissions', async t => { + const { root, options } = await fixture(t); + await assert.rejects( + setupWorker( + { ...options, workspaceDirectory: root }, + { + pair: async () => { + assert.fail('must not pair'); + }, + }, + ), + /outside the workspace/, + ); +}); + +test('setup never admits home or a parent as a project', async t => { + const { options } = await fixture(t); + for (const workspaceDirectory of [homedir(), '/']) { + await assert.rejects( + setupWorker({ ...options, workspaceDirectory }), + /Do not expose your home directory/, + ); + } +}); + +test('setup stops before storage or pairing when cancelled', async t => { + const { root, options } = await fixture(t); + const controller = new AbortController(); + controller.abort(new Error('cancelled by owner')); + await assert.rejects( + setupWorker({ ...options, signal: controller.signal }), + /cancelled by owner/, + ); + assert.deepEqual(await readdir(root), []); +}); + +test('setup accepts HTTPS prefixes and only permits insecure HTTP on loopback', () => { + assert.equal( + setupCodeApiUrl('https://code.example/proxy/v1/'), + 'https://code.example/proxy/v1', + ); + for (const host of ['localhost', '127.0.0.1', '[::1]']) { + assert.equal( + setupCodeApiUrl(`http://${host}:3000/v1`), + `http://${host}:3000/v1`, + ); + } + for (const value of [ + 'http://code.example/v1', + 'ftp://code.example', + 'https://user:secret@code.example/v1', + 'https://code.example/v1?token=secret', + 'https://code.example/v1#fragment', + 'https://code.example/\n', + ]) { + assert.throws(() => setupCodeApiUrl(value), /HTTPS|credentials/); + } +}); + +test('saved profile startup never merges legacy identity or permission settings', async t => { + const { options, pair } = await fixture(t); + const saved = await setupWorker(options, { pair }); + const selected = await profileRunConfiguration( + ['--profile', saved.profilePath], + {}, + ); + assert.equal(selected.profilePath, saved.profilePath); + assert.deepEqual(selected.args, ['--default-workspace']); + assert.equal(selected.profile?.identityPath, options.identityPath); + for (const key of [ + 'LIBRECHAT_CODE_WORKER_TOKEN', + 'LIBRECHAT_CODE_IDENTITY_FILE', + 'LIBRECHAT_CODE_ALLOW_WORKSPACE_COMMANDS', + ]) { + await assert.rejects( + profileRunConfiguration(['--profile', saved.profilePath], { + [key]: 'configured', + }), + /cannot be combined/, + ); + assert.deepEqual( + await profileRunConfiguration(['--worker-dir', '/legacy'], { + [key]: 'configured', + }), + { args: ['--worker-dir', '/legacy'] }, + ); + } + await assert.rejects( + profileRunConfiguration( + ['--profile', saved.profilePath, '--allow-workspace-commands'], + {}, + ), + /cannot be combined with run options/, + ); +}); + +test('explicit profile parse rejects missing, repeated, absent, or corrupt profiles', async t => { + const { options } = await fixture(t); + await assert.rejects( + profileRunConfiguration(['--profile'], {}), + /requires a file/, + ); + await assert.rejects( + profileRunConfiguration(['--profile=a', '--profile=b'], {}), + /only once/, + ); + await assert.rejects( + profileRunConfiguration(['--profile', options.profilePath!], {}), + /not found/, + ); + await mkdir(join(options.profilePath!, '..'), { recursive: true }); + await writeFile(options.profilePath!, '{}', { mode: 0o600 }); + await assert.rejects( + profileRunConfiguration(['--profile', options.profilePath!], {}), + /Invalid worker profile/, + ); +}); + +test('profile file environment variable selects the same profile without changing process environment', async t => { + const { options } = await fixture(t); + const profile = { + version: 1 as const, + identityPath: options.identityPath!, + workspace: { type: 'private' as const }, + allowWorkspaceWrites: true, + allowWorkspaceCommands: false, + }; + await saveWorkerProfile(options.profilePath!, profile); + const environment = { LIBRECHAT_CODE_PROFILE_FILE: options.profilePath! }; + const result = await profileRunConfiguration([], environment); + assert.deepEqual(result.args, [ + '--default-workspace', + '--allow-workspace-writes', + ]); + assert.deepEqual(environment, { + LIBRECHAT_CODE_PROFILE_FILE: options.profilePath!, + }); +}); + +test('setup CLI parsing rejects ambiguity and requires explicit non-interactive access', () => { + for (const args of [ + ['--worker-id'], + ['--url', '--worker-id'], + ['--unknown'], + ['--url=a', '--url=b'], + ['--default-workspace=true'], + ['--worker-dir', '/project', '--default-workspace'], + ['--non-interactive'], + ['--code-stdin'], + ]) { + assert.throws(() => parseSetupArguments(args)); + } + const parsed = parseSetupArguments([ + '--non-interactive', + '--code-stdin', + '--url=https://code.example/v1', + '--worker-id=vm1', + '--default-workspace', + ]); + assert.equal(parsed.get('--allow-workspace-commands'), undefined); + assert.equal(parsed.get('--allow-workspace-writes'), undefined); +}); diff --git a/packages/code/src/setup.ts b/packages/code/src/setup.ts new file mode 100644 index 00000000..46879207 --- /dev/null +++ b/packages/code/src/setup.ts @@ -0,0 +1,254 @@ +import { access, realpath, stat } from 'node:fs/promises'; +import { constants } from 'node:fs'; +import { homedir } from 'node:os'; +import { isAbsolute, relative, resolve, sep } from 'node:path'; + +import { pairBridgeWorker } from './pairing.js'; +import { isValidBridgeWorkerId } from './protocol.js'; +import { assertPrivateStorageSupported } from './private-storage.js'; +import { + assertIdentityPathIsPrivate, + defaultBridgeIdentityPath, + saveBridgeIdentity, +} from './storage.js'; +import { + assertWorkerProfileIsolation, + defaultWorkerProfilePath, + loadWorkerProfile, + saveWorkerProfile, + workerProfileArguments, +} from './worker-profile.js'; +import type { WorkerProfile } from './worker-profile.js'; + +export interface SetupWorkerOptions { + codeApiUrl: string; + workerId: string; + code: string; + profilePath?: string; + identityPath?: string; + workspaceDirectory?: string; + allowWorkspaceWrites: boolean; + allowWorkspaceCommands: boolean; + signal?: AbortSignal; +} + +export function setupCodeApiUrl(value: string): string { + if (value.length > 2048 || /[\s\x00-\x1f\x7f]/.test(value)) { + throw new Error( + 'Enter a Code API HTTPS URL without credentials, query, or fragment', + ); + } + let url: URL; + try { + url = new URL(value); + } catch { + throw new Error( + 'Enter a valid Code API HTTPS URL, including its /v1 prefix', + ); + } + const loopback = ['localhost', '127.0.0.1', '[::1]'].includes(url.hostname); + if ( + (url.protocol !== 'https:' && !(url.protocol === 'http:' && loopback)) || + url.username || + url.password || + url.search || + url.hash + ) { + throw new Error( + 'Code API requires HTTPS; HTTP is allowed only on loopback, without credentials, query, or fragment', + ); + } + return url.href.replace(/\/+$/, ''); +} + +export async function checkSetupCommands(): Promise { + const { SandboxManager } = await import('@anthropic-ai/sandbox-runtime'); + if (!SandboxManager.isSupportedPlatform()) { + throw new Error( + 'Native sandbox commands require a supported macOS or Linux host', + ); + } + const dependencies = await SandboxManager.checkDependenciesAsync(); + if (dependencies.errors.length) { + throw new Error( + `Install the native sandbox prerequisites before pairing: ${dependencies.errors.join('; ')}`, + ); + } + await access('/bin/bash', constants.X_OK); +} + +export async function setupWorkspaceDirectory(value: string): Promise { + if (!value.trim() || /[\x00-\x1f\x7f]/.test(value)) { + throw new Error('Choose an existing project directory'); + } + const path = await realpath(resolve(value)); + if (!(await stat(path)).isDirectory()) + throw new Error('Workspace must be a directory'); + const home = await realpath(homedir()); + const homeRelative = relative(path, home); + if ( + homeRelative === '' || + (!isAbsolute(homeRelative) && + homeRelative !== '..' && + !homeRelative.startsWith(`..${sep}`)) + ) { + throw new Error( + 'Do not expose your home directory or a directory containing it; choose a project or a private workspace', + ); + } + return path; +} + +export async function setupWorker( + options: SetupWorkerOptions, + dependencies: { + pair?: typeof pairBridgeWorker; + checkCommands?: typeof checkSetupCommands; + } = {}, +): Promise<{ profilePath: string; profile: WorkerProfile }> { + options.signal?.throwIfAborted(); + assertPrivateStorageSupported(); + const codeApiUrl = setupCodeApiUrl(options.codeApiUrl); + if (!isValidBridgeWorkerId(options.workerId)) + throw new Error('Enter the worker ID supplied by LibreChat'); + if (!/^[A-Za-z0-9_-]{16,256}$/.test(options.code)) + throw new Error('Enter a valid one-time pairing code from LibreChat'); + const profilePath = resolve( + options.profilePath ?? defaultWorkerProfilePath(), + ); + const identityPath = resolve( + options.identityPath ?? defaultBridgeIdentityPath(options.workerId), + ); + if ( + [profilePath, identityPath].some( + path => Buffer.byteLength(path) > 4096 || /[\x00-\x1f\x7f]/.test(path), + ) + ) { + throw new Error( + 'Profile and identity paths must contain no control characters and be at most 4096 bytes', + ); + } + if ( + typeof options.allowWorkspaceWrites !== 'boolean' || + typeof options.allowWorkspaceCommands !== 'boolean' + ) { + throw new Error( + 'Workspace permissions must be explicitly enabled or disabled', + ); + } + const workspaceDirectory = + options.workspaceDirectory == null + ? undefined + : await setupWorkspaceDirectory(options.workspaceDirectory); + const profile: WorkerProfile = { + version: 1, + identityPath, + workspace: + workspaceDirectory == null + ? { type: 'private' } + : { type: 'directory', path: workspaceDirectory }, + allowWorkspaceWrites: options.allowWorkspaceWrites, + allowWorkspaceCommands: options.allowWorkspaceCommands, + }; + if (profile.allowWorkspaceCommands) + await (dependencies.checkCommands ?? checkSetupCommands)(); + options.signal?.throwIfAborted(); + const profileReservation = await assertIdentityPathIsPrivate( + profilePath, + true, + ); + try { + const identityReservation = await assertIdentityPathIsPrivate( + identityPath, + true, + ); + try { + await assertWorkerProfileIsolation( + profilePath, + identityPath, + workspaceDirectory, + ); + const identity = await (dependencies.pair ?? pairBridgeWorker)({ + codeApiUrl, + workerId: options.workerId, + code: options.code, + signal: + options.signal == null + ? AbortSignal.timeout(30_000) + : AbortSignal.any([options.signal, AbortSignal.timeout(30_000)]), + }); + await saveBridgeIdentity(identityPath, identity); + try { + await saveWorkerProfile(profilePath, profile); + } catch { + throw new Error( + `Machine paired and identity saved to ${identityPath}, but the profile could not be saved. Keep this identity and use LIBRECHAT_CODE_IDENTITY_FILE with the existing run command; do not pair again automatically.`, + ); + } + return { profilePath, profile }; + } finally { + await identityReservation.release(); + } + } finally { + await profileReservation.release(); + } +} + +export interface ProfileRunConfiguration { + args: string[]; + profilePath?: string; + profile?: WorkerProfile; +} + +export async function profileRunConfiguration( + args: string[], + environment: NodeJS.ProcessEnv = process.env, +): Promise { + const profileArgs = args.filter( + arg => arg === '--profile' || arg.startsWith('--profile='), + ); + if (profileArgs.length > 1) throw new Error('Supply --profile only once'); + let profilePath = environment.LIBRECHAT_CODE_PROFILE_FILE?.trim(); + if (profileArgs.length) { + const flag = profileArgs[0]; + const value = + flag === '--profile' + ? args[args.indexOf(flag) + 1] + : flag.slice('--profile='.length); + if (!value || value.startsWith('--')) + throw new Error('--profile requires a file'); + profilePath = value; + args = args.filter( + (_, index) => + index !== args.indexOf(flag) && + (flag !== '--profile' || index !== args.indexOf(flag) + 1), + ); + } + const legacyConfiguration = Object.entries(environment).some( + ([key, value]) => + key.startsWith('LIBRECHAT_CODE_') && + key !== 'LIBRECHAT_CODE_PROFILE_FILE' && + value?.trim(), + ); + if (legacyConfiguration) { + if (profilePath) + throw new Error( + 'Saved profiles cannot be combined with LIBRECHAT_CODE_* settings; unset them or use the existing run configuration', + ); + return { args }; + } + const path = resolve(profilePath ?? defaultWorkerProfilePath()); + const profile = await loadWorkerProfile(path); + if (!profile) { + if (profilePath) + throw new Error( + 'Saved profile not found; run librechat-code setup first', + ); + return { args }; + } + if (args.length) + throw new Error( + 'Saved profiles cannot be combined with run options; use a separate profile or the existing environment-variable configuration', + ); + return { profilePath: path, profile, args: workerProfileArguments(profile) }; +} diff --git a/packages/code/src/storage.ts b/packages/code/src/storage.ts index 904a8562..b7fb094d 100644 --- a/packages/code/src/storage.ts +++ b/packages/code/src/storage.ts @@ -223,7 +223,18 @@ async function readGuardedFile( if ((mode & 0o077) !== 0) throw new BridgeProtocolError(exposed(mode.toString(8))); } await assertPrivateStorageAcl(handle, path); - return await handle.readFile('utf8'); + if (!Number.isFinite(maxBytes)) return await handle.readFile('utf8'); + const buffer = Buffer.alloc(maxBytes + 1); + let length = 0; + while (length < buffer.length) { + const { bytesRead } = await handle.read(buffer, length, buffer.length - length, null); + if (bytesRead === 0) break; + length += bytesRead; + } + if (length > maxBytes) { + throw new BridgeProtocolError('Invalid private storage file size or type'); + } + return buffer.toString('utf8', 0, length); } finally { await handle.close(); } @@ -360,6 +371,7 @@ async function assertSiblingPublishable(path: string): Promise { export async function assertIdentityPathIsPrivate( path: string, + createOnly = false, ): Promise { assertPrivateStorageSupported(); await assertWriteContainerPrivate(path); @@ -389,6 +401,11 @@ export async function assertIdentityPathIsPrivate( throw error; } if (!isRecord(error) || error.code !== 'EEXIST') throw error; + if (createOnly) { + throw new BridgeProtocolError( + `${path} already exists; choose a different identity/profile path.`, + ); + } /* Already present: judge what is there instead of the placeholder, and * prove the publish itself is possible - an unwritable parent would * otherwise surface as EACCES only after the code was spent. */ @@ -465,7 +482,25 @@ export async function saveEnvironmentPreparationKey(path: string, key: string): await savePrivateJson(path, { version: 1, key }); } -async function savePrivateJson(path: string, value: unknown): Promise { +export async function readPrivateJson(path: string, maxBytes: number): Promise { + assertPrivateStorageSupported(); + if (!Number.isSafeInteger(maxBytes) || maxBytes < 1) { + throw new BridgeProtocolError('Private JSON file limit must be a positive integer'); + } + const content = await readGuardedFile( + path, + (mode) => `Private JSON file ${path} must be owner-only (mode ${mode})`, + maxBytes, + ); + await assertReadPathPrivate(path); + try { + return JSON.parse(content) as unknown; + } catch { + throw new BridgeProtocolError(`Invalid private JSON file: ${path}`); + } +} + +export async function savePrivateJson(path: string, value: unknown): Promise { assertPrivateStorageSupported(); await assertWriteContainerPrivate(path); await mkdir(dirname(path), { recursive: true, mode: 0o700 }); diff --git a/packages/code/src/worker-profile.test.ts b/packages/code/src/worker-profile.test.ts new file mode 100644 index 00000000..585144c0 --- /dev/null +++ b/packages/code/src/worker-profile.test.ts @@ -0,0 +1,529 @@ +import assert from 'node:assert/strict'; +import { + chmod, + link, + mkdir, + mkdtemp, + open, + readFile, + readdir, + rm, + stat, + symlink, + writeFile, +} from 'node:fs/promises'; +import type { FileHandle } from 'node:fs/promises'; +import { homedir, tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { assertIdentityPathIsPrivate } from './storage.js'; +import { + assertWorkerProfileIsolation, + defaultWorkerProfilePath, + loadWorkerProfile, + saveWorkerProfile, + workerProfileArguments, +} from './worker-profile.js'; +import type { WorkerProfile } from './worker-profile.js'; + +function profile(directory: string): WorkerProfile { + return { + version: 1, + identityPath: join(directory, 'identity.json'), + workspace: { type: 'private' }, + allowWorkspaceWrites: false, + allowWorkspaceCommands: false, + }; +} + +test('the default worker profile is separate from identity storage', () => { + assert.equal( + defaultWorkerProfilePath(), + join(homedir(), '.config', 'librechat', 'code', 'profiles', 'default.json'), + ); +}); + +test('worker profiles roundtrip atomically with owner-only file and directory permissions', async t => { + const directory = await mkdtemp(join(tmpdir(), 'worker-profile-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'profiles', 'default.json'); + const value = profile(directory); + await saveWorkerProfile(path, value); + assert.deepEqual(await loadWorkerProfile(path), value); + assert.deepEqual(JSON.parse(await readFile(path, 'utf8')), value); + assert.equal((await stat(path)).mode & 0o777, 0o600); + assert.equal((await stat(join(directory, 'profiles'))).mode & 0o777, 0o700); + + const updated: WorkerProfile = { + ...value, + workspace: { + type: 'directory', + path: join(directory, 'project with spaces'), + }, + allowWorkspaceWrites: true, + allowWorkspaceCommands: true, + }; + await saveWorkerProfile(path, updated); + assert.deepEqual(await loadWorkerProfile(path), updated); + assert.equal((await stat(path)).mode & 0o777, 0o600); + assert.deepEqual(await readdir(join(directory, 'profiles')), [ + 'default.json', + ]); +}); + +test('worker profile arguments enable only explicitly granted permissions', () => { + for (const workspace of [ + { type: 'private' as const }, + { type: 'directory' as const, path: '/srv/project with spaces' }, + ]) { + for (const allowWorkspaceWrites of [false, true]) { + for (const allowWorkspaceCommands of [false, true]) { + const value = { + ...profile('/srv'), + workspace, + allowWorkspaceWrites, + allowWorkspaceCommands, + }; + assert.deepEqual(workerProfileArguments(value), [ + ...(workspace.type === 'private' + ? ['--default-workspace'] + : ['--worker-dir', workspace.path]), + ...(allowWorkspaceWrites ? ['--allow-workspace-writes'] : []), + ...(allowWorkspaceCommands ? ['--allow-workspace-commands'] : []), + ]); + } + } + } +}); + +test('only an absent worker profile returns undefined', async t => { + const directory = await mkdtemp(join(tmpdir(), 'worker-profile-missing-')); + t.after(() => rm(directory, { recursive: true, force: true })); + assert.equal( + await loadWorkerProfile(join(directory, 'absent.json')), + undefined, + ); + assert.equal( + await loadWorkerProfile(join(directory, 'missing', 'absent.json')), + undefined, + ); + await assert.rejects(loadWorkerProfile(directory), /size or type/); + + const dangling = join(directory, 'dangling.json'); + await symlink(join(directory, 'absent.json'), dangling); + await assert.rejects(loadWorkerProfile(dangling), { code: 'ENOENT' }); + const loop = join(directory, 'loop.json'); + await symlink(loop, loop); + await assert.rejects(loadWorkerProfile(loop), /Too many storage symlinks/); +}); + +test('malformed, empty, and oversized profiles fail without disclosing their contents', async t => { + const directory = await mkdtemp( + join(tmpdir(), 'worker-profile-invalid-json-'), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'profile.json'); + for (const content of ['', ' ', '{"credential":"do-not-disclose",']) { + await writeFile(path, content, { mode: 0o600 }); + await assert.rejects(loadWorkerProfile(path), (error: Error) => { + assert.match(error.message, /Invalid private JSON file/); + assert.doesNotMatch(error.message, /do-not-disclose/); + return true; + }); + } + await writeFile( + path, + `${JSON.stringify(profile(directory))}${' '.repeat(32 * 1024)}`, + { mode: 0o600 }, + ); + await assert.rejects(loadWorkerProfile(path), /size or type/); +}); + +test('profiles reject unknown fields, unsupported versions, invalid paths, and implicit grants', async t => { + const directory = await mkdtemp(join(tmpdir(), 'worker-profile-fields-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'profile.json'); + const value = profile(directory); + const cases: [string, unknown][] = [ + ['null', null], + ['array', []], + ['unsupported version', { ...value, version: 2 }], + ['unknown root field', { ...value, credential: 'not-allowed' }], + [ + 'unknown private workspace field', + { ...value, workspace: { type: 'private', path: '/srv' } }, + ], + [ + 'unknown directory workspace field', + { ...value, workspace: { type: 'directory', path: '/srv', extra: true } }, + ], + ['unknown workspace type', { ...value, workspace: { type: 'shared' } }], + ['missing workspace path', { ...value, workspace: { type: 'directory' } }], + ['relative identity path', { ...value, identityPath: 'identity.json' }], + [ + 'relative workspace path', + { ...value, workspace: { type: 'directory', path: './project' } }, + ], + ['empty identity path', { ...value, identityPath: '' }], + ['NUL identity path', { ...value, identityPath: '/srv/identity\0.json' }], + [ + 'NUL workspace path', + { ...value, workspace: { type: 'directory', path: '/srv/project\0' } }, + ], + [ + 'oversized identity path', + { ...value, identityPath: `/${'x'.repeat(4096)}` }, + ], + [ + 'oversized workspace path', + { + ...value, + workspace: { type: 'directory', path: `/${'é'.repeat(2048)}` }, + }, + ], + [ + 'missing write permission', + { + version: 1, + identityPath: value.identityPath, + workspace: value.workspace, + allowWorkspaceCommands: false, + }, + ], + ['string write permission', { ...value, allowWorkspaceWrites: 'true' }], + ['numeric command permission', { ...value, allowWorkspaceCommands: 1 }], + ]; + for (const [name, invalid] of cases) { + await t.test(name, async () => { + const serialized = JSON.stringify(invalid); + await writeFile(path, serialized, { mode: 0o600 }); + await assert.rejects(loadWorkerProfile(path), /worker profile/i); + await assert.rejects( + saveWorkerProfile(path, invalid as WorkerProfile), + /worker profile/i, + ); + assert.equal(await readFile(path, 'utf8'), serialized); + assert.throws( + () => workerProfileArguments(invalid as WorkerProfile), + /worker profile/i, + ); + }); + } + await assert.rejects( + loadWorkerProfile('relative.json'), + /paths must be absolute/, + ); + await assert.rejects( + saveWorkerProfile('relative.json', value), + /paths must be absolute/, + ); +}); + +test('save rejects non-JSON fields and bounds the serialized profile size', async t => { + const directory = await mkdtemp( + join(tmpdir(), 'worker-profile-serialization-'), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'profile.json'); + const value = profile(directory); + await assert.rejects( + saveWorkerProfile(path, { ...value, ignored: undefined } as WorkerProfile), + /fields/, + ); + await assert.rejects( + saveWorkerProfile(path, { ...value, [Symbol('ignored')]: true }), + /fields/, + ); + await assert.rejects( + saveWorkerProfile(path, { ...value, toJSON: () => value } as WorkerProfile), + /fields/, + ); + await assert.rejects( + saveWorkerProfile(path, { + ...value, + identityPath: `/${'"'.repeat(4095)}`, + workspace: { type: 'directory', path: `/${'"'.repeat(4095)}` }, + }), + /16 KiB limit/, + ); + await assert.rejects(stat(path), { code: 'ENOENT' }); +}); + +test('private profile reads remain bounded if the file grows after its size check', async t => { + const directory = await mkdtemp(join(tmpdir(), 'worker-profile-growth-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'profile.json'); + await saveWorkerProfile(path, profile(directory)); + const probe = await open(path, 'r'); + const prototype = Object.getPrototypeOf(probe) as FileHandle; + await probe.close(); + const originalRead = prototype.read; + let grown = false; + t.mock.method( + prototype, + 'read', + async function (this: FileHandle, ...args: unknown[]) { + if (!grown) { + grown = true; + await writeFile( + path, + `${JSON.stringify(profile(directory))}${' '.repeat(32 * 1024)}`, + ); + } + return Reflect.apply(originalRead, this, args); + }, + ); + await assert.rejects(loadWorkerProfile(path), /size or type/); + assert.equal(grown, true); +}); + +test('permission errors are not mistaken for an absent profile', async t => { + const directory = await mkdtemp(join(tmpdir(), 'worker-profile-permission-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'profile.json'); + await saveWorkerProfile(path, profile(directory)); + const probe = await open(path, 'r'); + const prototype = Object.getPrototypeOf(probe) as FileHandle; + await probe.close(); + t.mock.method(prototype, 'read', async () => { + throw Object.assign(new Error('Permission denied'), { code: 'EACCES' }); + }); + await assert.rejects(loadWorkerProfile(path), { code: 'EACCES' }); +}); + +test('exposed profiles and unsafe storage ancestry are rejected', async t => { + const directory = await mkdtemp(join(tmpdir(), 'worker-profile-modes-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'profile.json'); + const value = profile(directory); + await saveWorkerProfile(path, value); + await chmod(path, 0o644); + await assert.rejects(loadWorkerProfile(path), /owner-only.*644/); + await chmod(path, 0o600); + await chmod(directory, 0o777); + await assert.rejects(loadWorkerProfile(path), /writable by other accounts/); + await assert.rejects( + loadWorkerProfile(join(directory, 'missing.json')), + /writable by other accounts/, + ); + await assert.rejects( + saveWorkerProfile(path, value), + /writable by other accounts/, + ); +}); + +test('worker profile storage fails closed on unsupported platforms', async t => { + const descriptor = Object.getOwnPropertyDescriptor(process, 'platform')!; + t.after(() => Object.defineProperty(process, 'platform', descriptor)); + Object.defineProperty(process, 'platform', { + value: 'freebsd', + configurable: true, + }); + const value = profile('/srv'); + await assert.rejects( + loadWorkerProfile('/srv/missing.json'), + /unavailable on this platform/, + ); + await assert.rejects( + saveWorkerProfile('/srv/profile.json', value), + /unavailable on this platform/, + ); + await assert.rejects( + assertWorkerProfileIsolation('/srv/profile.json', value.identityPath), + /unavailable on this platform/, + ); +}); + +test('isolation accepts separate reserved files outside the workspace', async t => { + const directory = await mkdtemp(join(tmpdir(), 'worker-profile-isolation-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'profile.json'); + const identity = join(directory, 'identity.json'); + const workspace = join(directory, 'workspace'); + await mkdir(workspace); + const reservedProfile = await assertIdentityPathIsPrivate(path, true); + const reservedIdentity = await assertIdentityPathIsPrivate(identity, true); + await assertWorkerProfileIsolation(path, identity); + await assertWorkerProfileIsolation(path, identity, workspace); + await reservedIdentity.release(); + await reservedProfile.release(); + assert.deepEqual(await readdir(directory), ['workspace']); +}); + +test('isolation rejects lexical, symlink, and hardlink aliases of the identity', async t => { + const directory = await mkdtemp(join(tmpdir(), 'worker-profile-alias-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const identity = join(directory, 'identity.json'); + await writeFile(identity, '', { mode: 0o600 }); + const symlinkPath = join(directory, 'symlink.json'); + const hardlinkPath = join(directory, 'hardlink.json'); + await symlink(identity, symlinkPath); + await link(identity, hardlinkPath); + for (const path of [ + identity, + `${directory}/./identity.json`, + symlinkPath, + hardlinkPath, + ]) { + await assert.rejects( + assertWorkerProfileIsolation(path, identity), + /separate files, not aliases/, + ); + } +}); + +test('isolation rejects either private file inside a read-only workspace', async t => { + const directory = await mkdtemp( + join(tmpdir(), 'worker-profile-containment-'), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const workspace = join(directory, 'workspace'); + await mkdir(workspace); + const profilePath = join(directory, 'profile.json'); + const identity = join(directory, 'identity.json'); + const inside = join(workspace, 'storage.json'); + for (const path of [profilePath, identity, inside]) + await writeFile(path, '', { mode: 0o600 }); + await assert.rejects( + assertWorkerProfileIsolation(inside, identity, workspace), + /outside the workspace/, + ); + await assert.rejects( + assertWorkerProfileIsolation(profilePath, inside, workspace), + /outside the workspace/, + ); + const workspaceAlias = join(directory, 'workspace-alias'); + await symlink(workspace, workspaceAlias); + await assert.rejects( + assertWorkerProfileIsolation(profilePath, inside, workspaceAlias), + /outside the workspace/, + ); +}); + +test('isolation checks every intermediate symlink and traversed storage ancestor', async t => { + const directory = await mkdtemp( + join(tmpdir(), 'worker-profile-link-ancestry-'), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const workspace = join(directory, 'workspace'); + const outside = join(directory, 'workspace-private'); + await mkdir(workspace); + await mkdir(outside); + const profilePath = join(outside, 'profile.json'); + const identity = join(outside, 'identity.json'); + await writeFile(profilePath, '', { mode: 0o600 }); + await writeFile(identity, '', { mode: 0o600 }); + await assertWorkerProfileIsolation(profilePath, identity, workspace); + + const linkInWorkspace = join(workspace, 'storage'); + const indirect = join(directory, 'indirect'); + await symlink(outside, linkInWorkspace); + await symlink(linkInWorkspace, indirect); + const cases = [ + join(linkInWorkspace, 'profile.json'), + join(indirect, 'profile.json'), + `${workspace}/../workspace-private/profile.json`, + `${indirect}/../workspace-private/profile.json`, + ]; + for (const path of cases) { + await assert.rejects( + assertWorkerProfileIsolation(path, identity, workspace), + /outside the workspace/, + ); + } + await assert.rejects( + assertWorkerProfileIsolation( + profilePath, + join(indirect, 'identity.json'), + workspace, + ), + /outside the workspace/, + ); + const fileLink = join(workspace, 'identity-link.json'); + await symlink(identity, fileLink); + await assert.rejects( + assertWorkerProfileIsolation(profilePath, fileLink, workspace), + /outside the workspace/, + ); +}); + +test('isolation fails on missing or non-file storage and missing or non-directory workspaces', async t => { + const directory = await mkdtemp( + join(tmpdir(), 'worker-profile-isolation-invalid-'), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'profile.json'); + const identity = join(directory, 'identity.json'); + await writeFile(path, '', { mode: 0o600 }); + await writeFile(identity, '', { mode: 0o600 }); + await assert.rejects( + assertWorkerProfileIsolation(join(directory, 'missing.json'), identity), + { code: 'ENOENT' }, + ); + await assert.rejects( + assertWorkerProfileIsolation(directory, identity), + /regular files/, + ); + await assert.rejects( + assertWorkerProfileIsolation(path, identity, join(directory, 'missing')), + { code: 'ENOENT' }, + ); + await assert.rejects( + assertWorkerProfileIsolation(path, identity, path), + /must be a directory/, + ); +}); + +test('create-only reservations never overwrite existing profiles or identities', async t => { + const directory = await mkdtemp( + join(tmpdir(), 'worker-profile-create-only-'), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'profile.json'); + const value = profile(directory); + await saveWorkerProfile(path, value); + const before = await stat(path, { bigint: true }); + await assert.rejects( + assertIdentityPathIsPrivate(path, true), + /already exists; choose a different identity\/profile path/, + ); + assert.equal((await stat(path, { bigint: true })).ino, before.ino); + assert.deepEqual(await loadWorkerProfile(path), value); + const existing = await assertIdentityPathIsPrivate(path); + await existing.release(); + assert.deepEqual(await loadWorkerProfile(path), value); +}); + +test('only one concurrent create-only reservation can claim a path', async t => { + const directory = await mkdtemp( + join(tmpdir(), 'worker-profile-reservations-'), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'profile.json'); + const results = await Promise.allSettled([ + assertIdentityPathIsPrivate(path, true), + assertIdentityPathIsPrivate(path, true), + ]); + const successes = results.filter(result => result.status === 'fulfilled'); + const failures = results.filter(result => result.status === 'rejected'); + assert.equal(successes.length, 1); + assert.equal(failures.length, 1); + assert.match(String(failures[0].reason), /already exists/); + assert.equal((await stat(path)).size, 0); + await successes[0].value.release(); + await assert.rejects(stat(path), { code: 'ENOENT' }); +}); + +test('reservation cleanup preserves a profile published over its placeholder', async t => { + const directory = await mkdtemp( + join(tmpdir(), 'worker-profile-reservation-release-'), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const path = join(directory, 'profile.json'); + const reservation = await assertIdentityPathIsPrivate(path, true); + const value = profile(directory); + await saveWorkerProfile(path, value); + await reservation.release(); + assert.deepEqual(await loadWorkerProfile(path), value); +}); diff --git a/packages/code/src/worker-profile.ts b/packages/code/src/worker-profile.ts new file mode 100644 index 00000000..a466a5d1 --- /dev/null +++ b/packages/code/src/worker-profile.ts @@ -0,0 +1,200 @@ +import { lstat, realpath, stat } from 'node:fs/promises'; +import { homedir } from 'node:os'; +import { isAbsolute, join, relative, sep } from 'node:path'; + +import { assertPrivateStorageAncestors } from './private-storage.js'; +import { BridgeProtocolError } from './protocol.js'; +import { readPrivateJson, savePrivateJson } from './storage.js'; + +const MAX_PROFILE_BYTES = 16 * 1024; +const MAX_PATH_BYTES = 4096; + +export interface WorkerProfile { + version: 1; + identityPath: string; + workspace: { type: 'private' } | { type: 'directory'; path: string }; + allowWorkspaceWrites: boolean; + allowWorkspaceCommands: boolean; +} + +function hasExactKeys( + value: unknown, + keys: string[], +): value is Record { + return ( + typeof value === 'object' && + value !== null && + !Array.isArray(value) && + Reflect.ownKeys(value).length === keys.length && + keys.every(key => Object.hasOwn(value, key)) + ); +} + +function assertProfilePath(value: unknown): asserts value is string { + if ( + typeof value !== 'string' || + !isAbsolute(value) || + /[\x00-\x1f\x7f]/.test(value) || + Buffer.byteLength(value, 'utf8') > MAX_PATH_BYTES + ) { + throw new BridgeProtocolError( + 'Worker profile paths must be absolute, contain no control characters, and be at most 4096 bytes', + ); + } +} + +function parseWorkerProfile(value: unknown): WorkerProfile { + if ( + !hasExactKeys(value, [ + 'version', + 'identityPath', + 'workspace', + 'allowWorkspaceWrites', + 'allowWorkspaceCommands', + ]) || + value.version !== 1 || + typeof value.allowWorkspaceWrites !== 'boolean' || + typeof value.allowWorkspaceCommands !== 'boolean' + ) { + throw new BridgeProtocolError( + 'Invalid worker profile fields or unsupported version', + ); + } + assertProfilePath(value.identityPath); + let workspace: WorkerProfile['workspace']; + if ( + hasExactKeys(value.workspace, ['type']) && + value.workspace.type === 'private' + ) { + workspace = { type: 'private' }; + } else if ( + hasExactKeys(value.workspace, ['type', 'path']) && + value.workspace.type === 'directory' + ) { + assertProfilePath(value.workspace.path); + workspace = { type: 'directory', path: value.workspace.path }; + } else { + throw new BridgeProtocolError('Invalid worker profile workspace'); + } + return { + version: 1, + identityPath: value.identityPath, + workspace, + allowWorkspaceWrites: value.allowWorkspaceWrites, + allowWorkspaceCommands: value.allowWorkspaceCommands, + }; +} + +export function defaultWorkerProfilePath(): string { + return join( + homedir(), + '.config', + 'librechat', + 'code', + 'profiles', + 'default.json', + ); +} + +export async function loadWorkerProfile( + path: string, +): Promise { + assertProfilePath(path); + let value: unknown; + try { + value = await readPrivateJson(path, MAX_PROFILE_BYTES); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + // A dangling leaf symlink is not an absent profile. + const entry = await lstat(path).catch((failure: NodeJS.ErrnoException) => { + if (failure.code === 'ENOENT') return undefined; + throw failure; + }); + if (entry !== undefined) throw error; + return undefined; + } + return parseWorkerProfile(value); +} + +export async function saveWorkerProfile( + path: string, + profile: WorkerProfile, +): Promise { + assertProfilePath(path); + const value = parseWorkerProfile(profile); + if ( + Buffer.byteLength(`${JSON.stringify(value, null, 2)}\n`, 'utf8') > + MAX_PROFILE_BYTES + ) { + throw new BridgeProtocolError('Worker profile exceeds the 16 KiB limit'); + } + await savePrivateJson(path, value); +} + +export function workerProfileArguments(profile: WorkerProfile): string[] { + const value = parseWorkerProfile(profile); + const args = + value.workspace.type === 'private' + ? ['--default-workspace'] + : ['--worker-dir', value.workspace.path]; + if (value.allowWorkspaceWrites) args.push('--allow-workspace-writes'); + if (value.allowWorkspaceCommands) args.push('--allow-workspace-commands'); + return args; +} + +function isWithin(root: string, path: string): boolean { + const suffix = relative(root, path); + return ( + suffix === '' || + (suffix !== '..' && !suffix.startsWith(`..${sep}`) && !isAbsolute(suffix)) + ); +} + +export async function assertWorkerProfileIsolation( + profilePath: string, + identityPath: string, + workspaceRoot?: string, +): Promise { + assertProfilePath(profilePath); + assertProfilePath(identityPath); + if (workspaceRoot !== undefined) assertProfilePath(workspaceRoot); + const [profileAncestors, identityAncestors] = await Promise.all([ + assertPrivateStorageAncestors(profilePath), + assertPrivateStorageAncestors(identityPath), + ]); + const [profile, identity, profileMetadata, identityMetadata] = + await Promise.all([ + realpath(profilePath), + realpath(identityPath), + stat(profilePath, { bigint: true }), + stat(identityPath, { bigint: true }), + ]); + if (!profileMetadata.isFile() || !identityMetadata.isFile()) { + throw new BridgeProtocolError( + 'Worker profile and identity paths must be regular files', + ); + } + if ( + profile === identity || + (profileMetadata.dev === identityMetadata.dev && + profileMetadata.ino === identityMetadata.ino) + ) { + throw new BridgeProtocolError( + 'Worker profile and identity must use separate files, not aliases', + ); + } + if (workspaceRoot === undefined) return; + const root = await realpath(workspaceRoot); + if (!(await stat(root)).isDirectory()) { + throw new BridgeProtocolError('Worker workspace root must be a directory'); + } + if ( + [profile, identity, ...profileAncestors, ...identityAncestors].some(path => + isWithin(root, path), + ) + ) { + throw new BridgeProtocolError( + 'Worker profile, identity, and their storage ancestry must stay outside the workspace', + ); + } +} From 9ac42e4de33c60f1c7666e3e98013850b7766684 Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Wed, 21 Oct 2026 16:43:58 +0200 Subject: [PATCH 43/48] =?UTF-8?q?=F0=9F=94=91=20feat:=20Recover=20Enrolled?= =?UTF-8?q?=20Worker=20Credentials=20On=20Startup=20(#341)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Lia --- packages/code/src/credential-recovery.test.ts | 701 ++++++++++++++++++ packages/code/src/credential-recovery.ts | 223 ++++++ packages/code/src/worker.ts | 54 +- 3 files changed, 966 insertions(+), 12 deletions(-) create mode 100644 packages/code/src/credential-recovery.test.ts create mode 100644 packages/code/src/credential-recovery.ts diff --git a/packages/code/src/credential-recovery.test.ts b/packages/code/src/credential-recovery.test.ts new file mode 100644 index 00000000..1cce473e --- /dev/null +++ b/packages/code/src/credential-recovery.test.ts @@ -0,0 +1,701 @@ +import assert from 'node:assert/strict'; +import { randomBytes } from 'node:crypto'; +import test from 'node:test'; + +import { recoverWorkerCredential } from './credential-recovery.js'; +import { + createBridgeIdentity, + verifyBridgeRecovery, + verifyBridgeRecoveryStart, + verifyBridgeRequest, +} from './identity.js'; +import { BridgeProtocolError } from './protocol.js'; +import { BridgeWorker } from './worker.js'; +import type { BridgeWorkerOptions } from './worker.js'; + +const workerId = 'recovery:vm'; +const codeApiUrl = 'https://code.example/proxy/v1'; +const incarnationId = 'worker-recovery-incarnation'; +const oldCredential = 'expired-short-lived-credential-value'; +const newCredential = 'recovered-short-lived-credential-value'; +const capabilities = { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: ['bash'], +}; + +function challenge() { + return { + protocolVersion: 1 as const, + operation: 'credential.recover' as const, + serverId: 'https://code.example', + workerId, + enrollmentGeneration: randomBytes(18).toString('base64url'), + challenge: randomBytes(32).toString('base64url'), + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }; +} + +function credential() { + return { + protocolVersion: 1 as const, + workerId, + credential: newCredential, + expiresAt: new Date(Date.now() + 900_000).toISOString(), + }; +} + +function identity() { + return { + ...createBridgeIdentity(), + credential: oldCredential, + expiresAt: new Date(Date.now() - 60_000).toISOString(), + }; +} + +function workerOptions( + overrides: Partial = {}, +): BridgeWorkerOptions { + return { + codeApiUrl, + workerId, + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + identity: identity(), + capabilities, + ...overrides, + }; +} + +function expired() { + return Response.json( + { + code: 'CREDENTIAL_INVALID', + error: 'Worker credential is invalid or expired', + }, + { status: 401 }, + ); +} + +test('startup recovers the enrolled key and persists before registering with the new credential', async () => { + const key = identity(); + const recovered = credential(); + const issued = challenge(); + const requests: string[] = []; + let persisted = false; + const controller = new AbortController(); + const worker = new BridgeWorker( + workerOptions({ + identity: key, + onIdentityChange: next => { + assert.equal(key.credential, oldCredential); + assert.equal(next.privateKey, key.privateKey); + assert.equal(next.credential, newCredential); + persisted = true; + }, + fetchImpl: async (input, init) => { + const url = String(input); + const body = JSON.parse(String(init?.body)); + requests.push(url); + if (url.endsWith('/refresh')) return expired(); + if (url.endsWith('/challenge') || url.endsWith('/recover')) { + assert.equal( + (init?.headers as Record).Authorization, + undefined, + ); + assert.equal(init?.redirect, 'error'); + assert.ok(!String(init?.body).includes(key.privateKey)); + assert.ok(!String(init?.body).includes(oldCredential)); + assert.match( + url, + /\/proxy\/v1\/bridge\/workers\/recovery%3Avm\/credentials\//, + ); + } + if (url.endsWith('/challenge')) { + assert.equal(body.serverId, 'https://code.example'); + assert.equal(body.operation, 'credential.challenge'); + assert.equal(body.workerId, workerId); + assert.match(body.nonce, /^[A-Za-z0-9_-]{43}$/); + assert.equal( + verifyBridgeRecoveryStart(key.publicKey, body, body.signature), + true, + ); + return Response.json(issued); + } + if (url.endsWith('/recover')) { + assert.equal( + verifyBridgeRecovery(key.publicKey, body, body.signature), + true, + ); + assert.deepEqual( + { ...body, signature: undefined }, + { ...issued, signature: undefined }, + ); + return Response.json(recovered); + } + assert.equal(persisted, true); + const headers = init!.headers as Record; + assert.equal(headers.Authorization, `Bridge ${newCredential}`); + assert.equal( + verifyBridgeRequest( + key.publicKey, + { + credential: newCredential, + method: 'POST', + path: new URL(url).pathname, + timestamp: headers['X-LibreChat-Code-Timestamp'], + nonce: headers['X-LibreChat-Code-Nonce'], + body: String(init!.body), + }, + headers['X-LibreChat-Code-Signature'], + ), + true, + ); + if (url.endsWith('/register')) + return Response.json({ + protocolVersion: 1, + workerId, + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + }); + controller.abort(); + return Response.json({ protocolVersion: 1, serverElapsedMs: 0 }); + }, + }), + ); + await worker.run(controller.signal); + assert.equal(key.credential, newCredential); + assert.equal(requests.filter(url => url.endsWith('/challenge')).length, 1); + assert.equal(requests.filter(url => url.endsWith('/recover')).length, 1); + assert.ok( + !requests.some( + url => + url.includes('pairing') || + url.includes('reset') || + url.includes('revoke'), + ), + ); +}); + +test('ordinary successful refresh does not probe recovery and static workers do not recover', async () => { + let calls = 0; + const worker = new BridgeWorker( + workerOptions({ + fetchImpl: async input => { + assert.ok(String(input).endsWith('/refresh')); + calls++; + return Response.json(credential()); + }, + }), + ); + await worker.refreshCredential(); + assert.equal(calls, 1); + const staticWorker = new BridgeWorker( + workerOptions({ + identity: undefined, + token: 'legacy', + fetchImpl: async () => { + assert.fail('static worker has no credential refresh'); + }, + }), + ); + await staticWorker.refreshCredential(); +}); + +for (const [status, code] of [ + [401, undefined], + [401, 'PROOF_INVALID'], + [401, 'PROOF_REPLAYED'], + [403, 'CREDENTIAL_INVALID'], + [409, 'WORKER_FENCED'], + [429, 'RECOVERY_RATE_LIMITED'], + [503, undefined], +] as const) { + test(`refresh failure ${status}/${code} never triggers recovery`, async () => { + let calls = 0; + const worker = new BridgeWorker( + workerOptions({ + fetchImpl: async input => { + calls++; + assert.ok(String(input).endsWith('/refresh')); + return Response.json({ code }, { status }); + }, + }), + ); + await assert.rejects( + worker.refreshCredential(), + error => error instanceof BridgeProtocolError && error.status === status, + ); + assert.equal(calls, 1); + }); +} + +test('active workers never bootstrap recovery after registration', async () => { + const key = identity(); + let recoveries = 0; + const worker = new BridgeWorker( + workerOptions({ + identity: key, + fetchImpl: async input => { + const url = String(input); + if (url.endsWith('/register')) + return Response.json({ + protocolVersion: 1, + workerId, + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + }); + if (!url.endsWith('/refresh')) recoveries++; + return expired(); + }, + }), + ); + await worker.register(); + await assert.rejects(worker.refreshCredential(), { + status: 401, + code: 'CREDENTIAL_INVALID', + }); + assert.equal(recoveries, 0); +}); + +for (const status of [404, 405, 401, 403, 429, 503]) { + test(`recovery failure ${status} preserves enrollment, key and old local credential`, async () => { + const key = identity(); + const original = { ...key }; + let persisted = false; + const worker = new BridgeWorker( + workerOptions({ + identity: key, + onIdentityChange: () => { + persisted = true; + }, + fetchImpl: async input => { + if (String(input).endsWith('/refresh')) return expired(); + assert.ok(String(input).endsWith('/challenge')); + return status === 404 + ? new Response('upstream secret', { status }) + : Response.json( + { code: status === 401 ? 'ENROLLMENT_INVALID' : undefined }, + { status }, + ); + }, + }), + ); + await assert.rejects(worker.refreshCredential(), error => { + assert.ok(error instanceof BridgeProtocolError); + assert.equal( + error.status, + status === 404 || status === 405 ? 401 : status, + ); + assert.ok(!error.message.includes('upstream secret')); + if (status === 401) + assert.match(error.message, /revoked.*pairing|pairing.*revoked/i); + if (status === 404) + assert.match( + error.message, + /enable durable enrollment or approve a new pairing/, + ); + return true; + }); + assert.deepEqual(key, original); + assert.equal(persisted, false); + }); +} + +for (const codeApiUrl of [ + 'http://code.example/v1', + 'https://user:password@code.example/v1', + 'https://code.example/v1?token=secret', + 'https://code.example/v1#fragment', +]) { + test('recovery refuses unsafe server context without disclosing identity', async () => { + await assert.rejects( + recoverWorkerCredential({ + codeApiUrl, + workerId, + privateKey: identity().privateKey, + validThroughMs: Date.now(), + signal: AbortSignal.timeout(1000), + fetchImpl: async () => { + assert.fail('must not fetch'); + }, + }), + { code: 'RECOVERY_UNAVAILABLE', status: 401 }, + ); + }); +} + +const badChallenges: Array< + [string, (value: ReturnType) => unknown] +> = [ + ['null', () => null], + ['array', () => []], + ['wrong version', value => ({ ...value, protocolVersion: 2 })], + [ + 'wrong operation', + value => ({ ...value, operation: 'credential.challenge' }), + ], + ['other server', value => ({ ...value, serverId: 'https://other.example' })], + ['other machine', value => ({ ...value, workerId: 'other-worker' })], + ['bad generation', value => ({ ...value, enrollmentGeneration: 'short' })], + ['bad challenge', value => ({ ...value, challenge: 'short' })], + [ + 'expired', + value => ({ ...value, expiresAt: new Date(Date.now() - 1).toISOString() }), + ], + [ + 'unbounded lifetime', + value => ({ ...value, expiresAt: '2099-01-01T00:00:00Z' }), + ], + ['invalid date', value => ({ ...value, expiresAt: 'not a date' })], +]; +for (const [name, alter] of badChallenges) { + test(`recovery refuses ${name} challenge before signing completion`, async () => { + let calls = 0; + await assert.rejects( + recoverWorkerCredential({ + codeApiUrl, + workerId, + privateKey: identity().privateKey, + validThroughMs: Date.now(), + signal: AbortSignal.timeout(1000), + fetchImpl: async () => { + calls++; + return Response.json(alter(challenge())); + }, + }), + /invalid machine recovery challenge/, + ); + assert.equal(calls, 1); + }); +} + +for (const value of [ + null, + [], + { ...credential(), workerId: 'another-worker' }, + { ...credential(), credential: 'short' }, + { ...credential(), credential: 'x'.repeat(4097) }, + { ...credential(), expiresAt: 'bad' }, + { ...credential(), expiresAt: new Date(0).toISOString() }, +]) { + test('invalid recovered credentials are never adopted or persisted', async () => { + const key = identity(); + const worker = new BridgeWorker( + workerOptions({ + identity: key, + onIdentityChange: () => { + assert.fail('must not persist'); + }, + fetchImpl: async input => { + if (String(input).endsWith('/refresh')) return expired(); + if (String(input).endsWith('/challenge')) + return Response.json(challenge()); + return Response.json(value); + }, + }), + ); + await assert.rejects( + worker.refreshCredential(), + /invalid recovered worker credential/, + ); + assert.equal(key.credential, oldCredential); + }); +} + +test('oversized and non-JSON recovery responses have bounded, sanitized failures', async () => { + for (const source of ['sensitive upstream contents', 'x'.repeat(8193)]) { + await assert.rejects( + recoverWorkerCredential({ + codeApiUrl, + workerId, + privateKey: identity().privateKey, + validThroughMs: Date.now(), + signal: AbortSignal.timeout(1000), + fetchImpl: async () => new Response(source), + }), + error => { + assert.ok(!String(error).includes('sensitive upstream contents')); + assert.match( + String(error), + /invalid machine recovery response|exceeds 8 KiB/, + ); + return true; + }, + ); + } +}); + +test('failed persistence never adopts a recovered credential; retry proves the same key with a new nonce', async () => { + const key = identity(); + const nonces = new Set(); + let attempts = 0; + const worker = new BridgeWorker( + workerOptions({ + identity: key, + onIdentityChange: () => { + if (++attempts === 1) throw new Error('disk unavailable'); + }, + fetchImpl: async (input, init) => { + if (String(input).endsWith('/refresh')) return expired(); + if (String(input).endsWith('/challenge')) { + const body = JSON.parse(String(init?.body)); + assert.equal(nonces.has(body.nonce), false); + nonces.add(body.nonce); + assert.equal( + verifyBridgeRecoveryStart(key.publicKey, body, body.signature), + true, + ); + return Response.json(challenge()); + } + return Response.json(credential()); + }, + }), + ); + await assert.rejects(worker.refreshCredential(), /disk unavailable/); + assert.equal(key.credential, oldCredential); + await worker.refreshCredential(); + assert.equal(key.credential, newCredential); + assert.equal(attempts, 2); + assert.equal(nonces.size, 2); +}); + +test('concurrent callers share recovery and one cancellation cannot abort another waiter', async () => { + let release!: () => void; + let notify!: () => void; + const started = new Promise(resolve => { + notify = resolve; + }); + const ready = new Promise(resolve => { + release = resolve; + }); + const first = new AbortController(); + let challenges = 0; + let aborted = false; + const worker = new BridgeWorker( + workerOptions({ + fetchImpl: async (input, init) => { + if (String(input).endsWith('/refresh')) return expired(); + if (String(input).endsWith('/challenge')) { + challenges++; + return Response.json(challenge()); + } + init!.signal!.addEventListener('abort', () => { + aborted = true; + }); + notify(); + await ready; + return Response.json(credential()); + }, + }), + ); + const firstRefresh = worker.refreshCredential(first.signal); + await started; + const secondRefresh = worker.refreshCredential(); + first.abort(); + await assert.rejects(firstRefresh, { name: 'AbortError' }); + assert.equal(aborted, false); + release(); + await secondRefresh; + assert.equal(challenges, 1); + assert.equal(aborted, false); +}); + +test('last waiter cancellation aborts recovery without persisting or registering', async () => { + const controller = new AbortController(); + let notify!: () => void; + const started = new Promise(resolve => { + notify = resolve; + }); + let transportAborted = false; + const worker = new BridgeWorker( + workerOptions({ + onIdentityChange: () => { + assert.fail('must not persist'); + }, + fetchImpl: async (input, init) => { + if (String(input).endsWith('/refresh')) return expired(); + assert.ok(String(input).endsWith('/challenge')); + notify(); + return new Promise((_resolve, reject) => + init!.signal!.addEventListener( + 'abort', + () => { + transportAborted = true; + reject(new DOMException('aborted', 'AbortError')); + }, + { once: true }, + ), + ); + }, + }), + ); + const pending = worker.refreshCredential(controller.signal); + await started; + controller.abort(); + await assert.rejects(pending, { name: 'AbortError' }); + assert.equal(transportAborted, true); +}); + +test('challenge and recovery share the remaining credential-refresh deadline', async () => { + let challengeSignal: AbortSignal | null | undefined; + const worker = new BridgeWorker( + workerOptions({ + credentialRefreshTransportTimeoutMs: 100, + fetchImpl: async (input, init) => { + if (String(input).endsWith('/refresh')) return expired(); + if (String(input).endsWith('/challenge')) { + challengeSignal = init?.signal; + return Response.json(challenge()); + } + assert.equal(init?.signal, challengeSignal); + return new Promise((_resolve, reject) => + init!.signal!.addEventListener( + 'abort', + () => reject(new DOMException('aborted', 'AbortError')), + { once: true }, + ), + ); + }, + }), + ); + // Keep the event loop alive because AbortSignal.timeout timers are unref'ed. + const timer = setTimeout(() => {}, 2000); + try { + await assert.rejects(worker.refreshCredential(), { name: 'AbortError' }); + } finally { + clearTimeout(timer); + } + assert.equal(challengeSignal?.aborted, true); +}); + +test('revoked enrollment terminates startup instead of retrying or issuing a pairing', async () => { + let calls = 0; + const worker = new BridgeWorker( + workerOptions({ + onError: () => { + assert.fail('terminal auth failure must not reconnect'); + }, + fetchImpl: async input => { + calls++; + if (String(input).endsWith('/refresh')) return expired(); + assert.ok(String(input).endsWith('/challenge')); + return Response.json({ code: 'ENROLLMENT_INVALID' }, { status: 401 }); + }, + }), + ); + await assert.rejects(worker.run(), { + status: 401, + code: 'ENROLLMENT_INVALID', + }); + assert.equal(calls, 2); +}); + +test('recovery retries transient startup failure without replacing the enrolled key', async () => { + let challenges = 0; + let failures = 0; + const key = identity(); + const controller = new AbortController(); + const worker = new BridgeWorker( + workerOptions({ + identity: key, + reconnectDelayMs: 1, + reconnectMaxDelayMs: 1, + onError: error => { + assert.ok(error instanceof BridgeProtocolError); + assert.equal(error.status, 503); + failures++; + }, + fetchImpl: async (input, init) => { + const url = String(input); + if (url.endsWith('/refresh')) return expired(); + if (url.endsWith('/challenge')) { + const body = JSON.parse(String(init?.body)); + assert.equal( + verifyBridgeRecoveryStart(key.publicKey, body, body.signature), + true, + ); + return ++challenges === 1 + ? Response.json({}, { status: 503 }) + : Response.json(challenge()); + } + if (url.endsWith('/recover')) return Response.json(credential()); + if (url.endsWith('/register')) + return Response.json({ + protocolVersion: 1, + workerId, + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + }); + controller.abort(); + return Response.json({ protocolVersion: 1, serverElapsedMs: 0 }); + }, + }), + ); + await worker.run(controller.signal); + assert.equal(challenges, 2); + assert.equal(failures, 1); + assert.equal(key.credential, newCredential); +}); + +test('revocation between challenge and completion cannot restore access', async () => { + let persisted = false; + const worker = new BridgeWorker( + workerOptions({ + onIdentityChange: () => { + persisted = true; + }, + fetchImpl: async input => { + if (String(input).endsWith('/refresh')) return expired(); + if (String(input).endsWith('/challenge')) + return Response.json(challenge()); + assert.ok(String(input).endsWith('/recover')); + return Response.json({ code: 'ENROLLMENT_INVALID' }, { status: 401 }); + }, + }), + ); + await assert.rejects(worker.run(), { + code: 'ENROLLMENT_INVALID', + status: 401, + }); + assert.equal(persisted, false); +}); + +test('successful credential recovery never clears an existing workspace quarantine', async () => { + let checks = 0; + const worker = new BridgeWorker( + workerOptions({ + workspaceMutationQuarantine: { + assertAvailable: async () => { + checks++; + throw new BridgeProtocolError( + 'Workspace requires inspection', + 409, + 'WORKER_QUARANTINED', + ); + }, + arm: async () => { + assert.fail('must not arm'); + }, + clear: async () => { + assert.fail('must not clear'); + }, + quarantine: async () => { + assert.fail('must not change quarantine'); + }, + }, + fetchImpl: async input => { + if (String(input).endsWith('/refresh')) return expired(); + if (String(input).endsWith('/challenge')) + return Response.json(challenge()); + assert.ok(String(input).endsWith('/recover')); + return Response.json(credential()); + }, + }), + ); + await assert.rejects(worker.run(), { code: 'WORKER_QUARANTINED' }); + assert.equal(checks, 1); +}); diff --git a/packages/code/src/credential-recovery.ts b/packages/code/src/credential-recovery.ts new file mode 100644 index 00000000..01aae3fd --- /dev/null +++ b/packages/code/src/credential-recovery.ts @@ -0,0 +1,223 @@ +import { randomBytes } from 'node:crypto'; + +import { signBridgeRecovery, signBridgeRecoveryStart } from './identity.js'; +import { + BRIDGE_PROTOCOL_VERSION, + BridgeProtocolError, + bridgeWorkerPath, +} from './protocol.js'; +import type { + BridgeRecoveryChallengeResponse, + BridgeWorkerCredentialResponse, +} from './protocol.js'; + +interface RecoverWorkerCredentialOptions { + codeApiUrl: string; + workerId: string; + privateKey: string; + validThroughMs: number; + fetchImpl: typeof fetch; + signal: AbortSignal; +} + +const MAX_RESPONSE_BYTES = 8192; +const MAX_CHALLENGE_LIFETIME_MS = 6 * 60_000; + +async function responsePayload(response: Response): Promise { + const reader = response.body?.getReader(); + if (!reader) + throw new BridgeProtocolError( + 'Code API returned an empty machine recovery response', + 502, + ); + const chunks: Uint8Array[] = []; + let size = 0; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > MAX_RESPONSE_BYTES) { + throw new BridgeProtocolError( + 'Code API machine recovery response exceeds 8 KiB', + 502, + ); + } + chunks.push(value); + } + try { + return JSON.parse( + new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks)), + ) as unknown; + } catch { + throw new BridgeProtocolError( + 'Code API returned an invalid machine recovery response', + 502, + ); + } + } finally { + await reader.cancel().catch(() => undefined); + reader.releaseLock(); + } +} + +function record(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +async function recoveryRequest( + url: string, + body: object, + options: RecoverWorkerCredentialOptions, +): Promise { + options.signal.throwIfAborted(); + const response = await options.fetchImpl(url, { + method: 'POST', + redirect: 'error', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + signal: options.signal, + }); + if (!response.ok) { + // Older/disabled servers may return HTML. Do not echo untrusted response text. + let payload: unknown; + try { + payload = await responsePayload(response); + } catch { + /* Status is sufficient. */ + } + options.signal.throwIfAborted(); + if (response.status === 404 || response.status === 405) { + throw new BridgeProtocolError( + 'Machine credential expired or is unavailable. This Code API does not support enabled machine recovery; enable durable enrollment or approve a new pairing in LibreChat.', + 401, + 'RECOVERY_UNAVAILABLE', + ); + } + const code = + record(payload) && + typeof payload.code === 'string' && + [ + 'ENROLLMENT_INVALID', + 'PROOF_INVALID', + 'PROOF_REPLAYED', + 'CHALLENGE_INVALID', + 'RECOVERY_RATE_LIMITED', + ].includes(payload.code) + ? payload.code + : undefined; + const retryable = + response.status === 408 || + response.status === 429 || + response.status >= 500; + throw new BridgeProtocolError( + code === 'ENROLLMENT_INVALID' + ? 'Machine enrollment is unavailable or revoked. Approve a new pairing in LibreChat; recovery cannot restore revoked access.' + : `Machine credential recovery failed with HTTP ${response.status}${retryable ? '; reconnect will retry' : '; check enrollment and approve a new pairing in LibreChat if needed'}`, + retryable ? response.status : response.status === 403 ? 403 : 401, + code, + ); + } + const payload = await responsePayload(response); + options.signal.throwIfAborted(); + return payload; +} + +/** Recover only existing enrollment. Never pair, replace a key, or reset workspace state. */ +export async function recoverWorkerCredential( + options: RecoverWorkerCredentialOptions, +): Promise { + options.signal.throwIfAborted(); + const server = new URL(options.codeApiUrl); + if ( + server.protocol !== 'https:' || + server.username || + server.password || + server.search || + server.hash || + server.origin.length > 256 + ) { + throw new BridgeProtocolError( + 'Automatic machine recovery requires the enrolled Code API HTTPS origin; use the existing pairing flow for this endpoint.', + 401, + 'RECOVERY_UNAVAILABLE', + ); + } + const url = `${options.codeApiUrl.replace(/\/+$/, '')}${bridgeWorkerPath(options.workerId)}/credentials`; + const start = { + operation: 'credential.challenge' as const, + serverId: server.origin, + workerId: options.workerId, + timestamp: new Date().toISOString(), + nonce: randomBytes(32).toString('base64url'), + }; + const payload = await recoveryRequest( + `${url}/challenge`, + { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + ...start, + signature: signBridgeRecoveryStart(options.privateKey, start), + }, + options, + ); + if ( + !record(payload) || + payload.protocolVersion !== BRIDGE_PROTOCOL_VERSION || + payload.operation !== 'credential.recover' || + payload.serverId !== server.origin || + payload.workerId !== options.workerId || + typeof payload.enrollmentGeneration !== 'string' || + !/^[A-Za-z0-9_-]{24}$/.test(payload.enrollmentGeneration) || + typeof payload.challenge !== 'string' || + !/^[A-Za-z0-9_-]{43}$/.test(payload.challenge) || + typeof payload.expiresAt !== 'string' || + !Number.isFinite(Date.parse(payload.expiresAt)) || + Date.parse(payload.expiresAt) <= Date.now() || + Date.parse(payload.expiresAt) > Date.now() + MAX_CHALLENGE_LIFETIME_MS + ) { + throw new BridgeProtocolError( + 'Code API returned an invalid machine recovery challenge', + 502, + ); + } + // Sign only the validated recovery operation, never arbitrary server-supplied fields. + const challenge: BridgeRecoveryChallengeResponse = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'credential.recover', + serverId: server.origin, + workerId: options.workerId, + enrollmentGeneration: payload.enrollmentGeneration, + challenge: payload.challenge, + expiresAt: payload.expiresAt, + }; + const credential = await recoveryRequest( + `${url}/recover`, + { + ...challenge, + signature: signBridgeRecovery(options.privateKey, challenge), + }, + options, + ); + if ( + !record(credential) || + credential.protocolVersion !== BRIDGE_PROTOCOL_VERSION || + credential.workerId !== options.workerId || + typeof credential.credential !== 'string' || + credential.credential.length < 32 || + credential.credential.length > 4096 || + typeof credential.expiresAt !== 'string' || + !Number.isFinite(Date.parse(credential.expiresAt)) || + Date.parse(credential.expiresAt) <= options.validThroughMs + ) { + throw new BridgeProtocolError( + 'Code API returned an invalid recovered worker credential', + 502, + ); + } + return { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: options.workerId, + credential: credential.credential, + expiresAt: credential.expiresAt, + }; +} diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 49b96eb1..6b09f257 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -15,6 +15,7 @@ import { } from './protocol.js'; import { EndpointRuntimeSupervisor } from './runtime.js'; import { signBridgeRequest } from './identity.js'; +import { recoverWorkerCredential } from './credential-recovery.js'; import { isWorkspaceToolRequest, WorkspaceToolError } from './workspace.js'; import type { @@ -1317,20 +1318,49 @@ export class BridgeWorker { if (Date.parse(identity.expiresAt) > validThroughMs) { return; } - const credential = await this.timedRequest( - `${this.codeApiUrl}${bridgeWorkerPath(this.options.workerId)}` + - '/credentials/refresh', - { protocolVersion: BRIDGE_PROTOCOL_VERSION }, - Math.max( - 1, - Math.min( - transportTimeoutMs, - this.options.credentialRefreshTransportTimeoutMs ?? - DEFAULT_CONTROL_TRANSPORT_TIMEOUT_MS, - ), + const timeoutMs = Math.max( + 1, + Math.min( + transportTimeoutMs, + this.options.credentialRefreshTransportTimeoutMs ?? + DEFAULT_CONTROL_TRANSPORT_TIMEOUT_MS, ), - signal, ); + const deadline = performance.now() + timeoutMs; + let credential: BridgeWorkerCredentialResponse; + try { + credential = await this.timedRequest( + `${this.codeApiUrl}${bridgeWorkerPath(this.options.workerId)}/credentials/refresh`, + { protocolVersion: BRIDGE_PROTOCOL_VERSION }, + timeoutMs, + signal, + ); + } catch (error) { + // Only bootstrap recovery. Auth failures during active work retain existing fencing/settlement rules. + if ( + this.lastRegisteredAtMs !== 0 || + !(error instanceof BridgeProtocolError) || + error.status !== 401 || + error.code !== 'CREDENTIAL_INVALID' + ) throw error; + signal?.throwIfAborted(); + const remainingMs = Math.floor(deadline - performance.now()); + if (remainingMs <= 0) { + throw new DOMException('Credential recovery timed out', 'TimeoutError'); + } + const recoverySignal = AbortSignal.timeout(remainingMs); + credential = await recoverWorkerCredential({ + codeApiUrl: this.codeApiUrl, + workerId: this.options.workerId, + privateKey: identity.privateKey, + validThroughMs, + fetchImpl: this.fetchImpl, + signal: signal == null + ? recoverySignal + : AbortSignal.any([signal, recoverySignal]), + }); + } + signal?.throwIfAborted(); if ( credential.protocolVersion !== BRIDGE_PROTOCOL_VERSION || credential.workerId !== this.options.workerId || From a3207a75c8327ba1c9526eff8b06f3588f3739ed Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Thu, 22 Oct 2026 11:49:43 +0200 Subject: [PATCH 44/48] =?UTF-8?q?=F0=9F=AA=AA=20feat:=20Add=20Key-Bound=20?= =?UTF-8?q?Browser=20Enrollment=20And=20Resumable=20Setup=20(#342)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🪪 feat: Add Key-Bound Browser Enrollment And Resumable Setup * 🛡️ fix: Protect Setup State And Legacy Enrollment Fences * 🔐 fix: Remove Completed Setup Secrets And Bind Pending State To Identity * 🛡️ fix: Bind Setup Recovery State Across Startup Modes * 🗝️ fix: Protect The Entire Setup Journal Namespace --------- Co-authored-by: Lia --- packages/code/package.json | 4 + .../code/src/browser-enrollment-protocol.ts | 253 ++++++++++ packages/code/src/browser-protection.test.ts | 31 ++ packages/code/src/browser-setup-cli.ts | 103 ++++ packages/code/src/browser-setup.test.ts | 279 +++++++++++ packages/code/src/browser-setup.ts | 352 ++++++++++++++ packages/code/src/cli.ts | 31 +- packages/code/src/index.ts | 1 + packages/code/src/native-sandbox.test.ts | 25 + packages/code/src/pairing.ts | 2 + packages/code/src/setup-cli.ts | 35 +- packages/code/src/storage.ts | 7 +- packages/code/src/worker.test.ts | 20 +- packages/code/src/worker.ts | 8 +- service/rollup.config.js | 1 + .../bridge/browser-enrollment-router.test.ts | 184 +++++++ service/src/bridge/browser-enrollment.test.ts | 447 ++++++++++++++++++ service/src/bridge/pairing.ts | 337 ++++++++++++- service/src/bridge/router.ts | 73 +++ 19 files changed, 2182 insertions(+), 11 deletions(-) create mode 100644 packages/code/src/browser-enrollment-protocol.ts create mode 100644 packages/code/src/browser-protection.test.ts create mode 100644 packages/code/src/browser-setup-cli.ts create mode 100644 packages/code/src/browser-setup.test.ts create mode 100644 packages/code/src/browser-setup.ts create mode 100644 service/src/bridge/browser-enrollment-router.test.ts create mode 100644 service/src/bridge/browser-enrollment.test.ts diff --git a/packages/code/package.json b/packages/code/package.json index 95eea841..d3e6395b 100644 --- a/packages/code/package.json +++ b/packages/code/package.json @@ -19,6 +19,10 @@ "types": "./dist/identity.d.ts", "import": "./dist/identity.js" }, + "./browser-enrollment": { + "types": "./dist/browser-enrollment-protocol.d.ts", + "import": "./dist/browser-enrollment-protocol.js" + }, "./worker": { "types": "./dist/worker.d.ts", "import": "./dist/worker.js" diff --git a/packages/code/src/browser-enrollment-protocol.ts b/packages/code/src/browser-enrollment-protocol.ts new file mode 100644 index 00000000..ef1c712b --- /dev/null +++ b/packages/code/src/browser-enrollment-protocol.ts @@ -0,0 +1,253 @@ +import { createHash, createPublicKey, sign, verify } from 'node:crypto'; + +export type BrowserSetupRequest = BrowserEnrollmentRequest; +export type EnrollmentRedemption = BrowserEnrollmentRedemptionContract; +export type EnrollmentApproval = BrowserEnrollmentApproval; + +export const BROWSER_ENROLLMENT_PROTOCOL_VERSION = 1 as const; +export const BROWSER_ENROLLMENT_MAX_TTL_MS = 10 * 60_000; +export const BROWSER_SETUP_PROOF_DOMAIN = 'librechat-code:browser-setup:v1'; +export const BROWSER_ENROLLMENT_REDEMPTION_DOMAIN = 'librechat-code:browser-enrollment-redeem:v1'; + +export interface BrowserEnrollmentRequest { + version: 1; + instance: string; + attemptId: string; + createdAt: string; + publicKey: string; + machineName: string; + workspace: { type: 'private' | 'directory' }; + allowWorkspaceWrites: boolean; + allowWorkspaceCommands: boolean; +} + +export interface BrowserSetupProof { + operation: 'setup.start' | 'setup.poll' | 'setup.cancel' | 'setup.complete'; + instance: string; + attemptId: string; + requestHash: string; + timestamp: string; + nonce: string; +} + +export interface BrowserEnrollmentRedemptionContract { + operation: 'enrollment.redeem'; + serverId: string; + instance: string; + attemptId: string; + requestHash: string; + workerId: string; + expiresAt: string; +} + +export interface BrowserEnrollmentStatusRequest extends BrowserEnrollmentRedemptionContract { + protocolVersion: 1; +} + +export interface BrowserEnrollmentRedemption extends BrowserEnrollmentRedemptionContract { + protocolVersion: 1; + signature: string; +} + +export interface BrowserEnrollmentBinding { + tenantId: string; + principal: { type: 'deployment' | 'tenant' | 'user' | 'role' | 'group'; id: string }; +} + +export interface BrowserEnrollmentApproval extends BrowserEnrollmentRedemptionContract { + protocolVersion: 1; + publicKey: string; + binding: BrowserEnrollmentBinding; +} + +export interface BrowserEnrollmentCapabilities { + browserEnrollmentVersion: 1; + serverId: string; +} + +function record(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function keys(value: Record, names: string[]): boolean { + return Object.keys(value).length === names.length && names.every(name => Object.prototype.hasOwnProperty.call(value, name)); +} + +export function isBrowserEnrollmentTimestamp(value: unknown): value is string { + if (typeof value !== 'string' || value.length !== 24) return false; + const time = Date.parse(value); + return Number.isFinite(time) && new Date(time).toISOString() === value; +} + +export function isBrowserEnrollmentToken(value: unknown): value is string { + return typeof value === 'string' && /^[A-Za-z0-9_-]{43}$/.test(value) && + Buffer.from(value, 'base64url').toString('base64url') === value; +} + +export function isBrowserEnrollmentSignature(value: unknown): value is string { + return typeof value === 'string' && /^[A-Za-z0-9_-]{86}$/.test(value) && + Buffer.from(value, 'base64url').toString('base64url') === value; +} + +export function isBrowserEnrollmentPublicKey(value: unknown): value is string { + if (typeof value !== 'string' || value.length > 4096 || !value.startsWith('-----BEGIN PUBLIC KEY-----\n')) return false; + try { + const key = createPublicKey(value); + return key.asymmetricKeyType === 'ed25519' && key.export({ type: 'spki', format: 'pem' }) === value; + } catch { + return false; + } +} + +export function normalizeBrowserEnrollmentInstance(value: string): string { + if (value.length > 2048 || value !== value.trim()) throw new TypeError('Invalid LibreChat instance URL'); + const url = new URL(value); + if ( + (url.protocol !== 'https:' && !(url.protocol === 'http:' && ['localhost', '127.0.0.1', '[::1]'].includes(url.hostname))) || + url.username !== '' || url.password !== '' || url.search !== '' || url.hash !== '' + ) { + throw new TypeError('LibreChat instance requires HTTPS (or loopback HTTP), without credentials, query, or fragment'); + } + return `${url.origin}${url.pathname.replace(/\/+$/, '')}`; +} + +function instance(value: unknown): value is string { + try { + return typeof value === 'string' && normalizeBrowserEnrollmentInstance(value) === value; + } catch { + return false; + } +} + +function serverOrigin(value: unknown): value is string { + if (typeof value !== 'string' || value.length > 256) return false; + try { + const url = new URL(value); + return url.protocol === 'https:' && url.origin === value && instance(value); + } catch { + return false; + } +} + +function requestHash(value: unknown): value is string { + return typeof value === 'string' && /^[a-f0-9]{64}$/.test(value); +} + +export function isBrowserEnrollmentRequest(value: unknown): value is BrowserEnrollmentRequest { + return record(value) && keys(value, [ + 'version', 'instance', 'attemptId', 'createdAt', 'publicKey', 'machineName', + 'workspace', 'allowWorkspaceWrites', 'allowWorkspaceCommands', + ]) && value.version === 1 && instance(value.instance) && isBrowserEnrollmentToken(value.attemptId) && + isBrowserEnrollmentTimestamp(value.createdAt) && isBrowserEnrollmentPublicKey(value.publicKey) && + typeof value.machineName === 'string' && /^[A-Za-z0-9][A-Za-z0-9 ._()-]{0,79}$/.test(value.machineName) && + value.machineName === value.machineName.trim() && record(value.workspace) && keys(value.workspace, ['type']) && + (value.workspace.type === 'private' || value.workspace.type === 'directory') && + typeof value.allowWorkspaceWrites === 'boolean' && typeof value.allowWorkspaceCommands === 'boolean'; +} + +export function canonicalBrowserEnrollmentRequest(input: BrowserEnrollmentRequest): string { + if (!isBrowserEnrollmentRequest(input)) throw new TypeError('Invalid browser enrollment request'); + return JSON.stringify({ + version: input.version, + instance: input.instance, + attemptId: input.attemptId, + createdAt: input.createdAt, + publicKey: input.publicKey, + machineName: input.machineName, + workspace: { type: input.workspace.type }, + allowWorkspaceWrites: input.allowWorkspaceWrites, + allowWorkspaceCommands: input.allowWorkspaceCommands, + }); +} + +export function hashBrowserEnrollmentRequest(input: BrowserEnrollmentRequest): string { + return createHash('sha256').update(canonicalBrowserEnrollmentRequest(input)).digest('hex'); +} + +export function isBrowserSetupProof(value: unknown): value is BrowserSetupProof { + return record(value) && keys(value, ['operation', 'instance', 'attemptId', 'requestHash', 'timestamp', 'nonce']) && + typeof value.operation === 'string' && ['setup.start', 'setup.poll', 'setup.cancel', 'setup.complete'].includes(value.operation) && + instance(value.instance) && isBrowserEnrollmentToken(value.attemptId) && requestHash(value.requestHash) && + isBrowserEnrollmentTimestamp(value.timestamp) && isBrowserEnrollmentToken(value.nonce); +} + +export function canonicalBrowserSetupProof(input: BrowserSetupProof): string { + if (!isBrowserSetupProof(input)) throw new TypeError('Invalid browser setup proof'); + return [BROWSER_SETUP_PROOF_DOMAIN, input.operation, input.instance, input.attemptId, + input.requestHash, input.timestamp, input.nonce].join('\n'); +} + +export function isBrowserEnrollmentRedemptionContract(value: unknown): value is BrowserEnrollmentRedemptionContract { + return record(value) && value.operation === 'enrollment.redeem' && serverOrigin(value.serverId) && + instance(value.instance) && isBrowserEnrollmentToken(value.attemptId) && requestHash(value.requestHash) && + typeof value.workerId === 'string' && /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(value.workerId) && + isBrowserEnrollmentTimestamp(value.expiresAt); +} + +export function isBrowserEnrollmentBinding(value: unknown): value is BrowserEnrollmentBinding { + return record(value) && keys(value, ['tenantId', 'principal']) && + typeof value.tenantId === 'string' && /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(value.tenantId) && + record(value.principal) && keys(value.principal, ['type', 'id']) && + typeof value.principal.type === 'string' && ['deployment', 'tenant', 'user', 'role', 'group'].includes(value.principal.type) && + typeof value.principal.id === 'string' && /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(value.principal.id); +} + +const CONTRACT_KEYS = ['operation', 'serverId', 'instance', 'attemptId', 'requestHash', 'workerId', 'expiresAt']; + +export function isBrowserEnrollmentApproval(value: unknown): value is BrowserEnrollmentApproval { + return isBrowserEnrollmentRedemptionContract(value) && record(value) && + keys(value, [...CONTRACT_KEYS, 'protocolVersion', 'publicKey', 'binding']) && + value.protocolVersion === 1 && isBrowserEnrollmentPublicKey(value.publicKey) && isBrowserEnrollmentBinding(value.binding); +} + +export function isBrowserEnrollmentStatusRequest(value: unknown): value is BrowserEnrollmentStatusRequest { + return isBrowserEnrollmentRedemptionContract(value) && record(value) && + keys(value, [...CONTRACT_KEYS, 'protocolVersion']) && value.protocolVersion === 1; +} + +export function isBrowserEnrollmentRedemption(value: unknown): value is BrowserEnrollmentRedemption { + return isBrowserEnrollmentRedemptionContract(value) && record(value) && + keys(value, [...CONTRACT_KEYS, 'protocolVersion', 'signature']) && + value.protocolVersion === 1 && isBrowserEnrollmentSignature(value.signature); +} + +export function canonicalBrowserEnrollmentRedemption(input: BrowserEnrollmentRedemptionContract): string { + if (!isBrowserEnrollmentRedemptionContract(input)) throw new TypeError('Invalid browser enrollment redemption'); + return [BROWSER_ENROLLMENT_REDEMPTION_DOMAIN, input.operation, input.serverId, input.instance, + input.attemptId, input.requestHash, input.workerId, input.expiresAt].join('\n'); +} + +export function signBrowserSetupProof(privateKey: string, input: BrowserSetupProof): string { + return sign(null, Buffer.from(canonicalBrowserSetupProof(input)), privateKey).toString('base64url'); +} + +export function verifyBrowserSetupProof(publicKey: string, input: BrowserSetupProof, signature: string): boolean { + try { + return isBrowserEnrollmentPublicKey(publicKey) && isBrowserEnrollmentSignature(signature) && + verify(null, Buffer.from(canonicalBrowserSetupProof(input)), publicKey, Buffer.from(signature, 'base64url')); + } catch { + return false; + } +} + +export function signBrowserEnrollmentRedemption(privateKey: string, input: BrowserEnrollmentRedemptionContract): string { + return sign(null, Buffer.from(canonicalBrowserEnrollmentRedemption(input)), privateKey).toString('base64url'); +} + +export function verifyBrowserEnrollmentRedemption(publicKey: string, input: BrowserEnrollmentRedemptionContract, signature: string): boolean { + try { + return isBrowserEnrollmentPublicKey(publicKey) && isBrowserEnrollmentSignature(signature) && + verify(null, Buffer.from(canonicalBrowserEnrollmentRedemption(input)), publicKey, Buffer.from(signature, 'base64url')); + } catch { + return false; + } +} + +export const browserSetupRequestHash = hashBrowserEnrollmentRequest; +export const canonicalBrowserSetupRequest = canonicalBrowserEnrollmentRequest; +export const canonicalEnrollmentRedemption = canonicalBrowserEnrollmentRedemption; +export const signEnrollmentRedemption = signBrowserEnrollmentRedemption; +export const verifyEnrollmentRedemption = verifyBrowserEnrollmentRedemption; +export const validateBrowserSetupRequest = isBrowserEnrollmentRequest; +export const validateBrowserSetupProof = isBrowserSetupProof; +export const validateEnrollmentRedemption = isBrowserEnrollmentRedemptionContract; diff --git a/packages/code/src/browser-protection.test.ts b/packages/code/src/browser-protection.test.ts new file mode 100644 index 00000000..2da4e27d --- /dev/null +++ b/packages/code/src/browser-protection.test.ts @@ -0,0 +1,31 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { relative, sep } from 'node:path'; +import { SandboxManager } from '@anthropic-ai/sandbox-runtime'; + +import { browserSetupAttemptPath, browserSetupProtectedPaths } from './browser-setup.js'; + +test('browser setup namespace denies journal, lock and crash-left temporaries outside home', async t => { + const attempt = browserSetupAttemptPath('/srv/code-state/profile.json'); + const paths = browserSetupProtectedPaths(attempt); + assert.deepEqual(paths, ['/srv/code-state/profile.json.setup']); + const previous = SandboxManager.getConfig(); + t.after(async () => { + await SandboxManager.reset(); + if (previous) SandboxManager.updateConfig(previous); + }); + SandboxManager.updateConfig({ + network: { allowedDomains: [], deniedDomains: [] }, + filesystem: { denyRead: [], allowRead: [], allowWrite: [], denyWrite: paths }, + credentials: { files: paths.map(path => ({ path, mode: 'deny' as const })) }, + }); + const effective = SandboxManager.getFsReadConfig(); + for (const file of [attempt, `${attempt}.lock`, `${attempt}.0123456789abcdef.tmp`, `${attempt}.future.tmp`]) { + assert.ok(effective.denyOnly.some(directory => { + const child = relative(directory, file); + return child !== '..' && !child.startsWith(`..${sep}`); + }), 'every journal generation must remain beneath an effective directory deny'); + } + assert.deepEqual(browserSetupProtectedPaths(undefined), []); + assert.throws(() => browserSetupProtectedPaths('/srv/code-state/profile.json.setup.json'), /Unsupported setup journal/); +}); diff --git a/packages/code/src/browser-setup-cli.ts b/packages/code/src/browser-setup-cli.ts new file mode 100644 index 00000000..250fcdb7 --- /dev/null +++ b/packages/code/src/browser-setup-cli.ts @@ -0,0 +1,103 @@ +import { spawn } from 'node:child_process'; +import { createHash, createPublicKey } from 'node:crypto'; +import { hostname } from 'node:os'; +import { resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { cancelBrowserSetup, loadBrowserSetupAttempt, setupBrowserWorker, setupInstanceUrl } from './browser-setup.js'; +import { defaultWorkerProfilePath } from './worker-profile.js'; +import { setupWorkspaceDirectory } from './setup.js'; + +/** No shell, token, or inbound callback listener is involved. */ +export async function openSetupBrowser(url: string): Promise { + if (process.env.SSH_CONNECTION || process.env.SSH_TTY || + (process.platform === 'linux' && !process.env.DISPLAY && !process.env.WAYLAND_DISPLAY)) return false; + const command = process.platform === 'darwin' ? 'open' : process.platform === 'linux' ? 'xdg-open' : undefined; + if (!command) return false; + return new Promise(resolve => { + const child = spawn(command, [url], { stdio: 'ignore', shell: false }); + const timer = setTimeout(() => { child.kill(); resolve(false); }, 5000); + child.once('error', () => { clearTimeout(timer); resolve(false); }); + child.once('exit', code => { clearTimeout(timer); resolve(code === 0); }); + }); +} + +export async function runBrowserSetupCommand( + options: Map, + prompts: { + interactive: boolean; + ask(message: string): Promise; + yesNo(message: string): Promise; + signal: AbortSignal; + }, +): Promise { + const profilePath = resolve(String(options.get('--profile') ?? defaultWorkerProfilePath())); + const instance = setupInstanceUrl(String(options.get('--instance'))); + const saved = await loadBrowserSetupAttempt(profilePath); + if (saved && saved.request.instance !== instance) throw new Error('This profile has a setup attempt for another LibreChat instance. Resume the original instance or choose another profile'); + if (options.has('--cancel')) { + await cancelBrowserSetup(profilePath, prompts.signal); + process.stdout.write('Setup cancellation recorded. The local machine key and attempt were preserved.\n'); + return; + } + let workspaceDirectory = options.get('--worker-dir') as string | undefined; + if (saved) { + if (workspaceDirectory && (saved.profile.workspace.type !== 'directory' || await setupWorkspaceDirectory(workspaceDirectory) !== saved.profile.workspace.path)) + throw new Error('The workspace differs from the saved setup attempt'); + if (options.has('--default-workspace') && saved.profile.workspace.type !== 'private') throw new Error('The workspace differs from the saved setup attempt'); + if ((options.has('--allow-workspace-writes') && !saved.profile.allowWorkspaceWrites) || + (options.has('--allow-workspace-commands') && !saved.profile.allowWorkspaceCommands)) throw new Error('Permissions differ from the saved setup attempt'); + workspaceDirectory = saved.profile.workspace.type === 'directory' ? saved.profile.workspace.path : undefined; + process.stdout.write('Resuming the saved setup attempt with its original machine key and access choices.\n'); + } else if (prompts.interactive && !workspaceDirectory && !options.has('--default-workspace')) { + workspaceDirectory = (await prompts.ask('Project directory (Enter for a new private workspace): ')) || undefined; + } + if (workspaceDirectory !== undefined) workspaceDirectory = await setupWorkspaceDirectory(workspaceDirectory); + const machineName = String(options.get('--machine-name') ?? saved?.request.machineName ?? hostname().replace(/[^A-Za-z0-9 ._()-]/g, '-').slice(0, 80)); + const allowWorkspaceWrites = saved?.profile.allowWorkspaceWrites ?? (options.has('--allow-workspace-writes') || + (prompts.interactive && await prompts.yesNo('Allow file writes within this workspace?'))); + const allowWorkspaceCommands = saved?.profile.allowWorkspaceCommands ?? (options.has('--allow-workspace-commands') || + (prompts.interactive && await prompts.yesNo('Allow sandboxed commands? Commands can modify workspace files independently of file-write tooling'))); + process.stdout.write(`\nLibreChat: ${instance}\nMachine: ${machineName}\nWorkspace: ${workspaceDirectory ?? 'new private workspace'}\nFile writes: ${allowWorkspaceWrites ? 'enabled' : 'disabled'}\nCommands: ${allowWorkspaceCommands ? 'enabled, restricted sandbox policy' : 'disabled'}\nProfile: ${profilePath}\nPairing does not attest this host or replace tool approvals.\n`); + if (!saved && prompts.interactive && !await prompts.yesNo('Request browser approval with these choices?')) { + process.stdout.write('Setup cancelled before enrollment.\n'); + return; + } + await setupBrowserWorker({ + instance, machineName, profilePath, + identityPath: options.get('--identity') as string | undefined ?? saved?.profile.identityPath, + workspaceDirectory, allowWorkspaceWrites, allowWorkspaceCommands, + signal: prompts.signal, + }, { + onVerification: async status => { + const attempt = (await loadBrowserSetupAttempt(profilePath))!; + const fingerprint = createHash('sha256').update(createPublicKey(attempt.request.publicKey).export({ type: 'spki', format: 'der' })).digest('hex'); + process.stdout.write(`Machine key: SHA256:${fingerprint}\n`); + // Always show both, even when opening a browser succeeds. + process.stdout.write(`\nApprove this machine in LibreChat:\n ${status.verificationUrl}\nVerification code: ${status.verificationCode}\nExpires: ${status.expiresAt}\nCheck the matching code and key identity before approving. Waiting for your explicit decision.\n`); + if (!options.has('--no-browser') && prompts.interactive && !await openSetupBrowser(status.verificationUrl)) + process.stdout.write('Could not open a browser here. Open the URL on another device and compare the code.\n'); + }, + }); + process.stdout.write('\nMachine enrolled. Profile saved. It is not connected or ready until the worker starts and readiness is confirmed.\n'); + prompts.signal.throwIfAborted(); + const start = options.has('--start') || (prompts.interactive && await prompts.yesNo('Start the worker in the foreground now?')); + if (start) { + prompts.signal.throwIfAborted(); + process.stdout.write('Starting the saved profile in the foreground. Stop with Ctrl-C.\n'); + await new Promise((resolve, reject) => { + const child = spawn(process.execPath, [fileURLToPath(new URL('./cli.js', import.meta.url)), 'run', '--profile', profilePath], { stdio: 'inherit' }); + const abort = () => child.kill('SIGINT'); + prompts.signal.addEventListener('abort', abort, { once: true }); + child.once('error', reject); + child.once('exit', code => { + prompts.signal.removeEventListener('abort', abort); + if (code === 0 || prompts.signal.aborted) resolve(); + else reject(new Error('Worker stopped before a successful foreground session; enrollment and profile remain saved')); + }); + }); + } else { + const quoted = `'${profilePath.replace(/'/g, "'\\''")}'`; + process.stdout.write(`Start it with:\n librechat-code run${profilePath === resolve(defaultWorkerProfilePath()) ? '' : ` --profile ${quoted}`}\n`); + } +} diff --git a/packages/code/src/browser-setup.test.ts b/packages/code/src/browser-setup.test.ts new file mode 100644 index 00000000..077db673 --- /dev/null +++ b/packages/code/src/browser-setup.test.ts @@ -0,0 +1,279 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import test from 'node:test'; + +import { createBridgeIdentity } from './identity.js'; +import { browserSetupRequestHash, verifyBrowserSetupProof, verifyEnrollmentRedemption } from './browser-enrollment-protocol.js'; +import type { BrowserSetupRequest, BrowserSetupProof } from './browser-enrollment-protocol.js'; +import { browserSetupAttemptPath, browserSetupProtectedPaths, loadBrowserSetupAttempt, setupBrowserWorker, setupInstanceUrl, cancelBrowserSetup } from './browser-setup.js'; +import type { BrowserSetupOptions } from './browser-setup.js'; +import { loadBridgeIdentity, saveBridgeIdentity, savePrivateJson } from './storage.js'; +import { loadWorkerProfile } from './worker-profile.js'; +import { parseSetupArguments } from './setup-cli.js'; +import { runBrowserSetupCommand } from './browser-setup-cli.js'; + +async function fixture(t: test.TestContext) { + const root = await mkdtemp(join(tmpdir(), 'browser-setup-')); + t.after(() => rm(root, { recursive: true, force: true })); + const options: BrowserSetupOptions = { + instance: 'https://chat.example', machineName: 'headless-machine', + profilePath: join(root, 'profile.json'), + allowWorkspaceWrites: false, allowWorkspaceCommands: false, + }; + let completeCalls = 0; + let startCalls = 0; + let initialRequest: BrowserSetupRequest | undefined; + let lostStart = false; + let lostComplete = false; + let status: 'approved' | 'denied' | 'expired' | 'cancelled' = 'approved'; + const credential = { protocolVersion: 1 as const, workerId: 'approved-machine', credential: 'approved-credential-that-is-long-enough', expiresAt: '2099-01-01T00:00:00.000Z' }; + const fetchImpl = (async (url: string | URL | Request, init?: RequestInit) => { + if (String(url).endsWith('/capabilities')) return Response.json({ browserEnrollmentVersion: 1 }); + const body = JSON.parse(String(init?.body)) as { request: BrowserSetupRequest; proof: BrowserSetupProof; signature: string; redemptionSignature?: string }; + assert.equal((body.request as unknown as Record).privateKey, undefined); + assert.equal(verifyBrowserSetupProof(body.request.publicKey, body.proof, body.signature), true); + assert.equal(body.proof.requestHash, browserSetupRequestHash(body.request)); + const saved = await loadBrowserSetupAttempt(options.profilePath!); + assert.equal(saved?.request.publicKey, body.request.publicKey, 'key saved before remote effect'); + assert.equal(saved?.request.attemptId, body.request.attemptId); + initialRequest ??= body.request; + assert.deepEqual(body.request, initialRequest, 'retries retain immutable request and key'); + const expiresAt = new Date(Date.parse(body.request.createdAt) + 600_000).toISOString(); + const approval = { operation: 'enrollment.redeem' as const, serverId: 'https://code.example', instance: body.request.instance, + attemptId: body.request.attemptId, requestHash: browserSetupRequestHash(body.request), workerId: credential.workerId, + expiresAt, codeApiUrl: 'https://code.example/v1' }; + const base = { version: 1, expiresAt, verificationCode: 'ABCD-EFGH', verificationUrl: `https://chat.example/code/setup/${body.request.attemptId}` }; + if (String(url).endsWith('/start')) { + startCalls++; + if (lostStart) { lostStart = false; throw new Error('lost issuance response'); } + return Response.json({ ...base, status: 'pending' }); + } + if (String(url).endsWith('/cancel')) return Response.json({ ...base, status: 'cancelled' }); + if (String(url).endsWith('/complete')) { + completeCalls++; + assert.equal(body.proof.operation, 'setup.complete'); + assert.equal(verifyEnrollmentRedemption(body.request.publicKey, approval, body.redemptionSignature!), true); + if (lostComplete) { lostComplete = false; throw new Error('lost completion response'); } + return Response.json({ ...base, status: 'completed', approval, credential }); + } + return Response.json({ ...base, status, ...(status === 'approved' ? { approval } : {}) }); + }) as typeof fetch; + return { root, options, fetchImpl, credential, counts: () => ({ completeCalls, startCalls }), + loseStart: () => { lostStart = true; }, loseComplete: () => { lostComplete = true; }, setStatus: (value: typeof status) => { status = value; } }; +} + +test('headless setup displays URL and code and saves the existing least-privilege profile', async t => { + const f = await fixture(t); + let displayed = false; + const saved = await setupBrowserWorker(f.options, { fetchImpl: f.fetchImpl, pollIntervalMs: 0, onVerification: async value => { + assert.match(value.verificationUrl, /^https:\/\/chat.example\/code\/setup\//); + assert.equal(value.verificationCode, 'ABCD-EFGH'); + assert.ok(await loadBrowserSetupAttempt(f.options.profilePath!)); + displayed = true; + } }); + assert.equal(displayed, true); + assert.deepEqual(await loadWorkerProfile(saved.profilePath), saved.profile); + assert.equal(saved.profile.allowWorkspaceCommands, false); + assert.equal(saved.profile.allowWorkspaceWrites, false); + assert.deepEqual(saved.profile.workspace, { type: 'private' }); + const identity = await loadBridgeIdentity(saved.profile.identityPath); + assert.equal(identity.credential, f.credential.credential); + assert.equal((await stat(browserSetupAttemptPath(saved.profilePath))).mode & 0o777, 0o600); + assert.ok(!(await readFile(saved.profilePath, 'utf8')).includes('PRIVATE KEY')); + const receipt = JSON.parse(await readFile(browserSetupAttemptPath(saved.profilePath), 'utf8')); + assert.equal(receipt.privateKey, undefined); + assert.equal(receipt.identity, undefined); + assert.equal(receipt.completed, true); + assert.equal(identity.setupAttemptPath, browserSetupAttemptPath(saved.profilePath)); +}); + +for (const boundary of ['start', 'complete'] as const) test(`lost ${boundary} response resumes the identical key and attempt`, async t => { + const f = await fixture(t); + if (boundary === 'start') f.loseStart(); else f.loseComplete(); + await assert.rejects(setupBrowserWorker(f.options, { fetchImpl: f.fetchImpl, pollIntervalMs: 0 }), /lost/); + const before = await loadBrowserSetupAttempt(f.options.profilePath!); + assert.ok(before); + await setupBrowserWorker(f.options, { fetchImpl: f.fetchImpl, pollIntervalMs: 0 }); + const after = await loadBrowserSetupAttempt(f.options.profilePath!); + assert.deepEqual(after?.request, before.request); + assert.equal(after?.privateKey, before.privateKey); + assert.equal(f.counts().startCalls, 2); +}); + +test('SIGKILL during journal publish preserves the attempt and contains the secret temporary after completion', { timeout: 10_000 }, async t => { + const f = await fixture(t); + f.loseComplete(); + await assert.rejects(setupBrowserWorker(f.options, { fetchImpl: f.fetchImpl, pollIntervalMs: 0 }), /lost/); + const before = (await loadBrowserSetupAttempt(f.options.profilePath!))!; + const attempt = browserSetupAttemptPath(f.options.profilePath!); + const child = spawn(process.execPath, ['--input-type=module', '--eval', ` + import fs from 'node:fs/promises'; + import { syncBuiltinESMExports } from 'node:module'; + const rename = fs.rename; + fs.rename = async (from, to) => { + if (to === ${JSON.stringify(attempt)}) { + process.send('journal-written'); + await new Promise(resolve => setTimeout(resolve, 60_000)); + } + return rename(from, to); + }; + syncBuiltinESMExports(); + const { savePrivateJson } = await import(${JSON.stringify(new URL('./storage.js', import.meta.url).href)}); + await savePrivateJson(${JSON.stringify(attempt)}, JSON.parse(await fs.readFile(${JSON.stringify(attempt)}, 'utf8'))); + `], { stdio: ['ignore', 'ignore', 'pipe', 'ipc'] }); + let stderr = ''; + child.stderr!.on('data', chunk => { stderr = (stderr + String(chunk)).slice(-4096); }); + const closed = once(child, 'close'); + t.after(async () => { child.kill('SIGKILL'); await closed; }); + await Promise.race([ + once(child, 'message', { signal: t.signal }).then(([message]) => assert.equal(message, 'journal-written')), + closed.then(() => { throw new Error(`Journal writer exited before the crash boundary: ${stderr}`); }), + ]); + child.kill('SIGKILL'); + assert.deepEqual(await closed, [null, 'SIGKILL']); + const directory = dirname(attempt); + const orphanName = (await readdir(directory)).find(name => /^attempt\.json\.[a-f0-9]{16}\.tmp$/.test(name)); + assert.ok(orphanName, 'real atomic writer left its secret generation when killed before rename'); + const orphan = join(directory, orphanName); + const crashCopy = JSON.parse(await readFile(orphan, 'utf8')); + assert.equal(crashCopy.privateKey, before.privateKey); + assert.deepEqual(browserSetupProtectedPaths(attempt), [directory]); + const saved = await setupBrowserWorker(f.options, { fetchImpl: f.fetchImpl, pollIntervalMs: 0 }); + const identity = await loadBridgeIdentity(saved.profile.identityPath); + assert.equal(identity.privateKey, before.privateKey); + assert.equal(identity.setupAttemptPath, attempt); + assert.deepEqual((await loadBrowserSetupAttempt(saved.profilePath))?.request, before.request); + assert.equal(JSON.parse(await readFile(attempt, 'utf8')).privateKey, undefined); + assert.ok((await stat(orphan)).isFile(), 'crash leftovers stay inside the denied namespace after final-receipt scrubbing'); +}); + +test('an older draft journal is never silently replaced by a different key or attempt', async t => { + const f = await fixture(t); + const oldPath = `${f.options.profilePath}.setup.json`; + await writeFile(oldPath, 'preserve the existing draft key', { mode: 0o600 }); + await assert.rejects(setupBrowserWorker(f.options, { fetchImpl: async () => { assert.fail('no remote allocation'); } }), /older draft setup journal/); + assert.equal(await readFile(oldPath, 'utf8'), 'preserve the existing draft key'); +}); + +test('local profile failure after enrollment resumes without remote enrollment effects', async t => { + const f = await fixture(t); + await assert.rejects(setupBrowserWorker(f.options, { fetchImpl: f.fetchImpl, pollIntervalMs: 0, saveProfile: async () => { throw new Error('disk full'); } }), /enrolled.*profile persistence failed/); + assert.ok((await loadBrowserSetupAttempt(f.options.profilePath!))?.identity); + assert.equal(f.counts().completeCalls, 1); + const saved = await setupBrowserWorker(f.options, { fetchImpl: async () => { throw new Error('must not enroll again'); } }); + assert.ok(await loadWorkerProfile(saved.profilePath)); + assert.equal(f.counts().completeCalls, 1); +}); + +test('rerunning completed setup preserves a newer startup credential without network or redundant writes', async t => { + const f = await fixture(t); + const saved = await setupBrowserWorker(f.options, { fetchImpl: f.fetchImpl, pollIntervalMs: 0 }); + const identity = { ...await loadBridgeIdentity(saved.profile.identityPath), credential: 'newer-credential-recovered-after-startup' }; + await saveBridgeIdentity(saved.profile.identityPath, identity); + await setupBrowserWorker(f.options, { + fetchImpl: async () => { assert.fail('completed setup does not enroll again'); }, + saveIdentity: async () => { assert.fail('do not replace a newer credential'); }, + saveProfile: async () => { assert.fail('do not rewrite an unchanged profile'); }, + }); + assert.deepEqual(await loadBridgeIdentity(saved.profile.identityPath), identity); +}); + +test('identity persistence failure after enrollment preserves the completed local receipt', async t => { + const f = await fixture(t); + await assert.rejects(setupBrowserWorker(f.options, { fetchImpl: f.fetchImpl, pollIntervalMs: 0, saveIdentity: async () => { throw new Error('identity disk full'); } }), /disk full/); + assert.ok((await loadBrowserSetupAttempt(f.options.profilePath!))?.identity); + await setupBrowserWorker(f.options, { fetchImpl: async () => { throw new Error('must not enroll again'); } }); + assert.equal(f.counts().completeCalls, 1); +}); + +test('expired setup with a lost completion response recovers only existing key-bound enrollment', async t => { + const f = await fixture(t); + f.loseComplete(); + await assert.rejects(setupBrowserWorker(f.options, { fetchImpl: f.fetchImpl, pollIntervalMs: 0 }), /lost/); + const attempt = (await loadBrowserSetupAttempt(f.options.profilePath!))!; + const now = Date.now(); + t.mock.method(Date, 'now', () => now + 11 * 60_000); + const saved = await setupBrowserWorker(f.options, { + fetchImpl: async () => { assert.fail('an expired attempt must not allocate another session'); }, + recoverCredential: async options => { + assert.equal(options.privateKey, attempt.privateKey); + assert.equal(options.workerId, attempt.approval?.workerId); + assert.equal(options.codeApiUrl, attempt.approval?.codeApiUrl); + return f.credential; + }, + }); + assert.equal((await loadBridgeIdentity(saved.profile.identityPath)).publicKey, attempt.request.publicKey); + assert.equal(f.counts().completeCalls, 1); +}); + +test('unsupported servers fail closed and never invoke manual redemption', async t => { + const f = await fixture(t); + let calls = 0; + await assert.rejects(setupBrowserWorker(f.options, { fetchImpl: async url => { + calls++; + assert.ok(String(url).endsWith('/capabilities')); + return Response.json({ protocolVersion: 1 }); + } }), /will not downgrade/); + assert.equal(calls, 1); + assert.ok(await loadBrowserSetupAttempt(f.options.profilePath!)); +}); + +for (const status of ['denied', 'expired', 'cancelled'] as const) test(`${status} setup preserves the saved identity and does not complete`, async t => { + const f = await fixture(t); + f.setStatus(status); + await assert.rejects(setupBrowserWorker(f.options, { fetchImpl: f.fetchImpl, pollIntervalMs: 0 }), new RegExp(status)); + assert.equal(f.counts().completeCalls, 0); + assert.ok(await loadBrowserSetupAttempt(f.options.profilePath!)); +}); + +test('retry cannot widen permissions, rename a machine, switch instances, or replace a key', async t => { + const f = await fixture(t); + f.loseStart(); + await assert.rejects(setupBrowserWorker(f.options, { fetchImpl: f.fetchImpl }), /lost/); + for (const change of [{ allowWorkspaceWrites: true }, { machineName: 'other-machine' }, { instance: 'https://other.example' }]) { + await assert.rejects(setupBrowserWorker({ ...f.options, ...change }, { fetchImpl: async () => { assert.fail('no remote effects'); } }), /choices differ/); + } + const attempt = (await loadBrowserSetupAttempt(f.options.profilePath!))!; + await savePrivateJson(browserSetupAttemptPath(f.options.profilePath!), { ...attempt, privateKey: createBridgeIdentity().privateKey }); + await assert.rejects(loadBrowserSetupAttempt(f.options.profilePath!), /key does not match/); +}); + +test('cancellation is signed by the persisted machine key', async t => { + const f = await fixture(t); + f.loseStart(); + await assert.rejects(setupBrowserWorker(f.options, { fetchImpl: f.fetchImpl }), /lost/); + await cancelBrowserSetup(f.options.profilePath!, undefined, f.fetchImpl); +}); + +test('cancellation after enrollment saves the profile but prevents requested foreground startup', { timeout: 5000 }, async t => { + const f = await fixture(t); + const controller = new AbortController(); + t.mock.method(globalThis, 'fetch', async (input: Parameters[0], init?: RequestInit) => { + const response = await f.fetchImpl(input, init); + if (String(input).endsWith('/complete')) controller.abort(new Error('owner cancelled startup')); + return response; + }); + await assert.rejects(runBrowserSetupCommand(new Map([ + ['--instance', f.options.instance], ['--machine-name', f.options.machineName], + ['--profile', f.options.profilePath!], ['--default-workspace', true], ['--start', true], + ]), { + interactive: false, signal: controller.signal, + ask: async () => { assert.fail('no prompt expected'); }, + yesNo: async () => { assert.fail('no prompt expected'); }, + }), /owner cancelled startup/); + assert.ok(await loadWorkerProfile(f.options.profilePath!)); +}); + +test('browser flags preserve manual automation without accepting mixed credentials', () => { + const flags = parseSetupArguments(['--instance', 'https://chat.example', '--non-interactive', '--default-workspace', '--no-browser']); + assert.equal(flags.get('--instance'), 'https://chat.example'); + for (const manual of ['--url=https://code.example/v1', '--worker-id=worker', '--code-stdin']) + assert.throws(() => parseSetupArguments(['--instance=https://chat.example', manual]), /manual pairing/); + assert.throws(() => parseSetupArguments(['--no-browser']), /require --instance/); + assert.equal(setupInstanceUrl('https://chat.example/'), 'https://chat.example'); + assert.throws(() => setupInstanceUrl('https://user:secret@chat.example'), /credentials/); +}); diff --git a/packages/code/src/browser-setup.ts b/packages/code/src/browser-setup.ts new file mode 100644 index 00000000..7bbbe493 --- /dev/null +++ b/packages/code/src/browser-setup.ts @@ -0,0 +1,352 @@ +import { createPrivateKey, createPublicKey, randomBytes } from 'node:crypto'; +import { lstat } from 'node:fs/promises'; +import { basename, dirname, join, resolve } from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; + +import { createBridgeIdentity } from './identity.js'; +import { recoverWorkerCredential } from './credential-recovery.js'; +import { + browserSetupRequestHash, + signBrowserSetupProof, + signEnrollmentRedemption, + isBrowserEnrollmentRedemptionContract, +} from './browser-enrollment-protocol.js'; +import type { + BrowserSetupRequest, + BrowserSetupProof, + EnrollmentRedemption, +} from './browser-enrollment-protocol.js'; +import { assertPrivateStorageSupported } from './private-storage.js'; +import { withProcessLock } from './process-lock.js'; +import { + assertIdentityPathIsPrivate, + loadBridgeIdentity, + ensurePrivateWorkspaceDirectory, + readPrivateJson, + saveBridgeIdentity, + savePrivateJson, +} from './storage.js'; +import { checkSetupCommands, setupCodeApiUrl, setupWorkspaceDirectory } from './setup.js'; +import { + assertWorkerProfileIsolation, + defaultWorkerProfilePath, + loadWorkerProfile, + saveWorkerProfile, + workerProfileArguments, +} from './worker-profile.js'; +import type { WorkerProfile } from './worker-profile.js'; +import type { PairedBridgeWorkerIdentity } from './pairing.js'; + +export interface BrowserSetupAttempt { + version: 1; + request: BrowserSetupRequest; + privateKey: string; + profile: WorkerProfile; + approval?: BrowserSetupApproval; + identity?: PairedBridgeWorkerIdentity; + completed?: true; +} + +export type BrowserSetupApproval = EnrollmentRedemption & { codeApiUrl: string }; + +export interface BrowserSetupOptions { + instance: string; + machineName: string; + profilePath?: string; + identityPath?: string; + workspaceDirectory?: string; + allowWorkspaceWrites: boolean; + allowWorkspaceCommands: boolean; + signal?: AbortSignal; +} + +interface SetupStatus { + version: 1; + status: 'pending' | 'approved' | 'completed' | 'denied' | 'cancelled' | 'expired'; + expiresAt: string; + verificationUrl: string; + verificationCode: string; + approval?: BrowserSetupApproval; + credential?: { protocolVersion: 1; workerId: string; credential: string; expiresAt: string }; +} + +interface BrowserSetupDependencies { + fetchImpl?: typeof fetch; + checkCommands?: typeof checkSetupCommands; + onVerification?: (status: Pick) => void | Promise; + pollIntervalMs?: number; + saveProfile?: typeof saveWorkerProfile; + saveIdentity?: typeof saveBridgeIdentity; + recoverCredential?: typeof recoverWorkerCredential; +} + +function record(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +export function setupInstanceUrl(value: string): string { + return setupCodeApiUrl(value); +} + +export function browserSetupAttemptPath(profilePath: string): string { + return join(`${resolve(profilePath)}.setup`, 'attempt.json'); +} + +export function browserSetupProtectedPaths(attemptPath: string | undefined): string[] { + if (attemptPath === undefined) return []; + const directory = dirname(attemptPath); + if (basename(attemptPath) !== 'attempt.json' || !directory.endsWith('.setup')) + throw new Error('Unsupported setup journal location; preserve the original identity and journal'); + // Directory protection includes atomic-write temporaries, even after SIGKILL. + return [directory]; +} + +export async function loadBrowserSetupAttempt(profilePath: string): Promise { + // The earlier, unreleased draft stored journals beside profiles. Never rekey them silently. + const previous = await lstat(`${resolve(profilePath)}.setup.json`).catch((error: NodeJS.ErrnoException) => { + if (error.code === 'ENOENT') return undefined; + throw error; + }); + if (previous) throw new Error('An older draft setup journal exists; preserve its key and attempt instead of starting a new enrollment'); + let value: unknown; + try { value = await readPrivateJson(browserSetupAttemptPath(profilePath), 32 * 1024); } + catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + throw error; + } + if (!record(value) || value.version !== 1 || !record(value.request) || !record(value.profile)) + throw new Error('Invalid saved setup attempt; preserve it and choose another profile, do not replace its machine key'); + const attempt = value as unknown as BrowserSetupAttempt; + workerProfileArguments(attempt.profile); + if (attempt.completed === true) { + attempt.identity = await loadBridgeIdentity(attempt.profile.identityPath); + attempt.privateKey = attempt.identity.privateKey; + } + if (typeof attempt.privateKey !== 'string') throw new Error('Saved setup key is unavailable; preserve the original identity'); + if (setupInstanceUrl(attempt.request.instance) !== attempt.request.instance || + createPublicKey(createPrivateKey(attempt.privateKey)).export({ type: 'spki', format: 'pem' }) !== attempt.request.publicKey) + throw new Error('Saved setup key does not match its request'); + // The shared canonicalizer validates the immutable request before it is signed. + browserSetupRequestHash(attempt.request); + if (attempt.approval && (!isBrowserEnrollmentRedemptionContract(attempt.approval) || + attempt.approval.instance !== attempt.request.instance || attempt.approval.attemptId !== attempt.request.attemptId || + attempt.approval.requestHash !== browserSetupRequestHash(attempt.request) || + setupCodeApiUrl(attempt.approval.codeApiUrl) !== attempt.approval.codeApiUrl || + new URL(attempt.approval.codeApiUrl).origin !== attempt.approval.serverId)) + throw new Error('Saved approval does not match its setup request'); + if (attempt.identity && (!attempt.approval || attempt.identity.protocolVersion !== 1 || + attempt.identity.publicKey !== attempt.request.publicKey || attempt.identity.privateKey !== attempt.privateKey || + attempt.identity.workerId !== attempt.approval.workerId || attempt.identity.codeApiUrl !== attempt.approval.codeApiUrl || + typeof attempt.identity.credential !== 'string' || attempt.identity.credential.length < 32 || + attempt.identity.credential.length > 4096 || !Number.isFinite(Date.parse(attempt.identity.expiresAt)))) + throw new Error('Saved completion does not match its approved machine key'); + return attempt; +} + +async function boundedJson(response: Response): Promise { + const reader = response.body?.getReader(); + if (!reader) throw new Error('LibreChat returned an empty setup response'); + let size = 0; + const chunks: Uint8Array[] = []; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > 32 * 1024) throw new Error('LibreChat setup response exceeds 32 KiB'); + chunks.push(value); + } + try { return JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(Buffer.concat(chunks))) as unknown; } + catch { throw new Error('LibreChat returned an invalid setup response'); } + } finally { await reader.cancel().catch(() => undefined); reader.releaseLock(); } +} + +async function requestJson(instance: string, path: string, body: unknown, signal: AbortSignal | undefined, fetchImpl: typeof fetch): Promise { + const response = await fetchImpl(`${instance}/api/code-environments/setup/${path}`, { + method: body === undefined ? 'GET' : 'POST', + redirect: 'error', + headers: { 'Content-Type': 'application/json' }, + body: body === undefined ? undefined : JSON.stringify(body), + signal: signal ? AbortSignal.any([signal, AbortSignal.timeout(30_000)]) : AbortSignal.timeout(30_000), + }); + if (!response.ok) { + await response.body?.cancel(); + if ([404, 405, 501].includes(response.status)) + throw new Error('This LibreChat instance does not support browser-approved machine setup. Update compatible servers first, or explicitly use the existing manual pairing flow'); + throw new Error(`LibreChat setup request failed (HTTP ${response.status}). Your saved key and attempt are unchanged; rerun the same setup command to resume`); + } + return boundedJson(response); +} + +function machineBody(attempt: BrowserSetupAttempt, operation: BrowserSetupProof['operation']): object { + const proof: BrowserSetupProof = { + operation, + instance: attempt.request.instance, + attemptId: attempt.request.attemptId, + requestHash: browserSetupRequestHash(attempt.request), + timestamp: new Date().toISOString(), + nonce: randomBytes(32).toString('base64url'), + }; + return { + request: attempt.request, + proof, + signature: signBrowserSetupProof(attempt.privateKey, proof), + ...(operation === 'setup.complete' && attempt.approval ? { + redemptionSignature: signEnrollmentRedemption(attempt.privateKey, attempt.approval), + } : {}), + }; +} + +function parseStatus(value: unknown, attempt: BrowserSetupAttempt): SetupStatus { + if (!record(value) || value.version !== 1 || + !['pending', 'approved', 'completed', 'denied', 'cancelled', 'expired'].includes(String(value.status)) || + typeof value.expiresAt !== 'string' || !Number.isFinite(Date.parse(value.expiresAt)) || + Date.parse(value.expiresAt) > Date.parse(attempt.request.createdAt) + 10 * 60_000 || + typeof value.verificationUrl !== 'string' || typeof value.verificationCode !== 'string' || + !/^[A-Z0-9-]{6,20}$/.test(value.verificationCode)) + throw new Error('LibreChat returned an invalid setup session'); + let url: URL; + try { url = new URL(value.verificationUrl); } + catch { throw new Error('LibreChat returned an invalid approval destination'); } + if (url.href !== `${attempt.request.instance}/code/setup/${attempt.request.attemptId}` || + url.username || url.password || url.search || url.hash) + throw new Error('LibreChat returned an unexpected approval destination'); + if (value.approval !== undefined) { + const approval = value.approval as BrowserSetupApproval; + if (!isBrowserEnrollmentRedemptionContract(approval) || approval.operation !== 'enrollment.redeem' || + approval.instance !== attempt.request.instance || approval.attemptId !== attempt.request.attemptId || + approval.requestHash !== browserSetupRequestHash(attempt.request) || + approval.expiresAt !== value.expiresAt || typeof approval.workerId !== 'string' || + typeof approval.codeApiUrl !== 'string' || setupCodeApiUrl(approval.codeApiUrl) !== approval.codeApiUrl || + new URL(approval.codeApiUrl).origin !== approval.serverId) + throw new Error('LibreChat approval does not match this setup attempt'); + if (attempt.approval && JSON.stringify(attempt.approval) !== JSON.stringify(approval)) + throw new Error('LibreChat changed the saved approval; refusing to replace the machine binding'); + } + return value as unknown as SetupStatus; +} + +async function assertEmptyOrMatching(path: string, expected: unknown, load: (path: string) => Promise, same = (left: unknown, right: unknown) => JSON.stringify(left) === JSON.stringify(right)): Promise { + const metadata = await lstat(path); + if (metadata.isFile() && metadata.size === 0) return false; + if (!same(await load(path), expected)) + throw new Error('Existing profile or identity differs from the saved setup attempt; refusing to replace it'); + return true; +} + +function sameEnrolledKey(left: unknown, right: unknown): boolean { + return record(left) && record(right) && ['protocolVersion', 'workerId', 'codeApiUrl', 'publicKey', 'privateKey', 'setupAttemptPath'] + .every(key => left[key] === right[key]); +} + +/** Every remote mutation follows a durable, private key and immutable local request. */ +export async function setupBrowserWorker(options: BrowserSetupOptions, dependencies: BrowserSetupDependencies = {}): Promise<{ profilePath: string; profile: WorkerProfile }> { + options.signal?.throwIfAborted(); + assertPrivateStorageSupported(); + const instance = setupInstanceUrl(options.instance); + const profilePath = resolve(options.profilePath ?? defaultWorkerProfilePath()); + const attemptPath = browserSetupAttemptPath(profilePath); + await ensurePrivateWorkspaceDirectory(dirname(attemptPath)); + // This stable inode is deliberately retained. Removing an advisory lock can split its holders. + await assertIdentityPathIsPrivate(`${attemptPath}.lock`); + return withProcessLock(`${attemptPath}.lock`, async () => { + let attempt = await loadBrowserSetupAttempt(profilePath); + const workspaceDirectory = options.workspaceDirectory === undefined ? undefined : await setupWorkspaceDirectory(options.workspaceDirectory); + const profile: WorkerProfile = { + version: 1, + identityPath: resolve(options.identityPath ?? `${profilePath}.identity.json`), + workspace: workspaceDirectory ? { type: 'directory', path: workspaceDirectory } : { type: 'private' }, + allowWorkspaceWrites: options.allowWorkspaceWrites, + allowWorkspaceCommands: options.allowWorkspaceCommands, + }; + workerProfileArguments(profile); + if (attempt && (attempt.request.instance !== instance || attempt.request.machineName !== options.machineName || JSON.stringify(attempt.profile) !== JSON.stringify(profile))) + throw new Error('Setup choices differ from the saved attempt. Resume its original instance and permissions, or explicitly select another profile'); + if (profile.allowWorkspaceCommands) await (dependencies.checkCommands ?? checkSetupCommands)(); + const reservations = []; + try { + reservations.push(await assertIdentityPathIsPrivate(profilePath, !attempt)); + reservations.push(await assertIdentityPathIsPrivate(profile.identityPath, !attempt)); + reservations.push(await assertIdentityPathIsPrivate(attemptPath, !attempt)); + await assertWorkerProfileIsolation(profilePath, profile.identityPath, workspaceDirectory); + await assertWorkerProfileIsolation(attemptPath, profilePath, workspaceDirectory); + await assertWorkerProfileIsolation(attemptPath, profile.identityPath, workspaceDirectory); + if (!attempt) { + const key = createBridgeIdentity(); + attempt = { + version: 1, + request: { + version: 1, instance, attemptId: randomBytes(32).toString('base64url'), + createdAt: new Date().toISOString(), publicKey: key.publicKey, + machineName: options.machineName, workspace: { type: profile.workspace.type }, + allowWorkspaceWrites: profile.allowWorkspaceWrites, allowWorkspaceCommands: profile.allowWorkspaceCommands, + }, + privateKey: key.privateKey, profile, + }; + browserSetupRequestHash(attempt.request); + await savePrivateJson(attemptPath, attempt); + } + const persist = () => savePrivateJson(attemptPath, attempt); + const publish = async () => { + if (!await assertEmptyOrMatching(profile.identityPath, attempt!.identity, loadBridgeIdentity, sameEnrolledKey)) + await (dependencies.saveIdentity ?? saveBridgeIdentity)(profile.identityPath, attempt!.identity!); + if (!attempt!.completed) { + await savePrivateJson(attemptPath, { version: 1, request: attempt!.request, profile, approval: attempt!.approval, completed: true }); + attempt!.completed = true; + } + const profileSaved = await assertEmptyOrMatching(profilePath, profile, loadWorkerProfile); + try { if (!profileSaved) await (dependencies.saveProfile ?? saveWorkerProfile)(profilePath, profile); } + catch { throw new Error(`Machine enrolled and identity saved, but profile persistence failed. Rerun setup with this same profile to resume: ${profilePath}`); } + return { profilePath, profile }; + }; + if (attempt.identity) return await publish(); + const fetchImpl = dependencies.fetchImpl ?? fetch; + if (attempt.approval && Date.now() >= Date.parse(attempt.approval.expiresAt)) { + // Session expiry cannot undo enrollment. Recovery proves only existing trust. + const credential = await (dependencies.recoverCredential ?? recoverWorkerCredential)({ + codeApiUrl: attempt.approval.codeApiUrl, workerId: attempt.approval.workerId, + privateKey: attempt.privateKey, fetchImpl, validThroughMs: Date.now() + 60_000, + signal: options.signal ? AbortSignal.any([options.signal, AbortSignal.timeout(30_000)]) : AbortSignal.timeout(30_000), + }); + attempt.identity = { ...credential, codeApiUrl: attempt.approval.codeApiUrl, + publicKey: attempt.request.publicKey, privateKey: attempt.privateKey, setupAttemptPath: attemptPath }; + await persist(); + return await publish(); + } + const capabilities = await requestJson(instance, 'capabilities', undefined, options.signal, fetchImpl); + if (!record(capabilities) || capabilities.browserEnrollmentVersion !== 1) + throw new Error('This instance lacks compatible key-bound browser enrollment. Update compatible servers first; setup will not downgrade to manual pairing'); + let status = parseStatus(await requestJson(instance, 'start', machineBody(attempt, 'setup.start'), options.signal, fetchImpl), attempt); + await dependencies.onVerification?.(status); + for (;;) { + options.signal?.throwIfAborted(); + if (['denied', 'cancelled', 'expired'].includes(status.status)) throw new Error(`Machine setup ${status.status}. The original key and attempt were preserved; choose another profile only for an intentional new enrollment`); + if (status.approval) { attempt.approval = status.approval; await persist(); } + if (status.status === 'approved' || status.status === 'completed') { + if (!attempt.approval) throw new Error('LibreChat did not supply the key-bound approval contract'); + const completed = parseStatus(await requestJson(instance, 'complete', machineBody(attempt, 'setup.complete'), options.signal, fetchImpl), attempt); + const credential = completed.credential; + if (completed.status !== 'completed' || !credential || credential.protocolVersion !== 1 || + credential.workerId !== attempt.approval.workerId || typeof credential.credential !== 'string' || + credential.credential.length < 32 || credential.credential.length > 4096 || + !Number.isFinite(Date.parse(credential.expiresAt))) throw new Error('LibreChat returned an invalid enrollment credential'); + attempt.identity = { protocolVersion: 1, workerId: credential.workerId, credential: credential.credential, expiresAt: credential.expiresAt, + codeApiUrl: attempt.approval.codeApiUrl, publicKey: attempt.request.publicKey, privateKey: attempt.privateKey, setupAttemptPath: attemptPath }; + // Keep replayable local state even if either final file cannot be published. + await persist(); + return await publish(); + } + if (Date.now() >= Date.parse(status.expiresAt)) throw new Error('Machine setup expired; the saved key and attempt were preserved'); + await delay(dependencies.pollIntervalMs ?? 2000, undefined, { signal: options.signal }); + status = parseStatus(await requestJson(instance, 'poll', machineBody(attempt, 'setup.poll'), options.signal, fetchImpl), attempt); + } + } finally { for (const reservation of reservations.reverse()) await reservation.release(); } + }, options.signal, false); +} + +export async function cancelBrowserSetup(profilePath: string, signal?: AbortSignal, fetchImpl: typeof fetch = fetch): Promise { + const attempt = await loadBrowserSetupAttempt(profilePath); + if (!attempt) throw new Error('No saved setup attempt exists for this profile'); + if (attempt.identity) throw new Error('This machine is already enrolled; revoke it through LibreChat instead'); + await requestJson(attempt.request.instance, 'cancel', machineBody(attempt, 'setup.cancel'), signal, fetchImpl); +} diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 510d67e4..bee3697a 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -1,11 +1,13 @@ #!/usr/bin/env node import { createHash, createHmac, randomBytes } from 'node:crypto'; import { readFileSync } from 'node:fs'; -import { readdir, realpath, stat } from 'node:fs/promises'; +import { lstat, readdir, realpath, stat } from 'node:fs/promises'; import { basename, dirname, join, resolve, relative, isAbsolute, sep } from 'node:path'; import { pairBridgeWorker } from './pairing.js'; import { runSetupCommand } from './setup-cli.js'; +import { browserSetupProtectedPaths } from './browser-setup.js'; +import { assertPrivateStorageAncestors } from './private-storage.js'; import { profileRunConfiguration } from './setup.js'; import { assertWorkerProfileIsolation } from './worker-profile.js'; import type { WorkerProfile } from './worker-profile.js'; @@ -437,6 +439,7 @@ async function run( const pairedIdentity = identityPath ? await loadBridgeIdentity(identityPath) : undefined; + const setupProtectedPaths = browserSetupProtectedPaths(pairedIdentity?.setupAttemptPath); const workerId = required( 'LIBRECHAT_CODE_WORKER_ID', configuredWorkerId ?? pairedIdentity?.workerId, @@ -850,6 +853,19 @@ async function run( }), ]), ); + for (const path of setupProtectedPaths) { + const namespace = await lstat(path).catch((error: NodeJS.ErrnoException) => { + if (error.code === 'ENOENT') return undefined; + throw error; + }); + if (!namespace) continue; + if (!namespace.isDirectory() || namespace.isSymbolicLink()) + throw new Error('Setup journal namespace must be a private directory, not a symlink'); + const sourceParents = await assertPrivateStorageAncestors(path); + await assertEnvironmentDefinitionsOutsideRoots([{ + path, sourceParents, definition: { name: 'setup-state', root: path }, fingerprint: '', + }], roots); + } const preparationDirectory = join(dirname(identityPath ?? defaultBridgeIdentityPath(workerId)), 'environment-preparation'); if (environments.some(environment => environment.definition.setup?.reuse)) { const sourceParents = await prepareEnvironmentPreparationDirectory(preparationDirectory); @@ -865,13 +881,13 @@ async function run( definition: { name: 'shared-resource', root: resource.path }, fingerprint: '', }))); await assertEnvironmentResourceIsolation(environments.flatMap(environment => environment.resources ?? []), roots.map(root => root.root), [ - identityPath, preparationDirectory, github.privateKeyPath, + identityPath, ...setupProtectedPaths, preparationDirectory, github.privateKeyPath, ...environments.map(environment => environment.path), ...rootQuarantinePaths.values(), ].filter((path): path is string => path != null)); await assertEnvironmentDefinitionsOutsideRoots(resourceRoots, roots); const snapshotStores = environments.flatMap(environment => environment.snapshotStore ? [environment.snapshotStore] : []); await assertEnvironmentResourceIsolation(snapshotStores.map(store => ({ kind: 'npm-cache' as const, path: store.store, access: 'read-only' as const })), roots.map(root => root.root), [ - identityPath, preparationDirectory, github.privateKeyPath, ...environments.map(environment => environment.path), + identityPath, ...setupProtectedPaths, preparationDirectory, github.privateKeyPath, ...environments.map(environment => environment.path), ...rootQuarantinePaths.values(), ...environments.flatMap(environment => (environment.resources ?? []).map(resource => resource.path)), ].filter((path): path is string => path != null)); await assertEnvironmentDefinitionsOutsideRoots(snapshotStores.map(store => ({ path: store.store, sourceParents: store.controlPaths, @@ -1101,6 +1117,7 @@ async function run( protectedPaths: [ identityPath, savedProfile?.path, + ...setupProtectedPaths, ...(environments.some(environment => environment.definition.setup?.reuse) ? [preparationDirectory] : []), ...snapshotStores.map(store => store.store), ...environments.map(environment => environment.path), @@ -1491,8 +1508,14 @@ async function run( throw error; } worktreeRetirementScheduler?.start(); + let readinessReported = false; try { const worker = new BridgeWorker({ + onReady: () => { + if (readinessReported) return; + readinessReported = true; + process.stdout.write('librechat-code: connected and ready\n'); + }, instructionDescriptors: () => localWorkspaceTools?.instructionDescriptors() ?? Promise.resolve(undefined), codeApiUrl, token: configuredToken, @@ -1778,7 +1801,7 @@ async function main(): Promise { if (args.length === 1 && (args[0] === '--help' || args[0] === 'help')) { process.stdout.write(`Usage: librechat-code - setup Guided pairing and private profile setup + setup --instance Browser-approved enrollment and private profile setup run [--profile ] Start a saved profile or existing worker configuration pair --worker-id Existing one-time pairing flow projects --root Inspect local Git projects diff --git a/packages/code/src/index.ts b/packages/code/src/index.ts index 6f94b190..d8c53f41 100644 --- a/packages/code/src/index.ts +++ b/packages/code/src/index.ts @@ -1,5 +1,6 @@ export * from './protocol.js'; export * from './identity.js'; +export * from './browser-enrollment-protocol.js'; export * from './pairing.js'; export * from './storage.js'; export * from './runtime.js'; diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index 90215156..4f0a7094 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -32,6 +32,7 @@ import { NativeSrtWorkspaceCommandSandbox, } from './native-sandbox.js'; import { restoreScratchTraversal } from './native-scratch.js'; +import { browserSetupAttemptPath, browserSetupProtectedPaths } from './browser-setup.js'; import { WorkspaceToolError } from './workspace.js'; import { isWorkspaceToolResult } from './protocol.js'; @@ -1309,6 +1310,30 @@ test('fails closed when SRT dependencies are unavailable', async t => { ); }); +test('protects retained browser setup keys outside home from command reads and writes', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-enrollment-protection-')); + t.after(() => rm(root, { recursive: true, force: true })); + const workspace = join(root, 'workspace'); + const state = join(root, 'state'); + await mkdir(workspace); + await mkdir(state); + const attempt = browserSetupAttemptPath(join(state, 'profile.json')); + const paths = browserSetupProtectedPaths(attempt); + await mkdir(paths[0], { mode: 0o700 }); + for (const path of [attempt, `${attempt}.lock`, `${attempt}.0123456789abcdef.tmp`]) + await writeFile(path, 'private setup state', { mode: 0o600 }); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ workspaceRoot: workspace, protectedPaths: paths, manager: fake.manager }); + t.after(() => sandbox.close()); + await sandbox.prepare(); + for (const path of paths) { + const canonical = await realpath(path); + assert.ok(fake.config?.credentials?.files?.some(file => file.path === canonical && file.mode === 'deny'), + 'setup keys must be denied through the sandbox credential-file policy'); + assert.ok(fake.config?.filesystem.denyWrite.includes(canonical), 'commands must not replace setup state'); + } +}); + test('refuses workspace roots that expose worker home or control files', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); const controlDirectory = join(root, '.control'); diff --git a/packages/code/src/pairing.ts b/packages/code/src/pairing.ts index d1504e82..cef571eb 100644 --- a/packages/code/src/pairing.ts +++ b/packages/code/src/pairing.ts @@ -16,6 +16,8 @@ export interface PairedBridgeWorkerIdentity codeApiUrl: string; publicKey: string; privateKey: string; + /** Private setup journal to protect until completion removes its secret copies. */ + setupAttemptPath?: string; } function normalizedBaseUrl(value: string): string { diff --git a/packages/code/src/setup-cli.ts b/packages/code/src/setup-cli.ts index 877a5c17..08262153 100644 --- a/packages/code/src/setup-cli.ts +++ b/packages/code/src/setup-cli.ts @@ -9,15 +9,30 @@ import { } from './setup.js'; import { defaultWorkerProfilePath } from './worker-profile.js'; import { isValidBridgeWorkerId } from './protocol.js'; +import { runBrowserSetupCommand } from './browser-setup-cli.js'; export const SETUP_HELP = `Usage: librechat-code setup [options] +Enroll with browser approval: + librechat-code setup --instance https://chat.example.com + +The browser uses LibreChat's normal login, including configured OAuth/OIDC. +SSH/headless machines display a URL and matching verification code. No inbound +listener or CLI identity-provider integration is needed. Interrupted setup +resumes the same saved key and attempt. Never delete it to retry an enrollment. + +Existing manual pairing: Create a private worker profile using a one-time pairing from LibreChat. Open Settings > Code Environments in LibreChat to add the machine first. Use its Code API URL, worker ID, and pairing code. This command does not sign in to LibreChat or bypass its enrollment policy. - --url Code API URL, including its /v1 prefix + --instance LibreChat instance URL for browser approval + --machine-name Display name (default: host name) + --no-browser Display approval instructions without opening a browser + --start Start the saved worker in the foreground after setup + --cancel Cancel this profile's pending browser setup + --url Manual Code API URL, including its /v1 prefix --worker-id Worker ID supplied by LibreChat --worker-dir Expose one existing project directory --default-workspace Create a private workspace when the worker starts @@ -35,6 +50,8 @@ start a background service, or change the restricted command policy. `; const VALUE_FLAGS = new Set([ + '--instance', + '--machine-name', '--url', '--worker-id', '--worker-dir', @@ -42,6 +59,9 @@ const VALUE_FLAGS = new Set([ '--identity', ]); const BOOLEAN_FLAGS = new Set([ + '--no-browser', + '--start', + '--cancel', '--default-workspace', '--allow-workspace-writes', '--allow-workspace-commands', @@ -86,8 +106,15 @@ export function parseSetupArguments( if (options.has('--worker-dir') && options.has('--default-workspace')) { throw new Error('Choose --worker-dir or --default-workspace, not both'); } + if (options.has('--instance') && ['--url', '--worker-id', '--code-stdin'].some(flag => options.has(flag))) + throw new Error('Browser approval cannot be combined with manual pairing options'); + if (!options.has('--instance') && ['--machine-name', '--no-browser', '--start', '--cancel'].some(flag => options.has(flag))) + throw new Error('Browser setup options require --instance'); + if (options.has('--non-interactive') && options.has('--instance') && !options.has('--cancel') && + !options.has('--worker-dir') && !options.has('--default-workspace')) + throw new Error('Non-interactive browser setup requires --worker-dir or --default-workspace'); if ( - options.has('--non-interactive') && + options.has('--non-interactive') && !options.has('--instance') && (!options.has('--url') || !options.has('--worker-id') || !options.has('--code-stdin') || @@ -177,6 +204,10 @@ export async function runSetupCommand(args: string[]): Promise { } }; try { + if (options.has('--instance')) { + await runBrowserSetupCommand(options, { interactive, ask, yesNo, signal: controller.signal }); + return; + } if (interactive) process.stdout.write( 'Connect a machine using the pairing supplied by LibreChat.\nOnly selected folders and permissions will be exposed.\n\n', diff --git a/packages/code/src/storage.ts b/packages/code/src/storage.ts index b7fb094d..578f9fb6 100644 --- a/packages/code/src/storage.ts +++ b/packages/code/src/storage.ts @@ -11,7 +11,7 @@ import { } from 'node:fs/promises'; import type { FileHandle } from 'node:fs/promises'; import { homedir } from 'node:os'; -import { dirname, join, resolve } from 'node:path'; +import { dirname, isAbsolute, join, resolve } from 'node:path'; import { BRIDGE_PROTOCOL_VERSION, BridgeProtocolError } from './protocol.js'; import { assertPrivateStorageAcl, removePrivateStorageAcl, assertPrivateStorageAncestors, assertPrivateStorageSupported } from './private-storage.js'; @@ -34,7 +34,10 @@ function isPairedIdentity(value: unknown): value is PairedBridgeWorkerIdentity { typeof value.expiresAt === 'string' && Number.isFinite(Date.parse(value.expiresAt)) && typeof value.publicKey === 'string' && - typeof value.privateKey === 'string' + typeof value.privateKey === 'string' && + (value.setupAttemptPath === undefined || + (typeof value.setupAttemptPath === 'string' && isAbsolute(value.setupAttemptPath) && + Buffer.byteLength(value.setupAttemptPath) <= 4096 && !/[\x00-\x1f\x7f]/.test(value.setupAttemptPath))) ); } diff --git a/packages/code/src/worker.test.ts b/packages/code/src/worker.test.ts index e26f212a..b555c1cd 100644 --- a/packages/code/src/worker.test.ts +++ b/packages/code/src/worker.test.ts @@ -75,11 +75,12 @@ test('worker confirms readiness only after local registration activation succeed onRegistered: async () => { events.push('activated'); }, + onReady: () => { events.push('observed-ready'); }, }); await worker.register(); - assert.deepEqual(events, ['registered', 'activated', 'ready']); + assert.deepEqual(events, ['registered', 'activated', 'ready', 'observed-ready']); }); test('worker does not confirm readiness when local activation fails', async () => { @@ -110,12 +111,29 @@ test('worker does not confirm readiness when local activation fails', async () = onRegistered: async () => { throw new Error('relay activation failed'); }, + onReady: () => { assert.fail('failed activation is not readiness'); }, }); await assert.rejects(worker.register(), /relay activation failed/); assert.equal(readinessRequests, 0); }); +test('worker never reports ready for an absent or negative readiness acknowledgement', async () => { + for (const ready of [undefined, false]) { + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: false, sandboxProfile: 'nsjail', runtimes: ['bash'], requiresReadyConfirmation: true }, + fetchImpl: async input => Response.json(String(input).endsWith('/ready') + ? { protocolVersion: 1, ready } + : { protocolVersion: 1, workerId: 'vm-1', incarnationId, registrationGeneration: 1, + registeredAt: new Date().toISOString(), leaseTtlMs: 60_000 }), + onReady: () => { assert.fail('readiness was not confirmed'); }, + }); + await assert.rejects(worker.register(), /did not confirm worker readiness/); + } +}); + test('worker forwards a fenced assignment to the sandbox and settles the result', async () => { const requests: Array<{ url: string; init?: RequestInit }> = []; const fetchImpl: typeof fetch = async (input, init) => { diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 6b09f257..8978d4cd 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -98,6 +98,8 @@ export interface BridgeWorkerOptions { onRegistered?: ( registration: BridgeWorkerRegistrationResponse, ) => void | Promise; + /** Called only after registration and required readiness confirmation succeed. */ + onReady?: () => void | Promise; incarnationId?: string; } @@ -840,6 +842,7 @@ export class BridgeWorker { await this.confirmReady(registration, signal); } this.lastRegisteredAtMs = registrationStartedAtMs; + if (!this.maintenanceOnly) await this.options.onReady?.(); return registration; } @@ -856,7 +859,7 @@ export class BridgeWorker { 'Code API does not support explicit worker readiness confirmation', ); } - await this.timedRequest( + const ready = await this.timedRequest<{ ready?: boolean }>( `${this.codeApiUrl}${bridgeWorkerPath(this.options.workerId)}/ready`, { protocolVersion: BRIDGE_PROTOCOL_VERSION, @@ -870,6 +873,9 @@ export class BridgeWorker { ), signal, ); + if (ready.ready !== true) { + throw new BridgeProtocolError('Code API did not confirm worker readiness'); + } } async resetWorkspace( diff --git a/service/rollup.config.js b/service/rollup.config.js index 059c71d1..e2024c6f 100644 --- a/service/rollup.config.js +++ b/service/rollup.config.js @@ -43,6 +43,7 @@ export default { '../shared/telemetry-core.ts', '../packages/code/src/protocol.ts', '../packages/code/src/identity.ts', + '../packages/code/src/browser-enrollment-protocol.ts', ], sourceMap: true, declaration: false, diff --git a/service/src/bridge/browser-enrollment-router.test.ts b/service/src/bridge/browser-enrollment-router.test.ts new file mode 100644 index 00000000..f0418d24 --- /dev/null +++ b/service/src/bridge/browser-enrollment-router.test.ts @@ -0,0 +1,184 @@ +import { randomBytes } from 'crypto'; +import { createServer, type Server } from 'http'; + +import { afterAll, afterEach, describe, expect, test } from 'bun:test'; +import express, { json } from 'express'; +import Redis from 'ioredis'; +import RedisMock from 'ioredis-mock'; + +import { + browserSetupRequestHash, + signEnrollmentRedemption, +} from '../../../packages/code/src/browser-enrollment-protocol'; +import type { BrowserEnrollmentApproval, BrowserEnrollmentRedemption } from '../../../packages/code/src/browser-enrollment-protocol'; +import { createBridgeIdentity } from '../../../packages/code/src/identity'; +import { RedisBridgePairingStore } from './pairing'; +import type { BridgeRecoveryOptions } from './pairing'; +import { createBridgeRouter } from './router'; +import type { BridgeRouterOptions } from './router'; +import { RedisBridgeStore } from './store'; + +const redis = process.env.BROWSER_ENROLLMENT_REDIS_PORT + ? new Redis({ host: '127.0.0.1', port: Number(process.env.BROWSER_ENROLLMENT_REDIS_PORT) }) + : new RedisMock() as unknown as Redis; +const serverId = 'https://code.example.test'; +const adminToken = 'browser-enrollment-test-admin'; +const identity = createBridgeIdentity(); +const outsider = createBridgeIdentity(); +const servers: Server[] = []; + +function pairings(options: Partial = {}): RedisBridgePairingStore { + return new RedisBridgePairingStore(redis, 600, 900, 5_000, '', { serverId, ...options }); +} + +async function router(options: Partial = {}): Promise { + const app = express(); + app.set('trust proxy', 1); + app.use(json()); + app.use('/bridge', createBridgeRouter({ + store: new RedisBridgeStore(redis), pairings: pairings(), authMode: 'paired', adminToken, + allowDynamicWorkers: true, configuredWorkerId: 'browser-http-worker', ...options, + })); + const server = createServer(app); + servers.push(server); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Expected TCP listener'); + return `http://127.0.0.1:${address.port}/bridge`; +} + +function approved(): BrowserEnrollmentApproval { + const request = { + version: 1 as const, instance: 'https://chat.example.test', attemptId: randomBytes(32).toString('base64url'), + createdAt: new Date().toISOString(), publicKey: identity.publicKey, machineName: 'HTTP worker', + workspace: { type: 'private' as const }, allowWorkspaceWrites: false, allowWorkspaceCommands: false, + }; + return { + protocolVersion: 1, operation: 'enrollment.redeem', serverId, instance: request.instance, + attemptId: request.attemptId, requestHash: browserSetupRequestHash(request), workerId: 'browser-http-worker', + expiresAt: new Date(Date.now() + 60_000).toISOString(), publicKey: identity.publicKey, + binding: { tenantId: 'tenant-one', principal: { type: 'user', id: 'owner-one' } }, + }; +} + +function proof(approval: BrowserEnrollmentApproval, privateKey = identity.privateKey): BrowserEnrollmentRedemption { + const { publicKey: _publicKey, binding: _binding, ...contract } = approval; + return { ...contract, signature: signEnrollmentRedemption(privateKey, contract) }; +} + +function post(url: string, body: object, token = adminToken, headers: Record = {}): Promise { + return fetch(url, { + method: 'POST', headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${token}`, ...headers }, + body: JSON.stringify(body), + }); +} + +async function capability(url: string, token = adminToken): Promise { + return fetch(`${url}/enrollments/capabilities`, { headers: { Authorization: `Bearer ${token}` } }); +} + +afterEach(async () => { + await Promise.all(servers.splice(0).map(server => new Promise((resolve, reject) => { + server.close(error => error ? reject(error) : resolve()); + server.closeAllConnections(); + }))); + await redis.flushall(); +}); +afterAll(() => { redis.disconnect(); }); + +describe('browser enrollment HTTP boundary', () => { + test('advertises the version only for paired auth, durable recovery, dynamic workers and an authenticated admin', async () => { + const url = await router(); + const response = await capability(url); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ browserEnrollmentVersion: 1, serverId }); + expect((await capability(url, 'outsider')).status).toBe(401); + for (const options of [ + { enabled: false }, { authMode: 'static' as const }, { allowDynamicWorkers: false }, + { pairings: new RedisBridgePairingStore(redis) }, + ]) { + const disabled = await router(options); + expect((await capability(disabled)).status).toBe(404); + expect((await post(`${disabled}/enrollments`, approved())).status).toBe(404); + expect((await post(`${disabled}/enrollments/redeem`, proof(approved()))).status).toBe(404); + } + const unconfigured = await router({ adminToken: '' }); + expect((await capability(unconfigured)).status).toBe(503); + expect((await post(`${unconfigured}/enrollments`, approved())).status).toBe(503); + }); + + test('preserves old manual pairing when browser enrollment is disabled', async () => { + const url = await router({ allowDynamicWorkers: false, pairings: new RedisBridgePairingStore(redis) }); + const issue = await post(`${url}/pairings`, { workerId: 'browser-http-worker' }); + expect(issue.status).toBe(200); + const pairing = await issue.json() as { code: string }; + const result = await post(`${url}/pairings/redeem`, { + protocolVersion: 1, workerId: 'browser-http-worker', code: pairing.code, publicKey: identity.publicKey, + }, ''); + expect(result.status).toBe(200); + expect(await result.json()).toMatchObject({ protocolVersion: 1, workerId: 'browser-http-worker' }); + }); + + test('requires admin auth for approval, redemption and reconciliation, independently of machine proof', async () => { + const url = await router(); + const approval = approved(); + const redemption = proof(approval); + const { signature: _signature, ...statusRequest } = redemption; + for (const [path, body] of [ + ['enrollments', approval], ['enrollments/redeem', redemption], ['enrollments/status', statusRequest], + ] as const) expect((await post(`${url}/${path}`, body, 'outsider')).status).toBe(401); + expect(await redis.keys('codeapi:bridge:v1:browser-enrollment:*')).toHaveLength(0); + }); + + test('validates envelopes, binds approvals, and idempotently returns the same credential across HTTP replicas', async () => { + const first = await router(); + const second = await router(); + const approval = approved(); + for (const body of [ + { ...approval, protocolVersion: 2 }, { ...approval, publicKey: outsider.privateKey }, + { ...approval, binding: null }, { ...approval, extra: true }, + ]) expect((await post(`${first}/enrollments`, body)).status).toBe(400); + const issued = await post(`${first}/enrollments`, approval); + expect(issued.status).toBe(200); + expect(await issued.json()).toEqual(approval); + expect((await post(`${second}/enrollments`, approval)).status).toBe(200); + const { signature: _pendingSignature, ...pendingRequest } = proof(approval); + const pending = await post(`${second}/enrollments/status`, pendingRequest); + expect(pending.status).toBe(200); + expect(await pending.json()).toEqual({ protocolVersion: 1, status: 'pending', workerId: approval.workerId }); + expect((await post(`${second}/enrollments`, { ...approval, publicKey: outsider.publicKey })).status).toBe(409); + expect((await post(`${first}/enrollments/redeem`, { ...proof(approval), signature: '?' })).status).toBe(400); + expect((await post(`${first}/enrollments/redeem`, { ...proof(approval), operation: 'setup.complete' })).status).toBe(400); + expect((await post(`${first}/enrollments/redeem`, proof(approval, outsider.privateKey))).status).toBe(401); + expect((await post(`${first}/enrollments/redeem`, proof({ ...approval, serverId: 'https://other.example.test' }))).status).toBe(401); + expect((await post(`${first}/enrollments/redeem`, proof({ ...approval, instance: 'https://other.example.test' }))).status).toBe(409); + const responses = await Promise.all([first, second].map(url => post(`${url}/enrollments/redeem`, proof(approval)))); + expect(responses.map(response => response.status)).toEqual([200, 200]); + const [original, replay] = await Promise.all(responses.map(response => response.json())); + expect(original).toEqual(replay); + expect(original).toMatchObject({ protocolVersion: 1, workerId: approval.workerId }); + expect(JSON.stringify(original)).not.toMatch(/tenant-one|owner-one|publicKey|privateKey|binding/); + const { signature: _signature, ...statusRequest } = proof(approval); + expect((await post(`${second}/enrollments/status`, statusRequest)).status).toBe(200); + expect((await post(`${second}/enrollments/status`, { ...statusRequest, requestHash: 'f'.repeat(64) })).status).toBe(409); + expect((await post(`${first}/workers/${approval.workerId}/revoke`, {})).status).toBe(200); + expect((await post(`${second}/enrollments/redeem`, proof(approval))).status).toBe(409); + const revoked = await post(`${second}/enrollments/status`, statusRequest); + expect(revoked.status).toBe(200); + expect(await revoked.json()).toEqual({ protocolVersion: 1, status: 'revoked', workerId: approval.workerId }); + }); + + test('shares a connection-peer budget across replicas and ignores spoofed forwarded IPs', async () => { + const first = await router({ pairings: pairings({ maxUntrustedRequestsPerMinute: 2 }) }); + const second = await router({ pairings: pairings({ maxUntrustedRequestsPerMinute: 2 }) }); + const approval = approved(); + await post(`${first}/enrollments`, approval); + const forged = proof(approval, outsider.privateKey); + expect((await post(`${first}/enrollments/redeem`, forged, adminToken, { 'X-Forwarded-For': '198.51.100.1' })).status).toBe(401); + expect((await post(`${second}/enrollments/redeem`, forged, adminToken, { 'X-Forwarded-For': '198.51.100.2' })).status).toBe(401); + const blocked = await post(`${second}/enrollments/redeem`, proof(approved()), adminToken, { 'X-Forwarded-For': '198.51.100.3' }); + expect(blocked.status).toBe(429); + expect(blocked.headers.get('retry-after')).toBe('60'); + expect(await redis.keys('codeapi:bridge:v1:credential:*')).toHaveLength(0); + }); +}); diff --git a/service/src/bridge/browser-enrollment.test.ts b/service/src/bridge/browser-enrollment.test.ts new file mode 100644 index 00000000..ca2b6844 --- /dev/null +++ b/service/src/bridge/browser-enrollment.test.ts @@ -0,0 +1,447 @@ +import { createHash, randomBytes } from 'crypto'; + +import { afterAll, afterEach, describe, expect, test } from 'bun:test'; +import Redis from 'ioredis'; +import RedisMock from 'ioredis-mock'; + +import { + browserSetupRequestHash, + canonicalBrowserSetupProof, + canonicalBrowserSetupRequest, + canonicalEnrollmentRedemption, + isBrowserEnrollmentApproval, + isBrowserEnrollmentRedemption, + normalizeBrowserEnrollmentInstance, + signBrowserSetupProof, + signEnrollmentRedemption, + validateBrowserSetupProof, + validateBrowserSetupRequest, + verifyBrowserSetupProof, + verifyEnrollmentRedemption, +} from '../../../packages/code/src/browser-enrollment-protocol'; +import type { + BrowserEnrollmentApproval, + BrowserEnrollmentRedemption, + BrowserSetupProof, + BrowserSetupRequest, +} from '../../../packages/code/src/browser-enrollment-protocol'; +import { + createBridgeIdentity, + signBridgeRecovery, + signBridgeRecoveryStart, + signBridgeRequest, +} from '../../../packages/code/src/identity'; +import { RedisBridgePairingStore } from './pairing'; +import type { BridgeRecoveryOptions, BridgeWorkerCredential } from './pairing'; +import { RedisBridgeStore } from './store'; + +const redis = process.env.BROWSER_ENROLLMENT_REDIS_PORT + ? new Redis({ host: '127.0.0.1', port: Number(process.env.BROWSER_ENROLLMENT_REDIS_PORT) }) + : new RedisMock() as unknown as Redis; +const serverId = 'https://code.example.test'; +const adminToken = 'test-browser-enrollment-administrator-secret'; +const identity = createBridgeIdentity(); +const outsider = createBridgeIdentity(); +const workerId = 'browser-enrolled-worker'; + +function store(options: Partial = {}, credentialTtl = 900): RedisBridgePairingStore { + return new RedisBridgePairingStore(redis, 600, credentialTtl, 5_000, '', { serverId, ...options }); +} + +function request(): BrowserSetupRequest { + return { + version: 1, + instance: 'https://chat.example.test/base', + attemptId: randomBytes(32).toString('base64url'), + createdAt: new Date().toISOString(), + publicKey: identity.publicKey, + machineName: 'My VM (development)', + workspace: { type: 'directory' }, + allowWorkspaceWrites: true, + allowWorkspaceCommands: false, + }; +} + +function approval(overrides: Partial = {}): BrowserEnrollmentApproval { + const setup = request(); + return { + protocolVersion: 1, + operation: 'enrollment.redeem', + serverId, + instance: setup.instance, + attemptId: setup.attemptId, + requestHash: browserSetupRequestHash(setup), + workerId, + expiresAt: new Date(Date.now() + 60_000).toISOString(), + publicKey: identity.publicKey, + binding: { tenantId: 'tenant-one', principal: { type: 'user', id: 'owner-one' } }, + ...overrides, + }; +} + +function redemption(input: BrowserEnrollmentApproval, privateKey = identity.privateKey): BrowserEnrollmentRedemption { + const { operation, serverId, instance, attemptId, requestHash, workerId, expiresAt } = input; + const contract = { operation, serverId, instance, attemptId, requestHash, workerId, expiresAt }; + return { protocolVersion: 1, ...contract, signature: signEnrollmentRedemption(privateKey, contract) }; +} + +function authorized(credential: BridgeWorkerCredential): Parameters[0] { + const proof = { + credential: credential.credential, + method: 'POST', + path: `/bridge/workers/${workerId}/lease`, + timestamp: new Date().toISOString(), + nonce: randomBytes(32).toString('base64url'), + body: '{}', + }; + return { ...proof, workerId, signature: signBridgeRequest(identity.privateKey, proof) }; +} + +afterEach(async () => { await redis.flushall(); }); +afterAll(() => { redis.disconnect(); }); + +describe('browser enrollment proof contract', () => { + test('canonicalizes fixed request fields and preserves every authorization choice', () => { + const input = request(); + expect(normalizeBrowserEnrollmentInstance('https://CHAT.example.test:443/base///')).toBe(input.instance); + expect(() => normalizeBrowserEnrollmentInstance('http://untrusted.example.test')).toThrow(); + expect(() => normalizeBrowserEnrollmentInstance('https://user:password@chat.example.test')).toThrow(); + expect(() => normalizeBrowserEnrollmentInstance('https://chat.example.test/?query=1')).toThrow(); + const canonical = canonicalBrowserSetupRequest(input); + expect(Object.keys(JSON.parse(canonical))).toEqual([ + 'version', 'instance', 'attemptId', 'createdAt', 'publicKey', 'machineName', 'workspace', + 'allowWorkspaceWrites', 'allowWorkspaceCommands', + ]); + expect(browserSetupRequestHash(input)).toBe(createHash('sha256').update(canonical).digest('hex')); + const reordered = Object.fromEntries(Object.entries(input).reverse()) as unknown as BrowserSetupRequest; + expect(browserSetupRequestHash(reordered)).toBe(browserSetupRequestHash(input)); + for (const change of [ + { allowWorkspaceWrites: false }, { allowWorkspaceCommands: true }, { workspace: { type: 'private' as const } }, + { machineName: 'Different VM' }, { publicKey: outsider.publicKey }, + ]) expect(browserSetupRequestHash({ ...input, ...change })).not.toBe(browserSetupRequestHash(input)); + }); + + test('rejects noncanonical requests and malformed proofs without throwing from verification', () => { + const input = request(); + for (const malformed of [ + null, [], { ...input, extra: true }, { ...input, version: 2 }, { ...input, machineName: 'x\ny' }, + { ...input, machineName: 'x'.repeat(81) }, { ...input, machineName: ' leading' }, + { ...input, instance: `${input.instance}/` }, { ...input, attemptId: 'x'.repeat(43) }, + { ...input, createdAt: '2026-10-21' }, { ...input, publicKey: outsider.privateKey }, + { ...input, workspace: { type: 'directory', path: '/secret' } }, { ...input, allowWorkspaceWrites: 1 }, + ]) expect(validateBrowserSetupRequest(malformed)).toBe(false); + expect(verifyBrowserSetupProof('invalid key', {} as BrowserSetupProof, '!')).toBe(false); + }); + + test('binds operation, audience, attempt, request hash, time and nonce, with separate signing domains', () => { + const input = request(); + const proof: BrowserSetupProof = { + operation: 'setup.start', instance: input.instance, attemptId: input.attemptId, + requestHash: browserSetupRequestHash(input), timestamp: input.createdAt, + nonce: randomBytes(32).toString('base64url'), + }; + const signature = signBrowserSetupProof(identity.privateKey, proof); + expect(canonicalBrowserSetupProof(proof).split('\n')).toEqual([ + 'librechat-code:browser-setup:v1', proof.operation, proof.instance, proof.attemptId, + proof.requestHash, proof.timestamp, proof.nonce, + ]); + expect(validateBrowserSetupProof(proof)).toBe(true); + expect(verifyBrowserSetupProof(identity.publicKey, proof, signature)).toBe(true); + expect(verifyBrowserSetupProof(outsider.publicKey, proof, signature)).toBe(false); + for (const change of [ + { operation: 'setup.poll' as const }, { operation: 'setup.cancel' as const }, { operation: 'setup.complete' as const }, + { instance: 'https://different.example.test' }, { attemptId: randomBytes(32).toString('base64url') }, + { requestHash: 'f'.repeat(64) }, { timestamp: new Date(Date.now() + 1_000).toISOString() }, + { nonce: randomBytes(32).toString('base64url') }, + ]) expect(verifyBrowserSetupProof(identity.publicKey, { ...proof, ...change }, signature)).toBe(false); + const approved = approval({ instance: proof.instance, attemptId: proof.attemptId, requestHash: proof.requestHash }); + expect(canonicalEnrollmentRedemption(approved).split('\n')).toEqual([ + 'librechat-code:browser-enrollment-redeem:v1', approved.operation, approved.serverId, + approved.instance, approved.attemptId, approved.requestHash, approved.workerId, approved.expiresAt, + ]); + expect(verifyEnrollmentRedemption(identity.publicKey, approved, signature)).toBe(false); + expect(verifyBrowserSetupProof(identity.publicKey, proof, signEnrollmentRedemption(identity.privateKey, approved))).toBe(false); + expect(isBrowserEnrollmentRedemption({ ...redemption(approved), publicKey: identity.publicKey })).toBe(false); + expect(isBrowserEnrollmentApproval({ ...approved, binding: { tenantId: 'x', principal: { type: 'bad', id: 'x' } } })).toBe(false); + }); +}); + +describe('Redis browser enrollment ownership', () => { + test('retries lost issue and redemption responses across replicas without rotating keys or widening access', async () => { + const first = store(); + const approved = approval(); + await first.issueBrowserEnrollment(approved); + const second = store(); + expect(await second.issueBrowserEnrollment(approved)).toEqual(approved); + const original = await first.redeemBrowserEnrollment(redemption(approved), adminToken); + expect(await second.issueBrowserEnrollment(approved)).toEqual(approved); + expect(await second.redeemBrowserEnrollment(redemption(approved), adminToken)).toEqual(original); + const auth = await second.authorize(authorized(original)); + expect(auth).toMatchObject({ workerId, binding: approved.binding, pairingGeneration: 0 }); + const enrollmentRaw = await redis.get(`codeapi:bridge:v1:enrollment:${workerId}`); + const allValues = await redis.mget(...await redis.keys('codeapi:bridge:v1:*')); + expect(allValues.join('\n')).not.toContain(original.credential); + expect(enrollmentRaw).toContain(identity.publicKey.trim().split('\n')[1]); + await expect(second.redeemBrowserEnrollment(redemption(approved), 'wrong-encryption-key')).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + expect(await redis.get(`codeapi:bridge:v1:enrollment:${workerId}`)).toBe(enrollmentRaw); + }); + + test('concurrent duplicates commit exactly one enrollment and return the identical response', async () => { + const approved = approval(); + const replicas = [store(), store(), store()]; + const issued = await Promise.all(replicas.map(replica => replica.issueBrowserEnrollment(approved))); + expect(issued).toEqual([approved, approved, approved]); + const completed = await Promise.all(replicas.map(replica => replica.redeemBrowserEnrollment(redemption(approved), adminToken))); + expect(completed[1]).toEqual(completed[0]); + expect(completed[2]).toEqual(completed[0]); + expect(await redis.keys('codeapi:bridge:v1:credential:*')).toHaveLength(1); + expect(await redis.keys('codeapi:bridge:v1:enrollment:*')).toHaveLength(1); + }); + + test('does not depend on binding property order and retries after worker registration', async () => { + const replica = store(); + const approved = approval({ binding: { principal: { id: 'owner-one', type: 'user' }, tenantId: 'tenant-one' } }); + await replica.issueBrowserEnrollment(approved); + const credential = await replica.redeemBrowserEnrollment(redemption(approved), adminToken); + const auth = await replica.authorize(authorized(credential)); + const workers = new RedisBridgeStore(redis); + await workers.register({ + protocolVersion: 1, workerId, incarnationId: 'browser-test-incarnation', + identityId: auth.identityId, credentialId: auth.credentialId, binding: approved.binding, + capabilities: { statefulWorkspace: false, runtimes: [], sandboxProfile: 'native-srt' }, + }, auth); + expect(await replica.issueBrowserEnrollment(approved)).toEqual(approved); + expect(await replica.redeemBrowserEnrollment(redemption(approved), adminToken)).toEqual(credential); + }); + + test('conflicting concurrent approvals cannot replace the winner or allocate a second worker', async () => { + const approved = approval(); + const changed = { ...approved, workerId: 'other-worker', publicKey: outsider.publicKey }; + const results = await Promise.allSettled([store().issueBrowserEnrollment(approved), store().issueBrowserEnrollment(changed)]); + expect(results.filter(result => result.status === 'fulfilled')).toHaveLength(1); + expect(results.filter(result => result.status === 'rejected')).toHaveLength(1); + const winner = results.find(result => result.status === 'fulfilled'); + if (winner?.status !== 'fulfilled') throw new Error('Expected one committed approval'); + const key = winner.value.publicKey === identity.publicKey ? identity.privateKey : outsider.privateKey; + await store().redeemBrowserEnrollment(redemption(winner.value, key), adminToken); + expect(await redis.keys('codeapi:bridge:v1:enrollment:*')).toHaveLength(1); + }); + + test('bounded credential replay cannot remint after its TTL but durable status remains available', async () => { + const replica = store({}, 1); + const approved = approval(); + await replica.issueBrowserEnrollment(approved); + await replica.redeemBrowserEnrollment(redemption(approved), adminToken); + await new Promise(resolve => setTimeout(resolve, 1_100)); + await expect(store({}, 1).redeemBrowserEnrollment(redemption(approved), adminToken)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + const { signature: _signature, ...statusRequest } = redemption(approved); + expect(await store().browserEnrollmentStatus(statusRequest)).toMatchObject({ status: 'completed', workerId }); + expect(await redis.keys('codeapi:bridge:v1:credential:*')).toHaveLength(0); + }); + + test('rejects replacement of immutable approval fields and reuse of the allocated worker', async () => { + const replica = store(); + const approved = approval(); + await replica.issueBrowserEnrollment(approved); + for (const change of [ + { workerId: 'another-worker' }, { publicKey: outsider.publicKey }, { requestHash: 'f'.repeat(64) }, + { expiresAt: new Date(Date.now() + 120_000).toISOString() }, + { binding: { tenantId: 'tenant-two', principal: { type: 'deployment' as const, id: 'everyone' } } }, + ]) await expect(replica.issueBrowserEnrollment({ ...approved, ...change })).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(replica.issueBrowserEnrollment(approval())).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + const first = await replica.redeemBrowserEnrollment(redemption(approved), adminToken); + await expect(replica.issueBrowserEnrollment(approval())).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + expect(await replica.authorize(authorized(first))).toMatchObject({ binding: approved.binding }); + }); + + test('rejects wrong key, audience, operation and every contract mismatch', async () => { + const replica = store(); + const approved = approval(); + await replica.issueBrowserEnrollment(approved); + await expect(replica.redeemBrowserEnrollment(redemption(approved, outsider.privateKey), adminToken)).rejects.toMatchObject({ code: 'PROOF_INVALID' }); + for (const change of [ + { serverId: 'https://other-code.example.test' }, { instance: 'https://other-chat.example.test' }, + { attemptId: randomBytes(32).toString('base64url') }, { requestHash: 'f'.repeat(64) }, + { workerId: 'another-worker' }, { expiresAt: new Date(Date.now() + 120_000).toISOString() }, + ]) await expect(replica.redeemBrowserEnrollment(redemption({ ...approved, ...change }), adminToken)).rejects.toBeDefined(); + await expect(replica.redeemBrowserEnrollment({ ...redemption(approved), operation: 'setup.complete' } as unknown as BrowserEnrollmentRedemption, adminToken)).rejects.toMatchObject({ code: 'PROOF_INVALID' }); + await expect(replica.redeemBrowserEnrollment({ ...redemption(approved), signature: '?' }, adminToken)).rejects.toMatchObject({ code: 'PROOF_INVALID' }); + expect(await redis.keys('codeapi:bridge:v1:credential:*')).toHaveLength(0); + await replica.redeemBrowserEnrollment(redemption(approved), adminToken); + }); + + test('never extends the immutable expiry or recreates an expired approval', async () => { + const replica = store(); + for (const expiresAt of [new Date(Date.now() - 1).toISOString(), new Date(Date.now() + 601_000).toISOString()]) { + await expect(replica.issueBrowserEnrollment(approval({ expiresAt }))).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + } + const approved = approval({ expiresAt: new Date(Date.now() + 2_000).toISOString() }); + await replica.issueBrowserEnrollment(approved); + await new Promise(resolve => setTimeout(resolve, 2_100)); + await expect(replica.issueBrowserEnrollment(approved)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(replica.redeemBrowserEnrollment(redemption(approved), adminToken)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + const { signature: _signature, ...statusRequest } = redemption(approved); + expect(await replica.browserEnrollmentStatus(statusRequest)).toMatchObject({ status: 'revoked' }); + await expect(replica.issueBrowserEnrollment({ ...approved, workerId: 'replacement-worker', expiresAt: new Date(Date.now() + 60_000).toISOString() })).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + }); + + test('durable enrollment outlives setup expiry and recovers after restart and access credential loss', async () => { + const replica = store(); + const approved = approval({ expiresAt: new Date(Date.now() + 2_000).toISOString() }); + await replica.issueBrowserEnrollment(approved); + const original = await replica.redeemBrowserEnrollment(redemption(approved), adminToken); + const originalAuth = await replica.authorize(authorized(original)); + await new Promise(resolve => setTimeout(resolve, 2_100)); + expect(await store().redeemBrowserEnrollment(redemption(approved), adminToken)).toEqual(original); + await redis.del( + `codeapi:bridge:v1:identity:${workerId}`, `codeapi:bridge:v1:stable-identity:${workerId}`, + `codeapi:bridge:v1:credential:${createHash('sha256').update(original.credential).digest('hex')}`, + ); + const restarted = store(); + const { signature: _signature, ...statusRequest } = redemption(approved); + expect(await restarted.browserEnrollmentStatus(statusRequest)).toEqual({ protocolVersion: 1, status: 'completed', workerId }); + await expect(restarted.browserEnrollmentStatus({ ...statusRequest, requestHash: 'f'.repeat(64) })).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + const start = { + operation: 'credential.challenge' as const, serverId, workerId, + timestamp: new Date().toISOString(), nonce: randomBytes(32).toString('base64url'), + }; + const challenge = await restarted.createRecoveryChallenge(workerId, start, signBridgeRecoveryStart(identity.privateKey, start)); + const recovered = await restarted.recoverCredential(workerId, challenge, signBridgeRecovery(identity.privateKey, challenge)); + expect(await restarted.authorize(authorized(recovered))).toMatchObject({ identityId: originalAuth.identityId, binding: approved.binding }); + await restarted.revoke(workerId); + expect(await restarted.browserEnrollmentStatus(statusRequest)).toMatchObject({ status: 'revoked', workerId }); + await expect(restarted.redeemBrowserEnrollment(redemption(approved), adminToken)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + }); + + test('revocation fences approval, completion, exact replay and recovery without clearing quarantine', async () => { + const replica = store(); + const approved = approval(); + const quarantine = `codeapi:bridge:v1:worker:${workerId}:workspace:root:quarantined`; + await redis.set(quarantine, 'existing-fence'); + await replica.issueBrowserEnrollment(approved); + await replica.revoke(workerId); + await expect(replica.issueBrowserEnrollment(approved)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(replica.redeemBrowserEnrollment(redemption(approved), adminToken)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(replica.issueBrowserEnrollment(approval())).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + expect(await redis.get(quarantine)).toBe('existing-fence'); + expect(await redis.get(`codeapi:bridge:v1:pairing-generation:${workerId}`)).toBe('1'); + + const other = approval({ workerId: 'second-worker' }); + await replica.issueBrowserEnrollment(other); + const credential = await replica.redeemBrowserEnrollment(redemption(other), adminToken); + await replica.revoke(other.workerId); + await expect(replica.redeemBrowserEnrollment(redemption(other), adminToken)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + expect(await redis.get(`codeapi:bridge:v1:enrollment:${other.workerId}`)).toBeNull(); + expect(credential.workerId).toBe(other.workerId); + }); + + test('status distinguishes pending, completed, revoked and unknown exact contracts without issuing credentials', async () => { + const replica = store(); + const approved = approval(); + const { signature: _signature, ...statusRequest } = redemption(approved); + await expect(replica.browserEnrollmentStatus(statusRequest)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await replica.issueBrowserEnrollment(approved); + expect(await store().browserEnrollmentStatus(statusRequest)).toEqual({ protocolVersion: 1, status: 'pending', workerId }); + expect(await redis.keys('codeapi:bridge:v1:credential:*')).toHaveLength(0); + await expect(replica.browserEnrollmentStatus({ ...statusRequest, workerId: 'different-worker' })).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await replica.revoke(workerId); + expect(await store().browserEnrollmentStatus(statusRequest)).toEqual({ protocolVersion: 1, status: 'revoked', workerId }); + await expect(replica.issueBrowserEnrollment(approved)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + expect(await redis.keys('codeapi:bridge:v1:credential:*')).toHaveLength(0); + }); + + test('manual issue supersedes only the pending fence and cannot be undone by a retry', async () => { + const replica = store(); + const approved = approval(); + await replica.issueBrowserEnrollment(approved); + const manual = await replica.issue(workerId, approved.binding); + await expect(replica.issueBrowserEnrollment(approved)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(replica.redeemBrowserEnrollment(redemption(approved), adminToken)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + const issued = await replica.redeem({ workerId, code: manual.code, publicKey: outsider.publicKey }); + await expect(replica.issueBrowserEnrollment(approval())).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + expect(issued.workerId).toBe(workerId); + }); + + for (const afterScanWindow of [false, true]) test(`unindexed legacy codes cannot supersede browser approval ${afterScanWindow ? 'after' : 'during'} migration`, async () => { + const replica = store(); + const approved = approval(); + const legacyCode = randomBytes(24).toString('base64url'); + const legacyKey = `codeapi:bridge:v1:pairing:${createHash('sha256').update(legacyCode).digest('hex')}`; + await redis.set(legacyKey, JSON.stringify({ workerId, expiresAt: new Date(Date.now() + 60_000).toISOString(), + binding: { tenantId: 'other-tenant', principal: { type: 'user', id: 'other-owner' } } }), 'EX', 60); + if (afterScanWindow) await redis.set('codeapi:bridge:v1:migration:legacy-pairing-scan-until', String(Date.now() - 1)); + await replica.issueBrowserEnrollment(approved); + expect(await redis.get(`codeapi:bridge:v1:pairing-generation:${workerId}`)).toBe('0'); + if (!afterScanWindow) expect(await redis.get(legacyKey)).toBeNull(); + await expect(replica.redeem({ workerId, code: legacyCode, publicKey: outsider.publicKey })).rejects.toMatchObject({ code: 'PAIRING_INVALID' }); + const credential = await replica.redeemBrowserEnrollment(redemption(approved), adminToken); + expect(await replica.authorize(authorized(credential))).toMatchObject({ workerId, binding: approved.binding }); + }); + + test('cannot replace a preexisting manual enrollment or pending pairing', async () => { + const replica = store(); + const manual = await replica.issue(workerId); + await expect(replica.issueBrowserEnrollment(approval())).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await replica.redeem({ workerId, code: manual.code, publicKey: outsider.publicKey }); + await expect(replica.issueBrowserEnrollment(approval())).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(replica.issueBrowserEnrollment(approval({ serverId: 'https://wrong.example.test' }))).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(new RedisBridgePairingStore(redis).issueBrowserEnrollment(approval())).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + }); + + test('per-attempt budgets are shared by replicas and wrong signatures cannot create credentials', async () => { + const approved = approval(); + const first = store({ maxAttemptsPerMinute: 2 }); + const second = store({ maxAttemptsPerMinute: 2 }); + await first.issueBrowserEnrollment(approved); + const forged = redemption(approved, outsider.privateKey); + await expect(first.redeemBrowserEnrollment(forged, adminToken)).rejects.toMatchObject({ code: 'PROOF_INVALID' }); + await expect(second.redeemBrowserEnrollment(forged, adminToken)).rejects.toMatchObject({ code: 'PROOF_INVALID' }); + await expect(first.redeemBrowserEnrollment(redemption(approved), adminToken)).rejects.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + expect(await redis.keys('codeapi:bridge:v1:credential:*')).toHaveLength(0); + }); + + test('manual issue between signature verification and completion permanently supersedes the browser attempt', async () => { + const replica = store(); + const approved = approval(); + await replica.issueBrowserEnrollment(approved); + const originalEval = redis.eval.bind(redis); + let manualCode = ''; + redis.eval = (async (...args: Parameters): Promise => { + if (!manualCode && String(args[0]).includes('local replay =')) { + manualCode = (await replica.issue(workerId)).code; + } + return originalEval(...args); + }) as Redis['eval']; + try { + await expect(replica.redeemBrowserEnrollment(redemption(approved), adminToken)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + expect(manualCode.length).toBeGreaterThan(0); + await replica.redeem({ workerId, code: manualCode, publicKey: outsider.publicKey }); + await expect(replica.issueBrowserEnrollment(approved)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + } finally { + redis.eval = originalEval as Redis['eval']; + } + }); + + test('atomic generation comparison wins a revoke between proof validation and completion', async () => { + const replica = store(); + const approved = approval(); + await replica.issueBrowserEnrollment(approved); + const originalEval = redis.eval.bind(redis); + let intercepted = false; + redis.eval = (async (...args: Parameters): Promise => { + if (!intercepted && String(args[0]).includes('local replay =')) { + intercepted = true; + await replica.revoke(workerId); + } + return originalEval(...args); + }) as Redis['eval']; + try { + await expect(replica.redeemBrowserEnrollment(redemption(approved), adminToken)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + expect(intercepted).toBe(true); + expect(await redis.get(`codeapi:bridge:v1:enrollment:${workerId}`)).toBeNull(); + expect(await redis.keys('codeapi:bridge:v1:credential:*')).toHaveLength(0); + } finally { + redis.eval = originalEval as Redis['eval']; + } + }); +}); diff --git a/service/src/bridge/pairing.ts b/service/src/bridge/pairing.ts index b8a32162..c8170b69 100644 --- a/service/src/bridge/pairing.ts +++ b/service/src/bridge/pairing.ts @@ -1,4 +1,6 @@ import { + createCipheriv, + createDecipheriv, createHash, createPublicKey, randomBytes, @@ -6,6 +8,21 @@ import { import type Redis from 'ioredis'; +import { + BROWSER_ENROLLMENT_MAX_TTL_MS, + canonicalBrowserEnrollmentRedemption, + isBrowserEnrollmentApproval, + isBrowserEnrollmentRedemption, + isBrowserEnrollmentRedemptionContract, + isBrowserEnrollmentStatusRequest, + verifyBrowserEnrollmentRedemption, +} from '../../../packages/code/src/browser-enrollment-protocol'; +import type { + BrowserEnrollmentApproval, + BrowserEnrollmentRedemption, + BrowserEnrollmentRedemptionContract, + BrowserEnrollmentStatusRequest, +} from '../../../packages/code/src/browser-enrollment-protocol'; import { verifyBridgeRecovery, verifyBridgeRecoveryStart, @@ -74,6 +91,66 @@ else end return 1 `; +const ISSUE_BROWSER_ENROLLMENT_SCRIPT = ` +local now = redis.call('TIME') +local remaining = tonumber(ARGV[3]) - (tonumber(now[1]) * 1000.0 + math.floor(tonumber(now[2]) / 1000)) +if remaining <= 0 or remaining > tonumber(ARGV[4]) then return 0 end +if (redis.call('GET', KEYS[5]) or '0') ~= '0' then return 0 end +local saved = redis.call('GET', KEYS[1]) +local tombstone = redis.call('GET', KEYS[13]) +if tombstone and (not saved or tombstone ~= ARGV[5]) then return 0 end +if saved then + if saved ~= ARGV[1] or redis.call('GET', KEYS[4]) ~= KEYS[3] or redis.call('GET', KEYS[3]) ~= 'browser-enrollment' then return 0 end + local enrollment = redis.call('GET', KEYS[8]) + if enrollment then + if enrollment ~= redis.call('GET', KEYS[2]) then return 0 end + else + for i = 6, 12 do + if i ~= 8 and redis.call('EXISTS', KEYS[i]) == 1 then return 0 end + end + end + redis.call('SETNX', KEYS[5], '0') + return 1 +end +for i = 4, 12 do + if i ~= 5 and redis.call('EXISTS', KEYS[i]) == 1 then return 0 end +end +redis.call('SETNX', KEYS[5], '0') +redis.call('SET', KEYS[13], ARGV[5]) +redis.call('SET', KEYS[1], ARGV[1], 'PX', remaining) +redis.call('SET', KEYS[2], ARGV[2], 'PX', remaining) +redis.call('SET', KEYS[3], 'browser-enrollment', 'PX', remaining) +redis.call('SET', KEYS[4], KEYS[3], 'PX', remaining) +return 1 +`; +const REDEEM_BROWSER_ENROLLMENT_SCRIPT = ` +local now = redis.call('TIME') +local remaining = tonumber(ARGV[3]) - (tonumber(now[1]) * 1000.0 + math.floor(tonumber(now[2]) / 1000)) +if redis.call('GET', KEYS[1]) ~= ARGV[1] or redis.call('GET', KEYS[2]) ~= ARGV[2] or + redis.call('GET', KEYS[3]) ~= 'browser-enrollment' or redis.call('GET', KEYS[4]) ~= KEYS[3] or + (redis.call('GET', KEYS[5]) or '0') ~= '0' then return nil end +local replay = redis.call('GET', KEYS[11]) +if replay then + if redis.call('GET', KEYS[9]) ~= ARGV[2] or redis.call('GET', KEYS[10]) ~= ARGV[9] then return nil end + return replay +end +if remaining <= 0 or ARGV[11] ~= '1' then return nil end +for i = 7, 14 do + if i ~= 11 and redis.call('EXISTS', KEYS[i]) == 1 then return nil end +end +redis.call('SET', KEYS[6], ARGV[4], 'EX', ARGV[5]) +redis.call('SET', KEYS[7], ARGV[6], 'EX', ARGV[5]) +redis.call('SET', KEYS[8], ARGV[7], 'EX', ARGV[5]) +if tonumber(ARGV[8]) > 0 then + redis.call('SET', KEYS[9], ARGV[2], 'EX', ARGV[8]) +else + redis.call('SET', KEYS[9], ARGV[2]) +end +redis.call('SET', KEYS[10], ARGV[9]) +redis.call('SET', KEYS[11], ARGV[10], 'EX', ARGV[5]) +for i = 1, 4 do redis.call('EXPIRE', KEYS[i], ARGV[5]) end +return ARGV[10] +`; const ROTATE_CREDENTIAL_SCRIPT = ` if ARGV[6] ~= '' then if redis.call('GET', KEYS[5]) ~= ARGV[6] or (redis.call('GET', KEYS[6]) or '0') ~= ARGV[7] or redis.call('GET', KEYS[7]) ~= ARGV[8] then @@ -127,7 +204,7 @@ if redis.call('EXISTS', KEYS[6]) == 1 then return -2 end local count = redis.call('INCR', KEYS[2]) if count == 1 then redis.call('EXPIRE', KEYS[2], 60) end if count > tonumber(ARGV[2]) then return -1 end -if redis.call('SET', KEYS[3], ARGV[3], 'EX', ARGV[5], 'NX') ~= 'OK' then return 0 end +if not redis.call('SET', KEYS[3], ARGV[3], 'EX', ARGV[5], 'NX') then return 0 end redis.call('SET', KEYS[6], '1', 'EX', ARGV[7]) return 1 `; @@ -223,6 +300,7 @@ interface StoredEnrollment { pairingGeneration: number; publicKey: string; binding?: BridgeWorkerBinding; + browserEnrollment?: BrowserEnrollmentRedemptionContract; } export interface BridgeRecoveryOptions { @@ -277,6 +355,54 @@ function pairingKey(code: string): string { return `${PREFIX}:pairing:${digest(code)}`; } +function browserEnrollmentKey(instance: string, attemptId: string): string { + return `${PREFIX}:browser-enrollment:${digest(JSON.stringify([instance, attemptId]))}`; +} + +function canonicalApproval(approval: BrowserEnrollmentApproval): string { + return JSON.stringify({ + protocolVersion: 1, + operation: approval.operation, + serverId: approval.serverId, + instance: approval.instance, + attemptId: approval.attemptId, + requestHash: approval.requestHash, + workerId: approval.workerId, + expiresAt: approval.expiresAt, + publicKey: approval.publicKey, + binding: { + tenantId: approval.binding.tenantId, + principal: { type: approval.binding.principal.type, id: approval.binding.principal.id }, + }, + }); +} + +function enrollmentReplayKey(adminToken: string, serverId: string): Buffer { + if (!adminToken) throw new BridgePairingError('ENROLLMENT_INVALID', 'Browser enrollment requires an administrator token'); + return createHash('sha256').update('librechat-code:browser-enrollment-replay:v1\0') + .update(serverId).update('\0').update(adminToken).digest(); +} + +function encryptEnrollmentReplay(credential: BridgeWorkerCredential, approval: string, key: Buffer): string { + const iv = randomBytes(12); + const cipher = createCipheriv('aes-256-gcm', key, iv); + cipher.setAAD(Buffer.from(approval)); + const ciphertext = Buffer.concat([cipher.update(JSON.stringify(credential), 'utf8'), cipher.final()]); + return Buffer.concat([iv, cipher.getAuthTag(), ciphertext]).toString('base64url'); +} + +function decryptEnrollmentReplay(ciphertext: string, approval: string, key: Buffer): BridgeWorkerCredential { + try { + const bytes = Buffer.from(ciphertext, 'base64url'); + const decipher = createDecipheriv('aes-256-gcm', key, bytes.subarray(0, 12)); + decipher.setAAD(Buffer.from(approval)); + decipher.setAuthTag(bytes.subarray(12, 28)); + return JSON.parse(Buffer.concat([decipher.update(bytes.subarray(28)), decipher.final()]).toString('utf8')) as BridgeWorkerCredential; + } catch { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Browser enrollment replay is unavailable'); + } +} + function credentialDigestKey(credentialDigest: string): string { return `${PREFIX}:credential:${credentialDigest}`; } @@ -408,6 +534,215 @@ export class RedisBridgePairingStore { return this.recovery != null; } + get recoveryServerId(): string | undefined { + return this.recovery?.serverId; + } + + async limitBrowserEnrollmentPeer(peer: string): Promise { + if (this.recovery == null) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Browser enrollment is disabled'); + } + const allowed = await this.redis.eval( + LIMIT_RECOVERY_ATTEMPTS_SCRIPT, + 1, + `${PREFIX}:browser-enrollment:peer:${digest(peer)}`, + String(this.recovery.maxUntrustedRequestsPerMinute ?? DEFAULT_UNTRUSTED_RECOVERY_REQUESTS_PER_MINUTE), + ); + if (allowed !== 1) { + throw new BridgePairingError('RECOVERY_RATE_LIMITED', 'Too many browser enrollment requests from this peer'); + } + } + + async issueBrowserEnrollment(approval: BrowserEnrollmentApproval): Promise { + if (!isBrowserEnrollmentApproval(approval) || this.recovery == null || approval.serverId !== this.recovery.serverId || + Date.parse(approval.expiresAt) <= Date.now() || Date.parse(approval.expiresAt) - Date.now() > BROWSER_ENROLLMENT_MAX_TTL_MS) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Invalid browser enrollment approval'); + } + await this.removeLegacyPairings(approval.workerId); + const approvalRaw = canonicalApproval(approval); + const normalized = JSON.parse(approvalRaw) as BrowserEnrollmentApproval; + const enrollment: StoredEnrollment = { + workerId: approval.workerId, + serverId: approval.serverId, + identityId: randomBytes(18).toString('base64url'), + generation: randomBytes(18).toString('base64url'), + pairingGeneration: 0, + browserEnrollment: { + operation: approval.operation, + serverId: approval.serverId, + instance: approval.instance, + attemptId: approval.attemptId, + requestHash: approval.requestHash, + workerId: approval.workerId, + expiresAt: approval.expiresAt, + }, + publicKey: approval.publicKey, + binding: normalized.binding, + }; + const key = browserEnrollmentKey(approval.instance, approval.attemptId); + const workerPrefix = `${PREFIX}:worker:${encodeURIComponent(approval.workerId)}`; + // The manual-pairing index points at a separate fence, never the immutable + // approval. Manual issue/revoke consumes that fence without erasing the tombstone. + const accepted = await this.redis.eval( + ISSUE_BROWSER_ENROLLMENT_SCRIPT, + 13, + key, + `${key}:enrollment`, + `${key}:fence`, + workerPairingIndexKey(approval.workerId), + workerPairingGenerationKey(approval.workerId), + workerIdentityKey(approval.workerId), + workerStableIdentityKey(approval.workerId), + workerEnrollmentKey(approval.workerId), + workerEnrollmentRequiredKey(approval.workerId), + workerPrefix, + `${workerPrefix}:incarnation`, + `${workerPrefix}:registration-generation`, + `${key}:tombstone`, + approvalRaw, + JSON.stringify(enrollment), + String(Date.parse(approval.expiresAt)), + String(BROWSER_ENROLLMENT_MAX_TTL_MS), + `${digest(approvalRaw)}:${digest(canonicalBrowserEnrollmentRedemption(approval))}`, + ); + if (accepted !== 1) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Browser enrollment is expired, revoked, superseded, or conflicts with an existing worker'); + } + return normalized; + } + + async browserEnrollmentStatus(proof: BrowserEnrollmentStatusRequest): Promise<{ + protocolVersion: 1; + status: 'pending' | 'completed' | 'revoked'; + workerId: string; + }> { + if (!isBrowserEnrollmentStatusRequest(proof) || this.recovery == null || proof.serverId !== this.recovery.serverId) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Invalid browser enrollment status request'); + } + const key = browserEnrollmentKey(proof.instance, proof.attemptId); + const [raw, generation, requiredGeneration, approvalRaw, tombstone, fence, indexed, planned] = await this.redis.mget( + workerEnrollmentKey(proof.workerId), + workerPairingGenerationKey(proof.workerId), + workerEnrollmentRequiredKey(proof.workerId), + key, + `${key}:tombstone`, + `${key}:fence`, + workerPairingIndexKey(proof.workerId), + `${key}:enrollment`, + ); + const contract = canonicalBrowserEnrollmentRedemption(proof); + try { + const enrollment = this.checkedEnrollment(proof.workerId, raw, generation, requiredGeneration); + if (isBrowserEnrollmentRedemptionContract(enrollment.browserEnrollment) && + canonicalBrowserEnrollmentRedemption(enrollment.browserEnrollment) === contract) { + return { protocolVersion: 1, status: 'completed', workerId: proof.workerId }; + } + } catch (error) { + if (!(error instanceof BridgePairingError)) throw error; + } + if (tombstone?.split(':')[1] !== digest(contract)) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Browser enrollment does not match a known contract'); + } + let pending = false; + try { + const approval = JSON.parse(approvalRaw ?? 'null') as unknown; + pending = (generation ?? '0') === '0' && raw == null && requiredGeneration == null && planned != null && + isBrowserEnrollmentApproval(approval) && canonicalBrowserEnrollmentRedemption(approval) === contract && + Date.parse(approval.expiresAt) > Date.now() && fence === 'browser-enrollment' && indexed === `${key}:fence`; + } catch { + pending = false; + } + return { protocolVersion: 1, status: pending ? 'pending' : 'revoked', workerId: proof.workerId }; + } + + async redeemBrowserEnrollment( + proof: BrowserEnrollmentRedemption, + adminToken: string, + ): Promise { + if (!isBrowserEnrollmentRedemption(proof) || this.recovery == null || proof.serverId !== this.recovery.serverId) { + throw new BridgePairingError('PROOF_INVALID', 'Invalid browser enrollment proof'); + } + const replayKey = enrollmentReplayKey(adminToken, proof.serverId); + const key = browserEnrollmentKey(proof.instance, proof.attemptId); + const attempts = await this.redis.eval( + LIMIT_RECOVERY_ATTEMPTS_SCRIPT, + 1, + `${key}:attempts`, + String(this.recovery.maxAttemptsPerMinute ?? DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE), + ); + if (attempts !== 1) { + throw new BridgePairingError('RECOVERY_RATE_LIMITED', 'Too many browser enrollment attempts'); + } + const [raw, enrollmentRaw] = await this.redis.mget(key, `${key}:enrollment`); + let approval: BrowserEnrollmentApproval; + let enrollment: StoredEnrollment; + try { + approval = JSON.parse(raw ?? 'null') as BrowserEnrollmentApproval; + enrollment = JSON.parse(enrollmentRaw ?? 'null') as StoredEnrollment; + if (!isBrowserEnrollmentApproval(approval) || canonicalBrowserEnrollmentRedemption(approval) !== canonicalBrowserEnrollmentRedemption(proof) || + enrollment?.workerId !== approval.workerId || enrollment.serverId !== approval.serverId || + enrollment.publicKey !== approval.publicKey || enrollment.pairingGeneration !== 0 || + enrollment.browserEnrollment == null || canonicalBrowserEnrollmentRedemption(enrollment.browserEnrollment) !== canonicalBrowserEnrollmentRedemption(proof) || + JSON.stringify(enrollment.binding) !== JSON.stringify(approval.binding) || + !/^[A-Za-z0-9_-]{24}$/.test(enrollment.identityId) || !/^[A-Za-z0-9_-]{24}$/.test(enrollment.generation)) { + throw new Error('Invalid approval'); + } + } catch { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Browser enrollment is unavailable or does not match its approval'); + } + if (!verifyBrowserEnrollmentRedemption(approval.publicKey, proof, proof.signature)) { + throw new BridgePairingError('PROOF_INVALID', 'Browser enrollment requires the approved machine key'); + } + const credential: BridgeWorkerCredential = { + workerId: approval.workerId, + credential: randomBytes(32).toString('base64url'), + expiresAt: new Date(Date.now() + this.credentialTtlSeconds * 1000).toISOString(), + }; + const credentialDigest = digest(credential.credential); + const stored: StoredCredential = { + workerId: approval.workerId, + identityId: enrollment.identityId, + publicKey: approval.publicKey, + expiresAt: credential.expiresAt, + binding: approval.binding, + enrollmentGeneration: enrollment.generation, + }; + const workerPrefix = `${PREFIX}:worker:${encodeURIComponent(approval.workerId)}`; + const encrypted = await this.redis.eval( + REDEEM_BROWSER_ENROLLMENT_SCRIPT, + 14, + key, + `${key}:enrollment`, + `${key}:fence`, + workerPairingIndexKey(approval.workerId), + workerPairingGenerationKey(approval.workerId), + credentialDigestKey(credentialDigest), + workerIdentityKey(approval.workerId), + workerStableIdentityKey(approval.workerId), + workerEnrollmentKey(approval.workerId), + workerEnrollmentRequiredKey(approval.workerId), + `${key}:result`, + workerPrefix, + `${workerPrefix}:incarnation`, + `${workerPrefix}:registration-generation`, + raw!, + enrollmentRaw!, + String(Date.parse(approval.expiresAt)), + JSON.stringify(stored), + String(this.credentialTtlSeconds), + credentialDigest, + enrollment.identityId, + String(this.recovery.enrollmentTtlSeconds ?? 0), + enrollment.generation, + encryptEnrollmentReplay(credential, raw!, replayKey), + Date.parse(proof.expiresAt) > Date.now() ? '1' : '0', + ); + if (typeof encrypted !== 'string') { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Browser enrollment is expired, revoked, or superseded'); + } + return decryptEnrollmentReplay(encrypted, raw!, replayKey); + } + async limitUntrustedRecovery( peer: string, operation: 'challenge' | 'recover', diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 123f8c75..d035b032 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -6,6 +6,12 @@ import type { BridgeWorkerRegistration } from '../../../packages/code/src/protoc import type { BridgePrincipalType, BridgeWorkerBinding } from './pairing'; import type { CodeBridgeAssignment, CodeBridgeSettlement } from './store'; +import { + BROWSER_ENROLLMENT_PROTOCOL_VERSION, + isBrowserEnrollmentApproval, + isBrowserEnrollmentRedemption, + isBrowserEnrollmentStatusRequest, +} from '../../../packages/code/src/browser-enrollment-protocol'; import { BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, BRIDGE_PROTOCOL_VERSION, @@ -243,6 +249,73 @@ export function createBridgeRouter(options: BridgeRouterOptions): Router { const workerAuth = options.authMode === 'paired' ? pairedWorkerAuth : staticWorkerAuth; + const browserEnrollmentEnabled = options.authMode === 'paired' && + options.pairings.recoveryEnabled && options.allowDynamicWorkers === true && + options.adminToken.length > 0; + + const browserEnrollmentAvailable: RequestHandler = (_req, res, next) => { + if (!browserEnrollmentEnabled) { + res.status(404).json({ error: 'Browser enrollment is disabled' }); + return; + } + next(); + }; + + const sendEnrollmentError = (error: unknown, res: Response): void => { + if (!(error instanceof BridgePairingError)) throw error; + if (error.code === 'RECOVERY_RATE_LIMITED') { + res.set('Retry-After', '60').status(429).json({ error: error.message, code: error.code }); + return; + } + res.status(error.code === 'PROOF_INVALID' ? 401 : 409) + .json({ error: error.message, code: error.code }); + }; + + router.get('/enrollments/capabilities', adminAuth, browserEnrollmentAvailable, (_req, res) => { + res.json({ + browserEnrollmentVersion: BROWSER_ENROLLMENT_PROTOCOL_VERSION, + serverId: options.pairings.recoveryServerId, + }); + }); + + router.post('/enrollments', adminAuth, browserEnrollmentAvailable, asyncRoute(async (req, res) => { + if (!isBrowserEnrollmentApproval(req.body)) { + res.status(400).json({ error: 'Invalid browser enrollment approval' }); + return; + } + try { + res.json(await options.pairings.issueBrowserEnrollment(req.body)); + } catch (error) { + sendEnrollmentError(error, res); + } + })); + + router.post('/enrollments/status', adminAuth, browserEnrollmentAvailable, asyncRoute(async (req, res) => { + if (!isBrowserEnrollmentStatusRequest(req.body)) { + res.status(400).json({ error: 'Invalid browser enrollment status request' }); + return; + } + try { + res.json(await options.pairings.browserEnrollmentStatus(req.body)); + } catch (error) { + sendEnrollmentError(error, res); + } + })); + + router.post('/enrollments/redeem', adminAuth, browserEnrollmentAvailable, asyncRoute(async (req, res) => { + try { + await options.pairings.limitBrowserEnrollmentPeer(req.socket.remoteAddress ?? ''); + if (!isBrowserEnrollmentRedemption(req.body)) { + res.status(400).json({ error: 'Invalid browser enrollment redemption' }); + return; + } + const credential = await options.pairings.redeemBrowserEnrollment(req.body, options.adminToken); + res.json({ protocolVersion: BROWSER_ENROLLMENT_PROTOCOL_VERSION, ...credential }); + } catch (error) { + sendEnrollmentError(error, res); + } + })); + router.post('/pairings', adminAuth, asyncRoute(async (req, res) => { if (options.authMode !== 'paired') { res.status(409).json({ error: 'Paired worker authentication is disabled' }); From 0eb6c2dcd697373692c6e7acde222bede23c8f29 Mon Sep 17 00:00:00 2001 From: "lia-by-librechat[bot]" <328778573+lia-by-librechat[bot]@users.noreply.github.com> Date: Thu, 22 Oct 2026 15:09:55 +0200 Subject: [PATCH 45/48] =?UTF-8?q?=F0=9F=93=91=20feat:=20Add=20Standalone?= =?UTF-8?q?=20File=20Extraction=20Service=20(#306)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 📑 feat: Add Standalone File Extraction Service * fix: Restrict Worker Control and Preserve Nested Document Text * fix: Preserve Parser Policy Failures and Startup Restrictions * fix: Reject Unsupported DOCX Content and Validate OPC XML Parts * fix: Enforce Complete DOCX Content Structure * fix: Extract Merged DOCX Cells Once * fix: Reject Unmappable PDF Font Text * fix: Normalize Optional PDF Font Encodings --------- Co-authored-by: Lia --- .github/workflows/extraction.yml | 25 + packages/extraction/.gitattributes | 2 + packages/extraction/.gitignore | 1 + packages/extraction/Dockerfile | 32 ++ packages/extraction/compose.yaml | 22 + packages/extraction/native/guard.c | 156 ++++++ packages/extraction/package-lock.json | 64 +++ packages/extraction/package.json | 22 + packages/extraction/src/backend.ts | 166 ++++++ packages/extraction/src/boundary.test.ts | 22 + packages/extraction/src/health.ts | 15 + packages/extraction/src/main.ts | 64 +++ packages/extraction/src/protocol.test.ts | 68 +++ packages/extraction/src/protocol.ts | 107 ++++ packages/extraction/src/server.test.ts | 267 ++++++++++ packages/extraction/src/server.ts | 260 +++++++++ packages/extraction/tests/adversarial.py | 488 +++++++++++++++++ packages/extraction/tests/block-landlock.c | 21 + .../extraction/tests/container-client.mjs | 303 +++++++++++ packages/extraction/tests/container.sh | 61 +++ .../extraction/tests/fixtures/aggregate.pdf | Bin 0 -> 37605 bytes packages/extraction/tests/fixtures/body.docx | Bin 0 -> 36732 bytes .../extraction/tests/fixtures/compressed.pdf | Bin 0 -> 707 bytes packages/extraction/tests/fixtures/empty.docx | Bin 0 -> 36563 bytes packages/extraction/tests/fixtures/empty.pdf | Bin 0 -> 660 bytes .../extraction/tests/fixtures/generate.py | 37 ++ .../extraction/tests/fixtures/inflate.pdf | Bin 0 -> 8852 bytes packages/extraction/tests/fixtures/pages.pdf | Bin 0 -> 950 bytes packages/extraction/tests/isolation.mjs | 64 +++ packages/extraction/tests/isolation.py | 73 +++ packages/extraction/tsconfig.json | 15 + packages/extraction/worker/extract.py | 503 ++++++++++++++++++ packages/extraction/worker/requirements.txt | 4 + 33 files changed, 2862 insertions(+) create mode 100644 .github/workflows/extraction.yml create mode 100644 packages/extraction/.gitattributes create mode 100644 packages/extraction/.gitignore create mode 100644 packages/extraction/Dockerfile create mode 100644 packages/extraction/compose.yaml create mode 100644 packages/extraction/native/guard.c create mode 100644 packages/extraction/package-lock.json create mode 100644 packages/extraction/package.json create mode 100644 packages/extraction/src/backend.ts create mode 100644 packages/extraction/src/boundary.test.ts create mode 100644 packages/extraction/src/health.ts create mode 100644 packages/extraction/src/main.ts create mode 100644 packages/extraction/src/protocol.test.ts create mode 100644 packages/extraction/src/protocol.ts create mode 100644 packages/extraction/src/server.test.ts create mode 100644 packages/extraction/src/server.ts create mode 100644 packages/extraction/tests/adversarial.py create mode 100644 packages/extraction/tests/block-landlock.c create mode 100644 packages/extraction/tests/container-client.mjs create mode 100755 packages/extraction/tests/container.sh create mode 100644 packages/extraction/tests/fixtures/aggregate.pdf create mode 100644 packages/extraction/tests/fixtures/body.docx create mode 100644 packages/extraction/tests/fixtures/compressed.pdf create mode 100644 packages/extraction/tests/fixtures/empty.docx create mode 100644 packages/extraction/tests/fixtures/empty.pdf create mode 100644 packages/extraction/tests/fixtures/generate.py create mode 100644 packages/extraction/tests/fixtures/inflate.pdf create mode 100644 packages/extraction/tests/fixtures/pages.pdf create mode 100644 packages/extraction/tests/isolation.mjs create mode 100644 packages/extraction/tests/isolation.py create mode 100644 packages/extraction/tsconfig.json create mode 100644 packages/extraction/worker/extract.py create mode 100644 packages/extraction/worker/requirements.txt diff --git a/.github/workflows/extraction.yml b/.github/workflows/extraction.yml new file mode 100644 index 00000000..29a55217 --- /dev/null +++ b/.github/workflows/extraction.yml @@ -0,0 +1,25 @@ +name: Extraction Service +on: + pull_request: + paths: ['packages/extraction/**', '.github/workflows/extraction.yml'] + push: + branches: [main] + paths: ['packages/extraction/**', '.github/workflows/extraction.yml'] +permissions: + contents: read +jobs: + extraction: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 + with: + node-version: 24.16.0 + cache: npm + cache-dependency-path: packages/extraction/package-lock.json + - run: npm ci --ignore-scripts + working-directory: packages/extraction + - run: npm run format:check && npm run typecheck && npm test + working-directory: packages/extraction + - run: npm run test:container + working-directory: packages/extraction diff --git a/packages/extraction/.gitattributes b/packages/extraction/.gitattributes new file mode 100644 index 00000000..67da8786 --- /dev/null +++ b/packages/extraction/.gitattributes @@ -0,0 +1,2 @@ +tests/fixtures/*.pdf binary +tests/fixtures/*.docx binary diff --git a/packages/extraction/.gitignore b/packages/extraction/.gitignore new file mode 100644 index 00000000..2e3a563c --- /dev/null +++ b/packages/extraction/.gitignore @@ -0,0 +1 @@ +native/guard diff --git a/packages/extraction/Dockerfile b/packages/extraction/Dockerfile new file mode 100644 index 00000000..4acf5d9c --- /dev/null +++ b/packages/extraction/Dockerfile @@ -0,0 +1,32 @@ +FROM node:24.16.0-bookworm-slim AS build +WORKDIR /build +COPY packages/extraction/package*.json ./ +RUN npm ci --ignore-scripts --no-audit --no-fund +COPY packages/extraction/tsconfig.json ./ +COPY packages/extraction/src ./src +RUN npm run build +RUN apt-get update && apt-get install -y --no-install-recommends gcc libc6-dev linux-libc-dev && rm -rf /var/lib/apt/lists/* +COPY packages/extraction/native/guard.c ./guard.c +RUN gcc -O2 -Wall -Wextra -Werror -o guard guard.c +COPY packages/extraction/tests/block-landlock.c ./block-landlock.c +RUN gcc -O2 -Wall -Wextra -Werror -o block-landlock block-landlock.c + +FROM python:3.13-slim-bookworm@sha256:5024f48ba9441d4b13a95d3945abc6365538e3a31109833367a1923523c6efed AS runtime +RUN apt-get update && apt-get install -y --no-install-recommends libstdc++6 libatomic1 && rm -rf /var/lib/apt/lists/* +COPY packages/extraction/worker/requirements.txt /opt/extraction/requirements.txt +RUN pip install --no-cache-dir -r /opt/extraction/requirements.txt +COPY --from=build /usr/local/bin/node /usr/local/bin/node +COPY --from=build --chmod=0555 /build/guard /opt/extraction/guard +COPY --from=build --chmod=0555 /build/dist /opt/extraction/dist +COPY --chmod=0444 packages/extraction/package.json /opt/extraction/package.json +COPY --chmod=0444 packages/extraction/worker/extract.py /opt/extraction/extract.py +RUN groupadd -g 10001 extraction && useradd -u 10001 -g extraction -M extraction && \ + mkdir /socket /jobs && chown extraction:extraction /socket /jobs && chmod 0700 /jobs && chmod 0770 /socket +USER 10001:10001 +WORKDIR /opt/extraction +HEALTHCHECK --interval=10s --timeout=2s --start-period=10s --retries=3 CMD ["node", "dist/health.js"] +CMD ["node", "dist/main.js"] + +FROM runtime AS qualification +COPY --chown=10001:10001 packages/extraction/tests /tests +COPY --from=build --chmod=0555 /build/block-landlock /tests/block-landlock diff --git a/packages/extraction/compose.yaml b/packages/extraction/compose.yaml new file mode 100644 index 00000000..f6bbf9f4 --- /dev/null +++ b/packages/extraction/compose.yaml @@ -0,0 +1,22 @@ +services: + extraction: + image: ${EXTRACTION_IMAGE:-librechat-extraction:local} + build: + context: ../.. + dockerfile: packages/extraction/Dockerfile + target: runtime + init: true + user: '10001:10001' + read_only: true + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + network_mode: none + mem_limit: 768m + cpus: 2 + pids_limit: 32 + tmpfs: + - /jobs:rw,noexec,nosuid,nodev,size=96m,nr_inodes=1024,uid=10001,gid=10001,mode=0700 + volumes: + - extraction-socket:/socket +volumes: + extraction-socket: diff --git a/packages/extraction/native/guard.c b/packages/extraction/native/guard.c new file mode 100644 index 00000000..ee1d2cbc --- /dev/null +++ b/packages/extraction/native/guard.c @@ -0,0 +1,156 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Stable UAPI values missing from Debian 12's older build headers. */ +#ifndef LANDLOCK_ACCESS_FS_TRUNCATE +#define LANDLOCK_ACCESS_FS_TRUNCATE (1ULL << 14) +#endif +#ifndef __NR_fchmodat2 +#define __NR_fchmodat2 452 +#endif + +#if defined(__x86_64__) +#define ARCH AUDIT_ARCH_X86_64 +#define LOADER "/lib64/ld-linux-x86-64.so.2" +#elif defined(__aarch64__) +#define ARCH AUDIT_ARCH_AARCH64 +#define LOADER "/lib/ld-linux-aarch64.so.1" +#else +#error Unsupported architecture +#endif + +static void fail_at(int line) { fprintf(stderr, "ISOLATION_UNAVAILABLE:%d:%d\n", line, errno); exit(125); } +#define fail() fail_at(__LINE__) +static void limit(int resource, rlim_t value) { + struct rlimit r = {value, value}; + if (setrlimit(resource, &r)) fail(); +} +static void allow(int rules, const char *path, __u64 access) { + int fd = open(path, O_PATH | O_CLOEXEC); + if (fd < 0) fail(); + struct landlock_path_beneath_attr rule = {.allowed_access = access, .parent_fd = fd}; + if (syscall(SYS_landlock_add_rule, rules, LANDLOCK_RULE_PATH_BENEATH, &rule, 0)) fail(); + close(fd); +} +#define DENY(nr) BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, nr, 0, 1), BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | EPERM) +static void restrict_syscalls(void) { + struct sock_filter filter[] = { + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, arch)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, ARCH, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), +#ifdef __x86_64__ + BPF_JUMP(BPF_JMP | BPF_JGE | BPF_K, 0x40000000, 0, 1), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), +#endif + DENY(__NR_socket), DENY(__NR_socketpair), DENY(__NR_connect), + DENY(__NR_bind), DENY(__NR_listen), DENY(__NR_accept), DENY(__NR_accept4), + DENY(__NR_clone), DENY(__NR_clone3), +#ifdef __NR_fork + DENY(__NR_fork), DENY(__NR_vfork), +#endif + DENY(__NR_ptrace), DENY(__NR_process_vm_readv), DENY(__NR_process_vm_writev), + DENY(__NR_kill), DENY(__NR_tkill), DENY(__NR_tgkill), DENY(__NR_pidfd_send_signal), + DENY(__NR_rt_sigqueueinfo), DENY(__NR_rt_tgsigqueueinfo), + /* CPython marks opened files close-on-exec through FIOCLEX. */ + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_ioctl, 0, 3), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[1])), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, FIOCLEX, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | EPERM), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), + /* Async descriptor ownership can cause kernel-delivered signals. */ + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_fcntl, 0, 7), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[1])), + DENY(F_SETOWN), DENY(F_SETOWN_EX), DENY(F_SETSIG), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), + DENY(__NR_mq_open), DENY(__NR_mq_unlink), DENY(__NR_mq_timedsend), + DENY(__NR_mq_timedreceive), DENY(__NR_mq_notify), DENY(__NR_mq_getsetattr), + DENY(__NR_mount), DENY(__NR_umount2), DENY(__NR_pivot_root), DENY(__NR_chroot), + DENY(__NR_unshare), DENY(__NR_setns), DENY(__NR_bpf), DENY(__NR_userfaultfd), + DENY(__NR_io_uring_setup), DENY(__NR_perf_event_open), + DENY(__NR_setpriority), DENY(__NR_sched_setparam), DENY(__NR_sched_setscheduler), + DENY(__NR_sched_setaffinity), DENY(__NR_sched_setattr), + DENY(__NR_pidfd_open), DENY(__NR_pidfd_getfd), + DENY(__NR_keyctl), DENY(__NR_add_key), DENY(__NR_request_key), + DENY(__NR_shmget), DENY(__NR_shmat), DENY(__NR_shmctl), + DENY(__NR_semget), DENY(__NR_semop), DENY(__NR_semctl), + DENY(__NR_msgget), DENY(__NR_msgsnd), DENY(__NR_msgrcv), DENY(__NR_msgctl), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_prlimit64, 0, 3), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[0])), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, 0, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | EPERM), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), + DENY(__NR_utimensat), +#ifdef __NR_utime + DENY(__NR_utime), DENY(__NR_utimes), DENY(__NR_futimesat), +#endif +#ifdef __NR_fchmodat2 + DENY(__NR_fchmodat2), +#endif + DENY(__NR_open_by_handle_at), DENY(__NR_name_to_handle_at), + DENY(__NR_fchmod), DENY(__NR_fchmodat), DENY(__NR_fchown), DENY(__NR_fchownat), +#ifdef __NR_chmod + DENY(__NR_chmod), DENY(__NR_chown), DENY(__NR_lchown), +#endif + DENY(__NR_mknodat), DENY(__NR_setuid), DENY(__NR_setgid), + DENY(__NR_setreuid), DENY(__NR_setregid), DENY(__NR_setresuid), DENY(__NR_setresgid), + DENY(__NR_setgroups), DENY(__NR_setxattr), DENY(__NR_lsetxattr), DENY(__NR_fsetxattr), + DENY(__NR_removexattr), DENY(__NR_lremovexattr), DENY(__NR_fremovexattr), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW) + }; + struct sock_fprog program = {.len = sizeof(filter) / sizeof(filter[0]), .filter = filter}; + if (prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &program)) fail(); +} +int main(int argc, char **argv) { + if (argc != 5 || geteuid() == 0) fail(); + /* Paths are supervisor-owned, never supplied in the HTTP protocol. */ + if (chdir(argv[1])) fail(); + pid_t parent = getppid(); + if (parent == 1 || prctl(PR_SET_PDEATHSIG, SIGKILL) || getppid() != parent) fail(); + limit(RLIMIT_AS, 512ULL * 1024 * 1024); + limit(RLIMIT_CPU, 8); + limit(RLIMIT_FSIZE, 4ULL * 1024 * 1024); + limit(RLIMIT_NOFILE, 64); + limit(RLIMIT_CORE, 0); + if (prctl(PR_SET_DUMPABLE, 0) || prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) fail(); + if (syscall(SYS_close_range, 3, ~0U, 0)) fail(); + int abi = syscall(SYS_landlock_create_ruleset, NULL, 0, LANDLOCK_CREATE_RULESET_VERSION); + if (abi < 3) fail(); + __u64 read = LANDLOCK_ACCESS_FS_READ_FILE | LANDLOCK_ACCESS_FS_READ_DIR; + __u64 write = LANDLOCK_ACCESS_FS_WRITE_FILE | LANDLOCK_ACCESS_FS_TRUNCATE | LANDLOCK_ACCESS_FS_MAKE_REG; + struct landlock_ruleset_attr attr = { + .handled_access_fs = (1ULL << 15) - 1 /* ABI 3, including REFER and TRUNCATE */ + }; + int rules = syscall(SYS_landlock_create_ruleset, &attr, sizeof(attr), 0); + if (rules < 0) fail(); + allow(rules, "/usr", read); + allow(rules, "/lib", read); + if (access("/lib64", F_OK) == 0) allow(rules, "/lib64", read); + allow(rules, argv[2], LANDLOCK_ACCESS_FS_READ_FILE | LANDLOCK_ACCESS_FS_EXECUTE); + allow(rules, LOADER, LANDLOCK_ACCESS_FS_READ_FILE | LANDLOCK_ACCESS_FS_EXECUTE); + allow(rules, argv[3], LANDLOCK_ACCESS_FS_READ_FILE); + allow(rules, argv[1], read | write); + allow(rules, "/dev/null", LANDLOCK_ACCESS_FS_READ_FILE | LANDLOCK_ACCESS_FS_WRITE_FILE); + allow(rules, "/dev/urandom", LANDLOCK_ACCESS_FS_READ_FILE); + if (syscall(SYS_landlock_restrict_self, rules, 0)) fail(); + close(rules); + restrict_syscalls(); + if (clearenv() || setenv("LANG", "C.UTF-8", 1)) fail(); + execl(argv[2], argv[2], "-I", "-B", argv[3], argv[4], (char *)NULL); + fail(); +} diff --git a/packages/extraction/package-lock.json b/packages/extraction/package-lock.json new file mode 100644 index 00000000..86e21c9f --- /dev/null +++ b/packages/extraction/package-lock.json @@ -0,0 +1,64 @@ +{ + "name": "@librechat/extraction-service", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@librechat/extraction-service", + "version": "0.1.0", + "devDependencies": { + "@types/node": "24.10.1", + "prettier": "2.4.1", + "typescript": "5.9.3" + }, + "engines": { + "node": ">=24 <25" + } + }, + "node_modules/@types/node": { + "version": "24.10.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.1.tgz", + "integrity": "sha512-GNWcUTRBgIRJD5zj+Tq0fKOJ5XZajIiBroOF0yvj2bSU1WvNdYS/dn9UxwsujGW4JX06dnHyjV2y9rRaybH0iQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~7.16.0" + } + }, + "node_modules/prettier": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.4.1.tgz", + "integrity": "sha512-9fbDAXSBcc6Bs1mZrDYb3XKzDLm4EXXL9sC1LqKP5rZkT6KRr/rf9amVUcODVXgguK/isJz0d0hP72WeaKWsvA==", + "dev": true, + "license": "MIT", + "bin": { + "prettier": "bin-prettier.js" + }, + "engines": { + "node": ">=10.13.0" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/packages/extraction/package.json b/packages/extraction/package.json new file mode 100644 index 00000000..5934c126 --- /dev/null +++ b/packages/extraction/package.json @@ -0,0 +1,22 @@ +{ + "name": "@librechat/extraction-service", + "version": "0.1.0", + "private": true, + "type": "module", + "engines": { + "node": ">=24 <25" + }, + "scripts": { + "build": "tsc", + "typecheck": "tsc --noEmit", + "test": "npm run build && node --test dist/*.test.js", + "start": "node dist/main.js", + "test:container": "bash tests/container.sh", + "format:check": "prettier --check src tests/*.mjs *.json compose.yaml ../../.github/workflows/extraction.yml" + }, + "devDependencies": { + "@types/node": "24.10.1", + "prettier": "2.4.1", + "typescript": "5.9.3" + } +} diff --git a/packages/extraction/src/backend.ts b/packages/extraction/src/backend.ts new file mode 100644 index 00000000..a3831c65 --- /dev/null +++ b/packages/extraction/src/backend.ts @@ -0,0 +1,166 @@ +import { spawn } from 'node:child_process'; +import { constants } from 'node:fs'; +import { mkdtemp, open, rm, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { ExtractionError, policy, validateResult } from './protocol.js'; +import type { Format } from './protocol.js'; + +export interface Backend { + healthy: boolean; + create(): Promise; + run( + job: string, + format: Format, + digest: string, + signal: AbortSignal + ): Promise; + cleanup(job: string): Promise; +} +export interface Runtime { + jobs: string; + guard: string; + python: string; + worker: string; +} +const parserErrors = new Set([ + 'INVALID_DOCUMENT', + 'EMPTY_OUTPUT', + 'OUTPUT_LIMIT', + 'DECOMPRESSION_LIMIT', + 'STRUCTURE_LIMIT', + 'ENCRYPTED_DOCUMENT', + 'UNSUPPORTED_ENCODING', + 'UNSUPPORTED_CONTENT', + 'RESOURCE_LIMIT', +]); +export class RestrictedBackend implements Backend { + healthy = true; + constructor(private readonly runtime: Runtime) {} + async create(): Promise { + if (!this.healthy) throw new ExtractionError('UNAVAILABLE'); + return mkdtemp(join(this.runtime.jobs, 'job-')); + } + private async execute( + job: string, + mode: string, + signal: AbortSignal + ): Promise { + signal.throwIfAborted(); + const child = spawn( + this.runtime.guard, + [job, this.runtime.python, this.runtime.worker, mode], + { + detached: true, + stdio: ['ignore', 'pipe', 'pipe'], + env: {}, + } + ); + let output = ''; + let count = 0; + let overflow = false; + const kill = () => { + if ( + child.pid && + child.exitCode === null && + child.signalCode === null + ) { + try { + process.kill(-child.pid, 'SIGKILL'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ESRCH') + this.healthy = false; + } + } + }; + signal.addEventListener('abort', kill, { once: true }); + if (signal.aborted) kill(); + let spawnError = false; + child.on('error', () => { + spawnError = true; + }); + child.stdout.on('data', (chunk: Buffer) => { + count += chunk.length; + if (count > 1024) { + overflow = true; + kill(); + } else output += chunk.toString('ascii'); + }); + child.stderr.on('data', (chunk: Buffer) => { + count += chunk.length; + if (count > 2048) { + overflow = true; + kill(); + } + }); + const code = await new Promise(resolve => + child.once('close', resolve) + ); + signal.removeEventListener('abort', kill); + signal.throwIfAborted(); + if (spawnError || overflow) throw new ExtractionError('UNAVAILABLE'); + if (code !== 0) { + const value = output.trim(); + if (parserErrors.has(value)) + throw new ExtractionError(value as 'INVALID_DOCUMENT'); + throw new ExtractionError( + code === null ? 'RESOURCE_LIMIT' : 'UNAVAILABLE' + ); + } + return output; + } + async probe(): Promise { + const job = await this.create(); + try { + const output = await this.execute( + job, + 'probe', + AbortSignal.timeout(policy.deadlineMs) + ); + if (output.trim() !== 'READY') + throw new ExtractionError('UNAVAILABLE'); + } finally { + await this.cleanup(job); + } + } + async run( + job: string, + format: Format, + digest: string, + signal: AbortSignal + ): Promise { + await this.execute(job, format, signal); + const file = await open( + join(job, 'result'), + constants.O_RDONLY | constants.O_NOFOLLOW + ); + try { + const stat = await file.stat(); + if ( + !stat.isFile() || + stat.nlink !== 1 || + stat.size > policy.resultBytes + ) { + throw new ExtractionError('OUTPUT_LIMIT'); + } + const bytes = await file.readFile(); + signal.throwIfAborted(); + validateResult(bytes, format, digest); + return bytes; + } finally { + await file.close(); + } + } + async cleanup(job: string): Promise { + try { + await rm(job, { recursive: true, force: true, maxRetries: 0 }); + } catch { + this.healthy = false; + throw new ExtractionError('UNAVAILABLE'); + } + } +} + +/** Used by focused worker tests, not by HTTP callers. */ +export async function writeInput(job: string, input: Buffer): Promise { + await writeFile(join(job, 'input'), input, { flag: 'wx', mode: 0o600 }); +} diff --git a/packages/extraction/src/boundary.test.ts b/packages/extraction/src/boundary.test.ts new file mode 100644 index 00000000..da54749e --- /dev/null +++ b/packages/extraction/src/boundary.test.ts @@ -0,0 +1,22 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import { test } from 'node:test'; + +test('supervisor has no parser or application stack dependencies', async () => { + const manifest = JSON.parse( + await readFile(new URL('../package.json', import.meta.url), 'utf8') + ); + assert.equal(manifest.dependencies, undefined); + for (const name of ['server.ts', 'backend.ts', 'protocol.ts', 'main.ts']) { + const source = await readFile( + new URL(`../src/${name}`, import.meta.url), + 'utf8' + ); + for (const match of source.matchAll(/from ['"]([^'"]+)['"]/g)) { + assert.ok( + match[1]?.startsWith('node:') || match[1]?.startsWith('./'), + match[1] + ); + } + } +}); diff --git a/packages/extraction/src/health.ts b/packages/extraction/src/health.ts new file mode 100644 index 00000000..641575c4 --- /dev/null +++ b/packages/extraction/src/health.ts @@ -0,0 +1,15 @@ +import { request } from 'node:http'; +const req = request( + { + socketPath: '/socket/extraction.sock', + path: '/v1/capabilities', + timeout: 1000, + }, + res => { + res.resume(); + res.once('end', () => process.exit(res.statusCode === 200 ? 0 : 1)); + } +); +req.on('timeout', () => req.destroy()); +req.on('error', () => process.exit(1)); +req.end(); diff --git a/packages/extraction/src/main.ts b/packages/extraction/src/main.ts new file mode 100644 index 00000000..424a6ed6 --- /dev/null +++ b/packages/extraction/src/main.ts @@ -0,0 +1,64 @@ +import { chmod, lstat, readdir, unlink } from 'node:fs/promises'; +import { connect } from 'node:net'; +import { RestrictedBackend } from './backend.js'; +import { extractionServer } from './server.js'; + +// Deployment-owned locations, not caller-selected paths or runtimes. +const backend = new RestrictedBackend({ + jobs: '/jobs', + guard: '/opt/extraction/guard', + python: '/usr/local/bin/python3.13', + worker: '/opt/extraction/extract.py', +}); +try { + if ((await readdir('/jobs')).length) throw new Error('DIRTY_SCRATCH'); + try { + const stat = await lstat('/socket/extraction.sock'); + if (!stat.isSocket()) throw new Error('INVALID_SOCKET'); + await new Promise((resolve, reject) => { + const client = connect('/socket/extraction.sock'); + client.setTimeout(1000, () => { + client.destroy(); + reject(new Error('SOCKET_EXISTS')); + }); + client.once('connect', () => { + client.destroy(); + reject(new Error('SOCKET_EXISTS')); + }); + client.once('error', error => { + if ((error as NodeJS.ErrnoException).code === 'ECONNREFUSED') + resolve(); + else reject(error); + }); + }); + await unlink('/socket/extraction.sock'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + await backend.probe(); + const server = extractionServer(backend); + server.on('error', () => { + console.error('EXTRACTION_UNAVAILABLE'); + process.exit(1); + }); + process.umask(0o007); + server.listen('/socket/extraction.sock', async () => { + try { + await chmod('/socket/extraction.sock', 0o660); + } catch { + console.error('EXTRACTION_UNAVAILABLE'); + process.exit(1); + } + console.log('Extraction service ready'); + }); + const shutdown = () => { + server.close(); + server.closeAllConnections(); + setTimeout(() => process.exit(0), 12_000).unref(); + }; + process.once('SIGTERM', shutdown); + process.once('SIGINT', shutdown); +} catch { + console.error('EXTRACTION_UNAVAILABLE'); + process.exit(1); +} diff --git a/packages/extraction/src/protocol.test.ts b/packages/extraction/src/protocol.test.ts new file mode 100644 index 00000000..6688fa49 --- /dev/null +++ b/packages/extraction/src/protocol.test.ts @@ -0,0 +1,68 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { policy, validateResult } from './protocol.js'; +const digest = 'a'.repeat(64); +const value = (text = 'hello') => ({ + version: 1, + operation: 'document.extract-text', + format: 'pdf', + inputSha256: digest, + textBytes: Buffer.byteLength(text), + segments: [{ kind: 'page', index: 1, text }], +}); +const encode = (result: unknown) => Buffer.from(JSON.stringify(result)); +test('preserves ordered page segments and validates digest and byte count', () => { + const result = value(); + assert.deepEqual(validateResult(encode(result), 'pdf', digest), result); + for (const changed of [ + { ...result, inputSha256: 'b'.repeat(64) }, + { ...result, textBytes: 4 }, + { ...result, extra: true }, + { ...result, format: 'docx' }, + { ...result, segments: [{ kind: 'page', index: 0, text: 'hello' }] }, + { ...result, segments: [{ kind: 'sheet', index: 1, text: 'hello' }] }, + ]) + assert.throws(() => validateResult(encode(changed), 'pdf', digest)); +}); +test('rejects malformed UTF-8, lone surrogates, emptiness and invalid JSON', () => { + for (const bytes of [ + Buffer.from([0xff]), + Buffer.from('{'), + encode(value('\ud800')), + encode(value(' ')), + encode(null), + ]) { + assert.throws(() => validateResult(bytes, 'pdf', digest)); + } +}); +test('accepts exact text limit and rejects one byte more or oversized wire result', () => { + assert.equal( + validateResult( + encode(value('x'.repeat(policy.textBytes))), + 'pdf', + digest + ).textBytes, + policy.textBytes + ); + assert.throws(() => + validateResult( + encode(value('x'.repeat(policy.textBytes + 1))), + 'pdf', + digest + ) + ); + assert.throws(() => + validateResult(Buffer.alloc(policy.resultBytes + 1), 'pdf', digest) + ); +}); +test('rejects excess pages and does not accept PDF pages as DOCX structure', () => { + const result = value(); + result.segments = Array.from( + { length: policy.maxSegments + 1 }, + (_, index) => ({ kind: 'page', index: index + 1, text: 'hello' }) + ); + assert.throws(() => validateResult(encode(result), 'pdf', digest)); + assert.throws(() => + validateResult(encode({ ...value(), format: 'docx' }), 'docx', digest) + ); +}); diff --git a/packages/extraction/src/protocol.ts b/packages/extraction/src/protocol.ts new file mode 100644 index 00000000..935c902c --- /dev/null +++ b/packages/extraction/src/protocol.ts @@ -0,0 +1,107 @@ +import { createHash } from 'node:crypto'; + +export const policy = Object.freeze({ + inputBytes: 10 * 1024 * 1024, + textBytes: 1024 * 1024, + resultBytes: 3 * 1024 * 1024, + deadlineMs: 10_000, + maxSegments: 128, +}); +export type Format = 'pdf' | 'docx'; +export type ErrorCode = + | 'INVALID_REQUEST' + | 'INPUT_LIMIT' + | 'BUSY' + | 'DEADLINE' + | 'CANCELLED' + | 'UNAVAILABLE' + | 'INVALID_DOCUMENT' + | 'EMPTY_OUTPUT' + | 'OUTPUT_LIMIT' + | 'DECOMPRESSION_LIMIT' + | 'STRUCTURE_LIMIT' + | 'ENCRYPTED_DOCUMENT' + | 'UNSUPPORTED_ENCODING' + | 'UNSUPPORTED_CONTENT' + | 'RESOURCE_LIMIT'; +export class ExtractionError extends Error { + constructor(readonly code: ErrorCode) { + super(code); + } +} +export interface Segment { + kind: 'page' | 'document'; + index: number; + text: string; +} +export interface Result { + version: 1; + operation: 'document.extract-text'; + format: Format; + inputSha256: string; + textBytes: number; + segments: Segment[]; +} +const keys = (value: object, expected: string[]) => + Object.keys(value).length === expected.length && + expected.every(key => Object.hasOwn(value, key)); +export function validateResult( + bytes: Buffer, + format: Format, + digest: string +): Result { + if (bytes.length > policy.resultBytes) + throw new ExtractionError('OUTPUT_LIMIT'); + let result: Result; + try { + result = JSON.parse( + new TextDecoder('utf-8', { fatal: true }).decode(bytes) + ) as Result; + } catch { + throw new ExtractionError('INVALID_DOCUMENT'); + } + if ( + !result || + typeof result !== 'object' || + !keys(result, [ + 'version', + 'operation', + 'format', + 'inputSha256', + 'textBytes', + 'segments', + ]) || + result.version !== 1 || + result.operation !== 'document.extract-text' || + result.format !== format || + result.inputSha256 !== digest || + !Array.isArray(result.segments) || + !result.segments.length || + result.segments.length > (format === 'pdf' ? policy.maxSegments : 1) + ) { + throw new ExtractionError('INVALID_DOCUMENT'); + } + let size = 0; + let nonempty = false; + for (const [index, segment] of result.segments.entries()) { + if ( + !segment || + typeof segment !== 'object' || + !keys(segment, ['kind', 'index', 'text']) || + segment.kind !== (format === 'pdf' ? 'page' : 'document') || + segment.index !== index + 1 || + typeof segment.text !== 'string' || + !segment.text.isWellFormed() + ) + throw new ExtractionError('INVALID_DOCUMENT'); + size += Buffer.byteLength(segment.text); + nonempty ||= Boolean(segment.text.trim()); + } + if (size > policy.textBytes) throw new ExtractionError('OUTPUT_LIMIT'); + if (size !== result.textBytes || !nonempty) + throw new ExtractionError('INVALID_DOCUMENT'); + return result; +} +export function sha256(bytes: Buffer): string { + return createHash('sha256').update(bytes).digest('hex'); +} diff --git a/packages/extraction/src/server.test.ts b/packages/extraction/src/server.test.ts new file mode 100644 index 00000000..60f9bc63 --- /dev/null +++ b/packages/extraction/src/server.test.ts @@ -0,0 +1,267 @@ +import assert from 'node:assert/strict'; +import { mkdtemp, rm, readdir } from 'node:fs/promises'; +import { request } from 'node:http'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { once } from 'node:events'; +import { test } from 'node:test'; +import { extractionServer } from './server.js'; +import { ExtractionError, policy } from './protocol.js'; +import type { Backend } from './backend.js'; + +async function fixture(deadline = 500) { + const root = await mkdtemp(join(tmpdir(), 'extraction-test-')); + const events: string[] = []; + const backend: Backend = { + healthy: true, + create: async () => { + events.push('create'); + return mkdtemp(join(root, 'job-')); + }, + run: async () => { + events.push('run'); + return Buffer.from('{"segments":[]}'); + }, + cleanup: async job => { + events.push('cleanup'); + await rm(job, { recursive: true }); + }, + }; + const server = extractionServer(backend, deadline); + const socketPath = join(root, 'service.sock'); + server.listen(socketPath); + await once(server, 'listening'); + return { + backend, + events, + root, + socketPath, + server, + close: async () => { + server.closeAllConnections(); + await new Promise(resolve => server.close(() => resolve())); + await rm(root, { recursive: true, force: true }); + }, + }; +} +const headers = { + 'Content-Type': 'application/octet-stream', + 'Content-Length': '4', + 'X-Extraction-Version': '1', + 'X-Extraction-Format': 'pdf', +}; +async function send( + socketPath: string, + overrides = {}, + path = '/v1/extract-text' +) { + return new Promise<{ status: number; body: string }>((resolve, reject) => { + const req = request( + { + socketPath, + path, + method: 'POST', + headers: { ...headers, ...overrides }, + }, + res => { + let body = ''; + res.on('data', bytes => { + body += bytes; + }); + res.on('end', () => resolve({ status: res.statusCode!, body })); + } + ); + req.on('error', reject); + req.end('test'); + }); +} +test('binary upload executes once and cleanup completes before success', async () => { + const f = await fixture(); + try { + assert.equal((await send(f.socketPath)).status, 200); + assert.deepEqual(f.events, ['create', 'run', 'cleanup']); + assert.deepEqual(await readdir(f.root), ['service.sock']); + } finally { + await f.close(); + } +}); +test('closed parameters and declared input limit are rejected before admission', async () => { + const f = await fixture(); + try { + for (const changes of [ + { 'X-Extraction-Format': 'xls' }, + { 'X-Extraction-Version': '2' }, + { 'X-Extraction-Url': 'https://example.com' }, + { 'Content-Type': 'text/plain' }, + ]) { + assert.equal((await send(f.socketPath, changes)).status, 400); + } + assert.equal( + (await send(f.socketPath, {}, '/v1/extract-text?path=x')).status, + 404 + ); + assert.equal( + ( + await send(f.socketPath, { + 'Content-Length': String(policy.inputBytes + 1), + }) + ).status, + 413 + ); + assert.equal(f.events.length, 0); + } finally { + await f.close(); + } +}); +test('cleanup failure suppresses success and quarantines backend', async () => { + const f = await fixture(); + f.backend.cleanup = async () => { + f.backend.healthy = false; + throw new ExtractionError('UNAVAILABLE'); + }; + try { + assert.equal((await send(f.socketPath)).status, 503); + assert.equal((await send(f.socketPath)).status, 503); + assert.equal(f.events.filter(e => e === 'run').length, 1); + } finally { + await f.close(); + } +}); +test('deadline aborts execution and waits for its close before cleanup', async () => { + const f = await fixture(50); + f.backend.run = async (_job, _format, _digest, signal) => { + await new Promise(resolve => + signal.addEventListener( + 'abort', + () => { + setTimeout(() => { + f.events.push('closed'); + resolve(); + }, 15); + }, + { once: true } + ) + ); + signal.throwIfAborted(); + return Buffer.alloc(0); + }; + try { + assert.equal((await send(f.socketPath)).status, 504); + assert.deepEqual(f.events, ['create', 'closed', 'cleanup']); + } finally { + await f.close(); + } +}); +test('upload disconnect and stalled upload both release scratch and admission', async () => { + const f = await fixture(60); + try { + for (const disconnect of [true, false]) { + const req = request({ + socketPath: f.socketPath, + path: '/v1/extract-text', + method: 'POST', + headers, + }); + req.on('error', () => {}); + req.flushHeaders(); + req.write('t'); + await new Promise(resolve => setTimeout(resolve, 20)); + if (disconnect) req.destroy(); + await new Promise(resolve => setTimeout(resolve, 80)); + assert.deepEqual(await readdir(f.root), ['service.sock']); + } + assert.equal(f.events.includes('run'), false); + assert.equal((await send(f.socketPath)).status, 200); + } finally { + await f.close(); + } +}); +test('two active jobs bound concurrency without a queue', async () => { + const f = await fixture(200); + let active = 0; + f.backend.run = async (_job, _format, _digest, signal) => { + active++; + await new Promise(resolve => + signal.addEventListener('abort', () => resolve(), { once: true }) + ); + signal.throwIfAborted(); + return Buffer.alloc(0); + }; + try { + const a = send(f.socketPath), + b = send(f.socketPath); + while (active !== 2) + await new Promise(resolve => setTimeout(resolve, 5)); + assert.equal((await send(f.socketPath)).status, 429); + assert.equal((await a).status, 504); + assert.equal((await b).status, 504); + } finally { + await f.close(); + } +}); +test('deadline closes stalled output readers while keeping their slots admitted', async () => { + const f = await fixture(250); + f.backend.run = async () => Buffer.alloc(policy.resultBytes, 'a'); + const clients = [0, 1].map(() => { + const req = request( + { + socketPath: f.socketPath, + path: '/v1/extract-text', + method: 'POST', + headers, + }, + res => { + res.pause(); + res.on('error', () => {}); + } + ); + req.on('error', () => {}); + req.end('test'); + return req; + }); + try { + while (f.events.filter(e => e === 'cleanup').length !== 2) + await new Promise(resolve => setTimeout(resolve, 5)); + assert.equal((await send(f.socketPath)).status, 429); + await new Promise(resolve => setTimeout(resolve, 300)); + f.backend.run = async () => Buffer.from('{}'); + assert.equal((await send(f.socketPath)).status, 200); + } finally { + clients.forEach(client => client.destroy()); + await f.close(); + } +}); +test('duplicate extraction control headers are rejected before worker creation', async () => { + const f = await fixture(); + try { + assert.equal( + ( + await send(f.socketPath, { + 'X-Extraction-Format': ['pdf', 'pdf'], + }) + ).status, + 400 + ); + assert.equal(f.events.length, 0); + } finally { + await f.close(); + } +}); + +test('unsupported document content returns a safe failure after cleanup', async () => { + const f = await fixture(); + f.backend.run = async () => { + throw new ExtractionError('UNSUPPORTED_CONTENT'); + }; + try { + const response = await send(f.socketPath); + assert.equal(response.status, 422); + assert.deepEqual(JSON.parse(response.body), { + error: { code: 'UNSUPPORTED_CONTENT' }, + }); + assert.deepEqual(f.events, ['create', 'cleanup']); + assert.deepEqual(await readdir(f.root), ['service.sock']); + } finally { + await f.close(); + } +}); diff --git a/packages/extraction/src/server.ts b/packages/extraction/src/server.ts new file mode 100644 index 00000000..b3bd37c8 --- /dev/null +++ b/packages/extraction/src/server.ts @@ -0,0 +1,260 @@ +import { createHash } from 'node:crypto'; +import { createServer } from 'node:http'; +import { open } from 'node:fs/promises'; +import { join } from 'node:path'; +import { ExtractionError, policy } from './protocol.js'; +import type { IncomingMessage, ServerResponse } from 'node:http'; +import type { Backend } from './backend.js'; +import type { Format } from './protocol.js'; + +export const capabilities = Object.freeze({ + version: 1, + operation: 'document.extract-text', + formats: ['pdf', 'docx'], + structure: { + pdf: 'ordered pages, one-based index', + docx: 'document body, paragraphs and tables', + }, + limitations: { + pdf: 'Text only. Simple fonts need interpretable standard/Unicode mappings. Type3/composite fonts need complete Unicode maps. Unmapped glyphs, codec recovery and graphics-state font selection are rejected.', + docx: 'Body paragraphs/tables and direct runs/hyperlinks only. Other content structures, fields, images and wrappers are rejected; formatting is ignored.', + }, + limits: { + ...policy, + concurrency: 2, + expandedBytes: 32 * 1024 * 1024, + archiveEntries: 512, + tableColumns: 512, + }, +}); +const status = (code: string): number => { + if (code === 'BUSY') return 429; + if (code === 'UNAVAILABLE') return 503; + if (code === 'DEADLINE') return 504; + if ( + code === 'INPUT_LIMIT' || + code === 'OUTPUT_LIMIT' || + code === 'DECOMPRESSION_LIMIT' || + code === 'STRUCTURE_LIMIT' + ) + return 413; + if (code === 'INVALID_REQUEST') return 400; + return 422; +}; +function reply(res: ServerResponse, code: number, bytes: Buffer): void { + res.writeHead(code, { + 'Content-Type': 'application/json; charset=utf-8', + 'Content-Length': bytes.length, + Connection: 'close', + 'Cache-Control': 'no-store', + 'X-Content-Type-Options': 'nosniff', + }); + res.end(bytes); +} +function requestFormat(req: IncomingMessage): Format { + const seen = new Set(); + for (let i = 0; i < req.rawHeaders.length; i += 2) { + const name = req.rawHeaders[i]!.toLowerCase(); + if ( + name.startsWith('x-extraction-') && + !['x-extraction-version', 'x-extraction-format'].includes(name) + ) { + throw new ExtractionError('INVALID_REQUEST'); + } + if ( + [ + 'content-length', + 'content-type', + 'x-extraction-version', + 'x-extraction-format', + ].includes(name) + ) { + if (seen.has(name)) throw new ExtractionError('INVALID_REQUEST'); + seen.add(name); + } + } + if ( + req.headers['content-type'] !== 'application/octet-stream' || + req.headers['x-extraction-version'] !== '1' || + req.headers['transfer-encoding'] + ) { + throw new ExtractionError('INVALID_REQUEST'); + } + const format = req.headers['x-extraction-format']; + if (format !== 'pdf' && format !== 'docx') + throw new ExtractionError('INVALID_REQUEST'); + const length = req.headers['content-length']; + if (typeof length !== 'string' || !/^[1-9]\d*$/.test(length)) + throw new ExtractionError('INVALID_REQUEST'); + if ( + !Number.isSafeInteger(Number(length)) || + Number(length) > policy.inputBytes + ) + throw new ExtractionError('INPUT_LIMIT'); + return format; +} +export function extractionServer( + backend: Backend, + deadlineMs: number = policy.deadlineMs +) { + if ( + !Number.isInteger(deadlineMs) || + deadlineMs <= 0 || + deadlineMs > policy.deadlineMs + ) + throw new Error('INVALID_DEADLINE'); + let admitted = 0; + const server = createServer( + { + maxHeaderSize: 4096, + requireHostHeader: true, + connectionsCheckingInterval: 1000, + }, + (req, res) => { + void handle(req, res); + } + ); + server.maxConnections = 16; + server.maxRequestsPerSocket = 1; + server.headersTimeout = 5000; + server.requestTimeout = policy.deadlineMs; + server.setTimeout(policy.deadlineMs, socket => socket.destroy()); + server.on('checkContinue', (req, res) => { + reply(res, 400, Buffer.from('{"error":{"code":"INVALID_REQUEST"}}')); + }); + async function handle( + req: IncomingMessage, + res: ServerResponse + ): Promise { + if (req.method === 'GET' && req.url === '/v1/capabilities') { + reply( + res, + backend.healthy ? 200 : 503, + Buffer.from(JSON.stringify(capabilities)) + ); + return; + } + if (req.method !== 'POST' || req.url !== '/v1/extract-text') { + reply( + res, + 404, + Buffer.from('{"error":{"code":"INVALID_REQUEST"}}') + ); + return; + } + let format: Format; + try { + format = requestFormat(req); + } catch (error) { + const code = + error instanceof ExtractionError + ? error.code + : 'INVALID_REQUEST'; + reply( + res, + status(code), + Buffer.from(JSON.stringify({ error: { code } })) + ); + return; + } + if (!backend.healthy || admitted >= 2) { + const code = backend.healthy ? 'BUSY' : 'UNAVAILABLE'; + reply( + res, + status(code), + Buffer.from(JSON.stringify({ error: { code } })) + ); + return; + } + admitted++; + const controller = new AbortController(); + const timeout = setTimeout( + () => controller.abort(new ExtractionError('DEADLINE')), + deadlineMs + ); + const aborted = () => + controller.abort(new ExtractionError('CANCELLED')); + controller.signal.addEventListener( + 'abort', + () => { + if (!req.complete) req.destroy(); + if (res.headersSent) res.destroy(); + }, + { once: true } + ); + req.once('aborted', aborted); + res.once('close', () => { + if (!res.writableFinished) aborted(); + }); + const delivered = new Promise(resolve => { + res.once('finish', resolve); + res.once('close', resolve); + }); + let job: string | undefined; + let bytes: Buffer | undefined; + let failure: unknown; + try { + controller.signal.throwIfAborted(); + job = await backend.create(); + const file = await open(join(job, 'input'), 'wx', 0o600); + const digest = createHash('sha256'); + let size = 0; + try { + for await (const chunk of req) { + controller.signal.throwIfAborted(); + size += chunk.length; + if (size > policy.inputBytes) + throw new ExtractionError('INPUT_LIMIT'); + digest.update(chunk); + await file.writeFile(chunk); + } + } finally { + await file.close(); + } + if (size !== Number(req.headers['content-length'])) + throw new ExtractionError('INVALID_REQUEST'); + controller.signal.throwIfAborted(); + bytes = await backend.run( + job, + format, + digest.digest('hex'), + controller.signal + ); + } catch (error) { + failure = error; + } finally { + if (job) { + try { + await backend.cleanup(job); + } catch (error) { + failure = error; + } + } + } + try { + if (controller.signal.aborted && !failure) + failure = controller.signal.reason; + if (controller.signal.reason instanceof ExtractionError) + failure = controller.signal.reason; + if (!res.destroyed) { + if (failure || !bytes) { + const code = + failure instanceof ExtractionError + ? failure.code + : 'UNAVAILABLE'; + reply( + res, + status(code), + Buffer.from(JSON.stringify({ error: { code } })) + ); + } else reply(res, 200, bytes); + } + await delivered; + } finally { + clearTimeout(timeout); + req.removeListener('aborted', aborted); + admitted--; + } + } + return server; +} diff --git a/packages/extraction/tests/adversarial.py b/packages/extraction/tests/adversarial.py new file mode 100644 index 00000000..710c3a11 --- /dev/null +++ b/packages/extraction/tests/adversarial.py @@ -0,0 +1,488 @@ +"""Create bounded adversarial test documents inside the disposable test container.""" +from pathlib import Path +import struct +import zipfile +from docx import Document +from pypdf import PdfReader, PdfWriter + +root = Path('/tests/fixtures') +(root / 'bad.docx').write_bytes(b'PK\x03\x04not a zip') +with zipfile.ZipFile(root / 'entries.docx', 'w') as file: + for index in range(513): + file.writestr(str(index), 'x') +with zipfile.ZipFile(root / 'duplicate.docx', 'w') as file: + file.writestr('same', 'a') + file.writestr('same', 'b') +with zipfile.ZipFile(root / 'bomb.docx', 'w', compression=zipfile.ZIP_DEFLATED) as file: + file.writestr('content.xml', b'x' * (8 * 1024 * 1024 + 1)) +with zipfile.ZipFile(root / 'aggregate.docx', 'w', compression=zipfile.ZIP_DEFLATED) as file: + for index in range(5): + file.writestr(str(index), b'x' * (7 * 1024 * 1024)) +for name, value in [('utf8.docx', b'\xff'), ('entities.docx', b']>')]: + with zipfile.ZipFile(root / name, 'w') as file: + file.writestr('word/document.xml', value) +with zipfile.ZipFile(root / 'forged.docx', 'w', compression=zipfile.ZIP_DEFLATED) as file: + file.writestr('content.xml', 'forged-content' * 20) +value = bytearray((root / 'forged.docx').read_bytes()) +central = value.index(b'PK\x01\x02') +struct.pack_into('') + 2 + value = value[:declaration_end] + b']>' + value[declaration_end:] + value = value.replace(b'First paragraph', b'&controlled;') + elif payload == 'utf8': + value = value.replace(b'First paragraph', b'\xff') + elif member == 'word/_rels/document.xml.rels': + member = 'word/_rels/document.bin.rels' + target.writestr(member, value) +print('DOCX completeness and content-type fixtures generated') + +# The supported structure is closed, not an expanding list of forbidden tags. +for tag in ['dir', 'bdo', 'smartTag', 'unknownWrapper']: + for placement in ['paragraph', 'cell', 'hyperlink']: + document = Document() + document.add_paragraph('Ordinary text must not hide wrapped content') + paragraph = (document.add_table(rows=1, cols=1).cell(0, 0).add_paragraph() + if placement == 'cell' else document.add_paragraph()) + run = paragraph.add_run('Wrapped run text')._r + parent = run.getparent() + if placement == 'hyperlink': + link = OxmlElement('w:hyperlink') + link.set(qn('w:anchor'), 'test') + parent.append(link) + parent = link + wrapper = OxmlElement('w:' + tag) + if tag in ('dir', 'bdo'): + wrapper.set(qn('w:val'), 'rtl') + wrapper.append(run) + parent.append(wrapper) + document.save(root / f'wrapper-{tag}-{placement}.docx') + +# A malformed unknown block and run child cannot bypass the same invariant. +document = Document() +document.add_paragraph('Ordinary text') +paragraph = document.add_paragraph('Unknown body wrapper text') +parent = paragraph._p.getparent() +wrapper = OxmlElement('w:unknownWrapper') +wrapper.append(paragraph._p) +parent.insert(0, wrapper) +document.save(root / 'wrapper-block.docx') +document = Document() +document.add_paragraph('Ordinary text') +run = document.add_paragraph().add_run('Run wrapper text')._r +text = run[0] +wrapper = OxmlElement('w:unknownWrapper') +wrapper.append(text) +run.append(wrapper) +document.save(root / 'wrapper-run.docx') + +# Accepted formatting, hyperlinks, bookmarks and run text primitives retain order. +document = Document() +paragraph = document.add_paragraph() +bookmark = OxmlElement('w:bookmarkStart') +bookmark.set(qn('w:id'), '1') +bookmark.set(qn('w:name'), 'test') +paragraph._p.append(bookmark) +paragraph.add_run('Bold ').bold = True +link = OxmlElement('w:hyperlink') +link.set(qn('w:anchor'), 'test') +run = OxmlElement('w:r') +text = OxmlElement('w:t') +text.text = 'Link' +run.append(text) +link.append(run) +paragraph._p.append(link) +paragraph.add_run('\tTabbed\nNext') +run = paragraph.add_run()._r +run.append(OxmlElement('w:noBreakHyphen')) +run.append(OxmlElement('w:ptab')) +run.append(OxmlElement('w:cr')) +text = OxmlElement('w:t') +text.text = 'End' +run.append(text) +run.append(OxmlElement('w:lastRenderedPageBreak')) +bookmark = OxmlElement('w:bookmarkEnd') +bookmark.set(qn('w:id'), '1') +paragraph._p.append(bookmark) +document.save(root / 'supported-runs.docx') +print('DOCX closed-structure fixtures generated') + +# Physical cells are unique. A merged origin's text must never be repeated. +for name, end in [('horizontal.docx', (0, 1)), ('vertical.docx', (1, 0)), + ('rectangle.docx', (1, 1))]: + document = Document() + table = document.add_table(rows=2, cols=3) + for row in range(2): + for column in range(3): + table.cell(row, column).text = f'R{row}C{column}' + table.cell(0, 0).merge(table.cell(*end)).text = 'Merged text' + document.save(root / name) + +document = Document() +table = document.add_table(rows=2, cols=2) +origin = table.cell(0, 0).merge(table.cell(1, 1)) +origin.text = 'Before merged nested table' +origin.add_table(rows=1, cols=1).cell(0, 0).text = 'Unique nested content' +origin.add_paragraph('After merged nested table') +document.save(root / 'merged-nested.docx') + +for name, end in [('merged-large.docx', (0, 1)), ('merged-large-vertical.docx', (1, 0))]: + document = Document() + table = document.add_table(rows=2, cols=2) + table.cell(0, 0).merge(table.cell(*end)).text = 'x' * (600 * 1024) + document.save(root / name) + +document = Document() +table = document.add_table(rows=1, cols=3) +table.cell(0, 1).text = 'Middle cell' +row = table.rows[0]._tr +row.remove(row.tc_lst[-1]) +row.remove(row.tc_lst[0]) +properties = row.get_or_add_trPr() +for tag in ('gridBefore', 'gridAfter'): + value = OxmlElement('w:' + tag) + value.set(qn('w:val'), '1') + properties.append(value) +document.save(root / 'omitted-cells.docx') + +for name, mode in [('merge-content.docx', 'content'), ('merge-orphan.docx', 'orphan'), + ('merge-span.docx', 'span'), ('merge-legacy.docx', 'legacy'), + ('wide-table.docx', 'width')]: + document = Document() + table = document.add_table(rows=2, cols=2) + if mode == 'legacy': + value = OxmlElement('w:hMerge') + value.set(qn('w:val'), 'restart') + table.cell(0, 0)._tc.get_or_add_tcPr().append(value) + elif mode == 'width': + for _ in range(511): + table._tbl.tblGrid.append(OxmlElement('w:gridCol')) + else: + table.cell(0, 0).merge(table.cell(1, 0)).text = 'Origin text' + continuation = table.rows[1]._tr.tc_lst[0] + if mode == 'content': + continuation.p_lst[0].append(OxmlElement('w:r')) + text = OxmlElement('w:t') + text.text = 'Unexpected continuation text' + continuation.p_lst[0][-1].append(text) + elif mode == 'orphan': + table.rows[0]._tr.tc_lst[0].get_or_add_tcPr().remove( + table.rows[0]._tr.tc_lst[0].tcPr.vMerge) + elif mode == 'span': + continuation.get_or_add_tcPr().get_or_add_gridSpan().val = 2 + document.save(root / name) +print('DOCX merged and omitted-cell fixtures generated') + +from pypdf.generic import FloatObject, TextStringObject + +def font_pdf(name, subtype, raw=b'A', mapping=None, glyph='/unmappedGlyph', encoding=None, form=False): + writer = PdfWriter() + writer.append(root / 'pages.pdf') + page = writer.pages[0] + font = DictionaryObject({NameObject('/Type'): NameObject('/Font'), + NameObject('/Subtype'): NameObject('/' + subtype), + NameObject('/BaseFont'): NameObject('/Helvetica')}) + if subtype == 'Type3': + proc = DecodedStreamObject() + proc.set_data(b'600 0 0 0 500 700 d1 0 0 500 700 re f') + font[NameObject('/CharProcs')] = DictionaryObject({NameObject(glyph): writer._add_object(proc)}) + font[NameObject('/FontBBox')] = ArrayObject([NumberObject(v) for v in [0, 0, 500, 700]]) + font[NameObject('/FontMatrix')] = ArrayObject([FloatObject(v) for v in [0.001, 0, 0, 0.001, 0, 0]]) + font[NameObject('/Resources')] = DictionaryObject() + font[NameObject('/FirstChar')] = NumberObject(65) + font[NameObject('/LastChar')] = NumberObject(66) + font[NameObject('/Widths')] = ArrayObject([NumberObject(600), NumberObject(600)]) + if subtype == 'Type0': + descendant = DictionaryObject({ + NameObject('/Type'): NameObject('/Font'), NameObject('/Subtype'): NameObject('/CIDFontType2'), + NameObject('/BaseFont'): NameObject('/TestComposite'), + NameObject('/CIDSystemInfo'): DictionaryObject({ + NameObject('/Registry'): TextStringObject('Adobe'), + NameObject('/Ordering'): TextStringObject('Identity'), NameObject('/Supplement'): NumberObject(0), + }), + NameObject('/FontDescriptor'): DictionaryObject({ + NameObject('/Type'): NameObject('/FontDescriptor'), NameObject('/FontName'): NameObject('/TestComposite'), + NameObject('/Flags'): NumberObject(4), NameObject('/Ascent'): NumberObject(800), + NameObject('/Descent'): NumberObject(-200), NameObject('/CapHeight'): NumberObject(700), + NameObject('/ItalicAngle'): NumberObject(0), NameObject('/StemV'): NumberObject(80), + NameObject('/FontBBox'): ArrayObject([NumberObject(v) for v in [0, -200, 1000, 800]]), + }), NameObject('/CIDToGIDMap'): NameObject('/Identity'), + }) + font[NameObject('/DescendantFonts')] = ArrayObject([writer._add_object(descendant)]) + font[NameObject('/Encoding')] = NameObject('/Identity-H') + else: + font[NameObject('/Encoding')] = DictionaryObject({ + NameObject('/BaseEncoding'): NameObject('/WinAnsiEncoding'), + NameObject('/Differences'): ArrayObject([NumberObject(65), NameObject(glyph)]), + }) + if encoding: + font[NameObject('/Encoding')] = NameObject(encoding) + if mapping is not None: + cmap = DecodedStreamObject() + size = 2 if subtype == 'Type0' else 1 + start, end = ('0000', 'FFFF') if size == 2 else ('00', 'FF') + pairs = '\n'.join(f'<{code:0{size*2}X}> <{value.encode("utf-16-be", errors="surrogatepass").hex()}>' + for code, value in mapping.items()) + cmap.set_data((f'/CIDInit /ProcSet findresource begin\n12 dict begin\nbegincmap\n' + f'/CMapName /TestUnicode def\n/CMapType 2 def\n' + f'1 begincodespacerange\n<{start}> <{end}>\nendcodespacerange\n' + f'{len(mapping)} beginbfchar\n{pairs}\nendbfchar\nendcmap\n' + f'CMapName currentdict /CMap defineresource pop\nend\nend').encode('ascii')) + font[NameObject('/ToUnicode')] = writer._add_object(cmap.flate_encode()) + resources = DictionaryObject({NameObject('/Font'): DictionaryObject({NameObject('/F2'): writer._add_object(font)})}) + stream = DecodedStreamObject() + stream.set_data(b'BT /F2 12 Tf 72 700 Td <' + raw.hex().encode() + b'> Tj ET') + if form: + stream[NameObject('/Type')] = NameObject('/XObject') + stream[NameObject('/Subtype')] = NameObject('/Form') + stream[NameObject('/BBox')] = ArrayObject([NumberObject(v) for v in [0, 0, 612, 792]]) + stream[NameObject('/Resources')] = resources + page['/Resources'][NameObject('/XObject')] = DictionaryObject({NameObject('/FontForm'): writer._add_object(stream)}) + content = page['/Contents'].get_object() + content.set_data(content.get_data() + b'\n/FontForm Do') + else: + page[NameObject('/Resources')] = resources + page[NameObject('/Contents')] = writer._add_object(stream) + writer.write(root / name) + +font_pdf('font-type3-unmapped.pdf', 'Type3') +font_pdf('font-simple-unmapped.pdf', 'Type1') +font_pdf('font-form-unmapped.pdf', 'Type3', form=True) +font_pdf('font-partial-map.pdf', 'Type3', raw=b'AB', mapping={65: 'A'}) +font_pdf('font-empty-map.pdf', 'Type3', mapping={}) +font_pdf('font-bad-unicode.pdf', 'Type3', mapping={65: '\ud800'}) +font_pdf('font-composite-unmapped.pdf', 'Type0', raw=b'\x00A') +font_pdf('font-codec-fallback.pdf', 'Type0', raw=b'\x01', mapping={1: 'A'}) +font_pdf('font-composite-partial.pdf', 'Type0', raw=b'\x00\x01\x00\x02', mapping={1: 'A'}) +font_pdf('font-type3-unicode.pdf', 'Type3', mapping={65: '\u03a9'}) +font_pdf('font-simple-unicode.pdf', 'Type1', mapping={65: '\u03a9'}) +font_pdf('font-known-glyph.pdf', 'Type1', glyph='/Aacute') +font_pdf('font-ligature.pdf', 'Type3', mapping={65: 'fi'}) +font_pdf('font-emoji.pdf', 'Type3', mapping={65: '\U0001f600'}) +font_pdf('font-composite-unicode.pdf', 'Type0', raw=b'\x00\x01\x00\x02', mapping={1: 'A', 2: 'fi'}) +font_pdf('font-form-unicode.pdf', 'Type3', mapping={65: '\u03a9'}, form=True) +writer = PdfWriter() +writer.append(root / 'pages.pdf') +content = writer.pages[0]['/Contents'].get_object() +content.set_data(content.get_data().replace(b'/F1', b'/MissingFont')) +writer.write(root / 'font-missing.pdf') +print('PDF font mapping fixtures generated') + +font_pdf('font-replacement.pdf', 'Type3', mapping={65: '\ufffd'}) +font_pdf('font-partial-simple.pdf', 'Type1', raw=b'AB', mapping={65: 'A'}) +writer = PdfWriter() +writer.append(root / 'pages.pdf') +del writer.pages[0]['/Resources'] +writer.write(root / 'font-no-resources.pdf') +writer = PdfWriter() +writer.append(root / 'pages.pdf') +del writer.pages[0]['/Resources'] +writer.pages[0]['/Contents'].get_object().set_data(b'q 0 0 10 10 re f Q') +writer.write(root / 'no-resources-graphics.pdf') +writer = PdfWriter() +writer.append(root / 'pages.pdf') +page = writer.pages[0] +page['/Resources'][NameObject('/ExtGState')] = DictionaryObject({NameObject('/GS1'): DictionaryObject({ + NameObject('/Font'): ArrayObject([page['/Resources']['/Font'].raw_get('/F1'), NumberObject(12)]), +})}) +stream = page['/Contents'].get_object() +stream.set_data(b'/GS1 gs\n' + stream.get_data()) +writer.write(root / 'font-graphics-state.pdf') + +from pypdf.generic import NullObject + +# Null optional dictionary entries mean absent, including indirect nulls. +for subtype in ['Type1', 'TrueType']: + for mode in ['absent', 'null', 'indirect-null', 'empty-dict', 'null-base', 'invalid-number']: + writer = PdfWriter() + writer.append(root / 'pages.pdf') + font = writer.pages[0]['/Resources']['/Font']['/F1'] + font[NameObject('/Subtype')] = NameObject('/' + subtype) + font[NameObject('/BaseFont')] = NameObject('/UnknownCustomFont') + if mode == 'null': + font[NameObject('/Encoding')] = NullObject() + elif mode == 'indirect-null': + font[NameObject('/Encoding')] = writer._add_object(NullObject()) + elif mode == 'empty-dict': + font[NameObject('/Encoding')] = DictionaryObject() + elif mode == 'null-base': + font[NameObject('/Encoding')] = DictionaryObject({NameObject('/BaseEncoding'): NullObject()}) + elif mode == 'invalid-number': + font[NameObject('/Encoding')] = NumberObject(1) + writer.write(root / f'font-{subtype.lower()}-{mode}.pdf') + +for mode in ['null', 'indirect-null', 'null-unicode', 'indirect-null-unicode', 'null-base']: + writer = PdfWriter() + writer.append(root / 'pages.pdf') + font = writer.pages[0]['/Resources']['/Font']['/F1'] + if mode == 'null-base': + font[NameObject('/Encoding')] = DictionaryObject({NameObject('/BaseEncoding'): NullObject()}) + else: + key = '/ToUnicode' if 'unicode' in mode else '/Encoding' + font[NameObject(key)] = writer._add_object(NullObject()) if mode.startswith('indirect') else NullObject() + writer.write(root / f'font-core-{mode}.pdf') + +writer = PdfWriter() +writer.append(root / 'font-simple-unicode.pdf') +font = writer.pages[0]['/Resources']['/Font']['/F2'] +font[NameObject('/BaseFont')] = NameObject('/UnknownCustomFont') +font[NameObject('/Encoding')] = NullObject() +writer.write(root / 'font-null-with-unicode.pdf') +writer = PdfWriter() +writer.append(root / 'font-known-glyph.pdf') +writer.pages[0]['/Resources']['/Font']['/F2'][NameObject('/BaseFont')] = NameObject('/UnknownCustomFont') +writer.write(root / 'font-custom-standard.pdf') +print('PDF null/absent encoding fixtures generated') diff --git a/packages/extraction/tests/block-landlock.c b/packages/extraction/tests/block-landlock.c new file mode 100644 index 00000000..1185ec95 --- /dev/null +++ b/packages/extraction/tests/block-landlock.c @@ -0,0 +1,21 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include + +/* Negative startup probe: add a restriction to Docker's existing filter. */ +int main(void) { + struct sock_filter filter[] = { + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SYS_landlock_create_ruleset, 0, 1), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | 38), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW) + }; + struct sock_fprog program = {.len = sizeof(filter) / sizeof(filter[0]), .filter = filter}; + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) || prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &program)) return 125; + execl("/usr/local/bin/node", "node", "/opt/extraction/dist/main.js", (char *)NULL); + return 125; +} diff --git a/packages/extraction/tests/container-client.mjs b/packages/extraction/tests/container-client.mjs new file mode 100644 index 00000000..14e70f74 --- /dev/null +++ b/packages/extraction/tests/container-client.mjs @@ -0,0 +1,303 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { request } from 'node:http'; +import { createHash } from 'node:crypto'; + +const socketPath = '/socket/extraction.sock'; +function send(format, bytes, path = '/v1/extract-text', extra = {}) { + return new Promise((resolve, reject) => { + const req = request( + { + socketPath, + path, + method: 'POST', + headers: { + 'Content-Type': 'application/octet-stream', + 'Content-Length': bytes.length, + 'X-Extraction-Version': '1', + 'X-Extraction-Format': format, + ...extra, + }, + }, + res => { + const chunks = []; + res.on('data', chunk => chunks.push(chunk)); + res.on('end', () => + resolve({ + status: res.statusCode, + value: JSON.parse( + Buffer.concat(chunks).toString('utf8') + ), + }) + ); + res.on('error', reject); + } + ); + req.on('error', reject); + req.end(bytes); + }); +} +const fixtures = '/tests/fixtures'; +const pages = readFileSync(`${fixtures}/pages.pdf`); +const result = await send('pdf', pages); +assert.equal(result.status, 200); +assert.deepEqual( + result.value.segments.map(s => [s.kind, s.index, s.text]), + [ + ['page', 1, 'First page text'], + ['page', 2, 'Second page text'], + ] +); +assert.equal( + result.value.inputSha256, + createHash('sha256').update(pages).digest('hex') +); +assert.equal( + (await send('pdf', readFileSync(`${fixtures}/compressed.pdf`))).value + .segments[0].text, + 'Compressed page text' +); +const docx = await send('docx', readFileSync(`${fixtures}/body.docx`)); +assert.equal(docx.status, 200); +assert.deepEqual(docx.value.segments, [ + { + kind: 'document', + index: 1, + text: 'First paragraph\nLeft cell\tRight cell\nLast paragraph\n', + }, +]); +const form = await send('pdf', readFileSync(`${fixtures}/form-valid.pdf`)); +assert.equal(form.status, 200); +assert.match(form.value.segments[0].text, /Form content/); +const nested = await send('docx', readFileSync(`${fixtures}/nested.docx`)); +assert.equal(nested.status, 200); +assert.match( + nested.value.segments[0].text, + /Before nested table\nNested left\tNested right[\n]+After nested table/ +); +const nestedOnly = await send( + 'docx', + readFileSync(`${fixtures}/nested-only.docx`) +); +assert.equal(nestedOnly.status, 200); +assert.match(nestedOnly.value.segments[0].text, /Only nested text/); +for (const name of [ + 'renamed-valid.docx', + 'renamed-default.docx', + 'renamed-case.docx', +]) { + const response = await send('docx', readFileSync(`${fixtures}/${name}`)); + assert.equal(response.status, 200, name); + assert.deepEqual(response.value.segments, docx.value.segments, name); +} +const primitives = await send( + 'docx', + readFileSync(`${fixtures}/supported-runs.docx`) +); +assert.equal(primitives.status, 200); +assert.equal( + primitives.value.segments[0].text, + 'Bold Link\tTabbed\nNext-\t\nEnd\n' +); +for (const [name, expected] of [ + ['horizontal.docx', 'Merged text\t\tR0C2\nR1C0\tR1C1\tR1C2\n'], + ['vertical.docx', 'Merged text\tR0C1\tR0C2\n\tR1C1\tR1C2\n'], + ['rectangle.docx', 'Merged text\t\tR0C2\n\t\tR1C2\n'], + ['omitted-cells.docx', '\tMiddle cell\t\n'], +]) { + const response = await send('docx', readFileSync(`${fixtures}/${name}`)); + assert.equal(response.status, 200, name); + assert.equal(response.value.segments[0].text, expected, name); +} +const mergedNested = await send( + 'docx', + readFileSync(`${fixtures}/merged-nested.docx`) +); +assert.equal(mergedNested.status, 200); +assert.equal( + mergedNested.value.segments[0].text.match(/Unique nested content/g).length, + 1 +); +assert.match( + mergedNested.value.segments[0].text, + /Before merged nested table[\n]+Unique nested content[\n]+After merged nested table/ +); +for (const name of ['merged-large.docx', 'merged-large-vertical.docx']) { + const response = await send('docx', readFileSync(`${fixtures}/${name}`)); + assert.equal(response.status, 200, name); + assert.equal( + response.value.segments[0].text.match(/x/g).length, + 600 * 1024, + name + ); +} +for (const [name, expected] of [ + ['font-type3-unicode.pdf', '\u03a9'], + ['font-simple-unicode.pdf', '\u03a9'], + ['font-known-glyph.pdf', '\u00c1'], + ['font-ligature.pdf', 'fi'], + ['font-emoji.pdf', '\ud83d\ude00'], + ['font-composite-unicode.pdf', 'Afi'], +]) { + const response = await send('pdf', readFileSync(`${fixtures}/${name}`)); + assert.equal(response.status, 200, name); + assert.equal(response.value.segments[0].text, expected, name); +} +const fontForm = await send( + 'pdf', + readFileSync(`${fixtures}/font-form-unicode.pdf`) +); +assert.equal(fontForm.status, 200); +assert.match(fontForm.value.segments[0].text, /First page text[\n]+\u03a9/); +const noResources = await send( + 'pdf', + readFileSync(`${fixtures}/no-resources-graphics.pdf`) +); +assert.equal(noResources.status, 200); +assert.deepEqual( + noResources.value.segments.map(segment => segment.text), + ['', 'Second page text'] +); +for (const mode of [ + 'null', + 'indirect-null', + 'null-unicode', + 'indirect-null-unicode', + 'null-base', +]) { + const response = await send( + 'pdf', + readFileSync(`${fixtures}/font-core-${mode}.pdf`) + ); + assert.equal(response.status, 200, mode); + assert.deepEqual(response.value.segments, result.value.segments, mode); +} +for (const [name, expected] of [ + ['font-null-with-unicode.pdf', '\u03a9'], + ['font-custom-standard.pdf', '\u00c1'], +]) { + const response = await send('pdf', readFileSync(`${fixtures}/${name}`)); + assert.equal(response.status, 200, name); + assert.equal(response.value.segments[0].text, expected, name); +} +const rejectedFixtures = [ + ['font-replacement.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-partial-simple.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-no-resources.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-graphics-state.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-type3-unmapped.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-simple-unmapped.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-form-unmapped.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-partial-map.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-empty-map.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-bad-unicode.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-composite-unmapped.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-codec-fallback.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-composite-partial.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['font-missing.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['merge-content.docx', 'docx', 'UNSUPPORTED_CONTENT'], + ['merge-orphan.docx', 'docx', 'INVALID_DOCUMENT'], + ['merge-span.docx', 'docx', 'INVALID_DOCUMENT'], + ['merge-legacy.docx', 'docx', 'UNSUPPORTED_CONTENT'], + ['wide-table.docx', 'docx', 'STRUCTURE_LIMIT'], + ['control.docx', 'docx', 'UNSUPPORTED_CONTENT'], + ['control-cell.docx', 'docx', 'UNSUPPORTED_CONTENT'], + ['control-inline.docx', 'docx', 'UNSUPPORTED_CONTENT'], + ['control-only.docx', 'docx', 'UNSUPPORTED_CONTENT'], + ['custom-wrapper.docx', 'docx', 'UNSUPPORTED_CONTENT'], + ['revision.docx', 'docx', 'UNSUPPORTED_CONTENT'], + ['simple-field.docx', 'docx', 'UNSUPPORTED_CONTENT'], + ['renamed-entities.docx', 'docx', 'INVALID_DOCUMENT'], + ['renamed-default-entities.docx', 'docx', 'INVALID_DOCUMENT'], + ['renamed-case-entities.docx', 'docx', 'INVALID_DOCUMENT'], + ['renamed-utf8.docx', 'docx', 'INVALID_DOCUMENT'], + ['empty.pdf', 'pdf', 'EMPTY_OUTPUT'], + ['empty.docx', 'docx', 'EMPTY_OUTPUT'], + ['inflate.pdf', 'pdf', 'DECOMPRESSION_LIMIT'], + ['aggregate.pdf', 'pdf', 'DECOMPRESSION_LIMIT'], + ['raw-limit.pdf', 'pdf', 'DECOMPRESSION_LIMIT'], + ['form-limit.pdf', 'pdf', 'DECOMPRESSION_LIMIT'], + ['form-filter.pdf', 'pdf', 'UNSUPPORTED_ENCODING'], + ['aggregate.docx', 'docx', 'DECOMPRESSION_LIMIT'], + ['bad.docx', 'docx', 'INVALID_DOCUMENT'], + ['entries.docx', 'docx', 'STRUCTURE_LIMIT'], + ['duplicate.docx', 'docx', 'INVALID_DOCUMENT'], + ['forged.docx', 'docx', 'INVALID_DOCUMENT'], + ['forged-bomb.docx', 'docx', 'DECOMPRESSION_LIMIT'], + ['utf8.docx', 'docx', 'INVALID_DOCUMENT'], + ['entities.docx', 'docx', 'INVALID_DOCUMENT'], + ['bomb.docx', 'docx', 'DECOMPRESSION_LIMIT'], + ['output.docx', 'docx', 'OUTPUT_LIMIT'], + ['encrypted.pdf', 'pdf', 'ENCRYPTED_DOCUMENT'], + ['too-many-pages.pdf', 'pdf', 'STRUCTURE_LIMIT'], +]; +for (const subtype of ['type1', 'truetype']) { + for (const mode of [ + 'absent', + 'null', + 'indirect-null', + 'empty-dict', + 'null-base', + 'invalid-number', + ]) { + rejectedFixtures.push([ + `font-${subtype}-${mode}.pdf`, + 'pdf', + 'UNSUPPORTED_ENCODING', + ]); + } +} +for (const tag of ['dir', 'bdo', 'smartTag', 'unknownWrapper']) { + for (const placement of ['paragraph', 'cell', 'hyperlink']) { + rejectedFixtures.push([ + `wrapper-${tag}-${placement}.docx`, + 'docx', + 'UNSUPPORTED_CONTENT', + ]); + } +} +for (const name of ['wrapper-block.docx', 'wrapper-run.docx']) { + rejectedFixtures.push([name, 'docx', 'UNSUPPORTED_CONTENT']); +} +for (const [name, format, code] of rejectedFixtures) { + const response = await send(format, readFileSync(`${fixtures}/${name}`)); + assert.equal( + response.value.error?.code, + code, + `${name}: ${JSON.stringify(response)}` + ); +} +assert.equal( + (await send('pdf', readFileSync(`${fixtures}/raw-exact.pdf`))).status, + 200 +); +const exact = await send('docx', readFileSync(`${fixtures}/exact.docx`)); +assert.equal(exact.status, 200); +assert.equal(exact.value.textBytes, 1024 * 1024); +assert.equal( + (await send('pdf', Buffer.from('not a pdf'))).value.error.code, + 'INVALID_DOCUMENT' +); +assert.equal((await send('docx', pages)).value.error.code, 'INVALID_DOCUMENT'); +assert.equal((await send('xls', pages)).status, 400); +assert.equal((await send('pdf', pages, '/exec')).status, 404); +assert.equal( + (await send('pdf', pages, '/v1/extract-text?script=x')).status, + 404 +); +assert.equal( + ( + await send('pdf', pages, '/v1/extract-text', { + 'X-Extraction-Url': 'http://example.com', + }) + ).status, + 400 +); +// Exact input cap is accepted; one byte more is rejected before parsing. +const padded = Buffer.alloc(10 * 1024 * 1024, 32); +pages.copy(padded); +assert.equal((await send('pdf', padded)).status, 200); +assert.equal((await send('pdf', Buffer.alloc(padded.length + 1))).status, 413); +console.log( + `Container endpoint: PDF/DOCX success, structure, input limits and ${rejectedFixtures.length} adversarial fixtures passed` +); diff --git a/packages/extraction/tests/container.sh b/packages/extraction/tests/container.sh new file mode 100755 index 00000000..839670c5 --- /dev/null +++ b/packages/extraction/tests/container.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/../../.." +run_id=$(node -e 'console.log(require("node:crypto").randomUUID())') +project="extraction-test-$run_id" +image="librechat-extraction-test:$run_id" +export EXTRACTION_IMAGE="librechat-extraction:$run_id" +compose=(docker compose -p "$project" -f packages/extraction/compose.yaml) +cleanup() { + "${compose[@]}" down -v --remove-orphans >/dev/null + docker image rm "$image" "$EXTRACTION_IMAGE" >/dev/null 2>&1 || true +} +trap cleanup EXIT +"${compose[@]}" build extraction +docker build --target qualification -t "$image" -f packages/extraction/Dockerfile . +"${compose[@]}" up -d --wait --wait-timeout 30 extraction +id=$("${compose[@]}" ps -q extraction) +volume="${project}_extraction-socket" +# Inspect exactly the shipped recipe, not a relaxed test variant. +docker inspect "$id" | python3 -c ' +import json,sys +c=json.load(sys.stdin)[0]; h=c["HostConfig"] +assert h["NetworkMode"] == "none" and h["ReadonlyRootfs"] and not h["Privileged"] +assert h["CapDrop"] == ["ALL"] and h["PidsLimit"] == 32 +assert h["Memory"] == 768*1024*1024 and h["NanoCpus"] == 2*10**9 +assert "no-new-privileges:true" in h["SecurityOpt"] +assert c["Config"]["User"] == "10001:10001" +assert len(c["Mounts"]) == 1 and c["Mounts"][0]["Destination"] == "/socket" +assert set(h["Tmpfs"]) == {"/jobs"} +for flag in ("noexec", "nosuid", "nodev", "size=96m", "nr_inodes=1024", "uid=10001", "gid=10001", "mode=0700"): + assert flag in h["Tmpfs"]["/jobs"] +assert not c["Config"]["ExposedPorts"] if "ExposedPorts" in c["Config"] else True +print("Shipped Compose mounts, resource limits and security options verified")' +docker run --rm --init --network none --cap-drop ALL --security-opt no-new-privileges \ + --mount "type=volume,src=$volume,dst=/socket,readonly" "$image" \ + sh -c 'python /tests/adversarial.py && node /tests/container-client.mjs' +docker exec "$id" node -e 'require("node:fs").readdir("/jobs",(e,v)=>{if(e||v.length)process.exit(1); console.log("Endpoint scratch cleanup verified")})' +# Same outer boundary, with a test-only worker that attempts forbidden operations. +docker run --rm --init --network none --cap-drop ALL --security-opt no-new-privileges \ + --read-only --memory 768m --cpus 2 --pids-limit 32 \ + --tmpfs /jobs:rw,noexec,nosuid,nodev,size=96m,nr_inodes=1024,uid=10001,gid=10001,mode=0700 \ + --tmpfs /socket:rw,noexec,nosuid,nodev,size=1m,uid=10001,gid=10001,mode=0700 \ + "$image" node /tests/isolation.mjs +# Denying the mandatory isolation syscall must fail closed, never start unsandboxed. +set +e +docker run --rm --init --network none --cap-drop ALL --security-opt no-new-privileges \ + --read-only --memory 768m --cpus 2 --pids-limit 32 \ + --tmpfs /jobs:rw,noexec,nosuid,nodev,size=96m,uid=10001,gid=10001,mode=0700 \ + --tmpfs /socket:rw,noexec,nosuid,nodev,size=1m,uid=10001,gid=10001,mode=0700 \ + "$image" /tests/block-landlock > /dev/null 2>&1 +blocked=$? +set -e +if [ "$blocked" -ne 1 ]; then + printf 'FAIL: missing isolation did not reach fail-closed startup (exit %s)\n' "$blocked" >&2 + exit 1 +fi +printf 'Unavailable isolation fails closed\n' +# A stale socket left by SIGKILL must not prevent a safe restart. +docker kill -s KILL "$id" >/dev/null +"${compose[@]}" up -d --wait --wait-timeout 30 extraction +printf 'Hardened container restart passed\n' diff --git a/packages/extraction/tests/fixtures/aggregate.pdf b/packages/extraction/tests/fixtures/aggregate.pdf new file mode 100644 index 0000000000000000000000000000000000000000..70f476e3c7eb8a7c6236c212119a3d2c9d29752b GIT binary patch literal 37605 zcmeI*KTH#09L8~Ud>u4l(D7YJBt))v<$5h4kN{SV8c8u~Vn{sb3)a*gxgH8mXrgg3 zadE=INn;${j0~!_Yz9)z4zYtcc0$&$jy3!j>r+jNcG_eZm?EgW}AD3gz+Xx#{9$r|9n&s!}wEotkbJ=cbjkM*JyN z71mHNQ;DQ2th1$JRbG{@?pTms{F#eCZ8Jqd+UfY;Sk0iOC7F9=&>=ta%fXZ`#f&b^ ztc98rdKym12bD-zqch`CT$CBb7S^C&RpyAaPOI`Q6_qCZno&%N8>cn139F#0!Av-z z=QSAd`SBN9nt9_oG>COx6-)kLFsm;%)t!p}GPWbVoFm6f;i1wKuT-veDv{o;%1koC zs;AjcYRH$$kw&-jWj|8GY9c7=V$P|`RCHZ>cGfecSEEq*H^uDJ@9*xm>1|#<`P+H) zr7+*w5I_I{1Q0*~0R#|0009ILK%kWc=9ll^Zf`pv#kE&&3ta<`7N50$K6vlS!%II7 zzj^TZNGo4D*Mzw1WQ)6%Z`%4n&kc{F#tlgoB%FKdexeuz z5I_I{1Q0*~0R#|0009K{m;edq9{WKk2LS{SKmY**5I_KdtqU}naK=mKCgI|)MV$4k z?dnBs*Mn9f&aK}CjDP?F2q1s}0tg_000IagfI#aC5OKEdZ_AY-fB*srAbT7nKo^+<}T*q;} z`Z=6jHf`7U%jGgP<3*ugDyvZR^$BaVbVnIKs}KZ{G>LIRSZ6Aefz*SnEXP}o{4k28 P)1KpGL`TQT3wiMm|Keg% literal 0 HcmV?d00001 diff --git a/packages/extraction/tests/fixtures/body.docx b/packages/extraction/tests/fixtures/body.docx new file mode 100644 index 0000000000000000000000000000000000000000..85a4932cc073c3d721bdb7f4d90b5496aba1ccde GIT binary patch literal 36732 zcmagFWmsIvwm*yqcMC4T-QAsF!6mqB87GBBn9N544X1qDk=Bv%Gl67HCXiQWiRDxQ3UoKl zAEhy^5&_5jIV=6nk7T7@sN1)hT2DgqR9`2Mgls?aEGWKF>|&(}^$M|IVs zZ!|j$R(qo=?$6@^9uy#nK&hPPC1z^PY}-u4_8{K0D5DDv>2RaQ#6g!j_X&muXG;yI z`jx&SRL@}~0Q$_XeJ``em=?zoTzy&0jhgN{rcK!HDZn}J)d_1pCM-mi%z`RhsWj;wIzfMU*A=)6(L7)3e)W>~e*q`|K z$U)V2vQJ&bdu}Vi`KAhtDRg?z$;5>iJ)eyqeIAfXD&9&8_dP9^tnG=6-cjT~Vk;Uz zsQhCTy{O-d-h)SP9SQ;h3H;a4+1$>Bh57eqRpNv^G#h&0xvNMa`-2xGglvgc;VchV81ddam0PzvKu1eOizE{GSDi3`bJ422b;8aDgzu1u_GesyG82U06&3&gQ=-d3NHM z(f}*E)Sa)?oT6sbH9EYM96jXGCwW=ypry83PA)~eu^jG+jh#Y!-8=?&k_~~8Ye(TV zo~9OP)3C%a(umLJL1v$jbZ}l)OmJ5pbEN3`s**$d&wC$cuc$5gO-Fm9KM)`z*5*LV zWwPljEJX5C4%TyV&V^5sXSb>re?1ge!%$of*Az{pZKp&b+ry^&6%<4gl|-4?fKk>` zOahUGQTi1!DrL}EVbTC*cBuj(#CgjO#+{&AS9$wzU5|}mr5R<#wYL4i^olLOpI8?q z^h9RJNOeEUoVFH@o#@mZ1dEIY$szh7UDcyilYR5ksf)AI#*{m>eg$$%1Tt#M+pp87 zrIn(=U*T8v+}8nvMRg-{bK9#-E>wXcu4b(qwOG&1W4tt}#{OArUr|Rqge)TvvcKHj zTw1ebev!j25zB&+YnPp2?4HO!YzTOjd3n1;#Bk4Z34?!RiE2PM>ml*-)PF_z&vCg~ zS;0F7kBc1)1O)ou$Hmyu@sD9qA9Gk?MelrMfYEpTWc^0$1Ab!gg4ndg8bNJhR?FNd zh5TojHnwelKTsqNUpNaeN}xgDv2pZqQOx2^qmq>d&KM~Fn~KvKVpGkhk^Pm40}2|Cs zN}agZ-49+Qt28aGr+r{m{!{_|Tw_(?RXQcaC9?xO(n(21L6wFwe&&D~^Qy3>U5=E- zQzRo~GwQk+w}P=j@^J;$oBhJbv$WU46P%?KJtkpvyqZP6A4J^6GcDW{_5T5scC5%g zqg{c(`_y>cK&Dn-rWJV=dz#RrEjE(J#|8Z?K^yaZhBDd47|?fvk*giF>XguW+fXKu zfcpLx73M)eeCY{SlK!W}8(rdrq~>(2fVxq@^$8Cza#Gb5v^M0YtKG-_>{_fZqSi#q z?HKR;CZ~m^>c(m-BZvYauHy-Q*DL$ zw?Xa!XkfqC3gN5Bxa)3{-dPbmqcdOHvmI=L+=P&hJm7f!TN@BaQcn@=p1;S8kp#8c zZKNg(PiSqt6Nt#d)9|rV{+v3vA}3O$EYsDQ$@0WPa)FQux+Z*`2cD4rb7F{&HFV~I zFIGwTKQCAhfU_A3_^NfY2QN;)XGW36d%y}Sa`(9we)sO&Po;WBqzr7mPp+`sWL9^) z4E1(M$x>p8Nftf+U!LTDWd^(EV0}3egl!U#k_$`!d7Lz25l}* z%@4Vv zMDe#`DNdvL#D|;&!*VAc^3dkPM(tFe^-TKVszYag$cwL>vc{E^x-1(DTEcvibG68O zZ}MTDPx^2~UZ1R(A_dU8>3&{IO+4;SqNzJc5EC625uHvRwA_q+YV3Jxr`U*LA1WMX zM(jM>xH=oni6x6)NY{Gwcx%Avip%t^0+tlJL+$&0=hqFZdWmW19C4AV5cv-r0mN)k z%4v}yhZIU*9kUwDojpPB=e85ZF4x!IJ{4b8g$P8@PL+h+weQiN&@?#;24=>+%t#RP zFQDmvoiL~7eU~PqJn`IgFl+5yw1)n*F1Lq!irb;>WN()jN!>=oU;VH~kMh)0FwU6Fx@xO}4SF7gK5LrguB@&R1TI zha{WI}aVsArl&5TiEJ80*Q+b@^Kesmy3 z6JlK()nGm1yQZ%WqHQ?Uv4w}yYKh_XJi1UilGBN4#u2$WuHGFKmlM^kS6!91n<8r& zE`6Iy4VuCI7%+89^G5oVTvSHe=LbK4_hv)^&u(u?pM$GHx2Khzq;CHFoEy+-ckXXU8khNe zw#GBigVNomcdq$ZGI4g@0q}pu_Z0x`pLyz@<)o}UJ3OCdiY$%*UO@4$pvT4YxDt_; z;~o8b(As53>U_uJ!R#+!M?}QVk=Jv*;EVlh`+69Q|4&kbl#+|7frE(Dy{)m8ouT?vq5)snMzTPh7aK6Zx`qvxl23?=W`IRwmk53}6j@p0$jJuAV>wTc-+tZ3~ zR)%*GC5r?8O~NDtmj25*BR)C~?u*9ZO2&Dq6GycoXCm1zxWWp~GM6e_BM~9@hAbzYks^xo<+nbgif{A!J8R-AAaco5`W=$0n`CQuSmBYvzKSb9%DN; z4Iq4ZlsBHsp{8BAr*WB&7s3FnH?MB5pQ|F0;_(4t zEHB-w-5o8JW648o0rnlMe%&AS&byz15)T%tsU!nV8*40R8~$lhq`tB_r{u4LHJ~1o zzM8Rl>Xe2jSKSDIj0`FQ*A9fPi**Nl%(wf@VU8-hYr9_MEfsXS*n zJj%zfeg^0MLe;qri?3LN-6=7xKYLD<4p!m;+xNS#b-F#$IfUKqdeYn=x`$qF?>jas znwR^PeRrM%B)+oCq1TQ`BI;WcIWGg20{k9AO~j0C`1-URc@*Ocw5?J!HdWGo4N{Xz z;Xz5iNIKuhS${Y8?wgTQ>meFM19ebT@T|Ij{w8H|E^Vvi9t|Ri3b?6{LF7X%0Q}!R zrdM?nRN4C@T1JwVs!-=0 zfCIm3H~gxxO?7Nco>CVN_MBWQaSKZy;xR7T9v6+T)#+>o(=G2qUOV-x7p5LhXCEBz zoSsMIZm&g7&xNx-NH^)v&s_}cUtj+^8*s_uWC&{`D$veisGQl78f^pfBhr@PQ*F5vuPDZ6NJ$doLiX<7=Z@Y>O2X*XS8a3OpvTZf^KggD1s(#_92xZ*EW*~phv?vdssMOw58L&aba<4hsIfA`V*~yk!0Qtf0GPmWq zj37<-6XVO{nst{z_4#1o@~2uhjt#jbZTsyc^g?cEd3e=KL@JoR14$?HqojgR#wHzA z@r0kQbmZ`s>)5@9@YltG*r8r(qKH2`>Bx}m*DJFNWZ_jQ88Cw#6I%kwvUc?a;ym2}6>|gQK zO$1cTFf3k3-}>d&q5i@?m#14!@-CJXtr1rkme62r@t?K$mo(Be_=|Q7R~v=1W3p zYqMwfMnD^u-jclwk)-zAzEj8Y$e7&=V~h~9Udq$?(Ap!=txexy$VcpHU~%02tJRX{ zwch>kY;Mh#XU(s9VdaCp{qOl(_puuy`Yk|7(m020=4H|3k8$T*p9x+c9g!uNyrzuM z8CT1Q-`!!N={R9&)lHjLqruiq9gNkLj+mF;1)OVpG_IX=f2_`x(G_G#ieGDV_Zs0` zW=d$H-Es2u;7xyXz(e!cr8G8Tdi6T?n9qr|ZhC)k{BnP`?nEPZCfjv@zEyCnSKKTh z`^EKbO>I1*dyN@hy32D&M|4cViAHJo9(`Na{K{oxKvXs*0QhC*iki2_)kDo{67_p? z*s*-Q=Pc9+y$0@9nc+fa@bk;yMX~b182h>_NmQrYXr`+CSmx$aVmS{ioy5Yqq8uU# zd&8`kfN1jj(}jR6NldMG#dFJp92fK7yDJxKM8d8}nFF5CGhXf!fJr03l4tU{K#r)c zog?ncm0Li`H+}zmKOdYg?rEPUddFTnlg}01hz{FYnlC&+G7)lyTp4;D+yV{MoQlSIBi$0daEvC_d z${GBo3*^(P=V>2p3zz0i#Zm}&i9h)0rRDs#dY7U#al(y?a5mXaLjU_wI0GGj)PNi2 zn^`#`NSZbbCVRi(g?UTvgV*LQG0N&Uv8+N?b0}!eH4$qoxoYe6=;4nn1g758t%L@<1Y1&*WDgTQ({yt~6A{7=`OK0o$aoo! z`c}ce#FHFae`u59scwFRhC3i2yPx@T{9aALFvEQymHOcx(}cg@xlutb`8pZ7fjYxO zOzDm+?h$!tH6mc6q|{XDyU+-+n|#mv_UH{&je#oni`gA+x?&NBo(-v&XRb1j-1D1X z@q5pEvC%;H$DOOEvuB3asuf_#ENFhqsNeJ1DS-df@IE&l>E;QNq@L&6(GR}6+XKK7 ze=Kn&MRr%w(ds1qVeKHC;8RA5Vv5Kqi+(gdkE>VNs!)-0D%?h%e#IS##yUHym(#V(m(Nu0fQo(q~slPQ(6MBhe<@Ozf~`8Yz@(g6;imrquW ziP5oVA-j2;(;ZU6g;eHF%lqI23WhklAhpNtdNJ5vRy_c zE9G$lQPLYojvc(2SL->En7UIA7-ovOo+9jWba40xEVWMULXN2Kbmk`X#;WOj$J)q_ zv^Ym8a1HaZs5Jam;}nN~n-0cif1AH)g;JwN-Bd^nl*nnPX875uzPxVoLHQQp{Z05D z@)M?*;=0O58+V`bqalk*8CpXPP18-ogZ33)}K@c21mCX}iX_Ir%l|=wI}@V`$H_=NjZ>U@@eo62{}QHO;9O%T8~A^$M)k3HKBDq2`c_a)DoS3CY|qLKA%hrs#=pKIKtNL&eHXrUaWxNj zN4zt(NZ#q8ecM~+ITBij#CMIl@>XND3~RDl&Rf2*@aXiMJYs-I?k(3@3#T2 zNt`={b1_)&IF#CrhTXX{<$k*8+Dy?U3YF;BmV^^_FTq%U&iatsag3`U!-yOvvzbu$#teQL+F7sJLwHXZkhEL>F_dj788)xb+FXqE_@@-- ztI;Cf7g9wV6rm7F@c~+E%28$>v#IbQT7kl$WDU}j_FHV;QXj-6LZ2#x0%3uso$mHa z;gM^@`L9)P1{jwr=M>dxW@98YJtNdFc*-Zo#rID$GDWZaK5JZmLvOR;d8?3UPNLx= z9Qk3-FQ%eIT*PuOl%WY$l*6MP=(~SEdx}!-T~F@KAKn%~2*nw0fg3X2yJfXsFU*nJ zHox|%rp5%%-aM=<7+V8a$MZ-=&vgVnx$c^FgkD`KO{f#g*c90E@x{lUk4li1zEv>C}E4qPaLjZ70fx{0@P_oy%b<+C5s2_kh}X5$rjD zb7rCm*$@phRY?qMN`9e|Sa5ifUR17A|1U?KS*m;B{+@}&?vg#gJ%Xg<^FpnB;ctTx-w3_sxk&j6EtJ8UUwI})7 zvVkr{k6ChfBB4M>FMV5m`8?K)TCYPx)r62U-77tP>X^%7z?go(x5J&S{Gy0Paz2YS z=I`Ze4K_rTTl#BSOQLfU3E|$zvo9Adz^zW_y8b5x)2(B!#`9~petrAJF$t3O&;b9R zcBc7w&jwW_gLn-MX0E3necDLK-Z?aB3!UFZ^mx~h&~iSyr+jgdC;z#p!gU)&lVQpw z8qdZtozc5!kub-k3iocXcx{z=6UC~bBal(t-b_c_VOSJv>K1BaN-XIqYuWN7yqrwx zX80%6h$!GB+cBe{rOdRWdEl|SRN_!^^F5m^R1HzekIXoi&On~LiA$(@_^!|92wiWX zm7d5#X#VV8@k+ZyO*SN8PSk+yg#((&z+J{TMr4w z^4zNpU3bmef$fB}kHbrlI*es}e>g$pxc$)0V-k!d;L_A`h@vL;>9ERkm>a9B4sCUT zs4EuAl0T;o>qphQMaUcOx2==o+d{OwL0YKq4;4rF40;KF?0Z7$gsy+94_%Leq%qi3 zd_us=;!UsclLC|?s!9SV*>1SmF}juqqByb}uzsKjdO&pXT`!w#U6EjC^)f_I-4yQ- zv23DvRuq(*V|Y`JVS01C$`nkfy;s((L zyHj@T$XX+1%YSkNtu1{Eq8YnZ_wMTmYPB%K_IQlca>tE>xf2GSP|4zR_ zKhijD`09~FlCY?DoXecQo~6-H#qbd0O}7~am7~HkLGInE-c_1v7hxuVGdFf@|3$j% zVpuxVTjMe5aBUTS7dXdW7G26?Id3->FI-@N==n8}X70U9Bnh(y%MsNI`LYr$@XA}f z=mSe9IrdkB`EF)!p0MDm!% z-d#7;@W;)dAuuO4iCK1PQ0O_NS!nU=LL~}#6 zHXxj8?oj2DsECPyI_g+VB{O1^w*QDLtJc)^dZsK$5SB$hQk0WuzoL-e-HtEWv*Nl)>?QK$``u%!UM;|O{oav!vmXWYgmqB8S6a+YsboUv^aQw!Szp2cgdK-k?@5WCw z6r%(G%|k)<7mo*layvY{NNmKXK-f$*9{(V)B~h@Yf-g#F`S}swrze=8jxg74G~)Tu zEWT3YnP8}lc;-3gQRIc+ZNz_$lYRpy;8wUCTT#5O79L*lei={-bVl~Th zhK02ZF%o>0uyQL3;J*oEv4UNZD#=Rq22cEQr1&2;RJFo?vQ3p0{C}}YiVY)mMv1h= zWeLd$_-V|79Z6THE1Tw4)WFLMmqbm?e}LJ|p3h>2>YuVHrZ93s;?17d7NW1D$vbSV z&%FI*XHD;oPqg-w6POX&VuIC35Pw=kk!PbUin0^bAM`1ZU_C&DnsOpod4+;(S!u?d z9ro>5F6^?BT}#lc78es#BA`(o$a&jF-~*L7^=r$;4I5HFh+=c^!jvA00vqzzq0PU| zsUMcw(?hv;9vVBixOVA3I!7nNOpVgEUqrUJRA^a$#glD!o{=W=f#0Qus zr{_j5AJ9doFXDNvC+7uduVdFO%q57$r>5-5g8_t^ztP#$AQO_S)+-3HxKLysD1SPm zN|x+lO`W<=?-Xe}<~VmP}#KnSw()GfvjVdLxNwb`mE_cLg2Fw9?Umlq%0xU_NnnOqBRH^8o1f}<@0tY{0XJb#l#_OG=qaW{QJw}K@-Gf zM~s*QQ>6lZBzwdlS;%ZxxCCgv2V^oh_t1_;G7~m^V>aycQ%MWIu<`M>@MSEP$8nQy z(Pi`V5^W zPG%y?C~bgVmsVYi|7qNvqJbGbK_HFJ$4o0%2br%OGERw6dIO6z(}c}Co3E5AnvTS8 z#FXs<`XX`&yMBu^ZG1l~BQAtHOtdf%H4EljP?jhHI4ZO zPNP5LN-`ERFosh!WPhu)Hv}IVaBsHu{+mh;o{ZlrAM$>y6z&D9q=W~n6qr;z2w|fk zY`0Q_mqaXGAr(a}d?pn|KEa2!78^8a?>C`kUx^xT7Zt5;v1~rJ{TpKOmLe5v*H2B` zArpEJ-~i9Hz%v!o>g*@x7Bs;VTHp;#N-WeE6v^jsxW&S}UFtRj{#5JIZ2NUZ_PEd9 zT8j5|1I1HP^UF6wdHoU=ic&xg<7|$%&W+9H48{8G`PyYW88@?`JW>Ad>%||E;RCA+ zrt)0S*Umgz;x-d!qIj}{@SHzAdTS8b!qHeQa<^gI2ilkM^ins=*(EJmEUG4sn=o8w zpkr+evI}1-?kI$l)C}_4eqeG^+Rj>odppPnob6&isC+R|>(b}@-8jhCk&^h1+7?<* z8tVX@$HynjK!sXEor?V}kpa z?GwbZ)I`=8TqZ|D2pNGPgpn?Pv^RZu*AVr+b00LwR~Mt4HOrgyb%eobQdj-`ZzU~a za863kt~KL)4DrjgbMeb$|5k{6%6eW3y`5$Cr^4F*R+xH_C1Mx%WYIJojfi3+Gm-E< zMPiRXfd)0%K`h6eY9ftps(_vZ(H&m1shIzp3Dyp2@bj5#%+uK)EA-Us*^4=7<9X|u zF{WR3n?F>xVaFofe_FX(szyWxVCL`aCRoHg#iLAU&Z(E=7sMwtaa%sU!|GUU8t+Dm ztK_`MWHZi}P<6*IBVg~q7U}Q8dgB}M%eE<-YjHRhicLnN;ECf_r|kB!!RF(5d9gzt3{Z342%#m_88@Yu zkPeHnFN~_qA|_qCag*GDkFgJPz&I))tOMo!&zNlC0ZeFM9NhaSvaHkPO!)?_l?bZR zmU6iFt*6`#wJ__pNk$0FjNgIBql5V6nxb8zp1tVpiTi(3u^zd`U#LP*NN-Z_SJmI!)2~Fn61% zmi1rLwTb#9I-RwHslo>^$&ow=QySk2_=Qu!9TB6Y{IScPXk9C?_=~(o0!*ICkNyXF z>{l@P*8k}nYRaSYX+Py(_BpF40DPm1IL`*)P*kTU>oQ25ZI4dvG~#<2xFXE> z)POl~KR(&in0p_Ub`Y+2m!JCGC(egfm(k3h%7qac3jpg*n&}m&hi!w96-x1fi`J2X zXm#GuruSM{fOKlAh-b)ZXU`2V;68BZToa;Y9=)?nbW*-ip!B9T5>VsX;1x^CDd#`B z3Ri2qeN41cnmN)!pXNkYthn!2=;A@1ZbbdT8MKdjg8cX?0Qa4Ke!RFAo;B5UY$zP;7*f+YIOTR+@}r3`MNk?6M3%RCbhtClZqwQRd-wq*~RnK|1zofQ}vul@lTU{ zmA_4n==?T$ZvX#ha==>m(Qlor#2N@TnZc^~`z_hv=>>jQ>5nC+Rb&Q^s2$#r!&Jpp zX1pl}5wzYtVe7YfEainoqUF6a;?8`P25dd^48tnTG?|tf4=;#GVO-)pq7jI}HX@PG4*F%@a4C2(*j1_680cML;~BK# z#~F>cxh8|ag6)$Iy8}vVkXtti7b-O+5Ct!jG~i`2okSEFo&}7y4f`7n$wuf0e+(WN z%`>Qh$BH|rt>0jQ+Ca2RgR8UGJ|uMr>4Muc6c5o@qy;Y;!Lp@z0@(-NAW$}F6E>0v z!XPjbatpSVOs+-0s0pdc$?9WK=O zVC20ln1;qzL8bFIama~aal>Sxc%KX9Tw8AlGuOl(m7E8asD*w!75{`iOKwFBpoynj6#k8^{F; z)>X~67lIW63xbIY9Q;}~9N)+ugf@5R9j&ZIZMX$)@b#JT9-KA6vI_=@iJf|tM}*z% zl#$>p1j@L6T#gga*trcoZmM;1!?pN1`d`>wTuLJhNs9| zT{{zd+R3~lUp;mOYkyH3>kDq0A?u~k2A03Pi9cMRo{7IcCls3qQHS>WYG!l^5I5cM z5hZahJ(pe{G9HNqWv==+j~5dC`z}}D=PamthA*=Qo!1TINbSy zegf^tIEIy)e4Ev7)}H+pou>@#8k9dftr4#)uQZMg2Z~O9qV%v({dp9aVnxT%B0#a7y_V2E z7P^4Xpr(yR^=wdCKK4bc(VH^EKd72(Q@r)G|Dc-mZPso*fhQJA&+X02^Yfn{5WfX! z#3Qa8(C}BE2=|mbJ@!9k0N7S0g++MM5JjliM4)?_*m{@=g3?Go>361|ziIZ=1-RrI z9-o$A+=A$S6J7ZEH`%Va?rW`YuBZYo6vu0z$64@|^&_ypGJgRfAS2rF8VE5K_ILtI zYu&f=cShaOY6x0r#%sbXw(NZm4&I=az)X7o$545`%$cUPr&)JN?JvHFLH$)f0ohYs zo@7w+`FRN7IFnoACMI7|XNUSMyMZ~9KMDmoU)e-Lg7(YOw^|q~j%K?=%(ophuH|6^ z7~FiUjdT$*`3tJ=$f(0R_wX^V1?M7Zv5*;^F9e7o53_WY7q#3dGtRyND{L{U9{lZ@ zb+={8jpxy?r6&U(9z-?#U>3dkadprY^)(cKF(0z~ZGUj1sETVOpR&pBD~PGWVHCe{ zG?G$2Y$VZpZX}_<{1BXnX*lxUw<;^D0BMFtmxOc&2#4e!bRdugVAUSUR_jx)$twva?Jp#WDXAXV7Xe3d-u^Y1 z|66V92`BJ4CdEJCdVuTu#;@WSRz#gNmG0haY#^1CdqnN)C|;K%UJn%GN$i^m?Ayd; zZ7*s-E%#C%)D+pvcfJcyg7uy}D#32M0pt59iSeu@&RtB-5W6r)eyZ{#)m~3;(KXDj zk{Ulx1K}z$bSeq2V+81CYA2NPBEJ2reOWbSb&}F@qw5f8BNSR5_4qi4a!k>%hbnpa zZKZBQ(yPyRdmX^{R#-4J@NrJk%t4M&Tlz3tDW-3jeLh3>U2q$47(VV{ba{Kcva7sg(v3*$tXkw=)3u&GpXUM0uQXvfvd zwXZ`C^?(Lv{=Ul$6n5~lekSJ_q`u4QrT+kk_n`*^d{F)qprZ$ih=zAvir3-It>fU! z@T~EkYw!*ip&2&s?4`mg8O%s*i%h_#r`qTVEYXNYF%%IC$*@Y`&SnTceB2#E9MYWy z)xQ)F#8985VL`x*=xGZGXYtOqwTi*V5dz|0dM}H=EF|tf1Sw z%_o{zSjkCYjS$A*7BF?1tdK+Bq*Kq>bq!agqQPeai>AvyA2PydAO30PK64gX!}=s5 z*w8hx0_1QrRtW#g_F%Ybpv?M4k;-D_R5%AX-Jq}i>`e|6x9nihW(8#wIvRn-^+Uj( zKY4-cI5fgyPLFg`H5Zu~kHg!(jg6$YHO7Tdp~W5vNMlWlLgBTIlX)-WNZ=ZDAo2yB zHY#~!x@I#L>PWuBEtH{kk_mn@?44hgSR&NcwtM3^77K7VWw2Ml$fzyuOS%{iZ(46H z1THlz{UvL*R~nu+q)z|?8=HvFjp` zs~9_B9Y){JKK8ju6T{cN*I|HU?Q}r~25#=XI(E+hH4}5{T6XYyS*d^>mHf*Lg;R4s zem{xNr-0m6+%75EtItl*l%?NdjlPcEOH=a%FlK6g!QC=AnxLao?V(`-<(yZ^xR}e$ zQ92>tWi6=K9P<}f z$?EDC$F`3w47R>)z+eF8qaZDZXb6ET604Jb4(rgH6Ix%zD=;z&W0I<^BcqefR_2v_|!_Tf3r-|U#J2P zh^6@u+?Jpw+&Pv~C31Q)f+MDvCN%`!KSuOEXs3Z@{Dj8}T-kaJL=*EhV{;deUDTx? z(aRBOe`g#UC17Q(W6!P(8XgU9I~ZTiX4A$p$*g2dW3k9`nbBzhPm$1MDH`_6GVLe& zFaG-O?pV)9H;{_4JkQl`+jK(R-)&&K6#aZG8cMMCQ`U#_Eh@7*z9Mhao$&Xp$5J@& z)A#T%aFM@bj2x%*Cn@7%T|dAOxMUwkOeOublg$y{e6ScVclE&Yt7?O zL@J@=85PKNm8d*%kv#EhMdt@NoZ%POEm_XC3S#Y+wuyJUmnL+4 zy;3BC!GWhggu9NAD9M8dzxOjAi&4{}q0n%LuD|6`KW*)?3)=8HF#Dl&eH(y?{Ew znpxy>Zaq^xS4PVAiS?~yPw>l|!=gYqICKCuB5XO2g!H-U2D02zV69m8lBj)$iqE0Z z@7t4mQpV(V$V09UWvEd{ff4<7ntm@liXbC_d|K=Q`CGi(er_l5cT{?cu*9we>q3YP zBstlj1o3QHG5Z!3AK5>+%&KnX;kNQ1Pj@re?X-mY;+;2#J(Izr%ki;r=p zEF@49=d<{CrEi?7f#;#`3PokVY8K`-YLyBVs=PYQJdlU|_EE@+nDr~Ii!f|?r{AjZ zNAJKk@muZ3Ux(f#yVa4{^4#LZpP_sy-;u+LBf3$*HdTqAg7e~e=h6Oj0PDl+RD4o1 zl$QA+Mj8=ZiFN+4pUNlm*N(LavoPI`i z;*mUk3;49YXl@|pw|Loap{M?7bfRw)+BjTBP$TbO0$x15x*z@8)~dLV?>3NMx$pNK z+WeOUs~2)N0%+^jN5^yS)YzK-)g$n% zqoxM!{5l~rIdg4)_62~Jnd*_LoU4g8=3&UQGU735#LN5C zvv|61+_pkBydMobk6wE&YfD8jv}K8oxPEnm3z56Jv+=XuxsBtl8FO>KnzeYWU!nIMuD7f&GZgYtBhLMwI@j&+NOO|}ji}zNc#6mhE}fqQ%U%UR$l6bhy_UaNd;fso&?q_?>ahaHkjPeD&}jx zKT}opRANeqj4yP|`#D>*EuC6+%v1QV^Ao?H5-GxC4*=D!SfakE7@2?W$O5hze%I(> z$=TQtn4BmBRp0Dhn>>Sjo!ee+H%RO6()n)K*g!A-b+-yP%bG6Ax&YaAk~bI z9|Jx(fld8jzJq3MXJlOP0C95WH4VJncLzPvW9ZM7AQDhM&*{i^okschn=YTRcf!~={_ z*^E%ySW3>tDbG!&m1ZM+5KtR(zs-!GRBf;e_jeB=?ap26 z4FfkFZi1We!A%^uKoK8pqL*&amMk(?PxbB8*iWFnY+Y8)ZF+|x?#j@?={E2e^Bycu z@lgio;?`-(V^$(K^QCsfLG@P8bj%6fPKu>QIeN`T3lM9<>#wM@zP8a)F{0~Jkoa6R z+|M!<9kDmN4{~bjdUOMg7?gj`KTnbS@_|y$~LGV;lIJgIx&V6wT{&_30QPCzxN(BJjKgpQX07z5 z0)jT>Dim)eh6Cipy$;kuWOJbR`$0X32(f;qzyX3S++O2YN2JY*(XwU;R5f*Q2i;F_ z2iYmOBLUn|E8Pm~_-_cJ1^5<-=M2Igs_oxFe`Ws;@|>`TBq}cbjneoV{tuMD2mBr7 ze^-Ie)dK67mj4rqS7F)#k_gVa1p+lB6b#1o53T=_^b_ic2?GEQLjZ@_{wu5(;DEHr zD%JEGXAO!MdFD6H-vj>UxmLOnc219C9UO)U4*P8l)z-N}?g91o`YP@*Y>#|TkNT-b zE~yj3-*+y6Nrb6o3R6#*c} z*lrlT9z2~z?CoWb_TOQ}5Xy;oj`^><^SVEr@wfTD9@x$t+1_$hjqLCsm3WMy;l12@ zg5Xb`eSST!W_RNVW_PSV{$j3v)7)Y9DW{qQdAm)-ZRZ$;BxY`7dCx#eh}CRk0!Y|g z@f;s*R{7G#+_eR~iue2V{b2rK479s^^JHB&@zI$$@;Xed?Tj~T(ibEB325Ht`^?wY z;}K>mQnfIqf4d1B_#Qu|oM}Jfx4s)VIqUHymFvtAxNo29FA17GpGp;ITNOW}99>c1b$(y!pgfE%K@44Gif|ffJnb zbG!(}ZExN+{8_%Y2Qg;Cx!KrCX2LnyW__38!NeuKvo+EsU*gh zBX&wSU1*9-3oG`iUvfrZD+23^I|kh5ua(@lPep_e@-m;Eclfit43+!3caj8O>b{}4 zIWPNyo}S{RgfsX2uXUeicSh@<8bu7ai4@A2pIVpCd02>;9MAx_bDZgM8kgP5Bc7XW z{yw1A)f|)0MNR_}4k?Lsg47OQNu<-Ob`K`_4m6jma;B~)@tR;dRJNpnj1FE>PZ6_Vf+~C#F zIh2Gfh@8iHg@zR#F~r0|TY>OS-!ZnK$miidxQ@lusIk2Pddi20xLdU@yd|{QxK>-q zGYlCKob6C)7&xCLx-Rh;eUa}f?AS@k>Hgw^KWW-?J-O)N<-_Q|iR$^b9|8M#uFgwO zFxNBo(%~cO@PJdj(7Nl#2T7&11ixpp|VFDUefwQ=DP=a5i?{p5Bef9Unk~Im6LGC*(7(HMJqJ%;)ORC5+#H{qz43 z_7=c#Y)g}{n3f_nA7?)2GaOm@OKOcwnF08uAA{W3$b_O?#CLo4DIm?~+(vfGRzR-9+m#m@kk!ZO6>J)#->;%RaA)kQhKvv) ztm!+{e8V^*qhRr z@|V3^%)wY}>UJtT;k2v8(#9u!qYaFjQ{~b_qUwwvr9_-|AuIP~>pHRT-tK0_1!9WW zT1(l5DV>eh6U4QaJ`?<^+u14T*%?F+x>s`*w7zF^Bc)wBu`#;ma~|{e)^8eUki4#^ zhiHu*O%raW^A92J3+g=^nUZt6CZ zBDa`NPtjoaZdC~blobrFu!+C6yK(S3zB~Q!KV|Q+|1#bp^xal9SraLA&C%tdc;9xc z<5p#q(7h`a&>*JoQGOonPCO~+uIi)P{+KPT#yw5>*z%bWvRu9> z{kuYsp==#}zL(Lr%6Ws{PD(b<-Ta$blS%^@r+2_#Uq5-Z>py@c@=iXZ3}vja2_C(( zkTr&p@JT+L;KUc2EF!ydWb6w&ZoEA^{Td(L{5jf&pZH4@1%2OBK5Cqm6^bfDj+_)G zOy&~_)q!8yJvd5;nHh@sND~pRsq zsc;}u?`>I>7pTe1lcxZRWVO4g`R+kQ9nLdvj zI(v3D#GI8+Ku9!c{<>Ar#vibgJM>bi^#4|!sTzE7uI~EnYX0GgY0Wkl{mfu1|E}M1 zC)w5VJ@Kq{z(*SHI%BJc(Oi6W^G7?~laiVbQ>{hTdMj7e;Et&cSq|LyI$c zQt0^DQ+3K`3ag0jQd*LYS=qNMn<1n1l&N&h=g^b;G~wp@N9k#NljUcb)RIzUiOUHu zj!dfhifu_S5$oqym(Y-d1zM=g%YA2EckHt`K>f%xuIWaW_g*Hsh#-}&noZEkVTILE zR*7}P?ZE$b%fRw` z9T|BGn0(q%0Lz(*(P-%-Ah_|h>wyBUG((pRQoj>orKSxAhp+vBcVo~_On@D}`GGu2 zRae9@X;eOx8V}pN@n%fZqFXxTEtn%uB0ttBWlLpgVqE&q&~VYmp41i9v9iO8jyX* zkk`PI!D-tSsazm26ec^``1QB2!mO06RPnhE@{#q!sN5WX7{cw@fG3*zVn!MVT$orL)_*dFK~pGXpuJVx1%K=j9_xA z+Al6ryIWd(KyK00J6opgv0vu9afi2D1x$WIt>ghdHZsTiewhQbd&@exvn~g_{k_DH zwCxuElA7Y^@R!tIB`aNgQG+rZMz_|?UooztiwA{wwxS?5GHtmQ03F;{j(ZB=kM@pk zN-xJ-?)|oczyAN(0XQP>a=&gkbGC6~A%pvS?ZdZmuK{;B&tDg%GtdnRwLU_!k*R-o#uL?5msVE^9EmO9 zNT^&*D^_myOqB9wY5p>4uv(6i`EB2t+MgTSF3{N3-=5=8%7G^HYl*WXnP_j&cUbwo z&HeYSN2iBeOSTtR=*~t!Tk0Lyj|MB2`{PB!)H$T(6=u{#Y(|6BHuKc< zJB+#A==$2|`sv;&mV~@{_}|$Sx=#l98|V2430Pc_nl6h$aTvkHmUJuS`?4RFX&M|- zy=?9bf$Loi)a&IF7n4XuAu%?GNx&zH$B@?LqE?MQT|r#hu4bw(&ZnXkJ{2AAERIai zx!_jw;?@ve!xI9_bzN{49jK6sR*iH;f%9r*3iQy{l z(`wA?l$mNchiX^{1;NnP7=%s4V~|pP*WGIE(`q9m96snDkINuYDgOCq)#@cuGoCgW z?Q^@;X3RDMF73bX*(sPiE*8apxVhUXc-laA9jXow=-LeECMwF*kng$=(oUCSe|ERg z@U%$-5Bcuj(#;7{Xl7XEX_LIR)wqQxQSsH?W^EErafs=LtO3_*%dC{BVQf&>=DxF8 zVCy#B0Z4JTdfY;5C0?j|7vXX-lBtW6zMFFjRP0!IeGAWTj*UgFyLsZ^LE9mk%;i&8 z^MzUTaWZzUPl#!CYPP3z1hPZ5j6=0wt2dbekfOWIE$qHqQJ@3QuE0Y9yu-At1ENdc zEbnYfE0kdq0s^hW<U7>RkhWf4QipxH)``twQRYWdVLxYGYD8X)pSUu7`bg3rdp#Vq&*dek zBGMctwc0Q`Q}H>;UxO{sj>MFr&3{Z()6M6+x7#b2fIO;}OdgIo6|cBiy%}45zSI>t zc_77zJtZQ!3L22U6M6n=Osgx*;5D^ckk$9OQ>{cK#~8GZ5Cp%EYI~qK**oV9aO#${ zludGJ!7S!REmpx_-{tTb3L+J0v2=pbJMPKu*TW?(iwfOCrmB&6rR^#JlmO8y#k9_xh0fAW*b-_Di(eUeSYKAd z7ahovFBME=;8&VIY082nehX91&vNrr56I#Fv1u{|<=) z`tOkEs}{hJZWn)r@l&1A;K!hoY)D$de_W7+GEAd3G4{g<2d0!aAQf+8Kvlc*{Htgrt@DphFCTd<&%? zh7=543__nJb4JcuBNomlK%9=%f4LYf9JYx$QBpq7qg@AL07|?u=>}Ig41^Snp9Ay? zx6o_2;&bDK4Ju6fY`d)$CeG6eR}c&zTQIK;sup`NYoKmgKMbnY>sTK6V-00C94ZJ- zhCgXp4&NNZcWTq#j0xb`@ZUqTqyquc{y?%qNOtWZ1|cX4K@|eCceD6QcJ>ha72*b$ zX})9X70b%N_)8=QGbR}dH%XhjBi{eyBP#}BOy$a?s{<+d-i$~Z5F>BYgW~^56^Feo z7&i*Ve>fI_&+womrpO9SFTkS@&vGbmrseYU1c= z#-g4y8g=m+wMaYGN*2C=pRC5%p4$FI@;0h> z)G!)_*)P9Jr*kSM8eN<{UVSu_tN*y~dNfn}+F!{%Ur7wz*xyy;TnyE#Kb|$EQERGX zT&dYWeYiQP6Xc8h(S!P-?<336GuBi$#nC&9BevD$&W2DE8;2O*zu+*kFuHLuy%99J z-7!ksVMHK739tW1KQsojUn6(G|EqQ{}G_bNH0+HIU#Upz%~k#L4fb$cb1%$@|^ zbVh^myrGa%xVsKw8yndcOmvDa@WfK+E|0q5~>GDbJdcQFhg!%Lc#aXeDSH$&=(nW>%p&Tda zsYr1+XXEQz0@m&W5O3%_H5V`mg8e zS`WK{79(F}v&ioEg!yhWeIW@@1NXPJNlUojfWZDX)4*GLZklohgYhi2Z%CMdY=?Ym z7vq?qDNX+uJ!eRV@cRA#)Z^fbD@Wdfcz#ZybMkO!c2g+5mHM!^djQY-G}wShtEg%H zRptAa9g%@RiU*SR8!s_IZ8;jy0eFxK?P_K4YSge}67QG&Wp7%)q`=NY%`9?hIqWQ~ z4NP=wnPq8E&1fbcLJ&1VV^5*)30uCUFZ@|WA9Ty-7wL=edL_M93N0+}?aPyk{du4` zmQDJ*E^o~|8p-la-*P}OX!3*=2+=DXPn9&^md1F3}a8FWu1X`LO->!g4yNM3W@z0}7PobugWGsOrnBo=SDj6nOj%XUJ z>L7p z7m@jTA=??s^hGf3A6j3Sd4Oq5z}wcZ#RMQwD+;kLiQ`XHH8Ixn+pvL->qh zMk>zrA@$sZp;Z=+cw2#(UTYP~Y8VE$>tbS8yu!i=U^2uyB$Xb|wL7hZ#I#-)D@&dq z?_1H+U&Wh0#ewA>(n0zjG$*<3h&aMyXGl82yYVP~b5spe_3YoXC8ofpXhqHQKcZtJ zxbkVSr4Vrg4U#;uKM8TdU ze}5G55b?kV#iOYuTiCs_n3P0I!WOm+x|MVqKv`QGx|O=j@4_RN$-dP{HJVh_kHE9d z>k5WfDC`Qx!)Z4grp)Wgx(vJxxa9)lP|}wu8Qh#niKd`o_M1gGd?=(LV13HBp+8*u zC4H9E+7Tw3bE~4J1bWD*XNh;nXj=E3k)8!|&hj08cvebL(f{_B73gVytaw!U zH^gHZ&kn$ZBm-kDW>dqt@gLMGf31I^4&x;ko&AMc9}?d` zpCNFcjb)>Q0~N+efMP9EwOgjKqohIarv+XgQlt{Vv%*@06v)snnGPMsnFYg$X)}a2 zp({e)$+R4Xh&zNA#;N1C6sydwTcu139Z;^%cY#ke9BiEjumMAlGYX~I5|a~c_Z z2oDLSnPS;-k;{$7CQ}q>nwW~Fde%9PO0#;3ULg&`5FQ{XC4$rVkh(fA`7^f7PzZuC z;t(EV-`QwcCgXA$2J(m+Gofo3XOwmA%R$|O6`zownx$Mm4-Zr@ zRdan~xZm&|)kscqh1D+X5RWE(x=2`am+*cy52UNzE`3WxYA* z(#B9irWDFXG5%C|GUyijBuG!xX^7&=`{d=S!)Fnu9u?Bc@d#?HUbqT`weAE_a?Zqj zOObhMpAj%!RFZqt5GInWmFuY`Fku z1L@3}3u$?<|72>t7wXIbDv)|=Q!>d#s;o4ss?x*%L@k2-Cn^b22qp1i9aS2l+P^H& zs{>jP50CZ7g6YnP#3z#b)ySPPPt+t&P37UN#m)#iD6=hfwR-AbH3c0Kb6+tj|Kht3 z`3HHkGvd|-L2XXDf(JjiFzhV^+9qI47?V>lfls~ayLf781V;LaFlJ-w+<`+TRG`y1 z*@-5QGO!jrC1d6eq?}kF^*<<+f&G&*Puh&Rt^RO^A(xUwbS#A;Ca<7H{_hC0c|LaW z7-JJ&CMn{q3@qxVSG&nN&G%)>GCJRVl`gd=4-vG9B>@}@Xy;35Mr-FN&m*q_E`25${Ky?1(8Nrvv|@Q8wN$!ZNbKtM_GiG}1;=;%+_UeiwrCZ=3btVtG=s28KD z2lF_*k}A~5=R4G#&HNwD8ff7;m{SF{SN$w|Oa;MF?D*o}zDuuBMP;YG)k6j0!*D+f z(7|wv3ZK{iMBIhiiLb>V6faERp0Z1}zZe&irQ32$bngwL3rR)Ttf$) zGRqw;Sb1#IAkL9wS1)ACag+cJDGm&_^yiM#013kHCqGEEm`@)4GL!`NY)s6%*LUB* z?CPKXiWxt?nLl-a1PhG@@R%P1E;J}E3q;>PIdW;@jIl>aCVfdeX~Py;FjC<%hc zIr7CbZb`#hU?|FN=@>s?QxHh=q~>#Pu!aVK7Xc~py3aGjlJbl9AAn;?^E@t|lO#K~ zjkk3L4bJ?6?K!vc!SEvAbyU-GqM@%69`FxRJ=<4=>PeM>Ox9KF0y9BLbrjc0jdv9P zks9YHo{^g1ID`lz%aa8~vg=w95gCsbgFej4k27?}{KqLKF^4u2Abn~7paqv}@j{lE z)f~1&WG#~2RaL1-wSir#{aZ97p>0iQbNIqleAv*!+jKGQCj>MkimY2x+CPQdwaAIG zJ{$_?U6>}OWO%8(Z|QnGVAGR_1E^di8StSqAQV{uHBENqf$)LV*wT$RNT6BY%}x-% zPLb|{(8f7>BIP(@0XOcRfo6zvyeM~&$%GG0jhd9>dG*dE!3Ny@9hR(IOgkRqudsaS z`{h5A8R&fcS*4RyFfBhy9*lB_lJtd38m`R+-rE!2 zX#py4(`Fje$;{Y#HEdPc}U1=_%YVq<_M*0 zKg?4_{dVtTPU6UC9et$kpBu=?kivLK_WZ)cTFH>|@kc1jS5aL+_{FCBg{{@g*0XaP z!}v(fW zxQ`qo{JN_$gnI)$M2zB_#}V*55cDRL25B?yjg+vmOkfXr$0AIqNxtR?xSf{C%Qliw zY{h&Siv_B0Uskilcd~cC-+_oc60e{)&&Olf}RVP1TiCk+$w%O!m`glNxz5@9j62&R+r?$Fd>q6Lq17VUy$NiVo~@(#}OvIA?Hru|23$-EG9nXqr?LKzs0zC z{Qi9M&%0-fe~aOSCRz^Ym3q3*W*>7LY90W3rOE(IfwVJ zcw9Ref4q<%rP)Ky!}vi;Fx1aN;0W~Wzod}+{~m<-+cDwS{K^|>>Wty1B=_TBz7KVX zza7I7sp&H0%54;4BD}dPql<=d!McH-()K!H_(N>fPxX%^7Q#ZHY?YJR4;DBnRQ{lm ze`@je{~Z|nA0>0Y#ehXGnLM7EYoK5uvDbbWXUy0?OSs-|K1Axps78NAgV$!}kDME5 zd`IAD>>-z!0Ocsby`dl-H6|#!YuR(}A@_0aA)8Mz0|DDseV(kE*Bb+oPS-jItqyqx z-Q##S0a>JqDF>FCF|N2+oqq0FnP!25zQNip2JiJVh$ZI9ZIlV$`3#!V5jZcYN)qlp zZOhu@A^R*z*n-HZpa^UT8zWd}h+eLn-1X+;bW*O(#_nqgCG0F4k>Nl=F>rpUz*{5k zvOGE~=LrW0LYDfggASG%)b+7g<49`v{=Qe)sv8j!?2Y^C#<)W>sN6E7{J1yHVyIaa zQi-66ThqYcShX2SI1~Nt(JR$41$Ted%3|$7>Lw{00^-VfX5)~n4kWZ)4jeXhEyrKY zHG>dPy5_$Cyrj5FJp^$O1zHv^OheHD>{;eK)8AIZzm47yw3~4f7dL`lK>R_~&!FSw~hV5QVfZTkUJRK|p9+fD7>#W9HjXq|Sl* z$sggn9A8(hh+b4jv+w@TV4nb`yytTRRv`i5Gg;n!9bNvXsgSm!r@ArMgP4K**#S)kr<7-VYaK3OhtRBa?? z0ky5Uy*mSqTbRynC4Wf>*TvdI*f+fxJ)9wG_4?y+R|ueg_#%!>!Q=(wO_TAy?@a<% z?9(kz%lzy0(AnvhiRGi(+wN5ZLn8ic!dSv7sG$!>Eny(|ld25dLyvQ{>Inter5p_< z*|Zq_X}Sj52w)084b>dvpdrTSuVb96vp3cKl$Je*?HO)`#ITK|jWa(W=qtEUU_1|X zl$SmxQj@LtT?Zu0eu*M&>KJu~eXTHa3OGqiZp1dk}ZJU%Zg zEk7%15r;t)KqHGN%1f(fndPiP!yZSDsBiC)stkZj{&24D3|=-%LXUr>1Rjp^v+m`2 zafcJb2nEMhhW(w%s*L1Dx52i}AM9`0sgJWEh{lnEMxZ(lo;W?_Oj!LB66iO`^<^XT#Xxxa$< zR=L+w@gYU;yw2)y1PsJsJwDy-VsQp=u|P))BRN;wGTVWOw@R|)&OjTnW=io6Hi(eJ z>>1)T6ca3y?_R>OZ`IJ0S8A7JDJ880IjgkI@*EZ^>5xm|y>8qxCC`x?T(Ye^mjTm7 zNn~KU$mR*dfqYnGFd>66;xo0ia@U-fmjy#ly%Cx4X%TBeMl%WB;$mjx0tUAch9D_% z(n>Ws@n4*{*gc#4cxkAe<7}u4rQkv4a^?n+2g9=2qlfEkl>p|2ju`cjOv~gVysTop zw6rxkKuU{R!GBW{7bif`1c7F)9E$(P8D});eC#@x{05v9#0eRKDGT-%=K)U& z8frUQYN6^cFL><-j^Q2R2?|+o;XxW?{*%@+<*FvD;>n$St_U~6CGq?&Ynh+ezpZ%N z=X}+64R%tkf2g&4cIE9P!FC_eggak-XdwwZd0SA zck|)&`5pVqU+*1S#V9{37-2zu zEv&)@vh4U4nU7n2=8a+CEiM}db=GM^IcSP)MimA~NU~{T7laFcr$HaL+QoG~xTkW# zAEO;BPf(6iB|KP+EpQy_&R-P<6lx0~tbrlj{YAiIr|}?(3|8P@1gH+7crkIuoj9_+ zz$6sqDN2Ja#^M(;=fy}$NlIlUMF4I=5BrWP2{NJ0`C1|XDI>iEala?&>4b$2wN7nk z*?0@=gCk*c^u_R9*$qQiycT@nFgt63gwDcO>g z?lMFer~Ylo8p0nw$v}j^3@J$g2lA0w$qZuuA`qQEyDILi0~{ybZW($cXE^5yQ=m&y zx8p3gva;FrTp0LcR=3g%2?4HCQr3Q1@83E-t4NDOe`CgG`_-pSO}Ax zHjav?fDS&~q7NPmSP37MO>yOQ@bX#l??r&q;v|{uK74rW0Pe?NZdvNtx@=yk#3un` zi#e#T9Bt^;X0d;MDJ;N__*$RbGe1r8SB0dJd99C>QxFX_vTCv?$t)gYEwzsfh#~P9 zEFo*rQI{Ceu z8j&J1jCZq2XkbVnQ$W))-4(4H>#37IO~Pnt9Gi{Z@Ov|+Kkx4+Lj>+Vu6?kY%g3Z> z;4jfKui2q}Ab>E4`=M9wF99rrfMIE;&d^Mf(ca?vO90R2$P2{Cq-jDU{>ZGEUqo6@ewO@iASZWe$jBiUK{DFk_+RG7N=PeiIL)W%_w9pCPhH zS^xq|yA6e*ftf*tZ#?jlf$e&*jv|xbKZJ_-v)ol5JISZ?K)#vB+ubKW{vwmwr7=!* zH~cSkxB-9EiJ|;&b+!mVbv!WLes5=({{b^Scy@_*E&UtF9t#)xZY@fo{okb67;bNU z6SH4G{zrGXLLo_mUT_eQW7z-drQ~Af>S|?g@z--!%^I7|KX@>Dwo47$cFkd1oaFau zPbuoQ6sT;0n-73Ad;_t&=T1P#SKKCFj~GuePrJOpynXuK+v74cs>qy$_$3c9*;o*c zIbaS%oR*%B>uzqP)b`bggC!*_f`!&lkk1eITgUt5z96R|qjfr{CkoZ!p?6!eMsZ>| zZEIh6dmJ1@fK54X{qJ&abiln!^+@qSgqDty9g)RBAk&Ba+}X6|*kxcY|)ywgM+S-J_$PsVIpr76?U-A#pO5`LlRHBID0D(BNV4A)|Obk>sEtvM(2+ zsR+^!DS;am*r>3-h(~P59&Uaw$P1oioGWU|bCHm}Z;2bQ9`Dkp{7CuY&#<&Qv^)?; zC}6#+yec$5zl^6Ng_yiq&K37|BaGD0A|<^Q1IvdMWK!=?)|-IY24<}=(;J(wh_?O+ zAPHBkwt`fw8E^l@GFCT|a=(hu$83774@RPQFe4pV8W?~OcAox}8^DR3 z!djuBW>fxCJUH>GvY%>nC6c~@?fqIp_FO%1hbw2ggs`6A2YEjSVBmlv0b<f2ACM)PpX*S5?GD<;yn{1-J`lL?1djv2nRL|qgwk{yf!7@5=3-7? zz}>wK5X(*4vu^W!P$yFq{wO!A)^V*3w?5rhAKe`IU`*u-Gw*uXV=+l? zU3@y0YO6WR(tMfjA{O8J1Ulg~=4TWC{cEtwcn^0Vr=38ixlV7&UiOuLwh(Op^ zi?5CAr!fVo!3!;i$Kmy@QKqx*0GXU#FVQuFSq+}&`D7`-->yg6(!+fZ_qKp_3g0_$ z=I7(X0>((?z)1OBIn&}tU}}jqqUGulS{Pa?Ni-~i1}XC!u2tlUXRS|VPTvU+V$<)r z*K%bW^q=jIPIL}cL4d8MT7Z9l`h5P?YU-<*vFd+a@X@pV)`KhvAhTXQ0|yDJ`=(K> zq6us`gq=wuEy#`pl&sCbhoPpQf3a(G;dI_pk)D#4Z~5X)GwsOa+cmH;vq5R>04peyw=bV%}3v~((uH{4zN!&g5ocGT!~1bGEPs-#kOn|FrGjk zD}yO_jh;Bnz?k`_-@X%@Cf!T(ga{Bd+BK;-bulcDoh0FEcLT7xPdGWOg66r57<`m0 zqbJvIf*qdxq!6;(`q15+5D}q^NwI9+kJs&PA^+*v3vZr_39#4P3mmX%)W01w2lhc# zGcvaQt2^XoO27RN7J`wV-y&#Ivt(@(n=w%7Mnakc)&jD~gkFHtZ}t5~Quc1>Ykq5=IO!PLoU;XP2*gJDGwlCZLivdTGMW zGi+UL%!olkIEhWyT={5}<!$2I|_6 zlpv_)vK3Udnp$_GBiy}p#Xaf!^SOfx+=-bVs9%GX?mYasfN}HeMy#zV*W(sw8$2(G z(Da05&o%LIW5rLC&~VbZ9p_>6$iH(0D;pcv)#!3N?9(&Lvd6Yj7(_n5xhoxI zHuzA+=6YO6uyW_7!|q^4pU)jOMIMaXiZ^mI89)@C1z9nRhW&s*`P9nV$x}dOr7m(_ z933`4mQ8J0$6%RlFEfi|Io1pJ!}#5W^`p71*I=qQSI}M_A^dgbmI_TKylMiU#e+X? z)9N-ws6~HA&Cv0nKjls^MRl~QZqES{jCs2L=i#^T?{2TDJyhqVo(eUR72j_DM_qu{X4h12s(6Q$!|@p;`XqC?_h^>Voc z5v^4$wn<#qmPVQ6l%fam(y7nSXtQn*?xJO|u38xWV*V||p0EY!!~lnOErLeb`Dj?- z2PO0<*^_{GuYt<-(s?Km=u4QB{M)0>p^670$#x0S9(2LzLrBcR6W|{HpWoWUUBe)$#zsTB8=PkXgQ=n=!9kRV(az| z1`m`Mtn2*HAIqC_FzYY~7{zXvU?4835e`xT&Hp;R}$&klmh%*$(&pr$aWiyGO$Z9Q}uDavg zsXsqFOlDeHRb+5LUhCJcT&mA^Qt!^T6id)QVF0(lo|ln{0~_XmOmLlycHWduOfE@L zY}`6^#XG0U&g&$PeKpT0WecsGNLCH8ImK3}vN^y>Mc$PAs==DywRt(`jv ze%yspOSBpMYE_Dbg7p_h)R4Z zomwUs4Mr$oCBwKU81f+f&6H_Px0%rFBw}h8UtTJf*5?%*w;nf7VVZd5_W1;I*wbiR zAwY1?i%peewWL5xu;RGRGbJYYeJ`vdHeRbibaT*Tg&@k>d;rb`Q@J}tejKJ?=t~8u zxo76t*Hs~n-aFGyxpmv*H$Y4L%C_KJAKzw3laWzF>C3vPa7U#Zt;==j+#=WR(6%I- z|BTlU4@Lf;Ru*NYj8(FgKTSPXBCdSzIydLsW^`TG@r^{k3@7ES>pA%8WsICot`*K% z3LHc-7x>fnTK`Cj(h4VF$3X&<9H!Q4>tsSGlEM(IKSZTo9bRlQdc|O;^H!OvE=2gRk?CFj-qs|?0%Z( z!S73hQ`G390r~9bgLK6dX<76r&Bfvg$lC{5=qBGisqkx}i`I-lY% zErn$)<*Bp3nhL7BXnqU?IFxNhEOTSI?ga+6G;fu0F&GZ!*+m}^O8Gp1wG(SrqbAgE zx7ha2&skqUj{b_be{pjuC~^Ni$Cow`>9GKks(VaYC}P;qxWo52n(F zto5sVJMK!tF)2=J6Bu2601iY2PZQ%rkepd3yq@i^^sa?8wImIYhBR7m5Ye}5A+eY8 z5#Chly%}4%19ka}9acjvE~8zxpqAmOtq=PZ*(LWp82DA&MET&UAEMecENK>DkEpn1 z##?Ik4SxI%fVGawT^BGe&Z?E=-L{Y3ZnWj?!*V6ne4m8S9nU#+IwureAx1mJy{TNK zB2W2Nb$m#Pt6riba2mZl!2=IC_DmA`!XCa_MotPEpHj$BZFqle+g4=A;R%cml?0A=PLwAL7*)>y5Vzgb;56kJyk+vzA4XL5h$zx1uC=iIUC5)Lv zS`6A1Q=8_%a#J*q7AQ@wgS_dGQm?HphOb|9$MuW%^9}3xKX%g@V4Y)m-fBPphW=+R zmgC2_QVRUSg92u+fBMmzxVgAG*lF3>GF!Qt+5PpkR~=OkH^71c^2k6!Vc4W$&7=1|g0T>N8Uf6pztmHDk^uWpHg9T&AgW(9+uf&?Pu=xYP~Q zLk-kL^bcKscY*zny1!b$M$gm(^K%raoo+?aCd&#l3z~ebM>McCQAg6sxS5u)V)PC< z>8+juRd8IkEy9mqE_S}im2Dzav2$f570G@lj`vPnXBMlz_Q@nIPEGWZEN7ra&sT+? z`mNhdEpNibMaXd5SxOzM#~D!=)YC{(CWCmjm`rlq8m^2jW-~k9=bw5>JE>I(B+2iL zY>RM+V7Ka>GP(i@J8b;MnCY52(M)(kAJXbMBKnVTSEXlA-Zd1|<;y!Ygx*%(!2UU) z&7JZJyFkIlK;bz55dKH!tzLj)LZY}OI!vFwIoas4m}()Hw&(VQIvmu;AI}z=?sMIa#G>0| zLQ67n!lA1A+$HBT221`E3;0y2^Q$_yQ-+zc=()@MEZf@EiCmfnhuz+1?Oa9WX2xD8 zRFWpZ^QCw2R?R_p_119M2U1tWImNxj-~=(jnC1N=R5*2K zxnry>8&`6dCe!p)*+OLz?DA+|CZo}Pw|U>a0k4q1U#|lbwvfUxT)3_RbxXI!7>?wo zmiOO(rJhh-{8E8kLqq#M+vY25^|AXNJM)6NJ`agTS6fP>#rK2yX7z`3dXz1HG@#C>ag;i2za>S4osV1l-l(JNhd0$JxD4oA`VTZxr>oyjXLFiC)K5VR&w<|ODUQ3 zsdMK$(a3!9Fu8~+Qg$K-T4Of)8LY6f0A4w69^SL%Bn3!rN}03&|E z1hCAszUhH9G;B30N0`+@_M^+K9Of@7B<( zKz~Kj+?_jJp8ql|;9yvQgc2>KXfpz-hoT;@(p8!(eXie3-6mb^SO3~k+|B(}akBo< zl$K#w4s%_Et)<|CJx!-2W|lGjH~bIMTW4vd^g?VZn#}JD6bjO~S)P6Air=_n7aTJWLs8XRL=We@0C-&W zKExs~Uz&~29e){jQq>x)NsUyE)-_kY&>JaQ21CBY13pN+})qZ_@aKOBYQ3m8J5>6wB~Q{He23dIcF+ucv! z_Yb5d;&9_Ip2Ttu&glAQtQ+q&d#? zbeZAxI<_wuEif3Iy+!kaWkCQpa~HX9>Y21}u5sk4l)b>@as@eQZ6DYLu>rG0m?&sm zG}h^`ZI{`GXNU^m7z^P@)rG_6CBXCxwBGG-I)!1uKvpE)_^2hH2F`*YRFEEH4@oNf($i!5YC39sdb`srGp?+I#*RJ}=^!fVns zc-f#%o*U-vtJR|PVtODXfoB~VrwC~n>M@+4qdv2>epaa+Ab{RJ%Dir4{J3oR*({ka z_0krG(Q$<1WitWO3u-?;_ZIdMj*}QkK(s#k5&j9spojr0mp(P4!tRErTF|B#FBDy~ zV}fw3F}@aE>^mK&MU5&<65rDAjjg(iu*1FwXz*lAYk6Z7iekxOrWfmp#dgz_C}!ns zxfHKA%Ez=GMgpZrvLnfs2viF@d6!wrAF@k42LUKm(ZdH)o_fPgt?Y3#fx}Vo5HKQy zf}1EolZIf_Ndb_o2xOjJJE0&t5chDFH|$@9!GA!l@M1(yphtL`eG$rzu0*I&*5tuo z2$IHPP4wu14O*@y?$*pnwLh_w1%uuaD7a`?gEk0|= z^KuWa_oXlKxF&q{@kHT>2&FT}xni+t*C3R49 zaw=W^H=O}%LF@Om4U;X~E$jE23?}3sl=kSYfiHgRhW*B+$ z8u#bfj!k)o&i!+~cDb+$A3@O4%Eia3EnCO|+6!lAi{I5@o2bPC=6OV4QDz1G;^El@ zZR~MrWxvn7gWR4{=k03Y@aNv^RKPv41Ojmc$nYW zs-pT-KqP)T$zh#7fP;{akYfuEpa@e4uz;HVndd2P;7Yg6O?4+cK*bD^_5z=y4UOJ% zi0ZdO=~tBwtBaHqPOcAel5rRlrclqGYt#c0*Y)IaR_h9%qflBf7wN8l$_=Ksg%bU~ zi5soI6>y^nc7Y`RBHYsayBh=^dtQ}T#gApJZ(+@S#zYeN8jT7X*>OVKui1x$)yvHG zvSHN$a}9~B#xUn(XD>^=Xb#=5_=CvaW|pKCKQR0X$^)Udq16=l3AFJ+X!IoGJfgwDe5^roe5?83Q4(y* z=k-&bcmx-pes!%~uI|ajt*_;(YaTCKKCkZC&FAy#7Z#tsHa)H`_4exZdw*WL zZGEqDd-n5n{dV_0pS)N5+y3vH(}&}~%2-yd`Cf7VW&XLE-*>oywMc{Q$a()qbBpO}&m)p0+Ujw!0urM$PplGiLiomre78GD}wZrMndlmq# zap6F>diFNf+!A45hr0=@r@ zFk(U-*4{X}0q7lFgaH}#$Od5R_M)4G-giWpmDqr67HUrt-4yh$7Q&Q@Mrf#@biB~@ iqqheU`h{DOvKVUXFuambZ zPo=$j@^9(Ag$BHMiU4V!Ahm@ppM-90pXTvGK(Dksf>2Y&xToVibcg`8)hPMTV zGlpquc0cnFrJ3PqMU+WHd(=@0fp|rE&wRk3VT~6?ll%n3Zw0)${^GZtdi?O957^zL!M}5t=P8o{0b literal 0 HcmV?d00001 diff --git a/packages/extraction/tests/fixtures/empty.docx b/packages/extraction/tests/fixtures/empty.docx new file mode 100644 index 0000000000000000000000000000000000000000..38ea79cef69de713a138c159aa6d4c735f1d6973 GIT binary patch literal 36563 zcmagFWmp}_wm*yqcXvX7;1b+6Zoz^}aCdjuxN9J|ySuwXf+je@-JOlge`n^*xijb7 z`@UbAhpM%HvZ}hP=q?mLLc?G~KtRAly!92+sZ=Wbk_ZU_F#-nxfdy{W7PYf=HnDZq zQ}M7janfOOx3O+alv7v|LJv86#z|rpB=-_U!ztUee{aw5g;1g? z8?O3OF%*vJOGfIg59v~!K$mwDwYI3(iH=T&dj4DJV(4qt{w3&6w>ojTO>1s za5XI%-UO4JkStp@{k(dZ!Dk{lS^bo9~P$x;~Hjy1l&@^%Ku}USOhD%d|>IZ1d+%3D-YYl?c&qh#zpObseX#cQjLV zs9WkSM0FoRvO}NRw&`JV8&&^s2v=7cb*-$mifI|VePU-7L|rqz51pm{YRi{%w4_FF z-l3%#qvB(67$`;Pc`J&Vanut)OH)sw^A#?Qts}P5JLtCm_LjCV_MPDULYVID%(R@% zn5y45v=9zeK&?`{aBR}|4?JA24u_&|eLy)u+KW75@;)76dIInLg$BItE>ItL4Pbxa z-y!=~-AX-m7VWq!1>_mYG9=MzKPQqDV03>mc=UQeDlR7w6YPDOFJ9RJjNHD>d&HL0 zg;4y*D0)zT6uJYC-YOIX1QPhCo}-Di6EoBAYi0bn3^Xgc-zUWCRMS5MP0Fd zxdOqSl(EwyTcA|YuddEdVmcbQy`+0*TSlgH>G?zp^hMTz>V}%L0lD+N8f&x{4eff% z@JR?A{uJ&in?ah^X^|jGR3Q`UTTW9oazXl4tU+zs`qQ7nRU%?~bT(3`B|)VL7}rS7 z@6}y=7^m&v>?CBHI3kKGTeq#QCK^MRE7N3z*3%^2#7ni1kJdBW$!uDTiyWVFF+@!${(T!I3i%E;?Sa2PX?Oxpd(o8srC|PpL|7EoyA*5hqTL4r`OH&^qPF&7sJsNd&n+ zPf99;0&?M(wOv>31`2D3XJ$5+8J(#70M5oOA8N3kn?|{5k_~_vE4iq{ZUSbZ2$^4R zuP-cE)4xh%7Yk>=NViE%F?5aR?brLhO1=;*5YyjrUBKWUnxX2_PP>V|JoQ}?{c~Kd zmzMC3z~f>K0|9~l_i-_>xBp{UR7P!=SkOBj>0xx7BP`wsf5MLsm=pdfxb>xOe^aq(C0o3hbM#?6wX)A_t-G9JN6V9S6P1k;+*(Mg-?nb;YU?uebGvV;7`v?41I@C-`L_w8+GI;&~GwM-p@qRH( z0#A;dh*iJye9Roi63NRMRA+pEkz;0~jmJMtDRe~2V1GG{d^do&jb~J_E(H7mlCUq& zJbk|effvzmR8Ov4SE?R%8GRDhtsy*|!@~jnEc!m``xIrOrGZ`VHAc3E|FT0|%T0YL zUmWVk8&sGFK9PkdTrs*|qHnZF;u4xtv3zSs?5>WvxRDboFQGLcBQCcecQb3Sz6x0o zFScR4^O^W5C|)~SQ!cZ~R6!w7HAM3+iR96cjELTd_EdduPjp^OM_xM`C!2#1!Cg?18`x5hK$?7lVEz0(YLwK! z#d<9{QE*&+?Hyle2A-;ywZfO=fhB1`p@L**M>_KpGwC@(`u!Er>n!M)?4J_@)jDqI z7CbSK!RylR8`jOv(U=*0*Sgq%7pLDdqwwo^%r6%7;WM9z_sJRLs6W0+5Z&}d%#zaO zP6_%sWDNXTFj@@xD(FL#T~HRZ?-;RnwAVd6!7uO{pXjzbgs5nNJ}$lPFi4J7CKZ?s zaIJS)*OXM1pjYHaVOoZpOjl5FiHGEquF437F-g#>pJC3Nmd4jobCf>$jA+^;a=MFb z4DZ2&k7mW9*+PzCq5cbEYi`q3yKG|SK|sFGhh2KH<}-Sogm%f7i|V&OJ{z#Sp)3`h z#?~-cRqzavmCNLgkZJkYA0au$15%Kn5-{RCG`GaWj>|CNc-LRKEN}iwKsP4vwmm+% zYp1f~$2lTwU|r#l`7WQYA-#@%%lzltHH|T9u{sZtxg5Z%9a= z0~!voLEX2+UuDVko@?i1)3y2MmeiqXdI0qe1c{7^X+I8^uA|(4L_=vbJ-5ojZp;dKSWW1xkaOC4g z)0pv0F6WNW&SFCCW{rrY)M8L8_|D+)>=Aub^YKn1-G_LcC`L^u0z*8!z+6u23Vy=B zBZNLnA&;tBVM?s-s zPX{ZSVDg)xB_H+FOQ{N5Rgxp{*7S0AF7S)U{XD5nG_Tl``6~B>Sldc9P>1-gG1pe8 z6{jk?U|&KxKBSII3rbV$XMBo5Xm*yfXZzcW@v4@~&I+q_z)G4^@49@wT0kELOzpy~ zz78b^mHy`0-Ve~7F|qIS8(l=xhFN0Kq8pgAT#05bNPuN1lyAm6Cb!OF`eUSSwef;AGo_Xwv zo-Y-J5Sglys+vq$d>vn37Oo=gj(TQ-ipleHwr_{^8BmWbCjI$zg{!|CrK?r@Ozp9F z{Pe2b4)~1k&3C_h>aKN~m9+G1`+S-Xm>;%#xsQFlf1E#yDF!qjZRy!dAZfoD za4~DxOVieM-XKKYASZeJum*4n$b7*Sly#K6P}~>}4ZPE1UU~$Ja$ECVJwY2CK6zO2 z;r{Uq5E&OZQC|h`5p2A8C45PNg718(eFtBWZrZ0WPLJG1w=C=L@nulnxGx47b!MN$ zq(7b~f-c?DNVrC?BY}>Lp3gjT7Gq($nP!1+HKx#zt(DXOJosswP{Om-qSwxaC+XdQqjEJE$zB;?IH1$UJSmr zhY?fX7)pESGUwxW6R9DlZNk^3Y{{S)l%s8wps^~E^{J8>RtOG=`GnDWhfM=rT)VD^ zPb>y$5Or1V!vm&OfO+ebiP`U4#CB*9Nfp6Oz4U-jHFn_Nb{U=rED2t9?#84R-U!;t zFS(dN5z-a4y*pWZ?%-ztkR?@5y8T?%ic@0i3vC`wSg1suwFUJb)U5ebVH;^$8a^e@ z@9j7^Rp1tsJj7z0w?57rT&d7n4y2mh1-^FZRLxC3o=iX3-#R=GOW#}pPR;}~K1np{ z%+8$m?_OOUoc23qu+s;(5*MmRzxMCCmyu>%y;luHTKutdN9B7?Hm{K0oE&Kh^CQfP zKBA^foLdOv3#1sSK>=G=&vSdv1tn4B@~ei4xBp{yNtB&>EOVwsM_hA6@Vo}EE8gSD z%lUx~eFo{!6oxkr#SQ$Fus4a`d+De`O6~-~Gt+X_Z_8p&DipCYQlF$sFqOW3C=X)T z!KSD9$T%+pA1~k5UE#Y%%zURd(lLxZSJA~Hmenx`BLyol=PSZ_VLYrjSGnDfGx=R-DwW)#Qm!iX5*b5r0~Dus~QQZm|&PakiPXvuR{HW zeZn3OP9+D&(f*OD1x5=7lW9>X@pH@>#}~raM|)&ZMvqB-bcW?pl6SY5XqpaK>a{Tcv&TjA z*eO3cY;^fL`k2R#wQ6*?cl2_1y6Qk9eJa(thrW@2q+Qg+C-v2tpt>fO!L`~LFV*Qe zust#=|5&voWQVS`b9U*X!8bgU(hl@>>XMqf+u2Rod;;}*Q}B^Yo%=M@Fr6yyMycLh zdcgC`z%t2cJ;l$CEkV3^7dg zcSSRc10T+3zjsy4R|A4C$(Vef(9>S-;y?+*pyFqW8NUzVom+>T7fUyG#ou&*cRpS? zUtLoo#(PFzldR!lbhmBcCcH132)Nt4myn;_U(Pn4#LD5Jbx)^UC41Rj$M5o41(BZx z;6j>5F<@d$55%lW#|INe6K8fer7yQbFBNG)?;?++af@g)p|S?PX@Pju>$uuRT0~+io4nz`cTYhMe;VG?u zhKAcCB)^;bdh}6QRxi!9KbiXB4%3jg&#^&PI`JwIxt==BO<4YxJmwL3a5>a>t+>QU z{=2|1iHl74$F|5dCDr~)*YoKuPTC@XZTFh^%QHu*TlU%YLF~@+PIM&5^>OR+>GYZY zwQ>nmJbgdAq2K5J?BL6LqIZ`ai*)@2Nm|EsW$y#u)#YZ#9D5{sDNcS{-rnLM@o8l* zgfJp4NiGR+!mJaC&*khr+PXqM5&x6>qYPc&8yR`7HX7DK^Xj3_) zLa|OHTo7TandN}yQe#YJFV?Y?C;Fb@dNzg-ys(GEq^)u^5kc#+jnA{n4~KUcoa(Ekb2iW|9NV|_Ou z3diFmFAbv(IP8WckPtAWW;*OER%*9$bty5l0KW>&fT zSakAM!FPx;>S~gx)85whGpNKOxf40GuEUX&$P=r&;~h%_8`As?CEpdy=faYZ8`TpW z-c4E<%iT@h#wAMCY84{^;rn=YYh}GJ4s~U<6AubE2p_LQc95Sih2>TiKU=zbl^qV6 zR!F|rQ&lrs*V}7bLS{9dArAS*5qj*DY>qW5dFK*}NG7a??u%$Lk>Pi*G^jl_UF4&? zc;2z>6BCovrZ6|RC7=KE1FnKRYgLW$k^n56aJn>9)F!&)Hq{H| zCjlC5tnXI5`vx;nq})FpfVHdpHAPhdAQ+Vm&U{39=XYf#9K1oQtX0}Lk@g+qrWC$u^Ny-%a&T zmOfl$iFv0MYnIBH$31*x!}03B?k!axFz(YBNJ|+c7)zt4!=XNQCEyPLj(74(Y~8%r zif&Xo8BL(NrlJ&8OwkN=)ssQ|s8gP#p+p(d+s5-shxds~Tld}3)=Xa;HcMtADBsb? zwCN8%G@DBQnqX|qj5OwJMryM1Ovo&t8ZJyCrqpy?cKEny=bXU4RWK8U_3ndwoBoh1 zXS(z+CoRiK+IWFtotokhqOJuPi!T|UvYQWgjy>N?8!|8uvYr&TIi0J`VJtg1xkND_ z2TQKU)xI%?UxaqlZgLac5wuI#uKOIsx{wH)Q($2tOm_53oc&dQ9`7reoF$4tpqNPi zdke}DCNAU2kiqwS1%rvIWXEkc*xV&vhzmqsl?eHQe2rUOZ5M*WS9-I#m2dhP7Aj`s zRA{E7MAh6wRnED}CdNc|PtwwbE`7eJUVTGvwd5j@jW;1xbrKBwwBr+1UMvDI+X83J{MoZ%zFQ~wA-Q#SC8E085YNUWxHJG; z6;#XhNKVIb2tBdtoN|a>RUtv78O_k>*Zld#%fW&zOI^4PlIP=_#gl!^cLiBDRD-*e z)9g$b`=$=cGR{=Ik~t*_`-yj9kVP_6(ahWmfU+lD*!)8O&t_5gFXs({8QqaFd`2oc zy*$#aN_L|dFcc9%JL(|8@ewPeq=ONjhHQdm+3V^JN?4f~&SRvQpEVas=Vr5ZR3!>XNF(jUduf2z^ zwXSRyYf8Dtw!U&)z>)TqjxKrBY2J5K$M@U*)<#}oXafb0=?c^LvXy#E;))HO74-$7 z8PT{9Pvq&B^JdUShhuHuldRFk5l6$>l}n$F&HSh+>1vQK@Rzkw9^SKVCFuZOeZ8^s z$!D)tQu24UjT!=Hx1rsh)uiv)A6=8aI>}J{+EL`V@ux{M;t+~u{qQrbXWle!hEWOb z-9XXGGSfPWd40PdgNTi>riks35Z2@k)Y_zQ!c)eg*>Ol2x%lpzwEX5$ zqxPo$$Ep(1eYy3ItWr?b#7RHWW1KqtxN^oXpz7c|znCC&5a0auo+IvzhBD*Ls>S+I`EDNa znvc@S#VIJKcq942NkeWfO-|B)^qabN?*X5oNurjz)%YDS{N)VOA z>?m2UIoU8e7yH9MWY%N-K;d_T=;XOtG~Bo(#m?xV52d;;+9GCNM{zIDFEhdLq#VWc z{O~42mQmYNv!<%5Rt|d`~AiF}to4Cb*1WON`7e5S_4FrAPK`l}HNfiT#+K z(+)9s=Wq=?x-RCqYl^AaA0DH?C3+XV%KZi)@vu(%`W|YUPN8M! ztURBuEdJMGwcUrfDLBB4eu2Y!G9|6>Q^Nf9mLq8A%HS2(R0fXmd%?$zfy8?9t8&Uh zf(n%yZDBS}YU53q`ybj0{qPXoMOxWI_cxS4c{+6@@h2*Z%+>Re zGv%#jsMc4~KSX)bt_MS9DKd{!c($l?mLyw;8uQ`Ij2_v1mFPSlk_hrteN5P2S%%*R z&9Ie5mT;NPT93vG=IbK5=lapid~^yUWm086q*|g_l!pafdWsZ&V(y^8&efgmV)En) z4yZiJRZ}Jr>*Ogn)h@gy04en#!Sd(oriotLm!{2w;!o_Yivv78Ivb`+@K&WF6{{@) z#GUCD{vBCb&Mp|B-HJ3D)@ieukC4pR=Ruwps6YD)55+r;`Q9x_=db#ba4HZt5Ou@A zwkG3Dz&Mr)hkr|H&GIeDe+l8GgLL2#@_+}wB4H)mZuw;`C_(4r_FDIJ_HquyO`{<+ zAu)_vbf}l@-lv(*o6pYYVgt1DW+^4_r5DTPNr_`ynTnygpjzk>O*XYFafp^jML`|5 z&nJ`Xvr5=}#+6cTY<)dd;3o{upc^jCinm#k&FgBz7wevPn9j7Ho=-clmnSuVa+#i+ zWVo!ieIT%`tjb%nyv79q%T+q^fmU&!gvI$c9H@-GKJ+Ug^l$I);$Ar;*fnZNWL9#O z7D$eV&=S{O1y&8R(w%-f=O93yQPQSglEyaRUqbSJ<#oI<_q$e=8}->Hs#Fq)_NnTc zBD>F^7F3tSh?@rr>c9jwXBWTK@79xX2~^!v|oMyNx~RZG=aUNqC(w>gFwio@8+)r@@XS%2gDM zvdgRCr38z^CucvwY-i49FhTWAS{A+4cR=FKoYfGZtDwo*Z>dYaIk2{%^Ta1!dCKxj z3u-pRY9Nd~DSVq_sUU>1<=^KY;YYaY3!tVP4^UWoOTH*S<;n(2Fq#d!C~w{DKdsKe z2o-PFAOm8*X(jZ6il02#aB{(h)bW42zH@Fw2lW;k@?hWc-{w^IOKj+%Ts!s+Y@MAV z{n~<>wPr=%^OwRe$p4?!6ceUC_Z}wtEGds?ygc^53|{Cbm?wwl1`n_M^A2ysvl@5y z^ZT9lZI@ste`c@h(kD0id(^zOj>dY)z-;9neu(+GLK8oklPM+gL^lhH8c<-oVaX!q=n7>(`B@?>$7N%froZi-KI-*)qPq8^tR7 z{@vsYR4ukx5?j_J9MY*lq6XF*F+}4-zi*JTr0A^VO>ddQ?qt;O5XDXTzG&jXd?P{1 z5MXJW91A91fsmwuo4#8-YjeOKm+zR5AH+t}-K)jFyEqy!L|nATh}tuf&(}e+LG+h` z%yfo}gXVcaCWmtkYHuJnWYsZX#ZEmDGxZ4`8*2?&#A1FNGyL`w#opPF^?@{HSrKab z_?s@HrtHPG25F>!HrYppqBxYj8$;F&XT*k|Md)GJ->{*+K&Ob18wxQ<=%UxARMp@| zjJdw8XF`wTOQH2LR?pT%=4peBk!O%t!y-#JWcAGCDWQs_CG{CLVm*gG4;#d;+h9)_ z+s#Oe3FHhGD)2+ifcfU1A%uVqZOD2V5H{!>Jg8{ggq7;z@3;INbMb@Z!{0P&&dx-z zb)w2mfqhS&Zh(Cw2o4xV99hvs`tQ}$Z@~ZIn^6Ytn?HjLT$uk!nJdGKA(kwW385A| zlL;Xo<3n2r4;Z%f8NO#*3Lk3|5~^x8YdW&}8)DIh92HCFFEy({LpnE5Ki3uCGZo|V zbOci~8vpTo&^1g_G}I^*>6Z|=`GTBn>Q)5aWQ&qan^ifsm@l5{a(A`;MU&#Qi`RoW zeWIpv;viLn%nt;P4NWHWMLKPH8l_ul*V92<;lPh|A`gl1epUICIZo&+r*6$L>+w_J zT$%oOjuDTZs>D`sH0JZ1t=KkxHlLXF-|bek(uQ@MH6(lv@(f#8d+Za&`xi19^|-T`;QmFMIN=OszzV(7#7H0! z11OLv%;}Hz#;@<{!@qaz-VgB9Mk!=Wb0_2u(>qLPseJscq*)lwLH^mfdW?rYcCltA zc9Hzw3XxA(&Pt#+GtB=~So7ZsllL+J)-g|}jXxt1Q7k3L<31*d?(oLZpeEW1XSq_1 zr_fI3(~%;&!mBkF@qRPJ+9C^hK6Q?II{jmXo_sxhF}YuR-gst+>XX{!4U($gGED{k zRHzcK21wc&107xX3z;Ul6o^dNwG({&c?8C9%6@LK*cTbax{%?@JI*s&j`74*-ttQF z+1Rp%0i9T`y+aSI8Z$ZOhoYfaB~|mEKHMmt#Ve4*;00~-o=~GDS~emfosJkFgyEx# z1u-n!yq_{Xz9xvlFL7g++FaCKe;g|-vdw`3sm$Qa2KTY$gtNW|X4NV| zAAyPCJLqU+AP)uY0!*ERkNWX9wZVImV*`W^BeY)3?MA9ao!3+i;$G1XM~wig zkbX=GBsZd@1_C~x5GuGs611d0cG=@CEBU5>kynd?$^X+Io_($9(7G}33Q!xL-O*zS6cP-7ltP!`;e2+L{{&%=^- zqSdal6Q8^I*`TUYn%NWSU?KxPQ0;LOohhlE>FIY{S>|aH>~cM> zMV4I-Y19I$E;BnQ#TF;G7E>8Tv(@Bz7UXGatMsEvXNltMmC)>RIg8Ox%kye)NYM8W z+?ZJd&C@FGqeJCgrr5u?(4-Y%$YIrFmZsTd6iF+nlDK$GTWB5m961+TG}4Ee6NVKf z-l8_lc)h`tJA#nIGJl~0+&0dBLWc2tSkeATo6VF2^9oJFq)|ZOJ~hu6U^GqV&v%-l z{N=HYOOEMpVCie=)O^29$`xBw-g3-k7R^@u%cS-%r87pkKTYyf{5CnP`P<}~&Htat zehaNfpH+@x3lP|38jIZTuVe!!=lGo^KNcL8k?B8#Z*d3iC(A7};Z52C(0X%wV)l*xzVKmI6O`qwv6J?*8>$=A2oreY$hh zx-W{?_H1k}OYG2xcyi0PE4)_t%tIL{!)mq`(AGkUT>A#0*+J>I-M6%ZctSC$H$#G z^b#eW_Ylb{F$ho;0r@g-V=VWMpgYXHmkN~-{l@XTh@KGzx9sTz8ddelJ7{plWN{@- z4;=zC)vQZa%ZJ@5Yy2)$w{G#QwyWZF5NQQoFys!J8d7=dDfseNm5ev%0%ZO214#01 zeVW(o-$?BRHMMIWF0F*Gx%jQ|^cr*Toz}y$@%sx4pLmpo248QLlH$zyNjiUCjNwz= zx(PaJta0f=Y1)HT>(Pyq^p3qFy%4a867ohHWLjM&b5O%R7`ppqSDrg@uks1e{c}y= zOts5&s@@=;=VA26uzMH3dq%i?|Nd}Aq>(2WI9Lv)Qc~P0u0l^0jdbjv4kqn+D$z?= zyYnJgUvX3PSS|$Cuz)g#KsY~bL!b^j6f1zZU1N1QJ+c^to2vJS5$2igvLV-#QuOemg~rfA@1r|<%%HtYwd}s!H?B3PWZswN zVV}3?zbV!GgQ}?}$x~b752^|8CXJS3coN~%?4Fz)AK+}i$j!ZKEaK804R6)4V0W3r zW8YJn9qZDBAb=|c5kSQXfbL;r?PenMPa%uY=}0|$)8wvY=aj8?bW)6Qb5Hx5=-da` zXtmFvq-1w4H&zWj#*D9^6N;6~^z|MA8PSqkSAe0Q+Z|L=aY6oNu6kMCEh2Vodwz*rE`j2 zN9DJ5MNuFMF}N1EaS?3aEEWhvZ~&-yk5k$8U%0xB1Eta1Zz9%LD>bN7UNI{jpP zN`{IIvrDIA%ZMyo(+KX>j;y=9b_7+()amaM$)-%p$)+S=0WugW=SS|n)rt8(oO>U4 z-w5D5p;LpF{Y4k?a{{uvR_#W<&J*nw2V|GR09-KgD@c^!Z%5u3qb={%kEzclRkGCqQo=322kySBkXyTkv`)OX=Q3>gW&z&eQ)0 zev)Op`%^F?5G)wXGac)=qETHR`Mp`o>}rMnF885DtGD_|%2fMV%Jg5r!$`nNV(*H- zfTs#6ojS}p;Y!wvt8dfUc0fb-rR!#vy=Ve&4v%0}&P6#mah_x|fhsFf^^`~d0JeLl zcHW*5B@YXryR%4nTe3dv-g&gQaujeoj^;^sXVAbpnnjlx4M}%*aP(Xx>(S`|q`OPi zIv1VvfAE9^yN;HF1t=Gj?T`W~?Y8TWh8RwTV7pNt)WF_L1gq4}odywsOgK+qX?Ea} zkZpnBkbwSsd>M8u8pD~&y$aPi#Q|h}1*B2MRsFjFa3uT9UxWF-)uxVU9Ph(~$R}Jk zaDCtSlpnzYRLGKP?>t8PlgT)Tm2*e%I_>egp%{*%-;875#4l=iP}|jTF7!f8lD~ZC zIlqsy*pWdc+-}ol_!urama)LTjmaKp9Sq4!Rd%S<3EPAxKIUy+}i0 z1HbEMbc{mky{KCF4}e%NIxxT|g+Bq>yRnFAxL3uwZQtD354;Rb8|*j-Y;h17V{=bm z$S#w^3`aLh`mVbxj~v4i533eI5i^qxDf(@#2jauW+#%`CCY|ih0((@H#{7ElJ!nXy<3^vb& zHte&cgS^1OT=ZY*1l**nxDiKkjxw?W06F8czYTT0NUQLh%I^>~Ad)g6Um+ zCr=V(v*;Q%>liw(;3^eWc`RYkwAf|?hZ$@_BBt)rr;$}Hjza_Voa4*yZLdcQ;18_! zhAR6@Ew1ILOqWgsvp_%Vbu^wmDPUq2ZFO7Cq4a}BLeV&W@Y(Pt&T$+Cg__Q2ldUUd zBQxQB@U&@VCGBa6aw3v#wm|~XSiDDh>#>QGekW;9=sJD*D7c&Lp{*M>1HX3$X5K##1xerwFvRAC(Ml%5(0Txu4&3zkfe6g&+` zFJF3ARzpJ)6T`Va&8Wpp-mu70$Jz`>C*{5!$4=`CNN6mw>Fbrn&MwrEW2>&(oS%Pq;(;*=yV`E-ZUT>2K82NtbwZ*? zuQk6BbD!x7T`il3n%c45sFBGzXY;^FoTg@#o2n_4V@?Ibd^YEYl5v?%^L{9e)Si-o z5oc4A{J7-WBb&vLfJ`G3sM?Pe75Np`+p&_IkKRJ2L~@2Em90PoWkotK*kmU%@=vCRuUYcYE1R(lCiU>cL7ySZBEzNj!mqn|z)?a}k|Rst z*5*dbAWN9R8U5Y`$eASbJIC~XWAij*SutRl=*w#Fs5;<|E9*w4{);Nm0!*dv@;o5x_bMZkXRolJjV z(1Y?&kQM_~1;7=F)=WKvwe86Ys;lJY8y}e%o23wD!~I{W-T zGAsOtMgm&*#uhVKHLwiRD;QFkO*5RPG@HRwBq&jwhV7zM-q2+QZAb7 zxyogeR-o&R_H}MMirr@+R#uZ})2STgP3xZr(W#igyfQV{apb z*ugJ+0cJ}Ka6hZ{i|Q`Wn!afEV84W|mgu5Ig-485^YEl?W;N=RgU= z-b3pKqBGQV$LsHuapsEXdpOxK0n1%Pg5%ZfEJ`5Gt|Sz5$rN+%S9N>>5e)|Ju)=+^ z8%2YcQA~0~46N9MVFM7`gka}yJHOahSz^)M)$8xq65wB{9R(p$2_#M_Lar)?=ZFAu zM6ToK>N_YH~rkv64S zi$>+W#r2mE#B(Mx7##4*sgurR1qq|3D=u08isju-|9o^Brsry{HZ zX|GIeo^H$=P%~)PC|{yj_sEZev#vdfKz9vA*HDw$VyD>aO1&DT!)4OMUHX>_9e4OY zRlsDO*=t4DIjWXh;qd#8Sgen9s5+ONSNHFRqhV~S=YH#Ff)39pJV9g}#qz89)NvL}fQy;cbdhXHajPd5 z0}41_W%(R&mGcsG5V4&c90>D~s%cg9&1K&&B2OZmr%WJ(F!G%I>Z z^^%R#T-BcYxIxj*yp9o41cSega}wdQHdhfwV-vb-bQt9aF1iot>W^Rcaenz1P~{&$ zi2nwv2Lt847@dMd5kY`@2`PhPc}DGRgLc0^5VkZ0z|R&hgLElKkQ&Yxk?-=~*p>Xw zg5DJfN#&{)xpy5Kh%_5QP6cpRaD=Y%XqUYGUO4=TUth zo#cFpx|7FyoW{dJg{&k9itnR76PQp#x}#ecsW*3m<>fOvX;txuGIY)0)A~Z$ewg24 zrM?B708?m%2;y2lI1QtQ-M#p}xO;Rx`n0Z8avt5TA-{6o?b@~i7x=5@ve$gC&h)nI zXx5fbZX!Wjn;!MLU9Y>7JE!Td_h+vUFGp_e&)JisD>|2tpwss1YP7Sn>>KbA{?*PU z0e(h3-1hFy3eu+ZmEGwVJGAsC90v_NYGj2 z%5!OJGK!uRb7bh%s|#G9^yRIkkHywa3}^MIi{s_A>0{jjjjt_$rh5VC;q}aEaph90 z=kr;h`o_1&=04dJ=nLrzanCP)<#{z6bxIh{ z4oVMgPTukKZRvDd+HrF1PY0b^fILsQ7U$v<6!s+O$hfa+^z=fre&vKN4bE@-R-s%j zzieGdGEWZkRogmnuj7Ad2p?s3x23SV@}|)|_enfTv)%)$9W8a+aoJVRKfKDWF$ zTF`64iT&b@{&e%x)spLK!{Q=MHYYjs4EWTsYK=#hoxmsK`NSl?{p|ZXzoJveFJMdB zw^Ka0bOlT%9ZF>&3=P@^KPSh{xu8U$bO}`lft%_PIR+w4(Q2wVOF>ZH;eYy!5{6di%=o z`QF>H_2p)btnN0I=bDxE{smb3eA9pYtMB%VZN>h9a{g1ezq;C5C+>soZJWCWiLLR4 zMYrr+lkuH$`D8gZ`@{m^^`g4jc?{R?QSOCFx8CgwS1Ktj^5I77?f17g8!dQLcnkpW zSIoOQ@246C`HpUYh63u#z(uQjldacK7cf-`XDdkbDeHM|C7^0`Q9~6dsM9` zkzM%_BMBO|EE(#{YyHYbYyOC=03+T(l@C`%BPUhEE(;N!-7uBqFr}rL*i4MV%tT5_ zCc-B^eW7F?3=dS$y(Zhmv$lG6x!bud4E88hdQdBb zNUs`#Rjaxa0)xF@yp#K4lvUJ(iZoklHe#dpS)SFM+>UzrHacP(dZ9nCkmEAglphHS5o zO^fph#-So@cMhF;Rc@nvf(cBpUgIYPP=N|Hg@_uI3)-+17}j-AbJ%X$z>i`}Rer(& z&+0vdy%G%i(e{$N>pm)cY@xJeB&q$_Coy9NajufKL!p>866>-E8kEaW+!YwMkmGln zP;+5Tex4r(v>`%;`xyQD2{&+i45ICk*3U;un;=k?RlpszzrY>jC*Y1aa7T?q3#|RW zA&BSTn<1Xlh`OmZe+T`Q{X58g+y;`ksN^?F!*BRMQ2rk9ca;BK1wKbJtbIz}FDP!= zpSF<1a2CxFsDVLXFphs{{g0Ji;CVT-3x_|m zef9Bxsp?HryKzKT6)Ey&E5K#z2!%9iW^HjtS6+a{cx@a+)KvZ)8);ne(#q7i0lJL! zIrzRe`!IUHy?Fg(Q9J(GktFOYSh@9-J7dBdBlQVn((3)p)7tG8Yy_yB8`Zg42laoC z9aTuTnetiP_M4b?`JH{o*^uIXlA(mD}{>UcsN{A=ryD z7R=7XmNyp6$~5*mnh{0RcP;BZHixOFgxP(1FoXD}kp0Ffzj(PfAoq7LV3fS-bLy($;% zq1bYu+M@P;m)R?M*Ub}v;9gGp)AJT@riY$FZ`W1=|4Z#R6c@)u@B62xSaHGh9pIJL z^Yqq8-BSZVmy=kwjOnRm@r;X^n9&x^?q-HPHAeNKOJUf3y%p$n-?E%#_@&UHU(_}! zzLuZbHkVW)wQ_rJoM%sMu`+A&Y67nj_K6zl(4awqN3nC1teHpg!%Dk^+Wgt>(HFL#QHS~+>I7Du zKR<}cugtk0v~;`$I(})!V^}@TNY3UiMB4rKvq@hUQdh8cUu|TQzcN*3t|Ea17Q>6A zYdhNX?XII*zooZrJ5adjDh1Q7K~dTeH0FMGw2+t@Ij(bIuJ=J7(v6$Wzj!OLZ~CI4 z(t&3|B_PSR&`35A{Qq9{3odZr^WTiKF;xT-srIzS6gzqcwiN50#`Lp?GCSZ94_=`>uYObTOON8*n4syVT|Od&Xv( z`iuyexhPCADkkS{oi1FWs`$%)Cpw&`!S#^L1o;u4<#Iu1M-sU_0P`qvIo8lzGUd=} z2sdN_eKNZE-GU9@=(j3pw%r*!Dq$mpNvhfpYCh)?9VYBwcB*l)U%OX)Ik-Dt(C<#F z|LW+NCnmk$n`T0Nda=`1Jowds=W4Ek+V^a3q_j&rHb&=s&SU=G`du9boZI#E5T&uBX~NBP{vpJDp}n)> z==~XI-@He$l%wc}Id8M|nE}sI(>UW|IpoB)VC{#=P2FZvs5pU&mVne%PucYaoQKIl4R)@7s=b+$xXayLY9^H;Cwal%Get z6HdywtN7@&KW0m+a!!*!w&>x5*NZe=dXm*5>gj@fuYJ~la`sSV5&GU1#0BJ(Es@~4 z0ly#ONU_~Qq%n+$NBrRgE56WV5z&<` zV_(p5b|FJ)HpLU1Vx4nDKS)-)E6R(1HZI;P^1tuGbGWG zCIV~~XH(f%e>ks1ygK}==e@Y+jOdim1u2r!%$tT^We}L24PSN5RO=DDu{EolS&}s3 zjh-cjMTTP|YPUfyS~DAJZxoJAHK0RSbEB2Z!l}sdYgchoVSuJS+Oo(mkdqn7+wT#E z!S_@d^MuoQHPB`$X3ZS()k|G;UhWqcexFkFn#HkF$G*#MiUh}qd`;Lln5azOO&Eul zJgb);^>gM-t{bW2=YW zTy%BwXFJWaqN)!=twq*+hdjE6B#T)tl*EXggXu(vCVTRv!11xC%9PI(MiI@Wq&N$s zl5bZwT}JCEL+P5&p(p2Q!p-&1($n}ROFgO7l2Sym%Ly;GOp5x7ZE+AG>z6l|(2#=# zYKY9sePMN2!T=k0q|x)mnzu^{lTE~3i6FFt)Nbb<8!w*@LKb51(IX|T7umDw;C;v7*N|_`p}m73X?(L^fa!)5 zx7+4op+T`eS~jkJNh#_=Hl_GE>ayf8&|U9(w69{cd*2?HF5L<3NPX(X;S(Ypt!a5) zP!V@J8I30Qts5cvtlEAK5S;+#l(rAo*QEMuo{X+yQq`09uR>T!(`&o$uHY>4v^+o2 zp-cyh{bfX8wCswMFA(SolO1jR`dgTxR!UZ?cw7g$B4DaI#+G(AH2L7buH>wIP(!>_ z;*W}OucjAWzOT@FtrzxQ!5`}|&37{W(&2D)*x6DPQO9jY>ja%FBj3O^th0j~=ssny zjREZcW1yv#qH!EM&|U9}1+|emFB=H=R6??A$hnbOFW|Pp?ZxTnb+IKINQ5Of`69O2 zy=zN&1;2soJZ4>(V*JezyE(`U6s{#&sEzaeXvqgHn3SUStBb_$mL?B?TQud)mMLrO z*ZFSj;VoA_lV1=kdGendnd5y==JHy-Wu2T^mxJB@ULpuuc8h;WO>uPiOX^d}N*7Pm zpcI?Ytu^B}w5#aiLBXA^D6ox8TaE?!4$dpbJvq=vdq+3L*W)esep~*h|9^J!Y>{`l z-!`1t+c+^0LH)h!P#=x`83qM~F5u_3zHOqPptR>M8*(u?1)e<*R9h%FUjMQtm8`CzA%N z>bGjdxO5iN*`_Rzi&M{J!D$4y*NU5HUipG z??8SwSTQBH&gAgI$7Z(1uAE*~R+D?K2yN2lFE=CEhY;I~n^2GVf0tXO{OS$_r2|BK z>3L&XC0|=8s{wC7=cTnjUNlUZLtI{AMoGY8G)QSPPf5E&pWBV9uZ60g?ww+Z&z*<+ zgGH|UWPrDEo_7$B$rYjLvKSbP9#mvWr&6{r`(c@?!6DVl=FSi>-bF*cUOsU#iBu2} zVR4w0|3dZ{(z;yKs@|u=k4@dxOwq;uT(rWYtj(FlmgzYc+-hFj8p3UOf@is|1InZg z5mM2rp02=mUadri8rquDU3H_ny|*wEC%V_5g6J`^Tg`r2jeeanQw`%#4ecPuAKDrN zw+Vj?RI2Z~Tdj3kZG?cu1NrlD87L~nKmV*+twdtR(*~`5ZnxTu(T2~Z{r5d98Dq!A zqR3A-cN;lR8}P0}mEi#$n*p6f1*sa+UH3ui>2gdxcN=w2n>5gnAO0=f>_CNPhGm{M z$!lAUTR37B-`s81CUF#o7;Z=!u&uVtN(maq26b%iJDd5oZqpqA6nCq~Ei_l+1-f_P zE*B#iy4Y#E*{6U-js@4ZaQx<2m{hx)Ck`I89HL2FzH~KTm_;8aW9IsVm{zA|drC$i zI#f$JRQt7hlNbOfy4&1B@4FQRI&kgsJruw?OiMe!yY$U+&$hHe7&gJdQ9E4zT3<3r zPW{SNdzEE|n%KuJ!t;@`a6^d0sCqoR@aBM%5h`PqBhwzudH&8|=k-*l{hooa_4=AR z?8~uEXbz1uKavWqmvyNgc`<(CO84b6Nq_9^aL_)N8?TB$W0=@#!{|)e=Olj(xW%|oV&k!Ypusyr|r)Cax15t`y4T*y7*DWPlMNOO@2F%Xal zE7K6#!YuqgRmbYYO7%3sr92Z5)S|Ep&LNY!U)OXWkf6Q5m|G|mjZB2O2pAf<7&6J2 zen>(=ND`Xa^Jxbh6axWOM2609B$7Hk)Vk6NJ*%+3tcI`J;3Hov7)V5C_t3<|8L^w?`UB?M8<-`6vBogqyL!Pf%07AN5{1uYtF=Vmc z;$I=NAdpA|!}<{9`f`*fiits7tf_6#vDT>{qLG9mRnizm4QJ3s;OvbcNJR~ANCOd8 zJ5yX~M1c$~Awu#z&m@$%0BS2r?XN7q34;mdVPq^&h1P={*o8m=Ill_k@QUu-gg`aC zd=Um434;0^**!Vs7{6?G)GGOk(x_A9QI6!_2HZa!xBw&!28Fzu5A8UEwMnafnW2V5 zg2bBdvNlh{bX#i&h3skya~BMCkv@>tPyfdQIv>+nV6T+~`Cup@xamG56@@{rEYZZs z>MGB(!qCjBQ`P|XYShRv$=3ziYt%L_1B4DIjNc~X6AgCf>`b?QKGS=#`Fg^jI zG|c|X#VFy>O^k`+vUwiu+F%0^qK!#6*n(j|#2~zEz*pFXUc(i7jT1J=Q0cSnwpQp^ z&np~3P&_Qb+&0LXtijBII%)k-$eM3sd7zIqLDKv)_6#AP`=b96r_O?xvY0B6I0 zH_ehZ7*P8I(F#7vwTB28zc3g@2-M!q;*<33A?6#{4K~Ak$J85!m4WfsNH#`v5+qKd zHg`vy|H(&M1k9Mil|e@vT>PUMo;V;z)~E-`|BDJ1Yg;gO6q5gNEF6#FFYWnqBw)XM zY%Jq5D?D+1uC_l7=JS7UFnN5E-=nuq50AHidEc~+H$YavrSI~sakXm@CDptRr>bS_GhB^QQf14(J1tO*;N|tQxW0l;_UJ2 zqoG{=$9>nMncBDhO3wL8LdeGct|I4Rh+h5itSR+cQ$^!SjRwlY%}MPbU&PNItuA*KxSH5F`1t+>hmnQRjf?4xpwaD)QNj)*JTY=u{YNTqpF2-v zeczvhF=;Eg?~;i6z0CTbd<2mD3C61OhY;k=r?lybYq!a8(zd^)AB(ztv?8xxb{{zR z9wrsSM=;YdQqqqwaZFCuy_s;_ZELrcEUd3zxVwC&rnjc#+~2Fc-j%Z4tK`Q6Qoq2| zZg=Sit>iw|(H=cmeEI7BKH~oF&zq|teJ0bEjGMGrQfn%2KB>V%RliHSmU#POMn7;K zy0RU%!YDwFV_>SSX`W$V8I|v_Km;YOaZDw%t?O0-QJXt>{$*W= z_aSoyz;uD&aj5H~N(P~J8>`kAN5NbuobFQ1UQs!-CxJVi-e5d$D5Ta@Vd6Zs8LF00 z-1zC1#(S8U`JaeOCtQ6_vS$4YuGQW(zJ`1YQFvHJkP8#+(T zMTur)p{HY=W@HlXvU{=PP83G-V%u)obT@RpX7s$9-sJWKUE0(v0JA&=_CI681j)Bi=!8Nwl~e*Zu9 zIQU}AkhZ{|pOa~yJlvVx6iRNTKJ4utz;ZtiHo#LWXjp$!{_%B3Xdsa6fvEk)ON3ub zh6;E97N|n2S_!loIqaCo`*nZWo7yiau=7wOi&RnuGYew_9Tih*SrS+yn!yJjNEP4M zQ{YF!mT&1RZ&uMK&GPw0`Xa1uNw1Y$3)4sY^5kNF9x#?=lm4#Ddo!1MvTW1$93V8R zJV7~p)C$K_MfJ8I-IQ)PXY2!(F|&8w-C-=lSZQcsrbf&Xq$#4bwe~kRAm!{h$NLyI zQC62B@-XuUAkg#>O9(V9@(iaQZE-Cki~_AkmN0XqX`+k=mtag2zb_R-&~sDzX~JxQ zmgdK|E5MR&!UJ=>^GMTE$f-mbOQ3M3I0e{>hDnwq8U}0mRkd()bY9q^tbZ_G+6N+; z`v}k*f-TJ%Wr;I{*cx4)&w+|~h<8u$t-{QY!eo%9&wwFV;OF3|n55bEX%(&%Qz%0% z0U`?i!uXenGeHryKOz9E{@)^^heR7?y7m7Sz?=s?2S?jQV7^|+a)va05lsDu)>j5! zd_Pr6GQ4h5__--OCqRzzrBASgI0G15Jlwn|&GIYtHvpeM@gdlVGmL^)io~NzdqyA0 z-$15_Gs5@K3b3*N7UTSX6=SKv_G;B!Nu8dqFQiPt`l_wXd!|Xyzm4zeDRv^09 zT7{A-n!)Y5h{zSUpdcKG6rnazrN>L{PAfhkwb#YUlIQ2gR`m2Y(dI94Ai0M$;JydV zNp3qrj+;*PLxTngVERl-y}`}b@K$uP-Uk@NhIXjt&Bd|GVDgxr9G#82!`LY&Z; zLG@cu^rK^o|R++ zilZc93R(u;iaQM;t*s5+N?hi5;Sfq?->Rn?O{(Ze;MnGM1;fe}b_L^LwVMr-=XGUW z2HpnTa)7WY>Wh^OZqB4clTk7H&B7f%6jI?aKj+)f9xgpeo+Y(*gh}VzDyu32AJXeu z;vCYO*8QNTB>+A=v?WZ!O!}kZmWBoYZxtTELFW3lVS`etZy&dBieHugr9&m}7@*_B z<=;9!#y1*RNd6xr+zPa^e1{)i6jPM-zyDo4#fZ^C@bCe| zacD#0t+h9q$He5#8qV;Fb&nBT>lYuOuhntybeYz(8B^BW(os!zcI{xDy+1a8p&3lA zy|ZuE%X3H`sU=wQ;hcU|{={XOv5|--P=CjA z36WSEi#wP96>6o=b1*jNyMEqsAWEhr$qZGzc1o>=aEaGZk>l%PC`=8E+(m5yL` zBn!ogWG>n8HKGwu#5rIh!MX@9Sa2|WR@q;iu00<;so@+YdNod{$X~J$T=N~vtgYP~Y!$unmB8(jm$y%yv zw@iIUQJvOL6SO|0NI8IOg}DeJkgi=k9WsnP3yL1yW(Z|MM~JqQVL1#Qdk80tUE6Od zR*6%mN{JdWpj@Bl1jz=A#-ANswS#_{P2fLVl`KZ{s~SP;A&lXrq6wX9OeDdHwVb0t z!~kNDwP}{al7;I7)W7OgA>*?AQPc!fNmyv#qFJas2BXsLI!nCF)YDsE@~BZdoPiMn zYzz+>q>W<3MKi>mguX|#ELcTcE>C$g-iqD)%XFMw`dp}LSJcwu2TneTzWVG$O(eUq z2m~N_G=|}W$>KDIjjA-0aYZTu&k#2zKK`%IdsiJxrUS5IAG_AsqU?v(d6l`sFe-#1U0S zeAh7cDE-!wE$9DD76;Adqet(Hx`w}WEfMtxo4sDRF#A2@ga=EYIOD6kc;=_pl<$(FNn|05-xhf0~HL_9N+2gH@rvHlapMbwF)~# zqlup{64u-%s%YHIsw|Wu@PG^~PK4Y``97|%fJi8*87iWzHwRVH7(&35Ovxz5p8`h;)ncCr z;h8cGUQ}tHv|MHQEW*^ILQ*LnPLB}Qnhr);Oe-=k;-yl`kH-93L>wLc2&=3hUX?k@Vn2Q6OTTSi#Gn3)MtwzOTtzIx zlPWJ_ZBnw#c<0A)V#1V+{R0V=2zv6=xS|+&Ad0Hecn+ExSsj%mx*2@{&Do3@d_`J6 zBDR`pnn_-SxbmFR z*HxX8N-k1irczOn9R4S25%fP%i4a1_2^Z@q(%@D9Wr1!Tz=C*Kj6W7kcSa;W6Wy;y z?v!~VCwXcp4QDNOM$kZ*ZK?>_h+%fHgsMcK!q&wW=SYsihHU=_i8djj3}74w(>vPU9pe8UV_Gx!@@&Gj{;x!~%)` zL74>PpOm@MX3TB%hcgU06vd)r$rRAJ`7QE)N0`m?u!_bQn{YEo5N2gyP&U2UP1b3A zER&bf`0gursWy2Cp-e3CVOc;rUrN$jJ4Y!UfX^KeS~$R$o*f89f8ksW-YZ&H(h;=B zuqRVe?@KR;G{Tt0k=)oYD7tP7#zaEn9I>hS*rWC<{eC@#_<7?Dtbs2x=B5rmDGhRk zdXbxqraK1&OoU4)Afrq}d&2UTeu_6S<%(oYw3tM>7)?2t$L5t(p-MX6q3Uer|7g}g z4a>%u%CEKRXW3)Q4~k^R6aW50a*ZM?JMFz5A_y0X^M#KFic?tdy#5#bF2qiJEgHUP zVFKrrU9$bfxPUaxmSdum$8$*TuS(FyV5N)bpuxKcc(V~rNv=4@iMuoeanM>*B`26D zA(EwP!E@zwJMf>WQl_-CUAkM^MxX^&*3WQ)1E-?u+Nk7N?kK@ZW19wXjzqhsBsRIOPNECUG z`7yvigPi()1H>e5jFx7$LlsBqkJ1nrfKoD5fKquye$Y5ao_P8#NoWf+1?eqq;|EMK zJV~z9e9jH#&>+wv03~ksdAe9)UeW#oPz*_~$HjA^WXHDgwyvPTnJ4I;a~mHtFVbB{ zRZS-<+A6^T{~(pKeFcb~R4MRe9n~%n6Qoo}QSH=tN70|DagL%HsR@ol@KDlRSrA0K zt_2a1@hCBaJd#OWQtkMVM#>PB-&k7k%&|s*rnLN zML`hQ)_^pJEnLNg4lTS*7twl#LqQrDd|)-UbmJWyVAc<_6Zmga#JfP$agLq{IgS{B{jg^s z>EawO${nOKVM9}+CS|zZymN^#DW#|i-$I?K4eN0W^R6&dVR>X0*GgKz%byp*vsIQS`yq;&F=av zjH41YKY_$H_SyG%9PEo9W#BzwX`@{p0-`EzjCHp;Ty z1`-m4FfO7!zc7(j5`=u*5%Tg?WEUV_k*R(`YqhfV?A*pM9-^~Zy;K+%45v^G3RT}d zgqNq3oB_0WNExPxRZyznuTV21LU^~Gn_>JwD`EU;%)N{Ih%th1yUIh@H;_YwNWOV& z0lx!*Z$haMHsjuj@hi*t_7Hb0!UURRYmNZhX_>rkBM8J+%!e^qAp7=ZHCy~3`SAN4 z2+t+<27Cj#z#Q=#5anZI^#<_Jd+IOK_RTN>Kp=s?1M!8Y{J4QM)fEPOHTI1Jgtv}rAfJ?&!h&~K? z#F2f0if0LhVFw*Y7<7l6I{}|-kbRjaf(1n0`k|cBWB&}{dcXS+s1+j{{b>zuo0&gyZXj_T0j;rzSYiT@qX_ekgmBcD zpx~})&$)-#$G(SXKE((CY+LntwrXB)3`96x>m0N?TW(jjE7JI1TKY20F*8%~1TA=R$KYY!4PN!egvSI#pVha7c4q3trD(5Y)V{;IAS`0}M|{tJLhilfwn9}AwZ zW#Pg!6jh!z%baWa`)c_2(Hp#0Q!&4V4aQQBv-Y0zn?PWBmv7le0GQH0Fr|OP=*gEp zD^KG8`JGPIkrfhHA+^g^``T_002*7~h48B}<83HH=Rp1B&+uKgZ>v@WuPUS2cmHRw zFY=|_=W_#AAq%#_3sWkJnE#@)7QVTPAG7+)+oLKQOWq|k`$*|RZj{xBsn6=FW?=&Y zV~6dV711u+I%`~nyo*%+9d2oQ-Wc~EDe%4vQ%c|n{{hPGx7>@BQ3vCies|8;+#Sk< zcryfu)GKX=rvsqP3l3PdSP-FSA0?b+amup_naOiaJ7#3MWq3%Z_5p~&m9{hT-natR zkD1YeWF7f%Psg9nKwfA)}XpO+UgFl`VK{LJD-_%aX%VuUqgN zxPI%A-Zc49vd!_sgyH&lLElLwZcNo69bEnZ$BS*1?2S)Uqung7`&josWLwHy@_CoeY$0-8K2$`oti8?j)ALvV*hMP0h-&caG&2oqAf13A=`B-N8tlz&1}0zz`1raGf2 z6fn_LTDW&Y3v>HBgB*&&IhfN>?$@(^Tm1#kYC|l<-JZpv#!U@pWz@}0Xs^U=1?r+G ze?YV{@i-45=t@gDN0kP40VYZpVixF(GSYYx_3lA+9-o($mY>sw5AS{OMfX8N6(kgc|=y4mcd;XWh$kV-F{W;q#BJ4EsBiRAo6b zNQ&ZPy11K$O2LDY-R1^{@G2R;vI7>9SclDDl@>>+;I|{PEKE?>^puaSt z5dD({Msgy+nM2Y4IOB{0nU7iLlHY)p0zV;zH)X-v;ymCWi9gy>-QCJ`il9(N2K zcX8P;u(Ng>(m_*fGqRw(m^h0TW@*%EkwEkPivYzT1UEYNxD#8p7l@dGELmx=#aR48=DY|| zDN(7kxRAVC(8Io?a)MN7bH1jKyo8Z%f~eoKl8d?UvF<4C6qN6TbOuWz#C90he zk8VCrz9~ga{>#d@wW!SW>dG0qhG+l|$)Gz&PJJ(d*)%$?X3>ZQiDA5(RYC(@0*M@o zrs=M5-B?eZvEuk4&1zH{y-VnmG$ufB{n{$i?z%FkW7&v2NPy2G+l-vFv@S@p)^hP_VVc>izNBLFtpl`=o%R5lzGMj zFX>pW2kXc(@%}@oh&Rh!<*}1=N*Cz6dA!|y^5YYU#4eR_s=MKTslyKVqfP|rf2*^F z1E}MI>h^m-L;nw$>A{Ojyld&-K=v5ekaufQa_#>n#X@&`>zkPU_W3`Gt`!Q18}xz# z0UbmCSBWAQGgntDdyBu;Sv9L~I{)NC>)9?fY}+-5ZgG;`r#>aC+mfTO1#CV5)bJh1 z?w&INE?;4rbUk7`#XRlu0{!;+M{keI(5M1q7W~&d_+(>#7{-7(AW>>s8jicUl~UU` zBQ~a#um~ny}9_b)&fHo7tT0vZZKNY_x@+_UZ4ropch5 z>1+@NIFKoEu#Sj=*Obk=I6n-!iQ69Jzy3n6-=8|a9NdX=ep1$`#W*fHhD%Bz#|p$8 zN>_-B;CFlrEL>T|#OkE2cM3i-n$3Edg6E}-$MDPxrg(A80%u(4jE>7dgF6zLGIx)T zdZ8dEyjZ{&ItIteRN~Fz0*Z`3-#~$d!Ud1w_C%0@0L#8yh^D|xKO_fiRA8gb`YIZ+ zA$_>{qaZJMl76nJDbGbr`o1M@zd^8)96q1*s?w^!{QNSGwgi0g zW;sXP`;8z%LyLsuRtz)`MvzIpLs@SEdK-wf+)Qt5z5>enqdZZ#O0^ZZLd|&l7pAeg zku0A;T&2yG5#{?;xIRYHYkd$R-Gdp)$kM<7xUlo|=iC5x%oOGdbyb`4U!uW@&z1cY zqbrfL4J;qmV$$bofjb;I(A(H(O9Ar zF3wCLI5~H32c5lRkj zienD_wz&#}99qQ{8TbB;h`lZF)zyOMTU9W+QiYj!J@m1NIHwLS4{M|WdQauFM1439 zopoK=a0a4shp|#8?|sAAWXyeuKF6h5ed=)c#r!0Lm)4$Hc%q%t8+*-#MV5%K>us~C z>{b3t<)|-`+vJAalbl_#!T2Tw$EEoe3i#yLZBjxr+7kjG*45%`y7zK45TfO!hvJ5a`#JeHPS}IO3EQA6s^&6&D=!$EtPkBz?2?uP`@1@srWgGaP%PUW`4^@Bw zrKVZ{??2@;|0+oK&CFQkzb^Rb*?#LmCODv3ubzQ}1eJZ$C}!aV7A*YEB%u~WM?7-o zX24>o>6a%~Ee@>CdkW%H;_@wDoN0y~seHQz7Dg5b^}W1{MN`6Gp_?WnR8YK7(HI#Q zFiDr|Mfw%sF9^b6(L*H0o>s;&wdhc*=>EO zZcgyi>ewK+x}H_aWkdg{wEXO z$lo>Gq-_(M(U56ILYgKKaJ_&~@#-+v<1Yo<=U;#D+Qu?n)=72V+r@ca(oT^92S8!F zib#SIbAQe}8-L!v<7>BmqidiLrJ-s2)raFqHvr09yKCT}c7!?N#Nr;i`uo1wB|nVLT9x1v6g9k?MTFLMK3RV@9MI#g0E4 zc9tFq)YjmF>k$?SC4v*DK_ZE@%hSD`OvVxuP{|y|l1WA2jdZ4=RB9#N zNkmLkMj7yGRsRo~+1A~!;FnY74jKWArS*FL!Q7{TI`$(aa4NYh1y!x4*4?OZckf+s z&-(s6?!bI^BIXBb*C3@k4?i!U+&sJCYir7NIr-WK&r2dSJfT^0O+4I~anr=qowRSq zx#&Ie?;JtO#>REjyW9@@bj`BtbB0e(ch?sN5zlY#N=F$DK9w*z9v2d<+&O75I~Y;t zbB9e42jjNljoeHI;00%aR?MQIKf#c`v@&;c6;N2I37r>5hs}>=Q(D&1S!UZy%_3Ni z^}_r#{%~RbY;Nl{nCi{tx0i(rf1A0bK#>Zsn!si9;Emg~x=j&i(ce)ubUf%!x#Le! z8Lg_@a{vcnoUZ?M_&xlG+goZ6#d)cxT#b0e_nZGwZZb)Tq-_LHQLvK+2nhY(r=+W; znVlKqUq|M@Dw%1^IjnPF^*m`HbsQ|dth+^Yh+V8+E|ZosyN!>QK?sCQSSBsDZr`BuKzhZv&JX>$yg3K84h4r+ z>~;wPSohV70fvCadmHe#od#bcc25U96$Zq@{-;+r&bN~d* znV9mYI`>SdDXBP+VK(pt*U4z-P07UMk`#r;ty5Q=bBYY@kdhpQ1scwC zTs2g=FCq(6b#fMk>vkEIIM^BH{BobSo{jcJRuq=H0q@=3DCo_nf(t|_Oa%w#%rY_! zP0Dk%g^Q9|U!6!|$`~_T_~kk8$3j))CQ1OSRn5S z(1oG`jX@JRP6K&9DCQoS>mZ@ai3m7Dz(Y`F|CKDUznCAwJicS@xUo)H8?#3lb@O>c zxa4R`NZX+&j&Y>&e16p`P-_yF)Vkc;4M_Vc8T6{ZW3OVBJx!2jqTmq@!$#3LiCnTB zNg+&3d@Puviz{M2$s1@C`=_l@rjon_2wz_ws8c;Lq&wBrN#<%|Ob*5bgNStF$@2Hv z{36Fexu#kX(+*+fHfWLzO#8lYKmB_3SyHJnJE{#~v9G05%Xp)~a3#znX!m$S9>l*H zGOcMg6PlfbOzq;!OGVQ9yn^G_mG52W=A7G%j_W$Ek?_~yq`Y-q2S446k<-bw!Z}O6gGj~#f7)K_pGi@g;drcA z2q5Cal$vdw3~)uV=(Np)X*LZ*g+zI}Teg9 zCXRyh3+}$F?4=}iwbLvvE>jkZU*1}kD}`ezNXAO}NDgN40SjJSII{VaAP~AoKb0EN> zY%^k+6T@{cFtDY0tBixra4^p<`T$?T=K-XhP@@_-p??2NiFhZ9BJ2*O}K?g z+{2ir*^SMFrqh)KY27Uwl!*+r1+>`hOo@jlZ4y+=?W51szkG&8jq&;jB*o{KLz&C2 zup|iP4;Xpc-YrqrwWzQ|1+V_R+d0Px@yZ6hOZ=Y_V^bOc)?)Sy7Z@7Dq%ujM1$sg!#&wlW85vKKqdhMF8k zyDUL1!&6(I_AAm$?s-tKtG0=4?~+t8De*zpm+u~$lw^;4HcR6Hh9{JU=A46)m)tux=8F>v z&lw7|Q7Ud9gH z6~t!MNHvMka!o%hr!hy^CSx?Dgh(fiF{L7dBghsrW(;XDXje#WnghvA(KuQlH@Ob- zra?%(wz?R;e#;%#FW%2LtmFOMO{ar)j^%o<)%y+k&s;3Wk7uP6@P`Kh$X@@Hc{g!$ zadoiMw6kTjay7I2>u;|*svc&52@UAUM{GP{yo4tVZ9y4{5;-5HM(X7!L_$|Wk@4qN zSnrPeH^M-c{F-NSX0Mv@%fJj@76fcTqHXI4YBP;UWHD7gn zEJfrOh8hSiqswagoK5oJ+BTR>Q|F+iwf~`uf8ubd8=!|0po`!iy8f;L`yX|Gm4J<& zsR!ieNDw>S3dBv86=oJx`I?U?AZx;o#FcR~En&r|9Ws(zJq0SD*eqN4pHD7!zKE4= zLKLxcWhE8KekYFiP8??ztG@QhL@iEDwBk%>z(p@tgYXyW0`WU+{Kgn* znmSQTxI!P&>e(XtkFZxIXOKSB<f?@Q3rzR9ZbxF!Y%-uEnK)rlROxle_>4i5{=xvP zDs_HS<8;a}QxZORnV)4@yE>6cQ)jc=)6>dTP->>{bwVa;l7G4M4&JIc2(R864*Nvt ziZ~~`H~h`F#q7n_P&xtm`!0p#_x~8B#m$C>+W_Wd!U5Ucf5sg4Zg$3I&VX5&KiS>1 z=BE7~7gqOnsbIXgZgY4d8yH(tfuOoxv^Z(=9S{Xp-C6D!GtX@A5W<#6ccwwxpf&PgtH8KS_^so=N$<38QbiyaS|k=i#skZPQ^)&lGYc#LHP zjFS|!!wMZO_xInA(?#6q-!|pf1;Do{Bsex=F2lRkbt_Qc5Hxn@PM7Du4)Zw}79b!+ zODNckfa@Zu#VdD}=1QLHH&eDr7W>t|brg4Vep8sNKQyJL8yKK%3S3L_yVHZrF&O1nx)BzMYNPV=AlhM}6E2=@eskjz@>x+a z%}qNZo%0U|525s&oAtHZHTQyJ#$gDGdW-1coL6}+SKUvM$jjGe<8#L+<4%hDWbwo}b;QJ^kTGY+pg(`b^K{w43sVQ;^7JXx{IB`F?yNG!cdyhjAsAtFuSfzhK;W z=fTReoQPVzn1isE=hmaEMgX$iV5e^s`x{4>Q<(|{Yu-Pjoqj3InqYj{p60o2*5iDI z>Vcpz=sv1;f;duQJ3kT3@cy!m-FefjX7L79gCJWI60Mibo!Iw%>U9;}z+J<88~;1c zaYfA6&mLGgaX4JuT?9yAMSa~TcfA|e=G$EX9z8nN#Ox1y&wbnW+PXXw3&R?0jKw}a z){7d5$E(pt~?%*}b&-|@V*njp|1!Kz= z-QYk#K}0}6n1CJ&P@wy-ZVE8+0$iW=Mpm{A){bTtm+Cf-sA_2ZonD;OcGE~y^k|R+ zoh*Gt^un|t99BO|k~oU!Yn?)*66H`cD2%=>q{wH-zW_ai@_B#Oqp7-PIVsKs@F

A|a1+oFLgqtX6Tr}41ux*#xhNX)NU>gfzOVxqF z{%mBj8UNn1e|=!v}D9H z9y>>bk~-ZQ`W{)xj1*qW3GvIdJl_-87@>NTGKJfuYw)r`jWjpR+gGzi@zwM|Kn%w^ zGEM>9Fw|o>L0fHRYyGTJD}WEReUx$C#`tmBP|qxxC-u@6ir#UA?R7H&-3wwrKKB;- z5r&-*fls(T`VsaS%bj;@5OQPSYTT?mrIU{3VzI*x;3NkxkjW{Q74 z;Ae?ND{PLN!cIGv)XuGO`%3hKCd^Q^LI*D8VJ$vu$n$a!ruVfk@wg@fzk|VoI7h)5 zQg8raD0w0Ae0$~bV@x!PVBYO>_x`(`qFdv~t;KJj9?0D8xAC9=f&Sc~O3VnRR=)%# z(k&lyL~8251DuYVl>Xou{>9(~SB>fu_?j7ccT38k=HyhG{O{TW*8J8VYa1q8wp-R8 zHyI3wKgsP;TLWMH*2gdR_#shSGtZCs4<4Nz9Dk8_SK|JP_5W78h+CWD331*G1ROZb z^pmMy(H6@l&9qE%h}9I^8Qv^}`ULyS^;W`)OwBOj?Gqb zCQPoLHP@&ID6Z?-K|CAF%cMB={V-q`Ce=Fcd0ptQf^i{B>`FA%E zEatolp|T&-THnH&`;3V=;x!5dB%?PbHN1Ns^QM~z|5$)B(-xCvL@*ucKQhcR%H8a1<{9*nP$v2&-aPAMK-hxEEu{$ zC%Rqt97C%q&=W}GgV5+n`gwSRgZWs4)htk55T{k>7wLyOGKqw$iz zSluRNb(Z|-@>;$K81SkZZb_JD@$zt!yWAuH?YT+Nuk-7!Ht9f8#bxqrIh4TXV6OR` zz{`8Tq%_CpXze)1H{WG)IaS}sY`wC!`+0etcgv&I=C~%7f9r01>-(#~`^Ds-!2=$* zXVsJY_Nl`U@7K+zcMD%G-`&01UcHz1Sb@(1H|9S0Bz{Z1Rj?59AW~0l%s~zn|M=^D zXScx!j8nbJkk}cW!~~IOyjeE~IBeBIW6*7LRf~3Nk7*oz4Gihi?53PGxRY zePzY>QYOzrIlbj)0r0@(k`4UJn%JZ=7Bpfd{u_6EJ7gc=&B!Fe3>qHiU{I=eiCr7C zbmCNC3-uIm%nj7-fPn^}7z0DJGjK3CHLoN-q_QBjI2P18LpPvO&D-GzQ0YD3I5Ia( zH;`^%JO(rZ(#8ij?a?)Mv~dS#0+n0@Y7|7#_#7w#uX~d7i@+n%=-R*BzAgS5s6B^; zfk6O8dp%GDu064!0Gq4%awZ)64%E#9^fs#HGk_xCnFF8^rMW=AW7D43AT4AB)P5Rh zl?;mI`#7N5AwEYo1Zya;Us*r1SRdFF4rOKFM=?nYI4BQxNO4JJPAYi15cMi}wD7He-D-2n8CF2aEQb;t%_>-M6Xh2D2Wn6;xG z*(}taB)Tc+T`hzu=Nq7*g3|Fq*N@&FMCi|ILCRvNt-}CsR$vYSqWp D`Bw}5 literal 0 HcmV?d00001 diff --git a/packages/extraction/tests/fixtures/empty.pdf b/packages/extraction/tests/fixtures/empty.pdf new file mode 100644 index 0000000000000000000000000000000000000000..fc99c8291dbc2292fb29c408fdfbd0776a0fe03a GIT binary patch literal 660 zcmZWn-A=+V6uv)Cac-8tg?8Q8hza2WWYMURjNKV83hQtLSJHI`d`T~S1AP%sD?bY~ zS+agT=leOWH@!`+_;m>0&+peaaP(2lS1=d=n`)Ie3#qZQ*=_Qo3&SDED!0c$t1R1X zB(kYk${N^MHI+dQ?4itSe8GqcQ+#c;h8L}4VZ>T3Nzihjq{nYIK;NekLOY?>yw;pSwrFe%?hAiPoyjg*L zVh?=Ga_n?5Tj3Bu7NV8dXycDsP_VQRv`uAUB_VItyJWfh4~{pmu=94`d*AP;{??i+(RnfJv)ZrU zU%xW}9_Y=xY+-@%rqM~Sr3{?uZg-RRS+=;yRGzrc^~!HIx4Q~>Q+8Cr__FTh76jwh z(xiY}5a5q7+^)3xW39{5%1n1~p_>72c3f$=FxC+1+%mq=+qC6sCjd7t$wIj=z%Q%p zzOrdcR;D;1Lt}@E8IM(=d!~hZo#nE>d=cRGmO9Wjzpj#0F6lm6@@BpA2}K>|8g(>> z!i~bk|CJ+y+~T!>a5!N=&E1C@qBOIJmPeViYDKklq8MLOdB@&?dC_P%>4h~)Zn6He z-Ou9WN0>h5ozuIKl$}eTo_>4v=s3*0dwuoC#hvGSU+#T)$=9do5CI}U1c(3;AOb{y z2oM1xKm>>Y5g-B+OyI`Du7~!H9$9;#;neBXy{RW}PfW01vPJ}m01+SpM1Tko0U|&I zhyW2F0z_c42t5Az{!P&M%A%m}B0-fiMXC0NQX+6PSE(2lB&O zfZ2wTnh+Sw7_EhvXc(#0{qdR~R_0kF(@dE%L9&rPP$-YbT3c|zkC^c*d0T^vnUvXz S!b)RHAM>FfvRZ9*J!1RGaLMQZ literal 0 HcmV?d00001 diff --git a/packages/extraction/tests/fixtures/pages.pdf b/packages/extraction/tests/fixtures/pages.pdf new file mode 100644 index 0000000000000000000000000000000000000000..1e7acc2462718b0b8da896326f6a15871d0d0334 GIT binary patch literal 950 zcmcgr!A|2a5WUY=%%u_>z;V(fL_!=2M7!EuZI$MRIGAQqf=XODE-L&ahy8=)NAOq; zMCpYSmJ-?HnK$pvcqa3!{6wBcF!}%X`VS-qsP7)&{2avG=xVzpgGZa^O;sPm#RX7X z@pU+GE1x%nVy;%yfwy=B(NXQwALBRAN zAlI!0vDn^OPk?Di5Hr;g-=O$G>nF0!QVmd;PA~71H;95dy){dhmxcRyzmdp!Jr5^d z{GqB*Gre=dg44i%i3~B#LcC?-hFH9haFuASJ9e1#2>aUcx##ek+LgV>X~G;kYe+r9 ztVEGZbhu@WsY4F~EGs<98`D|bxay##oju0#0k2E$$Ubn7&&BTXzf0rUNF$$n4!y?o z3mO-+)NS|K{)x DfClz) literal 0 HcmV?d00001 diff --git a/packages/extraction/tests/isolation.mjs b/packages/extraction/tests/isolation.mjs new file mode 100644 index 00000000..d44ef800 --- /dev/null +++ b/packages/extraction/tests/isolation.mjs @@ -0,0 +1,64 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { mkdir, mkdtemp, writeFile, rm, readdir } from 'node:fs/promises'; +import { once } from 'node:events'; +import { RestrictedBackend } from '/opt/extraction/dist/backend.js'; + +await mkdir('/jobs/other'); +await writeFile('/jobs/other/secret', 'cross-job-canary'); +await writeFile('/socket/secret', 'transport-canary'); +const job = await mkdtemp('/jobs/job-'); +const child = spawn( + '/opt/extraction/guard', + [job, '/usr/local/bin/python3.13', '/tests/isolation.py', 'probe'], + { + env: { EXTRACTION_SECRET_CANARY: 'private-canary' }, + stdio: ['ignore', 'pipe', 'pipe'], + } +); +let output = '', + errors = ''; +child.stdout.on('data', bytes => { + output += bytes; +}); +child.stderr.on('data', bytes => { + errors += bytes; +}); +const [code] = await once(child, 'close'); +assert.equal(code, 0, errors); +assert.equal(output.trim(), 'ISOLATED'); +await rm(job, { recursive: true }); +const runtime = { + jobs: '/jobs', + guard: '/opt/extraction/guard', + python: '/usr/local/bin/python3.13', + worker: '/tests/isolation.py', +}; +const backend = new RestrictedBackend(runtime); +for (const abort of ['immediate', 'running']) { + const job = await backend.create(); + const signal = new AbortController(); + const pending = backend.run(job, 'hang', 'a'.repeat(64), signal.signal); + if (abort === 'running') + await new Promise(resolve => setTimeout(resolve, 100)); + signal.abort(new Error('CANCELLED')); + await assert.rejects(pending); + await backend.cleanup(job); +} +const job2 = await backend.create(); +await assert.rejects( + backend.run(job2, 'hang', 'a'.repeat(64), AbortSignal.timeout(100)) +); +await backend.cleanup(job2); +await rm('/jobs/other', { recursive: true }); +await rm('/socket/secret'); +assert.deepEqual(await readdir('/jobs'), []); +const missing = new RestrictedBackend({ ...runtime, guard: '/nonexistent' }); +const job3 = await missing.create(); +await assert.rejects( + missing.run(job3, 'pdf', 'a'.repeat(64), AbortSignal.timeout(1000)) +); +await missing.cleanup(job3); +console.log( + 'Container isolation: cross-job/credential paths, sockets, fork, signals, limits, startup/running cancellation, deadline and missing launcher passed' +); diff --git a/packages/extraction/tests/isolation.py b/packages/extraction/tests/isolation.py new file mode 100644 index 00000000..a50d0084 --- /dev/null +++ b/packages/extraction/tests/isolation.py @@ -0,0 +1,73 @@ +import ctypes +import errno +import fcntl +import os +import resource +import signal +import socket +import struct +import sys + +if sys.argv[1] == 'hang': + # Exercise supervisor cancellation/deadline termination, not a cooperative parser. + while True: + pass + +for path, mode in [('/jobs/other/secret', 'rb'), ('/jobs/other/secret', 'wb'), + ('/socket/secret', 'rb'), ('/proc/self/environ', 'rb'), ('/etc/passwd', 'rb')]: + try: + with open(path, mode) as file: + if mode == 'rb': + file.read() + except PermissionError: + pass + else: + raise RuntimeError('Cross-boundary file access') +for operation in [lambda: socket.socket(), lambda: socket.socket(socket.AF_UNIX), + lambda: os.fork(), lambda: os.kill(os.getppid(), 0), + lambda: os.chmod('/jobs/other/secret', 0o777), + lambda: fcntl.fcntl(1, fcntl.F_SETOWN, os.getppid()), + lambda: fcntl.fcntl(1, 15, struct.pack("ii", 1, os.getppid())), + lambda: fcntl.fcntl(1, fcntl.F_SETSIG, 0), + lambda: fcntl.ioctl(1, 0x8901, struct.pack("i", os.getppid()))]: + try: + operation() + except PermissionError: + pass + else: + raise RuntimeError('Forbidden syscall') +assert 'EXTRACTION_SECRET_CANARY' not in os.environ +assert resource.getrlimit(resource.RLIMIT_AS) == (512 * 1024 * 1024,) * 2 +assert resource.getrlimit(resource.RLIMIT_CPU) == (8, 8) +assert resource.getrlimit(resource.RLIMIT_FSIZE) == (4 * 1024 * 1024,) * 2 +assert resource.getrlimit(resource.RLIMIT_NOFILE) == (64, 64) + + +# Harmless signal-zero probes distinguish filtering from ordinary permissions. +class Sigval(ctypes.Union): + _fields_ = [("integer", ctypes.c_int), ("pointer", ctypes.c_void_p)] +libc = ctypes.CDLL(None, use_errno=True) +libc.sigqueue.argtypes = [ctypes.c_int, ctypes.c_int, Sigval] +assert libc.sigqueue(os.getppid(), 0, Sigval(integer=0)) == -1 +assert ctypes.get_errno() == errno.EPERM +number = {"x86_64": 297, "aarch64": 240}[os.uname().machine] +info = (ctypes.c_int * 32)() +info[2] = -1 # SI_QUEUE, with signal zero: never delivers a signal. +assert libc.syscall(number, os.getppid(), os.getppid(), 0, ctypes.byref(info)) == -1 +assert ctypes.get_errno() == errno.EPERM + + + +try: + fcntl.ioctl(1, 0, 0) +except PermissionError: + pass +else: + raise RuntimeError('Forbidden ioctl') + + + +libc.mq_open.argtypes = [ctypes.c_char_p, ctypes.c_int] +assert libc.mq_open(b"/extraction-negative-probe", os.O_RDONLY) == -1 +assert ctypes.get_errno() == errno.EPERM +print("ISOLATED", flush=True) diff --git a/packages/extraction/tsconfig.json b/packages/extraction/tsconfig.json new file mode 100644 index 00000000..aff02060 --- /dev/null +++ b/packages/extraction/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2023", + "lib": ["ES2024"], + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "noUncheckedIndexedAccess": true, + "esModuleInterop": true, + "skipLibCheck": true, + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src/**/*.ts"] +} diff --git a/packages/extraction/worker/extract.py b/packages/extraction/worker/extract.py new file mode 100644 index 00000000..20f0360f --- /dev/null +++ b/packages/extraction/worker/extract.py @@ -0,0 +1,503 @@ +"""Request-local parser. Only the supervisor can choose its mode and files.""" +import hashlib +import io +import json +import logging +import os +from pathlib import PurePosixPath +import socket +import struct +import sys +import xml.etree.ElementTree as ET +import zipfile +import zlib + +INPUT_BYTES = 10 * 1024 * 1024 +TEXT_BYTES = 1024 * 1024 +RESULT_BYTES = 3 * 1024 * 1024 +EXPANDED_BYTES = 32 * 1024 * 1024 +ENTRY_BYTES = 8 * 1024 * 1024 +SEGMENTS = 128 + + +class Rejected(Exception): + def __init__(self, code): + self.code = code + + +class Text: + def __init__(self): + self.size = 0 + self.parts = [] + + def add(self, value): + size = len(value.encode("utf-8", errors="strict")) + if self.size + size > TEXT_BYTES: + raise Rejected("OUTPUT_LIMIT") + self.size += size + self.parts.append(value) + + def value(self): + return "".join(self.parts) + + +def pdf_segments(data): + import pypdf + from pypdf import filters, _page + from pypdf.generic import DecodedStreamObject, DictionaryObject, NameObject, NullObject, StreamObject + from pypdf._cmap import charset_encoding + from pypdf._codecs.core_font_metrics import CORE_FONT_METRICS + from pypdf._text_extraction._text_extractor import TextExtraction + + used = 0 + failure = None + credited = {} + shown_bytes = 0 + + def reject(code): + nonlocal failure + failure = failure or code + raise Rejected(failure) + + def check_failure(): + if failure is not None: + raise Rejected(failure) + + class Warnings(logging.Handler): + def emit(self, record): + nonlocal failure + failure = failure or "INVALID_DOCUMENT" + + logger = logging.getLogger("pypdf") + logger.addHandler(Warnings()) + logger.setLevel(logging.WARNING) + logger.propagate = False + original = filters.decode_stream_data + original_data = DecodedStreamObject.get_data + original_show = TextExtraction._handle_tj_operation + original_resources = _page._get_page_resources + + def inflate(value): + nonlocal used + # Strict bounded decoding, without pypdf's recovery/fallback paths. + cap = min(ENTRY_BYTES, EXPANDED_BYTES - used) + decoder = zlib.decompressobj() + try: + result = decoder.decompress(value, cap + 1) + except zlib.error: + reject("INVALID_DOCUMENT") + if len(result) > cap or decoder.unconsumed_tail: + reject("DECOMPRESSION_LIMIT") + if not decoder.eof or decoder.unused_data: + reject("INVALID_DOCUMENT") + used += len(result) + credited[id(result)] = result + return result + + def decoded_data(stream): + nonlocal used + value = original_data(stream) + if not getattr(stream, "_extraction_counted", False): + if credited.get(id(value)) is not value: + used += len(value) + if len(value) > ENTRY_BYTES or used > EXPANDED_BYTES: + reject("DECOMPRESSION_LIMIT") + stream._extraction_counted = True + return value + + def decode(stream): + names = stream.get("/Filter", []) + if not isinstance(names, list): + names = [names] + if len(names) > 1 or any(str(name) not in { + "/FlateDecode", "/Fl", "/ASCIIHexDecode", "/AHx", "/ASCII85Decode", "/A85" + } for name in names): + reject("UNSUPPORTED_ENCODING") + try: + return original(stream) + except Rejected: + raise + except MemoryError: + reject("RESOURCE_LIMIT") + except Exception: + reject("INVALID_DOCUMENT") + + def resources_for_text(obj): + resources = original_resources(obj) + if resources: + fonts = resources.get("/Font") + if fonts is not None and not isinstance(fonts.get_object(), NullObject): + for ref in fonts.get_object().values(): + resource = ref.get_object() + for key in ("/Encoding", "/ToUnicode"): + if key in resource and isinstance(resource[key], NullObject): + del resource[key] + encoding = resource.get("/Encoding") + if encoding is not None: + encoding = encoding.get_object() + if not isinstance(encoding, (NameObject, DictionaryObject)): + reject("UNSUPPORTED_ENCODING") + if isinstance(encoding, DictionaryObject): + base = encoding.get("/BaseEncoding") + if base is not None and isinstance(base.get_object(), NullObject): + del encoding["/BaseEncoding"] + unicode_map = resource.get("/ToUnicode") + if unicode_map is not None and not isinstance(unicode_map.get_object(), StreamObject): + reject("UNSUPPORTED_ENCODING") + for state in resources.get("/ExtGState", DictionaryObject()).get_object().values(): + if "/Font" in state.get_object(): + reject("UNSUPPORTED_ENCODING") + return resources + # Do not let the parser skip text operators solely because resources + # are absent. A used missing font must reach the rejection below. + return DictionaryObject({NameObject("/Font"): DictionaryObject()}) + + def show_text(extractor, operands): + nonlocal shown_bytes + if operands and operands[0]: + font, resource = extractor.font, extractor.font_resource + if resource is None or not font.interpretable: + reject("UNSUPPORTED_ENCODING") + if font.sub_type not in {"Type1", "MMType1", "TrueType", "Type3", "Type0"}: + reject("UNSUPPORTED_ENCODING") + encoding = resource.get("/Encoding") + if encoding is not None: + encoding = encoding.get_object() + if isinstance(encoding, DictionaryObject): + encoding = encoding.get("/BaseEncoding") + if encoding is not None: + encoding = encoding.get_object() + standard_encoding = isinstance(encoding, NameObject) and encoding in charset_encoding + require_map = ("/ToUnicode" in resource or font.sub_type in {"Type3", "Type0"} + or isinstance(font.encoding, str) + or (not standard_encoding and font.name not in CORE_FONT_METRICS)) + value = operands[0] + if isinstance(value, bytes): + try: + if isinstance(font.encoding, str): + # Never take pypdf's charmap recovery on codec failure. + characters = value.decode(font.encoding, errors="surrogatepass") + else: + characters = (font.encoding[code] for code in value) + for character in characters: + if require_map and character not in font.character_map: + reject("UNSUPPORTED_ENCODING") + mapped = font.character_map.get(character, character) + # Unknown Adobe glyph names are preserved as NameObject. + if (not isinstance(mapped, str) or isinstance(mapped, NameObject) + or not mapped or "\ufffd" in mapped + or any(ord(char) < 32 and char not in "\t\n\r" for char in mapped)): + reject("UNSUPPORTED_ENCODING") + shown_bytes += len(mapped.encode("utf-8", errors="strict")) + if shown_bytes > TEXT_BYTES: + reject("OUTPUT_LIMIT") + except (LookupError, UnicodeError, TypeError): + reject("UNSUPPORTED_ENCODING") + elif not isinstance(value, str): + reject("UNSUPPORTED_ENCODING") + return original_show(extractor, operands) + + _page._get_page_resources = resources_for_text + TextExtraction._handle_tj_operation = show_text + DecodedStreamObject.get_data = decoded_data + filters.decompress = inflate + filters.decode_stream_data = decode + reader = pypdf.PdfReader(io.BytesIO(data), strict=True) + check_failure() + if reader.is_encrypted: + raise Rejected("ENCRYPTED_DOCUMENT") + if len(reader.pages) > SEGMENTS: + raise Rejected("STRUCTURE_LIMIT") + result = [] + text = Text() + for index, page in enumerate(reader.pages): + value = page.extract_text() or "" + check_failure() + text.add(value) + result.append({"kind": "page", "index": index + 1, "text": value}) + check_failure() + return result, text.size + + +def validate_xml(value): + xml = value.decode("utf-8", errors="strict") + if "\x00" in xml or " 512: + raise Rejected("STRUCTURE_LIMIT") + for entry in archive.infolist(): + name = entry.filename + if entry.orig_filename != name: + raise Rejected("INVALID_DOCUMENT") + path = PurePosixPath(name) + if (name in names or path.is_absolute() or ".." in path.parts or "\\" in name + or "\x00" in name or (entry.external_attr >> 16) & 0o170000 == 0o120000 + or entry.flag_bits & 1 or entry.compress_type not in (0, 8)): + raise Rejected("INVALID_DOCUMENT") + names.add(name) + if entry.file_size > ENTRY_BYTES: + raise Rejected("DECOMPRESSION_LIMIT") + # Let ZipFile validate local names, flags and entry overlap, but do + # not use ZipExtFile.read(): it trims data to untrusted file_size. + with archive.open(entry): + offset = entry.header_offset + if offset < 0 or offset + 30 > len(data): + raise Rejected("INVALID_DOCUMENT") + method = struct.unpack_from(" len(data): + raise Rejected("INVALID_DOCUMENT") + decoder = zlib.decompressobj(-zlib.MAX_WBITS) if entry.compress_type == 8 else None + chunks = [] + size = 0 + crc = 0 + for position in range(start, end, 64 * 1024): + chunk = data[position:min(position + 64 * 1024, end)] + if decoder: + cap = min(ENTRY_BYTES - size, EXPANDED_BYTES - total) + chunk = decoder.decompress(chunk, cap + 1) + size += len(chunk) + total += len(chunk) + if size > ENTRY_BYTES or total > EXPANDED_BYTES: + raise Rejected("DECOMPRESSION_LIMIT") + crc = zlib.crc32(chunk, crc) + chunks.append(chunk) + if decoder and (not decoder.eof or decoder.unused_data or decoder.unconsumed_tail): + raise Rejected("INVALID_DOCUMENT") + if size != entry.file_size or crc != entry.CRC: + raise Rejected("INVALID_DOCUMENT") + value = b"".join(chunks) + parts[name] = value + xml_parts(parts) + output = io.BytesIO() + with zipfile.ZipFile(output, "w") as clean: + for name, value in parts.items(): + # The document parser sees only measured, validated parts. + clean.writestr(name, value, compress_type=zipfile.ZIP_STORED) + return output.getvalue() + + +def validate_docx_body(body): + from lxml.etree import _Element + + literal = _Element.text.__get__ + word = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}" + # These are the exact content paths read by pinned python-docx. Unknown + # wrappers fail closed rather than hiding text from its direct-child XPath. + grammar = { + "body": {"p", "tbl", "sectPr"}, + "tbl": {"tblPr", "tblGrid", "tr"}, + "tr": {"trPr", "tc"}, + "tc": {"tcPr", "p", "tbl"}, + "p": {"pPr", "r", "hyperlink"}, + "hyperlink": {"r"}, + "r": {"rPr", "t", "tab", "br", "cr", "noBreakHyphen", "ptab", "lastRenderedPageBreak"}, + } + properties = {"sectPr", "tblPr", "tblGrid", "trPr", "tcPr", "pPr", "rPr"} + markers = {"bookmarkStart", "bookmarkEnd", "commentRangeStart", "commentRangeEnd", + "permStart", "permEnd", "proofErr"} + stack = [(body, "body")] + while stack: + parent, kind = stack.pop() + if literal(parent) and literal(parent).strip(): + raise Rejected("UNSUPPORTED_CONTENT") + for child in parent: + tag = child.tag.removeprefix(word) if isinstance(child.tag, str) else "" + marker = kind in {"body", "p", "hyperlink", "tc"} and tag in markers + if child.tag != word + tag or (tag not in grammar[kind] and not marker): + raise Rejected("UNSUPPORTED_CONTENT") + if child.tail and child.tail.strip(): + raise Rejected("UNSUPPORTED_CONTENT") + if tag in properties: + # Formatting is ignored, but cannot conceal textual content. + if any(node.tag in {word + "t", word + "instrText", word + "delText"} + or (literal(node) and literal(node).strip()) + or (node.tail and node.tail.strip()) for node in child.iter()): + raise Rejected("UNSUPPORTED_CONTENT") + elif tag in grammar: + stack.append((child, tag)) + elif len(child) or (tag != "t" and literal(child) and literal(child).strip()): + raise Rejected("UNSUPPORTED_CONTENT") + + +def docx_segments(data): + from docx import Document + from docx.table import Table, _Cell + + document = Document(io.BytesIO(safe_docx(data))) + validate_docx_body(document.element.body) + text = Text() + def blocks(container, depth=0, cell=False): + if depth > 32: + raise Rejected("STRUCTURE_LIMIT") + first = True + for block in container.iter_inner_content(): + if cell and not first: + text.add("\n") + first = False + if isinstance(block, Table): + width = len(block._tbl.tblGrid.gridCol_lst) + if width < 1 or width > 512: + raise Rejected("STRUCTURE_LIMIT") + previous = set() + for row_index, row in enumerate(block.rows): + if row_index: + text.add("\n") + before, after = row.grid_cols_before, row.grid_cols_after + if before < 0 or after < 0 or before + after >= width: + raise Rejected("INVALID_DOCUMENT") + column = before + text.add("\t" * before) + current = set() + for index, tc in enumerate(row._tr.tc_lst): + span, merge = tc.grid_span, tc.vMerge + if span < 1 or column + span > width - after: + raise Rejected("INVALID_DOCUMENT") + if tc.tcPr is not None and tc.tcPr.find( + "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}hMerge") is not None: + raise Rejected("UNSUPPORTED_CONTENT") + if index: + text.add("\t") + position = (column, span) + if merge == "continue": + if position not in previous: + raise Rejected("INVALID_DOCUMENT") + # Continuations occupy columns, not another copy of + # the originating cell. Hidden content is rejected. + if tc.xpath(".//w:t | .//w:tbl | .//w:tab | .//w:br | .//w:cr | .//w:noBreakHyphen | .//w:ptab"): + raise Rejected("UNSUPPORTED_CONTENT") + else: + blocks(_Cell(tc, block), depth + 1, cell=True) + if merge is not None: + if merge not in {"restart", "continue"}: + raise Rejected("INVALID_DOCUMENT") + current.add(position) + text.add("\t" * (span - 1)) + column += span + if column + after != width: + raise Rejected("INVALID_DOCUMENT") + text.add("\t" * after) + previous = current + if not cell: + text.add("\n") + else: + text.add(block.text) + if not cell: + text.add("\n") + + blocks(document) + return [{"kind": "document", "index": 1, "text": text.value()}], text.size + + +def probe(): + import pypdf + import docx + import lxml.etree + + for path in ("/socket", "/jobs", "/proc/self/environ", "/etc/passwd"): + try: + if path in ("/socket", "/jobs"): + os.listdir(path) + else: + open(path, "rb").read(1) + except PermissionError: + continue + raise Rejected("ISOLATION_UNAVAILABLE") + try: + socket.socket(socket.AF_INET, socket.SOCK_STREAM) + except PermissionError: + pass + else: + raise Rejected("ISOLATION_UNAVAILABLE") + try: + os.fork() + except PermissionError: + pass + else: + raise Rejected("ISOLATION_UNAVAILABLE") + if "EXTRACTION_SECRET_CANARY" in os.environ: + raise Rejected("ISOLATION_UNAVAILABLE") + print("READY", flush=True) + + +def main(): + if sys.argv[1] == "probe": + probe() + return + data = open("input", "rb").read(INPUT_BYTES + 1) + if not data or len(data) > INPUT_BYTES: + raise Rejected("INPUT_LIMIT") + mode = sys.argv[1] + if mode == "pdf": + if not data.startswith(b"%PDF-"): + raise Rejected("INVALID_DOCUMENT") + segments, size = pdf_segments(data) + elif mode == "docx": + if not data.startswith(b"PK\x03\x04"): + raise Rejected("INVALID_DOCUMENT") + segments, size = docx_segments(data) + else: + raise Rejected("UNSUPPORTED_FORMAT") + if not any(item["text"].strip() for item in segments): + raise Rejected("EMPTY_OUTPUT") + result = json.dumps({"version": 1, "operation": "document.extract-text", "format": mode, + "inputSha256": hashlib.sha256(data).hexdigest(), "textBytes": size, + "segments": segments}, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + if len(result) > RESULT_BYTES: + raise Rejected("OUTPUT_LIMIT") + with open("result", "xb") as target: + target.write(result) + + +if __name__ == "__main__": + try: + main() + except Rejected as error: + print(error.code, flush=True) + sys.exit(2) + except MemoryError: + print("RESOURCE_LIMIT", flush=True) + sys.exit(2) + except Exception: + print("INVALID_DOCUMENT", flush=True) + sys.exit(2) diff --git a/packages/extraction/worker/requirements.txt b/packages/extraction/worker/requirements.txt new file mode 100644 index 00000000..25ad0ede --- /dev/null +++ b/packages/extraction/worker/requirements.txt @@ -0,0 +1,4 @@ +pypdf==6.19.0 +python-docx==1.2.0 +lxml==6.0.2 +typing_extensions==4.15.0 From 3d29af8239fada541fbb28e65d5f80e85c54291e Mon Sep 17 00:00:00 2001 From: ThabangLenonyana Date: Thu, 22 Oct 2026 18:53:44 +0200 Subject: [PATCH 46/48] feat(sandbox): close skill package gaps and expose /etc-backed config to jobs Add qpdf, ImageMagick, Carlito/Caladea fonts, pypdfium2 and react/react-dom/react-icons, which the docx/pdf/pptx/xlsx skills use. NsJail jobs mount /usr but not /etc, so Debian's /usr symlinks into /etc dangled: LibreOffice's registry main.xcd (soffice aborted with uno::RuntimeException), alternatives (convert/magick, libblas/liblapack) and OpenSSL certs. Resolve them at build time and bind /usr copies of /etc/fonts and /etc/ImageMagick-7 into the jail. --- api/Dockerfile | 25 +++++++++++++++++++++++++ api/config/sandbox.cfg | 19 +++++++++++++++++++ build-packages.sh | 1 + docker/Dockerfile.worker-sandbox | 4 ++++ docker/package-init.sh | 2 ++ javascript-packages.txt | 3 +++ 6 files changed, 54 insertions(+) diff --git a/api/Dockerfile b/api/Dockerfile index 4ad34ece..1fae81c9 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -103,6 +103,8 @@ RUN apt-get update && \ file \ jq \ poppler-utils \ + qpdf \ + imagemagick \ pandoc \ libreoffice-writer \ libreoffice-calc \ @@ -112,6 +114,8 @@ RUN apt-get update && \ debianutils \ fonts-dejavu \ fonts-liberation \ + fonts-crosextra-carlito \ + fonts-crosextra-caladea \ libnl-route-3-200 \ libprotobuf32 \ libffi8 \ @@ -132,6 +136,27 @@ RUN apt-get update && \ # in its minimal build. RUN [ -x /usr/bin/mawk ] && ln -sf /usr/bin/mawk /usr/bin/awk +# NsJail jobs see /usr but not /etc (only ld.so.cache and a static hosts file +# are mounted). Debian keeps many /usr symlinks pointing into /etc: +# alternatives (ImageMagick's convert/magick, libblas/liblapack, which, +# pager), LibreOffice's registry main.xcd (without it soffice aborts with +# com::sun::star::uno::RuntimeException) and the OpenSSL certs. Re-point each +# link at its /usr target, or copy the /etc file in when it lives only in /etc. +# fontconfig and ImageMagick read /etc/fonts and /etc/ImageMagick-7 by +# absolute path, so stage copies under /usr for sandbox.cfg to bind there. +RUN find /usr -xdev -type l -lname '/etc/*' | while read -r link; do \ + target="$(readlink -f "$link")" || continue; \ + [ -e "$target" ] || continue; \ + case "$target" in \ + /usr/*) ln -sfn "$target" "$link" ;; \ + *) rm "$link" && cp -a --dereference "$target" "$link" ;; \ + esac; \ + done \ + && ! find /usr -xdev -type l -lname '/etc/*' -exec test -e {} \; -print | grep . \ + && mkdir -p /usr/local/share/sandbox-etc \ + && cp -a --dereference /etc/fonts /usr/local/share/sandbox-etc/fonts \ + && cp -a --dereference /etc/ImageMagick-7 /usr/local/share/sandbox-etc/ImageMagick-7 + RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen COPY --from=nsjail-builder /nsjail/nsjail /usr/sbin/nsjail diff --git a/api/config/sandbox.cfg b/api/config/sandbox.cfg index 4fdcc280..4d952b58 100644 --- a/api/config/sandbox.cfg +++ b/api/config/sandbox.cfg @@ -52,6 +52,25 @@ mount { mandatory: false } +# fontconfig (LibreOffice, matplotlib, ImageMagick) and ImageMagick read +# their config from /etc by absolute path. The rootfs build stages read-only +# copies under /usr so they exist in both direct and KVM modes. +mount { + src: "/usr/local/share/sandbox-etc/fonts" + dst: "/etc/fonts" + is_bind: true + rw: false + mandatory: false +} + +mount { + src: "/usr/local/share/sandbox-etc/ImageMagick-7" + dst: "/etc/ImageMagick-7" + is_bind: true + rw: false + mandatory: false +} + # Static /etc/hosts so "localhost" resolves for tool-call-server connections. # Uses inline content instead of bind-mounting the host's /etc/hosts, which # in Kubernetes leaks pod name, internal IP, and deployment metadata. diff --git a/build-packages.sh b/build-packages.sh index f3d4ed02..9954a143 100755 --- a/build-packages.sh +++ b/build-packages.sh @@ -215,6 +215,7 @@ install_python_packages() { vsdx \ pypdf \ pdfplumber \ + pypdfium2 \ defusedxml \ lxml \ validators \ diff --git a/docker/Dockerfile.worker-sandbox b/docker/Dockerfile.worker-sandbox index 99474503..8d0524e0 100644 --- a/docker/Dockerfile.worker-sandbox +++ b/docker/Dockerfile.worker-sandbox @@ -134,6 +134,8 @@ RUN apt-install \ bc \ jq \ poppler-utils \ + qpdf \ + imagemagick \ pandoc \ libreoffice-writer \ libreoffice-calc \ @@ -143,6 +145,8 @@ RUN apt-install \ debianutils \ fonts-dejavu \ fonts-liberation \ + fonts-crosextra-carlito \ + fonts-crosextra-caladea \ libnl-route-3-200 \ libprotobuf32 \ libffi8 \ diff --git a/docker/package-init.sh b/docker/package-init.sh index af7390ec..2ed75771 100644 --- a/docker/package-init.sh +++ b/docker/package-init.sh @@ -95,6 +95,7 @@ packages_ready() { [ -d "/pkgs/python/${PYTHON_VERSION}/lib/python${PYTHON_SITE_VERSION}/site-packages/statsmodels" ] && [ -d "/pkgs/python/${PYTHON_VERSION}/lib/python${PYTHON_SITE_VERSION}/site-packages/rasterio" ] && [ -d "/pkgs/python/${PYTHON_VERSION}/lib/python${PYTHON_SITE_VERSION}/site-packages/pdfplumber" ] && + [ -d "/pkgs/python/${PYTHON_VERSION}/lib/python${PYTHON_SITE_VERSION}/site-packages/pypdfium2" ] && [ -f "/pkgs/node/${NODE_VERSION}/.package-installed" ] && js_packages_ready "/pkgs/node/${NODE_VERSION}" && [ -f "/pkgs/bun/${BUN_VERSION}/.package-installed" ] && @@ -236,6 +237,7 @@ if [ -f "$PIP_PATH" ]; then vsdx \ pypdf \ pdfplumber \ + pypdfium2 \ defusedxml \ lxml \ validators \ diff --git a/javascript-packages.txt b/javascript-packages.txt index 2b502f98..2c335d28 100644 --- a/javascript-packages.txt +++ b/javascript-packages.txt @@ -33,3 +33,6 @@ archiver@7.0.1 adm-zip@0.5.17 pptxgenjs@4.0.1 docx@9.7.1 +react@19.3.1 +react-dom@19.3.1 +react-icons@5.7.0 From 3f19a5d070e17d9fc88ba5fae4133e984ba2fa7b Mon Sep 17 00:00:00 2001 From: ThabangLenonyana Date: Thu, 22 Oct 2026 18:53:44 +0200 Subject: [PATCH 47/48] feat(sandbox): SANDBOX_ALLOW_UNIX_SOCKETS opt-in for LibreOffice Upstream denies AF_UNIX socket() and socketpair() in both the job seccomp policy and the inherited supervisor filter. LibreOffice's process launcher needs them (ERROR 4 forking process). The opt-in re-allows only those two; sendmsg/sendmmsg and io_uring stay denied, so SCM_RIGHTS descriptor passing (CVE-2026-80521) remains blocked. Default off keeps upstream behaviour. --- api/src/config.ts | 4 ++++ api/src/nsjail.test.ts | 19 +++++++++++++++++++ api/src/nsjail.ts | 15 ++++++++++----- api/src/supervisor-policy.c | 17 ++++++++++++++++- tests/supervisor_policy.py | 15 +++++++++++++++ 5 files changed, 64 insertions(+), 6 deletions(-) diff --git a/api/src/config.ts b/api/src/config.ts index a5d0c993..d159a6a0 100644 --- a/api/src/config.ts +++ b/api/src/config.ts @@ -52,6 +52,10 @@ export const config = { ?? '/pkgs', disable_networking: (process.env.SANDBOX_DISABLE_NETWORKING ?? 'true') === 'true', use_cgroupv2: (process.env.SANDBOX_USE_CGROUPV2 ?? 'true') === 'true', + /** Fork opt-in: let jobs create AF_UNIX sockets and socketpairs (needed by + * LibreOffice). sendmsg/sendmmsg stay denied, so SCM_RIGHTS descriptor + * passing (CVE-2026-80521) remains blocked. Default false = upstream policy. */ + allow_unix_sockets: process.env.SANDBOX_ALLOW_UNIX_SOCKETS === 'true', allowed_local_network_port: Number(process.env.SANDBOX_ALLOWED_LOCAL_NETWORK_PORT ?? 0), output_max_size: safeInt(process.env.SANDBOX_OUTPUT_MAX_SIZE, 1024), max_process_count: Number(process.env.SANDBOX_MAX_PROCESS_COUNT ?? 64), diff --git a/api/src/nsjail.test.ts b/api/src/nsjail.test.ts index ab046b7f..c95c51b4 100644 --- a/api/src/nsjail.test.ts +++ b/api/src/nsjail.test.ts @@ -416,6 +416,25 @@ describe('NsJail seccomp policy', () => { expect(socketRule).toContain('domain != AF_VSOCK'); }); + test('SANDBOX_ALLOW_UNIX_SOCKETS opt-in permits only AF_UNIX and keeps send denials', () => { + const previous = config.allow_unix_sockets; + try { + config.allow_unix_sockets = true; + const policy = seccompPolicy(); + const errnoBlock = policy.split('ERRNO(1) {')[1]?.split(' }')[0] ?? ''; + const socketRule = errnoBlock.split('\n').find(line => line.includes('socket(domain)')); + expect(socketRule).toContain('domain != AF_VSOCK && domain != AF_UNIX'); + expect(policy).toContain('#define AF_UNIX 1'); + expect(errnoBlock).not.toMatch(/\bsocketpair\b/); + for (const name of ['sendmsg', 'sendmmsg', 'io_uring_setup']) { + expect(errnoBlock).toMatch(new RegExp(`\\b${name}\\b[,\\s]`)); + } + } finally { + config.allow_unix_sockets = previous; + } + expect(seccompPolicy()).toContain(' socketpair'); + }); + test('KILLs the defense-in-depth batch from the audit', () => { const policy = seccompPolicy(); for (const name of ['settimeofday', 'adjtimex', 'clock_adjtime', 'syslog']) { diff --git a/api/src/nsjail.ts b/api/src/nsjail.ts index 23c048a6..0ae9bff5 100644 --- a/api/src/nsjail.ts +++ b/api/src/nsjail.ts @@ -94,8 +94,9 @@ const archSpecificLowPrioritySyscalls = process.arch === 'arm64' ? '' : ' ioperm, iopl, modify_ldt, lookup_dcookie,'; -const SECCOMP_POLICY = [ +const seccompPolicy = (): string => [ ...syscallDefines, + '#define AF_UNIX 1', '#define AF_INET 2', '#define AF_INET6 10', '#define AF_NETLINK 16', @@ -185,9 +186,13 @@ const SECCOMP_POLICY = [ ' pidfd_send_signal,', /* Deny every socket() family. VSOCK keeps its stronger KILL action above. * Deny socketpairs as well; packaged runtime IPC uses anonymous pipes. - * Keep sendmsg/sendmmsg denied for inherited descriptors too. */ - ' socket(domain) { domain != AF_VSOCK },', - ' socketpair', + * Keep sendmsg/sendmmsg denied for inherited descriptors too. + * Fork opt-in (SANDBOX_ALLOW_UNIX_SOCKETS=true): allow AF_UNIX socket() and + * socketpair() for LibreOffice's IPC. The jail has its own network + * namespace, and sendmsg/sendmmsg above still block SCM_RIGHTS. */ + ...(config.allow_unix_sockets + ? [' socket(domain) { domain != AF_VSOCK && domain != AF_UNIX }'] + : [' socket(domain) { domain != AF_VSOCK },', ' socketpair']), ' }', '}', 'USE sandbox DEFAULT ALLOW', @@ -688,7 +693,7 @@ export function buildArgs(opts: BuildArgsOptions): string[] { const args: string[] = [ '--config', cfgPath ?? config.nsjail_config, '--log', logPath, - '--seccomp_string', SECCOMP_POLICY, + '--seccomp_string', seccompPolicy(), '--user', `${SANDBOX_INSIDE_UID}:${identity.uid}:1`, '--group', `${SANDBOX_INSIDE_GID}:${identity.gid}:1`, '-s', '/usr/bin:/bin', diff --git a/api/src/supervisor-policy.c b/api/src/supervisor-policy.c index b2091927..42f0c0cc 100644 --- a/api/src/supervisor-policy.c +++ b/api/src/supervisor-policy.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -30,8 +31,11 @@ int main(int argc, char **argv) { BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0x40000000, 0, 1), BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), #endif - DENY(socketpair), DENY(sendmsg), DENY(sendmmsg), + DENY(sendmsg), DENY(sendmmsg), DENY(io_uring_setup), DENY(io_uring_enter), DENY(io_uring_register), + /* The last 7 instructions are the Unix-socket denials; the + * SANDBOX_ALLOW_UNIX_SOCKETS opt-in below replaces them with ALLOW. */ + DENY(socketpair), BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_socket, 0, 3), BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[0])), BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, AF_UNIX, 0, 1), @@ -39,6 +43,17 @@ int main(int argc, char **argv) { BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), }; struct sock_fprog program = { .len = sizeof(instructions) / sizeof(instructions[0]), .filter = instructions }; + /* Fork opt-in (SANDBOX_ALLOW_UNIX_SOCKETS=true, also read by the job + * policy in nsjail.ts): this filter is inherited by NsJail and every job, + * so LibreOffice needs AF_UNIX socket() and socketpair() allowed here too. + * Rewrite the socketpair DENY into a jump straight to the final ALLOW. + * sendmsg/sendmmsg/io_uring stay denied, so SCM_RIGHTS passing + * (CVE-2026-80521) remains blocked. */ + const char *allow_unix = getenv("SANDBOX_ALLOW_UNIX_SOCKETS"); + if (allow_unix && strcmp(allow_unix, "true") == 0) { + const size_t first = program.len - 7; /* DENY(socketpair) jump */ + instructions[first] = (struct sock_filter)BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW); + } if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0 || prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &program) != 0) { perror("sandbox supervisor policy"); return 125; diff --git a/tests/supervisor_policy.py b/tests/supervisor_policy.py index caab12c0..e17743b6 100644 --- a/tests/supervisor_policy.py +++ b/tests/supervisor_policy.py @@ -34,11 +34,26 @@ def denied(action): peer, _ = listener.accept() client.sendall(b'tcp'); assert peer.recv(3) == b'tcp' print('PASS: supervisor denies Unix sockets, inherited descriptor sends and io_uring; TCP works') +elif len(sys.argv) > 1 and sys.argv[1] == '--allow-unix': + # Fork opt-in: Unix sockets/socketpairs allowed, descriptor sends still denied. + socket.socket(socket.AF_UNIX, socket.SOCK_STREAM).close() + left, right = socket.socketpair() + right.send(b'ok'); assert left.recv(2) == b'ok' + try: + left.sendmsg([b'x'], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, b'\0' * 4)]) + except OSError as error: + assert error.errno == errno.EPERM, error + else: + raise AssertionError('sendmsg allowed under SANDBOX_ALLOW_UNIX_SOCKETS') + print('PASS: SANDBOX_ALLOW_UNIX_SOCKETS allows Unix sockets and keeps sendmsg denied') else: left, right = socket.socketpair() result = subprocess.run([POLICY, sys.executable, __file__, '--filtered', str(left.fileno())], pass_fds=[left.fileno()], timeout=5) assert result.returncode == 0, result.returncode + result = subprocess.run([POLICY, sys.executable, __file__, '--allow-unix'], + env={**os.environ, 'SANDBOX_ALLOW_UNIX_SOCKETS': 'true'}, timeout=5) + assert result.returncode == 0, result.returncode # The pipe-based NsJail handshake must handle child setup/exec failure, # as well as the successful normal path tested by the runner canary. command = [POLICY, '/nsjail/nsjail', '--mode', 'o', '--disable_clone_newnet', From 56c0c6c8a841d8edbad4d0d041c653c36ae6e6ae Mon Sep 17 00:00:00 2001 From: ThabangLenonyana Date: Thu, 22 Oct 2026 18:53:44 +0200 Subject: [PATCH 48/48] ci(ecr): drop duplicated control-plane comment --- .github/workflows/ecr-lambda-microvm-runner.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/ecr-lambda-microvm-runner.yml b/.github/workflows/ecr-lambda-microvm-runner.yml index a1c221f0..9b471dce 100644 --- a/.github/workflows/ecr-lambda-microvm-runner.yml +++ b/.github/workflows/ecr-lambda-microvm-runner.yml @@ -100,12 +100,6 @@ jobs: # # Built for linux/amd64 (Momentum EKS x86 node groups). If the cluster moves # to Graviton/arm64, change CONTROL_PLANE_PLATFORM and the runner to match. - # egress-gateway, tool-call-server (standalone Dockerfiles) - # sandbox-runner (api/Dockerfile target sandbox-runner-baked: KVM mode with - # /pkgs baked in — this is the image that carries our fork's packages) - # - # Built for linux/amd64 (Momentum EKS x86 node groups). If the cluster moves - # to Graviton/arm64, change CONTROL_PLANE_PLATFORM and the runner to match. build-control-plane: name: Build and Push Control-Plane Images runs-on: ubuntu-latest