diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f0191f0a..7337e5a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,22 +93,22 @@ jobs: - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.4.2 + bun-version: 1.4.4 - name: Cache Bun packages uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 with: path: ~/.bun/install/cache - key: ${{ runner.os }}-codeapi-api-bun-1.4.2-${{ hashFiles('api/bun.lock') }} + key: ${{ runner.os }}-codeapi-api-bun-1.4.4-${{ hashFiles('api/bun.lock') }} restore-keys: | - ${{ runner.os }}-codeapi-api-bun-1.4.2- + ${{ runner.os }}-codeapi-api-bun-1.4.4- - name: Install dependencies run: bun ci - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 with: - node-version: 22 + node-version: 22.23.4 - name: Compile anonymous-pipe addon run: | @@ -149,7 +149,7 @@ jobs: --workdir /work/api \ --env SANDBOX_CLEANUP_TMPFS_TEST=1 \ --env SANDBOX_LOG_LEVEL=error \ - oven/bun:1.4.2-debian \ + oven/bun:1.4.4-debian \ bun test src/cleanup.integration.test.ts nsjail-ipc-tests: @@ -164,7 +164,7 @@ jobs: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.4.2 + bun-version: 1.4.4 - name: Render native sandbox policy working-directory: api run: | @@ -271,15 +271,15 @@ jobs: - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.4.2 + bun-version: 1.4.4 - name: Cache Bun packages uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 with: path: ~/.bun/install/cache - key: ${{ runner.os }}-codeapi-service-bun-1.4.2-${{ hashFiles('service/bun.lock') }} + key: ${{ runner.os }}-codeapi-service-bun-1.4.4-${{ hashFiles('service/bun.lock') }} restore-keys: | - ${{ runner.os }}-codeapi-service-bun-1.4.2- + ${{ runner.os }}-codeapi-service-bun-1.4.4- - name: Install dependencies run: bun ci @@ -306,7 +306,7 @@ jobs: strategy: fail-fast: false matrix: - node-version: ['22.21.0', '24.16.0'] + node-version: ['22.23.4', '24.22.1'] defaults: run: working-directory: packages/code @@ -343,7 +343,7 @@ jobs: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 with: - node-version: 24.16.0 + node-version: 24.22.1 - run: npm ci - run: npm run build - name: Selected project root containment tests @@ -370,7 +370,7 @@ jobs: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 with: - node-version: 24.16.0 + node-version: 24.22.1 - name: Install bubblewrap run: | sudo apt-get update -qq @@ -395,7 +395,7 @@ jobs: - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.4.2 + bun-version: 1.4.4 - name: Install service dependencies working-directory: service @@ -467,7 +467,7 @@ jobs: docker buildx inspect --bootstrap - name: Build Lambda MicroVM runner image - # The target pins Bun 1.4.2 and installs api/bun.lock frozen. Building + # The target pins Bun 1.4.4 and installs api/bun.lock frozen. Building # natively catches architecture-specific package and Docker layer # failures without retaining a second multi-gigabyte image copy. run: | diff --git a/.github/workflows/extraction.yml b/.github/workflows/extraction.yml index 29a55217..0d9a529d 100644 --- a/.github/workflows/extraction.yml +++ b/.github/workflows/extraction.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 with: - node-version: 24.16.0 + node-version: 24.22.1 cache: npm cache-dependency-path: packages/extraction/package-lock.json - run: npm ci --ignore-scripts diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml index 7c626767..ad1c36f3 100644 --- a/.github/workflows/images.yml +++ b/.github/workflows/images.yml @@ -33,6 +33,7 @@ on: - javascript-packages.txt - launcher/** - packages/code/src/** + - packages/extraction/** - service/** - shared/** pull_request: @@ -70,6 +71,8 @@ jobs: run: | images=$(jq -c . <<'JSON' [ + {"image": "code-interpreter-extraction", "dockerfile": "packages/extraction/Dockerfile", "target": "runtime", + "description": "Authenticated isolated PDF/DOCX extraction service", "timeout": 15}, {"image": "code-interpreter-api", "dockerfile": "service/Dockerfile", "target": "api", "description": "Code API HTTP server (Compose service api)", "timeout": 30}, {"image": "code-interpreter-worker", "dockerfile": "service/Dockerfile", "target": "worker", diff --git a/api/Dockerfile b/api/Dockerfile index 1fae81c9..59180391 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -35,7 +35,7 @@ COPY api/src/spec-guard.c /tmp/spec-guard.c COPY api/src/tool-call-pipe-bridge.c /tmp/tool-call-pipe-bridge.c COPY api/src/anonymous-pipes.c /tmp/anonymous-pipes.c COPY api/src/supervisor-policy.c /tmp/supervisor-policy.c -COPY --from=node:22-slim /usr/local/include/node /usr/local/include/node +COPY --from=node:22.23.4-slim /usr/local/include/node /usr/local/include/node COPY docker/rootfs-setup.c /tmp/rootfs-setup.c RUN gcc -shared -fPIC -O2 -Wall -Wextra -Werror -I/usr/local/include/node -o /usr/local/lib/anonymous-pipes.node /tmp/anonymous-pipes.c \ && gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/sandbox-supervisor-policy /tmp/supervisor-policy.c \ @@ -49,8 +49,8 @@ FROM nsjail-builder AS nsjail-ipc-test ENV SANDBOX_PIPE_ADDON=/usr/local/lib/anonymous-pipes.node COPY docker/python-runtime /python-runtime RUN python3 /python-runtime/install.py -COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node -COPY --from=oven/bun:1.4.2-debian /usr/local/bin/bun /usr/local/bin/bun +COPY --from=node:22.23.4-slim /usr/local/bin/node /usr/local/bin/node +COPY --from=oven/bun:1.4.4-debian /usr/local/bin/bun /usr/local/bin/bun # ============================================================================ # Stage 1b: Build language runtime packages (only consumed by sandbox-runner-baked) @@ -64,7 +64,7 @@ FROM buildpack-deps:bookworm AS package-builder ENV DEBIAN_FRONTEND=noninteractive ENV PYTHON_VERSION=3.14.4 -ENV BUN_VERSION=1.4.2 +ENV BUN_VERSION=1.4.4 COPY docker/apt-install.sh /usr/local/bin/apt-install RUN chmod +x /usr/local/bin/apt-install @@ -87,7 +87,7 @@ RUN chmod +x /package-init.sh && FORCE_REBUILD=true /package-init.sh # ============================================================================ # Stage 2: Build sandbox rootfs # ============================================================================ -FROM oven/bun:1.4.2-debian AS sandbox-build +FROM oven/bun:1.4.4-debian AS sandbox-build ENV DEBIAN_FRONTEND=noninteractive @@ -170,7 +170,7 @@ RUN chmod +x /usr/sbin/nsjail # The per-invocation broker starts Node after restore, so its OpenSSL process # state is not captured in the MicroVM snapshot. Pulling the Node binary # is ~30 MB and avoids inheriting npm/node_modules from a full apt install. -COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node +COPY --from=node:22.23.4-slim /usr/local/bin/node /usr/local/bin/node WORKDIR /sandbox_api diff --git a/api/bun.lock b/api/bun.lock index b2003e30..6e8b5dda 100644 --- a/api/bun.lock +++ b/api/bun.lock @@ -24,6 +24,9 @@ }, }, }, + "overrides": { + "proxy-addr": "2.0.8", + }, "packages": { "@opentelemetry/api": ["@opentelemetry/api@1.9.1", "", {}, "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q=="], @@ -185,7 +188,7 @@ "prom-client": ["prom-client@15.1.3", "", { "dependencies": { "@opentelemetry/api": "^1.4.0", "tdigest": "^0.1.1" } }, "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g=="], - "proxy-addr": ["proxy-addr@2.0.7", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg=="], + "proxy-addr": ["proxy-addr@2.0.8", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ=="], "qs": ["qs@6.16.0", "", { "dependencies": { "es-define-property": "^1.0.1", "side-channel": "^1.1.1" } }, "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA=="], diff --git a/api/package-lock.json b/api/package-lock.json index 8cf48ae4..a2dbad66 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -1008,15 +1008,20 @@ } }, "node_modules/proxy-addr": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", - "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.8.tgz", + "integrity": "sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ==", + "license": "MIT", "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" }, "engines": { "node": ">= 0.10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/qs": { @@ -1756,7 +1761,7 @@ "on-finished": "^2.4.1", "once": "^1.4.0", "parseurl": "^1.3.3", - "proxy-addr": "^2.0.7", + "proxy-addr": "2.0.8", "qs": "^6.14.0", "range-parser": "^1.2.1", "router": "^2.2.0", @@ -2009,9 +2014,9 @@ } }, "proxy-addr": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", - "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.8.tgz", + "integrity": "sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ==", "requires": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" diff --git a/api/package.json b/api/package.json index a27fd780..4b455865 100644 --- a/api/package.json +++ b/api/package.json @@ -27,5 +27,8 @@ "@types/semver": "^7.5.8", "typescript": "^5.7.3" }, - "license": "Apache-2.0" + "license": "Apache-2.0", + "overrides": { + "proxy-addr": "2.0.8" + } } diff --git a/api/src/api/session-inputs.routes.test.ts b/api/src/api/session-inputs.routes.test.ts index 48fe7d53..0ba649e4 100644 --- a/api/src/api/session-inputs.routes.test.ts +++ b/api/src/api/session-inputs.routes.test.ts @@ -26,6 +26,7 @@ let baseUrl: string; const savedSessionWorkspaceEnabled = config.session_workspace_enabled; beforeAll(async () => { + await fsp.rm(SESSION_INPUT_CACHE_DIR, { recursive: true, force: true }); config.session_workspace_enabled = true; const app = express(); /* Mirror index.ts: no global JSON parser, router mounted under /api/v2. */ diff --git a/api/src/job.ts b/api/src/job.ts index bfa99b63..88d3e5b4 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -169,8 +169,11 @@ export function ensureNodeModulesSymlink( if (!nodeModulesPath) return; const linkPath = path.join(submissionDir, 'node_modules'); try { - fs.lstatSync(linkPath); - return; + if (!fs.lstatSync(linkPath).isSymbolicLink()) return; + const target = path.resolve(submissionDir, fs.readlinkSync(linkPath)); + const packageRoot = `${path.resolve(config.packages_directory)}${path.sep}`; + if (!target.startsWith(packageRoot) || target === path.resolve(nodeModulesPath)) return; + fs.unlinkSync(linkPath); } catch (err) { if (errorCode(err) !== 'ENOENT') throw err; } diff --git a/api/src/package-init.test.ts b/api/src/package-init.test.ts new file mode 100644 index 00000000..5d71c22e --- /dev/null +++ b/api/src/package-init.test.ts @@ -0,0 +1,201 @@ +import { expect, test } from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +const initializer = fs.readFileSync( + path.resolve(__dirname, '../../docker/package-init.sh'), + 'utf8' +); + +for (const oldRuntimeExists of [false, true]) { + test(`cached PVC refreshes Node and Bun launchers with ${ + oldRuntimeExists ? 'retained' : 'removed' + } old runtimes`, () => { + const root = fs.mkdtempSync( + path.join(os.tmpdir(), 'cached-package-init-') + ); + try { + const packages = path.join(root, 'pkgs'); + const workspace = path.join(root, 'workspace'); + const manifest = path.join(root, 'javascript-packages.txt'); + fs.mkdirSync(workspace); + fs.writeFileSync(manifest, 'runtime-fixture@1.0.0\n'); + const runtimes = [ + { + directory: 'node/24.22.1', + binary: 'bin/node', + args: ['hello world', '--version'], + }, + { + directory: 'bun/1.4.4', + binary: 'bun', + args: ['run', 'hello world', '--version'], + }, + ]; + for (const runtime of runtimes) { + const destination = path.join(packages, runtime.directory); + const fixture = path.join( + destination, + 'node_modules/runtime-fixture' + ); + fs.mkdirSync(fixture, { recursive: true }); + fs.mkdirSync( + path.dirname(path.join(destination, runtime.binary)), + { recursive: true } + ); + fs.writeFileSync( + path.join(fixture, 'package.json'), + '{"version": "1.0.0"}' + ); + fs.writeFileSync( + path.join(destination, '.package-installed'), + 'installed' + ); + fs.writeFileSync( + path.join(destination, '.env'), + 'preserve runtime environment' + ); + fs.writeFileSync( + path.join(destination, runtime.binary), + '#!/bin/bash\nprintf "<%s>\\n" "$@"\n', + { mode: 0o755 } + ); + fs.writeFileSync( + path.join(destination, 'run'), + '#!/bin/bash\nexit 99\n', + { mode: 0o755 } + ); + } + const python = path.join(packages, 'python/3.14.4'); + for (const name of [ + 'PIL', + 'markitdown', + 'chdb', + 'statsmodels', + 'rasterio', + // Fork additions required by packages_ready() in docker/package-init.sh + 'pdfplumber', + 'pypdfium2', + ]) { + fs.mkdirSync( + path.join(python, 'lib/python3.14/site-packages', name), + { recursive: true } + ); + } + fs.writeFileSync( + path.join(python, 'lib/python3.14/.sandbox-pipe-runtime-v1'), + 'installed' + ); + fs.writeFileSync( + path.join(python, '.package-installed'), + 'installed' + ); + fs.mkdirSync(path.join(packages, 'bash/5.2.0'), { + recursive: true, + }); + fs.writeFileSync( + path.join(packages, 'bash/5.2.0/.package-installed'), + 'installed' + ); + fs.writeFileSync( + path.join(packages, '.initialized'), + 'original initialization' + ); + + const forbidden = path.join(root, 'forbidden'); + fs.mkdirSync(forbidden); + for (const binary of [ + 'curl', + 'wget', + 'make', + 'npm', + 'tar', + 'unzip', + ]) { + fs.writeFileSync( + path.join(forbidden, binary), + '#!/bin/bash\necho "unexpected reinstall" >&2\nexit 98\n', + { mode: 0o755 } + ); + } + const script = path.join(root, 'package-init.sh'); + fs.writeFileSync( + script, + initializer + .split('/pkgs') + .join(packages) + .split('/mnt/data/node_modules') + .join(path.join(workspace, 'node_modules')) + ); + const env = { + ...process.env, + PATH: `${forbidden}:${process.env.PATH}`, + FORCE_REBUILD: 'false', + PYTHON_VERSION: '3.14.4', + NODE_VERSION: '24.22.1', + BUN_VERSION: '1.4.4', + BASH_PACKAGE_VERSION: '5.2.0', + JS_PACKAGE_MANIFEST: manifest, + }; + for (let pass = 0; pass < 2; pass += 1) { + const initialized = Bun.spawnSync({ + cmd: ['bash', script], + env, + }); + expect(initialized.stderr.toString()).toBe(''); + expect(initialized.exitCode).toBe(0); + expect(initialized.stdout.toString()).toContain( + 'Packages already initialized' + ); + expect( + fs.readFileSync(path.join(packages, '.initialized'), 'utf8') + ).toBe('original initialization'); + for (const runtime of runtimes) { + const destination = path.join(packages, runtime.directory); + const oldModules = path.join( + packages, + runtime.directory.split('/')[0], + '0.0.1/node_modules' + ); + if (oldRuntimeExists) + fs.mkdirSync(oldModules, { recursive: true }); + const link = path.join(workspace, 'node_modules'); + fs.rmSync(link, { force: true }); + fs.symlinkSync(oldModules, link); + const launched = Bun.spawnSync({ + cmd: [ + path.join(destination, 'run'), + 'hello world', + '--version', + ], + }); + expect(launched.stderr.toString()).toBe(''); + expect(launched.exitCode).toBe(0); + expect(launched.stdout.toString()).toBe( + runtime.args.map(arg => `<${arg}>\n`).join('') + ); + expect(fs.readlinkSync(link)).toBe( + path.join(destination, 'node_modules') + ); + expect( + fs.readFileSync( + path.join(destination, '.package-installed'), + 'utf8' + ) + ).toBe('installed'); + expect( + fs.readFileSync(path.join(destination, '.env'), 'utf8') + ).toBe('preserve runtime environment'); + expect( + fs + .readdirSync(destination) + .some(name => name.startsWith('.run.')) + ).toBe(false); + } + } + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); +} diff --git a/api/src/pipe-process.test.ts b/api/src/pipe-process.test.ts index 608002c8..52d904ea 100644 --- a/api/src/pipe-process.test.ts +++ b/api/src/pipe-process.test.ts @@ -79,11 +79,19 @@ describe('anonymous process pipes', () => { if (process.platform === 'linux') { // Runtime watcher/pid descriptors can appear asynchronously. Assert the // exact pipe inodes we allocated, including duplicates held by FileSink. - const survivors = fs.readdirSync('/proc/self/fd').flatMap(fd => { + const survivors = () => fs.readdirSync('/proc/self/fd').flatMap(fd => { try { const inode = fs.readlinkSync(`/proc/self/fd/${fd}`); return ownedPipes.has(inode) ? [inode] : []; } catch { return []; } }); - expect(survivors).toEqual([]); + // Bun may finish releasing internal FD duplicates after child close. + // Allow bounded asynchronous cleanup, but still reject persistent leaks. + const deadline = performance.now() + 1000; + let remaining = survivors(); + while (remaining.length && performance.now() < deadline) { + await new Promise(resolve => setTimeout(resolve, 10)); + remaining = survivors(); + } + expect(remaining).toEqual([]); } }, 10000); }); diff --git a/api/src/runtime-module-links.test.ts b/api/src/runtime-module-links.test.ts new file mode 100644 index 00000000..6ef754ab --- /dev/null +++ b/api/src/runtime-module-links.test.ts @@ -0,0 +1,207 @@ +import { expect, test } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { config } from './config'; +import { ensureNodeModulesSymlink } from './job'; + +function fixture( + run: (workspace: string, packages: string, modules: string) => void +): void { + const root = fs.mkdtempSync( + path.join(os.tmpdir(), 'runtime-module-links-') + ); + const originalPackages = config.packages_directory; + const workspace = path.join(root, 'workspace'); + const packages = path.join(root, 'pkgs'); + const modules = path.join(packages, 'node', '24.22.1', 'node_modules'); + fs.mkdirSync(workspace); + fs.mkdirSync(modules, { recursive: true }); + config.packages_directory = packages; + try { + run(workspace, packages, modules); + } finally { + config.packages_directory = originalPackages; + fs.rmSync(root, { recursive: true, force: true }); + } +} + +const managers = [ + { + name: 'bash job preparation', + prepare: ensureNodeModulesSymlink, + }, +]; +for (const file of ['build-packages.sh', 'docker/package-init.sh']) { + const source = fs + .readFileSync(path.resolve(__dirname, '../..', file), 'utf8') + .replace(/\\(["$])/g, '$1'); + const launchers = + source.match(/MODULE_DIR=.*\nif \[ -d [\s\S]*?\nfi/g) ?? []; + expect(launchers).toHaveLength(2); + for (const [index, launcher] of launchers.entries()) { + managers.push({ + name: `${file} ${index === 0 ? 'Node' : 'Bun'} launcher`, + prepare(workspace: string, modules?: string): void { + const link = path.join(workspace, 'node_modules'); + const script = `SCRIPT_DIR=$1\n${launcher + .split('/mnt/data/node_modules') + .join('"$2"')}`; + const result = Bun.spawnSync({ + cmd: [ + 'bash', + '-c', + script, + 'runtime-launcher', + path.dirname(modules!), + link, + ], + }); + expect(result.exitCode).toBe(0); + }, + }); + } +} + +for (const manager of managers) { + test(`${manager.name}: links bundled modules into a fresh workspace`, () => { + fixture((workspace, _packages, modules) => { + manager.prepare(workspace, modules); + expect(fs.readlinkSync(path.join(workspace, 'node_modules'))).toBe( + modules + ); + }); + }); + + for (const oldRuntimeExists of [false, true]) { + test(`${manager.name}: refreshes a ${ + oldRuntimeExists ? 'retained' : 'removed' + } runtime link`, () => { + fixture((workspace, packages, modules) => { + const oldModules = path.join( + packages, + 'node', + '24.15.0', + 'node_modules' + ); + if (oldRuntimeExists) + fs.mkdirSync(oldModules, { recursive: true }); + fs.symlinkSync( + oldModules, + path.join(workspace, 'node_modules') + ); + manager.prepare(workspace, modules); + expect( + fs.readlinkSync(path.join(workspace, 'node_modules')) + ).toBe(modules); + }); + }); + } + + test(`${manager.name}: leaves the current runtime link unchanged`, () => { + fixture((workspace, _packages, modules) => { + const link = path.join(workspace, 'node_modules'); + fs.symlinkSync(modules, link); + const inode = fs.lstatSync(link).ino; + manager.prepare(workspace, modules); + expect(fs.lstatSync(link).ino).toBe(inode); + }); + }); + + test(`${manager.name}: preserves a user-managed directory`, () => { + fixture((workspace, _packages, modules) => { + const link = path.join(workspace, 'node_modules'); + fs.mkdirSync(link); + fs.writeFileSync(path.join(link, 'user-package'), 'keep'); + manager.prepare(workspace, modules); + expect( + fs.readFileSync(path.join(link, 'user-package'), 'utf8') + ).toBe('keep'); + }); + }); + + test(`${manager.name}: preserves a valid user link sharing the package-root prefix`, () => { + fixture((workspace, packages, modules) => { + const target = `${packages}-other/node_modules`; + const link = path.join(workspace, 'node_modules'); + fs.mkdirSync(target, { recursive: true }); + fs.symlinkSync(target, link); + manager.prepare(workspace, modules); + expect(fs.readlinkSync(link)).toBe(target); + }); + }); + + test(`${manager.name}: preserves a user link with a package-root traversal`, () => { + fixture((workspace, packages, modules) => { + const target = `${packages}/../user/node_modules`; + const link = path.join(workspace, 'node_modules'); + fs.mkdirSync(path.resolve(target), { recursive: true }); + fs.symlinkSync(target, link); + manager.prepare(workspace, modules); + expect(fs.readlinkSync(link)).toBe(target); + }); + }); + + for (const target of [ + '../user/node_modules', + '../pkgs-other/node_modules', + ]) { + test(`${manager.name}: preserves user link ${target}`, () => { + fixture((workspace, _packages, modules) => { + const link = path.join(workspace, 'node_modules'); + fs.symlinkSync(target, link); + manager.prepare(workspace, modules); + expect(fs.readlinkSync(link)).toBe(target); + }); + }); + } +} + +test('restored Node sessions resolve ESM packages after the old runtime is removed', () => { + fixture((workspace, packages, modules) => { + const oldModules = path.join( + packages, + 'node', + '24.15.0', + 'node_modules' + ); + fs.symlinkSync(oldModules, path.join(workspace, 'node_modules')); + const archive = path.join(path.dirname(workspace), 'checkpoint.tar'); + expect( + Bun.spawnSync({ + cmd: ['tar', '-cf', archive, '-C', workspace, '.'], + }).exitCode + ).toBe(0); + fs.rmSync(workspace, { recursive: true }); + fs.mkdirSync(workspace); + expect( + Bun.spawnSync({ cmd: ['tar', '-xf', archive, '-C', workspace] }) + .exitCode + ).toBe(0); + const pkg = path.join(modules, 'runtime-fixture'); + fs.mkdirSync(pkg); + fs.writeFileSync( + path.join(pkg, 'package.json'), + JSON.stringify({ type: 'module', exports: './index.js' }) + ); + fs.writeFileSync( + path.join(pkg, 'index.js'), + 'export default "upgraded";' + ); + + ensureNodeModulesSymlink(workspace, modules); + + const result = Bun.spawnSync({ + cmd: [ + process.env.NODE_RUNTIME_TEST_BINARY ?? 'node', + '--input-type=module', + '-e', + 'import value from "runtime-fixture"; console.log(value);', + ], + cwd: workspace, + }); + expect(result.stderr.toString()).toBe(''); + expect(result.exitCode).toBe(0); + expect(result.stdout.toString().trim()).toBe('upgraded'); + }); +}); diff --git a/api/src/workspace-command.test.ts b/api/src/workspace-command.test.ts index 38936a3c..e2a9eecb 100644 --- a/api/src/workspace-command.test.ts +++ b/api/src/workspace-command.test.ts @@ -1,10 +1,11 @@ -import { mkdtemp, mkdir, realpath, rm, symlink } from 'node:fs/promises'; +import { mkdtemp, mkdir, readFile, readlink, realpath, rm, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, expect, test } from 'bun:test'; import semver from 'semver'; -import type { Runtime } from './runtime'; +import { config } from './config'; +import { getRuntimes, type Runtime } from './runtime'; import { executeWorkspaceCommand, hasWorkspaceCommandToken, @@ -79,6 +80,80 @@ describe('sandboxed workspace commands', () => { } }); + for (const existingModules of ['absent', 'removed', 'retained', 'directory', 'external']) { + test(`prepares ${existingModules} modules before workspace-command ESM execution`, async () => { + const parent = await mkdtemp(join(tmpdir(), 'workspace-command-modules-')); + const savedPackages = config.packages_directory; + const registered = getRuntimes(); + const savedRuntimes = registered.splice(0); + try { + const packages = join(parent, 'pkgs'); + const root = join(parent, 'workspace'); + const modules = join(packages, 'node/24.22.1/node_modules'); + const link = join(root, 'node_modules'); + await mkdir(join(root, 'src'), { recursive: true }); + await mkdir(join(modules, 'runtime-fixture'), { recursive: true }); + await writeFile(join(modules, 'runtime-fixture/package.json'), JSON.stringify({ type: 'module', exports: './index.js' })); + await writeFile(join(modules, 'runtime-fixture/index.js'), 'export default "upgraded";'); + config.packages_directory = packages; + registered.push({ + ...runtime, + language: 'node', + version: new semver.SemVer('24.22.1'), + pkgdir: join(packages, 'node/24.22.1'), + env_vars: { NODE_PATH: modules }, + }); + let userTarget: string | undefined; + if (existingModules === 'directory') { + await mkdir(link); + await writeFile(join(link, 'keep'), 'user module'); + } else if (existingModules === 'external') { + userTarget = join(packages, '../user/node_modules'); + await mkdir(userTarget, { recursive: true }); + await symlink(userTarget, link); + } else if (existingModules !== 'absent') { + const oldModules = join(packages, 'node/24.15.0/node_modules'); + if (existingModules === 'retained') await mkdir(oldModules, { recursive: true }); + await symlink(oldModules, link); + } + let executed = false; + await executeWorkspaceCommand({ command: 'node app.mjs', cwd: 'src' }, { + workspaceRoot: root, + runtime, + executeNsJail: async options => { + executed = true; + expect(options.extraPkgdirs).toContain(join(packages, 'node/24.22.1')); + expect(options.envVars.NODE_PATH).toBe(modules); + if (existingModules === 'directory') { + expect(await readFile(join(link, 'keep'), 'utf8')).toBe('user module'); + } else if (existingModules === 'external') { + expect(await readlink(link)).toBe(userTarget!); + } else { + expect(await readlink(link)).toBe(modules); + const result = Bun.spawnSync({ + cmd: [process.env.NODE_RUNTIME_TEST_BINARY ?? 'node', '--input-type=module', '-e', + 'import value from "runtime-fixture"; console.log(value);'], + cwd: join(root, 'src'), + }); + expect(result.stderr.toString()).toBe(''); + expect(result.exitCode).toBe(0); + expect(result.stdout.toString().trim()).toBe('upgraded'); + } + return { + stdout: '', stderr: '', code: 0, signal: null, output: '', + memory: null, message: null, status: null, cpu_time: null, wall_time: 1, + }; + }, + }); + expect(executed).toBe(true); + } finally { + registered.splice(0, registered.length, ...savedRuntimes); + config.packages_directory = savedPackages; + await rm(parent, { recursive: true, force: true }); + } + }); + } + test('rejects a cwd symlink that escapes the mounted workspace', async () => { const parent = await mkdtemp(join(tmpdir(), 'workspace-command-')); try { diff --git a/api/src/workspace-command.ts b/api/src/workspace-command.ts index a2cd5339..34180da9 100644 --- a/api/src/workspace-command.ts +++ b/api/src/workspace-command.ts @@ -2,7 +2,7 @@ import * as path from 'node:path'; import * as fsp from 'node:fs/promises'; import { timingSafeEqual } from 'node:crypto'; -import { aggregateBashExtras } from './job'; +import { aggregateBashExtras, ensureNodeModulesSymlink } from './job'; import { execute, type NsJailResult } from './nsjail'; import { getLatestRuntimeMatchingLanguageVersion } from './runtime'; import type { Runtime } from './runtime'; @@ -169,7 +169,9 @@ export async function executeWorkspaceCommand( HOME: '/mnt/data', SANDBOX_LANGUAGE: 'bash', }; - const extraPkgdirs = aggregateBashExtras(runtime.pkgdir, envVars); + const linkTarget: { nodeModulesPath?: string } = {}; + const extraPkgdirs = aggregateBashExtras(runtime.pkgdir, envVars, undefined, linkTarget); + ensureNodeModulesSymlink(workspaceRoot, linkTarget.nodeModulesPath); const relativeCwd = path.relative(workspaceRoot, cwd).split(path.sep).join('/'); const result = await (options.executeNsJail ?? execute)({ command: [ diff --git a/build-packages.sh b/build-packages.sh index 9954a143..32d0ebcc 100755 --- a/build-packages.sh +++ b/build-packages.sh @@ -9,8 +9,8 @@ # # Environment Variables: # PYTHON_VERSION=3.14.4 # Python version to install -# NODE_VERSION=24.15.0 # Node.js version to install -# BUN_VERSION=1.4.2 # Bun version to install +# NODE_VERSION=24.22.1 # Node.js version to install +# BUN_VERSION=1.4.4 # Bun version to install # BASH_PACKAGE_VERSION=5.2.0 # Bash package registration version (semver-like x.y.z) # SKIP_PYTHON_PACKAGES=1 # Skip Python pip packages # SKIP_JS_PACKAGES=1 # Skip JavaScript npm packages for Node/Bun @@ -27,8 +27,8 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cd "$SCRIPT_DIR" PYTHON_VERSION="${PYTHON_VERSION:-3.14.4}" -NODE_VERSION="${NODE_VERSION:-24.15.0}" -BUN_VERSION="${BUN_VERSION:-1.4.2}" +NODE_VERSION="${NODE_VERSION:-24.22.1}" +BUN_VERSION="${BUN_VERSION:-1.4.4}" PACKAGES_DIR="./data/pkgs" JS_PACKAGE_MANIFEST="${JS_PACKAGE_MANIFEST:-${SCRIPT_DIR}/javascript-packages.txt}" @@ -277,8 +277,18 @@ EOF" #!/bin/bash SCRIPT_DIR=\"\$(cd \"\$(dirname \"\${BASH_SOURCE[0]}\")\" && pwd)\" MODULE_DIR=\"\${SCRIPT_DIR}/node_modules\" -if [ -d \"\${MODULE_DIR}\" ] && [ ! -e /mnt/data/node_modules ]; then - ln -s \"\${MODULE_DIR}\" /mnt/data/node_modules 2>/dev/null || true +if [ -d \"\${MODULE_DIR}\" ]; then + if [ -L /mnt/data/node_modules ]; then + LINK_TARGET=\"\$(readlink -m /mnt/data/node_modules)\" + if [ \"\${LINK_TARGET}\" != \"\${MODULE_DIR}\" ]; then + case \"\${LINK_TARGET}\" in + \"\${SCRIPT_DIR%/*/*}/\"*/node_modules) + ln -sfnT \"\${MODULE_DIR}\" /mnt/data/node_modules ;; + esac + fi + elif [ ! -e /mnt/data/node_modules ]; then + ln -s \"\${MODULE_DIR}\" /mnt/data/node_modules 2>/dev/null || true + fi fi \"\${SCRIPT_DIR}/bin/node\" \"\$@\" EOF @@ -379,8 +389,18 @@ EOF" #!/bin/bash SCRIPT_DIR=\"\$(cd \"\$(dirname \"\${BASH_SOURCE[0]}\")\" && pwd)\" MODULE_DIR=\"\${SCRIPT_DIR}/node_modules\" -if [ -d \"\${MODULE_DIR}\" ] && [ ! -e /mnt/data/node_modules ]; then - ln -s \"\${MODULE_DIR}\" /mnt/data/node_modules 2>/dev/null || true +if [ -d \"\${MODULE_DIR}\" ]; then + if [ -L /mnt/data/node_modules ]; then + LINK_TARGET=\"\$(readlink -m /mnt/data/node_modules)\" + if [ \"\${LINK_TARGET}\" != \"\${MODULE_DIR}\" ]; then + case \"\${LINK_TARGET}\" in + \"\${SCRIPT_DIR%/*/*}/\"*/node_modules) + ln -sfnT \"\${MODULE_DIR}\" /mnt/data/node_modules ;; + esac + fi + elif [ ! -e /mnt/data/node_modules ]; then + ln -s \"\${MODULE_DIR}\" /mnt/data/node_modules 2>/dev/null || true + fi fi \"\${SCRIPT_DIR}/bun\" run \"\$@\" EOF diff --git a/docker-compose.yaml b/docker-compose.yaml index db517b4c..a4a43cdd 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -15,6 +15,7 @@ services: - CODEAPI_BRIDGE_DYNAMIC_WORKERS=${CODEAPI_BRIDGE_DYNAMIC_WORKERS:-true} - CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS=${CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS:-1} - CODEAPI_BRIDGE_LANE_GIT=${CODEAPI_BRIDGE_LANE_GIT:-false} + - CODEAPI_BRIDGE_IMPORT_FILE_MAX_BYTES=${CODEAPI_BRIDGE_IMPORT_FILE_MAX_BYTES:-104857600} - CODEAPI_BRIDGE_WORKER_ID=${CODEAPI_BRIDGE_WORKER_ID:-} - CODEAPI_BRIDGE_RECOVERY_SERVER_ID=${CODEAPI_BRIDGE_RECOVERY_SERVER_ID:-} - CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS=${CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS:-0} diff --git a/docker/Dockerfile.worker-sandbox b/docker/Dockerfile.worker-sandbox index 8d0524e0..c692f08f 100644 --- a/docker/Dockerfile.worker-sandbox +++ b/docker/Dockerfile.worker-sandbox @@ -48,7 +48,7 @@ COPY api/src/spec-guard.c /tmp/spec-guard.c COPY api/src/tool-call-pipe-bridge.c /tmp/tool-call-pipe-bridge.c COPY api/src/anonymous-pipes.c /tmp/anonymous-pipes.c COPY api/src/supervisor-policy.c /tmp/supervisor-policy.c -COPY --from=node:22-slim /usr/local/include/node /usr/local/include/node +COPY --from=node:22.23.4-slim /usr/local/include/node /usr/local/include/node COPY docker/rootfs-setup.c /tmp/rootfs-setup.c RUN gcc -shared -fPIC -O2 -Wall -Wextra -Werror -I/usr/local/include/node -o /usr/local/lib/anonymous-pipes.node /tmp/anonymous-pipes.c \ && gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/sandbox-supervisor-policy /tmp/supervisor-policy.c \ @@ -68,7 +68,7 @@ FROM buildpack-deps:bookworm AS package-builder ENV DEBIAN_FRONTEND=noninteractive ENV PYTHON_VERSION=3.14.4 -ENV BUN_VERSION=1.4.2 +ENV BUN_VERSION=1.4.4 COPY docker/apt-install.sh /usr/local/bin/apt-install RUN chmod +x /usr/local/bin/apt-install @@ -91,7 +91,7 @@ RUN chmod +x /package-init.sh && FORCE_REBUILD=true /package-init.sh # ============================================================================ # Stage 2: Build worker # ============================================================================ -FROM oven/bun:1.4.2 AS worker-builder +FROM oven/bun:1.4.4 AS worker-builder WORKDIR /app COPY service/package.json service/bun.lock ./ RUN bun install --frozen-lockfile @@ -104,7 +104,7 @@ RUN bun build ./src/worker-server.ts --minify --outdir .build --target bun --ext # ============================================================================ # Stage 3: Build sandbox API # ============================================================================ -FROM oven/bun:1.4.2 AS sandbox-builder +FROM oven/bun:1.4.4 AS sandbox-builder WORKDIR /app COPY api/package.json ./ RUN bun install --frozen-lockfile @@ -117,7 +117,7 @@ RUN bun build ./src/index.ts --minify --outdir .build --target bun --external '@ # ============================================================================ # Stage 4: Build sandbox rootfs (full OS layer for the microVM guest) # ============================================================================ -FROM oven/bun:1.4.2-debian AS sandbox-rootfs +FROM oven/bun:1.4.4-debian AS sandbox-rootfs ENV DEBIAN_FRONTEND=noninteractive @@ -171,7 +171,7 @@ COPY --from=nsjail-builder /usr/local/bin/sandbox-supervisor-policy /usr/local/b COPY --from=nsjail-builder /usr/local/lib/anonymous-pipes.node /usr/local/lib/anonymous-pipes.node ENV SANDBOX_PIPE_ADDON=/usr/local/lib/anonymous-pipes.node RUN chmod +x /usr/sbin/nsjail -COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node +COPY --from=node:22.23.4-slim /usr/local/bin/node /usr/local/bin/node WORKDIR /sandbox_api diff --git a/docker/package-init.sh b/docker/package-init.sh index 2ed75771..d274ccd7 100644 --- a/docker/package-init.sh +++ b/docker/package-init.sh @@ -20,8 +20,8 @@ PYTHON_ALIAS="python${PYTHON_SITE_VERSION}" # below — that's inherent to installing unpinned packages, not specific to uv. PYTHON_PACKAGE_INSTALLER="${PYTHON_PACKAGE_INSTALLER:-uv}" UV_VERSION="${UV_VERSION:-0.11.26}" -NODE_VERSION="${NODE_VERSION:-24.15.0}" -BUN_VERSION="${BUN_VERSION:-1.4.2}" +NODE_VERSION="${NODE_VERSION:-24.22.1}" +BUN_VERSION="${BUN_VERSION:-1.4.4}" BASH_PACKAGE_VERSION="${BASH_PACKAGE_VERSION:-5.2.0}" INSTALL_FAILED=false JS_PACKAGE_MANIFEST="${JS_PACKAGE_MANIFEST:-${SCRIPT_DIR}/javascript-packages.txt}" @@ -103,8 +103,64 @@ packages_ready() { [ -f "/pkgs/bash/${BASH_PACKAGE_VERSION}/.package-installed" ] } +write_node_launcher() { + local destination="$1" + local launcher + launcher="$(mktemp "${destination}/.run.XXXXXX")" + cat > "$launcher" << 'EOF' +#!/bin/bash +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MODULE_DIR="${SCRIPT_DIR}/node_modules" +if [ -d "$MODULE_DIR" ]; then + if [ -L /mnt/data/node_modules ]; then + LINK_TARGET="$(readlink -m /mnt/data/node_modules)" + if [ "$LINK_TARGET" != "$MODULE_DIR" ]; then + case "$LINK_TARGET" in + "${SCRIPT_DIR%/*/*}/"*/node_modules) + ln -sfnT "$MODULE_DIR" /mnt/data/node_modules ;; + esac + fi + elif [ ! -e /mnt/data/node_modules ]; then + ln -s "$MODULE_DIR" /mnt/data/node_modules 2>/dev/null || true + fi +fi +"${SCRIPT_DIR}/bin/node" "$@" +EOF + chmod 0755 "$launcher" + mv -f "$launcher" "${destination}/run" +} + +write_bun_launcher() { + local destination="$1" + local launcher + launcher="$(mktemp "${destination}/.run.XXXXXX")" + cat > "$launcher" << 'EOF' +#!/bin/bash +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MODULE_DIR="${SCRIPT_DIR}/node_modules" +if [ -d "$MODULE_DIR" ]; then + if [ -L /mnt/data/node_modules ]; then + LINK_TARGET="$(readlink -m /mnt/data/node_modules)" + if [ "$LINK_TARGET" != "$MODULE_DIR" ]; then + case "$LINK_TARGET" in + "${SCRIPT_DIR%/*/*}/"*/node_modules) + ln -sfnT "$MODULE_DIR" /mnt/data/node_modules ;; + esac + fi + elif [ ! -e /mnt/data/node_modules ]; then + ln -s "$MODULE_DIR" /mnt/data/node_modules 2>/dev/null || true + fi +fi +"${SCRIPT_DIR}/bun" run "$@" +EOF + chmod 0755 "$launcher" + mv -f "$launcher" "${destination}/run" +} + if [ -f "$MARKER_FILE" ] && [ "$FORCE_REBUILD" != "true" ]; then if packages_ready; then + write_node_launcher "/pkgs/node/${NODE_VERSION}" + write_bun_launcher "/pkgs/bun/${BUN_VERSION}" echo "Packages already initialized (marker file exists)" echo "Set FORCE_REBUILD=true to force reinstall" echo "" @@ -309,16 +365,7 @@ if [ -n "$NODE_ARCH" ]; then } EOF - cat > "$NODE_DEST/run" << 'EOF' -#!/bin/bash -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -MODULE_DIR="${SCRIPT_DIR}/node_modules" -if [ -d "$MODULE_DIR" ] && [ ! -e /mnt/data/node_modules ]; then - ln -s "$MODULE_DIR" /mnt/data/node_modules 2>/dev/null || true -fi -"${SCRIPT_DIR}/bin/node" "$@" -EOF - chmod +x "$NODE_DEST/run" + write_node_launcher "$NODE_DEST" { echo "PATH=${NODE_DEST}/bin:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin:." @@ -421,16 +468,7 @@ if [ -n "$BUN_ARCH" ]; then } EOF - cat > "$BUN_DEST/run" << 'EOF' -#!/bin/bash -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -MODULE_DIR="${SCRIPT_DIR}/node_modules" -if [ -d "$MODULE_DIR" ] && [ ! -e /mnt/data/node_modules ]; then - ln -s "$MODULE_DIR" /mnt/data/node_modules 2>/dev/null || true -fi -"${SCRIPT_DIR}/bun" run "$@" -EOF - chmod +x "$BUN_DEST/run" + write_bun_launcher "$BUN_DEST" { echo "PATH=${BUN_DEST}:/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin:." diff --git a/docs/SECURITY-MAINTENANCE.md b/docs/SECURITY-MAINTENANCE.md index f33e509f..05631de6 100644 --- a/docs/SECURITY-MAINTENANCE.md +++ b/docs/SECURITY-MAINTENANCE.md @@ -120,7 +120,7 @@ does not mount. The namespace-disabled native policy tests cover those APIs when shared memory exists, not their availability in the default jail. With runtime packages built by `build-packages.sh` (or a compatible `/pkgs` -tree containing Python 3.14.4 and Bun 1.4.2), run from the repository root: +tree containing Python 3.14.4 and Bun 1.4.4), run from the repository root: ```sh bun service/scripts/dump-pipe-preamble.ts > /tmp/codeapi-pipe-client.py @@ -180,7 +180,7 @@ Remaining service audit findings at this change: - `braces` (GHSA-vfj7-8cjw-p6xm): development-only dependency with no published fix. Do not feed untrusted patterns to build tooling; track an upstream fix. -Bun 1.4.2 subprocess spawning with piped stdio also uses socketpairs and is +Bun 1.4.4 subprocess spawning with piped stdio also uses socketpairs and is denied by this policy. Basic JS/TS execution remains supported. Replacing Bun subprocess IPC requires a separate runtime change; do not re-enable socketpairs to provide that compatibility. diff --git a/docs/adr/0014-workspace-file-import.md b/docs/adr/0014-workspace-file-import.md new file mode 100644 index 00000000..e1aac4a7 --- /dev/null +++ b/docs/adr/0014-workspace-file-import.md @@ -0,0 +1,65 @@ +# 0014: Import chat uploads into attached workspaces + +Status: Accepted + +## Context + +A LibreChat chat can run on a personal machine through the remote bridge. Files +the user uploads in that chat stay in Code API storage: only programmatic Bash +stages them, per run, in a private scratch directory. `write_file` accepts +1 MiB of UTF-8 text, so binaries such as archives, images, and PDFs cannot reach +the workspace at all. + +## Decision + +Add one additive workspace operation, `import_file`, under protocol version 1. + +- **Request.** `{ path, source: { storage_session_id, id }, bytes, sha256, + overwrite? }` plus the usual workspace selectors. `sha256` is lowercase hex. +- **Ownership.** Code API, not the caller, decides whether `source` may be read. + It applies the `GET /download/:session_id/:fileId` rule for a user-kind + object: the live `session:` key must name the caller's storage key, and an + upload record must exist for that key, session, and object. +- **Grant.** Code API asks the egress gateway for a grant whose only input is + that object, with no upload capacity, two reads, and an expiry of the + request's admission plus execution budget plus 30 seconds. The relayed + request carries the grant and opaque handles in place of the raw ids. A + caller-supplied `egressGrant` is rejected. Synchronous requests revoke the + grant when they finish; durable requests let it expire. +- **Worker.** The worker streams the object from + `LIBRECHAT_CODE_FILE_RELAY_UPSTREAM`, the route programmatic inputs already + use, and holds no storage credential. It applies `write_file`'s path rules, + refuses `.git` segments and their filesystem aliases, refuses symlinked + parents, creates missing parents as 0755, enforces `bytes` while streaming + into a same-directory temporary file, verifies size and SHA-256, syncs, and + installs the file atomically with mode 0644. The size and hash are checked on + the synced temporary file itself. A create links exclusively; an overwrite + rechecks the inspected inode immediately before its rename, as `write_file` + does. +- **Idempotency.** Identical existing bytes return `unchanged: true` and + `bytesWritten: 0` without a transfer. Different bytes return `EDIT_CONFLICT` + unless `overwrite: true`. +- **Limits.** Code API (`CODEAPI_BRIDGE_IMPORT_FILE_MAX_BYTES`) and the worker + (`LIBRECHAT_CODE_IMPORT_FILE_MAX_BYTES`) each enforce a per-file ceiling, + 100 MiB by default, as `WRITE_LIMIT_EXCEEDED`. The worker also requires the + file plus 64 MiB to fit in the target filesystem's free space. +- **Policy.** `import_file` is a file write. It shares the mutation guard with + `write_file`, is never advertised for a read-only root, and a read-only root + refuses it with `WRITE_DISABLED`. +- **Negotiation.** Code API lists `import_file` in + `supportedWorkspaceToolOperations` only when `EGRESS_GATEWAY_URL` is set. A + worker advertises it by default whenever writes are enabled, a relay + upstream is configured, and the Code API listed it; operators opt out with + `--no-workspace-imports` or `LIBRECHAT_CODE_ALLOW_WORKSPACE_IMPORTS=false`. + LibreChat sends it only when advertised. LibreChat releases that predate + LibreChat-AI/LibreChat#17376 reject a worker status listing an unknown + operation, so LibreChat is updated first and the opt-out is the escape + hatch for workers serving an older one. + +## Consequences + +Uploads land in the workspace without a model round trip, and older Code APIs, +workers, and LibreChat releases keep their current behavior. Deployments +without an egress gateway do not offer the operation. Parent directories are +created only beneath a descriptor-held root, as for `write_file`; a legacy +pathname-only root reports a missing parent instead. diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index dc7cb98b..40465e00 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -172,6 +172,11 @@ whitespace-tolerant matching and multi-location replacement (see text, require an existing in-workspace parent directory, reject symlinks, and commit atomically. The worker capability is an enforcement boundary; LibreChat should still route every mutation through its configurable tool-approval hooks. +Writable workers with a file relay upstream also offer `import_file` by +default (opt out with `LIBRECHAT_CODE_ALLOW_WORKSPACE_IMPORTS=false`), which +copies a chat upload the caller owns into the workspace through a one-object +egress grant (ADR 0014). It is a file write: a worker without writes enabled +never advertises it and refuses it with `WRITE_DISABLED`. The workspace root can be an existing project, a Git repository, or an empty directory; Git is not required. This boundary keeps that directory local to the diff --git a/docs/remote-bridge/worker-runbook.md b/docs/remote-bridge/worker-runbook.md index 99f19992..859104c6 100644 --- a/docs/remote-bridge/worker-runbook.md +++ b/docs/remote-bridge/worker-runbook.md @@ -561,6 +561,22 @@ To keep every worktree, add `--no-worktree-retirement` (or plist. - **Worker online but not ready:** check native sandbox preparation, definition validation, setup, quarantine, and readiness logs. +- **Files work but commands return `503 COMMAND_UNAVAILABLE`:** inspect the + worker's stderr/service journal for `workspace_command_unavailable`, and + correlate its `assignmentId` with the bridge request. Updated workers emit + one JSON record for a rejected command or programmatic assignment, with + `stage`, `reason`, workspace/worker/incarnation IDs, mutation uncertainty, + quarantine requirement, and an allowlisted `osCode` when available. Stages + distinguish allocation, sandbox startup, programmatic prerequisites, + credential resolution, command wrapping, dispatch, executor transport, and + cleanup. For example, `sandbox_startup` with `osCode: ENOSPC` points to + storage; `credential_resolution` identifies credential setup rather than + command execution. Missing errno does not rule out either cause. + Commands, arguments, paths, credentials, output, and free-form exception + messages are excluded from these records. Diagnostics stay worker-local; + the bridge response and quarantine/retry behavior are unchanged. A worker + package update is required; a Code API restart alone does not enable this. + These records cover assignment failures, not failures before worker startup. - **Setup repeats on restart:** setup is intentionally per-start; make it idempotent, remove it, or opt in to the bounded `setup.reuse` readiness contract in the [worker package guide](../../packages/code/README.md#reusing-a-prepared-checkout). diff --git a/helm/codeapi/values.yaml b/helm/codeapi/values.yaml index 4858fd71..33c32af0 100644 --- a/helm/codeapi/values.yaml +++ b/helm/codeapi/values.yaml @@ -310,8 +310,8 @@ workerSandbox: tag: latest pullPolicy: Always pythonVersion: "3.14.4" - nodeVersion: "24.15.0" - bunVersion: "1.4.2" + nodeVersion: "24.22.1" + bunVersion: "1.4.4" bashPackageVersion: "5.2.0" backoffLimit: 2 ttlSecondsAfterFinished: 3600 diff --git a/launcher/Dockerfile b/launcher/Dockerfile index bad5cd2f..95a4d48d 100644 --- a/launcher/Dockerfile +++ b/launcher/Dockerfile @@ -40,7 +40,7 @@ COPY api/src/spec-guard.c /tmp/spec-guard.c COPY api/src/tool-call-pipe-bridge.c /tmp/tool-call-pipe-bridge.c COPY api/src/anonymous-pipes.c /tmp/anonymous-pipes.c COPY api/src/supervisor-policy.c /tmp/supervisor-policy.c -COPY --from=node:22-slim /usr/local/include/node /usr/local/include/node +COPY --from=node:22.23.4-slim /usr/local/include/node /usr/local/include/node COPY docker/rootfs-setup.c /tmp/rootfs-setup.c RUN gcc -shared -fPIC -O2 -Wall -Wextra -Werror -I/usr/local/include/node -o /usr/local/lib/anonymous-pipes.node /tmp/anonymous-pipes.c \ && gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/sandbox-supervisor-policy /tmp/supervisor-policy.c \ @@ -49,7 +49,7 @@ RUN gcc -shared -fPIC -O2 -Wall -Wextra -Werror -I/usr/local/include/node -o /us && gcc -O2 -static -o /usr/local/bin/sandbox-rootfs-setup /tmp/rootfs-setup.c \ && chmod 0111 /usr/local/bin/spec-guard /usr/local/bin/sandbox-rootfs-setup -FROM oven/bun:1.4.2-debian AS sandbox-build +FROM oven/bun:1.4.4-debian AS sandbox-build ENV DEBIAN_FRONTEND=noninteractive @@ -79,7 +79,7 @@ COPY --from=nsjail-builder /usr/local/bin/tool-call-pipe-bridge /usr/local/bin/t COPY --from=nsjail-builder /usr/local/bin/sandbox-supervisor-policy /usr/local/bin/sandbox-supervisor-policy COPY --from=nsjail-builder /usr/local/lib/anonymous-pipes.node /usr/local/lib/anonymous-pipes.node ENV SANDBOX_PIPE_ADDON=/usr/local/lib/anonymous-pipes.node -COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node +COPY --from=node:22.23.4-slim /usr/local/bin/node /usr/local/bin/node RUN chmod +x /usr/sbin/nsjail WORKDIR /sandbox_api diff --git a/packages/code/Dockerfile b/packages/code/Dockerfile index 93ed9c57..34c54899 100644 --- a/packages/code/Dockerfile +++ b/packages/code/Dockerfile @@ -1,11 +1,11 @@ -FROM node:24-alpine AS build +FROM node:24.22.1-alpine AS build WORKDIR /app COPY package.json package-lock.json tsconfig.json ./ RUN npm ci COPY src ./src RUN npm run build -FROM node:24-alpine +FROM node:24.22.1-alpine ENV NODE_ENV=production RUN apk add --no-cache bash bubblewrap ripgrep socat \ && addgroup -S librechat-code \ diff --git a/packages/code/README.md b/packages/code/README.md index 0437ecfb..f11e0072 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -630,6 +630,34 @@ with `--allow-workspace-writes` or `overwrite: false` to require an atomic create that returns `EDIT_CONFLICT` if the target already exists. Code API dispatches that mode only after the worker and server negotiate `create` in `writeFileModes`. +`write_file` and `import_file` create missing parent directories one level at +a time through `mkdirat` on a held root descriptor, so a swapped symlink cannot +redirect them outside the root. Selected projects, linked-worktree lanes and +conversation worktrees hold that descriptor for every request; a root +registered by path alone (`--worker-dir`, `--workspace`, `--environment`) holds +one for just the write or import whose parent is missing. A root the worker +cannot hold by descriptor is never advertised `import_file`. + +`import_file` copies one stored chat upload into a writable workspace as a +binary-safe file. It is on by default whenever the worker can perform it: +writes are enabled and `LIBRECHAT_CODE_FILE_RELAY_UPSTREAM` names the +deployment's egress gateway. It is advertised only to a Code API that lists it +in `supportedWorkspaceToolOperations` (one that can mint egress grants). Opt out +with `--no-workspace-imports` or `LIBRECHAT_CODE_ALLOW_WORKSPACE_IMPORTS=false`. +Update LibreChat before workers: LibreChat releases that predate +LibreChat-AI/LibreChat#17376 reject a worker status that lists an operation +they do not know, so a worker serving one of those must opt out. Code API checks that the caller owns the object +and relays a short-lived grant scoped to that object alone; the worker holds no +storage credential. The worker applies `write_file`'s path rules, refuses any +`.git` segment (including case, Unicode-ignorable, trailing-dot and `GIT~1` +aliases) and symlinked parents, creates missing parents as 0755 under a held +root descriptor, streams into a same-directory temporary file with `bytes` as a hard cap, +verifies size and SHA-256, syncs, and installs the file atomically with mode +0644. Identical existing bytes return `unchanged: true` without a transfer; +different bytes return `EDIT_CONFLICT` unless `overwrite: true`. A failure +removes the temporary file and any parents it created. The per-file ceiling is +`LIBRECHAT_CODE_IMPORT_FILE_MAX_BYTES` (`--import-file-max-bytes`, default +100 MiB), and the filesystem must keep 64 MiB free after the file lands. `edit_file` accepts either the legacy `oldText`/`newText` pair or an ordered `edits` array; every exact replacement is validated before the updated file is installed as one atomic mutation. Code API dispatches the batch form only after diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index bee3697a..a0352d34 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -5,6 +5,7 @@ import { lstat, readdir, realpath, stat } from 'node:fs/promises'; import { basename, dirname, join, resolve, relative, isAbsolute, sep } from 'node:path'; import { pairBridgeWorker } from './pairing.js'; +import { commandFailureLogLine } from './command-diagnostics.js'; import { runSetupCommand } from './setup-cli.js'; import { browserSetupProtectedPaths } from './browser-setup.js'; import { assertPrivateStorageAncestors } from './private-storage.js'; @@ -39,6 +40,7 @@ import { } from './storage.js'; import { BridgeWorker } from './worker.js'; import { LocalWorkspaceTools, SandboxWorkspaceTools } from './workspace.js'; +import { createWorkspaceFileImporter } from './workspace-import.js'; import { DockerRuntimeSupervisor, EndpointRuntimeSupervisor, @@ -78,6 +80,7 @@ import type { RuntimeSupervisor } from './runtime.js'; import type { GitHubCredentialProvider } from './github.js'; import type { WorkspaceToolExecutor } from './workspace.js'; import { + BRIDGE_WORKSPACE_IMPORT_FILE_DEFAULT_MAX_BYTES, BRIDGE_WORKSPACE_NAME_MAX_LENGTH, BridgeProtocolError, isValidBridgeWorkerCapabilities, @@ -914,10 +917,25 @@ async function run( ]), ) : undefined; + // A chat upload reaches the workspace only through the same relay upstream + // programmatic inputs use; without one the worker never advertises import_file. + const workspaceImporter = createWorkspaceFileImporter({ + optOut: args.includes('--no-workspace-imports'), + enabled: process.env.LIBRECHAT_CODE_ALLOW_WORKSPACE_IMPORTS, + writable: allowWorkspaceWrites, + upstreamUrl: fileRelayUpstream, + maxBytes: positiveInteger( + 'LIBRECHAT_CODE_IMPORT_FILE_MAX_BYTES', + option(args, '--import-file-max-bytes') ?? + process.env.LIBRECHAT_CODE_IMPORT_FILE_MAX_BYTES, + BRIDGE_WORKSPACE_IMPORT_FILE_DEFAULT_MAX_BYTES, + ), + }); const localWorkspaceTools = workerDirectory ? await LocalWorkspaceTools.create({ workspaces: roots, repositoryInstructions: args.includes('--repository-instructions'), + importer: workspaceImporter, }) : undefined; let workspaceTools: WorkspaceToolExecutor | undefined = localWorkspaceTools; @@ -1298,6 +1316,7 @@ async function run( }, repositoryInstructions: args.includes('--repository-instructions'), writable: root.writable ?? false, + importer: workspaceImporter, }, ]), ), @@ -1333,6 +1352,7 @@ async function run( command: nativeOptionsForWorkspace(root.id), repositoryInstructions: args.includes('--repository-instructions'), writable: root.writable ?? false, + importer: workspaceImporter, }, ]), ), @@ -1511,6 +1531,9 @@ async function run( let readinessReported = false; try { const worker = new BridgeWorker({ + onCommandFailure: event => { + process.stderr.write(commandFailureLogLine(event)); + }, onReady: () => { if (readinessReported) return; readinessReported = true; diff --git a/packages/code/src/command-diagnostics.test.ts b/packages/code/src/command-diagnostics.test.ts new file mode 100644 index 00000000..76e48f30 --- /dev/null +++ b/packages/code/src/command-diagnostics.test.ts @@ -0,0 +1,165 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + commandFailureDiagnostic, + commandFailureEvent, + commandFailureLogLine, + readCommandFailureDiagnostic, +} from './command-diagnostics.js'; + +test('command diagnostics retain only fixed classifications and allowlisted errno', () => { + const error = Object.assign( + new Error('token secret /private/path echo password'), + { + code: 'ENOSPC', + command: 'echo password', + token: 'secret', + }, + ); + assert.deepEqual( + commandFailureDiagnostic('sandbox_startup', 'startup_failed', error), + { + stage: 'sandbox_startup', + reason: 'startup_failed', + osCode: 'ENOSPC', + }, + ); + assert.deepEqual( + commandFailureDiagnostic('credential_resolution', 'credential_failed', { + code: 'private-provider-value', + }), + { stage: 'credential_resolution', reason: 'credential_failed' }, + ); +}); + +test('IPC diagnostics are reconstructed rather than forwarding arbitrary metadata', () => { + const diagnostic = { + stage: 'sandbox_startup', + reason: 'startup_failed', + osCode: 'EPERM', + message: 'secret', + command: 'echo password', + path: '/private/path', + }; + assert.deepEqual(readCommandFailureDiagnostic(diagnostic), { + stage: 'sandbox_startup', + reason: 'startup_failed', + osCode: 'EPERM', + }); + for (const value of [ + null, + 'secret', + {}, + { stage: 'secret', reason: 'startup_failed' }, + { stage: 'sandbox_startup', reason: 'secret' }, + ]) { + assert.equal(readCommandFailureDiagnostic(value), undefined); + } + assert.deepEqual( + readCommandFailureDiagnostic({ ...diagnostic, osCode: 'secret' }), + { + stage: 'sandbox_startup', + reason: 'startup_failed', + }, + ); +}); + +test('failure events bound correlation IDs and omit free-form error data', () => { + const event = commandFailureEvent(undefined, { + assignmentId: 'assignment-1', + workerId: 'worker-1', + incarnationId: 'incarnation-1', + workspaceId: '/private/path\nsecret', + operation: 'execute_command', + mutationMayHaveCommitted: false, + requiresQuarantine: false, + }); + assert.deepEqual(JSON.parse(JSON.stringify(event)), { + event: 'workspace_command_unavailable', + stage: 'dispatch', + reason: 'executor_rejected', + assignmentId: 'assignment-1', + workerId: 'worker-1', + incarnationId: 'incarnation-1', + operation: 'execute_command', + mutationMayHaveCommitted: false, + requiresQuarantine: false, + }); + assert.equal( + commandFailureEvent(undefined, { + ...event, + assignmentId: 'a'.repeat(129), + }).assignmentId, + undefined, + ); +}); + +test('diagnostic correlation retains every valid worker/workspace identifier character', () => { + const event = commandFailureEvent(undefined, { + assignmentId: '_assignment-1', + workerId: 'vm.example', + workspaceId: 'repo:api', + incarnationId: 'worker_v1-2', + operation: 'execute_command', + mutationMayHaveCommitted: false, + requiresQuarantine: false, + }); + assert.equal(event.workerId, 'vm.example'); + assert.equal(event.workspaceId, 'repo:api'); + assert.equal(event.incarnationId, 'worker_v1-2'); + assert.equal(event.assignmentId, '_assignment-1'); + assert.equal( + commandFailureEvent(undefined, { ...event, workerId: 'vm\n' }).workerId, + undefined, + ); +}); + +test('diagnostic extraction cannot invoke exception getters or mask a failure', () => { + const error = { + get code() { + throw new Error('private getter'); + }, + }; + assert.deepEqual( + commandFailureDiagnostic( + 'credential_resolution', + 'credential_failed', + error, + ), + { + stage: 'credential_resolution', + reason: 'credential_failed', + }, + ); + assert.deepEqual( + commandFailureDiagnostic( + 'credential_resolution', + 'credential_failed', + new Proxy( + {}, + { + getOwnPropertyDescriptor() { + throw new Error('private proxy'); + }, + }, + ), + ), + { + stage: 'credential_resolution', + reason: 'credential_failed', + }, + ); +}); + +test('CLI diagnostic lines are complete JSON records without a text prefix', () => { + const event = commandFailureEvent(undefined, { + assignmentId: 'assignment-1', + operation: 'execute_command', + mutationMayHaveCommitted: false, + requiresQuarantine: false, + }); + const line = commandFailureLogLine(event); + assert.equal(line.endsWith('\n'), true); + assert.equal(line.split('\n').length, 2); + assert.deepEqual(JSON.parse(line), JSON.parse(JSON.stringify(event))); +}); diff --git a/packages/code/src/command-diagnostics.ts b/packages/code/src/command-diagnostics.ts new file mode 100644 index 00000000..86b42c5b --- /dev/null +++ b/packages/code/src/command-diagnostics.ts @@ -0,0 +1,155 @@ +import { BRIDGE_WORKER_ID_PATTERN } from './protocol.js'; + +/** Worker-local diagnostics: never add these fields to bridge settlements. */ +export type CommandFailureStage = + | 'allocation' + | 'sandbox_startup' + | 'programmatic_preflight' + | 'credential_resolution' + | 'command_wrapping' + | 'dispatch' + | 'executor_transport' + | 'cleanup'; + +const reasons = [ + 'workspace_busy', + 'capacity_exhausted', + 'allocation_failed', + 'startup_failed', + 'prerequisite_failed', + 'credential_failed', + 'wrapping_failed', + 'executor_rejected', + 'executor_unavailable', + 'executor_lost', + 'send_failed', + 'rpc_timeout', + 'cleanup_failed', +] as const; +export type CommandFailureReason = (typeof reasons)[number]; + +const setupReasons = { + sandbox_startup: 'startup_failed', + programmatic_preflight: 'prerequisite_failed', + credential_resolution: 'credential_failed', + command_wrapping: 'wrapping_failed', +} as const; +export type CommandSetupStage = keyof typeof setupReasons; + +export function commandSetupFailureDiagnostic( + stage: CommandSetupStage, + error: unknown, +): CommandFailureDiagnostic { + return commandFailureDiagnostic(stage, setupReasons[stage], error); +} +export interface CommandFailureDiagnostic { + stage: CommandFailureStage; + reason: CommandFailureReason; + osCode?: string; +} +const stages = new Set([ + 'allocation', + 'sandbox_startup', + 'programmatic_preflight', + 'credential_resolution', + 'command_wrapping', + 'dispatch', + 'executor_transport', + 'cleanup', +]); +const osCodes = new Set([ + 'ENOSPC', + 'EDQUOT', + 'EACCES', + 'EPERM', + 'ENOMEM', + 'EMFILE', + 'ENFILE', + 'EIO', + 'ENOENT', + 'EPIPE', + 'ECONNRESET', + 'ETIMEDOUT', +]); + +export function commandFailureDiagnostic( + stage: CommandFailureStage, + reason: CommandFailureReason, + error?: unknown, +): CommandFailureDiagnostic { + let code: unknown; + try { + // Diagnostic extraction must not execute an exception object's getters. + if (error != null && typeof error === 'object') { + code = Object.getOwnPropertyDescriptor(error, 'code')?.value; + } + } catch { + // Even an unusual provider error/proxy cannot replace the original failure. + } + return { + stage, + reason, + ...(typeof code === 'string' && osCodes.has(code) + ? { osCode: code } + : {}), + }; +} + +/** Reconstruct only allowlisted fields, even across the trusted executor IPC. */ +export function readCommandFailureDiagnostic( + value: unknown, +): CommandFailureDiagnostic | undefined { + if (value == null || typeof value !== 'object') return; + const v = value as Partial; + if (!stages.has(v.stage ?? '') || !reasons.includes(v.reason!)) return; + return commandFailureDiagnostic(v.stage!, v.reason!, { code: v.osCode }); +} + +export interface CommandFailureEvent extends CommandFailureDiagnostic { + event: 'workspace_command_unavailable'; + assignmentId?: string; + workspaceId?: string; + workerId?: string; + incarnationId?: string; + operation: 'execute_command' | 'workspace_programmatic'; + mutationMayHaveCommitted: boolean; + requiresQuarantine: boolean; +} + +/** One complete JSON record, suitable for a JSON-lines log collector. */ +export function commandFailureLogLine(event: CommandFailureEvent): string { + return `${JSON.stringify(event)}\n`; +} + +export function commandFailureEvent( + diagnostic: CommandFailureDiagnostic | undefined, + context: Omit< + CommandFailureEvent, + keyof CommandFailureDiagnostic | 'event' + >, +): CommandFailureEvent { + const id = ( + value: string | undefined, + pattern: RegExp, + ): string | undefined => + value && + value.length <= 128 && + !/[\r\n]/.test(value) && + pattern.test(value) + ? value + : undefined; + // Assignment/incarnation IDs are opaque and may begin with '-' or '_'. + const opaqueIdPattern = /^[A-Za-z0-9._:-]{1,128}$/; + return { + event: 'workspace_command_unavailable', + ...(readCommandFailureDiagnostic(diagnostic) ?? + commandFailureDiagnostic('dispatch', 'executor_rejected')), + assignmentId: id(context.assignmentId, opaqueIdPattern), + workspaceId: id(context.workspaceId, BRIDGE_WORKER_ID_PATTERN), + workerId: id(context.workerId, BRIDGE_WORKER_ID_PATTERN), + incarnationId: id(context.incarnationId, opaqueIdPattern), + operation: context.operation, + mutationMayHaveCommitted: context.mutationMayHaveCommitted, + requiresQuarantine: context.requiresQuarantine, + }; +} diff --git a/packages/code/src/linked-worktrees.ts b/packages/code/src/linked-worktrees.ts index 225abebd..609bfbd8 100644 --- a/packages/code/src/linked-worktrees.ts +++ b/packages/code/src/linked-worktrees.ts @@ -18,7 +18,7 @@ import type { WorkspaceToolRequest, WorkspaceToolResult, } from './protocol.js'; -import type { WorkspaceToolExecutor } from './workspace.js'; +import type { WorkspaceFileImporter, WorkspaceToolExecutor } from './workspace.js'; /** Linked worktrees are only admitted from this directory beneath a checkout. */ export const LINKED_WORKTREE_DIRECTORY = '.worktrees'; @@ -60,6 +60,7 @@ export interface LinkedWorktreeSource { command?: NativeProcessSandboxOptions; repositoryInstructions: boolean; writable: boolean; + importer?: WorkspaceFileImporter; } export interface VerifiedLinkedWorktree { @@ -324,6 +325,7 @@ export class LinkedWorktreeWorkspaceTools implements WorkspaceToolExecutor, Link fingerprint, value: LocalWorkspaceTools.create({ repositoryInstructions: source.repositoryInstructions, + importer: source.importer, workspaces: [ { id: internalId, identity: lane.identity, root: lane.root, writable: source.writable }, ], diff --git a/packages/code/src/native-pool.test.ts b/packages/code/src/native-pool.test.ts index ce25caa4..2a339d59 100644 --- a/packages/code/src/native-pool.test.ts +++ b/packages/code/src/native-pool.test.ts @@ -14,6 +14,37 @@ const request = (workspaceId: string): WorkspaceExecuteCommandRequest => ({ command: 'fixture', }); +test('pool distinguishes same-root contention from exhausted machine capacity', async () => { + let release!: () => void; + let entered!: () => void; + const running = new Promise(resolve => { entered = resolve; }); + const finish = new Promise(resolve => { release = resolve; }); + const pool = new NativeWorkspaceCommandPool(roots, 1, () => ({ + async prepare() {}, async close() {}, + async execute(req) { + entered(); + await finish; + return { ...req, stdout: '', stderr: '', exitCode: 0, timedOut: false, truncated: false }; + }, + })); + const first = pool.execute(request('a')); + await running; + try { + for (const [root, reason] of [['a', 'workspace_busy'], ['b', 'capacity_exhausted']]) { + await assert.rejects(pool.execute(request(root!)), error => { + assert.ok(error instanceof WorkspaceToolError); + assert.deepEqual(error.commandDiagnostic, { stage: 'allocation', reason }); + assert.equal(error.mutationMayHaveCommitted, false); + return true; + }); + } + } finally { + release(); + await first; + await pool.close(); + } +}); + test('failed provisioning roots are removed only after confirmed executor cleanup', async () => { let failClose = true; const pool = new NativeWorkspaceCommandPool(roots, 2, () => ({ diff --git a/packages/code/src/native-pool.ts b/packages/code/src/native-pool.ts index 59db3e0a..60ba4b43 100644 --- a/packages/code/src/native-pool.ts +++ b/packages/code/src/native-pool.ts @@ -1,5 +1,6 @@ import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; import { WorkspaceToolError } from './workspace.js'; +import { commandFailureDiagnostic } from './command-diagnostics.js'; import type { NativeProcessSandboxOptions } from './native-process.js'; import type { BridgeWorkspaceProgrammaticRequest, @@ -122,7 +123,8 @@ export class NativeWorkspaceCommandPool { if (entry?.busy) throw new WorkspaceToolError( 'Native workspace already executing', - 'COMMAND_UNAVAILABLE' + 'COMMAND_UNAVAILABLE', false, false, + commandFailureDiagnostic('allocation', 'workspace_busy'), ); if (!entry) { if (this.entries.size >= this.capacity) { @@ -132,9 +134,18 @@ export class NativeWorkspaceCommandPool { if (!idle) throw new WorkspaceToolError( 'Native executor capacity reached', - 'COMMAND_UNAVAILABLE' + 'COMMAND_UNAVAILABLE', false, false, + commandFailureDiagnostic('allocation', 'capacity_exhausted'), ); - await idle[1].sandbox.close(); + try { + await idle[1].sandbox.close(); + } catch (error) { + const diagnostic = commandFailureDiagnostic('cleanup', 'cleanup_failed', error); + throw error instanceof WorkspaceToolError + ? new WorkspaceToolError(error.message, error.code, error.mutationMayHaveCommitted, + error.requiresQuarantine, error.commandDiagnostic ?? diagnostic) + : new WorkspaceToolError('Native executor allocation failed', 'COMMAND_UNAVAILABLE', false, false, diagnostic); + } this.entries.delete(idle[0]); } entry = { @@ -151,10 +162,15 @@ export class NativeWorkspaceCommandPool { const checked = pending.catch((error) => { // Allocation/idle eviction precedes dispatch into the requested root. // Do not turn a pool resource failure into an uncertain mutation there. - if (error instanceof WorkspaceToolError) throw error; + if (error instanceof WorkspaceToolError) { + if (error.commandDiagnostic) throw error; + throw new WorkspaceToolError(error.message, error.code, error.mutationMayHaveCommitted, + error.requiresQuarantine, commandFailureDiagnostic('allocation', 'allocation_failed', error)); + } throw new WorkspaceToolError( 'Native executor allocation failed', - 'COMMAND_UNAVAILABLE' + 'COMMAND_UNAVAILABLE', false, false, + commandFailureDiagnostic('allocation', 'allocation_failed', error), ); }); this.allocation = checked.catch(() => undefined); @@ -236,7 +252,8 @@ export class NativeWorkspaceCommandPool { if (!entry.sandbox.executeProgrammatic) { throw new WorkspaceToolError( 'Native programmatic executor is unavailable', - 'COMMAND_UNAVAILABLE' + 'COMMAND_UNAVAILABLE', false, false, + commandFailureDiagnostic('programmatic_preflight', 'prerequisite_failed'), ); } return await entry.sandbox.executeProgrammatic( diff --git a/packages/code/src/native-process-child.ts b/packages/code/src/native-process-child.ts index c3b9614b..e0e0d322 100644 --- a/packages/code/src/native-process-child.ts +++ b/packages/code/src/native-process-child.ts @@ -1,6 +1,7 @@ import { NativeSrtWorkspaceCommandSandbox } from './native-sandbox.js'; import { NativeWorkspaceProgrammaticExecutor } from './native-programmatic.js'; import { WorkspaceToolError } from './workspace.js'; +import { commandFailureDiagnostic } from './command-diagnostics.js'; import type { NativeSrtWorkspaceCommandSandboxOptions } from './native-sandbox.js'; import type { BridgeWorkspaceProgrammaticRequest, @@ -189,6 +190,14 @@ process.on('message', async (raw: unknown) => { reply({ id: message.id, ok: false, + diagnostic: + error instanceof WorkspaceToolError && error.commandDiagnostic + ? error.commandDiagnostic + : commandFailureDiagnostic( + message.type === 'prepare' ? 'sandbox_startup' : message.type === 'close' ? 'cleanup' : 'dispatch', + message.type === 'prepare' ? 'startup_failed' : message.type === 'close' ? 'cleanup_failed' : 'executor_rejected', + error, + ), code: error instanceof WorkspaceToolError ? error.code diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts index 8abf263c..afdad6f8 100644 --- a/packages/code/src/native-process.test.ts +++ b/packages/code/src/native-process.test.ts @@ -484,6 +484,7 @@ test('programmatic staging watchdog expires without claiming a workspace mutatio execution, (error: unknown) => error instanceof WorkspaceToolError && + error.commandDiagnostic?.reason === 'rpc_timeout' && !error.mutationMayHaveCommitted && !error.requiresQuarantine, ); @@ -876,3 +877,92 @@ test('executor classifies every pre-dispatch setup failure as mutation-atomic', await sandbox.close(); } }); + +test('executor diagnoses setup stages without dispatching or retaining private exception values', async () => { + for (const failure of ['fork', 'credential', 'wrapper'] as const) { + const fake = fixture(); + const privateError = Object.assign(new Error('secret /private/path command'), { code: 'ENOSPC' }); + const sandbox = new NativeProcessWorkspaceCommandSandbox({ + workspaceRoot: '/workspace', + maskedEnvironment: { + variables: [], + async resolve() { + if (failure === 'credential') throw privateError; + return {}; + }, + wrapCommand(command) { + if (failure === 'wrapper') throw privateError; + return command; + }, + }, + }, failure === 'fork' ? () => { throw privateError; } : fake.fork); + await assert.rejects(sandbox.execute(request), error => { + assert.ok(error instanceof WorkspaceToolError); + assert.equal(error.code, 'COMMAND_UNAVAILABLE'); + assert.equal(error.mutationMayHaveCommitted, false); + assert.equal(error.requiresQuarantine, false); + assert.deepEqual(error.commandDiagnostic, { + stage: failure === 'fork' ? 'sandbox_startup' : failure === 'credential' ? 'credential_resolution' : 'command_wrapping', + reason: failure === 'fork' ? 'startup_failed' : failure === 'credential' ? 'credential_failed' : 'wrapping_failed', + osCode: 'ENOSPC', + }); + assert.equal(JSON.stringify(error.commandDiagnostic).includes('secret'), false); + return true; + }); + assert.equal(fake.messages.some(m => m.type === 'execute'), false); + await sandbox.close(); + } +}); + +test('executor preserves sanitized child diagnostics and mutation uncertainty', async () => { + const fake = fixture((child, message) => child.emit('message', { + id: message.id, ok: false, mutation: true, + diagnostic: { stage: 'dispatch', reason: 'executor_rejected', osCode: 'EIO', secret: 'token' }, + })); + const sandbox = new NativeProcessWorkspaceCommandSandbox({ workspaceRoot: '/workspace' }, fake.fork); + await assert.rejects(sandbox.execute(request), error => { + assert.ok(error instanceof WorkspaceToolError); + assert.deepEqual(error.commandDiagnostic, { stage: 'dispatch', reason: 'executor_rejected', osCode: 'EIO' }); + assert.equal(error.mutationMayHaveCommitted, true); + assert.equal(error.requiresQuarantine, true); + return true; + }); + await sandbox.close(); +}); + +test('programmatic startup diagnoses child preparation failures before mutation', async () => { + const fake = fixture(undefined, (child, message) => child.emit('message', { + id: message.id, ok: false, + diagnostic: { stage: 'sandbox_startup', reason: 'startup_failed', osCode: 'ENOSPC' }, + })); + const sandbox = new NativeProcessWorkspaceCommandSandbox({ + workspaceRoot: '/workspace', programmaticFileUpstream: 'http://127.0.0.1:3190', + }, fake.fork); + await assert.rejects(sandbox.executeProgrammatic('primary', { + body: { language: 'bash', version: '5.2', session_id: 'session-1', files: [{ name: 'main.sh', content: 'echo ok' }] }, headers: {}, + }), error => { + assert.ok(error instanceof WorkspaceToolError); + assert.deepEqual(error.commandDiagnostic, { stage: 'sandbox_startup', reason: 'startup_failed', osCode: 'ENOSPC' }); + assert.equal(error.mutationMayHaveCommitted, false); + return true; + }); + assert.equal(fake.messages.some(m => m.type === 'programmatic'), false); + await sandbox.close(); +}); + +test('real executor IPC carries a startup errno without including the workspace path in diagnostics', async t => { + const directory = await mkdtemp(join(tmpdir(), 'command-diagnostic-')); + const sandbox = new NativeProcessWorkspaceCommandSandbox({ workspaceRoot: join(directory, 'missing-private-root') }); + t.after(async () => { + await sandbox.close(); + await rm(directory, { recursive: true, force: true }); + }); + await assert.rejects(sandbox.execute(request), error => { + assert.ok(error instanceof WorkspaceToolError); + assert.equal(error.code, 'COMMAND_UNAVAILABLE'); + assert.equal(error.mutationMayHaveCommitted, false); + assert.deepEqual(error.commandDiagnostic, { stage: 'sandbox_startup', reason: 'startup_failed', osCode: 'ENOENT' }); + assert.equal(JSON.stringify(error.commandDiagnostic).includes(directory), false); + return true; + }); +}); diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index 588f4c33..acb1c043 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -5,6 +5,12 @@ import { access, realpath } from 'node:fs/promises'; import { isAbsolute, join, relative, resolve, sep } from 'node:path'; import { promisify } from 'node:util'; import { WorkspaceToolError } from './workspace.js'; +import { + commandFailureDiagnostic, + commandSetupFailureDiagnostic, + readCommandFailureDiagnostic, +} from './command-diagnostics.js'; +import type { CommandFailureReason, CommandSetupStage } from './command-diagnostics.js'; import { NATIVE_PROGRAMMATIC_COMMAND } from './native-programmatic.js'; import { BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, @@ -171,11 +177,13 @@ export function nativeExecutorEnvironment( /** The executor process was lost, refused a send, or stalled past its * deadline, as opposed to a failure the executor reported explicitly. */ class NativeExecutorUnavailableError extends WorkspaceToolError { - constructor(mutation: boolean) { + constructor(mutation: boolean, reason: CommandFailureReason = 'executor_unavailable') { super( 'Native executor is unavailable', 'COMMAND_UNAVAILABLE', mutation, + mutation, + commandFailureDiagnostic('executor_transport', reason), ); this.name = 'NativeExecutorUnavailableError'; } @@ -227,8 +235,8 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan return this.ready; } - private unavailable(mutation: boolean): WorkspaceToolError { - return new NativeExecutorUnavailableError(mutation); + private unavailable(mutation: boolean, reason?: CommandFailureReason): WorkspaceToolError { + return new NativeExecutorUnavailableError(mutation, reason); } private async resolveProgrammaticExecutables(): Promise<{ @@ -271,6 +279,7 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan errorMessage?: unknown; fatal?: unknown; phase?: unknown; + diagnostic?: unknown; }; if ( !message || @@ -323,13 +332,14 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan code, mutationMayHaveCommitted, mutationMayHaveCommitted && !processTerminationConfirmed, + readCommandFailureDiagnostic(message.diagnostic) ?? commandFailureDiagnostic('dispatch', 'executor_rejected'), ), ); } }); const lost = () => { this.failed = true; - this.pending?.reject(this.unavailable(this.pending.mutation)); + this.pending?.reject(this.unavailable(this.pending.mutation, 'executor_lost')); }; child.on('error', lost); child.on('exit', lost); @@ -407,6 +417,8 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan throw new WorkspaceToolError( 'Native programmatic file transport is unavailable', 'COMMAND_UNAVAILABLE', + false, false, + commandFailureDiagnostic('programmatic_preflight', 'prerequisite_failed'), ); } const active = this.executeProgrammaticOnce( @@ -435,16 +447,20 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan let credentials: Record | undefined; let wrappedCommand: string | undefined; let programmaticExecutables: { shellPath: string; jqPath: string }; + let setupStage: CommandSetupStage = 'sandbox_startup'; try { await this.prepare(); if (signal?.aborted) throw new Error('aborted'); + setupStage = 'programmatic_preflight'; programmaticExecutables = await this.resolveProgrammaticExecutables(); if (signal?.aborted) throw new Error('aborted'); + setupStage = 'credential_resolution'; credentials = await this.options.maskedEnvironment?.resolve( signal, this.options.workspaceRoot, ); if (signal?.aborted) throw new Error('aborted'); + setupStage = 'command_wrapping'; wrappedCommand = this.options.maskedEnvironment?.wrapCommand?.( NATIVE_PROGRAMMATIC_COMMAND, process.platform, @@ -458,11 +474,13 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan 'EXECUTION_ABORTED', ); } + const diagnostic = commandSetupFailureDiagnostic(setupStage, error); throw error instanceof WorkspaceToolError - ? new WorkspaceToolError(error.message, error.code, false) + ? new WorkspaceToolError(error.message, error.code, false, false, error.commandDiagnostic ?? diagnostic) : new WorkspaceToolError( 'Native programmatic executor setup failed before dispatch', 'COMMAND_UNAVAILABLE', + false, false, diagnostic, ); } const result = await this.rpc( @@ -534,12 +552,15 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan ); let credentials: Record | undefined; let wrappedCommand: string | undefined; + let setupStage: CommandSetupStage = 'sandbox_startup'; try { await this.prepare(); if (signal?.aborted) throw new Error('aborted'); const cwd = resolve(this.options.workspaceRoot, request.cwd ?? '.'); + setupStage = 'credential_resolution'; credentials = await this.options.maskedEnvironment?.resolve(signal, cwd); if (signal?.aborted) throw new Error('aborted'); + setupStage = 'command_wrapping'; wrappedCommand = this.options.maskedEnvironment?.wrapCommand?.( request.command, process.platform, @@ -554,11 +575,13 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan 'Command aborted', 'EXECUTION_ABORTED', ); + const diagnostic = commandSetupFailureDiagnostic(setupStage, error); throw error instanceof WorkspaceToolError - ? new WorkspaceToolError(error.message, error.code, false) + ? new WorkspaceToolError(error.message, error.code, false, false, error.commandDiagnostic ?? diagnostic) : new WorkspaceToolError( 'Native executor setup failed before dispatch', 'COMMAND_UNAVAILABLE', + false, false, diagnostic, ); } const result = await this.rpc( @@ -610,8 +633,8 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan if (timer) clearTimeout(timer); timer = this.scheduleRpcTimeout(() => { this.failed = true; + reject(this.unavailable(this.pending?.mutation ?? mutation, 'rpc_timeout')); this.terminate(); - reject(this.unavailable(this.pending?.mutation ?? mutation)); }, timeoutMs); }; this.pending = { @@ -627,8 +650,8 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan signal?.addEventListener('abort', abort, { once: true }); const sendFailed = () => { this.failed = true; + reject(this.unavailable(mutation, 'send_failed')); this.terminate(); - reject(this.unavailable(mutation)); }; try { child.send({ type, id, ...payload }, error => { diff --git a/packages/code/src/native-programmatic.test.ts b/packages/code/src/native-programmatic.test.ts index 902c707a..076f141c 100644 --- a/packages/code/src/native-programmatic.test.ts +++ b/packages/code/src/native-programmatic.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert/strict'; import { createHash } from 'node:crypto'; import { createServer } from 'node:http'; -import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; +import { chmod, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import test from 'node:test'; @@ -389,6 +389,112 @@ test('preflights copy-on-write isolation and removes its private snapshot', asyn } }); +test('programmatic prerequisite failures retain their stage before executor dispatch', async t => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-diagnostic-')); + t.after(() => rm(scratch, { recursive: true, force: true })); + for (const failure of ['missing_probe', 'private_storage', 'isolation_probe'] as const) { + let executions = 0; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + sandbox: { + async createExecutionDirectory() { + if (failure === 'private_storage') throw Object.assign(new Error('secret path'), { code: 'ENOSPC' }); + return await mkdtemp(join(scratch, 'execution-')); + }, + ...(failure === 'missing_probe' ? {} : { + async createProgrammaticProbeWorkspace() { + throw new WorkspaceToolError('Probe isolation unavailable', 'COMMAND_UNAVAILABLE'); + }, + }), + async executeProgrammatic() { executions += 1; throw new Error('must not run'); }, + }, + }); + await assert.rejects(executor.prepare(), error => { + assert.ok(error instanceof WorkspaceToolError); + assert.equal(error.code, 'COMMAND_UNAVAILABLE'); + assert.equal(error.mutationMayHaveCommitted, false); + assert.equal(error.requiresQuarantine, false); + assert.deepEqual(error.commandDiagnostic, { + stage: 'programmatic_preflight', reason: 'prerequisite_failed', + ...(failure === 'private_storage' ? { osCode: 'ENOSPC' } : {}), + }); + assert.equal(JSON.stringify(error.commandDiagnostic).includes('secret'), false); + return true; + }); + assert.equal(executions, 0); + assert.deepEqual(await readdir(scratch), []); + } +}); + +for (const phase of ['preflight', 'probe', 'commit'] as const) { + test(`programmatic ${phase} cleanup failures retain cleanup diagnostics and mutation flags`, { + skip: process.platform === 'win32' || process.getuid?.() === 0, + }, async t => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-cleanup-diagnostic-')); + t.after(async () => { + await chmod(scratch, 0o700); + await rm(scratch, { recursive: true, force: true }); + }); + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + sandbox: { + async createExecutionDirectory() { return await mkdtemp(join(scratch, 'execution-')); }, + async createProgrammaticProbeWorkspace(directory) { + const workspace = join(directory, 'workspace'); + await mkdir(workspace); + if (phase === 'preflight') await chmod(scratch, 0o500); + return workspace; + }, + async executeProgrammatic(request) { + await chmod(scratch, 0o500); + if (phase === 'probe') throw new WorkspaceToolError('Probe failed', 'COMMAND_UNAVAILABLE'); + return { protocolVersion: 1, operation: 'execute_command', workspaceId: request.workspaceId, + stdout: '', stderr: '', exitCode: 0, timedOut: false, truncated: false }; + }, + }, + }); + const execution = phase === 'preflight' ? executor.prepare() : executor.execute({ + headers: {}, body: { language: 'bash', version: '5.2.0', session_id: 'session', + replay_tool_count: phase === 'probe' ? 1 : 0, files: [{ name: 'main.sh', content: 'true' }] }, + }, 'primary'); + await assert.rejects(execution, error => { + assert.ok(error instanceof WorkspaceToolError); + assert.equal(error.code, 'COMMAND_UNAVAILABLE'); + assert.equal(error.commandDiagnostic?.stage, 'cleanup'); + assert.equal(error.commandDiagnostic?.reason, 'cleanup_failed'); + assert.ok(['EACCES', 'EPERM'].includes(error.commandDiagnostic?.osCode ?? '')); + assert.equal(error.mutationMayHaveCommitted, phase === 'commit'); + assert.equal(error.requiresQuarantine, phase === 'commit'); + return true; + }); + }); +} + +test('programmatic commit normalization preserves the executor diagnostic', async t => { + const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-commit-diagnostic-')); + t.after(() => rm(scratch, { recursive: true, force: true })); + const diagnostic = { stage: 'executor_transport' as const, reason: 'executor_lost' as const }; + const executor = new NativeWorkspaceProgrammaticExecutor({ + upstreamUrl: 'http://127.0.0.1:1', + sandbox: { + async createExecutionDirectory() { return await mkdtemp(join(scratch, 'execution-')); }, + async executeProgrammatic() { + throw new WorkspaceToolError('Executor unavailable', 'COMMAND_UNAVAILABLE', false, false, diagnostic); + }, + }, + }); + await assert.rejects(executor.execute({ + headers: {}, body: { language: 'bash', version: '5.2.0', session_id: 'session', + files: [{ name: 'main.sh', content: 'true' }] }, + }, 'primary'), error => { + assert.ok(error instanceof WorkspaceToolError); + assert.equal(error.mutationMayHaveCommitted, true); + assert.equal(error.requiresQuarantine, true); + assert.deepEqual(error.commandDiagnostic, diagnostic); + return true; + }); +}); + test('keeps replay probes read-only and commits the script exactly once', async () => { const scratch = await mkdtemp(join(tmpdir(), 'native-ptc-probe-test-')); const phases: boolean[] = []; @@ -622,7 +728,8 @@ for (const failure of ['truncated', 'process-error']) test(`a failed speculative async createProgrammaticProbeWorkspace(directory) { const root = join(directory, 'workspace'); await mkdir(root); return root; }, async executeProgrammatic(request, _directory, _signal, options) { assert.equal(options?.probe, true); - if (failure === 'process-error') throw new WorkspaceToolError('probe output exceeded its limit', 'COMMAND_UNAVAILABLE', true, true); + if (failure === 'process-error') throw new WorkspaceToolError('probe output exceeded its limit', 'COMMAND_UNAVAILABLE', true, true, + { stage: 'executor_transport', reason: 'executor_lost' }); return { protocolVersion: 1, operation: 'execute_command', workspaceId: request.workspaceId, exitCode: 0, stdout: '', stderr: '', truncated: true, timedOut: false }; }, @@ -635,6 +742,8 @@ for (const failure of ['truncated', 'process-error']) test(`a failed speculative assert.match(String(error), /probe output exceeded/); assert.equal((error as { requiresQuarantine: boolean }).requiresQuarantine, false); assert.equal((error as { mutationMayHaveCommitted: boolean }).mutationMayHaveCommitted, false); + if (failure === 'process-error') assert.deepEqual((error as WorkspaceToolError).commandDiagnostic, + { stage: 'executor_transport', reason: 'executor_lost' }); return true; }); }); diff --git a/packages/code/src/native-programmatic.ts b/packages/code/src/native-programmatic.ts index 9b6eea9e..eca7316d 100644 --- a/packages/code/src/native-programmatic.ts +++ b/packages/code/src/native-programmatic.ts @@ -15,8 +15,13 @@ import { isSafePortableRelativePath, isSupportedBridgeArtifactName, } from './protocol.js'; -import { validateFileRelayUpstream } from './relay.js'; +import { + FILE_RELAY_GRANT_HEADER, + fileRelayObjectUrl, + validateFileRelayUpstream, +} from './relay.js'; import { WorkspaceToolError } from './workspace.js'; +import { commandFailureDiagnostic, commandSetupFailureDiagnostic } from './command-diagnostics.js'; import type { BridgeProgrammaticPayloadFile, @@ -25,7 +30,6 @@ import type { } from './protocol.js'; import type { NativeSrtWorkspaceCommandSandbox } from './native-sandbox.js'; -const EGRESS_GRANT_HEADER = 'X-CodeAPI-Egress-Grant'; const EXECUTION_MAIN_FILE = 'main.sh'; const EXECUTION_HISTORY_FILE = '_ptc_history.json'; const EXECUTION_CONTROL_FILE = '_ptc_pending_result.json'; @@ -248,6 +252,25 @@ export class NativeWorkspaceProgrammaticExecutor { * real replay will use, then removes the snapshot before registration. */ async prepare(signal?: AbortSignal): Promise { + try { + await this.prepareOnce(signal); + } catch (error) { + const diagnostic = commandSetupFailureDiagnostic( + 'programmatic_preflight', error, + ); + throw error instanceof WorkspaceToolError + ? new WorkspaceToolError( + error.message, error.code, error.mutationMayHaveCommitted, + error.requiresQuarantine, error.commandDiagnostic ?? diagnostic, + ) + : new WorkspaceToolError( + 'Native executor request failed', 'COMMAND_UNAVAILABLE', + false, false, diagnostic, + ); + } + } + + private async prepareOnce(signal?: AbortSignal): Promise { const createProbeWorkspace = this.options.sandbox.createProgrammaticProbeWorkspace; if (createProbeWorkspace == null) { @@ -265,7 +288,14 @@ export class NativeWorkspaceProgrammaticExecutor { signal, ); } finally { - await rm(executionDirectory, { recursive: true, force: true }); + try { + await rm(executionDirectory, { recursive: true, force: true }); + } catch (error) { + throw new WorkspaceToolError( + 'Native executor request failed', 'COMMAND_UNAVAILABLE', false, false, + commandFailureDiagnostic('cleanup', 'cleanup_failed', error), + ); + } } } @@ -327,12 +357,9 @@ export class NativeWorkspaceProgrammaticExecutor { const timer = setTimeout(() => controller.abort(), transferTimeoutMs); try { const response = await this.fetchImpl( - new URL( - `sessions/${encodeURIComponent(file.storage_session_id)}/objects/${encodeURIComponent(file.id)}`, - `${this.upstream.toString().replace(/\/+$/, '')}/`, - ), + fileRelayObjectUrl(this.upstream, file.storage_session_id, file.id), { - headers: { [EGRESS_GRANT_HEADER]: grant }, + headers: { [FILE_RELAY_GRANT_HEADER]: grant }, redirect: 'error', signal: controller.signal, }, @@ -704,14 +731,11 @@ export class NativeWorkspaceProgrammaticExecutor { let response: Response; try { response = await this.fetchImpl( - new URL( - `sessions/${encodeURIComponent(outputSessionId!)}/objects/${id}`, - `${this.upstream.toString().replace(/\/+$/, '')}/`, - ), + fileRelayObjectUrl(this.upstream, outputSessionId!, id), { method: 'PUT', headers: { - [EGRESS_GRANT_HEADER]: grant!, + [FILE_RELAY_GRANT_HEADER]: grant!, 'Content-Type': bridgeArtifactMediaType(name), 'Content-Length': String(bytes.byteLength), 'X-Original-Filename': encodeURIComponent(name), @@ -773,6 +797,9 @@ export class NativeWorkspaceProgrammaticExecutor { error instanceof WorkspaceToolError ? error.code : 'COMMAND_UNAVAILABLE', false, false, + error instanceof WorkspaceToolError && error.commandDiagnostic + ? error.commandDiagnostic + : commandSetupFailureDiagnostic('programmatic_preflight', error), ); } if (error instanceof WorkspaceToolError) { @@ -787,6 +814,7 @@ export class NativeWorkspaceProgrammaticExecutor { error.code, true, true, + error.commandDiagnostic, ); } throw new WorkspaceToolError( @@ -794,16 +822,18 @@ export class NativeWorkspaceProgrammaticExecutor { 'COMMAND_UNAVAILABLE', true, true, + commandFailureDiagnostic('dispatch', 'executor_rejected', error), ); } finally { try { await rm(executionDirectory, { recursive: true, force: true }); - } catch { + } catch (error) { throw new WorkspaceToolError( 'Native programmatic execution cleanup failed', 'COMMAND_UNAVAILABLE', commandDispatched, commandDispatched, + commandFailureDiagnostic('cleanup', 'cleanup_failed', error), ); } } diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index 4f0a7094..4b31658d 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -1739,7 +1739,9 @@ test('cleans allocated command state exactly once on every execution exit', asyn (outcome === 'abort-after-spawn') && error.requiresQuarantine === (outcome === 'abort-after-spawn' && - process.platform === 'win32'), + process.platform === 'win32') && + (outcome.startsWith('abort') || (error.commandDiagnostic?.stage === + (outcome === 'wrap-throw' ? 'command_wrapping' : 'dispatch'))), ); } assert.equal( diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 2701fe44..d994b1f9 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -15,6 +15,8 @@ import type { Dirent, Stats } from 'node:fs'; import { access, mkdtemp, open, readdir, realpath, rm, stat } from 'node:fs/promises'; import type { FileHandle } from 'node:fs/promises'; import { matchesWorkspaceRoot } from './root-identity.js'; +import { commandFailureDiagnostic, commandSetupFailureDiagnostic } from './command-diagnostics.js'; +import type { CommandSetupStage } from './command-diagnostics.js'; import { assertEnvironmentResourcesStable, environmentResourceVariables } from './environment-resources.js'; import type { WorkspaceRootIdentity } from './root-identity.js'; import { withWorkspaceRoot, WorkspaceRootAccessError, spawnWithinWorkspace, realpath as rootedRealpath, stat as rootedStat } from './root-access.js'; @@ -1071,9 +1073,10 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox // permissive proxy session through a live updateConfig. await this.manager.reset(); await this.manager.initialize(config, config.network.strictAllowlist ? undefined : async () => true); - } catch { + } catch (error) { this.resetFailed = true; - throw new WorkspaceToolError('Native probe network cleanup failed', 'COMMAND_UNAVAILABLE'); + throw new WorkspaceToolError('Native probe network cleanup failed', 'COMMAND_UNAVAILABLE', false, false, + commandFailureDiagnostic('cleanup', 'cleanup_failed', error)); } } } @@ -1175,10 +1178,13 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox let wrapped: Awaited< ReturnType >; + let setupStage: CommandSetupStage = 'command_wrapping'; try { const commandConfig = customConfig ?? (await this.rootCommandConfig()); + setupStage = 'credential_resolution'; const credentialEnvironment = await this.options.maskedEnvironment?.resolve(signal, cwd); + setupStage = 'command_wrapping'; const sandboxedCommand = this.options.maskedEnvironment?.wrapCommand ? this.options.maskedEnvironment.wrapCommand( request.command, @@ -1215,6 +1221,10 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox throw new WorkspaceToolError( 'Native sandbox command could not start', 'COMMAND_UNAVAILABLE', + false, false, + error instanceof WorkspaceToolError && error.commandDiagnostic + ? error.commandDiagnostic + : commandSetupFailureDiagnostic(setupStage, error), ); } try { @@ -1314,11 +1324,13 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox }, ); child.stdin.end(); - } catch { + } catch (error) { reject( new WorkspaceToolError( 'Native sandbox command could not start', 'COMMAND_UNAVAILABLE', + false, false, + commandFailureDiagnostic('dispatch', 'executor_rejected', error), ), ); return; @@ -1344,7 +1356,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox clearTimeout(timer); signal?.removeEventListener('abort', abort); }; - child.once('error', () => { + child.once('error', error => { if (settled) return; settled = true; const mayHaveStarted = child.pid != null; @@ -1355,6 +1367,8 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox 'Native sandbox command could not start', 'COMMAND_UNAVAILABLE', mayHaveStarted, + mayHaveStarted, + commandFailureDiagnostic('dispatch', 'executor_rejected', error), ), ); }); diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index 1451986d..0fcb6090 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -35,6 +35,12 @@ export function programmaticTransferReserveMs(jobTimeoutMs: number): number { export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES = 10 * 1024 * 1024; export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_HISTORY_BYTES = 40_000_000; export const BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_TOTAL_BYTES = 100 * 1024 * 1024; +/** Default per-file `import_file` ceiling; Code API and each worker may configure their own. */ +export const BRIDGE_WORKSPACE_IMPORT_FILE_DEFAULT_MAX_BYTES = 100 * 1024 * 1024; +/** Longest execution budget Code API grants one `import_file` transfer. */ +export const BRIDGE_WORKSPACE_IMPORT_FILE_TIMEOUT_MS = 5 * 60_000; +const BRIDGE_WORKSPACE_IMPORT_FILE_HANDLE_MAX_LENGTH = 32_768; +const BRIDGE_WORKSPACE_IMPORT_FILE_GRANT_MAX_LENGTH = 256 * 1024; /** How long Code API drains a clean rejection after Stop cancels a workspace mutation. */ export const BRIDGE_CANCELLED_WORKSPACE_SETTLEMENT_GRACE_MS = 5_000; @@ -313,7 +319,20 @@ export type BridgeWorkspaceToolOperation = | 'write_file' | 'preview_edit' | 'edit_file' - | 'execute_command'; + | 'execute_command' + | 'import_file'; + +/** Operations that may change workspace contents and therefore arm the durable mutation guard. */ +export function isWorkspaceMutationOperation( + operation: BridgeWorkspaceToolOperation, +): boolean { + return ( + operation === 'write_file' || + operation === 'edit_file' || + operation === 'execute_command' || + operation === 'import_file' + ); +} export type WorkspaceWriteFileMode = 'replace' | 'create'; export type WorkspaceEditFileMode = 'single' | 'batch'; @@ -674,6 +693,54 @@ export interface WorkspaceExecuteCommandResult { laneGit?: WorkspaceLaneGit; } +/** A stored object the caller owns, named exactly as Code API file storage names it. */ +export interface WorkspaceImportFileSource { + storage_session_id: string; + id: string; +} + +/** + * Copies one stored object into the workspace as a binary-safe file. The + * worker streams it through its file relay upstream, refuses `.git` paths and + * symlinked parents, verifies `bytes` and `sha256`, and installs it atomically + * with mode 0644. + */ +export interface WorkspaceImportFileRequest { + protocolVersion: BridgeProtocolVersion; + operation: 'import_file'; + workspaceId: string; + workspaceInstanceId?: string; + /** Linked worktree lane at `.worktrees/`; paths and cwd are relative to it. */ + worktree?: string; + path: string; + source: WorkspaceImportFileSource; + /** Exact object size, enforced as a hard cap while streaming. */ + bytes: number; + /** Lowercase hex SHA-256 of the object's bytes. */ + sha256: string; + /** False or omitted refuses to replace a file whose content differs. */ + overwrite?: boolean; + /** + * Set only by Code API when it relays the request to a worker: a short-lived + * read grant scoped to `source`, whose ids are then opaque grant handles. + * Callers never send it. + */ + egressGrant?: string; +} + +export interface WorkspaceImportFileResult { + protocolVersion: BridgeProtocolVersion; + operation: 'import_file'; + workspaceId: string; + path: string; + created: boolean; + /** True when the target already held these exact bytes; nothing was written. */ + unchanged?: boolean; + /** Bytes installed, or 0 when `unchanged`. */ + bytesWritten: number; + sha256: string; +} + export type WorkspaceToolRequest = | WorkspaceReadFileRequest | WorkspaceSearchTextRequest @@ -681,7 +748,8 @@ export type WorkspaceToolRequest = | WorkspaceWriteFileRequest | WorkspacePreviewEditRequest | WorkspaceEditFileRequest - | WorkspaceExecuteCommandRequest; + | WorkspaceExecuteCommandRequest + | WorkspaceImportFileRequest; export type WorkspaceToolResult = | WorkspaceReadFileResult | WorkspaceSearchTextResult @@ -689,7 +757,8 @@ export type WorkspaceToolResult = | WorkspaceWriteFileResult | WorkspacePreviewEditResult | WorkspaceEditFileResult - | WorkspaceExecuteCommandResult; + | WorkspaceExecuteCommandResult + | WorkspaceImportFileResult; const WORKSPACE_READ_REQUEST_KEYS = new Set([ 'instructionSha256', @@ -771,6 +840,30 @@ const WORKSPACE_COMMAND_REQUEST_KEYS = new Set([ 'timeoutMs', 'maxOutputBytes', ]); +const WORKSPACE_IMPORT_REQUEST_KEYS = new Set([ + 'protocolVersion', + 'operation', + 'workspaceId', + 'workspaceInstanceId', + 'worktree', + 'path', + 'source', + 'bytes', + 'sha256', + 'overwrite', + 'egressGrant', +]); +const WORKSPACE_IMPORT_SOURCE_KEYS = new Set(['storage_session_id', 'id']); +const WORKSPACE_IMPORT_RESULT_KEYS = new Set([ + 'protocolVersion', + 'operation', + 'workspaceId', + 'path', + 'created', + 'unchanged', + 'bytesWritten', + 'sha256', +]); const WORKSPACE_READ_RESULT_KEYS = new Set([ 'protocolVersion', 'operation', @@ -1275,6 +1368,94 @@ export function isSafePortableRelativePath(value: unknown): value is string { return value.split('/').every(segment => segment !== '..'); } +/** + * Whether a path segment names Git metadata on any host filesystem: case + * folding (macOS, Windows), ignorable code points (HFS+), trailing dots and + * spaces (Windows), NTFS stream syntax such as `.git::$INDEX_ALLOCATION`, and + * the NTFS short name all alias `.git`. + */ +function isDotGitPathSegment(segment: string): boolean { + const folded = segment + .split(':', 1)[0]! + .toLowerCase() + .replace(/\p{Default_Ignorable_Code_Point}/gu, '') + .replace(/[. ]+$/, ''); + return folded === '.git' || folded === 'git~1'; +} + +/** `write_file` path rules, plus no `.git` segment and a file-like final segment. */ +export function isWorkspaceImportFilePath(value: unknown): value is string { + if (!isSafePortableRelativePath(value)) return false; + const segments = value.split('/'); + const last = segments[segments.length - 1]; + return ( + last !== undefined && + last !== '' && + last !== '.' && + !segments.some(isDotGitPathSegment) + ); +} + +function isWorkspaceImportFileHandle(value: unknown): value is string { + return ( + typeof value === 'string' && + value.length > 0 && + value.length <= BRIDGE_WORKSPACE_IMPORT_FILE_HANDLE_MAX_LENGTH && + !/[\0\r\n]/.test(value) + ); +} + +function isWorkspaceImportFileRequestShape( + request: Record, +): boolean { + const source = request.source; + return ( + hasOnlyKeys(request, WORKSPACE_IMPORT_REQUEST_KEYS) && + isWorkspaceImportFilePath(request.path) && + typeof source === 'object' && + source !== null && + !Array.isArray(source) && + hasOnlyKeys(source as Record, WORKSPACE_IMPORT_SOURCE_KEYS) && + isWorkspaceImportFileHandle((source as Record).storage_session_id) && + isWorkspaceImportFileHandle((source as Record).id) && + Number.isSafeInteger(request.bytes) && + Number(request.bytes) >= 0 && + typeof request.sha256 === 'string' && + /^[a-f0-9]{64}$/.test(request.sha256) && + (request.overwrite === undefined || typeof request.overwrite === 'boolean') && + (request.egressGrant === undefined || + (typeof request.egressGrant === 'string' && + request.egressGrant.length > 0 && + request.egressGrant.length <= BRIDGE_WORKSPACE_IMPORT_FILE_GRANT_MAX_LENGTH && + !/[\0\r\n]/.test(request.egressGrant))) + ); +} + +/** + * The caller-facing `import_file` request LibreChat sends to Code API. A + * caller may never supply `egressGrant`; Code API mints it when relaying. + */ +export function isBridgeWorkspaceImportFileRequest( + value: unknown, +): value is WorkspaceImportFileRequest & { egressGrant?: never } { + return ( + isWorkspaceToolRequest(value) && + value.operation === 'import_file' && + value.egressGrant === undefined + ); +} + +/** The relayed `import_file` request a worker executes: it must carry Code API's grant. */ +export function isRelayedWorkspaceImportFileRequest( + value: unknown, +): value is WorkspaceImportFileRequest & { egressGrant: string } { + return ( + isWorkspaceToolRequest(value) && + value.operation === 'import_file' && + value.egressGrant !== undefined + ); +} + function normalizePortableRelativePath(value: string): string { return ( value @@ -1573,6 +1754,9 @@ export function isWorkspaceToolRequest( BRIDGE_WORKSPACE_COMMAND_MAX_OUTPUT_BYTES)) ); } + if (request.operation === 'import_file') { + return isWorkspaceImportFileRequestShape(request); + } return false; } @@ -1706,6 +1890,26 @@ export function isWorkspaceToolResult( ); } + if (request.operation === 'import_file') { + if ( + !hasOnlyKeys(result, WORKSPACE_IMPORT_RESULT_KEYS) || + result.path !== request.path || + result.sha256 !== request.sha256 || + typeof result.created !== 'boolean' || + (result.unchanged !== undefined && typeof result.unchanged !== 'boolean') || + !Number.isSafeInteger(result.bytesWritten) + ) { + return false; + } + if (result.unchanged === true) { + return result.created === false && result.bytesWritten === 0; + } + return ( + result.bytesWritten === request.bytes && + (request.overwrite === true || result.created === true) + ); + } + if (request.operation === 'edit_file') { const replacements = request.edits?.length ?? 1; return ( @@ -1808,7 +2012,7 @@ export function isValidBridgeWorkspaceToolCapabilities( capabilities.protocolVersion !== BRIDGE_PROTOCOL_VERSION || !Array.isArray(capabilities.operations) || capabilities.operations.length < 1 || - capabilities.operations.length > 7 || + capabilities.operations.length > 8 || !capabilities.operations.every( operation => operation === 'read_file' || @@ -1817,7 +2021,8 @@ export function isValidBridgeWorkspaceToolCapabilities( operation === 'write_file' || operation === 'preview_edit' || operation === 'edit_file' || - operation === 'execute_command', + operation === 'execute_command' || + operation === 'import_file', ) || new Set(capabilities.operations).size !== capabilities.operations.length || diff --git a/packages/code/src/relay.ts b/packages/code/src/relay.ts index 07aa8714..1cab6e65 100644 --- a/packages/code/src/relay.ts +++ b/packages/code/src/relay.ts @@ -34,6 +34,21 @@ const LOCAL_HTTP_HOSTS = new Set([ class RelayPayloadTooLargeError extends Error {} class UpstreamPayloadTooLargeError extends Error {} +/** Header carrying Code API's egress grant on every file relay object request. */ +export const FILE_RELAY_GRANT_HEADER = 'X-CodeAPI-Egress-Grant'; + +/** `/sessions//objects/`, the egress gateway's object route. */ +export function fileRelayObjectUrl( + upstream: URL, + storageSessionId: string, + objectId: string, +): URL { + return new URL( + `sessions/${encodeURIComponent(storageSessionId)}/objects/${encodeURIComponent(objectId)}`, + `${upstream.toString().replace(/\/+$/, '')}/`, + ); +} + export function validateFileRelayUpstream(value: string): URL { const upstream = new URL(value); if ( diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 8978d4cd..c2ae9924 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -7,6 +7,7 @@ import { bridgeWorkerPath, isBridgeWorkspaceProgrammaticRequest, isValidLinkedWorktreeName, + isWorkspaceMutationOperation, isWorkspaceToolResult, workspaceIsolationKey, workspaceIsolationKeysConflict, @@ -17,6 +18,8 @@ import { EndpointRuntimeSupervisor } from './runtime.js'; import { signBridgeRequest } from './identity.js'; import { recoverWorkerCredential } from './credential-recovery.js'; import { isWorkspaceToolRequest, WorkspaceToolError } from './workspace.js'; +import { commandFailureEvent } from './command-diagnostics.js'; +import type { CommandFailureEvent } from './command-diagnostics.js'; import type { WorkspaceCommandResultFeature, @@ -94,6 +97,8 @@ export interface BridgeWorkerOptions { credentialRefreshTransportTimeoutMs?: number; fetchImpl?: typeof fetch; onError?: (error: unknown) => void; + /** Local-only, allowlisted command failure metadata; never contains request data. */ + onCommandFailure?: (event: CommandFailureEvent) => void | Promise; onIdentityChange?: (identity: BridgeWorkerIdentity) => void | Promise; onRegistered?: ( registration: BridgeWorkerRegistrationResponse, @@ -564,10 +569,7 @@ export class BridgeWorker { } if ( options.capabilities.workspaceTools?.operations.some( - (operation) => - operation === 'write_file' || - operation === 'edit_file' || - operation === 'execute_command', + isWorkspaceMutationOperation, ) === true && options.workspaceMutationQuarantine == null && options.workspaceQuarantines == null && @@ -1885,10 +1887,9 @@ export class BridgeWorker { 'Workspace listing feature is not advertised', ); } - const isMutation = - workspaceRequest.operation === 'write_file' || - workspaceRequest.operation === 'edit_file' || - workspaceRequest.operation === 'execute_command'; + const isMutation = isWorkspaceMutationOperation( + workspaceRequest.operation, + ); if (isMutation) { this.mutationGuardArmed = true; try { @@ -2117,6 +2118,35 @@ export class BridgeWorker { result: payload, }; } catch (error) { + if ( + error instanceof WorkspaceToolError && + error.code === 'COMMAND_UNAVAILABLE' && + (assignment.executionKind === 'workspace_programmatic' || + (assignment.executionKind === 'workspace_tool' && + isWorkspaceToolRequest(assignment.request) && + assignment.request.operation === 'execute_command')) + ) { + try { + const event = commandFailureEvent(error.commandDiagnostic, { + assignmentId: assignment.assignmentId, + workerId: this.options.workerId, + incarnationId: this.incarnationId, + workspaceId: assignment.executionKind === 'workspace_programmatic' + ? assignment.workspaceId + : (isWorkspaceToolRequest(assignment.request) ? assignment.request.workspaceId : undefined), + operation: assignment.executionKind === 'workspace_programmatic' + ? 'workspace_programmatic' + : 'execute_command', + mutationMayHaveCommitted: error.mutationMayHaveCommitted, + requiresQuarantine: error.requiresQuarantine, + }); + void Promise.resolve(this.options.onCommandFailure?.(event)).catch(() => { + // A rejected asynchronous log sink must not interrupt settlement. + }); + } catch { + // Observability must never change settlement or quarantine decisions. + } + } if ( error instanceof BridgeWorkspaceQuarantinedError && !workspaceMutationArmed @@ -2566,9 +2596,7 @@ export class BridgeWorker { (assignment.executionKind === 'workspace_programmatic' || (assignment.executionKind === 'workspace_tool' && isWorkspaceToolRequest(assignment.request) && - (assignment.request.operation === 'write_file' || - assignment.request.operation === 'edit_file' || - assignment.request.operation === 'execute_command'))); + isWorkspaceMutationOperation(assignment.request.operation))); if (signal?.aborted === true) { if (assignment.runtimeSessionId != null || fulfilledWorkspaceMutation) { throw await this.quarantineWorkspace( diff --git a/packages/code/src/workspace-import.test.ts b/packages/code/src/workspace-import.test.ts new file mode 100644 index 00000000..d77e9173 --- /dev/null +++ b/packages/code/src/workspace-import.test.ts @@ -0,0 +1,606 @@ +import assert from 'node:assert/strict'; +import { createHash, randomBytes } from 'node:crypto'; +import { createServer } from 'node:http'; +import { chmod, lstat, mkdir, mkdtemp, readdir, readFile, realpath, rename, rm, stat, symlink, writeFile } from 'node:fs/promises'; +import { setTimeout as delay } from 'node:timers/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import type { IncomingMessage, ServerResponse } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import type { TestContext } from 'node:test'; + +import { + isBridgeWorkspaceImportFileRequest, + isRelayedWorkspaceImportFileRequest, + isValidBridgeWorkspaceToolCapabilities, + isWorkspaceImportFilePath, + isWorkspaceToolRequest, + isWorkspaceToolResult, +} from './protocol.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; +import { BridgeWorker } from './worker.js'; +import { createWorkspaceFileImporter, HttpWorkspaceFileImporter } from './workspace-import.js'; +import { LocalWorkspaceTools, WorkspaceToolError } from './workspace.js'; + +import type { WorkspaceImportFileRequest, WorkspaceToolRequest } from './protocol.js'; +import type { HttpWorkspaceFileImporterOptions } from './workspace-import.js'; + +const GRANT = 'ceg1.grant-for-one-object'; +const SESSION = 'session-handle'; +const OBJECT = 'object-handle'; +const incarnationId = 'incarnation-00000001'; + +const sha256 = (bytes: Uint8Array): string => createHash('sha256').update(bytes).digest('hex'); + +function importRequest( + bytes: Buffer, + overrides: Partial = {}, +): WorkspaceImportFileRequest { + return { + protocolVersion: 1, + operation: 'import_file', + workspaceId: 'primary', + path: 'uploads/archive.zip', + source: { storage_session_id: SESSION, id: OBJECT }, + bytes: bytes.byteLength, + sha256: sha256(bytes), + egressGrant: GRANT, + ...overrides, + }; +} + +interface EgressStub { + url: string; + hits: number; + body: Buffer; + /** Replace the default response (exact body, declared length, valid grant). */ + respond?: (req: IncomingMessage, res: ServerResponse) => void; +} + +/** A local egress gateway: serves one object to one grant, like the real object route. */ +async function egressStub(t: TestContext, body: Buffer): Promise { + const stub: EgressStub = { url: '', hits: 0, body }; + const server = createServer((req, res) => { + stub.hits += 1; + if (stub.respond) return stub.respond(req, res); + if ( + req.method !== 'GET' || + req.url !== `/v1/egress/sessions/${SESSION}/objects/${OBJECT}` || + req.headers['x-codeapi-egress-grant'] !== GRANT + ) { + res.writeHead(403).end(); + return; + } + res.writeHead(200, { 'content-length': String(stub.body.byteLength) }).end(stub.body); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + t.after(() => new Promise((resolve) => { + server.closeAllConnections(); + server.close(() => resolve()); + })); + stub.url = `http://127.0.0.1:${(server.address() as AddressInfo).port}/v1/egress`; + return stub; +} + +async function workspace( + t: TestContext, + stub: EgressStub, + options: { writable?: boolean; held?: boolean; importer?: Partial } = {}, +) { + const root = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-import-'))); + t.after(() => rm(root, { recursive: true, force: true })); + const tools = await LocalWorkspaceTools.create({ + importer: new HttpWorkspaceFileImporter({ upstreamUrl: stub.url, ...options.importer }), + workspaces: [{ + id: 'primary', + root, + writable: options.writable ?? true, + ...(options.held === false ? {} : { identity: await captureWorkspaceRootIdentity(root) }), + }], + }); + return { root, tools }; +} + +async function rejection(promise: Promise): Promise { + try { + await promise; + } catch (error) { + assert.ok(error instanceof WorkspaceToolError, String(error)); + return error; + } + assert.fail('expected a WorkspaceToolError'); +} + +async function temporaryFiles(directory: string): Promise { + return (await readdir(directory).catch(() => [] as string[])).filter((name) => name.endsWith('.tmp')); +} + +test('import requests follow write_file path rules and refuse Git metadata', () => { + const bytes = Buffer.from('payload'); + assert.equal(isWorkspaceToolRequest(importRequest(bytes)), true); + for (const path of [ + '../escape.bin', '/etc/passwd', 'C:/x.bin', 'a\\b.bin', 'dir/', 'dir/.', '.git', '.git/config', + 'src/.git/hooks/pre-commit', '.GIT/config', '.Git./x', 'a/.git /x', 'GIT~1/config', '.g\u200cit/config', + '.git::$INDEX_ALLOCATION/config', '.GIT:stream', '.git. ::$INDEX_ALLOCATION/hooks/x', 'git~1::$INDEX_ALLOCATION/x', + ]) { + assert.equal(isWorkspaceImportFilePath(path), false, path); + assert.equal(isWorkspaceToolRequest(importRequest(bytes, { path })), false, path); + } + for (const path of ['archive.zip', 'uploads/photo.png', '.gitignore', '.github/workflow.yml', 'docs/.gitkeep', 'notes/10:30 standup.md']) { + assert.equal(isWorkspaceImportFilePath(path), true, path); + } + for (const invalid of [ + { sha256: 'A'.repeat(64) }, + { sha256: 'a'.repeat(63) }, + { bytes: -1 }, + { bytes: 1.5 }, + { source: { storage_session_id: SESSION } }, + { source: { storage_session_id: SESSION, id: OBJECT, name: 'x' } }, + { source: { storage_session_id: 'a\nb', id: OBJECT } }, + { overwrite: 'yes' }, + { egressGrant: '' }, + ] as Array>) { + assert.equal(isWorkspaceToolRequest({ ...importRequest(bytes), ...invalid }), false, JSON.stringify(invalid)); + } + assert.equal(isWorkspaceToolRequest({ ...importRequest(bytes), content: 'x' }), false); + + const { egressGrant: _grant, ...callerRequest } = importRequest(bytes); + assert.equal(isBridgeWorkspaceImportFileRequest(callerRequest), true); + assert.equal(isBridgeWorkspaceImportFileRequest(importRequest(bytes)), false, 'callers never supply the grant'); + assert.equal(isRelayedWorkspaceImportFileRequest(importRequest(bytes)), true); + assert.equal(isRelayedWorkspaceImportFileRequest(callerRequest), false); +}); + +test('import results must echo the request and account for every byte', () => { + const bytes = Buffer.from('payload'); + const request = importRequest(bytes); + const result = { + protocolVersion: 1, + operation: 'import_file', + workspaceId: 'primary', + path: request.path, + created: true, + bytesWritten: bytes.byteLength, + sha256: request.sha256, + }; + assert.equal(isWorkspaceToolResult(request, result), true); + assert.equal(isWorkspaceToolResult(request, { ...result, created: false, unchanged: true, bytesWritten: 0 }), true); + assert.equal(isWorkspaceToolResult(request, { ...result, created: false, unchanged: true }), false); + assert.equal(isWorkspaceToolResult(request, { ...result, unchanged: true, bytesWritten: 0 }), false); + assert.equal(isWorkspaceToolResult(request, { ...result, created: false }), false, 'a create cannot replace'); + assert.equal(isWorkspaceToolResult({ ...request, overwrite: true }, { ...result, created: false }), true); + assert.equal(isWorkspaceToolResult(request, { ...result, bytesWritten: 3 }), false); + assert.equal(isWorkspaceToolResult(request, { ...result, sha256: 'b'.repeat(64) }), false); + assert.equal(isWorkspaceToolResult(request, { ...result, path: 'other.zip' }), false); + assert.equal(isWorkspaceToolResult(request, { ...result, content: 'x' }), false); +}); + +test('a worker imports by default whenever it can, and an explicit opt-out removes it', async (t) => { + const upstreamUrl = 'https://code.example.test/egress'; + const enabled = createWorkspaceFileImporter({ optOut: false, writable: true, upstreamUrl }); + assert.ok(enabled instanceof HttpWorkspaceFileImporter, 'on by default'); + for (const value of ['true', ' TRUE ', '']) { + assert.ok(createWorkspaceFileImporter({ optOut: false, enabled: value, writable: true, upstreamUrl })); + } + assert.equal(createWorkspaceFileImporter({ optOut: true, writable: true, upstreamUrl }), undefined, '--no-workspace-imports'); + assert.equal(createWorkspaceFileImporter({ optOut: false, enabled: 'false', writable: true, upstreamUrl }), undefined); + assert.equal(createWorkspaceFileImporter({ optOut: false, enabled: 'False', writable: true, upstreamUrl }), undefined); + assert.equal(createWorkspaceFileImporter({ optOut: false, writable: true }), undefined, 'no relay upstream'); + assert.equal(createWorkspaceFileImporter({ optOut: false, writable: true, upstreamUrl: ' ' }), undefined); + assert.equal(createWorkspaceFileImporter({ optOut: false, writable: false, upstreamUrl }), undefined, 'writes disabled'); + assert.throws(() => createWorkspaceFileImporter({ optOut: false, enabled: 'yes', writable: true, upstreamUrl }), /must be true or false/); + assert.equal(createWorkspaceFileImporter({ optOut: false, writable: true, upstreamUrl, maxBytes: 4096 })?.maxBytes, 4096); + + const root = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-import-'))); + t.after(() => rm(root, { recursive: true, force: true })); + const advertised = await LocalWorkspaceTools.create({ importer: enabled, workspaces: [{ id: 'primary', root, writable: true }] }); + assert.ok(advertised.capabilities.operations.includes('import_file')); + const optedOut = await LocalWorkspaceTools.create({ + importer: createWorkspaceFileImporter({ optOut: true, writable: true, upstreamUrl }), + workspaces: [{ id: 'primary', root, writable: true }], + }); + assert.equal(optedOut.capabilities.operations.includes('import_file'), false); +}); + +test('import_file is advertised only beside writes, on writable roots, with an importer', async (t) => { + const stub = await egressStub(t, Buffer.from('x')); + const writable = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-import-'))); + const readOnly = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-import-'))); + t.after(() => Promise.all([rm(writable, { recursive: true }), rm(readOnly, { recursive: true })])); + const importer = new HttpWorkspaceFileImporter({ upstreamUrl: stub.url }); + + const mixed = await LocalWorkspaceTools.create({ + importer, + workspaces: [{ id: 'primary', root: writable, writable: true }, { id: 'docs', root: readOnly }], + }); + assert.ok(mixed.capabilities.operations.includes('import_file')); + assert.ok(mixed.capabilities.workspaces[0]?.operations?.includes('import_file')); + assert.equal(mixed.capabilities.workspaces[1]?.operations?.includes('import_file'), false); + assert.equal(isValidBridgeWorkspaceToolCapabilities(mixed.capabilities), true); + const denied = await rejection(mixed.execute({ ...importRequest(Buffer.from('x')), workspaceId: 'docs' })); + assert.equal(denied.code, 'WRITE_DISABLED'); + assert.equal(stub.hits, 0); + + const allReadOnly = await LocalWorkspaceTools.create({ importer, workspaces: [{ id: 'docs', root: readOnly }] }); + assert.equal(allReadOnly.capabilities.operations.includes('import_file'), false); + const withoutRelay = await LocalWorkspaceTools.create({ workspaces: [{ id: 'primary', root: writable, writable: true }] }); + assert.equal(withoutRelay.capabilities.operations.includes('import_file'), false); + assert.equal((await rejection(withoutRelay.execute(importRequest(Buffer.from('x'))))).code, 'WRITE_UNAVAILABLE'); +}); + +test('imports binary bytes atomically into new 0755 parents as a 0644 file, whatever the umask', async (t) => { + const bytes = Buffer.concat([Buffer.from([0, 0xff, 0x50, 0x4b, 3, 4, 0]), randomBytes(256 * 1024)]); + const stub = await egressStub(t, bytes); + const { root, tools } = await workspace(t, stub); + const request = importRequest(bytes, { path: 'uploads/2026/archive.zip' }); + const previousUmask = process.umask(0o077); + t.after(() => process.umask(previousUmask)); + + const result = await tools.execute(request); + assert.deepEqual(result, { + protocolVersion: 1, + operation: 'import_file', + workspaceId: 'primary', + path: request.path, + created: true, + bytesWritten: bytes.byteLength, + sha256: request.sha256, + }); + assert.equal(isWorkspaceToolResult(request, result), true); + const target = join(root, 'uploads', '2026', 'archive.zip'); + assert.deepEqual(await readFile(target), bytes); + assert.equal((await stat(target)).mode & 0o777, 0o644); + assert.equal((await stat(join(root, 'uploads'))).mode & 0o777, 0o755); + assert.equal((await stat(join(root, 'uploads', '2026'))).mode & 0o777, 0o755); + assert.deepEqual(await temporaryFiles(join(root, 'uploads', '2026')), []); + assert.equal(stub.hits, 1); +}); + +test('a pathname-only root takes chat attachments into a folder it creates, as LibreChat sends them', async (t) => { + const bytes = randomBytes(8192); + const stub = await egressStub(t, bytes); + const { root, tools } = await workspace(t, stub, { held: false }); + assert.ok(tools.capabilities.workspaces[0]?.operations?.includes('import_file')); + const previousUmask = process.umask(0o077); + t.after(() => process.umask(previousUmask)); + + const ignore = await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: '.librechat/attachments/.gitignore', + content: '*\n', + overwrite: false, + }); + assert.equal((ignore as { created: boolean }).created, true); + const request = importRequest(bytes, { path: '.librechat/attachments/report.pdf', overwrite: false }); + assert.equal((await tools.execute(request) as { created: boolean }).created, true); + assert.deepEqual(await readFile(join(root, '.librechat', 'attachments', 'report.pdf')), bytes); + + const direct = importRequest(bytes, { path: 'uploads/2026/archive.zip' }); + assert.equal((await tools.execute(direct) as { created: boolean }).created, true); + assert.equal((await stat(join(root, 'uploads'))).mode & 0o777, 0o755); + assert.equal((await stat(join(root, 'uploads', '2026'))).mode & 0o777, 0o755); + assert.equal((await stat(join(root, 'uploads', '2026', 'archive.zip'))).mode & 0o777, 0o644); + assert.equal(stub.hits, 2); +}); + +test('a failed import on a pathname-only root removes the parents it created', async (t) => { + const stub = await egressStub(t, Buffer.from('tampered bytes')); + const { root, tools } = await workspace(t, stub, { held: false }); + + const mismatch = await rejection( + tools.execute(importRequest(Buffer.from('expected bytes'), { path: '.librechat/attachments/file.bin' })), + ); + assert.equal(mismatch.code, 'INVALID_REQUEST'); + assert.equal(mismatch.mutationMayHaveCommitted, false); + assert.deepEqual(await readdir(root), []); +}); + +test('re-importing identical bytes is idempotent and never downloads again', async (t) => { + const bytes = randomBytes(4096); + const stub = await egressStub(t, bytes); + const { root, tools } = await workspace(t, stub); + const request = importRequest(bytes, { path: 'photo.png' }); + await tools.execute(request); + const installed = await stat(join(root, 'photo.png')); + + const again = await tools.execute(request); + assert.deepEqual(again, { + protocolVersion: 1, + operation: 'import_file', + workspaceId: 'primary', + path: 'photo.png', + created: false, + unchanged: true, + bytesWritten: 0, + sha256: request.sha256, + }); + assert.equal(isWorkspaceToolResult(request, again), true); + assert.equal(stub.hits, 1); + assert.equal((await stat(join(root, 'photo.png'))).ino, installed.ino); +}); + +test('different existing content conflicts unless overwrite is explicit', async (t) => { + const bytes = Buffer.from('new report'); + const stub = await egressStub(t, bytes); + const { root, tools } = await workspace(t, stub); + const target = join(root, 'report.pdf'); + await writeFile(target, 'old report'); + await chmod(target, 0o755); + + const conflict = await rejection(tools.execute(importRequest(bytes, { path: 'report.pdf' }))); + assert.equal(conflict.code, 'EDIT_CONFLICT'); + assert.equal(conflict.mutationMayHaveCommitted, false); + assert.equal(await readFile(target, 'utf8'), 'old report'); + assert.equal(stub.hits, 0, 'a conflict is decided before any transfer'); + + const replaced = await tools.execute(importRequest(bytes, { path: 'report.pdf', overwrite: true })); + assert.equal((replaced as { created: boolean }).created, false); + assert.equal(await readFile(target, 'utf8'), 'new report'); + assert.equal((await stat(target)).mode & 0o777, 0o644, 'an import is never executable'); +}); + +test('a hash or size mismatch installs nothing and removes the parents it created', async (t) => { + const bytes = Buffer.from('expected bytes'); + const stub = await egressStub(t, Buffer.from('tampered bytes')); + const { root, tools } = await workspace(t, stub); + + const mismatch = await rejection(tools.execute(importRequest(bytes, { path: 'fresh/dir/file.bin' }))); + assert.equal(mismatch.code, 'INVALID_REQUEST'); + assert.equal(mismatch.mutationMayHaveCommitted, false); + assert.match(mismatch.message, /SHA-256/); + assert.deepEqual(await readdir(root), [], 'created parents are rolled back'); + + stub.respond = (_req, res) => { + res.writeHead(200); + res.write(bytes); + res.end(Buffer.from('and then some')); + }; + const oversize = await rejection(tools.execute(importRequest(bytes, { path: 'file.bin' }))); + assert.equal(oversize.code, 'INVALID_REQUEST'); + assert.match(oversize.message, /larger than its declared size/); + + stub.respond = (_req, res) => res.writeHead(200, { 'content-length': '3' }).end('abc'); + const declared = await rejection(tools.execute(importRequest(bytes, { path: 'file.bin' }))); + assert.equal(declared.code, 'INVALID_REQUEST'); + + stub.respond = (_req, res) => res.writeHead(200).end(bytes.subarray(0, 4)); + const short = await rejection(tools.execute(importRequest(bytes, { path: 'file.bin' }))); + assert.match(short.message, /smaller than its declared size/); + assert.deepEqual(await readdir(root), []); +}); + +test('the staged file is verified after it is synced, not only the bytes that arrived', async (t) => { + const bytes = randomBytes(64 * 1024); + const stub = await egressStub(t, bytes); + const { root } = await workspace(t, stub); + const relay = new HttpWorkspaceFileImporter({ upstreamUrl: stub.url }); + const tools = await LocalWorkspaceTools.create({ + importer: { + maxBytes: relay.maxBytes, + availableBytes: (directory) => relay.availableBytes(directory), + async open(request, signal) { + const chunks = await relay.open(request, signal); + return (async function* tamper() { + let first = true; + for await (const chunk of chunks) { + yield chunk; + if (!first) continue; + first = false; + // Another local writer rewrites already-staged bytes before the transfer finishes. + const [staged] = await temporaryFiles(root); + assert.ok(staged); + await writeFile(join(root, staged), Buffer.alloc(chunk.byteLength, 0x41), { flag: 'r+' }); + } + })(); + }, + }, + workspaces: [{ id: 'primary', root, writable: true, identity: await captureWorkspaceRootIdentity(root) }], + }); + const tampered = await rejection(tools.execute(importRequest(bytes, { path: 'payload.bin' }))); + assert.equal(tampered.code, 'INVALID_REQUEST'); + assert.match(tampered.message, /SHA-256/); + assert.deepEqual(await readdir(root), []); +}); + +test('rollback never removes a directory that replaced one the import created', async (t) => { + const bytes = randomBytes(32 * 1024); + const stub = await egressStub(t, Buffer.concat([bytes.subarray(0, 1024), Buffer.alloc(bytes.byteLength - 1024)])); + const { root } = await workspace(t, stub); + const relay = new HttpWorkspaceFileImporter({ upstreamUrl: stub.url }); + const tools = await LocalWorkspaceTools.create({ + importer: { + maxBytes: relay.maxBytes, + availableBytes: (directory) => relay.availableBytes(directory), + async open(request, signal) { + const chunks = await relay.open(request, signal); + return (async function* swap() { + for await (const chunk of chunks) yield chunk; + // Another local process moves the new parent aside and makes its own at that path. + await rename(join(root, 'incoming'), join(root, 'moved')); + await mkdir(join(root, 'incoming')); + })(); + }, + }, + workspaces: [{ id: 'primary', root, writable: true, identity: await captureWorkspaceRootIdentity(root) }], + }); + const failed = await rejection(tools.execute(importRequest(bytes, { path: 'incoming/file.bin' }))); + assert.equal(failed.code, 'INVALID_REQUEST'); + assert.equal(failed.mutationMayHaveCommitted, true, 'an unremovable created parent is reported, not hidden'); + assert.equal((await stat(join(root, 'incoming'))).isDirectory(), true); +}); + +test('the stall timeout covers waiting on the source, not slow consumer writes', async (t) => { + const bytes = randomBytes(8 * 1024); + const stub = await egressStub(t, bytes); + stub.respond = (_req, res) => { + res.writeHead(200, { 'content-length': String(bytes.byteLength) }); + res.write(bytes.subarray(0, 4096)); + setTimeout(() => res.end(bytes.subarray(4096)), 10); + }; + const importer = new HttpWorkspaceFileImporter({ upstreamUrl: stub.url, stallTimeoutMs: 50 }); + const received: Buffer[] = []; + for await (const chunk of await importer.open(importRequest(bytes) as WorkspaceImportFileRequest & { egressGrant: string })) { + received.push(Buffer.from(chunk)); + await delay(150); + } + assert.deepEqual(Buffer.concat(received), bytes); +}); + +test('file and free-space limits fail before anything is written', async (t) => { + const bytes = randomBytes(2048); + const stub = await egressStub(t, bytes); + const small = await workspace(t, stub, { importer: { maxBytes: 1024 } }); + const tooLarge = await rejection(small.tools.execute(importRequest(bytes, { path: 'big.bin' }))); + assert.equal(tooLarge.code, 'WRITE_LIMIT_EXCEEDED'); + assert.equal(stub.hits, 0); + + const full = await workspace(t, stub, { importer: { availableBytes: async () => 4096 } }); + const noSpace = await rejection(full.tools.execute(importRequest(bytes, { path: 'big.bin' }))); + assert.equal(noSpace.code, 'WRITE_LIMIT_EXCEEDED'); + assert.match(noSpace.message, /free space/); + assert.equal(stub.hits, 0); + assert.deepEqual(await readdir(full.root), []); +}); + +test('imports refuse symlinked parents, Git metadata, and requests without a grant', async (t) => { + const bytes = Buffer.from('payload'); + const stub = await egressStub(t, bytes); + const { root, tools } = await workspace(t, stub); + const outside = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-import-outside-'))); + t.after(() => rm(outside, { recursive: true, force: true })); + await mkdir(join(root, 'real')); + await symlink(join(root, 'real'), join(root, 'alias')); + await symlink(outside, join(root, 'escape')); + await mkdir(join(root, '.git')); + + for (const path of ['alias/file.bin', 'escape/file.bin']) { + const refused = await rejection(tools.execute(importRequest(bytes, { path }))); + assert.equal(refused.code, 'INVALID_PATH', path); + } + assert.deepEqual(await readdir(join(root, 'real')), []); + assert.deepEqual(await readdir(outside), []); + + const git = await rejection(tools.execute(importRequest(bytes, { path: '.git/hooks/post-checkout' }))); + assert.equal(git.code, 'INVALID_REQUEST'); + assert.deepEqual(await readdir(join(root, '.git')), []); + + await symlink(join(root, 'real', 'target.bin'), join(root, 'link.bin')); + const linkTarget = await rejection(tools.execute(importRequest(bytes, { path: 'link.bin', overwrite: true }))); + assert.equal(linkTarget.code, 'INVALID_PATH'); + assert.equal((await lstat(join(root, 'link.bin'))).isSymbolicLink(), true); + + const { egressGrant: _grant, ...ungranted } = importRequest(bytes, { path: 'file.bin' }); + assert.equal((await rejection(tools.execute(ungranted))).code, 'INVALID_REQUEST'); + assert.equal(stub.hits, 0); +}); + +test('an unavailable or refusing relay fails cleanly as WRITE_UNAVAILABLE', async (t) => { + const bytes = Buffer.from('payload'); + const stub = await egressStub(t, bytes); + const { root, tools } = await workspace(t, stub); + const refused = await rejection(tools.execute(importRequest(bytes, { egressGrant: 'ceg1.another-object' }))); + assert.equal(refused.code, 'WRITE_UNAVAILABLE'); + assert.equal(refused.mutationMayHaveCommitted, false); + assert.deepEqual(await readdir(root), []); + + stub.respond = (_req, res) => { + res.writeHead(200, { 'content-length': String(bytes.byteLength) }); + res.write(bytes.subarray(0, 2)); + }; + const stalled = await workspace(t, stub, { importer: { stallTimeoutMs: 50 } }); + const timedOut = await rejection(stalled.tools.execute(importRequest(bytes))); + assert.equal(timedOut.code, 'WRITE_UNAVAILABLE'); + assert.match(timedOut.message, /stalled/); + assert.deepEqual(await readdir(stalled.root), []); +}); + +test('a worker negotiates import_file and executes it as a guarded mutation', async (t) => { + const bytes = randomBytes(8192); + const stub = await egressStub(t, bytes); + const { root, tools } = await workspace(t, stub); + const lifecycle: string[] = []; + const registrations: string[][] = []; + const settlements: Array> = []; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: false, sandboxProfile: 'anthropic-srt', runtimes: [], workspaceTools: tools.capabilities }, + workspaceTools: tools, + workspaceMutationQuarantine: { + async assertAvailable() {}, + async arm() { lifecycle.push('arm'); }, + async clear() { lifecycle.push('clear'); }, + async quarantine() { lifecycle.push('quarantine'); }, + }, + fetchImpl: async (input, init) => { + const url = String(input); + const body = (init?.body == null ? {} : JSON.parse(String(init.body))) as Record; + if (url.endsWith('/register')) { + registrations.push((body.capabilities as { workspaceTools: { operations: string[] } }).workspaceTools.operations); + return Response.json({ + protocolVersion: 1, workerId: 'vm-1', incarnationId, registeredAt: new Date().toISOString(), leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: ['read_file', 'search_text', 'list_files', 'write_file', 'preview_edit', 'edit_file', 'import_file'], + supportedWorkspaceWriteFileModes: ['replace', 'create'], + }); + } + if (!url.endsWith('/settle')) return Response.json({ protocolVersion: 1, cancelled: false }); + lifecycle.push('settle'); + settlements.push(body); + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + await worker.register(); + assert.ok(registrations.at(-1)?.includes('import_file')); + + const request: WorkspaceToolRequest = importRequest(bytes, { path: 'chat/upload.bin' }); + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-import', + workerId: 'vm-1', + incarnationId, + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 10_000).toISOString(), + remainingMs: 10_000, + executionKind: 'workspace_tool', + request, + }); + assert.deepEqual(lifecycle, ['arm', 'settle', 'clear']); + assert.equal(settlements[0]?.status, 'fulfilled'); + assert.deepEqual(await readFile(join(root, 'chat', 'upload.bin')), bytes); +}); + +test('a worker never advertises import_file to a Code API that cannot relay it', async (t) => { + const stub = await egressStub(t, Buffer.from('x')); + const { tools } = await workspace(t, stub); + const registrations: string[][] = []; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: false, sandboxProfile: 'anthropic-srt', runtimes: [], workspaceTools: tools.capabilities }, + workspaceTools: tools, + workspaceMutationQuarantine: { async assertAvailable() {}, async arm() {}, async clear() {}, async quarantine() {} }, + fetchImpl: async (_input, init) => { + const body = JSON.parse(String(init?.body)) as { capabilities: { workspaceTools: { operations: string[] } } }; + registrations.push(body.capabilities.workspaceTools.operations); + return Response.json({ + protocolVersion: 1, workerId: 'vm-1', incarnationId, registeredAt: new Date().toISOString(), leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: ['read_file', 'search_text', 'list_files', 'write_file', 'preview_edit', 'edit_file'], + supportedWorkspaceWriteFileModes: ['replace', 'create'], + }); + }, + }); + await worker.register(); + assert.ok(registrations.length > 0); + for (const operations of registrations) assert.equal(operations.includes('import_file'), false); +}); diff --git a/packages/code/src/workspace-import.ts b/packages/code/src/workspace-import.ts new file mode 100644 index 00000000..e4c927d3 --- /dev/null +++ b/packages/code/src/workspace-import.ts @@ -0,0 +1,188 @@ +import { statfs } from 'node:fs/promises'; + +import { + BRIDGE_WORKSPACE_IMPORT_FILE_DEFAULT_MAX_BYTES, + BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_TIMEOUT_MS, +} from './protocol.js'; +import { + FILE_RELAY_GRANT_HEADER, + fileRelayObjectUrl, + validateFileRelayUpstream, +} from './relay.js'; +import { WorkspaceToolError } from './workspace.js'; + +import type { WorkspaceImportFileRequest } from './protocol.js'; +import type { WorkspaceFileImporter } from './workspace.js'; + +export interface HttpWorkspaceFileImporterOptions { + /** The egress gateway this worker already uses for programmatic inputs. */ + upstreamUrl: string; + maxBytes?: number; + /** Longest wait for response headers or for the next body chunk. */ + stallTimeoutMs?: number; + fetchImpl?: typeof fetch; + availableBytes?: (directory: string) => Promise; +} + +/** + * The importer a worker runs with, if any. `import_file` is on by default + * whenever the worker can perform it: writes enabled and a relay upstream to + * reach stored objects. `--no-workspace-imports` or + * `LIBRECHAT_CODE_ALLOW_WORKSPACE_IMPORTS=false` opts out; any other value of + * that variable fails startup rather than silently choosing either way. + */ +export function createWorkspaceFileImporter(input: { + optOut: boolean; + enabled?: string; + writable: boolean; + upstreamUrl?: string; + maxBytes?: number; +}): HttpWorkspaceFileImporter | undefined { + const enabled = input.enabled?.trim().toLowerCase() ?? ''; + if (enabled !== '' && enabled !== 'true' && enabled !== 'false') { + throw new Error('LIBRECHAT_CODE_ALLOW_WORKSPACE_IMPORTS must be true or false'); + } + const upstreamUrl = input.upstreamUrl?.trim() ?? ''; + if (input.optOut || enabled === 'false' || !input.writable || upstreamUrl === '') { + return undefined; + } + return new HttpWorkspaceFileImporter({ upstreamUrl, maxBytes: input.maxBytes }); +} + +async function filesystemAvailableBytes(directory: string): Promise { + const stats = await statfs(directory); + return Number(stats.bavail) * Number(stats.bsize); +} + +function importAborted(): WorkspaceToolError { + return new WorkspaceToolError('Workspace import aborted', 'EXECUTION_ABORTED'); +} + +function importUnavailable(message: string): WorkspaceToolError { + return new WorkspaceToolError(message, 'WRITE_UNAVAILABLE'); +} + +/** + * Streams one granted object from the file relay upstream. The worker holds no + * storage credential: Code API's grant authorizes exactly this object, briefly. + */ +export class HttpWorkspaceFileImporter implements WorkspaceFileImporter { + readonly maxBytes: number; + private readonly upstream: URL; + private readonly stallTimeoutMs: number; + private readonly fetchImpl: typeof fetch; + private readonly freeBytes: (directory: string) => Promise; + + constructor(options: HttpWorkspaceFileImporterOptions) { + this.upstream = validateFileRelayUpstream(options.upstreamUrl); + this.maxBytes = options.maxBytes ?? BRIDGE_WORKSPACE_IMPORT_FILE_DEFAULT_MAX_BYTES; + this.stallTimeoutMs = options.stallTimeoutMs ?? BRIDGE_WORKSPACE_PROGRAMMATIC_TRANSFER_TIMEOUT_MS; + if (!Number.isSafeInteger(this.maxBytes) || this.maxBytes < 1) { + throw new RangeError('Workspace import file limit must be a positive safe integer'); + } + if (!Number.isSafeInteger(this.stallTimeoutMs) || this.stallTimeoutMs < 1) { + throw new RangeError('Workspace import stall timeout must be a positive safe integer'); + } + this.fetchImpl = options.fetchImpl ?? fetch; + this.freeBytes = options.availableBytes ?? filesystemAvailableBytes; + } + + availableBytes(directory: string): Promise { + return this.freeBytes(directory); + } + + async open( + request: WorkspaceImportFileRequest & { egressGrant: string }, + signal?: AbortSignal, + ): Promise> { + if (signal?.aborted) throw importAborted(); + const controller = new AbortController(); + const abort = (): void => controller.abort(signal?.reason); + signal?.addEventListener('abort', abort, { once: true }); + let stalled = false; + let timer: ReturnType | undefined; + const arm = (): void => { + clearTimeout(timer); + timer = setTimeout(() => { + stalled = true; + controller.abort(); + }, this.stallTimeoutMs); + }; + const release = (): void => { + clearTimeout(timer); + signal?.removeEventListener('abort', abort); + }; + const failure = (): WorkspaceToolError => + signal?.aborted + ? importAborted() + : importUnavailable(stalled ? 'Import source transfer stalled' : 'Import source download failed'); + + arm(); + let response: Response; + try { + response = await this.fetchImpl( + fileRelayObjectUrl(this.upstream, request.source.storage_session_id, request.source.id), + { + headers: { + [FILE_RELAY_GRANT_HEADER]: request.egressGrant, + 'Accept-Encoding': 'identity', + }, + redirect: 'error', + signal: controller.signal, + }, + ); + } catch { + release(); + throw failure(); + } + // A compressed transfer declares its encoded length; only the decoded bytes are checked then. + const encoding = response.headers.get('content-encoding')?.trim().toLowerCase(); + const declaredLength = encoding == null || encoding === 'identity' + ? response.headers.get('content-length') + : null; + let rejection: WorkspaceToolError | undefined; + if (!response.ok) { + rejection = importUnavailable(`Import source download failed with HTTP ${response.status}`); + } else if (declaredLength != null && declaredLength !== String(request.bytes)) { + rejection = new WorkspaceToolError( + 'Import source size does not match the declared size', + 'INVALID_REQUEST', + ); + } else if (response.body == null && request.bytes > 0) { + rejection = importUnavailable('Import source returned no content'); + } + if (rejection != null) { + release(); + await response.body?.cancel().catch(() => undefined); + throw rejection; + } + const body = response.body; + async function* chunks(): AsyncGenerator { + if (body == null) { + release(); + return; + } + const reader = body.getReader(); + try { + for (;;) { + let next: Awaited>; + // The stall timer covers only waiting on the source, never the consumer's writes. + arm(); + try { + next = await reader.read(); + } catch { + throw failure(); + } finally { + clearTimeout(timer); + } + if (next.done) return; + yield next.value; + } + } finally { + release(); + await reader.cancel().catch(() => undefined); + } + } + return chunks(); + } +} diff --git a/packages/code/src/workspace-instances.ts b/packages/code/src/workspace-instances.ts index 68e76a10..7927152b 100644 --- a/packages/code/src/workspace-instances.ts +++ b/packages/code/src/workspace-instances.ts @@ -12,12 +12,13 @@ import type { WorkspaceToolRequest, WorkspaceToolResult, } from './protocol.js'; -import type { WorkspaceToolExecutor } from './workspace.js'; +import type { WorkspaceFileImporter, WorkspaceToolExecutor } from './workspace.js'; interface WorkspaceInstanceSource { command?: NativeProcessSandboxOptions; repositoryInstructions: boolean; writable: boolean; + importer?: WorkspaceFileImporter; } export interface GitWorktreeWorkspaceToolsOptions { @@ -119,6 +120,7 @@ export class GitWorktreeWorkspaceTools implements WorkspaceToolExecutor { identity: instance.identity, value: LocalWorkspaceTools.create({ repositoryInstructions: source.repositoryInstructions, + importer: source.importer, workspaces: [ { id: internalId, diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index 99fcdd02..46d117bd 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -8,6 +8,61 @@ import { SandboxWorkspaceTools, WorkspaceToolError } from './workspace.js'; const incarnationId = 'incarnation-00000001'; +test('worker logs command failures locally without changing bridge settlements', async () => { + for (const sink of ['sync', 'async', 'pending'] as const) { + for (const operation of ['execute_command', 'list_files'] as const) { + const events: unknown[] = []; + let settlement: Record | undefined; + const capabilities = { + protocolVersion: 1 as const, + operations: ['execute_command' as const, 'list_files' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: false, sandboxProfile: 'anthropic-srt', runtimes: [], workspaceTools: capabilities }, + workspaceTools: { + capabilities, mutationFailuresAreAtomic: true, + async execute() { + throw new WorkspaceToolError('private error', 'COMMAND_UNAVAILABLE', false, false, + { stage: 'credential_resolution', reason: 'credential_failed' }); + }, + }, + workspaceQuarantines: new Map([['primary', mutationQuarantine()]]), + onCommandFailure(event) { + events.push(event); + if (sink === 'sync') throw new Error('private log sink error'); + if (sink === 'async') return Promise.reject(new Error('private log sink error')); + return new Promise(() => {}); + }, + fetchImpl: async (_input, init) => { + settlement = JSON.parse(String(init?.body)); + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + await worker.executeAndSettle({ + protocolVersion: 1, assignmentId: 'assignment-1', workerId: 'vm-1', incarnationId, generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', expiresAt: new Date(Date.now() + 5000).toISOString(), + executionKind: 'workspace_tool', + request: operation === 'execute_command' + ? { protocolVersion: 1, operation, workspaceId: 'primary', command: 'echo secret' } + : { protocolVersion: 1, operation, workspaceId: 'primary', path: '.' }, + }); + assert.equal(events.length, operation === 'execute_command' ? 1 : 0); + if (events.length) assert.deepEqual(events[0], { + event: 'workspace_command_unavailable', stage: 'credential_resolution', reason: 'credential_failed', + assignmentId: 'assignment-1', workerId: 'vm-1', incarnationId, workspaceId: 'primary', + operation: 'execute_command', mutationMayHaveCommitted: false, requiresQuarantine: false, + }); + assert.equal(settlement?.status, 'rejected'); + assert.equal(settlement?.errorCode, 'COMMAND_UNAVAILABLE'); + assert.equal('diagnostic' in settlement!, false); + assert.equal(JSON.stringify(events).includes('secret'), false); + } + } +}); + test('instance advertisement requires a guard resolver even for reads and with base guards', () => { for (const operation of ['read_file', 'write_file'] as const) { const workspaceTools = { protocolVersion: 1 as const, operations: [operation], workspaces: [{ id: 'primary', workspaceInstances: ['git_worktree'] as ['git_worktree'] }] }; @@ -1308,6 +1363,7 @@ test('worker executes programmatic Bash in the selected workspace and preserves test('worker keeps a selected workspace usable after an atomic programmatic setup failure', async () => { const lifecycle: string[] = []; + const diagnostics: unknown[] = []; let settlement: Record | undefined; const workspaceCapabilities = { protocolVersion: 1 as const, @@ -1343,6 +1399,10 @@ test('worker keeps a selected workspace usable after an atomic programmatic setu ); }, }, + onCommandFailure(event) { + diagnostics.push(event); + throw new Error('logging sink failed'); + }, workspaceQuarantines: new Map([ [ 'primary', @@ -1383,6 +1443,14 @@ test('worker keeps a selected workspace usable after an atomic programmatic setu assert.deepEqual(lifecycle, ['arm', 'clear']); assert.equal(settlement?.status, 'rejected'); assert.equal(settlement?.errorCode, 'COMMAND_UNAVAILABLE'); + assert.deepEqual(diagnostics, [{ + event: 'workspace_command_unavailable', stage: 'dispatch', reason: 'executor_rejected', + assignmentId: 'assignment-programmatic-setup-failure', workerId: 'vm-1', incarnationId, + workspaceId: 'primary', operation: 'workspace_programmatic', + mutationMayHaveCommitted: false, requiresQuarantine: false, + }]); + assert.equal('commandDiagnostic' in settlement!, false); + assert.equal('diagnostic' in settlement!, false); }); test('worker keeps a selected workspace usable after confirmed programmatic cancellation cleanup', async () => { diff --git a/packages/code/src/workspace.test.ts b/packages/code/src/workspace.test.ts index 787fbe7d..266defee 100644 --- a/packages/code/src/workspace.test.ts +++ b/packages/code/src/workspace.test.ts @@ -1812,55 +1812,67 @@ test('writes create missing parent directories inside a held workspace root', as assert.equal((await stat(join(root, 'src', 'feature', 'deep'))).isDirectory(), true); }); -test('parent creation never follows symlinks, crosses files, or leaves the workspace', async (t) => { - const parent = await realpath( - await mkdtemp(join(tmpdir(), 'librechat-code-workspace-parent-')), - ); - t.after(() => rm(parent, { recursive: true, force: true })); - const root = join(parent, 'root'); - const outside = join(parent, 'outside'); - await mkdir(join(root, 'src'), { recursive: true }); - await mkdir(outside); - await writeFile(join(root, 'file.txt'), 'file'); - await symlink(outside, join(root, 'linked-outside')); - await symlink(join(root, 'src'), join(root, 'alias')); - await symlink(join(parent, 'missing-target'), join(root, 'dangling')); - const tools = await heldRootTools(root); +/** A root registered by path alone, as `--worker-dir`, `--workspace` and `--environment` do. */ +async function pathnameRootTools(root: string): Promise { + return LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); +} - for (const [path, code] of [ - ['linked-outside/new/notes.txt', 'INVALID_PATH'], - ['alias/new/notes.txt', 'INVALID_PATH'], - ['dangling/new/notes.txt', 'INVALID_PATH'], - ['file.txt/new/notes.txt', 'INVALID_PATH'], - ['../outside/new/notes.txt', 'INVALID_REQUEST'], - ['new/../../outside/notes.txt', 'INVALID_REQUEST'], - ] as const) { - await assert.rejects( - tools.execute({ - protocolVersion: 1, - operation: 'write_file', - workspaceId: 'primary', - path, - content: 'blocked', - }), - (error: unknown) => - error instanceof WorkspaceToolError && - error.code === code && - !error.mutationMayHaveCommitted, - path, +for (const [kind, createTools] of [ + ['held', heldRootTools], + ['pathname-only', pathnameRootTools], +] as const) { + test(`parent creation never follows symlinks, crosses files, or leaves a ${kind} workspace`, async (t) => { + const parent = await realpath( + await mkdtemp(join(tmpdir(), 'librechat-code-workspace-parent-')), ); - } - assert.deepEqual(await readdir(outside), []); - assert.deepEqual(await readdir(join(root, 'src')), []); - assert.deepEqual((await readdir(root)).sort(), [ - 'alias', - 'dangling', - 'file.txt', - 'linked-outside', - 'src', - ]); - await assert.rejects(stat(join(parent, 'missing-target'))); -}); + t.after(() => rm(parent, { recursive: true, force: true })); + const root = join(parent, 'root'); + const outside = join(parent, 'outside'); + await mkdir(join(root, 'src'), { recursive: true }); + await mkdir(outside); + await writeFile(join(root, 'file.txt'), 'file'); + await symlink(outside, join(root, 'linked-outside')); + await symlink(join(root, 'src'), join(root, 'alias')); + await symlink(join(parent, 'missing-target'), join(root, 'dangling')); + const tools = await createTools(root); + + for (const [path, code] of [ + ['linked-outside/new/notes.txt', 'INVALID_PATH'], + ['alias/new/notes.txt', 'INVALID_PATH'], + ['dangling/new/notes.txt', 'INVALID_PATH'], + ['file.txt/new/notes.txt', 'INVALID_PATH'], + ['../outside/new/notes.txt', 'INVALID_REQUEST'], + ['new/../../outside/notes.txt', 'INVALID_REQUEST'], + ] as const) { + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path, + content: 'blocked', + }), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === code && + !error.mutationMayHaveCommitted, + path, + ); + } + assert.deepEqual(await readdir(outside), []); + assert.deepEqual(await readdir(join(root, 'src')), []); + assert.deepEqual((await readdir(root)).sort(), [ + 'alias', + 'dangling', + 'file.txt', + 'linked-outside', + 'src', + ]); + await assert.rejects(stat(join(parent, 'missing-target'))); + }); +} test('a write finishes once it has created parents, even if cancelled', async (t) => { const root = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-workspace-'))); @@ -1914,12 +1926,65 @@ test('a write cancelled before creating parents leaves the workspace unchanged', assert.deepEqual(await readdir(root), []); }); -test('a pathname-only workspace root reports a missing parent instead of creating it', async (t) => { - const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); +test('a pathname-only workspace root creates missing parents like a held root', async (t) => { + const root = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-workspace-'))); t.after(() => rm(root, { recursive: true, force: true })); - const tools = await LocalWorkspaceTools.create({ - workspaces: [{ id: 'primary', root, writable: true }], - }); + const tools = await pathnameRootTools(root); + + for (const [path, overwrite] of [ + ['.librechat/attachments/.gitignore', false], + ['missing/deep/notes.txt', undefined], + ] as const) { + const result = await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path, + content: '*\n', + ...(overwrite === undefined ? {} : { overwrite }), + }); + assert.deepEqual(result, { + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path, + created: true, + bytesWritten: 2, + }); + assert.equal(await readFile(join(root, ...path.split('/')), 'utf8'), '*\n'); + } + assert.equal((await stat(join(root, '.librechat', 'attachments'))).isDirectory(), true); + + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: '.librechat/attachments/.gitignore', + content: '!keep\n', + overwrite: false, + }), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'EDIT_CONFLICT' && + !error.mutationMayHaveCommitted, + 'a create never replaces a file in a folder it just created', + ); + assert.equal(await readFile(join(root, '.librechat', 'attachments', '.gitignore'), 'utf8'), '*\n'); +}); + +test('a pathname-only root that has been replaced by a symlink creates nothing', async (t) => { + const parent = await realpath( + await mkdtemp(join(tmpdir(), 'librechat-code-workspace-parent-')), + ); + t.after(() => rm(parent, { recursive: true, force: true })); + const root = join(parent, 'root'); + const elsewhere = join(parent, 'elsewhere'); + await mkdir(root); + await mkdir(elsewhere); + const tools = await pathnameRootTools(root); + await rm(root, { recursive: true }); + await symlink(elsewhere, root); await assert.rejects( tools.execute({ @@ -1931,10 +1996,10 @@ test('a pathname-only workspace root reports a missing parent instead of creatin }), (error: unknown) => error instanceof WorkspaceToolError && - error.code === 'NOT_FOUND' && - error.message === 'Workspace path does not exist', + error.code === 'INVALID_PATH' && + !error.mutationMayHaveCommitted, ); - assert.deepEqual(await readdir(root), []); + assert.deepEqual(await readdir(elsewhere), []); }); test('a missing workspace path is NOT_FOUND only when reached through the workspace', async (t) => { @@ -2603,7 +2668,8 @@ test('fails closed on invalid or failed sandbox command results', async (t) => { for (const execute of [ async () => ({ ...request, exitCode: 0, stdout: 'ok', stderr: '' }), async () => { throw new Error('container details'); }, - async () => { throw new WorkspaceToolError('untrusted clean claim', 'COMMAND_UNAVAILABLE'); }, + async () => { throw new WorkspaceToolError('untrusted clean claim', 'COMMAND_UNAVAILABLE', false, false, + { stage: 'command_wrapping', reason: 'wrapping_failed' }); }, ]) { const tools = new SandboxWorkspaceTools({ workspaceTools: local, @@ -2616,6 +2682,7 @@ test('fails closed on invalid or failed sandbox command results', async (t) => { error instanceof WorkspaceToolError && error.code === 'COMMAND_UNAVAILABLE' && error.mutationMayHaveCommitted === true && + (error.message !== 'untrusted clean claim' || error.commandDiagnostic?.stage === 'command_wrapping') && !error.message.includes('container details'), ); assert.equal(tools.mutationFailuresAreAtomic, undefined); diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index d740d48a..ec5a4218 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -1,10 +1,11 @@ import { createHash, randomBytes } from 'node:crypto'; import { constants } from 'node:fs'; -import { deferWorkspaceRootCleanup, holdsWorkspaceRoot, link, lstat, mkdir, open, realpath, rename, stat, unlink, spawn, withWorkspaceRoot, WorkspaceRootAccessError } from './root-access.js'; +import { deferWorkspaceRootCleanup, holdsWorkspaceRoot, link, lstat, mkdir, open, realpath, rename, rmdir, stat, unlink, spawn, withWorkspaceRoot, WorkspaceRootAccessError } from './root-access.js'; import { basename, dirname, isAbsolute, relative, resolve, sep } from 'node:path'; import type { FileHandle } from 'node:fs/promises'; -import { matchesWorkspaceRoot } from './root-identity.js'; +import type { CommandFailureDiagnostic } from './command-diagnostics.js'; +import { captureWorkspaceRootIdentity, matchesWorkspaceRoot } from './root-identity.js'; import type { WorkspaceRootIdentity } from './root-identity.js'; import { @@ -17,6 +18,7 @@ import { BRIDGE_WORKSPACE_SEARCH_TEXT_MAX_LENGTH, comparePortableRelativePaths, isSafePortableRelativePath, + isWorkspaceImportFilePath, isValidBridgeWorkspaceToolCapabilities, isWorkspaceToolRequest, isWorkspaceToolResult, @@ -42,6 +44,8 @@ import type { WorkspacePreviewEditResult, WorkspaceExecuteCommandRequest, WorkspaceExecuteCommandResult, + WorkspaceImportFileRequest, + WorkspaceImportFileResult, WorkspaceListFilesRequest, WorkspaceListFilesResult, WorkspaceSearchMatch, @@ -65,6 +69,8 @@ export type { WorkspacePreviewEditResult, WorkspaceExecuteCommandRequest, WorkspaceExecuteCommandResult, + WorkspaceImportFileRequest, + WorkspaceImportFileResult, WorkspaceListFilesRequest, WorkspaceListFilesResult, WorkspaceSearchMatch, @@ -85,9 +91,27 @@ export interface LocalWorkspaceConfig { writable?: boolean; } +/** + * Fetches a stored object for `import_file`. Only a worker with a configured + * file relay upstream has one, and `import_file` is advertised only then. + */ +export interface WorkspaceFileImporter { + /** Largest file this worker imports; larger requests fail before any transfer. */ + readonly maxBytes: number; + /** Streams the granted object. Failures are WorkspaceToolErrors. */ + open( + request: WorkspaceImportFileRequest & { egressGrant: string }, + signal?: AbortSignal, + ): Promise>; + /** Bytes a new file in `directory` may still use on its filesystem. */ + availableBytes(directory: string): Promise; +} + export interface LocalWorkspaceToolsOptions { workspaces: LocalWorkspaceConfig[]; repositoryInstructions?: boolean; + /** Enables `import_file` on writable workspaces. */ + importer?: WorkspaceFileImporter; } export interface WorkspaceToolExecutor { @@ -144,6 +168,12 @@ const READ_OPERATIONS = [ 'list_files', ] as const; const WRITE_OPERATIONS = ['write_file', 'preview_edit', 'edit_file'] as const; +const IMPORT_OPERATIONS = ['import_file'] as const; +const IMPORT_DIRECTORY_MODE = 0o755; +const IMPORT_FILE_MODE = 0o644; +const IMPORT_HASH_CHUNK_BYTES = 1024 * 1024; +/** Headroom left on the target filesystem after an import lands. */ +export const WORKSPACE_IMPORT_FREE_SPACE_RESERVE_BYTES = 64 * 1024 * 1024; function isUtf8ScalarString(value: string): boolean { return Buffer.from(value).toString('utf8') === value; @@ -197,6 +227,8 @@ export class WorkspaceToolError extends Error { public readonly mutationMayHaveCommitted = false, /** Retain the durable mutation guard when process or write settlement is uncertain. */ public readonly requiresQuarantine = mutationMayHaveCommitted, + /** Worker-local diagnostic metadata; never serialized into bridge settlements. */ + public readonly commandDiagnostic?: CommandFailureDiagnostic, ) { super(message); this.name = 'WorkspaceToolError'; @@ -580,6 +612,8 @@ interface WorkspaceRoot { identity?: WorkspaceRootIdentity; root: string; writable: boolean; + /** Writable, an importer is configured, and this host can create the parents an import needs. */ + imports: boolean; } async function verifyDirectoryPathHasNoSymlinks( @@ -672,8 +706,10 @@ async function classifyMissingWorkspacePath( * Creates each missing ancestor of a write target as a real directory beneath * the root, one verified level at a time, and syncs each new entry into its * parent. Creation needs a held root descriptor, so `mkdirat` is anchored to - * the root; a pathname-only root creates nothing and its write reports the - * missing parent. An existing symlink or file stops the walk. + * the root; a pathname-only root borrows one through + * {@link withAnchoredParentCreation}, and where none can be held creates + * nothing and its write reports the missing parent. An existing symlink or + * file stops the walk. * * Created directories are a committed side effect, never rolled back: once one * exists, a later failure of this write is reported as a possible mutation, so @@ -682,6 +718,7 @@ async function classifyMissingWorkspacePath( async function createMissingParentDirectories( root: string, candidate: string, + mode = 0o777, ): Promise { const created: string[] = []; if (!holdsWorkspaceRoot()) return created; @@ -699,7 +736,7 @@ async function createMissingParentDirectories( let createdHere = false; if (entry == null) { try { - await mkdir(current, 0o777); + await mkdir(current, mode); created.push(current); createdHere = true; } catch (error) { @@ -723,6 +760,82 @@ async function createMissingParentDirectories( } } +/** The identity to hold `root` by, while it is still a real directory at that exact path. */ +async function anchorableRootIdentity(root: string): Promise { + try { + const identity = await captureWorkspaceRootIdentity(root); + return identity.path === root ? identity : undefined; + } catch { + return undefined; + } +} + +/** + * Whether this host can hold `root` by descriptor, which creating a missing + * parent needs. Exercises the native `openat` path, not only the open. + */ +async function canAnchorWorkspaceRoot( + root: string, + identity?: WorkspaceRootIdentity, +): Promise { + const held = identity ?? (await anchorableRootIdentity(root)); + if (held == null) return false; + try { + return (await withWorkspaceRoot(root, held, () => realpath(root))) === root; + } catch { + return false; + } +} + +/** Whether `path` names a file whose parent directory is missing beneath `root`. */ +async function hasMissingParent(root: string, path: string): Promise { + let parent: string; + try { + parent = dirname(resolveWorkspacePath(root, path)); + } catch { + return false; + } + if (parent === root) return false; + try { + await lstat(parent); + return false; + } catch (error) { + return isMissingEntry(error); + } +} + +/** + * Runs a write or import on a pathname-only root (`--worker-dir`, `--workspace` + * or `--environment`) under a descriptor held for just this request when its + * parent is missing, so the mutation, parent creation included, takes the same + * anchored path a selected project root does and a swapped symlink cannot + * redirect it outside the root. A held root or an existing parent runs the + * mutation as before; a root that is no longer a real directory at its path + * runs it unanchored and is refused there, creating nothing. A root that + * changes between its capture and its open is refused as `INVALID_PATH`. + */ +async function withAnchoredParentCreation( + root: string, + path: string, + action: () => Promise, +): Promise { + if (holdsWorkspaceRoot() || !(await hasMissingParent(root, path))) return action(); + const identity = await anchorableRootIdentity(root); + if (identity == null) return action(); + try { + return await withWorkspaceRoot(root, identity, action); + } catch (error) { + /** + * Raised only while opening the root, before `action` ran: the root changed after its + * identity was captured. Never retry unanchored against whatever replaced it. + */ + if (error instanceof WorkspaceRootAccessError) { + throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + } + throw error; + } +} + function afterCreatedDirectories( created: readonly string[], error: WorkspaceToolError, @@ -1070,6 +1183,373 @@ async function writeWorkspaceFile( }; } +interface ImportTargetIdentity { + dev: bigint | number; + ino: bigint | number; +} + +type ImportTarget = + | { state: 'missing' } + | { state: 'unchanged' } + | ({ state: 'different' } & ImportTargetIdentity); + +function importIntegrityError(message: string): WorkspaceToolError { + return new WorkspaceToolError(message, 'INVALID_REQUEST'); +} + +/** Size and SHA-256 of everything an open file holds now, read through its descriptor. */ +async function hashOpenFile( + handle: FileHandle, + signal?: AbortSignal, +): Promise<{ bytes: number; sha256: string }> { + const hash = createHash('sha256'); + const chunk = Buffer.allocUnsafe(IMPORT_HASH_CHUNK_BYTES); + let position = 0; + for (;;) { + throwIfAborted(signal); + const { bytesRead } = await handle.read(chunk, 0, chunk.byteLength, position); + if (bytesRead === 0) break; + hash.update(chunk.subarray(0, bytesRead)); + position += bytesRead; + } + return { bytes: position, sha256: hash.digest('hex') }; +} + +/** Hash an existing target only when its size already matches the import. */ +async function inspectImportTarget( + root: string, + request: WorkspaceImportFileRequest, + signal?: AbortSignal, +): Promise { + try { + return await withConfinedFile(root, request.path, async (handle, size) => { + const { dev, ino } = await handle.stat(); + if (size !== request.bytes) return { state: 'different', dev, ino }; + const existing = await hashOpenFile(handle, signal); + return existing.bytes === request.bytes && existing.sha256 === request.sha256 + ? { state: 'unchanged' } + : { state: 'different', dev, ino }; + }); + } catch (error) { + if (error instanceof WorkspaceToolError && error.code === 'NOT_FOUND') { + return { state: 'missing' }; + } + throw error; + } +} + +/** + * Undo parent directories this import created; true when every one is gone + * again. A directory is removed only while it is still the inode this import + * created, so a replacement made by another local process is never deleted. + */ +async function removeCreatedDirectories( + created: readonly string[], + identities: readonly ImportTargetIdentity[], +): Promise { + if (identities.length !== created.length) return false; + for (let index = created.length - 1; index >= 0; index -= 1) { + const directory = created[index]!; + try { + const current = await lstat(directory); + if ( + current.isSymbolicLink() || + !current.isDirectory() || + current.dev !== identities[index]!.dev || + current.ino !== identities[index]!.ino + ) { + return false; + } + await rmdir(directory); + await syncWorkspaceDirectory(dirname(directory)); + } catch { + return false; + } + } + return true; +} + +/** + * Record each directory this import created, then give it the documented mode + * regardless of the worker's umask. `identities` fills as each is opened. + */ +async function setImportDirectoryModes( + created: readonly string[], + identities: ImportTargetIdentity[], +): Promise { + for (const directory of created) { + const handle = await open( + directory, + constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW, + ); + try { + const { dev, ino } = await handle.stat(); + identities.push({ dev, ino }); + await handle.chmod(IMPORT_DIRECTORY_MODE); + } finally { + await handle.close(); + } + } +} + +async function writeImportChunk(handle: FileHandle, chunk: Uint8Array, position: number): Promise { + let offset = 0; + while (offset < chunk.byteLength) { + const { bytesWritten } = await handle.write(chunk, offset, chunk.byteLength - offset, position + offset); + if (bytesWritten < 1) throw new Error('short workspace write'); + offset += bytesWritten; + } +} + +/** Confirm the installed entry is the staged inode, inside the root, after syncing its directory. */ +async function confirmInstalledImport( + root: string, + installTarget: string, + staged: ImportTargetIdentity, +): Promise { + let handle: FileHandle | undefined; + try { + await syncWorkspaceDirectory(dirname(installTarget)); + handle = await open( + installTarget, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ); + const [installed, canonicalPath] = await Promise.all([handle.stat(), realpath(installTarget)]); + if ( + !installed.isFile() || + !isWithinRoot(root, canonicalPath) || + installed.dev !== staged.dev || + installed.ino !== staged.ino + ) { + throw new Error('installed workspace import changed'); + } + } catch { + throw new WorkspaceToolError( + 'Workspace import durability could not be confirmed', + 'WRITE_UNAVAILABLE', + true, + ); + } finally { + await handle?.close().catch(() => undefined); + } +} + +/** + * Stream one object into a same-directory temporary file, verify the synced + * file's size and SHA-256 through its own descriptor, then install it + * atomically. A create links exclusively so a racing file is never replaced. + * An overwrite rechecks the inspected inode immediately before its rename, the + * same check-then-rename `write_file` uses; bridge mutations of one root are + * serialized, so only an out-of-band local writer can race that last step. + */ +async function installImportedFile( + root: string, + candidate: string, + request: WorkspaceImportFileRequest & { egressGrant: string }, + importer: WorkspaceFileImporter, + target: Exclude, + signal?: AbortSignal, +): Promise<{ created: boolean }> { + const parent = dirname(candidate); + let canonicalParent: string; + let parentIdentity: Awaited>; + try { + canonicalParent = await realpath(parent); + parentIdentity = await verifyDirectoryPathHasNoSymlinks(root, parent); + if (!isWithinRoot(root, canonicalParent) || !parentIdentity.isDirectory()) { + throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + } + } catch (error) { + if (isMissingEntry(error)) throw await classifyMissingWorkspacePath(root, parent); + throw classifyWritePathValidationError(error); + } + let available: number; + try { + available = await importer.availableBytes(canonicalParent); + } catch { + throw new WorkspaceToolError('Workspace free space could not be determined', 'WRITE_UNAVAILABLE'); + } + if (available < request.bytes + WORKSPACE_IMPORT_FREE_SPACE_RESERVE_BYTES) { + throw new WorkspaceToolError( + 'Workspace filesystem does not have enough free space for this file', + 'WRITE_LIMIT_EXCEEDED', + ); + } + + const temporary = resolve(canonicalParent, `.librechat-code-${randomBytes(18).toString('hex')}.tmp`); + const installTarget = resolve(canonicalParent, basename(candidate)); + let handle: FileHandle | undefined; + let temporaryNeedsCleanup = false; + try { + handle = await open( + temporary, + constants.O_RDWR | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, + 0o600, + ); + temporaryNeedsCleanup = true; + let written = 0; + for await (const chunk of await importer.open(request, signal)) { + throwIfAborted(signal); + if (written + chunk.byteLength > request.bytes) { + throw importIntegrityError('Imported file is larger than its declared size'); + } + await writeImportChunk(handle, chunk, written); + written += chunk.byteLength; + } + throwIfAborted(signal); + if (written !== request.bytes) { + throw importIntegrityError('Imported file is smaller than its declared size'); + } + await handle.chmod(IMPORT_FILE_MODE); + await handle.sync(); + // Verify what will be installed, not what arrived: the staged file is a workspace entry. + const staged = await hashOpenFile(handle, signal); + if (staged.bytes !== request.bytes) { + throw importIntegrityError('Imported file does not match its declared size'); + } + if (staged.sha256 !== request.sha256) { + throw importIntegrityError('Imported file does not match its declared SHA-256'); + } + const stagedStat = await handle.stat(); + const stagedIdentity = { dev: stagedStat.dev, ino: stagedStat.ino }; + if (process.platform === 'win32') { + await handle.close(); + handle = undefined; + } + + const [currentParent, currentParentIdentity] = await Promise.all([ + realpath(parent), + verifyDirectoryPathHasNoSymlinks(root, parent), + ]); + if ( + currentParent !== canonicalParent || + currentParentIdentity.dev !== parentIdentity.dev || + currentParentIdentity.ino !== parentIdentity.ino + ) { + throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + } + let current: Awaited> | undefined; + try { + current = await lstat(candidate); + } catch (error) { + if (!isMissingEntry(error)) throw error; + } + if ( + target.state === 'missing' + ? current != null + : current == null || current.isSymbolicLink() || current.dev !== target.dev || current.ino !== target.ino + ) { + throw new WorkspaceToolError( + 'Workspace file changed before import could be committed', + 'EDIT_CONFLICT', + ); + } + throwIfAborted(signal); + if (target.state === 'different') { + await rename(temporary, installTarget); + temporaryNeedsCleanup = false; + } else { + try { + await link(temporary, installTarget); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'EEXIST') { + throw new WorkspaceToolError('Workspace file already exists', 'EDIT_CONFLICT'); + } + throw error; + } + try { + await unlink(temporary); + temporaryNeedsCleanup = false; + } catch { + throw new WorkspaceToolError( + 'Workspace import cleanup could not be confirmed', + 'WRITE_UNAVAILABLE', + true, + ); + } + } + await confirmInstalledImport(root, installTarget, stagedIdentity); + return { created: target.state === 'missing' }; + } catch (error) { + if (error instanceof WorkspaceToolError) throw error; + throw new WorkspaceToolError('Workspace storage is unavailable', 'WRITE_UNAVAILABLE'); + } finally { + await handle?.close().catch(() => undefined); + if (temporaryNeedsCleanup) await removeImportStaging(temporary); + } +} + +/** A staging file that cannot be removed is upload bytes left in the workspace: never a clean failure. */ +async function removeImportStaging(temporary: string): Promise { + try { + await unlink(temporary); + } catch (error) { + if (isMissingEntry(error)) return; + throw new WorkspaceToolError( + 'Workspace import staging file could not be removed', + 'WRITE_UNAVAILABLE', + true, + ); + } +} + +async function importWorkspaceFile( + root: string, + request: WorkspaceImportFileRequest, + importer: WorkspaceFileImporter, + signal?: AbortSignal, +): Promise { + throwIfAborted(signal); + if (!isWorkspaceImportFilePath(request.path)) { + throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + } + const { egressGrant } = request; + if (egressGrant === undefined) { + throw new WorkspaceToolError('Import source grant is unavailable', 'INVALID_REQUEST'); + } + if (request.bytes > importer.maxBytes) { + throw new WorkspaceToolError('Imported file exceeds the worker file limit', 'WRITE_LIMIT_EXCEEDED'); + } + const result = (created: boolean, bytesWritten: number, unchanged = false): WorkspaceImportFileResult => ({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'import_file', + workspaceId: request.workspaceId, + path: request.path, + created, + ...(unchanged ? { unchanged: true } : {}), + bytesWritten, + sha256: request.sha256, + }); + const candidate = resolveWorkspacePath(root, request.path); + const target = await inspectImportTarget(root, request, signal); + if (target.state === 'unchanged') return result(false, 0, true); + if (target.state === 'different' && request.overwrite !== true) { + throw new WorkspaceToolError( + 'Workspace file already exists with different content', + 'EDIT_CONFLICT', + ); + } + const directories = await createMissingParentDirectories(root, candidate, IMPORT_DIRECTORY_MODE); + const identities: ImportTargetIdentity[] = []; + try { + await setImportDirectoryModes(directories, identities); + const { created } = await installImportedFile( + root, + candidate, + { ...request, egressGrant }, + importer, + target, + signal, + ); + return result(created, request.bytes); + } catch (error) { + const failure = classifyWritePathValidationError(error); + if (directories.length === 0 || failure.mutationMayHaveCommitted) throw failure; + if (await removeCreatedDirectories(directories, identities)) throw failure; + throw afterCreatedDirectories(directories, failure); + } +} + async function editWorkspaceFile( root: string, request: WorkspaceEditFileRequest, @@ -1878,6 +2358,7 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { editFileModes?: BridgeWorkspaceToolCapabilities['editFileModes'], editFileFeatures?: BridgeWorkspaceToolCapabilities['editFileFeatures'], listFileFeatures?: BridgeWorkspaceToolCapabilities['listFileFeatures'], + private readonly importer?: WorkspaceFileImporter, ) { this.capabilities = { protocolVersion: BRIDGE_PROTOCOL_VERSION, @@ -1894,12 +2375,43 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { options: LocalWorkspaceToolsOptions, ): Promise { const roots = new Map(); + for (const workspace of options.workspaces) { + let canonicalRoot: string; + try { + canonicalRoot = await realpath(workspace.root); + if (workspace.identity && !await matchesWorkspaceRoot(canonicalRoot, workspace.identity)) throw new Error(); + await withWorkspaceRoot(canonicalRoot, workspace.identity, async () => undefined); + if (!(await stat(canonicalRoot)).isDirectory()) throw new Error(); + } catch { + throw new WorkspaceToolError( + 'Invalid workspace registration', + 'REGISTRATION_INVALID', + ); + } + const writable = workspace.writable === true; + roots.set(workspace.id, { + identity: workspace.identity, + root: canonicalRoot, + writable, + /** Imports land in folders that may not exist yet, so only where parents can be created. */ + imports: + writable && + options.importer != null && + (await canAnchorWorkspaceRoot(canonicalRoot, workspace.identity)), + }); + } const anyWritable = options.workspaces.some( (workspace) => workspace.writable === true, ); + const anyImports = [...roots.values()].some((root) => root.imports); + const mutationsFor = (root: WorkspaceRoot | undefined) => [ + ...(root?.writable ? WRITE_OPERATIONS : []), + ...(root?.imports ? IMPORT_OPERATIONS : []), + ]; const operations: BridgeWorkspaceToolCapabilities['operations'] = [ ...READ_OPERATIONS, ...(anyWritable ? WRITE_OPERATIONS : []), + ...(anyImports ? IMPORT_OPERATIONS : []), ]; const workspaces: BridgeWorkspaceDescriptor[] = options.workspaces.map( (workspace) => ({ @@ -1909,7 +2421,7 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { ? { operations: [ ...READ_OPERATIONS, - ...(workspace.writable === true ? WRITE_OPERATIONS : []), + ...mutationsFor(roots.get(workspace.id)), ], } : {}), @@ -1932,25 +2444,6 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { 'REGISTRATION_INVALID', ); } - for (const workspace of options.workspaces) { - let canonicalRoot: string; - try { - canonicalRoot = await realpath(workspace.root); - if (workspace.identity && !await matchesWorkspaceRoot(canonicalRoot, workspace.identity)) throw new Error(); - await withWorkspaceRoot(canonicalRoot, workspace.identity, async () => undefined); - if (!(await stat(canonicalRoot)).isDirectory()) throw new Error(); - } catch { - throw new WorkspaceToolError( - 'Invalid workspace registration', - 'REGISTRATION_INVALID', - ); - } - roots.set(workspace.id, { - identity: workspace.identity, - root: canonicalRoot, - writable: workspace.writable === true, - }); - } const tools = new LocalWorkspaceTools( roots, operations, @@ -1959,6 +2452,7 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { capabilities.editFileModes, capabilities.editFileFeatures, capabilities.listFileFeatures, + anyImports ? options.importer : undefined, ); tools.repositoryInstructions = options.repositoryInstructions === true; return tools; @@ -2000,6 +2494,25 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { } const { root } = workspace; + if (request.operation === 'import_file') { + if (!workspace.writable) { + throw new WorkspaceToolError( + 'Workspace mutations are disabled by the worker', + 'WRITE_DISABLED', + ); + } + const { importer } = this; + if (importer == null || !workspace.imports) { + throw new WorkspaceToolError( + 'Workspace file import is not configured on this worker', + 'WRITE_UNAVAILABLE', + ); + } + return withAnchoredParentCreation(root, request.path, () => + importWorkspaceFile(root, request, importer, signal), + ); + } + if ( request.operation === 'write_file' || request.operation === 'preview_edit' || @@ -2018,7 +2531,9 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { ); } if (request.operation === 'write_file') { - return writeWorkspaceFile(root, request, signal); + return withAnchoredParentCreation(root, request.path, () => + writeWorkspaceFile(root, request, signal), + ); } return request.operation === 'preview_edit' ? previewWorkspaceEdit(root, request, signal) @@ -2163,7 +2678,7 @@ export class SandboxWorkspaceTools implements WorkspaceToolExecutor { ) { throw error; } - throw new WorkspaceToolError(error.message, error.code, true); + throw new WorkspaceToolError(error.message, error.code, true, true, error.commandDiagnostic); } throw new WorkspaceToolError( 'Sandboxed command execution unavailable', diff --git a/packages/extraction/Dockerfile b/packages/extraction/Dockerfile index 4acf5d9c..0ae3d5ce 100644 --- a/packages/extraction/Dockerfile +++ b/packages/extraction/Dockerfile @@ -1,4 +1,4 @@ -FROM node:24.16.0-bookworm-slim AS build +FROM node:24.22.1-bookworm-slim AS build WORKDIR /build COPY packages/extraction/package*.json ./ RUN npm ci --ignore-scripts --no-audit --no-fund @@ -6,8 +6,8 @@ COPY packages/extraction/tsconfig.json ./ COPY packages/extraction/src ./src RUN npm run build RUN apt-get update && apt-get install -y --no-install-recommends gcc libc6-dev linux-libc-dev && rm -rf /var/lib/apt/lists/* -COPY packages/extraction/native/guard.c ./guard.c -RUN gcc -O2 -Wall -Wextra -Werror -o guard guard.c +COPY packages/extraction/native/guard.c packages/extraction/native/gateway.c ./ +RUN gcc -O2 -Wall -Wextra -Werror -o guard guard.c && gcc -O2 -Wall -Wextra -Werror -o gateway gateway.c COPY packages/extraction/tests/block-landlock.c ./block-landlock.c RUN gcc -O2 -Wall -Wextra -Werror -o block-landlock block-landlock.c @@ -17,14 +17,16 @@ COPY packages/extraction/worker/requirements.txt /opt/extraction/requirements.tx RUN pip install --no-cache-dir -r /opt/extraction/requirements.txt COPY --from=build /usr/local/bin/node /usr/local/bin/node COPY --from=build --chmod=0555 /build/guard /opt/extraction/guard +COPY --from=build --chmod=0555 /build/gateway /opt/extraction/gateway COPY --from=build --chmod=0555 /build/dist /opt/extraction/dist COPY --chmod=0444 packages/extraction/package.json /opt/extraction/package.json COPY --chmod=0444 packages/extraction/worker/extract.py /opt/extraction/extract.py RUN groupadd -g 10001 extraction && useradd -u 10001 -g extraction -M extraction && \ - mkdir /socket /jobs && chown extraction:extraction /socket /jobs && chmod 0700 /jobs && chmod 0770 /socket + mkdir -p /run/extraction /jobs && chown extraction:extraction /run/extraction /jobs && chmod 0700 /jobs && chmod 0755 /run/extraction USER 10001:10001 WORKDIR /opt/extraction -HEALTHCHECK --interval=10s --timeout=2s --start-period=10s --retries=3 CMD ["node", "dist/health.js"] +HEALTHCHECK --interval=10s --timeout=2s --start-period=10s --retries=3 CMD ["/opt/extraction/gateway", "node", "dist/health.js"] +ENTRYPOINT ["/opt/extraction/gateway"] CMD ["node", "dist/main.js"] FROM runtime AS qualification diff --git a/packages/extraction/compose.yaml b/packages/extraction/compose.yaml index f6bbf9f4..87d90964 100644 --- a/packages/extraction/compose.yaml +++ b/packages/extraction/compose.yaml @@ -1,22 +1,47 @@ services: + extraction-init: + image: ${EXTRACTION_IMAGE:-librechat-extraction:local} + build: + context: ../.. + dockerfile: packages/extraction/Dockerfile + target: runtime + command: [node, dist/provision.js] + user: '10001:10001' + read_only: true + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + network_mode: none + mem_limit: 128m + cpus: 1 + pids_limit: 16 + healthcheck: + disable: true + volumes: + - extraction-token:/run/extraction extraction: image: ${EXTRACTION_IMAGE:-librechat-extraction:local} build: context: ../.. dockerfile: packages/extraction/Dockerfile target: runtime + depends_on: + extraction-init: + condition: service_completed_successfully init: true user: '10001:10001' read_only: true cap_drop: [ALL] security_opt: [no-new-privileges:true] - network_mode: none + networks: [extraction] mem_limit: 768m cpus: 2 pids_limit: 32 tmpfs: - /jobs:rw,noexec,nosuid,nodev,size=96m,nr_inodes=1024,uid=10001,gid=10001,mode=0700 volumes: - - extraction-socket:/socket + - extraction-token:/run/extraction:ro +networks: + extraction: + internal: true volumes: - extraction-socket: + extraction-token: diff --git a/packages/extraction/native/gateway.c b/packages/extraction/native/gateway.c new file mode 100644 index 00000000..a91bbaa7 --- /dev/null +++ b/packages/extraction/native/gateway.c @@ -0,0 +1,63 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__x86_64__) +#define ARCH AUDIT_ARCH_X86_64 +#elif defined(__aarch64__) +#define ARCH AUDIT_ARCH_AARCH64 +#else +#error Unsupported architecture +#endif + +#define DENY(nr) BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, nr, 0, 1), BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | EPERM) +int main(int argc, char **argv) { + if (argc < 2 || geteuid() == 0) goto fail; + /* No inherited socket, including a caller-supplied socket on standard IO. */ + for (int fd = 0; fd < 3; fd++) { + struct stat stat; + if (fstat(fd, &stat)) goto fail; + if (S_ISSOCK(stat.st_mode)) { + int null = open("/dev/null", O_RDWR | O_CLOEXEC); + if (null < 0 || dup2(null, fd) < 0) goto fail; + close(null); + } + } + if (syscall(SYS_close_range, 3, ~0U, 0)) goto fail; + struct sock_filter filter[] = { + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, arch)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, ARCH, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), +#ifdef __x86_64__ + BPF_JUMP(BPF_JMP | BPF_JGE | BPF_K, 0x40000000, 0, 1), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), +#endif + DENY(__NR_socketpair), + DENY(__NR_sendmsg), DENY(__NR_recvmsg), DENY(__NR_sendmmsg), DENY(__NR_recvmmsg), + /* Only IP sockets. Docker's default filter remains in force underneath. */ + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_socket, 0, 5), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[0])), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, AF_INET, 2, 0), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, AF_INET6, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | EPERM), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW) + }; + struct sock_fprog program = {.len = sizeof(filter) / sizeof(filter[0]), .filter = filter}; + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) || prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &program)) goto fail; + execvp(argv[1], &argv[1]); +fail: + fputs("EXTRACTION_GATEWAY_UNAVAILABLE\n", stderr); + return 125; +} diff --git a/packages/extraction/native/guard.c b/packages/extraction/native/guard.c index ee1d2cbc..77d11e91 100644 --- a/packages/extraction/native/guard.c +++ b/packages/extraction/native/guard.c @@ -59,6 +59,9 @@ static void restrict_syscalls(void) { BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), #endif DENY(__NR_socket), DENY(__NR_socketpair), DENY(__NR_connect), + DENY(__NR_sendmsg), DENY(__NR_recvmsg), DENY(__NR_sendmmsg), DENY(__NR_recvmmsg), + DENY(__NR_sendto), DENY(__NR_recvfrom), DENY(__NR_setsockopt), DENY(__NR_getsockopt), + DENY(__NR_shutdown), DENY(__NR_getsockname), DENY(__NR_getpeername), DENY(__NR_bind), DENY(__NR_listen), DENY(__NR_accept), DENY(__NR_accept4), DENY(__NR_clone), DENY(__NR_clone3), #ifdef __NR_fork @@ -119,6 +122,11 @@ static void restrict_syscalls(void) { int main(int argc, char **argv) { if (argc != 5 || geteuid() == 0) fail(); /* Paths are supervisor-owned, never supplied in the HTTP protocol. */ + struct stat input, output, errors, null; + if (fstat(0, &input) || fstat(1, &output) || fstat(2, &errors) || stat("/dev/null", &null)) fail(); + if (!S_ISCHR(input.st_mode) || input.st_rdev != null.st_rdev || + !S_ISCHR(errors.st_mode) || errors.st_rdev != null.st_rdev || + !S_ISREG(output.st_mode) || output.st_nlink != 1 || output.st_size != 0) fail(); if (chdir(argv[1])) fail(); pid_t parent = getppid(); if (parent == 1 || prctl(PR_SET_PDEATHSIG, SIGKILL) || getppid() != parent) fail(); diff --git a/packages/extraction/src/backend.ts b/packages/extraction/src/backend.ts index a3831c65..b6d3c430 100644 --- a/packages/extraction/src/backend.ts +++ b/packages/extraction/src/backend.ts @@ -46,67 +46,79 @@ export class RestrictedBackend implements Backend { signal: AbortSignal ): Promise { signal.throwIfAborted(); - const child = spawn( - this.runtime.guard, - [job, this.runtime.python, this.runtime.worker, mode], - { - detached: true, - stdio: ['ignore', 'pipe', 'pipe'], - env: {}, - } + const status = await open( + join(job, 'status'), + constants.O_CREAT | + constants.O_EXCL | + constants.O_NOFOLLOW | + constants.O_RDWR, + 0o600 ); - let output = ''; - let count = 0; - let overflow = false; - const kill = () => { - if ( - child.pid && - child.exitCode === null && - child.signalCode === null - ) { - try { - process.kill(-child.pid, 'SIGKILL'); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ESRCH') - this.healthy = false; + try { + signal.throwIfAborted(); + const child = spawn( + this.runtime.guard, + [job, this.runtime.python, this.runtime.worker, mode], + { + detached: true, + stdio: ['ignore', status.fd, 'ignore'], + env: {}, } - } - }; - signal.addEventListener('abort', kill, { once: true }); - if (signal.aborted) kill(); - let spawnError = false; - child.on('error', () => { - spawnError = true; - }); - child.stdout.on('data', (chunk: Buffer) => { - count += chunk.length; - if (count > 1024) { - overflow = true; - kill(); - } else output += chunk.toString('ascii'); - }); - child.stderr.on('data', (chunk: Buffer) => { - count += chunk.length; - if (count > 2048) { - overflow = true; - kill(); - } - }); - const code = await new Promise(resolve => - child.once('close', resolve) - ); - signal.removeEventListener('abort', kill); - signal.throwIfAborted(); - if (spawnError || overflow) throw new ExtractionError('UNAVAILABLE'); - if (code !== 0) { - const value = output.trim(); - if (parserErrors.has(value)) - throw new ExtractionError(value as 'INVALID_DOCUMENT'); - throw new ExtractionError( - code === null ? 'RESOURCE_LIMIT' : 'UNAVAILABLE' ); + const kill = () => { + if ( + child.pid && + child.exitCode === null && + child.signalCode === null + ) { + try { + process.kill(-child.pid, 'SIGKILL'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ESRCH') + this.healthy = false; + } + } + }; + signal.addEventListener('abort', kill, { once: true }); + if (signal.aborted) kill(); + let spawnError = false; + child.on('error', () => { + spawnError = true; + }); + const code = await new Promise(resolve => + child.once('close', resolve) + ); + signal.removeEventListener('abort', kill); + signal.throwIfAborted(); + const stat = await status.stat(); + if ( + spawnError || + !stat.isFile() || + stat.nlink !== 1 || + stat.size > 1024 + ) + throw new ExtractionError('UNAVAILABLE'); + const buffer = Buffer.alloc(1025); + const { bytesRead } = await status.read( + buffer, + 0, + buffer.length, + 0 + ); + if (bytesRead > 1024) throw new ExtractionError('UNAVAILABLE'); + const output = buffer.toString('ascii', 0, bytesRead); + if (code !== 0) { + const value = output.trim(); + if (parserErrors.has(value)) + throw new ExtractionError(value as 'INVALID_DOCUMENT'); + throw new ExtractionError( + code === null ? 'RESOURCE_LIMIT' : 'UNAVAILABLE' + ); + } + return output; + } finally { + await status.close(); } - return output; } async probe(): Promise { const job = await this.create(); @@ -131,7 +143,7 @@ export class RestrictedBackend implements Backend { await this.execute(job, format, signal); const file = await open( join(job, 'result'), - constants.O_RDONLY | constants.O_NOFOLLOW + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK ); try { const stat = await file.stat(); @@ -142,7 +154,11 @@ export class RestrictedBackend implements Backend { ) { throw new ExtractionError('OUTPUT_LIMIT'); } - const bytes = await file.readFile(); + const buffer = Buffer.alloc(stat.size + 1); + const { bytesRead } = await file.read(buffer, 0, buffer.length, 0); + if (bytesRead > policy.resultBytes) + throw new ExtractionError('OUTPUT_LIMIT'); + const bytes = buffer.subarray(0, bytesRead); signal.throwIfAborted(); validateResult(bytes, format, digest); return bytes; diff --git a/packages/extraction/src/config.ts b/packages/extraction/src/config.ts new file mode 100644 index 00000000..f2d94238 --- /dev/null +++ b/packages/extraction/src/config.ts @@ -0,0 +1,9 @@ +import { isIP } from 'node:net'; + +export function endpoint() { + const host = process.env.EXTRACTION_HOST ?? '0.0.0.0'; + const port = process.env.EXTRACTION_PORT ?? '8000'; + if (!isIP(host) || !/^[1-9]\d{0,4}$/.test(port) || Number(port) > 65535) + throw new Error('INVALID_ENDPOINT'); + return { host, port: Number(port) }; +} diff --git a/packages/extraction/src/credential.test.ts b/packages/extraction/src/credential.test.ts new file mode 100644 index 00000000..5298c089 --- /dev/null +++ b/packages/extraction/src/credential.test.ts @@ -0,0 +1,94 @@ +import assert from 'node:assert/strict'; +import { + chmod, + mkdir, + mkdtemp, + readFile, + rm, + symlink, + writeFile, +} from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { test } from 'node:test'; +import { provision } from './provision.js'; +import { readToken } from './credential.js'; +import { endpoint } from './config.js'; + +test('atomic provisioning is concurrent, idempotent and restart safe', async () => { + const root = await mkdtemp(join(tmpdir(), 'credential-')); + await chmod(root, 0o755); + const path = join(root, 'token'); + try { + await Promise.all(Array.from({ length: 8 }, () => provision(path))); + const token = await readToken(path); + assert.match(token, /^[a-f0-9]{64}$/); + await provision(path); + assert.equal(await readToken(path), token); + } finally { + await rm(root, { recursive: true }); + } +}); + +test('existing invalid tokens, symlinks and non-files fail closed without regeneration', async () => { + const root = await mkdtemp(join(tmpdir(), 'credential-')); + await chmod(root, 0o755); + try { + const target = join(root, 'valid'); + await writeFile(target, 'a'.repeat(64), { mode: 0o444 }); + for (const [name, value, mode] of [ + ['short', 'a', 0o444], + ['uppercase', 'A'.repeat(64), 0o444], + ['oversize', 'a'.repeat(4096), 0o444], + ['writable', 'a'.repeat(64), 0o644], + ] as const) { + const path = join(root, name); + await writeFile(path, value, { mode }); + await assert.rejects(provision(path)); + assert.equal(await readFile(path, 'utf8'), value); + } + for (const [name, destination] of [ + ['link', target], + ['dangling', join(root, 'missing')], + ]) { + const path = join(root, name!); + await symlink(destination!, path); + await assert.rejects(provision(path)); + } + const directory = join(root, 'directory'); + await mkdir(directory); + await assert.rejects(readToken(directory)); + } finally { + await rm(root, { recursive: true }); + } +}); + +test('endpoint defaults and deployment overrides are bounded literal IP addresses', () => { + const previous = { + host: process.env.EXTRACTION_HOST, + port: process.env.EXTRACTION_PORT, + }; + try { + delete process.env.EXTRACTION_HOST; + delete process.env.EXTRACTION_PORT; + assert.deepEqual(endpoint(), { host: '0.0.0.0', port: 8000 }); + process.env.EXTRACTION_HOST = '::'; + process.env.EXTRACTION_PORT = '9000'; + assert.deepEqual(endpoint(), { host: '::', port: 9000 }); + for (const port of ['0', '65536', '08', '-1', 'x']) { + process.env.EXTRACTION_PORT = port; + assert.throws(endpoint); + } + process.env.EXTRACTION_PORT = '8000'; + process.env.EXTRACTION_HOST = '/socket/forbidden'; + assert.throws(endpoint); + } finally { + for (const [key, value] of [ + ['EXTRACTION_HOST', previous.host], + ['EXTRACTION_PORT', previous.port], + ]) { + if (value === undefined) delete process.env[key!]; + else process.env[key!] = value; + } + } +}); diff --git a/packages/extraction/src/credential.ts b/packages/extraction/src/credential.ts new file mode 100644 index 00000000..afee844f --- /dev/null +++ b/packages/extraction/src/credential.ts @@ -0,0 +1,25 @@ +import { constants } from 'node:fs'; +import { open } from 'node:fs/promises'; + +export const tokenFile = () => + process.env.EXTRACTION_TOKEN_FILE ?? '/run/extraction/token'; + +export async function readToken(path: string = tokenFile()): Promise { + const file = await open( + path, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK + ); + try { + const stat = await file.stat(); + if (!stat.isFile() || stat.size !== 64 || (stat.mode & 0o777) !== 0o444) + throw new Error('INVALID_TOKEN_FILE'); + const bytes = Buffer.alloc(65); + const { bytesRead } = await file.read(bytes, 0, bytes.length, 0); + const token = bytes.toString('utf8', 0, bytesRead); + if (bytesRead !== 64 || !/^[a-f0-9]{64}$/.test(token)) + throw new Error('INVALID_TOKEN_FILE'); + return token; + } finally { + await file.close(); + } +} diff --git a/packages/extraction/src/health.ts b/packages/extraction/src/health.ts index 641575c4..fa18be07 100644 --- a/packages/extraction/src/health.ts +++ b/packages/extraction/src/health.ts @@ -1,15 +1,26 @@ import { request } from 'node:http'; -const req = request( - { - socketPath: '/socket/extraction.sock', - path: '/v1/capabilities', - timeout: 1000, - }, - res => { - res.resume(); - res.once('end', () => process.exit(res.statusCode === 200 ? 0 : 1)); - } -); -req.on('timeout', () => req.destroy()); -req.on('error', () => process.exit(1)); -req.end(); +import { endpoint } from './config.js'; +import { readToken } from './credential.js'; + +try { + const { host, port } = endpoint(); + const req = request( + { + host: + host === '::' ? '::1' : host === '0.0.0.0' ? '127.0.0.1' : host, + port, + path: '/v1/capabilities', + headers: { Authorization: `Bearer ${await readToken()}` }, + timeout: 1000, + }, + res => { + res.resume(); + res.once('end', () => process.exit(res.statusCode === 200 ? 0 : 1)); + } + ); + req.on('timeout', () => req.destroy()); + req.on('error', () => process.exit(1)); + req.end(); +} catch { + process.exit(1); +} diff --git a/packages/extraction/src/main.ts b/packages/extraction/src/main.ts index 424a6ed6..2d103a81 100644 --- a/packages/extraction/src/main.ts +++ b/packages/extraction/src/main.ts @@ -1,9 +1,9 @@ -import { chmod, lstat, readdir, unlink } from 'node:fs/promises'; -import { connect } from 'node:net'; +import { readdir } from 'node:fs/promises'; import { RestrictedBackend } from './backend.js'; +import { endpoint } from './config.js'; +import { readToken } from './credential.js'; import { extractionServer } from './server.js'; -// Deployment-owned locations, not caller-selected paths or runtimes. const backend = new RestrictedBackend({ jobs: '/jobs', guard: '/opt/extraction/guard', @@ -11,46 +11,16 @@ const backend = new RestrictedBackend({ worker: '/opt/extraction/extract.py', }); try { + const address = endpoint(); + const token = await readToken(); if ((await readdir('/jobs')).length) throw new Error('DIRTY_SCRATCH'); - try { - const stat = await lstat('/socket/extraction.sock'); - if (!stat.isSocket()) throw new Error('INVALID_SOCKET'); - await new Promise((resolve, reject) => { - const client = connect('/socket/extraction.sock'); - client.setTimeout(1000, () => { - client.destroy(); - reject(new Error('SOCKET_EXISTS')); - }); - client.once('connect', () => { - client.destroy(); - reject(new Error('SOCKET_EXISTS')); - }); - client.once('error', error => { - if ((error as NodeJS.ErrnoException).code === 'ECONNREFUSED') - resolve(); - else reject(error); - }); - }); - await unlink('/socket/extraction.sock'); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; - } await backend.probe(); - const server = extractionServer(backend); + const server = extractionServer(backend, token); server.on('error', () => { console.error('EXTRACTION_UNAVAILABLE'); process.exit(1); }); - process.umask(0o007); - server.listen('/socket/extraction.sock', async () => { - try { - await chmod('/socket/extraction.sock', 0o660); - } catch { - console.error('EXTRACTION_UNAVAILABLE'); - process.exit(1); - } - console.log('Extraction service ready'); - }); + server.listen(address, () => console.log('Extraction service ready')); const shutdown = () => { server.close(); server.closeAllConnections(); diff --git a/packages/extraction/src/provision.ts b/packages/extraction/src/provision.ts new file mode 100644 index 00000000..0bdc329b --- /dev/null +++ b/packages/extraction/src/provision.ts @@ -0,0 +1,70 @@ +import { randomBytes } from 'node:crypto'; +import { constants } from 'node:fs'; +import { link, lstat, open, unlink } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { readToken, tokenFile } from './credential.js'; + +export async function provision(path: string = tokenFile()): Promise { + try { + await readToken(path); + return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + const directory = dirname(path); + const stat = await lstat(directory); + if (!stat.isDirectory() || (stat.mode & 0o777) !== 0o755) + throw new Error('INVALID_TOKEN_DIRECTORY'); + const temporary = join( + directory, + `.token-${randomBytes(16).toString('hex')}` + ); + const file = await open( + temporary, + constants.O_WRONLY | + constants.O_CREAT | + constants.O_EXCL | + constants.O_NOFOLLOW, + 0o600 + ); + try { + try { + await file.writeFile(randomBytes(32).toString('hex'), 'ascii'); + await file.chmod(0o444); + await file.sync(); + } finally { + await file.close(); + } + try { + // Publish complete bytes without replacing a concurrent provisioner's token. + await link(temporary, path); + const directoryFile = await open( + directory, + constants.O_RDONLY | + constants.O_DIRECTORY | + constants.O_NOFOLLOW + ); + try { + await directoryFile.sync(); + } finally { + await directoryFile.close(); + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + } + await readToken(path); + } finally { + await unlink(temporary); + } +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + provision().catch(() => { + console.error('EXTRACTION_PROVISION_UNAVAILABLE'); + process.exitCode = 1; + }); +} diff --git a/packages/extraction/src/server.test.ts b/packages/extraction/src/server.test.ts index 60f9bc63..8aa55c37 100644 --- a/packages/extraction/src/server.test.ts +++ b/packages/extraction/src/server.test.ts @@ -27,15 +27,15 @@ async function fixture(deadline = 500) { await rm(job, { recursive: true }); }, }; - const server = extractionServer(backend, deadline); - const socketPath = join(root, 'service.sock'); - server.listen(socketPath); + const server = extractionServer(backend, 'a'.repeat(64), deadline); + server.listen(0, '127.0.0.1'); await once(server, 'listening'); + const port = (server.address() as import('node:net').AddressInfo).port; return { backend, events, root, - socketPath, + port, server, close: async () => { server.closeAllConnections(); @@ -45,23 +45,25 @@ async function fixture(deadline = 500) { }; } const headers = { + Authorization: `Bearer ${'a'.repeat(64)}`, 'Content-Type': 'application/octet-stream', 'Content-Length': '4', 'X-Extraction-Version': '1', 'X-Extraction-Format': 'pdf', }; -async function send( - socketPath: string, - overrides = {}, - path = '/v1/extract-text' -) { +async function send(port: number, overrides = {}, path = '/v1/extract-text') { return new Promise<{ status: number; body: string }>((resolve, reject) => { const req = request( { - socketPath, + host: '127.0.0.1', + port, path, - method: 'POST', - headers: { ...headers, ...overrides }, + method: path === '/v1/capabilities' ? 'GET' : 'POST', + headers: Object.fromEntries( + Object.entries({ ...headers, ...overrides }).filter( + ([, value]) => value !== undefined + ) + ), }, res => { let body = ''; @@ -78,9 +80,9 @@ async function send( test('binary upload executes once and cleanup completes before success', async () => { const f = await fixture(); try { - assert.equal((await send(f.socketPath)).status, 200); + assert.equal((await send(f.port)).status, 200); assert.deepEqual(f.events, ['create', 'run', 'cleanup']); - assert.deepEqual(await readdir(f.root), ['service.sock']); + assert.deepEqual(await readdir(f.root), []); } finally { await f.close(); } @@ -94,15 +96,15 @@ test('closed parameters and declared input limit are rejected before admission', { 'X-Extraction-Url': 'https://example.com' }, { 'Content-Type': 'text/plain' }, ]) { - assert.equal((await send(f.socketPath, changes)).status, 400); + assert.equal((await send(f.port, changes)).status, 400); } assert.equal( - (await send(f.socketPath, {}, '/v1/extract-text?path=x')).status, + (await send(f.port, {}, '/v1/extract-text?path=x')).status, 404 ); assert.equal( ( - await send(f.socketPath, { + await send(f.port, { 'Content-Length': String(policy.inputBytes + 1), }) ).status, @@ -120,8 +122,8 @@ test('cleanup failure suppresses success and quarantines backend', async () => { throw new ExtractionError('UNAVAILABLE'); }; try { - assert.equal((await send(f.socketPath)).status, 503); - assert.equal((await send(f.socketPath)).status, 503); + assert.equal((await send(f.port)).status, 503); + assert.equal((await send(f.port)).status, 503); assert.equal(f.events.filter(e => e === 'run').length, 1); } finally { await f.close(); @@ -146,7 +148,7 @@ test('deadline aborts execution and waits for its close before cleanup', async ( return Buffer.alloc(0); }; try { - assert.equal((await send(f.socketPath)).status, 504); + assert.equal((await send(f.port)).status, 504); assert.deepEqual(f.events, ['create', 'closed', 'cleanup']); } finally { await f.close(); @@ -157,7 +159,8 @@ test('upload disconnect and stalled upload both release scratch and admission', try { for (const disconnect of [true, false]) { const req = request({ - socketPath: f.socketPath, + host: '127.0.0.1', + port: f.port, path: '/v1/extract-text', method: 'POST', headers, @@ -168,10 +171,10 @@ test('upload disconnect and stalled upload both release scratch and admission', await new Promise(resolve => setTimeout(resolve, 20)); if (disconnect) req.destroy(); await new Promise(resolve => setTimeout(resolve, 80)); - assert.deepEqual(await readdir(f.root), ['service.sock']); + assert.deepEqual(await readdir(f.root), []); } assert.equal(f.events.includes('run'), false); - assert.equal((await send(f.socketPath)).status, 200); + assert.equal((await send(f.port)).status, 200); } finally { await f.close(); } @@ -188,11 +191,11 @@ test('two active jobs bound concurrency without a queue', async () => { return Buffer.alloc(0); }; try { - const a = send(f.socketPath), - b = send(f.socketPath); + const a = send(f.port), + b = send(f.port); while (active !== 2) await new Promise(resolve => setTimeout(resolve, 5)); - assert.equal((await send(f.socketPath)).status, 429); + assert.equal((await send(f.port)).status, 429); assert.equal((await a).status, 504); assert.equal((await b).status, 504); } finally { @@ -205,7 +208,8 @@ test('deadline closes stalled output readers while keeping their slots admitted' const clients = [0, 1].map(() => { const req = request( { - socketPath: f.socketPath, + host: '127.0.0.1', + port: f.port, path: '/v1/extract-text', method: 'POST', headers, @@ -222,10 +226,10 @@ test('deadline closes stalled output readers while keeping their slots admitted' try { while (f.events.filter(e => e === 'cleanup').length !== 2) await new Promise(resolve => setTimeout(resolve, 5)); - assert.equal((await send(f.socketPath)).status, 429); + assert.equal((await send(f.port)).status, 429); await new Promise(resolve => setTimeout(resolve, 300)); f.backend.run = async () => Buffer.from('{}'); - assert.equal((await send(f.socketPath)).status, 200); + assert.equal((await send(f.port)).status, 200); } finally { clients.forEach(client => client.destroy()); await f.close(); @@ -236,7 +240,7 @@ test('duplicate extraction control headers are rejected before worker creation', try { assert.equal( ( - await send(f.socketPath, { + await send(f.port, { 'X-Extraction-Format': ['pdf', 'pdf'], }) ).status, @@ -254,13 +258,95 @@ test('unsupported document content returns a safe failure after cleanup', async throw new ExtractionError('UNSUPPORTED_CONTENT'); }; try { - const response = await send(f.socketPath); + const response = await send(f.port); assert.equal(response.status, 422); assert.deepEqual(JSON.parse(response.body), { error: { code: 'UNSUPPORTED_CONTENT' }, }); assert.deepEqual(f.events, ['create', 'cleanup']); - assert.deepEqual(await readdir(f.root), ['service.sock']); + assert.deepEqual(await readdir(f.root), []); + } finally { + await f.close(); + } +}); + +test('authentication fails before validation, health checks or admission', async () => { + const f = await fixture(); + f.backend.healthy = false; + try { + for (const authorization of [ + undefined, + '', + `Bearer ${'b'.repeat(64)}`, + `Bearer ${'A'.repeat(64)}`, + [`Bearer ${'a'.repeat(64)}`, `Bearer ${'a'.repeat(64)}`], + ]) { + for (const path of ['/v1/capabilities', '/v1/extract-text']) { + const response = await send( + f.port, + { + Authorization: authorization, + 'X-Extraction-Format': 'invalid', + }, + path + ); + assert.equal(response.status, 401); + assert.deepEqual(JSON.parse(response.body), { + error: { code: 'UNAUTHORIZED' }, + }); + } + } + assert.deepEqual(f.events, []); + const response = await new Promise((resolve, reject) => { + const req = request( + { + host: '127.0.0.1', + port: f.port, + path: '/v1/capabilities', + headers: { Authorization: headers.Authorization }, + }, + res => { + res.resume(); + res.on('end', () => resolve(res.statusCode!)); + } + ); + req.on('error', reject); + req.end(); + }); + assert.equal(response, 503); + } finally { + await f.close(); + } +}); + +test('unauthenticated expectations and stalled bodies close without admission', async () => { + const f = await fixture(); + try { + for (const expectation of [undefined, '100-continue', 'unsupported']) { + await new Promise((resolve, reject) => { + const req = request( + { + host: '127.0.0.1', + port: f.port, + method: 'POST', + path: '/v1/extract-text', + headers: { + 'Content-Length': '1000', + ...(expectation ? { Expect: expectation } : {}), + }, + }, + res => { + assert.equal(res.statusCode, 401); + assert.equal(res.headers.connection, 'close'); + res.resume(); + res.once('end', resolve); + } + ); + req.on('error', reject); + req.flushHeaders(); + }); + } + assert.deepEqual(f.events, []); } finally { await f.close(); } diff --git a/packages/extraction/src/server.ts b/packages/extraction/src/server.ts index b3bd37c8..54428438 100644 --- a/packages/extraction/src/server.ts +++ b/packages/extraction/src/server.ts @@ -1,4 +1,4 @@ -import { createHash } from 'node:crypto'; +import { createHash, timingSafeEqual } from 'node:crypto'; import { createServer } from 'node:http'; import { open } from 'node:fs/promises'; import { join } from 'node:path'; @@ -95,8 +95,31 @@ function requestFormat(req: IncomingMessage): Format { } export function extractionServer( backend: Backend, + token: string, deadlineMs: number = policy.deadlineMs ) { + if (!/^[a-f0-9]{64}$/.test(token)) throw new Error('INVALID_TOKEN'); + const expected = Buffer.from(`Bearer ${token}`, 'ascii'); + function authorized(req: IncomingMessage): boolean { + let count = 0; + for (let i = 0; i < req.rawHeaders.length; i += 2) + if (req.rawHeaders[i]!.toLowerCase() === 'authorization') count++; + const value = req.headers.authorization; + if ( + count !== 1 || + typeof value !== 'string' || + value.length !== expected.length + ) + return false; + const actual = Buffer.from(value, 'utf8'); + return ( + actual.length === expected.length && + timingSafeEqual(actual, expected) + ); + } + function unauthorized(res: ServerResponse): void { + reply(res, 401, Buffer.from('{"error":{"code":"UNAUTHORIZED"}}')); + } if ( !Number.isInteger(deadlineMs) || deadlineMs <= 0 || @@ -120,12 +143,18 @@ export function extractionServer( server.requestTimeout = policy.deadlineMs; server.setTimeout(policy.deadlineMs, socket => socket.destroy()); server.on('checkContinue', (req, res) => { + if (!authorized(req)) return unauthorized(res); + reply(res, 400, Buffer.from('{"error":{"code":"INVALID_REQUEST"}}')); + }); + server.on('checkExpectation', (req, res) => { + if (!authorized(req)) return unauthorized(res); reply(res, 400, Buffer.from('{"error":{"code":"INVALID_REQUEST"}}')); }); async function handle( req: IncomingMessage, res: ServerResponse ): Promise { + if (!authorized(req)) return unauthorized(res); if (req.method === 'GET' && req.url === '/v1/capabilities') { reply( res, diff --git a/packages/extraction/tests/block-landlock.c b/packages/extraction/tests/block-landlock.c index 1185ec95..c4f5fbee 100644 --- a/packages/extraction/tests/block-landlock.c +++ b/packages/extraction/tests/block-landlock.c @@ -7,15 +7,18 @@ #include /* Negative startup probe: add a restriction to Docker's existing filter. */ -int main(void) { +int main(int argc, char **argv) { + (void)argv; + int gateway = argc == 2; struct sock_filter filter[] = { BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), - BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, SYS_landlock_create_ruleset, 0, 1), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, gateway ? SYS_prctl : SYS_landlock_create_ruleset, 0, 1), BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | 38), BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW) }; struct sock_fprog program = {.len = sizeof(filter) / sizeof(filter[0]), .filter = filter}; if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) || prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &program)) return 125; - execl("/usr/local/bin/node", "node", "/opt/extraction/dist/main.js", (char *)NULL); + if (gateway) execl("/opt/extraction/gateway", "gateway", "node", "dist/main.js", (char *)NULL); + else execl("/usr/local/bin/node", "node", "/opt/extraction/dist/main.js", (char *)NULL); return 125; } diff --git a/packages/extraction/tests/container-client.mjs b/packages/extraction/tests/container-client.mjs index 14e70f74..d053a9e9 100644 --- a/packages/extraction/tests/container-client.mjs +++ b/packages/extraction/tests/container-client.mjs @@ -3,21 +3,27 @@ import { readFileSync } from 'node:fs'; import { request } from 'node:http'; import { createHash } from 'node:crypto'; -const socketPath = '/socket/extraction.sock'; +const host = process.env.EXTRACTION_HOST ?? 'extraction'; +const token = readFileSync('/run/extraction/token', 'ascii'); function send(format, bytes, path = '/v1/extract-text', extra = {}) { return new Promise((resolve, reject) => { const req = request( { - socketPath, + host, + port: 8000, path, - method: 'POST', - headers: { - 'Content-Type': 'application/octet-stream', - 'Content-Length': bytes.length, - 'X-Extraction-Version': '1', - 'X-Extraction-Format': format, - ...extra, - }, + method: path === '/v1/capabilities' ? 'GET' : 'POST', + family: 4, + headers: Object.fromEntries( + Object.entries({ + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/octet-stream', + 'Content-Length': bytes.length, + 'X-Extraction-Version': '1', + 'X-Extraction-Format': format, + ...extra, + }).filter(([, value]) => value !== undefined) + ), }, res => { const chunks = []; @@ -37,6 +43,15 @@ function send(format, bytes, path = '/v1/extract-text', extra = {}) { req.end(bytes); }); } +for (const Authorization of [undefined, `Bearer ${'0'.repeat(64)}`]) { + for (const path of ['/v1/capabilities', '/v1/extract-text']) { + const response = await send('pdf', Buffer.from('test'), path, { + Authorization, + }); + assert.equal(response.status, 401); + assert.deepEqual(response.value, { error: { code: 'UNAUTHORIZED' } }); + } +} const fixtures = '/tests/fixtures'; const pages = readFileSync(`${fixtures}/pages.pdf`); const result = await send('pdf', pages); diff --git a/packages/extraction/tests/container.sh b/packages/extraction/tests/container.sh index 839670c5..520b7805 100755 --- a/packages/extraction/tests/container.sh +++ b/packages/extraction/tests/container.sh @@ -13,40 +13,65 @@ cleanup() { trap cleanup EXIT "${compose[@]}" build extraction docker build --target qualification -t "$image" -f packages/extraction/Dockerfile . -"${compose[@]}" up -d --wait --wait-timeout 30 extraction -id=$("${compose[@]}" ps -q extraction) -volume="${project}_extraction-socket" -# Inspect exactly the shipped recipe, not a relaxed test variant. -docker inspect "$id" | python3 -c ' +"${compose[@]}" up -d --scale extraction=2 --wait --wait-timeout 30 extraction +mapfile -t ids < <("${compose[@]}" ps -q extraction) +[ "${#ids[@]}" -eq 2 ] +volume="${project}_extraction-token" +network="${project}_extraction" +docker network inspect "$network" | python3 -c 'import json,sys; assert json.load(sys.stdin)[0]["Internal"]' +for id in "${ids[@]}"; do + # Inspect the shipped recipe, not a relaxed test variant. + docker inspect "$id" | python3 -c ' import json,sys c=json.load(sys.stdin)[0]; h=c["HostConfig"] -assert h["NetworkMode"] == "none" and h["ReadonlyRootfs"] and not h["Privileged"] +assert h["ReadonlyRootfs"] and not h["Privileged"] assert h["CapDrop"] == ["ALL"] and h["PidsLimit"] == 32 assert h["Memory"] == 768*1024*1024 and h["NanoCpus"] == 2*10**9 assert "no-new-privileges:true" in h["SecurityOpt"] assert c["Config"]["User"] == "10001:10001" -assert len(c["Mounts"]) == 1 and c["Mounts"][0]["Destination"] == "/socket" +assert c["Config"]["Entrypoint"] == ["/opt/extraction/gateway"] +assert len(c["NetworkSettings"]["Networks"]) == 1 and not h["PortBindings"] +assert len(c["Mounts"]) == 1 and c["Mounts"][0]["Destination"] == "/run/extraction" and not c["Mounts"][0]["RW"] assert set(h["Tmpfs"]) == {"/jobs"} for flag in ("noexec", "nosuid", "nodev", "size=96m", "nr_inodes=1024", "uid=10001", "gid=10001", "mode=0700"): assert flag in h["Tmpfs"]["/jobs"] -assert not c["Config"]["ExposedPorts"] if "ExposedPorts" in c["Config"] else True -print("Shipped Compose mounts, resource limits and security options verified")' -docker run --rm --init --network none --cap-drop ALL --security-opt no-new-privileges \ - --mount "type=volume,src=$volume,dst=/socket,readonly" "$image" \ - sh -c 'python /tests/adversarial.py && node /tests/container-client.mjs' -docker exec "$id" node -e 'require("node:fs").readdir("/jobs",(e,v)=>{if(e||v.length)process.exit(1); console.log("Endpoint scratch cleanup verified")})' -# Same outer boundary, with a test-only worker that attempts forbidden operations. +assert not c["Config"].get("ExposedPorts") +print("Shipped Compose mounts, private network, resource limits and security options verified")' + docker exec "$id" /opt/extraction/gateway node dist/health.js + docker exec "$id" node -e 'require("node:fs").readdir("/jobs",(e,v)=>{if(e||v.length)process.exit(1)})' +done +# One-shot init is idempotent; arbitrary caller UIDs can read but not write. +before=$(docker exec "${ids[0]}" node -e 'console.log(require("node:fs").statSync("/run/extraction/token").ino)') +"${compose[@]}" run --rm --no-deps extraction-init +after=$(docker exec "${ids[0]}" node -e 'console.log(require("node:fs").statSync("/run/extraction/token").ino)') +[ "$before" = "$after" ] +docker run --rm --user 12345:12345 --network none --cap-drop ALL --security-opt no-new-privileges \ + --mount "type=volume,src=$volume,dst=/run/extraction,readonly" "$image" \ + node --input-type=module -e 'import {readToken} from "/opt/extraction/dist/credential.js"; await readToken(); console.log("Cross-UID credential read verified")' +docker run --rm --init --network "$network" --cap-drop ALL --security-opt no-new-privileges \ + --mount "type=volume,src=$volume,dst=/run/extraction,readonly" "$image" \ + sh -c 'python /tests/gateway.py && node --input-type=module -e '\''import {lookup, resolve4} from "node:dns/promises"; const a=await lookup("extraction", {all:true, family:4}); const b=await resolve4("extraction"); if(a.length !== 2 || b.length !== 2) process.exit(1); console.log("Node IPv4 libc and c-ares DNS qualified for two replicas")'\'' && python /tests/adversarial.py && node /tests/container-client.mjs' +for id in "${ids[@]}"; do + docker exec "$id" node -e 'require("node:fs").readdir("/jobs",(e,v)=>{if(e||v.length)process.exit(1); console.log("Endpoint scratch cleanup verified")})' +done +# Same outer boundary; the worker independently exercises its stronger filter. docker run --rm --init --network none --cap-drop ALL --security-opt no-new-privileges \ --read-only --memory 768m --cpus 2 --pids-limit 32 \ --tmpfs /jobs:rw,noexec,nosuid,nodev,size=96m,nr_inodes=1024,uid=10001,gid=10001,mode=0700 \ - --tmpfs /socket:rw,noexec,nosuid,nodev,size=1m,uid=10001,gid=10001,mode=0700 \ + --mount "type=volume,src=$volume,dst=/run/extraction,readonly" \ "$image" node /tests/isolation.mjs -# Denying the mandatory isolation syscall must fail closed, never start unsandboxed. +# A test controller without the gateway supplies socket stdio deliberately. +# Both native launchers must reject/remove it before running their target. +docker run --rm --init --network none --cap-drop ALL --security-opt no-new-privileges \ + --read-only --memory 768m --cpus 2 --pids-limit 32 \ + --tmpfs /jobs:rw,noexec,nosuid,nodev,size=96m,uid=10001,gid=10001,mode=0700 \ + --entrypoint node "$image" /tests/stdio.mjs +# Mandatory isolation unavailable: fail closed, never start unsandboxed. set +e docker run --rm --init --network none --cap-drop ALL --security-opt no-new-privileges \ --read-only --memory 768m --cpus 2 --pids-limit 32 \ --tmpfs /jobs:rw,noexec,nosuid,nodev,size=96m,uid=10001,gid=10001,mode=0700 \ - --tmpfs /socket:rw,noexec,nosuid,nodev,size=1m,uid=10001,gid=10001,mode=0700 \ + --mount "type=volume,src=$volume,dst=/run/extraction,readonly" \ "$image" /tests/block-landlock > /dev/null 2>&1 blocked=$? set -e @@ -55,7 +80,17 @@ if [ "$blocked" -ne 1 ]; then exit 1 fi printf 'Unavailable isolation fails closed\n' -# A stale socket left by SIGKILL must not prevent a safe restart. -docker kill -s KILL "$id" >/dev/null -"${compose[@]}" up -d --wait --wait-timeout 30 extraction -printf 'Hardened container restart passed\n' +set +e +docker run --rm --init --network none --cap-drop ALL --security-opt no-new-privileges \ + --read-only --memory 128m --pids-limit 16 \ + "$image" /tests/block-landlock gateway > /dev/null 2>&1 +blocked=$? +set -e +[ "$blocked" -eq 125 ] +printf 'Unavailable pre-runtime gateway fails closed\n' +# Restart uses the same credential and fresh per-replica scratch. +for id in "${ids[@]}"; do docker kill -s KILL "$id" >/dev/null; done +"${compose[@]}" up -d --scale extraction=2 --wait --wait-timeout 30 extraction +after=$(docker exec "${ids[0]}" node -e 'console.log(require("node:fs").statSync("/run/extraction/token").ino)') +[ "$before" = "$after" ] +printf 'Hardened two-replica restart and credential persistence passed\n' diff --git a/packages/extraction/tests/gateway.py b/packages/extraction/tests/gateway.py new file mode 100644 index 00000000..0d947345 --- /dev/null +++ b/packages/extraction/tests/gateway.py @@ -0,0 +1,21 @@ +"""Probe the pre-runtime guard independently of the stronger worker filter.""" +import ctypes +import errno +import socket +import os +import stat + +libc = ctypes.CDLL(None, use_errno=True) +for fd in (0, 1, 2): + assert not stat.S_ISSOCK(os.fstat(fd).st_mode) +for name, args in [ + ('socket', (socket.AF_UNIX, socket.SOCK_STREAM, 0)), + ('socketpair', (socket.AF_UNIX, socket.SOCK_STREAM, 0, 0)), + ('sendmsg', (-1, 0, 0)), ('recvmsg', (-1, 0, 0)), + ('sendmmsg', (-1, 0, 0, 0)), ('recvmmsg', (-1, 0, 0, 0, 0)), +]: + assert getattr(libc, name)(*args) == -1, name + assert ctypes.get_errno() == errno.EPERM, (name, ctypes.get_errno()) +with socket.socket(socket.AF_INET, socket.SOCK_STREAM): + pass +print('Pre-runtime guard: AF_UNIX, socketpair and descriptor-message syscalls denied; IP available') diff --git a/packages/extraction/tests/isolation.mjs b/packages/extraction/tests/isolation.mjs index d44ef800..fb93cef9 100644 --- a/packages/extraction/tests/isolation.mjs +++ b/packages/extraction/tests/isolation.mjs @@ -1,31 +1,42 @@ import assert from 'node:assert/strict'; +import { + mkdir, + mkdtemp, + writeFile, + rm, + readdir, + open, + readFile, +} from 'node:fs/promises'; +import { constants } from 'node:fs'; import { spawn } from 'node:child_process'; -import { mkdir, mkdtemp, writeFile, rm, readdir } from 'node:fs/promises'; import { once } from 'node:events'; import { RestrictedBackend } from '/opt/extraction/dist/backend.js'; await mkdir('/jobs/other'); await writeFile('/jobs/other/secret', 'cross-job-canary'); -await writeFile('/socket/secret', 'transport-canary'); + const job = await mkdtemp('/jobs/job-'); +const status = await open( + `${job}/status`, + constants.O_RDWR | + constants.O_CREAT | + constants.O_EXCL | + constants.O_NOFOLLOW, + 0o600 +); const child = spawn( '/opt/extraction/guard', [job, '/usr/local/bin/python3.13', '/tests/isolation.py', 'probe'], { env: { EXTRACTION_SECRET_CANARY: 'private-canary' }, - stdio: ['ignore', 'pipe', 'pipe'], + stdio: ['ignore', status.fd, 'ignore'], } ); -let output = '', - errors = ''; -child.stdout.on('data', bytes => { - output += bytes; -}); -child.stderr.on('data', bytes => { - errors += bytes; -}); const [code] = await once(child, 'close'); -assert.equal(code, 0, errors); +await status.close(); +const output = await readFile(`${job}/status`, 'ascii'); +assert.equal(code, 0, output); assert.equal(output.trim(), 'ISOLATED'); await rm(job, { recursive: true }); const runtime = { @@ -45,13 +56,19 @@ for (const abort of ['immediate', 'running']) { await assert.rejects(pending); await backend.cleanup(job); } +const flood = await backend.create(); +await assert.rejects( + backend.run(flood, 'flood', 'a'.repeat(64), AbortSignal.timeout(1000)), + /UNAVAILABLE/ +); +await backend.cleanup(flood); const job2 = await backend.create(); await assert.rejects( backend.run(job2, 'hang', 'a'.repeat(64), AbortSignal.timeout(100)) ); await backend.cleanup(job2); await rm('/jobs/other', { recursive: true }); -await rm('/socket/secret'); + assert.deepEqual(await readdir('/jobs'), []); const missing = new RestrictedBackend({ ...runtime, guard: '/nonexistent' }); const job3 = await missing.create(); diff --git a/packages/extraction/tests/isolation.py b/packages/extraction/tests/isolation.py index a50d0084..e46119b4 100644 --- a/packages/extraction/tests/isolation.py +++ b/packages/extraction/tests/isolation.py @@ -6,15 +6,44 @@ import signal import socket import struct +import stat import sys +for fd in (0, 2): + assert stat.S_ISCHR(os.fstat(fd).st_mode) + assert os.fstat(fd).st_rdev == os.stat('/dev/null').st_rdev +assert stat.S_ISREG(os.fstat(1).st_mode) +for fd in range(3, 64): + try: + os.fstat(fd) + except OSError as error: + assert error.errno == errno.EBADF + else: + raise RuntimeError('Inherited descriptor') + +libc = ctypes.CDLL(None, use_errno=True) +# Invalid descriptors would return EBADF without filtering. No messages are sent. +for name, args in [ + ('socketpair', (socket.AF_UNIX, socket.SOCK_STREAM, 0, 0)), + ('sendmsg', (-1, 0, 0)), ('recvmsg', (-1, 0, 0)), + ('sendmmsg', (-1, 0, 0, 0)), ('recvmmsg', (-1, 0, 0, 0, 0)), + ('sendto', (-1, 0, 0, 0, 0, 0)), ('recvfrom', (-1, 0, 0, 0, 0, 0)), + ('setsockopt', (-1, 0, 0, 0, 0)), ('getsockopt', (-1, 0, 0, 0, 0)), + ('sendmsg', (1, 0, 0)), ('recvmsg', (1, 0, 0)), +]: + assert getattr(libc, name)(*args) == -1, name + assert ctypes.get_errno() == errno.EPERM, (name, ctypes.get_errno()) + +if sys.argv[1] == 'flood': + os.write(1, b'x' * 2048) + sys.exit(0) if sys.argv[1] == 'hang': # Exercise supervisor cancellation/deadline termination, not a cooperative parser. while True: pass for path, mode in [('/jobs/other/secret', 'rb'), ('/jobs/other/secret', 'wb'), - ('/socket/secret', 'rb'), ('/proc/self/environ', 'rb'), ('/etc/passwd', 'rb')]: + ('/run/extraction/token', 'rb'), ('/proc/self/environ', 'rb'), ('/etc/passwd', 'rb')]: try: with open(path, mode) as file: if mode == 'rb': diff --git a/packages/extraction/tests/stdio.mjs b/packages/extraction/tests/stdio.mjs new file mode 100644 index 00000000..d6260d98 --- /dev/null +++ b/packages/extraction/tests/stdio.mjs @@ -0,0 +1,31 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { mkdtemp, rm } from 'node:fs/promises'; + +const job = await mkdtemp('/jobs/job-'); +try { + const worker = spawn( + '/opt/extraction/guard', + [job, '/usr/local/bin/python3.13', '/tests/isolation.py', 'probe'], + { stdio: ['pipe', 'pipe', 'pipe'] } + ); + worker.stdin.end(); + worker.stdout.resume(); + worker.stderr.resume(); + assert.equal((await once(worker, 'close'))[0], 125); + const gateway = spawn( + '/opt/extraction/gateway', + ['python', '/tests/gateway.py'], + { stdio: ['pipe', 'pipe', 'pipe'] } + ); + gateway.stdin.end(); + gateway.stdout.resume(); + gateway.stderr.resume(); + assert.equal((await once(gateway, 'close'))[0], 0); + console.log( + 'Socket standard IO rejected by worker and removed by pre-runtime gateway' + ); +} finally { + await rm(job, { recursive: true }); +} diff --git a/packages/extraction/worker/extract.py b/packages/extraction/worker/extract.py index 20f0360f..65022025 100644 --- a/packages/extraction/worker/extract.py +++ b/packages/extraction/worker/extract.py @@ -434,9 +434,9 @@ def probe(): import docx import lxml.etree - for path in ("/socket", "/jobs", "/proc/self/environ", "/etc/passwd"): + for path in ("/run/extraction", "/jobs", "/proc/self/environ", "/etc/passwd"): try: - if path in ("/socket", "/jobs"): + if path in ("/run/extraction", "/jobs"): os.listdir(path) else: open(path, "rb").read(1) diff --git a/service/Dockerfile b/service/Dockerfile index d7bf9036..fae912a0 100644 --- a/service/Dockerfile +++ b/service/Dockerfile @@ -1,5 +1,5 @@ # File Server Dockerfile -FROM oven/bun:1.4.2 AS base +FROM oven/bun:1.4.4 AS base WORKDIR /app # Install dependencies @@ -27,7 +27,7 @@ RUN bun build ./src/worker-server.ts --minify --outdir .build-worker --target bu RUN bun build ./src/egress-gateway.ts --minify --outdir .build-egress-gateway --target bun --external '@opentelemetry/*' # File server production -FROM oven/bun:1.4.2 AS production +FROM oven/bun:1.4.4 AS production ENV NODE_ENV=production WORKDIR /app COPY --from=install /temp/prod/node_modules ./node_modules @@ -35,7 +35,7 @@ COPY --from=builder /app/.build ./.build CMD ["bun", "run", ".build/file-server.js"] # API server (HTTP on port 3112) -FROM oven/bun:1.4.2 AS api +FROM oven/bun:1.4.4 AS api ENV NODE_ENV=production WORKDIR /app COPY --from=install /temp/prod/node_modules ./node_modules @@ -45,7 +45,7 @@ COPY --from=builder /app/src/*.py ./src/ CMD ["bun", "run", ".build-api/api-server.js"] # Worker server (job processor, health on port 3113) -FROM oven/bun:1.4.2 AS worker +FROM oven/bun:1.4.4 AS worker ENV NODE_ENV=production WORKDIR /app COPY --from=install /temp/prod/node_modules ./node_modules @@ -54,12 +54,12 @@ COPY --from=builder /app/src/*.py ./src/ CMD ["bun", "run", ".build-worker/worker-server.js"] # Egress gateway (sandbox outbound delegation) -FROM oven/bun:1.4.2 AS egress-gateway +FROM oven/bun:1.4.4 AS egress-gateway ENV NODE_ENV=production WORKDIR /app COPY --from=install /temp/prod/node_modules ./node_modules COPY --from=builder /app/.build-egress-gateway ./.build-egress-gateway -CMD ["bun", "run", ".build-egress-gateway/egress-gateway.js"] +CMD ["bun", "run", "--max-http-header-size=262144", ".build-egress-gateway/egress-gateway.js"] # Development stage FROM base AS development diff --git a/service/Dockerfile.api b/service/Dockerfile.api index 1da68c46..6a571db8 100644 --- a/service/Dockerfile.api +++ b/service/Dockerfile.api @@ -1,7 +1,7 @@ # API-Only Server Dockerfile # This builds the HTTP API server without workers # Scale this based on HTTP traffic -FROM oven/bun:1.4.2 AS base +FROM oven/bun:1.4.4 AS base WORKDIR /app # Install dependencies @@ -27,7 +27,7 @@ RUN bun build ./src/api-server.ts --minify --outdir .build --target bun --extern RUN bun build ./scripts/rehydrate-session-cache.ts --minify --outdir .build-migrations --target bun --external '@opentelemetry/*' # Production stage -FROM oven/bun:1.4.2 AS production +FROM oven/bun:1.4.4 AS production ENV NODE_ENV=production WORKDIR /app # Install curl for healthcheck (not included in bun base image) diff --git a/service/Dockerfile.bun b/service/Dockerfile.bun index d545f32c..941c7183 100644 --- a/service/Dockerfile.bun +++ b/service/Dockerfile.bun @@ -1,5 +1,5 @@ # Base stage -FROM oven/bun:1.4.2 AS base +FROM oven/bun:1.4.4 AS base WORKDIR /usr/src/app # Install dependencies diff --git a/service/Dockerfile.egress-gateway b/service/Dockerfile.egress-gateway index bd1d2b64..9f78fa7a 100644 --- a/service/Dockerfile.egress-gateway +++ b/service/Dockerfile.egress-gateway @@ -1,5 +1,5 @@ # Egress Gateway Dockerfile -FROM oven/bun:1.4.2 AS base +FROM oven/bun:1.4.4 AS base WORKDIR /app FROM base AS install @@ -11,7 +11,7 @@ RUN mkdir -p /temp/prod COPY service/package.json service/bun.lock /temp/prod/ RUN cd /temp/prod && bun install --frozen-lockfile --production -FROM oven/bun:1.4.2 AS production +FROM oven/bun:1.4.4 AS production ENV NODE_ENV=production WORKDIR /app RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* @@ -22,7 +22,7 @@ COPY packages/code/src /packages/code/src EXPOSE 3190 HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ CMD curl -f http://localhost:3190/health || exit 1 -CMD ["bun", "run", "src/egress-gateway.ts"] +CMD ["bun", "run", "--max-http-header-size=262144", "src/egress-gateway.ts"] FROM base AS development ENV NODE_ENV=development @@ -31,4 +31,4 @@ COPY service/src ./src COPY shared /shared COPY packages/code/src /packages/code/src EXPOSE 3190 9230 -CMD ["bun", "run", "--watch", "src/egress-gateway.ts"] +CMD ["bun", "run", "--watch", "--max-http-header-size=262144", "src/egress-gateway.ts"] diff --git a/service/Dockerfile.local b/service/Dockerfile.local index 4ed93ee4..216e9c31 100644 --- a/service/Dockerfile.local +++ b/service/Dockerfile.local @@ -1,5 +1,5 @@ # Local development Dockerfile - no authentication required -FROM oven/bun:1.4.2 AS base +FROM oven/bun:1.4.4 AS base WORKDIR /app # Install dependencies @@ -22,7 +22,7 @@ COPY service/tsconfig.json ./ RUN bun build ./src/local-api.ts --minify --outdir .build --target bun --external '@opentelemetry/*' # Production stage -FROM oven/bun:1.4.2 AS production +FROM oven/bun:1.4.4 AS production ENV NODE_ENV=production WORKDIR /app COPY --from=install /temp/prod/node_modules ./node_modules diff --git a/service/Dockerfile.node b/service/Dockerfile.node index 8a6006e4..dc964fce 100644 --- a/service/Dockerfile.node +++ b/service/Dockerfile.node @@ -1,5 +1,5 @@ # Base stage for shared steps -FROM node:22-slim AS base +FROM node:22.23.4-slim AS base WORKDIR /app COPY service/package*.json ./ RUN npm ci @@ -13,7 +13,7 @@ COPY service/tsconfig.json ./ RUN npx tsc -p tsconfig.json # Production stage -FROM node:22-slim AS production +FROM node:22.23.4-slim AS production ENV NODE_ENV=production WORKDIR /app COPY service/package*.json ./ diff --git a/service/Dockerfile.service b/service/Dockerfile.service index d5d4e6d9..dea1cea3 100644 --- a/service/Dockerfile.service +++ b/service/Dockerfile.service @@ -1,5 +1,5 @@ # Service API Dockerfile -FROM oven/bun:1.4.2 AS base +FROM oven/bun:1.4.4 AS base WORKDIR /usr/src/app # Install dependencies diff --git a/service/Dockerfile.tool-call-server b/service/Dockerfile.tool-call-server index 76afb837..95543ee7 100644 --- a/service/Dockerfile.tool-call-server +++ b/service/Dockerfile.tool-call-server @@ -1,5 +1,5 @@ # Tool Call Server Dockerfile -FROM oven/bun:1.4.2 AS base +FROM oven/bun:1.4.4 AS base WORKDIR /app # Install dependencies @@ -13,7 +13,7 @@ COPY service/package.json service/bun.lock /temp/prod/ RUN cd /temp/prod && bun install --frozen-lockfile --production # Production stage -FROM oven/bun:1.4.2 AS production +FROM oven/bun:1.4.4 AS production ENV NODE_ENV=production WORKDIR /app COPY --from=install /temp/prod/node_modules ./node_modules diff --git a/service/Dockerfile.worker b/service/Dockerfile.worker index bb7b6945..059ce2aa 100644 --- a/service/Dockerfile.worker +++ b/service/Dockerfile.worker @@ -2,7 +2,7 @@ # This builds the job processing worker without HTTP server # Deploy alongside a sandbox sidecar for execution # Scale this based on queue depth -FROM oven/bun:1.4.2 AS base +FROM oven/bun:1.4.4 AS base WORKDIR /app # Install dependencies @@ -25,7 +25,7 @@ COPY service/tsconfig.json ./ RUN bun build ./src/worker-server.ts --minify --outdir .build --target bun --external '@opentelemetry/*' # Production stage -FROM oven/bun:1.4.2 AS production +FROM oven/bun:1.4.4 AS production ENV NODE_ENV=production WORKDIR /app # Install curl for healthcheck diff --git a/service/bun.lock b/service/bun.lock index 3f3afb59..f42e2a8e 100644 --- a/service/bun.lock +++ b/service/bun.lock @@ -53,6 +53,7 @@ }, "overrides": { "decode-uri-component": "0.5.0", + "proxy-addr": "2.0.8", "qs": "6.16.0", "uuid": "11.1.1", }, @@ -853,7 +854,7 @@ "prom-client": ["prom-client@15.1.3", "", { "dependencies": { "@opentelemetry/api": "^1.4.0", "tdigest": "^0.1.1" } }, "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g=="], - "proxy-addr": ["proxy-addr@2.0.7", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg=="], + "proxy-addr": ["proxy-addr@2.0.8", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-5nnx0yGyVUcY6t9RnWcARWtwT9F1D8O9rt08htPvnd49W1IgZtmLkhu9WfMzQj1cFxjHIO6connUNVW5k7AVyQ=="], "proxy-from-env": ["proxy-from-env@2.1.0", "", {}, "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA=="], diff --git a/service/openapi.yml b/service/openapi.yml index 07d2d7e3..929fab71 100644 --- a/service/openapi.yml +++ b/service/openapi.yml @@ -72,7 +72,12 @@ components: protocolVersion: { type: integer, enum: [1] } operation: type: string - enum: [list_files, search_text, read_file, write_file, preview_edit, edit_file, execute_command] + enum: [list_files, search_text, read_file, write_file, preview_edit, edit_file, execute_command, import_file] + description: >- + `import_file` copies a stored object the caller owns into the + workspace: `path`, `source` ({ storage_session_id, id }), `bytes`, + lowercase hex `sha256`, optional `overwrite`. Send it only when the + worker advertises it. Callers never send `egressGrant`. workspaceId: { type: string } additionalProperties: true diff --git a/service/package.json b/service/package.json index 80daea5e..f9b8ffde 100644 --- a/service/package.json +++ b/service/package.json @@ -14,7 +14,7 @@ "start:api:dev": "ts-node src/api-server.ts", "start:worker": "node .build-service/src/worker-server.js", "start:worker:dev": "ts-node src/worker-server.ts", - "start:egress-gateway": "bun run src/egress-gateway.ts", + "start:egress-gateway": "bun run --max-http-header-size=262144 src/egress-gateway.ts", "build": "rollup -c", "deploy": "serverless deploy", "test": "bun test ./src/*.test.ts ./src/*.spec.ts ./src/**/*.test.ts ./src/**/*.spec.ts", @@ -77,6 +77,7 @@ "overrides": { "decode-uri-component": "0.5.0", "uuid": "11.1.1", - "qs": "6.16.0" + "qs": "6.16.0", + "proxy-addr": "2.0.8" } } diff --git a/service/src/bridge/enabled.ts b/service/src/bridge/enabled.ts index 540a844a..e0cba569 100644 --- a/service/src/bridge/enabled.ts +++ b/service/src/bridge/enabled.ts @@ -7,3 +7,8 @@ export function isBridgeEnabled(): boolean { || env.BRIDGE_DYNAMIC_WORKERS || env.BRIDGE_TOKEN.length > 0; } + +/** `import_file` relays need the egress gateway to mint each one-object read grant. */ +export function isWorkspaceImportEnabled(): boolean { + return env.EGRESS_GATEWAY_URL.trim() !== ''; +} diff --git a/service/src/bridge/index.ts b/service/src/bridge/index.ts index f7ba03b4..93370dfa 100644 --- a/service/src/bridge/index.ts +++ b/service/src/bridge/index.ts @@ -3,7 +3,7 @@ import { env } from '../config'; import { RedisBridgePairingStore } from './pairing'; import { createBridgeRouter } from './router'; import { RedisBridgeStore } from './store'; -import { isBridgeEnabled } from './enabled'; +import { isBridgeEnabled, isWorkspaceImportEnabled } from './enabled'; export const bridgeStore = new RedisBridgeStore( connection, @@ -39,4 +39,5 @@ export default createBridgeRouter({ configuredWorkerId: env.BRIDGE_WORKER_ID, allowDynamicWorkers: env.BRIDGE_DYNAMIC_WORKERS, maxCommandTimeoutMs: env.JOB_TIMEOUT, + workspaceImportFile: isWorkspaceImportEnabled(), }); diff --git a/service/src/bridge/router.test.ts b/service/src/bridge/router.test.ts index 5e76876e..a478f623 100644 --- a/service/src/bridge/router.test.ts +++ b/service/src/bridge/router.test.ts @@ -216,6 +216,53 @@ describe('paired bridge HTTP API', () => { }); } + for (const workspaceImportFile of [false, true]) { + test(`registration ${workspaceImportFile ? 'offers' : 'omits'} import_file when grants ${workspaceImportFile ? 'can' : 'cannot'} be minted`, async () => { + const app = express(); + app.use(json()); + app.use( + '/v1/bridge', + createBridgeRouter({ + store: new RedisBridgeStore(redis), + pairings: new RedisBridgePairingStore(redis), + authMode: 'static', + adminToken: 'strong-administrator-bootstrap-token', + configuredWorkerId: 'vm-1', + workspaceImportFile, + }), + ); + server = createServer(app); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Expected TCP listener'); + const response = await fetch(`http://127.0.0.1:${address.port}/v1/bridge/workers/register`, { + method: 'POST', + headers: { + Authorization: 'Bearer strong-administrator-bootstrap-token', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['read_file', 'write_file', 'import_file'], + workspaces: [{ id: 'primary' }], + }, + }, + }), + }); + expect(response.status).toBe(200); + const body = (await response.json()) as { supportedWorkspaceToolOperations: string[] }; + expect(body.supportedWorkspaceToolOperations.includes('import_file')).toBe(workspaceImportFile); + }); + } + test('rejects a malformed optional binding for a configured worker', async () => { const app = express(); app.use(json()); diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index d035b032..c3b5aead 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -45,6 +45,8 @@ export interface BridgeRouterOptions { configuredWorkerId?: string; allowDynamicWorkers?: boolean; maxCommandTimeoutMs?: number; + /** Offer `import_file`; true only when this Code API can mint its egress grant. */ + workspaceImportFile?: boolean; } function sameToken(left: string, right: string): boolean { @@ -606,6 +608,7 @@ router.post( 'preview_edit', 'edit_file', 'execute_command', + ...(options.workspaceImportFile === true ? ['import_file' as const] : []), ], supportedWorkspaceWriteFileModes: ['replace', 'create'], supportedWorkspaceEditFileModes: ['single', 'batch'], diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 0ca74c45..f9919f49 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -15,6 +15,7 @@ import { BRIDGE_PROTOCOL_VERSION, isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, + isWorkspaceMutationOperation, isWorkspaceToolRequest, isWorkspaceToolResult, workspaceIsolationKey, @@ -169,7 +170,7 @@ export interface BridgeWorkerStatus { } function isWorkspaceMutation(request: WorkspaceToolRequest): boolean { - return request.operation === 'write_file' || request.operation === 'edit_file' || request.operation === 'execute_command'; + return isWorkspaceMutationOperation(request.operation); } function supportsWorkspaceTool( diff --git a/service/src/config.spec.ts b/service/src/config.spec.ts index 3c2ca9d2..e5af81cf 100644 --- a/service/src/config.spec.ts +++ b/service/src/config.spec.ts @@ -1,4 +1,6 @@ import { describe, expect, it } from 'bun:test'; +import { readFileSync } from 'fs'; +import * as path from 'path'; import { jobCompletionWaitTimeoutMs, jobDeadlineAtMs, @@ -22,11 +24,22 @@ describe('node language configuration', () => { it('maps Node.js requests to the node sandbox runtime', () => { expect(languageConfig[Languages.node]).toEqual({ language: 'node', - version: '24.15.0', + version: '24.22.1', fileName: 'index.js', }); }); + it('aligns package installers and Helm with the selected Node runtime', () => { + const root = path.resolve(__dirname, '../..'); + const version = languageConfig[Languages.node]!.version; + for (const file of ['build-packages.sh', 'docker/package-init.sh']) { + expect(readFileSync(path.join(root, file), 'utf8')) + .toContain(`NODE_VERSION="\${NODE_VERSION:-${version}}"`); + } + expect(readFileSync(path.join(root, 'helm/codeapi/values.yaml'), 'utf8')) + .toContain(`nodeVersion: "${version}"`); + }); + it('creates a valid Node.js execution payload', () => { const req = { body: { @@ -45,7 +58,7 @@ describe('node language configuration', () => { expect(payload).toMatchObject({ language: 'node', - version: '24.15.0', + version: '24.22.1', session_id: 'session-node', args: ['--trace-warnings'], files: [ @@ -67,15 +80,15 @@ describe('runtime version configuration', () => { }); }); - it('maps Bun JavaScript and TypeScript requests to Bun 1.4.2', () => { + it('maps Bun JavaScript and TypeScript requests to Bun 1.4.4', () => { expect(languageConfig[Languages.js]).toMatchObject({ language: 'bun-js', - version: '1.4.2', + version: '1.4.4', fileName: 'index.js', }); expect(languageConfig[Languages.ts]).toMatchObject({ language: 'bun-ts', - version: '1.4.2', + version: '1.4.4', fileName: 'main.ts', }); }); diff --git a/service/src/config.ts b/service/src/config.ts index b3e397b8..8abfa54f 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -7,13 +7,14 @@ import { resolveExecutionProfile, resolveExecutionProfileSource, } from './execution-profile'; +import { BRIDGE_WORKSPACE_IMPORT_FILE_DEFAULT_MAX_BYTES } from '../../packages/code/src/protocol'; export const languageConfig: Record = { [Languages.bash]: { language: 'bash', version: '5.2.0', fileName: 'script.sh' }, - [Languages.js]: { language: 'bun-js', version: '1.4.2', fileName: 'index.js' }, - [Languages.node]: { language: 'node', version: '24.15.0', fileName: 'index.js' }, + [Languages.js]: { language: 'bun-js', version: '1.4.4', fileName: 'index.js' }, + [Languages.node]: { language: 'node', version: '24.22.1', fileName: 'index.js' }, [Languages.py]: { language: 'python', version: '3.14.4', fileName: 'main.py' }, - [Languages.ts]: { language: 'bun-ts', version: '1.4.2', fileName: 'main.ts' }, + [Languages.ts]: { language: 'bun-ts', version: '1.4.4', fileName: 'main.ts' }, }; const languageAliases: Record = { @@ -425,6 +426,14 @@ export const env = { * Off by default: enable only after the LibreChat talking to this service accepts the field. */ BRIDGE_LANE_GIT: process.env.CODEAPI_BRIDGE_LANE_GIT === 'true', + /** + * Largest stored object relayed into a worker by `import_file`. The operation + * is offered only when EGRESS_GATEWAY_URL can mint its one-object grant. + */ + BRIDGE_IMPORT_FILE_MAX_BYTES: resolvePositiveIntEnv( + process.env.CODEAPI_BRIDGE_IMPORT_FILE_MAX_BYTES, + BRIDGE_WORKSPACE_IMPORT_FILE_DEFAULT_MAX_BYTES, + ), /** Outbound worker selected by the remote-bridge backend. */ BRIDGE_WORKER_ID: process.env.CODEAPI_BRIDGE_WORKER_ID ?? '', /** Static compatibility auth or short-lived proof-of-possession credentials. */ diff --git a/service/src/egress-headers.test.ts b/service/src/egress-headers.test.ts new file mode 100644 index 00000000..9591acf2 --- /dev/null +++ b/service/src/egress-headers.test.ts @@ -0,0 +1,49 @@ +import { test, expect } from 'bun:test'; +import { mkdtempSync, readFileSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +test('deployed egress command accepts scoped grants above 16 KiB, with a finite cap', async () => { + const dockerfile = readFileSync(new URL('../Dockerfile.egress-gateway', import.meta.url), 'utf8'); + const command: string[] = JSON.parse(dockerfile.match(/^CMD (.+)$/m)![1]); + const bundledDockerfile = readFileSync(new URL('../Dockerfile', import.meta.url), 'utf8'); + const commands: string[][] = [...dockerfile.matchAll(/^CMD (.+)$/gm), + ...bundledDockerfile.matchAll(/^CMD (.+egress-gateway.+)$/gm)] + .map(match => JSON.parse(match[1])); + const packageJson = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')); + commands.push(packageJson.scripts['start:egress-gateway'].split(' ')); + expect(commands).toHaveLength(4); + for (const entrypoint of commands) expect(entrypoint).toContain('--max-http-header-size=262144'); + const probe = ` + const http = require('node:http'); + const server = http.createServer((req, res) => res.end('ok')); + server.listen(0, '127.0.0.1', async () => { + try { + const statuses = []; + for (const size of [32768, 280000]) { + const response = await fetch('http://127.0.0.1:' + server.address().port, { + headers: { 'X-CodeAPI-Egress-Grant': 'a'.repeat(size) }, + }); + statuses.push(response.status); + await response.text(); + } + console.log(JSON.stringify(statuses)); + } finally { server.close(); } + });`; + const directory = mkdtempSync(join(tmpdir(), 'codeapi-header-test-')); + const script = join(directory, 'probe.cjs'); + writeFileSync(script, probe); + const run = async (options: string[]) => { + const child = Bun.spawn([process.execPath, ...options, script], { stdout: 'pipe', stderr: 'pipe' }); + const stdout = await new Response(child.stdout).text(); + const stderr = await new Response(child.stderr).text(); + if (await child.exited !== 0) throw new Error(stderr); + expect(await child.exited).toBe(0); + return JSON.parse(stdout); + }; + try { + expect(await run(['run'])).toEqual([431, 431]); + // Preserve the deployed command's argument order, replacing only the file. + expect(await run(command.slice(1, -1))).toEqual([200, 431]); + } finally { rmSync(directory, { recursive: true }); } +}); diff --git a/service/src/session-ownership.ts b/service/src/session-ownership.ts index bf4ba6dd..1841ed06 100644 --- a/service/src/session-ownership.ts +++ b/service/src/session-ownership.ts @@ -94,7 +94,7 @@ export type SessionOwnershipResult = * simply gone. */ export async function authorizeSessionOwnership( - store: SessionOwnershipStore, + store: Pick, args: { session_id: string; expectedSessionKey: string; diff --git a/service/src/workspace-tools/import.test.ts b/service/src/workspace-tools/import.test.ts new file mode 100644 index 00000000..7aa22aef --- /dev/null +++ b/service/src/workspace-tools/import.test.ts @@ -0,0 +1,295 @@ +import { createServer } from 'node:http'; +import type { Server } from 'node:http'; + +import { afterEach, expect, test } from 'bun:test'; +import express, { json } from 'express'; +import RedisMock from 'ioredis-mock'; +import type Redis from 'ioredis'; + +import type * as t from '../types'; +import type { CodeApiPrincipal } from '../auth/principal'; +import type { WorkspaceImportFileRequest, WorkspaceToolRequest } from '../../../packages/code/src/protocol'; + +import { applyPrincipal } from '../auth/principal'; +import { RedisBridgeStore } from '../bridge/store'; +import { resolveSessionKey } from '../session-key'; +import { recordSessionOwnership } from '../session-ownership'; +import { openEgressGrant, openEgressHandle, prepareSandboxEgress } from '../egress-grant'; +import { createWorkspaceToolsRouter } from './router'; +import { RedisWorkspaceRequests } from './requests'; +import { + WorkspaceImportError, + authorizeWorkspaceImportSource, + createWorkspaceImportService, +} from './import'; + +const SECRET = 'workspace-import-test-secret-of-sufficient-length'; +const SESSION = 'sessAAAAAAAAAAAAAAAAA'; +const OBJECT = 'fileAAAAAAAAAAAAAAAAA'; +const OTHER_SESSION = 'sessBBBBBBBBBBBBBBBBB'; +const OTHER_OBJECT = 'fileBBBBBBBBBBBBBBBBB'; +const owner: CodeApiPrincipal = { userId: 'user-1', tenantId: 'tenant-1', principalSource: 'librechat_jwt', codeWorkerId: 'worker' }; +const stranger: CodeApiPrincipal = { userId: 'user-2', tenantId: 'tenant-1', principalSource: 'librechat_jwt', codeWorkerId: 'worker' }; + +const importBody = (overrides: Record = {}): WorkspaceImportFileRequest => ({ + protocolVersion: 1, + operation: 'import_file', + workspaceId: 'primary', + path: 'uploads/report.pdf', + source: { storage_session_id: SESSION, id: OBJECT }, + bytes: 4096, + sha256: 'a'.repeat(64), + ...overrides, +}); + +function principalRequest(principal: CodeApiPrincipal): t.AuthenticatedRequest { + const req = { headers: {} } as t.AuthenticatedRequest; + applyPrincipal(req, principal); + return req; +} + +/** Uploads as LibreChat's code-environment upload does: user-kind session key plus upload record. */ +async function upload(redis: Redis, principal: CodeApiPrincipal, sessionId: string, objectId: string): Promise { + const sessionKey = resolveSessionKey(principalRequest(principal), { kind: 'user', id: principal.userId }); + await recordSessionOwnership(redis, sessionId, sessionKey); + await redis.set(`upload:${sessionKey}${sessionId}${objectId}`, '1'); +} + +function importService(redis: Redis, minted: string[] = [], revoked: string[] = [], maxBytes = 1024 * 1024) { + return createWorkspaceImportService({ + maxBytes, + store: redis, + async createGrant({ payload, claims }) { + const grantId = `grant-${minted.length + 1}`; + minted.push(grantId); + return { grant_id: grantId, ...prepareSandboxEgress({ payload, claims, grantId, secret: SECRET }) }; + }, + async revokeGrant(grantId) { + revoked.push(grantId); + }, + }); +} + +// ioredis-mock instances share one keyspace across test files; clear it like the bridge suites do. +const redis = new RedisMock() as unknown as Redis; +let server: Server | undefined; +afterEach(async () => { + server?.close(); + server = undefined; + await redis.flushall(); +}); + +async function listen(app: express.Express): Promise { + server = createServer(app); + await new Promise(resolve => server!.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Missing listener'); + return `http://127.0.0.1:${address.port}`; +} + +function appFor(router: express.Router): express.Express { + const app = express(); + app.use(json()); + app.use((req, _res, next) => { + applyPrincipal(req, req.header('X-Test-User') === 'user-2' ? stranger : owner); + next(); + }); + app.use(router); + return app; +} + +function post(url: string, body: unknown, headers: Record = {}, signal?: AbortSignal): Promise { + return fetch(url, { method: 'POST', headers: { 'Content-Type': 'application/json', ...headers }, body: JSON.stringify(body), signal }); +} + +test('ownership follows the download rule and rejects another principal\'s object', async () => { + await upload(redis, owner, SESSION, OBJECT); + await upload(redis, stranger, OTHER_SESSION, OTHER_OBJECT); + const req = principalRequest(owner); + + expect(await authorizeWorkspaceImportSource({ req, store: redis, source: { storage_session_id: SESSION, id: OBJECT } })) + .toBe(resolveSessionKey(req, { kind: 'user', id: 'user-1' })); + const denials = [ + { storage_session_id: OTHER_SESSION, id: OTHER_OBJECT }, + { storage_session_id: SESSION, id: OTHER_OBJECT }, + { storage_session_id: OTHER_SESSION, id: OBJECT }, + ]; + for (const source of denials) { + const error = await authorizeWorkspaceImportSource({ req, store: redis, source }).catch(caught => caught); + expect(error).toBeInstanceOf(WorkspaceImportError); + expect(error).toMatchObject({ status: 403, code: 'IMPORT_SOURCE_UNAUTHORIZED' }); + } + const malformed = await authorizeWorkspaceImportSource({ + req, store: redis, source: { storage_session_id: '../escape', id: OBJECT }, + }).catch(caught => caught); + expect(malformed).toMatchObject({ status: 400, code: 'INVALID_WORKSPACE_TOOL_REQUEST' }); + + await redis.del(`session:${SESSION}`); + const expired = await authorizeWorkspaceImportSource({ req, store: redis, source: { storage_session_id: SESSION, id: OBJECT } }) + .catch(caught => caught); + expect(expired).toMatchObject({ status: 403, code: 'IMPORT_SOURCE_UNAUTHORIZED' }); +}); + +test('the grant reads exactly one object, briefly, and can upload nothing', async () => { + await upload(redis, owner, SESSION, OBJECT); + const imports = importService(redis); + const req = principalRequest(owner); + const sessionKey = await imports.authorize(req, importBody()); + const before = Math.floor(Date.now() / 1000); + const prepared = await imports.prepare({ req, request: importBody(), sessionKey, ttlSeconds: 60 }); + + const grant = openEgressGrant(prepared.request.egressGrant, SECRET); + expect(grant.input_files).toEqual([{ id: OBJECT, session_id: SESSION, name: 'report.pdf' }]); + expect(grant.read_sessions).toEqual([SESSION]); + expect(grant).toMatchObject({ max_upload_bytes: 0, max_output_files: 0, max_requests: 2, session_key: sessionKey }); + expect(grant.exp - before).toBeGreaterThanOrEqual(90); + expect(grant.exp - before).toBeLessThanOrEqual(91); + expect(prepared.request.source.id).not.toBe(OBJECT); + expect(prepared.request.source.storage_session_id).not.toBe(SESSION); + expect(openEgressHandle(prepared.request.source.id, SECRET)).toMatchObject({ typ: 'object', object_id: OBJECT, session_id: SESSION }); + expect(openEgressHandle(prepared.request.source.storage_session_id, SECRET)).toMatchObject({ typ: 'session', dir: 'read', session_id: SESSION }); + const { egressGrant: _grant, source: _source, ...unchanged } = prepared.request; + const { source: _callerSource, ...callerFields } = importBody(); + expect(unchanged).toEqual(callerFields); +}); + +test('a synchronous import relays the granted request and revokes its grant', async () => { + await upload(redis, owner, SESSION, OBJECT); + await upload(redis, stranger, OTHER_SESSION, OTHER_OBJECT); + const minted: string[] = []; + const revoked: string[] = []; + const dispatched: Array<{ request: WorkspaceToolRequest; executionTimeoutMs?: number }> = []; + const url = await listen(appFor(createWorkspaceToolsRouter({ + backend: 'remote-bridge', configuredWorkerId: 'worker', dynamicWorkers: false, timeoutMs: 120_000, + imports: importService(redis, minted, revoked, 8192), + store: { async dispatchWorkspaceTool(args) { + dispatched.push({ request: args.request, executionTimeoutMs: args.executionTimeoutMs }); + const request = args.request as WorkspaceImportFileRequest; + return { protocolVersion: 1, generation: 1, leaseToken: 'lease', incarnationId: 'incarnation', status: 'fulfilled', result: { + protocolVersion: 1, operation: 'import_file', workspaceId: 'primary', path: request.path, + created: true, bytesWritten: request.bytes, sha256: request.sha256, + } }; + } }, + }))); + const endpoint = `${url}/workspace-tools/execute`; + + const response = await post(endpoint, importBody()); + expect(response.status).toBe(200); + expect(await response.json()).toMatchObject({ operation: 'import_file', created: true, bytesWritten: 4096 }); + expect(dispatched).toHaveLength(1); + const relayed = dispatched[0]!.request as WorkspaceImportFileRequest; + expect(typeof relayed.egressGrant).toBe('string'); + expect(relayed.source.id).not.toBe(OBJECT); + expect(dispatched[0]!.executionTimeoutMs).toBe(120_000); + await Bun.sleep(0); + expect(revoked).toEqual(minted); + + const callerGrant = await post(endpoint, { ...importBody(), egressGrant: 'ceg1.forged' }); + expect(callerGrant.status).toBe(400); + const foreign = await post(endpoint, importBody({ source: { storage_session_id: OTHER_SESSION, id: OTHER_OBJECT } })); + expect(foreign.status).toBe(403); + expect(await foreign.json()).toMatchObject({ code: 'IMPORT_SOURCE_UNAUTHORIZED' }); + const stolen = await post(endpoint, importBody(), { 'X-Test-User': 'user-2' }); + expect(stolen.status).toBe(403); + const oversized = await post(endpoint, importBody({ bytes: 8193 })); + expect(oversized.status).toBe(413); + expect(await oversized.json()).toMatchObject({ code: 'WRITE_LIMIT_EXCEEDED' }); + expect(dispatched).toHaveLength(1); + expect(minted).toHaveLength(1); +}); + +test('a caller that disconnects while its grant is minted never dispatches, and the grant is revoked', async () => { + await upload(redis, owner, SESSION, OBJECT); + const minted: string[] = []; + const revoked: string[] = []; + const base = importService(redis, minted, revoked); + const grantRequested = Promise.withResolvers(); + const releaseGrant = Promise.withResolvers(); + let dispatched = 0; + const url = await listen(appFor(createWorkspaceToolsRouter({ + backend: 'remote-bridge', configuredWorkerId: 'worker', dynamicWorkers: false, + imports: { ...base, async prepare(args) { + grantRequested.resolve(); + await releaseGrant.promise; + return base.prepare(args); + } }, + store: { async dispatchWorkspaceTool(): Promise { dispatched++; throw new Error('must not dispatch'); } }, + }))); + const caller = new AbortController(); + const response = post(`${url}/workspace-tools/execute`, importBody(), {}, caller.signal).catch(() => undefined); + await grantRequested.promise; + caller.abort(); + await response; + await Bun.sleep(50); + releaseGrant.resolve(); + for (let attempt = 0; attempt < 100 && revoked.length === 0; attempt++) await Bun.sleep(10); + expect(dispatched).toBe(0); + expect(revoked).toEqual(minted); + expect(minted).toHaveLength(1); +}); + +test('a Code API that cannot mint grants refuses import_file without dispatching', async () => { + let dispatched = 0; + const url = await listen(appFor(createWorkspaceToolsRouter({ + backend: 'remote-bridge', configuredWorkerId: 'worker', dynamicWorkers: false, + store: { async dispatchWorkspaceTool(): Promise { dispatched++; throw new Error('must not dispatch'); } }, + }))); + const response = await post(`${url}/workspace-tools/execute`, importBody()); + expect(response.status).toBe(503); + expect(await response.json()).toMatchObject({ code: 'IMPORT_UNAVAILABLE' }); + expect(dispatched).toBe(0); +}); + +test('a durable import fingerprints the caller request and mints its grant once', async () => { + await upload(redis, owner, SESSION, OBJECT); + const bridge = new RedisBridgeStore(redis); + await bridge.register({ protocolVersion: 1, workerId: 'worker', incarnationId: 'incarnation-00000001', capabilities: { + sandboxProfile: 'anthropic-srt', statefulWorkspace: false, runtimes: [], + workspaceTools: { protocolVersion: 1, operations: ['read_file', 'import_file'], workspaces: [{ id: 'primary' }] }, + } }); + const minted: string[] = []; + const revoked: string[] = []; + const url = await listen(appFor(createWorkspaceToolsRouter({ + backend: 'remote-bridge', configuredWorkerId: 'worker', dynamicWorkers: false, + imports: importService(redis, minted, revoked), + requests: new RedisWorkspaceRequests(redis, bridge), + store: { async dispatchWorkspaceTool(): Promise { throw new Error('durable requests never dispatch synchronously'); } }, + }))); + const headers = { 'X-LibreChat-Workspace-Request-Id': 'import-request-000001' }; + for (let attempt = 0; attempt < 2; attempt++) { + const response = await post(`${url}/workspace-tools/requests`, importBody(), headers); + expect(response.status).toBe(202); + expect(await response.json()).toMatchObject({ state: 'queued', workspaceId: 'primary' }); + } + expect(minted).toHaveLength(1); + + // A retry of the same request answers from its record even once the upload is gone. + await redis.del(`session:${SESSION}`); + const retried = await post(`${url}/workspace-tools/requests`, importBody(), headers); + expect(retried.status).toBe(202); + expect(minted).toHaveLength(1); + + const keys = (await redis.keys('codeapi:workspace-requests:v1:*')).filter(key => !key.endsWith(':active')); + const records = await Promise.all(keys.map(key => redis.hget(key, 'record'))); + const stored = records.flatMap(record => record == null ? [] : [JSON.parse(record).request as WorkspaceImportFileRequest]); + expect(stored.length).toBeGreaterThan(0); + for (const request of stored) { + expect(openEgressGrant(request.egressGrant!, SECRET).grant_id).toBe(minted[0]); + expect(request.source.id).not.toBe(OBJECT); + } + + // Racing submissions of one new request ID: only the adopted grant survives. + const raceHeaders = { 'X-LibreChat-Workspace-Request-Id': 'import-request-000002' }; + await upload(redis, owner, SESSION, OBJECT); + const before = minted.length; + const raced = await Promise.all([0, 1].map(() => post(`${url}/workspace-tools/requests`, importBody(), raceHeaders))); + expect(raced.map(response => response.status)).toEqual([202, 202]); + const racedGrants = minted.slice(before); + expect(racedGrants.length).toBeGreaterThan(0); + expect(revoked).toHaveLength(racedGrants.length - 1); + + const mintedBeforeConflict = minted.length; + const conflict = await post(`${url}/workspace-tools/requests`, importBody({ bytes: 1 }), headers); + expect(conflict.status).toBe(409); + expect(minted).toHaveLength(mintedBeforeConflict); +}); diff --git a/service/src/workspace-tools/import.ts b/service/src/workspace-tools/import.ts new file mode 100644 index 00000000..6f22b55b --- /dev/null +++ b/service/src/workspace-tools/import.ts @@ -0,0 +1,182 @@ +import { posix } from 'path'; +import { nanoid } from 'nanoid'; + +import type * as t from '../types'; +import type { GatewayPreparedEgress } from '../egress-gateway-client'; +import type { ExecutionManifestClaims } from '../execution-manifest'; +import type { SessionOwnershipStore } from '../session-ownership'; +import type { WorkspaceImportFileRequest } from '../../../packages/code/src/protocol'; + +import { isValidId } from '../utils'; +import { authorizeSessionOwnership } from '../session-ownership'; +import { resolveSessionKey, SessionKeyResolutionError } from '../session-key'; +import { buildExecutionManifestClaims } from '../execution-manifest-claims'; + +/** One download plus one retry of the same object; nothing else is readable. */ +const IMPORT_GRANT_MAX_REQUESTS = 2; +/** Clock-skew slack beyond the request's own admission and execution budget. */ +const IMPORT_GRANT_TTL_SLACK_SECONDS = 30; + +export type WorkspaceImportErrorCode = + | 'INVALID_WORKSPACE_TOOL_REQUEST' + | 'IMPORT_SOURCE_UNAUTHORIZED' + | 'IMPORT_UNAVAILABLE'; + +export class WorkspaceImportError extends Error { + constructor( + readonly status: 400 | 403 | 503, + readonly code: WorkspaceImportErrorCode, + message: string, + ) { + super(message); + this.name = 'WorkspaceImportError'; + } +} + +/** Redis reads used to prove ownership: the session cache key and the upload record. */ +export interface WorkspaceImportSourceStore extends Pick { + exists(key: string): Promise; +} + +export type WorkspaceImportGrantFactory = (args: { + payload: t.PayloadBody; + claims: ExecutionManifestClaims; +}) => Promise; + +export interface PreparedWorkspaceImport { + grantId: string; + request: WorkspaceImportFileRequest & { egressGrant: string }; +} + +export interface WorkspaceImportService { + /** Largest file Code API relays, checked before ownership or dispatch. */ + readonly maxBytes: number; + /** Proves the caller owns `source` and returns the storage session key that owns it. */ + authorize(req: t.AuthenticatedRequest, request: WorkspaceImportFileRequest): Promise; + /** Mints the one-object grant and rewrites `source` into its opaque handles. */ + prepare(args: { + req: t.AuthenticatedRequest; + request: WorkspaceImportFileRequest; + sessionKey: string; + ttlSeconds: number; + }): Promise; + revoke(grantId: string): Promise; +} + +const unauthorized = (): WorkspaceImportError => + new WorkspaceImportError(403, 'IMPORT_SOURCE_UNAUTHORIZED', 'Unauthorized import source'); + +/** + * Applies the `GET /download/:session_id/:fileId` rule for a user-kind object: + * the live session cache key must name the caller's own storage key, and the + * object must have been uploaded under that key and session. + */ +export async function authorizeWorkspaceImportSource(args: { + req: t.AuthenticatedRequest; + source: WorkspaceImportFileRequest['source']; + store: WorkspaceImportSourceStore; +}): Promise { + const { req, source, store } = args; + if (!isValidId(source.storage_session_id) || !isValidId(source.id)) { + throw new WorkspaceImportError(400, 'INVALID_WORKSPACE_TOOL_REQUEST', 'Invalid import source'); + } + const userId = req.codeApiAuthContext?.userId; + if (!userId) throw unauthorized(); + let sessionKey: string; + try { + sessionKey = resolveSessionKey(req, { kind: 'user', id: userId }); + } catch (error) { + if (error instanceof SessionKeyResolutionError && error.status === 400) throw unauthorized(); + throw error; + } + const ownership = await authorizeSessionOwnership(store, { + session_id: source.storage_session_id, + expectedSessionKey: sessionKey, + allowExpiredCache: false, + }); + if (!ownership.authorized) throw unauthorized(); + const uploaded = await store.exists(`upload:${sessionKey}${source.storage_session_id}${source.id}`); + if (uploaded !== 1) throw unauthorized(); + return sessionKey; +} + +/** Claims for a read-only grant over exactly one object, with no upload capacity. */ +export function workspaceImportGrantClaims(args: { + req: t.AuthenticatedRequest; + request: WorkspaceImportFileRequest; + sessionKey: string; + ttlSeconds: number; + nowSeconds?: number; +}): { payload: t.PayloadBody; claims: ExecutionManifestClaims } { + const now = args.nowSeconds ?? Math.floor(Date.now() / 1000); + const payload: t.PayloadBody = { + language: 'workspace-import', + version: '1', + files: [{ + id: args.request.source.id, + storage_session_id: args.request.source.storage_session_id, + name: posix.basename(args.request.path), + }], + }; + const claims = buildExecutionManifestClaims({ + req: args.req, + executionId: `import_${nanoid()}`, + userId: args.req.codeApiAuthContext?.userId ?? '', + sessionKey: args.sessionKey, + outputSessionId: `import_${nanoid()}`, + payload, + nowSeconds: now, + }); + return { + payload, + claims: { + ...claims, + max_upload_bytes: 0, + max_output_files: 0, + max_requests: IMPORT_GRANT_MAX_REQUESTS, + tool_call_socket: false, + exp: now + Math.max(1, Math.ceil(args.ttlSeconds)) + IMPORT_GRANT_TTL_SLACK_SECONDS, + }, + }; +} + +function isPayloadFileRef(file: t.PayloadBody['files'][number] | undefined): file is t.PayloadFileRef { + return file != null && 'id' in file && typeof file.id === 'string' && + typeof file.storage_session_id === 'string'; +} + +export function createWorkspaceImportService(options: { + maxBytes: number; + store: WorkspaceImportSourceStore; + createGrant: WorkspaceImportGrantFactory; + revokeGrant: (grantId: string) => Promise; +}): WorkspaceImportService { + if (!Number.isSafeInteger(options.maxBytes) || options.maxBytes < 1) { + throw new RangeError('Workspace import file limit must be a positive safe integer'); + } + return { + maxBytes: options.maxBytes, + authorize: (req, request) => authorizeWorkspaceImportSource({ req, source: request.source, store: options.store }), + async prepare({ req, request, sessionKey, ttlSeconds }) { + let prepared: GatewayPreparedEgress; + try { + prepared = await options.createGrant(workspaceImportGrantClaims({ req, request, sessionKey, ttlSeconds })); + } catch { + throw new WorkspaceImportError(503, 'IMPORT_UNAVAILABLE', 'Import source grant could not be issued'); + } + const [file] = prepared.payload.files; + if (!isPayloadFileRef(file) || prepared.payload.files.length !== 1 || !prepared.egressGrantToken) { + throw new WorkspaceImportError(503, 'IMPORT_UNAVAILABLE', 'Import source grant is invalid'); + } + return { + grantId: prepared.grant_id, + request: { + ...request, + source: { storage_session_id: file.storage_session_id, id: file.id }, + egressGrant: prepared.egressGrantToken, + }, + }; + }, + revoke: options.revokeGrant, + }; +} diff --git a/service/src/workspace-tools/index.ts b/service/src/workspace-tools/index.ts index 3c7caee2..58c043d9 100644 --- a/service/src/workspace-tools/index.ts +++ b/service/src/workspace-tools/index.ts @@ -9,7 +9,9 @@ import { connection } from '../queue'; import logger from '../logger'; import { checkServiceStartUp, checkServiceShutDown } from '../lifecycle'; import { withSpan } from '../telemetry'; -import { isBridgeEnabled } from '../bridge/enabled'; +import { isBridgeEnabled, isWorkspaceImportEnabled } from '../bridge/enabled'; +import { createWorkspaceImportService } from './import'; +import { createGatewayEgressGrant, revokeGatewayEgressGrant } from '../egress-gateway-client'; import { workspaceRequestCoordinationScope } from './coordination'; const router = Router(); @@ -53,6 +55,14 @@ router.use(['/workspace-tools/execute', '/workspace-tools/requests'], (req, res, if (req.method === 'POST') executionLimiter(req, res, next); else next(); }); +const imports = isWorkspaceImportEnabled() + ? createWorkspaceImportService({ + maxBytes: env.BRIDGE_IMPORT_FILE_MAX_BYTES, + store: connection, + createGrant: args => createGatewayEgressGrant(args), + revokeGrant: grantId => revokeGatewayEgressGrant({ grantId, reason: 'completed' }), + }) + : undefined; router.use( createWorkspaceToolsRouter({ store: bridgeStore, @@ -61,6 +71,7 @@ router.use( configuredWorkerId: env.BRIDGE_WORKER_ID, dynamicWorkers: env.BRIDGE_DYNAMIC_WORKERS, timeoutMs: env.JOB_TIMEOUT, + imports, }), ); diff --git a/service/src/workspace-tools/outcome.ts b/service/src/workspace-tools/outcome.ts index e8dc100f..03b54c5e 100644 --- a/service/src/workspace-tools/outcome.ts +++ b/service/src/workspace-tools/outcome.ts @@ -30,6 +30,7 @@ export function getWorkspaceToolOutcome(res: Response, route = '/workspace-tools if (existing != null) return existing; const startedAt = performance.now(); let responseEndedAt: number | undefined; + let responseFinished = false; let logged = false; const outcome: WorkspaceToolOutcome = { dispatchPending: false, @@ -37,7 +38,7 @@ export function getWorkspaceToolOutcome(res: Response, route = '/workspace-tools if (logged || responseEndedAt == null || outcome.dispatchPending) return; logged = true; outcomes.delete(res); - const finished = res.writableFinished; + const finished = responseFinished; logger.log(finished && res.statusCode < 400 ? 'info' : 'warn', 'Workspace tool request completed', { route, operation: outcome.operation, @@ -55,6 +56,8 @@ export function getWorkspaceToolOutcome(res: Response, route = '/workspace-tools res.removeListener('finish', onResponseEnd); res.removeListener('close', onResponseEnd); responseEndedAt = performance.now(); + // Late dispatch settlement can write to a closed response. Preserve its state at close. + responseFinished = res.writableFinished; outcome.flush(); }; outcomes.set(res, outcome); diff --git a/service/src/workspace-tools/requests.ts b/service/src/workspace-tools/requests.ts index 2988c7cc..902651fc 100644 --- a/service/src/workspace-tools/requests.ts +++ b/service/src/workspace-tools/requests.ts @@ -99,6 +99,13 @@ export class RedisWorkspaceRequests { request: WorkspaceToolRequest; queueWaitMs: number; executionTimeoutMs: number; + /** + * Derives the request a worker receives, such as an import carrying its + * egress grant. Runs only for a new request ID; the fingerprint, and so + * idempotent retries, cover the caller's request. `release` undoes the + * preparation when the queue does not adopt this submission. + */ + prepare?: () => Promise<{ request: WorkspaceToolRequest; release?: () => Promise }>; }): Promise { if (!WORKSPACE_REQUEST_ID_PATTERN.test(args.requestId)) { throw new BridgeStoreError('ASSIGNMENT_INVALID', 'Invalid durable workspace request ID'); @@ -117,21 +124,31 @@ export class RedisWorkspaceRequests { const registration = await this.bridge.prepareDurableWorkspaceTool({ ...args, tenantId: args.owner.tenantId, }); + const prepared = args.prepare == null ? undefined : await args.prepare(); + const release = async (): Promise => { await prepared?.release?.().catch(() => undefined); }; + const request = prepared?.request ?? args.request; const now = Date.now(); const record: StoredWorkspaceRequest = { id: `durable-${randomBytes(18).toString('base64url')}`, key, fingerprint, workerId: args.workerId, tenantId: args.owner.tenantId, requireTenantBinding: args.requireTenantBinding, - request: args.request, registration, createdAtMs: now, + request, registration, createdAtMs: now, queueDeadlineAtMs: now + args.queueWaitMs, executionTimeoutMs: args.executionTimeoutMs, state: 'queued', }; const queue = new BridgeAdmissionQueue(this.redis); - const accepted = await queue.submit({ - workerId: record.workerId, id: record.id, deadlineAtMs: record.queueDeadlineAtMs, - workspaceId: workspaceAdmissionId(record.request.workspaceId, record.request.workspaceInstanceId, record.request.worktree), - key, activeKey: this.activeKey, fingerprint, record: JSON.stringify(record), retentionMs: RETENTION_MS, - }); + let accepted: Awaited>; + try { + accepted = await queue.submit({ + workerId: record.workerId, id: record.id, deadlineAtMs: record.queueDeadlineAtMs, + workspaceId: workspaceAdmissionId(record.request.workspaceId, record.request.workspaceInstanceId, record.request.worktree), + key, activeKey: this.activeKey, fingerprint, record: JSON.stringify(record), retentionMs: RETENTION_MS, + }); + } catch (error) { + await release(); + throw error; + } + if (accepted !== 'accepted') await release(); if (accepted === 'conflict') throw new WorkspaceRequestConflict('Request ID was already used for different work'); if (accepted === 'full') throw new BridgeStoreError('WORKER_QUEUE_FULL', 'Bridge worker pending request limit reached'); const submitted = (await this.read(key))!; diff --git a/service/src/workspace-tools/router.test.ts b/service/src/workspace-tools/router.test.ts index 8a753c08..e3ed54bd 100644 --- a/service/src/workspace-tools/router.test.ts +++ b/service/src/workspace-tools/router.test.ts @@ -534,11 +534,9 @@ test('logs a disconnected dispatch once without inventing HTTP 200', async () => const settlementGate = Promise.withResolvers(); let dispatchAborted = false; let queueRemaining: number | undefined; - let closeConnection = (): void => { throw new Error('connection not ready'); }; app.use(json()); app.use((req, res, next) => { applyPrincipal(req, { userId: 'user-1', tenantId: 'tenant-1', principalSource: 'librechat_jwt', codeWorkerId: 'user-worker' }); - closeConnection = (): void => { res.destroy(); }; res.once('close', () => closed.resolve()); next(); }); @@ -578,7 +576,7 @@ test('logs a disconnected dispatch once without inventing HTTP 200', async () => body: JSON.stringify({ protocolVersion: 1, operation: 'list_files', workspaceId: 'primary' }), }); await started.promise; - closeConnection(); + controller.abort(); await expect(response).rejects.toThrow(); await closed.promise; expect(queueRemaining).toBeGreaterThan(29_000); diff --git a/service/src/workspace-tools/router.ts b/service/src/workspace-tools/router.ts index d4d65d14..bf31e272 100644 --- a/service/src/workspace-tools/router.ts +++ b/service/src/workspace-tools/router.ts @@ -4,14 +4,19 @@ import type { RequestHandler, Response } from 'express'; import type { AuthenticatedRequest } from '../types'; import type { RedisBridgeStore } from '../bridge/store'; import type { WorkspaceToolRequest } from '../../../packages/code/src/protocol'; +import type { PreparedWorkspaceImport, WorkspaceImportService } from './import'; +import logger from '../logger'; import { getWorkspaceToolOutcome } from './outcome'; +import { WorkspaceImportError } from './import'; import { getPrincipalOrReject } from '../auth/principal'; import { BridgeStoreError } from '../bridge/store'; import { checkServiceShutDown } from '../lifecycle'; import { isWorkspaceToolRequest, + isBridgeWorkspaceImportFileRequest, BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, + BRIDGE_WORKSPACE_IMPORT_FILE_TIMEOUT_MS, } from '../../../packages/code/src/protocol'; import { CODEAPI_BRIDGE_WORKER_HEADER, @@ -35,6 +40,8 @@ interface WorkspaceToolsRouterOptions { timeoutMs?: number; queueTimeoutMs?: number; isShuttingDown?: () => boolean; + /** Present only when this Code API can mint egress grants for `import_file`. */ + imports?: WorkspaceImportService; } function asyncRoute(handler: (req: AuthenticatedRequest, res: Response) => Promise): RequestHandler { @@ -82,13 +89,28 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) res.status(503).json({ error: 'Service is shutting down' }); return; } - if (!isWorkspaceToolRequest(req.body)) { + if ( + !isWorkspaceToolRequest(req.body) || + (req.body.operation === 'import_file' && !isBridgeWorkspaceImportFileRequest(req.body)) + ) { outcome.errorCode = 'INVALID_WORKSPACE_TOOL_REQUEST'; res.status(400).json({ error: 'Invalid workspace tool request', }); return; } + if (req.body.operation === 'import_file') { + if (options.imports == null) { + outcome.errorCode = 'IMPORT_UNAVAILABLE'; + res.status(503).json({ error: 'Workspace file import is not available on this Code API', code: 'IMPORT_UNAVAILABLE' }); + return; + } + if (req.body.bytes > options.imports.maxBytes) { + outcome.errorCode = 'WRITE_LIMIT_EXCEEDED'; + res.status(413).json({ error: 'Imported file exceeds the Code API file limit', code: 'WRITE_LIMIT_EXCEEDED' }); + return; + } + } const advertisedQueueWait = req.header(WORKSPACE_QUEUE_WAIT_HEADER); if (advertisedQueueWait !== undefined && !/^[1-9]\d*$/.test(advertisedQueueWait)) { outcome.errorCode = 'INVALID_WORKSPACE_QUEUE_WAIT'; @@ -121,9 +143,14 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) options.timeoutMs ?? Number.MAX_SAFE_INTEGER, ) } : principalRequest; - const executionBudgetMs = request.operation === 'execute_command' - ? request.timeoutMs! + 5_000 - : Math.min(options.timeoutMs ?? 30_000, 30_000); + let executionBudgetMs = Math.min(options.timeoutMs ?? 30_000, 30_000); + if (request.operation === 'execute_command') executionBudgetMs = request.timeoutMs! + 5_000; + if (request.operation === 'import_file') { + executionBudgetMs = Math.min( + options.timeoutMs ?? BRIDGE_WORKSPACE_IMPORT_FILE_TIMEOUT_MS, + BRIDGE_WORKSPACE_IMPORT_FILE_TIMEOUT_MS, + ); + } outcome.deadlineBudgetMs = queueBudgetMs + executionBudgetMs; let selection: { workerId: string; explicit: boolean } | undefined; @@ -152,20 +179,47 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) } outcome.workerId = selection.workerId; + const sendImportError = (error: unknown): boolean => { + if (!(error instanceof WorkspaceImportError)) return false; + outcome.errorCode = error.code; + res.status(error.status).json({ error: error.message, code: error.code }); + return true; + }; + const imports = options.imports; + const importRequest = request.operation === 'import_file' ? request : undefined; + /** Ownership and the grant are settled only for work about to be queued or dispatched, + * so a durable retry of an existing request never depends on the upload still existing. */ + const prepareImport = imports == null || importRequest == null + ? undefined + : async (): Promise => { + const sessionKey = await imports.authorize(req, importRequest); + // The grant must outlive admission and execution, never more. + const ttlSeconds = Math.ceil((queueBudgetMs + executionBudgetMs) / 1000); + return imports.prepare({ req, request: importRequest, sessionKey, ttlSeconds }); + }; + if (durable) { if (options.requests == null) { res.status(404).json({ error: 'Durable workspace requests are unavailable' }); return; } try { + const prepare = prepareImport; const status = await options.requests.submit({ owner: principal, requestId: req.header('X-LibreChat-Workspace-Request-Id') ?? '', workerId: selection.workerId, requireTenantBinding: selection.explicit && (options.dynamicWorkers || selection.workerId !== options.configuredWorkerId), request, queueWaitMs: queueBudgetMs, executionTimeoutMs: executionBudgetMs, + ...(prepare == null || imports == null ? {} : { + prepare: async () => { + const prepared = await prepare(); + return { request: prepared.request, release: () => imports.revoke(prepared.grantId) }; + }, + }), }); res.status(202).json(status); } catch (error) { + if (sendImportError(error)) return; if (error instanceof WorkspaceRequestConflict) { res.status(409).json({ error: error.message, code: 'REQUEST_CONFLICT' }); } else if (error instanceof BridgeStoreError) { @@ -182,7 +236,24 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) if (!res.writableEnded) abort(); }; res.once('close', abortClosedResponse); + // Events that fired before these listeners existed are visible only on the socket. + if (req.socket?.destroyed === true) abort(); + let preparedImport: PreparedWorkspaceImport | undefined; try { + if (prepareImport != null) { + try { + preparedImport = await prepareImport(); + } catch (error) { + if (sendImportError(error)) return; + throw error; + } + // A caller that left while ownership or the grant was settled must not start a mutation. + if (controller.signal.aborted) { + outcome.errorCode = 'CLIENT_DISCONNECTED'; + return; + } + } + const dispatchRequest: WorkspaceToolRequest = preparedImport?.request ?? request; outcome.dispatchPending = true; const dispatchStartedAt = performance.now(); const settlement = await options.store.dispatchWorkspaceTool({ @@ -190,7 +261,7 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) tenantId: principal.tenantId, requireTenantBinding: selection.explicit && (options.dynamicWorkers || selection.workerId !== options.configuredWorkerId), - request, + request: dispatchRequest, deadlineAtMs: Date.now() + queueBudgetMs, executionTimeoutMs: executionBudgetMs, signal: controller.signal, @@ -243,6 +314,12 @@ export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions) outcome.flush(); req.removeListener('aborted', abort); res.removeListener('close', abortClosedResponse); + if (preparedImport != null && options.imports != null) { + const { grantId } = preparedImport; + void options.imports.revoke(grantId).catch(error => { + logger.warn('Failed to revoke workspace import grant', { grantId, error }); + }); + } } }); router.post('/workspace-tools/execute', execute(false)); diff --git a/tests/compose-bridge-config.cjs b/tests/compose-bridge-config.cjs index bafea2fe..79268b52 100644 --- a/tests/compose-bridge-config.cjs +++ b/tests/compose-bridge-config.cjs @@ -16,6 +16,7 @@ function render(overrides) { CODEAPI_BRIDGE_TOKEN: '', CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS: '', CODEAPI_BRIDGE_LANE_GIT: '', + CODEAPI_BRIDGE_IMPORT_FILE_MAX_BYTES: '', CODEAPI_BRIDGE_RECOVERY_SERVER_ID: '', CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS: '', CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS: '', @@ -36,6 +37,7 @@ for (const overrides of [ CODEAPI_BRIDGE_WORKER_ID: 'test-worker', CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS: '4', CODEAPI_BRIDGE_LANE_GIT: 'true', + CODEAPI_BRIDGE_IMPORT_FILE_MAX_BYTES: '1048576', CODEAPI_BRIDGE_RECOVERY_SERVER_ID: 'https://code.example.test', CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS: '86400', CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS: '90', @@ -61,6 +63,10 @@ for (const overrides of [ assert.equal(env.CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE, overrides.CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE ?? '30'); assert.equal(env.CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE, overrides.CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE ?? '240'); } + assert.equal( + config.services.api.environment.CODEAPI_BRIDGE_IMPORT_FILE_MAX_BYTES, + overrides.CODEAPI_BRIDGE_IMPORT_FILE_MAX_BYTES ?? '104857600', + ); for (const name of ['egress_gateway', 'sandbox-runner']) { assert.equal(config.services[name].environment.CODEAPI_BRIDGE_TOKEN, undefined); assert.equal(config.services[name].environment.CODEAPI_BRIDGE_RECOVERY_SERVER_ID, undefined); diff --git a/tests/package_init_runtime_rollout.sh b/tests/package_init_runtime_rollout.sh index 3bb8cf6c..839d9a7c 100755 --- a/tests/package_init_runtime_rollout.sh +++ b/tests/package_init_runtime_rollout.sh @@ -15,6 +15,12 @@ render() { --set workerSandbox.enabled=true \ --set workerSandbox.packages.source=pvc "$@" } +# The hook installs the runtime selected by service requests. +render | grep -A 1 'name: NODE_VERSION' | grep -q 'value: "24.22.1"' +render | grep -A 1 'name: BUN_VERSION' | grep -q 'value: "1.4.4"' +# Explicit runtime selections still override the chart default. +render --set workerSandbox.packages.initJob.nodeVersion=24.22.0 \ + | grep -A 1 'name: NODE_VERSION' | grep -q 'value: "24.22.0"' # Default mutable image must refresh on every pre-install/pre-upgrade hook. render | grep -q 'imagePullPolicy: Always' # Reused historical values must not bypass the runtime migration. diff --git a/tests/test_tool_call_pipes.py b/tests/test_tool_call_pipes.py new file mode 100644 index 00000000..3907ce05 --- /dev/null +++ b/tests/test_tool_call_pipes.py @@ -0,0 +1,36 @@ +import unittest +from unittest.mock import patch + +from tool_call_pipes import wait_for_killed_descendant + + +class DescendantReapingTest(unittest.TestCase): + def test_already_reaped_by_parent(self): + with patch('tool_call_pipes.os.waitpid', side_effect=ChildProcessError), \ + patch('tool_call_pipes.os.path.exists', return_value=False): + wait_for_killed_descendant(123) + + def test_waits_for_adoption(self): + with patch('tool_call_pipes.os.waitpid', side_effect=[ChildProcessError, (123, 9)]), \ + patch('tool_call_pipes.os.path.exists', return_value=True), \ + patch('tool_call_pipes.time.sleep'): + wait_for_killed_descendant(123) + + def test_rejects_wrong_exit_status(self): + with patch('tool_call_pipes.os.waitpid', return_value=(123, 0)): + with self.assertRaises(AssertionError): + wait_for_killed_descendant(123) + + def test_live_nonchild_still_fails_at_deadline(self): + with patch('tool_call_pipes.os.waitpid', side_effect=ChildProcessError), \ + patch('tool_call_pipes.os.path.exists', return_value=True), \ + patch('tool_call_pipes.time.monotonic', side_effect=[0, 0, 6]), \ + patch('tool_call_pipes.time.sleep'), \ + patch('tool_call_pipes.os.kill') as kill: + with self.assertRaisesRegex(AssertionError, 'orphan survived'): + wait_for_killed_descendant(123) + kill.assert_called_once_with(123, 9) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/tool_call_pipes.py b/tests/tool_call_pipes.py index c9388530..df37fc39 100644 --- a/tests/tool_call_pipes.py +++ b/tests/tool_call_pipes.py @@ -36,6 +36,25 @@ def do_POST(self): self.wfile.write(body) +def wait_for_killed_descendant(pid, timeout=5): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + reaped, status = os.waitpid(pid, os.WNOHANG) + except ChildProcessError: + # Adoption is asynchronous, and the original parent may reap it. + # ECHILD alone does not prove that a live descendant has exited. + if not os.path.exists(f'/proc/{pid}'): + return + else: + if reaped: + assert os.waitstatus_to_exitcode(status) == -9 + return + time.sleep(0.01) + os.kill(pid, 9) + raise AssertionError('orphan survived broker death') + + def main(): policy, broker, preamble = sys.argv[1:] server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), Upstream) @@ -148,16 +167,7 @@ async def check(): running.kill() running.communicate(timeout=5) for pid in [job_pid, relay_pid, broker_pid]: - deadline = time.monotonic() + 5 - while time.monotonic() < deadline: - reaped, status = os.waitpid(pid, os.WNOHANG) - if reaped: - assert os.waitstatus_to_exitcode(status) == -9 - break - time.sleep(0.01) - else: - os.kill(pid, 9) - raise AssertionError('orphan survived broker death') + wait_for_killed_descendant(pid) finally: if running.poll() is None: running.kill() diff --git a/tests/tool_call_runner.cjs b/tests/tool_call_runner.cjs index 61ddcb9e..7028d9cc 100644 --- a/tests/tool_call_runner.cjs +++ b/tests/tool_call_runner.cjs @@ -118,8 +118,8 @@ if __name__ == '__main__': const plain = await execute("import os, asyncio; assert not os.path.exists('/proc/self/fd/3'); asyncio.run(asyncio.to_thread(lambda: 1)); print('PASS: ordinary asyncio execution without pipe capability')", false); console.log(plain.run.stdout.trim()); for (const language of ['bun-js', 'bun-ts']) { - const js = await execute("let denied = false; try { Bun.spawn([process.execPath, '-e', '0'], {stdout:'pipe', stderr:'pipe'}); } catch (error) { if (error.code !== 'EPERM' || error.syscall !== 'socketpair') throw error; denied = true; } if (!denied) throw new Error('socketpair spawn allowed'); console.log('PASS: Bun ' + Bun.version + ', socketpair-based spawn denied')", false, language, '1.4.2'); - assert(js.run.stdout.includes('PASS: Bun 1.4.2'), JSON.stringify(js)); + const js = await execute("let denied = false; try { Bun.spawn([process.execPath, '-e', '0'], {stdout:'pipe', stderr:'pipe'}); } catch (error) { if (error.code !== 'EPERM' || error.syscall !== 'socketpair') throw error; denied = true; } if (!denied) throw new Error('socketpair spawn allowed'); console.log('PASS: Bun ' + Bun.version + ', socketpair-based spawn denied')", false, language, '1.4.4'); + assert(js.run.stdout.includes('PASS: Bun 1.4.4'), JSON.stringify(js)); console.log(language + ': ' + js.run.stdout.trim()); } } finally {