From d2bbb7aeb08d831cc1f2e1ff35916c6602be0864 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Tue, 27 Oct 2026 15:54:28 -0400 Subject: [PATCH 01/12] feat(coding-agent): share installed artifact acceptance helpers --- scripts/artifact-acceptance.mjs | 117 +------- scripts/artifact-installation.mjs | 119 ++++++++ scripts/verify-candidate-artifacts.py | 395 +++++++++++++------------- 3 files changed, 324 insertions(+), 307 deletions(-) create mode 100644 scripts/artifact-installation.mjs diff --git a/scripts/artifact-acceptance.mjs b/scripts/artifact-acceptance.mjs index f8f60f200..ac4188808 100644 --- a/scripts/artifact-acceptance.mjs +++ b/scripts/artifact-acceptance.mjs @@ -1,6 +1,7 @@ +import { assertContained, installConsumer, isolatedEnvironment, verifyTarballs } from "./artifact-installation.mjs"; +export { assertContained, installConsumer, isolatedEnvironment, verifyTarballs } from "./artifact-installation.mjs"; import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; -import { createHash } from "node:crypto"; import { cpSync, existsSync, @@ -13,20 +14,14 @@ import { writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; -import { dirname, isAbsolute, join, relative, resolve } from "node:path"; +import { dirname, join, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { cliDistribution } from "./cli-distribution.mjs"; import { getPublicWorkspacePackages } from "./release-packages.mjs"; const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); -const digest = (bytes) => createHash("sha256").update(bytes).digest("hex"); const readJson = (file) => JSON.parse(readFileSync(file, "utf8")); -export function assertContained(file, directory) { - const local = relative(realpathSync(directory), realpathSync(file)); - assert(local === "" || (!local.startsWith("..") && !isAbsolute(local)), `Path escapes isolated consumer: ${file}`); -} - export function packageClosure(name, packages) { const result = []; const visiting = new Set(); @@ -52,51 +47,6 @@ export function packageClosure(name, packages) { return result; } -export function verifyTarballs(directory, records, packages) { - assert.equal(records.length, packages.length, "Incomplete tarball inventory"); - assert.equal(new Set(records.map((pkg) => pkg.name)).size, packages.length, "Duplicate tarball identity"); - for (const pkg of packages) { - const record = records.find((entry) => entry.name === pkg.name); - assert(record && record.version === pkg.version, `Missing tarball: ${pkg.name}`); - assert.equal(record.filename, `${pkg.name.replace("@", "").replace("/", "-")}-${pkg.version}.tgz`); - assert.equal( - digest(readFileSync(join(directory, record.filename))), - record.sha256, - `Tarball checksum mismatch: ${pkg.name}`, - ); - } -} - -export function isolatedEnvironment(directory) { - const env = {}; - for (const key of ["PATH", "SystemRoot", "WINDIR", "COMSPEC", "PATHEXT", "LANG"]) { - if (process.env[key]) env[key] = process.env[key]; - } - for (const dir of ["home", "config", "cache", "prefix", "project", "agent", "temp"]) - mkdirSync(join(directory, dir), { recursive: true }); - return { - ...env, - HOME: join(directory, "home"), - USERPROFILE: join(directory, "home"), - XDG_CONFIG_HOME: join(directory, "config"), - XDG_CACHE_HOME: join(directory, "cache"), - npm_config_cache: join(directory, "cache", "npm"), - npm_config_prefix: join(directory, "prefix"), - npm_config_userconfig: join(directory, "empty.npmrc"), - npm_config_globalconfig: join(directory, "empty-global.npmrc"), - npm_config_registry: "https://registry.npmjs.org", - npm_config_update_notifier: "false", - BUN_INSTALL_CACHE_DIR: join(directory, "cache", "bun"), - TMPDIR: join(directory, "temp"), - TMP: join(directory, "temp"), - TEMP: join(directory, "temp"), - DIFFEX_CODING_AGENT_DIR: join(directory, "agent"), - DIFFEX_NO_LOCAL_LLM: "1", - DIFFEX_OFFLINE: "1", - TERM: "xterm-256color", - }; -} - function walk(directory) { return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => { const path = join(directory, entry.name); @@ -154,67 +104,6 @@ export const operations = { "@diffexai/diffex-coding-agent-server": `const reporter = api.createDurableUsageReporter({ endpoint: "http://127.0.0.1:1/usage", sharedSecret: "local-artifact-fixture-only-32-bytes", outboxDirectory: path.join(process.env.DIFFEX_CODING_AGENT_DIR, "outbox"), fetch: () => { throw new Error("Unexpected usage"); } }); await reporter.close();`, }; -export function installConsumer( - directory, - manager, - selected, - records, - tarballDirectory, - cacheDirectory = join(directory, "cache"), -) { - mkdirSync(directory, { recursive: true }); - const env = isolatedEnvironment(directory); - mkdirSync(cacheDirectory, { recursive: true }); - env.npm_config_cache = join(cacheDirectory, "npm"); - env.BUN_INSTALL_CACHE_DIR = join(cacheDirectory, "bun"); - writeFileSync(join(directory, ".npmrc"), "@diffexai:registry=http://127.0.0.1:9/\n"); - const dependencies = Object.fromEntries( - selected.map((pkg) => [ - pkg.name, - `file:${join(tarballDirectory, records.find((record) => record.name === pkg.name).filename)}`, - ]), - ); - writeFileSync( - join(directory, "package.json"), - JSON.stringify({ private: true, type: "module", dependencies, overrides: dependencies }), - ); - execFileSync( - manager, - manager === "npm" - ? ["install", "--omit=dev", "--ignore-scripts", "--no-audit", "--no-fund"] - : ["install", "--production", "--ignore-scripts", "--network-concurrency=8"], - { cwd: directory, env, stdio: "inherit", timeout: 600_000 }, - ); - for (const pkg of selected) { - const installed = join(directory, "node_modules", pkg.name); - assertContained(installed, directory); - const info = readJson(join(installed, "package.json")); - assert.equal(info.name, pkg.name); - assert.equal(info.version, pkg.version); - const record = records.find((entry) => entry.name === pkg.name); - const inventory = JSON.parse( - execFileSync("tar", ["-xOf", join(tarballDirectory, record.filename), "package/distribution-files.json"], { - encoding: "utf8", - maxBuffer: 10 * 1024 * 1024, - }), - ); - assert.deepEqual( - readJson(join(installed, "distribution-files.json")), - inventory, - "Installed inventory differs from the checked tarball", - ); - for (const file of inventory.files) { - assertContained(join(installed, file.path), installed); - assert.equal( - digest(readFileSync(join(installed, file.path))), - file.sha256, - `Installed bytes differ: ${pkg.name}/${file.path}`, - ); - } - } - return env; -} - export function acceptArtifacts(artifactDirectory, outputDirectory, { managers = ["npm", "bun"] } = {}) { assert( managers.length && managers.every((manager) => ["npm", "bun"].includes(manager)), diff --git a/scripts/artifact-installation.mjs b/scripts/artifact-installation.mjs new file mode 100644 index 000000000..5421e7568 --- /dev/null +++ b/scripts/artifact-installation.mjs @@ -0,0 +1,119 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdirSync, readFileSync, realpathSync, writeFileSync } from "node:fs"; +import { isAbsolute, join, relative } from "node:path"; +const digest = (bytes) => createHash("sha256").update(bytes).digest("hex"); +const readJson = (file) => JSON.parse(readFileSync(file, "utf8")); + +export function assertContained(file, directory) { + const local = relative(realpathSync(directory), realpathSync(file)); + assert(local === "" || (!local.startsWith("..") && !isAbsolute(local)), `Path escapes isolated consumer: ${file}`); +} + +export function verifyTarballs(directory, records, packages) { + assert.equal(records.length, packages.length, "Incomplete tarball inventory"); + assert.equal(new Set(records.map((pkg) => pkg.name)).size, packages.length, "Duplicate tarball identity"); + for (const pkg of packages) { + const record = records.find((entry) => entry.name === pkg.name); + assert(record && record.version === pkg.version, `Missing tarball: ${pkg.name}`); + assert.equal(record.filename, `${pkg.name.replace("@", "").replace("/", "-")}-${pkg.version}.tgz`); + assert.equal( + digest(readFileSync(join(directory, record.filename))), + record.sha256, + `Tarball checksum mismatch: ${pkg.name}`, + ); + } +} + +export function isolatedEnvironment(directory) { + const env = {}; + for (const key of ["PATH", "SystemRoot", "WINDIR", "COMSPEC", "PATHEXT", "LANG"]) { + if (process.env[key]) env[key] = process.env[key]; + } + for (const dir of ["home", "config", "cache", "prefix", "project", "agent", "temp"]) + mkdirSync(join(directory, dir), { recursive: true }); + return { + ...env, + HOME: join(directory, "home"), + USERPROFILE: join(directory, "home"), + XDG_CONFIG_HOME: join(directory, "config"), + XDG_CACHE_HOME: join(directory, "cache"), + npm_config_cache: join(directory, "cache", "npm"), + npm_config_prefix: join(directory, "prefix"), + npm_config_userconfig: join(directory, "empty.npmrc"), + npm_config_globalconfig: join(directory, "empty-global.npmrc"), + npm_config_registry: "https://registry.npmjs.org", + npm_config_update_notifier: "false", + BUN_INSTALL_CACHE_DIR: join(directory, "cache", "bun"), + TMPDIR: join(directory, "temp"), + TMP: join(directory, "temp"), + TEMP: join(directory, "temp"), + DIFFEX_CODING_AGENT_DIR: join(directory, "agent"), + DIFFEX_NO_LOCAL_LLM: "1", + DIFFEX_OFFLINE: "1", + TERM: "xterm-256color", + }; +} + +export function installConsumer( + directory, + manager, + selected, + records, + tarballDirectory, + cacheDirectory = join(directory, "cache"), +) { + mkdirSync(directory, { recursive: true }); + const env = isolatedEnvironment(directory); + mkdirSync(cacheDirectory, { recursive: true }); + env.npm_config_cache = join(cacheDirectory, "npm"); + env.BUN_INSTALL_CACHE_DIR = join(cacheDirectory, "bun"); + writeFileSync(join(directory, ".npmrc"), "@diffexai:registry=http://127.0.0.1:9/\n"); + const dependencies = Object.fromEntries( + selected.map((pkg) => [ + pkg.name, + `file:${join(tarballDirectory, records.find((record) => record.name === pkg.name).filename)}`, + ]), + ); + writeFileSync( + join(directory, "package.json"), + JSON.stringify({ private: true, type: "module", dependencies, overrides: dependencies }), + ); + execFileSync( + manager, + manager === "npm" + ? ["install", "--omit=dev", "--ignore-scripts", "--no-audit", "--no-fund"] + : ["install", "--production", "--ignore-scripts", "--network-concurrency=8"], + { cwd: directory, env, stdio: "inherit", timeout: 600_000 }, + ); + for (const pkg of selected) { + const installed = join(directory, "node_modules", pkg.name); + assertContained(installed, directory); + const info = readJson(join(installed, "package.json")); + assert.equal(info.name, pkg.name); + assert.equal(info.version, pkg.version); + const record = records.find((entry) => entry.name === pkg.name); + const inventory = JSON.parse( + execFileSync("tar", ["-xOf", join(tarballDirectory, record.filename), "package/distribution-files.json"], { + encoding: "utf8", + maxBuffer: 10 * 1024 * 1024, + }), + ); + assert.deepEqual( + readJson(join(installed, "distribution-files.json")), + inventory, + "Installed inventory differs from the checked tarball", + ); + for (const file of inventory.files) { + assertContained(join(installed, file.path), installed); + assert.equal( + digest(readFileSync(join(installed, file.path))), + file.sha256, + `Installed bytes differ: ${pkg.name}/${file.path}`, + ); + } + } + return env; +} + diff --git a/scripts/verify-candidate-artifacts.py b/scripts/verify-candidate-artifacts.py index 61ef60bb2..6db297145 100644 --- a/scripts/verify-candidate-artifacts.py +++ b/scripts/verify-candidate-artifacts.py @@ -1,199 +1,208 @@ import os, sys, json, subprocess, tempfile, shutil, pathlib, pty, select, time, re, fcntl, termios, struct -if len(sys.argv) not in (2, 3): - raise SystemExit( - "Usage: python3 scripts/verify-candidate-artifacts.py ARTIFACT_DIRECTORY" - ) -out = pathlib.Path(sys.argv[1]).resolve() -fixture_source = ( - pathlib.Path(__file__).resolve().parent / "fixtures/artifact-handoff.mjs" -) -executables = { - "node": str(out / "node/node_modules/.bin/diffex"), - "bun-install": str(out / "bun-install/node_modules/.bin/diffex"), -} -if len(sys.argv) == 3: - executables = {"consumer": str(pathlib.Path(sys.argv[2]).resolve() / "node_modules/.bin/diffex")} -results = [] -for runtime, exe in executables.items(): - root = pathlib.Path(tempfile.mkdtemp(prefix="diffex artifact terminal ")) - fixture = None - try: - for d in ["home", "agent", "project"]: - (root / d).mkdir() - env = { - "PATH": os.environ["PATH"], - "HOME": str(root / "home"), - "USERPROFILE": str(root / "home"), - "XDG_CONFIG_HOME": str(root / "home/config"), - "XDG_CACHE_HOME": str(root / "home/cache"), - "TMPDIR": str(root), - "DIFFEX_CODING_AGENT_DIR": str(root / "agent"), - "DIFFEX_NO_LOCAL_LLM": "1", - "TERM": "xterm-256color", - "LANG": "en_US.UTF-8", - "DIFFEX_HANDOFF_RECORD": str(root / "events.jsonl"), - } - (root / "agent/settings.json").write_text( - json.dumps({"theme": "dark", "packages": []}) - ) - (root / "events.jsonl").write_text("") - fixture_fd, fixture_path = tempfile.mkstemp(prefix="artifact-fixture-", suffix=".mjs", dir=pathlib.Path(exe).parent.parent.parent) - os.close(fixture_fd) - fixture = pathlib.Path(fixture_path) - shutil.copyfile(fixture_source, fixture) - args = [ - exe, - "--no-session", - "--no-extensions", - "--no-skills", - "--no-context-files", - "--no-prompt-templates", - "-e", - str(fixture), - "--model", - "handoff/fixture", - ] - for command in [["--version"], ["--help"], ["auth", "--help"]]: - p = subprocess.run( - [exe] + command, - cwd=root / "project", - env={**env, "DIFFEX_OFFLINE": "1"}, - capture_output=True, - text=True, - timeout=30, - ) - assert p.returncode == 0, p.stderr - results.append( - {"runtime": runtime, "command": command, "exitCode": p.returncode} - ) - for command, expected in [ - (["--list-models"], "fixture"), - (["-p", "Reply with the fixture confirmation."], "handoff-fixture-ok"), - ]: - p = subprocess.run( - args + command, - cwd=root / "project", - env=env, - capture_output=True, - text=True, - timeout=30, - ) - assert p.returncode == 0, (p.stdout, p.stderr) - assert expected in p.stdout, (p.stdout, p.stderr) - results.append( - { - "runtime": runtime, - "command": command, - "exitCode": p.returncode, - "expected": expected, - } - ) - pid, fd = pty.fork() - if pid == 0: - os.chdir(root / "project") - fcntl.ioctl(1, termios.TIOCSWINSZ, struct.pack("HHHH", 30, 100, 0, 0)) - os.execve(exe, args, env) - raw = b"" - text = "" - stage = "startup" - deadline = time.monotonic() + 30 - mark = 0 + +def verify(out, executables, fixture_source=None): + out = pathlib.Path(out).resolve() + out.mkdir(parents=True, exist_ok=True) + fixture_source = fixture_source or pathlib.Path(__file__).resolve().parent / "fixtures/artifact-handoff.mjs" + results = [] + for runtime, exe in executables.items(): + root = pathlib.Path(tempfile.mkdtemp(prefix="diffex artifact terminal ")) + fixture = None try: - while time.monotonic() < deadline: - if not select.select([fd], [], [], 0.1)[0]: - continue - try: - chunk = os.read(fd, 65536) - except OSError: - break - if not chunk: - break - raw += chunk - text = re.sub( - r"\x1b\[[0-?]*[ -/]*[@-~]", "", raw.decode(errors="replace") + for d in ["home", "agent", "project"]: + (root / d).mkdir() + env = { + "PATH": os.environ["PATH"], + "HOME": str(root / "home"), + "USERPROFILE": str(root / "home"), + "XDG_CONFIG_HOME": str(root / "home/config"), + "XDG_CACHE_HOME": str(root / "home/cache"), + "TMPDIR": str(root), + "DIFFEX_CODING_AGENT_DIR": str(root / "agent"), + "DIFFEX_NO_LOCAL_LLM": "1", + "TERM": "xterm-256color", + "LANG": "en_US.UTF-8", + "DIFFEX_HANDOFF_RECORD": str(root / "events.jsonl"), + } + (root / "agent/settings.json").write_text( + json.dumps({"theme": "dark", "packages": []}) + ) + (root / "events.jsonl").write_text("") + fixture_fd, fixture_path = tempfile.mkstemp(prefix="artifact-fixture-", suffix=".mjs", dir=pathlib.Path(exe).parent.parent.parent) + os.close(fixture_fd) + fixture = pathlib.Path(fixture_path) + shutil.copyfile(fixture_source, fixture) + args = [ + exe, + "--no-session", + "--no-extensions", + "--no-skills", + "--no-context-files", + "--no-prompt-templates", + "-e", + str(fixture), + "--model", + "handoff/fixture", + ] + for command in [["--version"], ["--help"], ["auth", "--help"]]: + p = subprocess.run( + [exe] + command, + cwd=root / "project", + env={**env, "DIFFEX_OFFLINE": "1"}, + capture_output=True, + text=True, + timeout=30, ) - if stage == "startup" and "Diffex v" in text: - os.write(fd, b"Reply with the fixture confirmation.\r") - stage = "reply" - elif stage == "reply" and "handoff-fixture-ok" in text: - os.write(fd, b"/login\r") - stage = "login" - elif stage == "login" and "Select authentication method:" in text: - os.write(fd, b"\r") - stage = "accounts" - elif stage == "accounts" and all( - name in text for name in ["ChatGPT", "OpenRouter", "xAI"] - ): - assert not re.search(r"\bRadius\b", text), text - mark = len(raw) - os.write(fd, b"radius") - stage = "account-search" - elif stage == "account-search" and b"No matching providers" in raw[mark:]: - mark = len(raw) - os.write(fd, b"\x03") - stage = "back" - elif stage == "back" and b"Select authentication method:" in raw[mark:]: - mark = len(raw) - os.write(fd, b"\x1b[B\r") - stage = "api-keys" - elif stage == "api-keys" and b"Select provider to configure:" in raw[mark:]: - mark = len(raw) - os.write(fd, b"radius") - stage = "api-key-search" - elif stage == "api-key-search" and b"No matching providers" in raw[mark:]: - mark = len(raw) - os.write(fd, b"\x03") - stage = "api-key-back" - elif stage == "api-key-back" and b"Select authentication method:" in raw[mark:]: - mark = len(raw) - os.write(fd, b"\x03") - stage = "editor" - elif stage == "editor" and b"Ask Diffex" in raw[mark:]: - os.write(fd, b"/exit\r") - stage = "exit" - (out / f"{runtime}-terminal.txt").write_text(text) - assert stage == "exit", (stage, text[-4000:]) - ended, status = os.waitpid(pid, os.WNOHANG) - reap_deadline = time.monotonic() + 2 - while not ended and time.monotonic() < reap_deadline: - time.sleep(0.02) + assert p.returncode == 0, p.stderr + results.append( + {"runtime": runtime, "command": command, "exitCode": p.returncode} + ) + for command, expected in [ + (["--list-models"], "fixture"), + (["-p", "Reply with the fixture confirmation."], "handoff-fixture-ok"), + ]: + p = subprocess.run( + args + command, + cwd=root / "project", + env=env, + capture_output=True, + text=True, + timeout=30, + ) + assert p.returncode == 0, (p.stdout, p.stderr) + assert expected in p.stdout, (p.stdout, p.stderr) + results.append( + { + "runtime": runtime, + "command": command, + "exitCode": p.returncode, + "expected": expected, + } + ) + pid, fd = pty.fork() + if pid == 0: + os.chdir(root / "project") + fcntl.ioctl(1, termios.TIOCSWINSZ, struct.pack("HHHH", 30, 100, 0, 0)) + os.execve(exe, args, env) + raw = b"" + text = "" + stage = "startup" + deadline = time.monotonic() + 30 + mark = 0 + try: + while time.monotonic() < deadline: + if not select.select([fd], [], [], 0.1)[0]: + continue + try: + chunk = os.read(fd, 65536) + except OSError: + break + if not chunk: + break + raw += chunk + text = re.sub( + r"\x1b\[[0-?]*[ -/]*[@-~]", "", raw.decode(errors="replace") + ) + if stage == "startup" and "Diffex v" in text: + os.write(fd, b"Reply with the fixture confirmation.\r") + stage = "reply" + elif stage == "reply" and "handoff-fixture-ok" in text: + os.write(fd, b"/login\r") + stage = "login" + elif stage == "login" and "Select authentication method:" in text: + os.write(fd, b"\r") + stage = "accounts" + elif stage == "accounts" and all( + name in text for name in ["ChatGPT", "OpenRouter", "xAI"] + ): + assert not re.search(r"\bRadius\b", text), text + mark = len(raw) + os.write(fd, b"radius") + stage = "account-search" + elif stage == "account-search" and b"No matching providers" in raw[mark:]: + mark = len(raw) + os.write(fd, b"\x03") + stage = "back" + elif stage == "back" and b"Select authentication method:" in raw[mark:]: + mark = len(raw) + os.write(fd, b"\x1b[B\r") + stage = "api-keys" + elif stage == "api-keys" and b"Select provider to configure:" in raw[mark:]: + mark = len(raw) + os.write(fd, b"radius") + stage = "api-key-search" + elif stage == "api-key-search" and b"No matching providers" in raw[mark:]: + mark = len(raw) + os.write(fd, b"\x03") + stage = "api-key-back" + elif stage == "api-key-back" and b"Select authentication method:" in raw[mark:]: + mark = len(raw) + os.write(fd, b"\x03") + stage = "editor" + elif stage == "editor" and b"Ask Diffex" in raw[mark:]: + os.write(fd, b"/exit\r") + stage = "exit" + (out / f"{runtime}-terminal.txt").write_text(text) + assert stage == "exit", (stage, text[-4000:]) ended, status = os.waitpid(pid, os.WNOHANG) - assert ended == pid, "Terminal did not exit within its bounded deadline" - pid = None - assert os.waitstatus_to_exitcode(status) == 0 - (out / f"{runtime}-terminal.txt").write_text(text) - events = [ - json.loads(l) for l in (root / "events.jsonl").read_text().splitlines() - ] - assert not any(e["type"] == "blocked-network" for e in events), events - assert sum(e["type"] == "inference" for e in events) == 2, events - requests = [e["url"] for e in events if e["type"] == "request"] - assert requests and set(requests) == { - "https://api.diffex.ai/api/latest-version" - }, events - results.append( - { - "runtime": runtime, - "command": "interactive prompt, subscription and API-key Radius exclusion, exit", - "exitCode": 0, - "events": events, - "accounts": ["ChatGPT", "OpenRouter", "xAI"], - "cleanup": "process exited and isolated directories removed", - } - ) + reap_deadline = time.monotonic() + 2 + while not ended and time.monotonic() < reap_deadline: + time.sleep(0.02) + ended, status = os.waitpid(pid, os.WNOHANG) + assert ended == pid, "Terminal did not exit within its bounded deadline" + pid = None + assert os.waitstatus_to_exitcode(status) == 0 + (out / f"{runtime}-terminal.txt").write_text(text) + events = [ + json.loads(l) for l in (root / "events.jsonl").read_text().splitlines() + ] + assert not any(e["type"] == "blocked-network" for e in events), events + assert sum(e["type"] == "inference" for e in events) == 2, events + requests = [e["url"] for e in events if e["type"] == "request"] + assert requests and set(requests) == { + "https://api.diffex.ai/api/latest-version" + }, events + results.append( + { + "runtime": runtime, + "command": "interactive prompt, subscription and API-key Radius exclusion, exit", + "exitCode": 0, + "events": events, + "accounts": ["ChatGPT", "OpenRouter", "xAI"], + "cleanup": "process exited and isolated directories removed", + } + ) + finally: + os.close(fd) + if pid: + try: + os.kill(pid, 9) + except ProcessLookupError: + pass + os.waitpid(pid, 0) finally: - os.close(fd) - if pid: - try: - os.kill(pid, 9) - except ProcessLookupError: - pass - os.waitpid(pid, 0) - finally: - if fixture is not None: - fixture.unlink(missing_ok=True) - shutil.rmtree(root) - assert not root.exists() -(out / "smoke.json").write_text(json.dumps(results, indent=2) + "\n") -print(json.dumps(results, indent=2)) + if fixture is not None: + fixture.unlink(missing_ok=True) + shutil.rmtree(root) + assert not root.exists() + (out / "smoke.json").write_text(json.dumps(results, indent=2) + "\n") + print(json.dumps(results, indent=2)) + return results + +if __name__ == "__main__": + if len(sys.argv) not in (2, 3): + raise SystemExit( + "Usage: python3 scripts/verify-candidate-artifacts.py ARTIFACT_DIRECTORY" + ) + out = pathlib.Path(sys.argv[1]).resolve() + fixture_source = ( + pathlib.Path(__file__).resolve().parent / "fixtures/artifact-handoff.mjs" + ) + executables = { + "node": str(out / "node/node_modules/.bin/diffex"), + "bun-install": str(out / "bun-install/node_modules/.bin/diffex"), + } + if len(sys.argv) == 3: + executables = {"consumer": str(pathlib.Path(sys.argv[2]).resolve() / "node_modules/.bin/diffex")} + verify(out, executables, fixture_source) From 93e37c9185c2a72b82fb0aba02806af6eae8a403 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Tue, 27 Oct 2026 15:59:09 -0400 Subject: [PATCH 02/12] feat(coding-agent): add installed CLI and terminal behavior scenarios --- .../baselines/startup-120x40.txt | 40 ++ .../baselines/startup-80x24.txt | 24 ++ .../baselines/startup-resize-80x24.txt | 24 ++ scripts/daytona-behaviors/fixture.mjs | 235 ++++++++++ scripts/daytona-behaviors/fixture.test.mjs | 144 +++++++ scripts/daytona-behaviors/install.mjs | 22 + scripts/daytona-behaviors/installed.py | 408 ++++++++++++++++++ scripts/daytona-behaviors/live-guard.mjs | 35 ++ scripts/daytona-behaviors/live-guard.test.mjs | 41 ++ scripts/daytona-behaviors/test_installed.py | 69 +++ 10 files changed, 1042 insertions(+) create mode 100644 scripts/daytona-behaviors/baselines/startup-120x40.txt create mode 100644 scripts/daytona-behaviors/baselines/startup-80x24.txt create mode 100644 scripts/daytona-behaviors/baselines/startup-resize-80x24.txt create mode 100644 scripts/daytona-behaviors/fixture.mjs create mode 100644 scripts/daytona-behaviors/fixture.test.mjs create mode 100644 scripts/daytona-behaviors/install.mjs create mode 100644 scripts/daytona-behaviors/installed.py create mode 100644 scripts/daytona-behaviors/live-guard.mjs create mode 100644 scripts/daytona-behaviors/live-guard.test.mjs create mode 100644 scripts/daytona-behaviors/test_installed.py diff --git a/scripts/daytona-behaviors/baselines/startup-120x40.txt b/scripts/daytona-behaviors/baselines/startup-120x40.txt new file mode 100644 index 000000000..13db84d0a --- /dev/null +++ b/scripts/daytona-behaviors/baselines/startup-120x40.txt @@ -0,0 +1,40 @@ + + /----\_/----\ + | \|/ | + \---\ /---/ Diffex v + X + /---/ \---\ + | /|\ | + \____/^\____/ +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +› Ask Diffex to do something for you. +──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── +fixture · thinking off · context 0.0% / 131k · harness default · ctrl+p model · /hotkeys + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/scripts/daytona-behaviors/baselines/startup-80x24.txt b/scripts/daytona-behaviors/baselines/startup-80x24.txt new file mode 100644 index 000000000..157153cd5 --- /dev/null +++ b/scripts/daytona-behaviors/baselines/startup-80x24.txt @@ -0,0 +1,24 @@ + + /----\_/----\ + | \|/ | + \---\ /---/ Diffex v + X + /---/ \---\ + | /|\ | + \____/^\____/ +──────────────────────────────────────────────────────────────────────────────── + +──────────────────────────────────────────────────────────────────────────────── +› Ask Diffex to do something for you. +──────────────────────────────────────────────────────────────────────────────── +fixture · thinking off · context 0.0% / 131k · harness default · /hotkeys + + + + + + + + + + diff --git a/scripts/daytona-behaviors/baselines/startup-resize-80x24.txt b/scripts/daytona-behaviors/baselines/startup-resize-80x24.txt new file mode 100644 index 000000000..157153cd5 --- /dev/null +++ b/scripts/daytona-behaviors/baselines/startup-resize-80x24.txt @@ -0,0 +1,24 @@ + + /----\_/----\ + | \|/ | + \---\ /---/ Diffex v + X + /---/ \---\ + | /|\ | + \____/^\____/ +──────────────────────────────────────────────────────────────────────────────── + +──────────────────────────────────────────────────────────────────────────────── +› Ask Diffex to do something for you. +──────────────────────────────────────────────────────────────────────────────── +fixture · thinking off · context 0.0% / 131k · harness default · /hotkeys + + + + + + + + + + diff --git a/scripts/daytona-behaviors/fixture.mjs b/scripts/daytona-behaviors/fixture.mjs new file mode 100644 index 000000000..b15bc767f --- /dev/null +++ b/scripts/daytona-behaviors/fixture.mjs @@ -0,0 +1,235 @@ +// Installed acceptance extension: only provider inference/HTTP are synthetic. +// Sessions, built-in tools, auth resolution, agent orchestration and UI remain real. +import { appendFileSync } from "node:fs"; +import { syncBuiltinESMExports } from "node:module"; +import { Socket } from "node:net"; +import { createAssistantMessageEventStream } from "@diffexai/diffex-ai"; + +const toolSteps = [ + { name: "read", arguments: { path: "input.txt" } }, + { name: "write", arguments: { path: "output.txt", content: "behavior-written\n" } }, + { + name: "edit", + arguments: { path: "output.txt", edits: [{ oldText: "written", newText: "edited" }] }, + }, + { name: "bash", arguments: { command: "cat output.txt > bash-result.txt" } }, +]; + +function textOf(message) { + return typeof message.content === "string" + ? message.content + : message.content.filter((block) => block.type === "text").map((block) => block.text).join("\n"); +} + +function checkedResults(messages, promptIndex) { + const following = messages.slice(promptIndex + 1); + const calls = following.flatMap((message) => + message.role === "assistant" ? message.content.filter((block) => block.type === "toolCall") : [], + ); + const results = following.filter((message) => message.role === "toolResult"); + if (results.length > toolSteps.length || calls.length !== results.length) { + throw new Error("Behavior tools context has missing or extra tool results"); + } + for (const [index, result] of results.entries()) { + const step = toolSteps[index]; + const call = calls[index]; + if ( + result.isError || + result.toolName !== step.name || + call.name !== step.name || + call.id !== result.toolCallId || + JSON.stringify(call.arguments) !== JSON.stringify(step.arguments) + ) { + throw new Error(`Behavior ${step.name} tool result/context mismatch`); + } + } + return results; +} + +function authResponse() { + const text = "behavior-auth-ok"; + const item = { + type: "message", + id: "msg_behavior_auth", + role: "assistant", + status: "completed", + content: [{ type: "output_text", text, annotations: [] }], + }; + const events = [ + { type: "response.created", response: { id: "resp_behavior_auth", status: "in_progress" } }, + { type: "response.output_item.added", output_index: 0, item: { ...item, status: "in_progress", content: [] } }, + { type: "response.content_part.added", output_index: 0, content_index: 0, item_id: item.id, part: { type: "output_text", text: "", annotations: [] } }, + { type: "response.output_text.delta", output_index: 0, content_index: 0, item_id: item.id, delta: text }, + { type: "response.output_text.done", output_index: 0, content_index: 0, item_id: item.id, text }, + { type: "response.output_item.done", output_index: 0, item }, + { + type: "response.completed", + response: { + id: "resp_behavior_auth", + object: "response", + model: "gpt-4.1-mini", + status: "completed", + output: [item], + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2, input_tokens_details: { cached_tokens: 0 }, output_tokens_details: { reasoning_tokens: 0 } }, + }, + }, + ]; + const encoder = new TextEncoder(); + return new Response(new ReadableStream({ + start(controller) { + for (const [sequence_number, event] of events.entries()) { + controller.enqueue(encoder.encode(`event: ${event.type}\ndata: ${JSON.stringify({ ...event, sequence_number })}\n\n`)); + } + controller.close(); + }, + }), { headers: { "content-type": "text/event-stream", "x-request-id": "req_behavior_auth" } }); +} + +export default function (diffex) { + const recordPath = process.env.DIFFEX_BEHAVIOR_RECORD; + if (!recordPath) throw new Error("DIFFEX_BEHAVIOR_RECORD is required"); + const record = (value) => appendFileSync(recordPath, `${JSON.stringify(value)}\n`); + diffex.on("agent_settled", () => { record({ type: "settled" }); }); + diffex.on("tool_result", (event) => { record({ ...event, type: "tool_result" }); }); + const transport = process.env.DIFFEX_BEHAVIOR_TRANSPORT === "1"; + const blocked = (details) => { + record({ type: "blocked-network", ...details }); + throw new Error("Network disabled by behavior acceptance fixture"); + }; + globalThis.fetch = async (input, init) => { + const url = input instanceof Request ? input.url : String(input); + const method = init?.method ?? (input instanceof Request ? input.method : "GET"); + record({ type: "request", url, method }); + if (url === "https://api.diffex.ai/api/latest-version" && method === "GET") { + return Response.json({ error: "no_published_release" }, { status: 404 }); + } + if (!transport || url !== "https://api.openai.com/v1/responses" || method !== "POST") { + return blocked({ transport: "fetch", url, method }); + } + const headers = new Headers(input instanceof Request ? input.headers : undefined); + new Headers(init?.headers).forEach((value, name) => headers.set(name, value)); + const allowedHeaders = new Set([ + "accept", "authorization", "content-type", "user-agent", "session_id", "x-client-request-id", + "x-stainless-arch", "x-stainless-lang", "x-stainless-os", "x-stainless-package-version", + "x-stainless-runtime", "x-stainless-runtime-version", "x-stainless-retry-count", "x-stainless-timeout", + ]); + for (const name of headers.keys()) { + if (!allowedHeaders.has(name)) return blocked({ transport: "fetch", url, header: name }); + } + if (process.env.OPENAI_API_KEY !== "fake" || headers.get("authorization") !== "Bearer fake" || headers.get("content-type") !== "application/json") { + return blocked({ transport: "fetch", url, reason: "Unexpected auth or content-type" }); + } + const body = init?.body ?? (input instanceof Request ? await input.text() : undefined); + if (typeof body !== "string") return blocked({ transport: "fetch", url, reason: "Expected JSON body" }); + const payload = JSON.parse(body); + if (payload.model !== "gpt-4.1-mini" || payload.stream !== true || !Array.isArray(payload.input)) { + return blocked({ transport: "fetch", url, reason: "Unexpected model or streaming payload" }); + } + record({ type: "inference", provider: "openai", model: payload.model, transport: "fetch", credentialVerified: true }); + record({ type: "context", transport: "fetch", input: payload.input }); + record({ type: "auth-request", url, credentialVerified: true, headers: { ...Object.fromEntries(headers), authorization: "Bearer [fixture]" }, payload }); + return authResponse(); + }; + Socket.prototype.connect = function () { + return blocked({ transport: "socket" }); + }; + syncBuiltinESMExports(); + // Crucially, transport mode does not register/override any provider or auth. + if (transport) return; + + diffex.registerProvider("behavior", { + baseUrl: "https://behavior.invalid", + apiKey: "local-fixture", + api: "behavior", + models: [{ + id: "fixture", + name: "Behavior fixture", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 131072, + maxTokens: 1024, + }], + streamSimple(model, context, options = {}) { + const stream = createAssistantMessageEventStream(); + const message = { + role: "assistant", content: [], api: model.api, provider: model.provider, model: model.id, + usage: { input: 1, output: 1, cacheRead: 0, cacheWrite: 0, totalTokens: 2, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, + stopReason: "pending", timestamp: Date.now(), + }; + queueMicrotask(async () => { + try { + const messages = context.messages; + const promptIndex = messages.findLastIndex((entry) => entry.role === "user"); + const prompt = promptIndex < 0 ? "" : textOf(messages[promptIndex]); + const modes = prompt.match(/\bBEHAVIOR_(?:PRINT|TOOLS|CANCEL|RECOVER|RESUME)\b/g) ?? []; + if (new Set(modes).size !== 1) throw new Error("Expected exactly one BEHAVIOR mode in latest user prompt"); + const mode = modes[0]; + record({ type: "inference", provider: model.provider, model: model.id, mode, prompt }); + record({ type: "context", mode, messages, tools: context.tools?.map((tool) => tool.name) ?? [] }); + for (const result of messages.filter((entry) => entry.role === "toolResult")) { + record({ type: "tool-result", mode, ...result }); + } + await options.onPayload?.({ model: model.id, messages }, model); + await options.onResponse?.({ status: 200, headers: {} }, model); + if (options.signal?.aborted) throw new Error("Request was aborted"); + stream.push({ type: "start", partial: message }); + let text; + if (mode === "BEHAVIOR_TOOLS") { + const results = checkedResults(messages, promptIndex); + if (results.length < toolSteps.length) { + const step = toolSteps[results.length]; + if (!context.tools?.some((tool) => tool.name === step.name)) throw new Error(`Missing built-in ${step.name} tool`); + const call = { type: "toolCall", id: `behavior_${promptIndex}_${step.name}`, ...step }; + message.content.push(call); + stream.push({ type: "toolcall_start", contentIndex: 0, partial: message }); + stream.push({ type: "toolcall_delta", contentIndex: 0, delta: JSON.stringify(step.arguments), partial: message }); + stream.push({ type: "toolcall_end", contentIndex: 0, toolCall: call, partial: message }); + message.stopReason = "toolUse"; + stream.push({ type: "done", reason: "toolUse", message }); + stream.end(message); + return; + } + text = "behavior-tools-ok"; + } else if (mode === "BEHAVIOR_RESUME") { + const earlierIndex = messages.findLastIndex((entry, index) => index < promptIndex && entry.role === "user" && /\bBEHAVIOR_TOOLS\b/.test(textOf(entry))); + if (earlierIndex < 0 || checkedResults(messages.slice(0, promptIndex), earlierIndex).length !== toolSteps.length || !messages.slice(earlierIndex + 1, promptIndex).some((entry) => entry.role === "assistant" && textOf(entry) === "behavior-tools-ok")) { + throw new Error("Resumed context is missing the earlier BEHAVIOR_TOOLS prompt, results or final answer"); + } + record({ type: "resume-verified", prompt: textOf(messages[earlierIndex]), toolNames: toolSteps.map((step) => step.name) }); + text = "behavior-resumed-ok"; + } else { + text = { BEHAVIOR_PRINT: "behavior-print-ok", BEHAVIOR_RECOVER: "behavior-recovered-ok", BEHAVIOR_CANCEL: "behavior-cancel-started" }[mode]; + } + message.content.push({ type: "text", text }); + stream.push({ type: "text_start", contentIndex: 0, partial: message }); + stream.push({ type: "text_delta", contentIndex: 0, delta: text, partial: message }); + if (mode === "BEHAVIOR_CANCEL") { + if (!options.signal) throw new Error("Cancellation fixture requires an AbortSignal"); + record({ type: "cancel-started" }); + await new Promise((resolve) => { + // Keep print/RPC processes alive as a real pending HTTP request would. + const keepAlive = setInterval(() => {}, 1000); + const abort = () => { clearInterval(keepAlive); resolve(); }; + options.signal.addEventListener("abort", abort, { once: true }); + if (options.signal.aborted) { options.signal.removeEventListener("abort", abort); abort(); } + }); + record({ type: "abort", mode, aborted: options.signal.aborted }); + throw new Error("Behavior fixture request aborted"); + } + stream.push({ type: "text_end", contentIndex: 0, content: text, partial: message }); + message.stopReason = "stop"; + stream.push({ type: "done", reason: "stop", message }); + stream.end(message); + } catch (error) { + message.stopReason = options.signal?.aborted ? "aborted" : "error"; + message.errorMessage = error instanceof Error ? error.message : String(error); + record({ type: "stream-error", reason: message.stopReason, message: message.errorMessage }); + stream.push({ type: "error", reason: message.stopReason, error: message }); + stream.end(message); + } + }); + return stream; + }, + }); +} diff --git a/scripts/daytona-behaviors/fixture.test.mjs b/scripts/daytona-behaviors/fixture.test.mjs new file mode 100644 index 000000000..11c837d0d --- /dev/null +++ b/scripts/daytona-behaviors/fixture.test.mjs @@ -0,0 +1,144 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { Socket } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; +import fixture from "./fixture.mjs"; + +const cli = fileURLToPath(new URL("../../packages/coding-agent/dist/cli.js", import.meta.url)); +const extension = fileURLToPath(new URL("./fixture.mjs", import.meta.url)); + +function records(path) { + return readFileSync(path, "utf8").trim().split("\n").map((line) => JSON.parse(line)); +} + +function runCli(directory, record, args, env = {}) { + const result = spawnSync(process.execPath, [cli, "--no-extensions", "-e", extension, ...args], { + cwd: directory, + env: { + ...process.env, + DIFFEX_CODING_AGENT_DIR: join(directory, "config"), + DIFFEX_BEHAVIOR_RECORD: record, + DIFFEX_BEHAVIOR_TRANSPORT: "0", + ...env, + }, + encoding: "utf8", + timeout: 30000, + }); + assert.equal(result.error, undefined); + assert.equal(result.status, 0, result.stderr || result.stdout); + return result.stdout; +} + +test("installed extension drives real print, built-in tools and persisted resume", () => { + const directory = mkdtempSync(join(tmpdir(), "behavior-fixture-")); + try { + const record = join(directory, "record.jsonl"); + const session = join(directory, "session.jsonl"); + writeFileSync(join(directory, "input.txt"), "behavior-input\n"); + assert.match(runCli(directory, record, ["--model", "behavior/fixture", "--no-session", "-p", "BEHAVIOR_PRINT"]), /behavior-print-ok/); + assert.match(runCli(directory, record, ["--model", "behavior/fixture", "--session", session, "-p", "BEHAVIOR_TOOLS"]), /behavior-tools-ok/); + assert.equal(readFileSync(join(directory, "output.txt"), "utf8"), "behavior-edited\n"); + assert.equal(readFileSync(join(directory, "bash-result.txt"), "utf8"), "behavior-edited\n"); + assert.match(runCli(directory, record, ["--model", "behavior/fixture", "--session", session, "-p", "BEHAVIOR_RESUME"]), /behavior-resumed-ok/); + const events = records(record); + assert.equal(events.filter((event) => event.type === "settled").length, 3); + assert.deepEqual(events.filter((event) => event.type === "tool_result").map((event) => event.toolName), ["read", "write", "edit", "bash"]); + assert.equal(events.filter((event) => event.type === "resume-verified").length, 1); + assert.equal(events.some((event) => event.type === "blocked-network"), false); + const resumed = events.find((event) => event.type === "context" && event.mode === "BEHAVIOR_RESUME"); + assert.equal(resumed.messages.filter((message) => message.role === "toolResult").length, 4); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test("built-in OpenAI provider resolves fake environment credentials and consumes Responses SSE", () => { + const directory = mkdtempSync(join(tmpdir(), "behavior-auth-")); + try { + const record = join(directory, "record.jsonl"); + assert.match(runCli(directory, record, ["--model", "openai/gpt-4.1-mini", "--no-session", "-p", "BEHAVIOR_AUTH"], { + DIFFEX_BEHAVIOR_TRANSPORT: "1", OPENAI_API_KEY: "fake", + }), /behavior-auth-ok/); + const events = records(record); + assert(events.some((event) => event.type === "auth-request" && event.credentialVerified)); + assert.equal(events.filter((event) => event.type === "settled").length, 1); + assert.equal(events.some((event) => event.type === "blocked-network"), false); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); + +test("cancellation stays pending until AbortSignal, then recovery and missing-resume checks work", async () => { + const directory = mkdtempSync(join(tmpdir(), "behavior-cancel-")); + const originalFetch = globalThis.fetch; + const originalConnect = Socket.prototype.connect; + const originalEnv = { ...process.env }; + try { + const record = join(directory, "record.jsonl"); + process.env.DIFFEX_BEHAVIOR_RECORD = record; + process.env.DIFFEX_BEHAVIOR_TRANSPORT = "0"; + let provider; + fixture({ on() {}, registerProvider(_name, config) { provider = config; } }); + const model = { api: "behavior", provider: "behavior", id: "fixture" }; + const controller = new AbortController(); + const stream = provider.streamSimple(model, { messages: [{ role: "user", content: "BEHAVIOR_CANCEL", timestamp: 1 }] }, { signal: controller.signal }); + let settled = false; + stream.result().then(() => { settled = true; }); + for await (const event of stream) { + if (event.type === "text_delta") { + assert.equal(event.delta, "behavior-cancel-started"); + await new Promise((resolve) => setTimeout(resolve, 20)); + assert.equal(settled, false); + assert.equal(records(record).some((entry) => entry.type === "abort"), false); + controller.abort(); + } + } + assert.equal((await stream.result()).stopReason, "aborted"); + assert(records(record).some((entry) => entry.type === "abort" && entry.aborted)); + const recovered = await provider.streamSimple(model, { messages: [{ role: "user", content: "BEHAVIOR_RECOVER", timestamp: 2 }] }).result(); + assert.equal(recovered.content[0].text, "behavior-recovered-ok"); + const invalidResume = await provider.streamSimple(model, { messages: [{ role: "user", content: "BEHAVIOR_RESUME", timestamp: 3 }] }).result(); + assert.equal(invalidResume.stopReason, "error"); + assert.match(invalidResume.errorMessage, /missing the earlier/); + await assert.rejects(fetch("https://example.com/"), /Network disabled/); + assert.throws(() => new Socket().connect(443, "example.com"), /Network disabled/); + assert.equal((await fetch("https://api.diffex.ai/api/latest-version")).status, 404); + } finally { + globalThis.fetch = originalFetch; + Socket.prototype.connect = originalConnect; + process.env = originalEnv; + rmSync(directory, { recursive: true, force: true }); + } +}); + +test("transport rejects unexpected hosts, credentials and custom headers without registering providers", async () => { + const directory = mkdtempSync(join(tmpdir(), "behavior-network-")); + const originalFetch = globalThis.fetch; + const originalConnect = Socket.prototype.connect; + const originalEnv = { ...process.env }; + try { + process.env.DIFFEX_BEHAVIOR_RECORD = join(directory, "record.jsonl"); + process.env.DIFFEX_BEHAVIOR_TRANSPORT = "1"; + process.env.OPENAI_API_KEY = "fake"; + fixture({ on() {}, registerProvider() { assert.fail("Transport fixture must not register a provider"); } }); + const request = { + method: "POST", + headers: { authorization: "Bearer fake", "content-type": "application/json" }, + body: JSON.stringify({ model: "gpt-4.1-mini", stream: true, input: [] }), + }; + await assert.rejects(fetch("https://unexpected.example/v1/responses", request), /Network disabled/); + await assert.rejects(fetch("https://api.openai.com/v1/responses", { ...request, headers: { ...request.headers, authorization: "Bearer real-secret" } }), /Network disabled/); + await assert.rejects(fetch("https://api.openai.com/v1/responses", { ...request, headers: { ...request.headers, "x-api-key": "unexpected" } }), /Network disabled/); + assert.equal(records(process.env.DIFFEX_BEHAVIOR_RECORD).filter((event) => event.type === "blocked-network").length, 3); + assert.equal(readFileSync(process.env.DIFFEX_BEHAVIOR_RECORD, "utf8").includes("real-secret"), false); + } finally { + globalThis.fetch = originalFetch; + Socket.prototype.connect = originalConnect; + process.env = originalEnv; + rmSync(directory, { recursive: true, force: true }); + } +}); diff --git a/scripts/daytona-behaviors/install.mjs b/scripts/daytona-behaviors/install.mjs new file mode 100644 index 000000000..10f221cee --- /dev/null +++ b/scripts/daytona-behaviors/install.mjs @@ -0,0 +1,22 @@ +import assert from "node:assert/strict"; +import { readFileSync, realpathSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { installConsumer, verifyTarballs } from "../artifact-installation.mjs"; + +const [artifacts, consumer, manager, commit] = process.argv.slice(2); +assert(artifacts && consumer && ["npm", "bun"].includes(manager) && /^[a-f0-9]{40}$/.test(commit), + "Usage: node install.mjs ARTIFACTS CONSUMER npm|bun COMMIT"); +const manifest = JSON.parse(readFileSync(join(artifacts, "manifest.json"), "utf8")); +assert.equal(manifest.source.commit, commit, "Retained source does not match triggering commit"); +assert.equal(manifest.schemaVersion, 1); +const packages = manifest.packages; +assert.equal(packages.length, 1, "Expected the standalone CLI distribution"); +assert.equal(packages[0].name, "@diffexai/diffex"); +verifyTarballs(artifacts, packages, packages); +installConsumer(resolve(consumer), manager, packages, packages, resolve(artifacts)); +const executable = join(resolve(consumer), "node_modules/.bin/diffex"); +assert(realpathSync(executable).startsWith(`${resolve(consumer)}/node_modules/`)); +writeFileSync(join(consumer, "installation.json"), `${JSON.stringify({ + manager, executable, resolved: realpathSync(executable), commit, + artifacts: packages.map(({ name, sha256 }) => ({ name, sha256 })), integrity: "passed", +}, null, 2)}\n`); diff --git a/scripts/daytona-behaviors/installed.py b/scripts/daytona-behaviors/installed.py new file mode 100644 index 000000000..9b8ee57dd --- /dev/null +++ b/scripts/daytona-behaviors/installed.py @@ -0,0 +1,408 @@ +"""Installed-package acceptance. No repository imports, provider keys in evidence, or UI mocks.""" +import argparse +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import re +import shutil +import shlex +import subprocess +import sys +import tempfile +import time +import uuid +import xml.etree.ElementTree as ET + +HERE = Path(__file__).resolve().parent + + +def read_events(path): + return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else [] + + +def write_json(path, value): + path.write_text(json.dumps(value, indent=2) + "\n") + + +class Terminal: + """Real tmux PTY, isolated server, bounded state waits, and incremental capture.""" + def __init__(self, executable, args, env, project, evidence, deadline): + self.socket = "diffex-behavior-" + uuid.uuid4().hex + self.evidence, self.deadline = evidence, deadline + self.env = env + (evidence / "tmux.conf").write_text("set -g extended-keys on\nset -gq extended-keys-format csi-u\n") + self.command("new-session", "-d", "-s", "behavior", "-x", "80", "-y", "24", + "-c", str(project), + " ".join(shlex.quote(a) for a in [executable, *args])) + self.command("set-option", "-g", "remain-on-exit", "on") + self.command("pipe-pane", "-o", "cat >> " + shlex.quote(str(evidence / "terminal.raw"))) + + def command(self, *args): + result = subprocess.run(["tmux", "-f", str(self.evidence / "tmux.conf"), "-L", self.socket, *args], capture_output=True, text=True, + timeout=10, env=self.env) + if result.returncode: + raise RuntimeError("tmux: " + result.stderr) + return result.stdout + + def send(self, text): + self.command("send-keys", "-l", text) + self.command("send-keys", "Enter") + + def capture(self): + screen = self.command("capture-pane", "-p", "-S", "-2000") + (self.evidence / "terminal.txt").write_text(screen) + return screen + + def wait(self, text, seconds=30): + end = min(self.deadline, time.monotonic() + seconds) + while time.monotonic() < end: + screen = self.capture() + if text in screen: + return screen + time.sleep(.1) + raise AssertionError(f"Terminal did not reach {text!r} within bounded wait") + + def settled(self, record, previous, seconds=30): + end = min(self.deadline, time.monotonic() + seconds) + while time.monotonic() < end: + self.capture() + if sum(e["type"] == "settled" for e in read_events(record)) > previous: + return + time.sleep(.1) + raise AssertionError("Agent did not settle before deadline") + + def screen(self, name, size, baseline): + width, height = size + self.command("resize-window", "-x", str(width), "-y", str(height)) + # Wait for a stable rendered layout, not an arbitrary startup sleep. + end = min(self.deadline, time.monotonic() + 5) + previous, stable = None, 0 + while time.monotonic() < end: + screen = self.command("capture-pane", "-p") + stable = stable + 1 if screen == previous else 0 + previous = screen + if stable >= 3: + break + time.sleep(.1) + assert stable >= 3, "Terminal layout never stabilized" + (self.evidence / (name + ".txt")).write_text(screen) + normalized = normalize_screen(screen, self.env) + (self.evidence / (name + ".normalized.txt")).write_text(normalized) + expected = HERE / "baselines" / (name + ".txt") + if baseline: + expected.parent.mkdir(exist_ok=True) + expected.write_text(normalized) + assert expected.exists(), f"Missing reviewed baseline: {expected.name}" + assert normalized == expected.read_text(), f"Layout differs: {expected.name} (see captures)" + assert len(screen.splitlines()) == height, "Incorrect terminal height" + assert all(len(line) <= width for line in screen.splitlines()), "Terminal overflow" + + def exit(self): + self.send("/exit") + end = min(self.deadline, time.monotonic() + 10) + while time.monotonic() < end: + self.capture() + dead = self.command("display-message", "-p", "#{pane_dead}:#{pane_dead_status}").strip() + if dead.startswith("1:"): + assert dead == "1:0", f"Unclean terminal exit: {dead}" + return + time.sleep(.1) + raise AssertionError("Terminal failed to exit") + + def close(self): + try: + self.capture() + finally: + self.command("kill-server") + + +def normalize_screen(screen, env): + # Only machine-specific identity/timing is normalized; whitespace and geometry remain exact. + screen = screen.replace(str(Path(env["DIFFEX_PROJECT"]).resolve()), "") + screen = screen.replace(env["HOME"], "").replace(env["DIFFEX_PROJECT"], "") + screen = re.sub(r"Diffex v[\w.\-]+", "Diffex v", screen) + screen = re.sub(r"\b\d+(?:\.\d+)?[km]?/\d+(?:\.\d+)?[km]?", "", screen) + screen = re.sub(r"\b\d+(?:\.\d+)?(?:ms|s)\b", "