diff --git a/.github/workflows/verify-release-publication.yml b/.github/workflows/verify-release-publication.yml index 62e6d1c99..236b3ed66 100644 --- a/.github/workflows/verify-release-publication.yml +++ b/.github/workflows/verify-release-publication.yml @@ -8,6 +8,11 @@ on: type: boolean default: false + check_client_canaries: + description: Verify isolated source CLI startup and self-update against staging + type: boolean + default: false + permissions: contents: read @@ -29,6 +34,9 @@ jobs: uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: '22' + - name: Install canary dependencies without lifecycle scripts + if: inputs.check_client_canaries + run: npm ci --ignore-scripts - name: Verify AWS CLI run: aws --version - name: Check controlled publisher regressions @@ -40,8 +48,11 @@ jobs: AWS_DEFAULT_REGION: auto AWS_EC2_METADATA_DISABLED: 'true' CHECK_PUBLIC_API: ${{ inputs.check_public_api }} + CHECK_CLIENT_CANARIES: ${{ inputs.check_client_canaries }} run: | - if [ "$CHECK_PUBLIC_API" = "true" ]; then + if [ "$CHECK_CLIENT_CANARIES" = "true" ]; then + node scripts/verify-release-publication.mjs --staging --client-canaries + elif [ "$CHECK_PUBLIC_API" = "true" ]; then node scripts/verify-release-publication.mjs --staging --public-api else node scripts/verify-release-publication.mjs --staging diff --git a/docs/release/client-canaries.md b/docs/release/client-canaries.md new file mode 100644 index 000000000..c71436ce4 --- /dev/null +++ b/docs/release/client-canaries.md @@ -0,0 +1,110 @@ +# Staging release client acceptance + +Verified on 2026-09-10 with Node 22.19.0 and AWS CLI 1.46.1. +All 21 source CLI canaries passed against real R2 and the deployed staging Worker. +The CLI source is based on `9b6f2dcab` from PR #204, with the reusable canary harness added on `feat/staging-version-canaries`. +The cloud source and regression checks use `01c1205` from Diffex Cloud PR #145. +The [cloud staging record](https://github.com/diffexai/diffex-cloud/blob/feat/serve-verified-releases/docs/public-api-staging.md) documents the existing deployment; this run made no Worker deployment. + +## Executed command and boundary + +From the CLI repository root, with the dedicated staging credentials supplied through the environment: + +```sh +node scripts/verify-release-publication.mjs --staging --client-canaries +``` + +The real publisher verifies controlled npm responses for all 11 public workspace packages and writes only the private `diffex-artifacts-staging` bucket in account `6b24afa849f8254ab5e6e2d1956239cc`. +It created synthetic versions `0.1789086098745.0` and `0.1789086098745.1`; these are test fixtures, not published npm releases. +Every client requested `GET https://api.diffex.ai/api/latest-version` with manual redirect handling. +Only the test preload remapped that exact URL to `https://diffex-public-api-staging.yibai14651.workers.dev/api/latest-version`. +Actual sockets were restricted to that hostname on port 443. +Requests were anonymous, with no cookies or R2 credentials supplied to the clients. +Every response had a JSON content type and `Cache-Control: no-store`. + +Manual commands ran the source CLI with `update --self`, adding `--force` where listed. +Separate temporary manifests modeled installed versions below, equal to, and above the staging record. +Interactive clients ran with `--no-session --no-extensions --no-skills` in a real 100x30 PTY, waited for the staging response and welcome screen, and exited through `/exit` without submitting a model prompt. +The harness removed every temporary home, project, credential directory, event record, and simulated installation prefix. + +## Observed behavior + +| Canary | Version requests | HTTP | Exit | Install attempts | +| --- | --- | --- | --- | --- | +| missing release | 1 | 404 | 1 | 0 | +| forced missing release | 1 | 404 | 1 | 0 | +| interactive missing release | 1 | 404 | 0 | 0 | +| newer release | 1 | 200 | 0 | 1 | +| equal release | 1 | 200 | 0 | 0 | +| older release | 1 | 200 | 0 | 0 | +| forced equal release | 1 | 200 | 0 | 1 | +| forced older release | 1 | 200 | 0 | 1 | +| manual automatic opt-out | 1 | 200 | 0 | 1 | +| manual offline | 0 | none | 1 | 0 | +| interactive newer release | 1 | 200 | 0 | 0 | +| interactive offline | 0 | none | 0 | 0 | +| interactive automatic opt-out | 0 | none | 0 | 0 | +| forced invalid JSON | 3 | 503 | 1 | 0 | +| interactive unavailable | 1 | 503 | 0 | 0 | +| forced wrong schema | 3 | 503 | 1 | 0 | +| forced prerelease semver | 3 | 503 | 1 | 0 | +| forced missing canonical package | 3 | 503 | 1 | 0 | +| forced duplicate canonical package | 3 | 503 | 1 | 0 | +| forced package version mismatch | 3 | 503 | 1 | 0 | +| forced foreign package | 3 | 503 | 1 | 0 | + +Missing releases printed `No published Diffex release is available yet.` and exited 1 after one request. +Invalid stored records produced HTTP 503 and `Diffex update service is unavailable. Try again later.`, with three manual requests and no package commands even under force. +Interactive failures stayed quiet and made at most one version request. +Equal and older releases printed `already up to date` unless forced. +Offline mode suppressed all version requests, while automatic opt-out still allowed manual update. + +All four intercepted install attempts matched: + +```sh +npm --prefix install -g --ignore-scripts --min-release-age=0 @diffexai/diffex@0.1789086098745.0 +``` + +No package-manager process executed, no uninstall was attempted, and no installed software changed. +The preload intercepted the separate `https://pi.dev/api/report-install` call during non-offline startup and returned a controlled failure without forwarding it. +There was no Pi version-check traffic or any Pi socket connection. +Install telemetry, model catalogs, and Radius remain separate migration work. + +## Regression and artifact checks + +The focused checker, source CLI, package-command, and consumer-contract suites passed all 155 tests before the harness extraction. +The changed source CLI test file passed all 32 cases after extraction, and the final full isolated `./test.sh` run passed. +That full run includes the publisher and shared contract corpus, all changed tests, and default tests with live E2E disabled. +The checker tests cover invalid HTTP JSON, unsupported semver, exact identity validation, timeouts, HTTP errors, all supported redirect statuses without forwarding, retry limits, offline mode, and automatic opt-out. +The stored-record corpus covers wrong schemas, missing and duplicate packages, and identity/version mismatches. + +The CLI `npm run check`, workflow validation with actionlint 1.7.12, and Cloud-to-CLI contract document and fixture comparisons passed. +Cloud root `npm run check` passed 546 unit and 23 integration tests and its typecheck, lint, formatting, database, build, and audit checks. +The isolated Worker `npm run check` passed 335 tests, typecheck, lint, formatting, staging deployment dry run, and audit. + +The initial local artifact scan found the historical Pi endpoint in `packages/coding-agent/dist/utils/version-check.js`. +With explicit owner approval, `npm run build` from `packages/coding-agent` refreshed the local artifact without publishing or modifying generated files manually. +The final scan returned no matches: + +```sh +rg -n 'pi\.(dev|codes)/api/latest-version' packages/coding-agent/src packages/coding-agent/dist --glob '!*.map' +``` + +Both `src/utils/version-check.ts` and rebuilt `dist/utils/version-check.js` contain the canonical Diffex URL. +Generated build output remains untracked and is not included in the PR. + +## Cleanup and limits + +The run verified byte-preserving repeated publication, failed npm verification before and after publication, conditional advancement, older-announcement rejection, and interrupted-publication recovery. +Cleanup verified these exact keys absent before releasing the lock: + +- `releases/v1/latest.json` +- `releases/v1/releases/0.1789086098745.0.json` +- `releases/v1/releases/0.1789086098745.1.json` +- `verification/release-publication.lock` + +The final staging latest response was HTTP 404 `no_published_release`, and readiness was HTTP 200 with `release: "missing"`. +No production object, route, DNS record, Worker, Render service, landing application, npm package, or GitHub Release was changed. +The opt-in workflow input becomes dispatchable after the workflow reaches the default branch; this evidence comes from running the checked-in command locally. +These are source CLI canaries with intercepted installation, not acceptance of an installed published client or public binary distribution. +Production activation and real installation remain separately authorized release work. diff --git a/docs/release/publication.md b/docs/release/publication.md index 56d446b99..029d21eeb 100644 --- a/docs/release/publication.md +++ b/docs/release/publication.md @@ -102,7 +102,8 @@ If cleanup fails, the lock remains for investigation; inspect its recorded keys Do not empty the bucket or delete another run's records. `.github/workflows/verify-release-publication.yml` exposes the same verification as a manual-only workflow using the staging environment. -It requires no dependency installation and is excluded from default PR CI. +Publisher-only verification requires no dependency installation and is excluded from default PR CI. +The optional client canaries install the locked dependencies with lifecycle scripts disabled. Dispatch it from `main` once the workflow is merged; adding a workflow on a pull request does not make it dispatchable from the default branch. ## Verification evidence @@ -145,3 +146,27 @@ Both release keys, the latest pointer, and the shared lock were confirmed absent The production latest record remained absent. The [cloud staging acceptance record](https://github.com/diffexai/diffex-cloud/blob/feat/serve-verified-releases/docs/public-api-staging.md) records Worker deployment identities, the separate missing-binding failure and recovery test, and unchanged Render routing. This proves publisher-to-R2-to-HTTP behavior; the installed CLI authority migration and canary remain separate work. + +## Source CLI staging canaries + +Use the staging-only publication credentials and an AWS CLI supporting conditional writes: + +```sh +node scripts/verify-release-publication.mjs --staging --client-canaries +``` + +The manual workflow exposes this as `check_client_canaries`, disabled by default, and automatically includes the HTTP checks. +The command uses the same exclusive staging lock, ownership checks, conditional writes, and verified cleanup as publication verification. +It adds manual and interactive source CLI clients while each controlled record is present in real R2. +Each client gets an isolated home, project, credentials directory, and simulated npm prefix. +The test-only preload maps the exact production version URL to the fixed staging HTTPS endpoint and rejects redirects. +It permits sockets only to that staging hostname on port 443, intercepts every package-manager subprocess, and never passes storage credentials into the client. +The production client has no endpoint override. + +Canaries cover missing releases, newer/equal/older releases, normal and forced updates, manual retries, offline mode, automatic opt-out, and quiet interactive startup and `/exit` in a real 100x30 terminal. +Malformed JSON, wrong schema, prerelease versions, missing or duplicate canonical packages, package-version mismatches, and foreign identities must produce an unavailable response and no install even with force. +Every successful install attempt must match the complete expected npm argument list, pinned to `@diffexai/diffex@` without uninstall. +The JSON evidence captures requested and actual URLs, response status and public body, socket destination, intercepted install command, and verified local cleanup. +Install telemetry is intercepted separately without network traffic; its Pi dependency remains separate migration work. +The source canary does not establish published-package installation or binary acceptance. +See [the client canary acceptance record](client-canaries.md) for executed results and remaining limits. diff --git a/packages/coding-agent/test/fixtures/version-update-client.ts b/packages/coding-agent/test/fixtures/version-update-client.ts new file mode 100644 index 000000000..c74242741 --- /dev/null +++ b/packages/coding-agent/test/fixtures/version-update-client.ts @@ -0,0 +1,156 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { stripVTControlCharacters } from "node:util"; + +const packageDir = resolve(import.meta.dirname, "../.."); +const repositoryDir = resolve(packageDir, "../.."); +const packageJson = readFileSync(join(packageDir, "package.json"), "utf8"); +export const installed = JSON.parse(packageJson) as { name: string; version: string }; +export const endpoint = "https://api.diffex.ai/api/latest-version"; + +export interface RecordedEvent { + type: string; + status?: number; + body?: unknown; + cacheControl?: string; + contentType?: string; + host?: string; + port?: number; + url?: string; + method?: string; + userAgent?: string; + redirect?: string; + command?: string; + args?: string[]; +} + +export async function reproduce(options: { + staging?: boolean; + installedVersion?: string; + interactive?: boolean; + force?: boolean; + offline?: boolean; + skip?: boolean; + failure?: boolean; + packageName?: string; + version?: string; + response?: string; + status?: number; + redirect?: string; +}) { + const root = mkdtempSync(join(tmpdir(), "diffex-version-repro-")); + const agentDir = join(root, "agent"); + const recordPath = join(root, "events.jsonl"); + const homeDir = join(root, "home"); + const cwd = join(root, "project"); + for (const dir of [agentDir, homeDir, cwd]) mkdirSync(dir); + writeFileSync(join(agentDir, "settings.json"), JSON.stringify({ theme: "dark", packages: [] })); + writeFileSync(recordPath, ""); + const env: NodeJS.ProcessEnv = { + PATH: process.env.PATH, + HOME: homeDir, + USERPROFILE: homeDir, + TMPDIR: root, + XDG_CONFIG_HOME: join(homeDir, ".config"), + XDG_CACHE_HOME: join(homeDir, ".cache"), + TERM: "xterm-256color", + LANG: "en_US.UTF-8", + TSX_TSCONFIG_PATH: join(repositoryDir, "tsconfig.json"), + DIFFEX_CODING_AGENT_DIR: agentDir, + DIFFEX_NO_LOCAL_LLM: "1", + DIFFEX_REPRO_RECORD: recordPath, + DIFFEX_REPRO_STAGING: options.staging ? "1" : "", + DIFFEX_REPRO_VERSION: options.version ?? (options.force ? installed.version : "9999.0.0"), + DIFFEX_REPRO_PACKAGE: options.packageName ?? "@foreign-fixture/untrusted-cli", + DIFFEX_REPRO_RESPONSE: options.response, + DIFFEX_REPRO_STATUS: String(options.status ?? 200), + DIFFEX_REPRO_REDIRECT: options.redirect, + DIFFEX_REPRO_FAILURE: options.failure ? "1" : "", + DIFFEX_OFFLINE: options.offline ? "1" : "", + DIFFEX_SKIP_VERSION_CHECK: options.skip ? "1" : "", + }; + if (!options.interactive) { + const fakePackageDir = join(root, "lib", "node_modules", ...installed.name.split("/")); + mkdirSync(fakePackageDir, { recursive: true }); + writeFileSync( + join(fakePackageDir, "package.json"), + JSON.stringify({ ...installed, version: options.installedVersion ?? installed.version }), + ); + env.DIFFEX_PACKAGE_DIR = fakePackageDir; + } + const nodeArgs = [ + `--import=${import.meta.resolve("tsx")}`, + `--import=${join(import.meta.dirname, "version-update-guard.mjs")}`, + join(packageDir, "src/cli.ts"), + ...(options.interactive ? ["--no-session", "--no-extensions", "--no-skills"] : ["update", "--self"]), + ...(options.force ? ["--force"] : []), + ]; + const child = options.interactive + ? spawn("python3", [join(import.meta.dirname, "version-update-terminal.py"), process.execPath, ...nodeArgs], { + cwd, + env, + stdio: "pipe", + }) + : spawn(process.execPath, nodeArgs, { cwd, env, stdio: "pipe" }); + let output = ""; + let quitSent = false; + let deadlineReached = false; + const completion = new Promise((resolveExit, reject) => { + child.once("error", reject); + child.once("close", resolveExit); + }); + const sendExit = () => { + const awaitingResponse = + options.staging && + !options.offline && + !options.skip && + !readFileSync(recordPath, "utf8").includes('"type":"staging-response"'); + if ( + options.interactive && + !awaitingResponse && + !quitSent && + stripVTControlCharacters(output).includes("Diffex v") + ) { + quitSent = true; + child.stdin.write("/exit\r"); + } + }; + const exitPoll = setInterval(sendExit, 50); + const capture = (chunk: Buffer) => { + output += chunk.toString(); + sendExit(); + }; + child.stdout.on("data", capture); + child.stderr.on("data", capture); + const timeout = setTimeout(() => { + deadlineReached = true; + child.kill("SIGTERM"); + }, 20_000); + try { + const code = await completion; + const events = readFileSync(recordPath, "utf8") + .trim() + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as RecordedEvent); + assert.equal(deadlineReached, false, stripVTControlCharacters(output)); + assert.deepEqual( + events.filter((event) => event.type === "blocked-network" || event.type === "blocked-process"), + [], + output, + ); + return { code, events, output: stripVTControlCharacters(output), quitSent, prefix: root }; + } finally { + clearTimeout(timeout); + clearInterval(exitPoll); + if (child.exitCode === null && child.signalCode === null) { + child.kill("SIGTERM"); + await completion.catch(() => {}); + } + rmSync(root, { recursive: true, force: true }); + assert.equal(existsSync(root), false); + } +} diff --git a/packages/coding-agent/test/fixtures/version-update-guard.mjs b/packages/coding-agent/test/fixtures/version-update-guard.mjs index f0b92f782..edb036362 100644 --- a/packages/coding-agent/test/fixtures/version-update-guard.mjs +++ b/packages/coding-agent/test/fixtures/version-update-guard.mjs @@ -5,12 +5,18 @@ import { appendFileSync } from "node:fs"; import { syncBuiltinESMExports } from "node:module"; import { Socket } from "node:net"; import { basename, dirname, join } from "node:path"; +import { Agent, fetch as requestStaging } from "undici"; import "../../src/core/http-dispatcher.ts"; const recordPath = process.env.DIFFEX_REPRO_RECORD; if (!recordPath) throw new Error("The reproduction requires an isolated event record"); const record = (event) => appendFileSync(recordPath, `${JSON.stringify(event)}\n`); +const staging = process.env.DIFFEX_REPRO_STAGING === "1"; +const stagingUrl = "https://diffex-public-api-staging.yibai14651.workers.dev/api/latest-version"; +const stagingHost = new URL(stagingUrl).hostname; +const dispatcher = staging ? new Agent() : undefined; + globalThis.fetch = async (input, init) => { const url = input instanceof Request ? input.url : String(input); const headers = new Headers(init?.headers); @@ -22,6 +28,22 @@ globalThis.fetch = async (input, init) => { redirect: init?.redirect, }); if (url !== "https://api.diffex.ai/api/latest-version") return new Response(null, { status: 503 }); + if (staging) { + if ((init?.method ?? "GET") !== "GET" || init?.redirect !== "manual") { + throw new Error("Unexpected staging request options"); + } + const response = await requestStaging(stagingUrl, { ...init, dispatcher, redirect: "manual" }); + const body = await response.text(); + record({ + type: "staging-response", + url: stagingUrl, + status: response.status, + body, + cacheControl: response.headers.get("cache-control"), + contentType: response.headers.get("content-type"), + }); + return new Response(body, { status: response.status, headers: response.headers }); + } if (process.env.DIFFEX_REPRO_FAILURE === "1") throw new Error("controlled network failure"); if (process.env.DIFFEX_REPRO_RESPONSE) { return new Response(process.env.DIFFEX_REPRO_RESPONSE, { @@ -39,7 +61,13 @@ globalThis.fetch = async (input, init) => { }; // Prevent alternate HTTP clients from reaching the network too. -Socket.prototype.connect = function () { +const originalConnect = Socket.prototype.connect; +Socket.prototype.connect = function (...args) { + const options = Array.isArray(args[0]) ? args[0][0] : args[0]; + if (staging && options?.host === stagingHost && Number(options.port) === 443) { + record({ type: "staging-connect", host: stagingHost, port: 443 }); + return originalConnect.apply(this, args); + } record({ type: "blocked-network" }); throw new Error("Network disabled by the version-update reproduction"); }; diff --git a/packages/coding-agent/test/version-update-e2e.test.ts b/packages/coding-agent/test/version-update-e2e.test.ts index 028b45057..37302422f 100644 --- a/packages/coding-agent/test/version-update-e2e.test.ts +++ b/packages/coding-agent/test/version-update-e2e.test.ts @@ -1,136 +1,5 @@ -import { spawn } from "node:child_process"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; -import { stripVTControlCharacters } from "node:util"; import { describe, expect, it } from "vitest"; - -const packageDir = resolve(import.meta.dirname, ".."); -const repositoryDir = resolve(packageDir, "../.."); -const packageJson = readFileSync(join(packageDir, "package.json"), "utf8"); -const installed = JSON.parse(packageJson) as { name: string; version: string }; -const endpoint = "https://api.diffex.ai/api/latest-version"; - -interface RecordedEvent { - type: string; - url?: string; - method?: string; - userAgent?: string; - redirect?: string; - command?: string; - args?: string[]; -} - -async function reproduce(options: { - interactive?: boolean; - force?: boolean; - offline?: boolean; - skip?: boolean; - failure?: boolean; - packageName?: string; - version?: string; - response?: string; - status?: number; - redirect?: string; -}) { - const root = mkdtempSync(join(tmpdir(), "diffex-version-repro-")); - const agentDir = join(root, "agent"); - const recordPath = join(root, "events.jsonl"); - const homeDir = join(root, "home"); - const cwd = join(root, "project"); - for (const dir of [agentDir, homeDir, cwd]) mkdirSync(dir); - writeFileSync(join(agentDir, "settings.json"), JSON.stringify({ theme: "dark", packages: [] })); - writeFileSync(recordPath, ""); - const env: NodeJS.ProcessEnv = { - PATH: process.env.PATH, - HOME: homeDir, - USERPROFILE: homeDir, - TMPDIR: root, - XDG_CONFIG_HOME: join(homeDir, ".config"), - XDG_CACHE_HOME: join(homeDir, ".cache"), - TERM: "xterm-256color", - LANG: "en_US.UTF-8", - TSX_TSCONFIG_PATH: join(repositoryDir, "tsconfig.json"), - DIFFEX_CODING_AGENT_DIR: agentDir, - DIFFEX_NO_LOCAL_LLM: "1", - DIFFEX_REPRO_RECORD: recordPath, - DIFFEX_REPRO_VERSION: options.version ?? (options.force ? installed.version : "9999.0.0"), - DIFFEX_REPRO_PACKAGE: options.packageName ?? "@foreign-fixture/untrusted-cli", - DIFFEX_REPRO_RESPONSE: options.response, - DIFFEX_REPRO_STATUS: String(options.status ?? 200), - DIFFEX_REPRO_REDIRECT: options.redirect, - DIFFEX_REPRO_FAILURE: options.failure ? "1" : "", - DIFFEX_OFFLINE: options.offline ? "1" : "", - DIFFEX_SKIP_VERSION_CHECK: options.skip ? "1" : "", - }; - if (!options.interactive) { - const fakePackageDir = join(root, "lib", "node_modules", ...installed.name.split("/")); - mkdirSync(fakePackageDir, { recursive: true }); - writeFileSync(join(fakePackageDir, "package.json"), packageJson); - env.DIFFEX_PACKAGE_DIR = fakePackageDir; - } - const nodeArgs = [ - `--import=${import.meta.resolve("tsx")}`, - `--import=${join(import.meta.dirname, "fixtures/version-update-guard.mjs")}`, - join(packageDir, "src/cli.ts"), - ...(options.interactive ? ["--no-session", "--no-extensions", "--no-skills"] : ["update", "--self"]), - ...(options.force ? ["--force"] : []), - ]; - const child = options.interactive - ? spawn( - "python3", - [join(import.meta.dirname, "fixtures/version-update-terminal.py"), process.execPath, ...nodeArgs], - { - cwd, - env, - stdio: "pipe", - }, - ) - : spawn(process.execPath, nodeArgs, { cwd, env, stdio: "pipe" }); - let output = ""; - let quitSent = false; - let deadlineReached = false; - const completion = new Promise((resolveExit, reject) => { - child.once("error", reject); - child.once("close", resolveExit); - }); - const capture = (chunk: Buffer) => { - output += chunk.toString(); - // /exit travels through the real editor after the welcome screen is rendered. - if (options.interactive && !quitSent && stripVTControlCharacters(output).includes("Diffex v")) { - quitSent = true; - child.stdin.write("/exit\r"); - } - }; - child.stdout.on("data", capture); - child.stderr.on("data", capture); - const timeout = setTimeout(() => { - deadlineReached = true; - child.kill("SIGTERM"); - }, 20_000); - try { - const code = await completion; - const events = readFileSync(recordPath, "utf8") - .trim() - .split("\n") - .filter(Boolean) - .map((line) => JSON.parse(line) as RecordedEvent); - expect(deadlineReached, stripVTControlCharacters(output)).toBe(false); - expect( - events.filter((event) => event.type === "blocked-network" || event.type === "blocked-process"), - output, - ).toEqual([]); - return { code, events, output: stripVTControlCharacters(output), quitSent, prefix: root }; - } finally { - clearTimeout(timeout); - if (child.exitCode === null && child.signalCode === null) { - child.kill("SIGTERM"); - await completion.catch(() => {}); - } - rmSync(root, { recursive: true, force: true }); - expect(existsSync(root)).toBe(false); - } -} +import { endpoint, installed, type RecordedEvent, reproduce } from "./fixtures/version-update-client.ts"; function versionRequests(events: RecordedEvent[]) { return events.filter((event) => event.type === "request" && event.url === endpoint); diff --git a/scripts/fixtures/release-client-canary.mjs b/scripts/fixtures/release-client-canary.mjs new file mode 100644 index 000000000..e03817fcf --- /dev/null +++ b/scripts/fixtures/release-client-canary.mjs @@ -0,0 +1,81 @@ +import assert from "node:assert/strict"; +import { endpoint, reproduce } from "../../packages/coding-agent/test/fixtures/version-update-client.ts"; + +export async function checkStagingClient(name, options, expected) { + const result = await reproduce({ ...options, staging: true }); + assert.equal(result.code, expected.code, result.output); + const requests = result.events.filter((event) => event.type === "request" && event.url === endpoint); + const interceptedOtherRequests = result.events.filter((event) => event.type === "request" && event.url !== endpoint); + for (const request of interceptedOtherRequests) { + // Install telemetry is a separate migration; the preload returns 503 without forwarding it. + assert.match(request.url, /^https:\/\/pi\.dev\/api\/report-install\?version=[0-9.]+$/); + } + const connections = result.events.filter((event) => event.type === "staging-connect"); + assert.equal(connections.length > 0, expected.requests > 0); + for (const connection of connections) { + assert.equal(connection.host, "diffex-public-api-staging.yibai14651.workers.dev"); + assert.equal(connection.port, 443); + } + assert.equal(requests.length, expected.requests); + for (const request of requests) { + assert.equal(request.url, endpoint); + assert.equal(request.method, "GET"); + assert.equal(request.redirect, "manual"); + assert.match(request.userAgent, /^diffex\//); + } + const responses = result.events.filter((event) => event.type === "staging-response"); + assert.equal(responses.length, expected.requests); + for (const response of responses) { + assert.equal(response.url, "https://diffex-public-api-staging.yibai14651.workers.dev/api/latest-version"); + assert.equal(response.status, expected.status); + assert.equal(response.cacheControl, "no-store"); + assert.match(response.contentType, /^application\/json\b/); + assert.deepEqual(JSON.parse(response.body), expected.body); + } + const commands = result.events.filter((event) => event.type === "spawn"); + assert.deepEqual( + commands, + expected.installVersion + ? [ + { + type: "spawn", + command: "npm", + args: [ + "--prefix", + result.prefix, + "install", + "-g", + "--ignore-scripts", + "--min-release-age=0", + `@diffexai/diffex@${expected.installVersion}`, + ], + }, + ] + : [], + ); + if (expected.message) assert.ok(result.output.includes(expected.message), result.output); + if (options.interactive) { + assert.equal(result.quitSent, true); + assert.doesNotMatch( + result.output, + /New version|Could not determine latest|update service|No published Diffex release|Refusing self-update/, + ); + } + console.log( + JSON.stringify({ + canary: name, + code: result.code, + requests, + responses, + interceptedOtherRequests, + connections, + commands: commands.map((command) => ({ + ...command, + args: command.args.map((arg) => (arg === result.prefix ? "" : arg)), + })), + terminalExited: result.quitSent, + message: expected.message ?? null, + cleanup: "verified", + }), + ); +} diff --git a/scripts/verify-release-publication.mjs b/scripts/verify-release-publication.mjs index ab028583a..51937e0a5 100644 --- a/scripts/verify-release-publication.mjs +++ b/scripts/verify-release-publication.mjs @@ -4,11 +4,20 @@ import { spawnSync } from "node:child_process"; import { randomBytes } from "node:crypto"; import { readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; +import { checkStagingClient } from "./fixtures/release-client-canary.mjs"; import { createPublicationFixture } from "./fixtures/release-publication/harness.mjs"; -const checkPublicApi = process.argv.length === 4 && process.argv[3] === "--public-api"; -if (process.argv[2] !== "--staging" || (process.argv.length !== 3 && !checkPublicApi)) { - throw new Error("Use --staging [--public-api] with staging-scoped AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY."); +const flags = process.argv.slice(2); +const checkClients = flags.includes("--client-canaries"); +const checkPublicApi = checkClients || flags.includes("--public-api"); +if ( + flags[0] !== "--staging" || + new Set(flags).size !== flags.length || + flags.some((flag) => !["--staging", "--public-api", "--client-canaries"].includes(flag)) +) { + throw new Error( + "Use --staging [--public-api] [--client-canaries] with staging-scoped AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY.", + ); } if (!process.env.AWS_ACCESS_KEY_ID || !process.env.AWS_SECRET_ACCESS_KEY) { throw new Error("Staging verification requires AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY."); @@ -53,9 +62,7 @@ function aws(operation, args, { allowMissing = false } = {}) { const code = /An error occurred \(([^)]+)\)/.exec(result.stderr ?? "")?.[1]; if (allowMissing && ["NoSuchKey", "404", "NotFound"].includes(code)) return undefined; if (result.error || result.status !== 0) - throw new Error( - `Staging ${operation} failed; check credentials, concurrency lock, and conditional-write support.`, - ); + throw new Error(`Staging ${operation} failed; check credentials, concurrency lock, and conditional-write support.`); return JSON.parse(result.stdout || "{}"); } function read(key) { @@ -75,7 +82,10 @@ function publish(value, options = {}) { function removeOwned(key) { const object = read(key); if (!object) return; - assert.ok(ownedInvalidBodies.has(object.body) || JSON.parse(object.body).sourceCommit === commit, "Refusing cleanup of an object owned by another writer."); + assert.ok( + ownedInvalidBodies.has(object.body) || JSON.parse(object.body).sourceCommit === commit, + "Refusing cleanup of an object owned by another writer.", + ); aws("delete-object", ["--key", key]); assert.equal(read(key), undefined, "Staging object remains after cleanup."); } @@ -90,6 +100,22 @@ try { ready = true; await expectWorker(404, { error: "no_published_release" }); await expectWorker(200, { status: "ready", release: "missing" }, "/health/ready"); + if (checkClients) { + const missing = { + code: 1, + requests: 1, + status: 404, + body: { error: "no_published_release" }, + message: "No published Diffex release is available yet.", + }; + await checkStagingClient("missing release", {}, missing); + await checkStagingClient("forced missing release", { force: true }, missing); + await checkStagingClient( + "interactive missing release", + { interactive: true }, + { ...missing, code: 0, message: undefined }, + ); + } console.log(`Staging publication fixture: ${version}, ${newer}`); for (const failure of ["package", "tarball"]) { @@ -105,7 +131,45 @@ try { assert.equal(JSON.parse(initial.body).packages.length, fixture.packageCount); console.log("First verified publication creates matching versioned and latest records."); await expectWorker(200, { version, packageName: "@diffexai/diffex" }); - await expectWorker(200, { version, packageName: "@diffexai/diffex" }, "/api/latest-version?key=unrelated&packageName=foreign&version=99.0.0"); + if (checkClients) { + const valid = { code: 0, requests: 1, status: 200, body: { version, packageName: "@diffexai/diffex" } }; + await checkStagingClient("newer release", {}, { ...valid, installVersion: version }); + await checkStagingClient( + "equal release", + { installedVersion: version }, + { ...valid, message: "already up to date" }, + ); + await checkStagingClient("older release", { installedVersion: newer }, { ...valid, message: "already up to date" }); + await checkStagingClient( + "forced equal release", + { installedVersion: version, force: true }, + { ...valid, installVersion: version }, + ); + await checkStagingClient( + "forced older release", + { installedVersion: newer, force: true }, + { ...valid, installVersion: version }, + ); + await checkStagingClient("manual automatic opt-out", { skip: true }, { ...valid, installVersion: version }); + await checkStagingClient( + "manual offline", + { offline: true, force: true }, + { code: 1, requests: 0, message: "Could not determine latest diffex version." }, + ); + await checkStagingClient("interactive newer release", { interactive: true }, valid); + await checkStagingClient("interactive offline", { interactive: true, offline: true }, { code: 0, requests: 0 }); + await checkStagingClient( + "interactive automatic opt-out", + { interactive: true, skip: true }, + { code: 0, requests: 0 }, + ); + } + + await expectWorker( + 200, + { version, packageName: "@diffexai/diffex" }, + "/api/latest-version?key=unrelated&packageName=foreign&version=99.0.0", + ); await expectWorker(200, { status: "ready", release: "published" }, "/health/ready"); for (const method of ["POST", "PUT", "PATCH", "DELETE", "OPTIONS", "HEAD"]) await expectWorker(405, { error: "method_not_allowed" }, "/api/latest-version", method); @@ -114,19 +178,69 @@ try { assert.equal(read(latestKey).body, initial.body, "HTTP requests must not mutate release storage."); if (checkPublicApi) { const invalidFile = join(fixture.directory, "invalid-record.json"); - for (const invalid of [ - `{invalid-json-${commit}`, - JSON.stringify({ ...JSON.parse(initial.body), schemaVersion: 2 }), - JSON.stringify({ ...JSON.parse(initial.body), version: `${version}-beta.1` }), + const record = JSON.parse(initial.body); + const canonical = record.packages.find((pkg) => pkg.name === "@diffexai/diffex"); + for (const [name, invalid] of [ + ["invalid JSON", `{invalid-json-${commit}`], + ["wrong schema", JSON.stringify({ ...record, schemaVersion: 2 })], + ["prerelease semver", JSON.stringify({ ...record, version: `${version}-beta.1` })], + [ + "missing canonical package", + JSON.stringify({ ...record, packages: record.packages.filter((pkg) => pkg !== canonical) }), + ], + ["duplicate canonical package", JSON.stringify({ ...record, packages: [...record.packages, canonical] })], + [ + "package version mismatch", + JSON.stringify({ + ...record, + packages: record.packages.map((pkg) => (pkg === canonical ? { ...pkg, version: newer } : pkg)), + }), + ], + [ + "foreign package", + JSON.stringify({ + ...record, + packages: record.packages.map((pkg) => + pkg === canonical ? { ...pkg, name: "@foreign-fixture/untrusted-cli" } : pkg, + ), + }), + ], ]) { ownedInvalidBodies.add(invalid); writeFileSync(invalidFile, invalid); aws("put-object", ["--key", latestKey, "--body", invalidFile, "--if-match", read(latestKey).metadata.ETag]); await expectWorker(503, { error: "release_unavailable" }); await expectWorker(503, { status: "unavailable", release: "unavailable" }, "/health/ready"); + if (checkClients) { + const unavailable = { + code: 1, + requests: 3, + status: 503, + body: { error: "release_unavailable" }, + message: "Diffex update service is unavailable.", + }; + await checkStagingClient(`forced ${name}`, { force: true }, unavailable); + if (name === "invalid JSON") + await checkStagingClient( + "interactive unavailable", + { interactive: true }, + { ...unavailable, code: 0, requests: 1, message: undefined }, + ); + } } writeFileSync(invalidFile, initial.body); - aws("put-object", ["--key", latestKey, "--body", invalidFile, "--if-match", read(latestKey).metadata.ETag, "--content-type", "application/json", "--cache-control", "no-store"]); + aws("put-object", [ + "--key", + latestKey, + "--body", + invalidFile, + "--if-match", + read(latestKey).metadata.ETag, + "--content-type", + "application/json", + "--cache-control", + "no-store", + ]); await expectWorker(200, { version, packageName: "@diffexai/diffex" }); } publish(version);