From 7fc85b6c18d7c71662cab634c9be057ceac83d7f Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Sat, 12 Sep 2026 19:01:04 -0400 Subject: [PATCH 1/2] feat(coding-agent): verify packed distributions - Detect missing exports and runtime dependencies in isolated consumers - Gate publication on native acceptance of retained artifact bytes --- .github/workflows/artifact-acceptance.yml | 56 ++++ .github/workflows/build-binaries.yml | 11 +- .../workflows/native-artifact-acceptance.yml | 55 +++ AGENTS.md | 2 + docs/release/artifact-acceptance.md | 79 +++++ scripts/artifact-acceptance.mjs | 314 ++++++++++++++++++ scripts/artifact-acceptance.test.mjs | 80 +++++ scripts/fixtures/artifact-network-guard.mjs | 13 + scripts/fixtures/artifact-server.mjs | 88 +++++ scripts/local-release.mjs | 20 +- scripts/prepare-candidate-artifacts.mjs | 7 +- scripts/release-distribution-manifest.mjs | 3 +- .../release-distribution-manifest.test.mjs | 3 + scripts/verify-candidate-artifacts.py | 13 +- 14 files changed, 735 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/artifact-acceptance.yml create mode 100644 .github/workflows/native-artifact-acceptance.yml create mode 100644 docs/release/artifact-acceptance.md create mode 100644 scripts/artifact-acceptance.mjs create mode 100644 scripts/artifact-acceptance.test.mjs create mode 100644 scripts/fixtures/artifact-network-guard.mjs create mode 100644 scripts/fixtures/artifact-server.mjs diff --git a/.github/workflows/artifact-acceptance.yml b/.github/workflows/artifact-acceptance.yml new file mode 100644 index 000000000..da698454d --- /dev/null +++ b/.github/workflows/artifact-acceptance.yml @@ -0,0 +1,56 @@ +name: Artifact Acceptance + +on: + pull_request: + workflow_dispatch: + +permissions: + contents: read + actions: read + +concurrency: + group: artifact-acceptance-${{ github.ref }} + cancel-in-progress: true + +jobs: + prepare: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + outputs: + commit: ${{ steps.source.outputs.commit }} + steps: + - name: Checkout candidate + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Resolve candidate commit + id: source + run: echo "commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + - name: Setup Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22.19.0' + - name: Install pinned npm and locked dependencies + run: | + npm install -g npm@11.16.0 --ignore-scripts + npm ci --ignore-scripts + - name: Freeze and build reviewed candidate artifacts + run: | + npm run hydrate:model-data + node scripts/release-artifacts.mjs freeze .artifacts/release + npm run build:offline + node scripts/release-artifacts.mjs prepare .artifacts/release + - name: Retain exact private release bytes + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: private-release-${{ steps.source.outputs.commit }} + path: .artifacts/release/ + include-hidden-files: true + if-no-files-found: error + retention-days: 14 + + native-acceptance: + needs: prepare + uses: ./.github/workflows/native-artifact-acceptance.yml + with: + source_commit: ${{ needs.prepare.outputs.commit }} diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index 5c8d77f74..66c3c80ea 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -126,8 +126,17 @@ jobs: include-hidden-files: true retention-days: 90 - publish-npm: + native-acceptance: needs: [resolve-source, prepare-artifacts] + permissions: + contents: read + actions: read + uses: ./.github/workflows/native-artifact-acceptance.yml + with: + source_commit: ${{ needs.resolve-source.outputs.commit }} + + publish-npm: + needs: [resolve-source, prepare-artifacts, native-acceptance] runs-on: ubuntu-latest environment: npm-publish permissions: diff --git a/.github/workflows/native-artifact-acceptance.yml b/.github/workflows/native-artifact-acceptance.yml new file mode 100644 index 000000000..941a6a4f1 --- /dev/null +++ b/.github/workflows/native-artifact-acceptance.yml @@ -0,0 +1,55 @@ +name: Native Artifact Acceptance + +on: + workflow_call: + inputs: + source_commit: + required: true + type: string + +permissions: + contents: read + actions: read + +jobs: + accept: + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + runner: [ubuntu-24.04, ubuntu-24.04-arm, macos-15, macos-15-intel] + runs-on: ${{ matrix.runner }} + env: + DIFFEX_SOURCE_COMMIT: ${{ inputs.source_commit }} + steps: + - name: Checkout exact artifact source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.source_commit }} + persist-credentials: false + - name: Setup Node + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22.19.0' + - name: Setup Bun package manager + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: '1.3.11' + - name: Install pinned npm + run: npm install -g npm@11.16.0 --ignore-scripts + - name: Download exact retained tarballs + env: + GH_TOKEN: ${{ github.token }} + run: gh run download "$GITHUB_RUN_ID" --repo diffexai/diffex --name "private-release-$DIFFEX_SOURCE_COMMIT" --dir .artifacts/release + - name: Verify frozen source and archive bytes + run: node scripts/release-artifacts.mjs verify .artifacts/release + - name: Accept all packages and process boundaries outside the workspace + run: node scripts/artifact-acceptance.mjs .artifacts/release .artifacts/acceptance + - name: Retain private native acceptance evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: native-acceptance-${{ matrix.runner }}-${{ inputs.source_commit }} + path: .artifacts/acceptance/ + if-no-files-found: error + retention-days: 90 diff --git a/AGENTS.md b/AGENTS.md index 56b9105c8..79e86d2bc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -203,3 +203,5 @@ If the user's instructions conflict with any rule in this document, ask for expl - 2026-09-12: Release preflight treated the private monorepo version as the public lockstep version -> Derive release identity from public workspaces and test against the actual repository inventory. - 2026-09-12: A shell commit body retained a literal newline escape -> Write multiline commit and PR text to a file and pass the file to Git or GitHub CLI. + +- 2026-09-12: Artifact smoke assumed an Agent mutator and reusable old consumers -> Check installed public API types, create fresh consumers, and retain failed acceptance with verified cleanup. diff --git a/docs/release/artifact-acceptance.md b/docs/release/artifact-acceptance.md new file mode 100644 index 000000000..e31332b85 --- /dev/null +++ b/docs/release/artifact-acceptance.md @@ -0,0 +1,79 @@ +# Local package and native acceptance + +The release gate installs every package returned by `scripts/release-packages.mjs` from its reviewed local tarball. +Each package receives a separate consumer with only its public dependency closure, ordered before the package that requires it. +Consumers live outside the repository in paths containing spaces, with separate homes, configuration, projects, temporary files, package prefixes, and npm/Bun caches. +Lifecycle scripts are disabled and the Diffex registry scope points to an unavailable loopback endpoint, so a missing local override fails instead of fetching an older Diffex package. +External npm dependencies still resolve from the public registry. + +Declared runtime exports, wildcard exports, declarations, main files, and executable targets must exist inside the installed package. +Runtime imports and representative operations execute in fresh Node processes. +Installed runtime inventories must equal the inventory extracted from the checked tarball, and every recorded installed file must match its SHA-256 digest. +The operation inventory fails closed if a new public package has no representative operation. + +| Package | Representative operation | +| --- | --- | +| `diffex-agent-core` | Construct an agent and change its state | +| `diffex-ai` | Complete and consume an assistant event stream | +| `diffex-client` | Construct and dispose a client; additionally list a hosted session over WebSocket | +| `diffex` | Initialize model runtime, load clipboard native module, execute image WASM, export a session to HTML | +| `diffex-coding-agent-server` | Close usage reporter; start its actual executable, reject wrong credentials, list its session, terminate cleanly | +| `diffex-harness-state` | Persist and reload the initial harness version | +| `diffex-protocol` | Encode and decode CBOR | +| `diffex-server` | Start and close a loopback WebSocket listener | +| `diffex-session-backend-sqlite-node` | Create, write, and query SQLite | +| `diffex-telemetry` | Execute a telemetry span callback | +| `diffex-tui` | Render text and load the current platform's native modifier helper where applicable | + +The packed CLI additionally executes help, version, model listing, controlled inference, a terminal prompt/reply, account-menu inspection, and clean exit. +The terminal path exercises packaged themes and the HTML export path exercises its templates, CSS, and bundled renderer resources. +Clipboard coverage loads the native module without reading or replacing the user's clipboard. +Socket guards reject non-loopback process traffic; the terminal fixture supplies the known no-release HTTP response and fails on unexpected socket attempts. +No provider credentials are inherited. + +## Supported distribution scope + +The approved channel remains npm, installed with npm or Bun and executed on Node 22.19.0 or newer. +Bun package installation is distinct from a Bun standalone executable. +Standalone publication remains deferred for every platform under the approved redistribution policy, and the manifest advertises no binary assets or binary platforms. +Windows npm targets are now explicitly deferred until equivalent terminal acceptance is available. +The advertised npm targets are Darwin arm64/x64 and Linux arm64/x64. +The reusable native workflow uses clean native runners for those four targets, downloads the same retained tarballs, verifies their source and checksums, and runs the complete acceptance command before npm publication can proceed. +Runner labels follow the [GitHub-hosted runner reference](https://docs.github.com/en/actions/reference/runners/github-hosted-runners). + +## Commands and evidence + +Build, pack, and accept the current local candidate: + +```sh +npm run release:local -- --out '/tmp/diffex local release' +``` + +Accept existing reviewed artifacts without rebuilding or repacking: + +```sh +node scripts/artifact-acceptance.mjs /path/to/artifacts /path/to/private-evidence +``` + +The command accepts both the local manifest/tarballs layout and the frozen release artifact layout. +`acceptance.json` records the source commit, OS, architecture, Node and package-manager versions, each package's imports and operation result, exact tarball checksums, process outcomes, and cleanup. +A failed run records failure and the completed subset; it cannot produce a passing overall status. +Terminal transcripts and smoke events are retained beside it in private evidence storage. +Every test-owned consumer, session, cache, configuration directory, and server process is removed, including on failure. + +Verify final tarball bytes from their directory using the retained checksum list: + +```sh +shasum -a 256 -c /path/to/private-evidence/SHA256SUMS +``` + +On Linux, `sha256sum -c` accepts the same checksum file. +Regression coverage alters archive bytes and confirms verification fails before installation. +The release workflow retains the source snapshot and manifests privately, while npm publication supplies registry integrity and provenance through the existing publisher. +Public acceptance must compare the actual downloaded npm bytes with these retained identities after separately authorized publication. + +The baseline CLI-only acceptance passed on Darwin arm64 but emitted no per-package operation or hosted-server result. +The expanded local acceptance covers all eleven packages for both managers and the server lifecycle. +The owner specified that no live provider test environment is available, so real login and inference remain pending. +Native runner results and exact committed-candidate evidence are recorded in the implementation pull request and accompanying acceptance record. +Public downloads and public-to-public upgrades remain unexecuted until publication; local fixture inference does not establish those results. diff --git a/scripts/artifact-acceptance.mjs b/scripts/artifact-acceptance.mjs new file mode 100644 index 000000000..a33f1b1b8 --- /dev/null +++ b/scripts/artifact-acceptance.mjs @@ -0,0 +1,314 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + cpSync, + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, isAbsolute, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { getPublicWorkspacePackages } from "./release-packages.mjs"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const digest = (bytes) => createHash("sha256").update(bytes).digest("hex"); +const readJson = (file) => JSON.parse(readFileSync(file, "utf8")); + +export function assertContained(file, directory) { + const local = relative(realpathSync(directory), realpathSync(file)); + assert(local === "" || (!local.startsWith("..") && !isAbsolute(local)), `Path escapes isolated consumer: ${file}`); +} + +export function packageClosure(name, packages) { + const result = []; + const visiting = new Set(); + const visited = new Set(); + function visit(current) { + if (visited.has(current)) return; + assert(!visiting.has(current), `Public dependency cycle: ${current}`); + const pkg = packages.find((candidate) => candidate.name === current); + assert(pkg, `Missing local public dependency: ${current}`); + visiting.add(current); + for (const dependency of Object.keys({ + ...pkg.dependencies, + ...pkg.optionalDependencies, + ...pkg.peerDependencies, + })) { + if (dependency.startsWith("@diffexai/")) visit(dependency); + } + visiting.delete(current); + visited.add(current); + result.push(pkg); + } + visit(name); + return result; +} + +export function verifyTarballs(directory, records, packages) { + assert.equal(records.length, packages.length, "Incomplete tarball inventory"); + assert.equal(new Set(records.map((pkg) => pkg.name)).size, packages.length, "Duplicate tarball identity"); + for (const pkg of packages) { + const record = records.find((entry) => entry.name === pkg.name); + assert(record && record.version === pkg.version, `Missing tarball: ${pkg.name}`); + assert.equal(record.filename, `${pkg.name.replace("@", "").replace("/", "-")}-${pkg.version}.tgz`); + assert.equal( + digest(readFileSync(join(directory, record.filename))), + record.sha256, + `Tarball checksum mismatch: ${pkg.name}`, + ); + } +} + +export function isolatedEnvironment(directory) { + const env = {}; + for (const key of ["PATH", "SystemRoot", "WINDIR", "COMSPEC", "PATHEXT", "LANG"]) { + if (process.env[key]) env[key] = process.env[key]; + } + for (const dir of ["home", "config", "cache", "prefix", "project", "agent", "temp"]) + mkdirSync(join(directory, dir), { recursive: true }); + return { + ...env, + HOME: join(directory, "home"), + USERPROFILE: join(directory, "home"), + XDG_CONFIG_HOME: join(directory, "config"), + XDG_CACHE_HOME: join(directory, "cache"), + npm_config_cache: join(directory, "cache", "npm"), + npm_config_prefix: join(directory, "prefix"), + npm_config_userconfig: join(directory, "empty.npmrc"), + npm_config_globalconfig: join(directory, "empty-global.npmrc"), + npm_config_registry: "https://registry.npmjs.org", + npm_config_update_notifier: "false", + BUN_INSTALL_CACHE_DIR: join(directory, "cache", "bun"), + TMPDIR: join(directory, "temp"), + TMP: join(directory, "temp"), + TEMP: join(directory, "temp"), + DIFFEX_CODING_AGENT_DIR: join(directory, "agent"), + DIFFEX_NO_LOCAL_LLM: "1", + DIFFEX_OFFLINE: "1", + TERM: "xterm-256color", + }; +} + +function walk(directory) { + return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => { + const path = join(directory, entry.name); + return entry.isDirectory() ? walk(path) : [path]; + }); +} + +export function declaredEntrypoints(pkg, directory) { + const files = walk(directory).map((file) => relative(directory, file).replaceAll("\\", "/")); + const imports = []; + function target(value, specifier, condition) { + if (typeof value === "string") { + const pattern = value.replace(/^\.\//, ""); + const [prefix, suffix] = pattern.split("*"); + const matches = pattern.includes("*") + ? files.filter((file) => file.startsWith(prefix) && file.endsWith(suffix)) + : [pattern]; + assert(matches.length, `Empty export pattern: ${pkg.name} ${value}`); + for (const file of matches) { + assertContained(join(directory, file), directory); + if (condition !== "types" && /\.[cm]?js$/.test(file)) { + imports.push(specifier.replace("*", file.slice(prefix.length, suffix ? -suffix.length : undefined))); + } + } + } else if (value && typeof value === "object") { + for (const [key, nested] of Object.entries(value)) target(nested, specifier, key); + } + } + if (pkg.exports) { + for (const [key, value] of Object.entries(pkg.exports)) + target(value, key === "." ? pkg.name : pkg.name + key.slice(1)); + } else target(pkg.main, pkg.name); + for (const file of [pkg.main, pkg.types, ...Object.values(pkg.bin ?? {})].filter(Boolean)) + assertContained(join(directory, file), directory); + return [...new Set(imports)]; +} + +export const operations = { + "@diffexai/diffex-agent-core": `const agent = new api.Agent({ streamFn: () => { throw new Error("Unexpected inference"); } }); agent.state.systemPrompt = "artifact"; assert.equal(agent.state.systemPrompt, "artifact");`, + "@diffexai/diffex-ai": `const stream = api.createAssistantMessageEventStream(); stream.end({ marker: "artifact" }); assert.equal((await stream.result()).marker, "artifact");`, + "@diffexai/diffex-client": `const client = new api.DiffexClient({ transportFactory: async () => { throw new Error("Unexpected transport"); } }); await client.dispose(); assert.equal(client.disposed, true);`, + "@diffexai/diffex-protocol": `assert.deepEqual(api.decodeCbor(api.encodeCbor({ artifact: 42 })), { artifact: 42 });`, + "@diffexai/diffex-harness-state": `const store = new api.HarnessStateStore(process.env.DIFFEX_CODING_AGENT_DIR); const state = await store.initialize(); assert.equal((await store.loadCurrent()).manifest.version, state.manifest.version);`, + "@diffexai/diffex-telemetry": `let called = false; await api.NOOP_TELEMETRY_CONTEXT.startSpan({ name: "artifact" }, () => { called = true; }); assert(called);`, + "@diffexai/diffex-session-backend-sqlite-node": `const db = await api.createNodeSqliteFactory().open(":memory:"); try { db.exec("CREATE TABLE acceptance (value TEXT)"); db.prepare("INSERT INTO acceptance VALUES (?)").run("artifact"); assert.equal(db.prepare("SELECT value FROM acceptance").get().value, "artifact"); } finally { db.close(); }`, + "@diffexai/diffex-tui": `assert(new api.Text("artifact", 0, 0).render(40).join("\\n").includes("artifact")); if (process.platform === "darwin" || process.platform === "win32") { const platform = process.platform; const filename = platform === "darwin" ? "darwin-modifiers.node" : "win32-console-mode.node"; const helper = require(path.join(packageRoot, "native", platform, "prebuilds", platform + "-" + process.arch, filename)); assert.equal(typeof helper.isModifierPressed, "function"); }`, + "@diffexai/diffex": `const runtime = await api.ModelRuntime.create({ authPath: path.join(process.env.DIFFEX_CODING_AGENT_DIR, "auth.json"), modelsPath: path.join(process.env.DIFFEX_CODING_AGENT_DIR, "models.json"), refreshOnCreate: false }); assert(runtime); const cliRequire = createRequire(path.join(packageRoot, "package.json")); const clipboard = cliRequire("@mariozechner/clipboard"); assert.equal(typeof clipboard.getText, "function"); const photon = cliRequire("@silvia-odwyer/photon-node"); const image = new photon.PhotonImage(new Uint8Array([0, 0, 0, 255]), 1, 1); assert.equal(image.get_width(), 1); image.free(); + const sm = api.SessionManager.create(process.cwd(), path.join(process.env.DIFFEX_CODING_AGENT_DIR, "export-session")); + sm.appendMessage({ role: "user", content: "artifact export", timestamp: Date.now() }); + sm.appendMessage({ role: "assistant", content: [{ type: "text", text: "artifact exported reply" }], api: "openai-completions", provider: "fixture", model: "fixture", usage: { input: 1, output: 1, cacheRead: 0, cacheWrite: 0, totalTokens: 2, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } }, stopReason: "stop", timestamp: Date.now() }); + const html = path.join(process.cwd(), "artifact export.html"); + execFileSync(process.execPath, [path.join(packageRoot, "dist/cli.js"), "--export", sm.getSessionFile(), html], { env: process.env, timeout: 30000 }); + const exported = readFileSync(html, "utf8"); const embedded = exported.match(/