diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index 1a61be7da..db2c0304c 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -272,8 +272,9 @@ jobs: announce-diffex-release: runs-on: ubuntu-latest + timeout-minutes: 20 needs: publish-npm - environment: diffex-model-upload + environment: diffex-release-upload concurrency: group: announce-diffex-release cancel-in-progress: false @@ -286,7 +287,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.DIFFEX_ARTIFACTS_R2_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: auto AWS_EC2_METADATA_DISABLED: 'true' - R2_ENDPOINT: https://67c0d357268b0fca6e0b465bb9d01b84.r2.cloudflarestorage.com + R2_ENDPOINT: https://6b24afa849f8254ab5e6e2d1956239cc.r2.cloudflarestorage.com steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.github/workflows/verify-release-publication.yml b/.github/workflows/verify-release-publication.yml new file mode 100644 index 000000000..f3e303b2c --- /dev/null +++ b/.github/workflows/verify-release-publication.yml @@ -0,0 +1,37 @@ +name: Verify Release Publication + +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: verify-release-publication-staging + cancel-in-progress: false + +jobs: + verify: + runs-on: ubuntu-latest + timeout-minutes: 15 + environment: diffex-release-staging + steps: + - name: Checkout reviewed source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22' + - name: Verify AWS CLI + run: aws --version + - name: Check controlled publisher regressions + run: node --test scripts/publish-release-announcement.test.mjs + - name: Verify publication against staging storage + env: + AWS_ACCESS_KEY_ID: ${{ secrets.DIFFEX_ARTIFACTS_R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.DIFFEX_ARTIFACTS_R2_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: auto + AWS_EC2_METADATA_DISABLED: 'true' + run: node scripts/verify-release-publication.mjs --staging diff --git a/docs/release/publication.md b/docs/release/publication.md new file mode 100644 index 000000000..15a575c5a --- /dev/null +++ b/docs/release/publication.md @@ -0,0 +1,122 @@ +# Verified release publication + +## Ownership and release order + +The CLI monorepo owns npm verification and release-record publication through `scripts/publish-release-announcement.mjs` and `.github/workflows/build-binaries.yml`. +The independent public API Worker in `diffex-cloud` consumes these records; Worker, Render, and landing deployments do not advance them. + +The release workflow builds binaries, stages a draft GitHub Release, publishes npm packages, announces verified npm availability, and finally publishes the GitHub Release. +The R2 marker means that all public workspace packages resolved at their exact release version and their tarballs were available. +It does not independently guarantee that binary assets are already public. +This setup does not authorize running a release or bypass the repository's release requirements. + +## Credentials and environment boundaries + +Release storage belongs to account `6b24afa849f8254ab5e6e2d1956239cc`. +The publisher endpoint is `https://6b24afa849f8254ab5e6e2d1956239cc.r2.cloudflarestorage.com`. + +| Environment | Bucket | Credential | +| --- | --- | --- | +| `diffex-release-upload` | `diffex-artifacts` | Dedicated production Object Read & Write token restricted to this bucket. | +| `diffex-release-staging` | `diffex-artifacts-staging` | Separate staging Object Read & Write token restricted to this bucket. | + +Each environment stores `DIFFEX_ARTIFACTS_R2_ACCESS_KEY_ID` and `DIFFEX_ARTIFACTS_R2_SECRET_ACCESS_KEY` as GitHub environment secrets. +The workflow maps them to `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` only in the corresponding job. +Production environment policies allow the `main` branch and `v*` tags; staging allows only `main`. +The repository's GitHub plan rejected required-reviewer protection with HTTP 422. +The owner approved branch/tag restrictions without required reviewers on 2026-09-10. +GitHub reports `can_admins_bypass: true`; there is no enforced human approval gate. +Review release tags and any manually selected recovery source before dispatching a workflow. +These restrictions apply to the workflow ref, so a recovery `source_ref` still needs maintainer review. +Missing credentials fail before npm verification or storage access. +Use an AWS CLI that supports `put-object --if-match` and `--if-none-match`. + +The original `diffex-model-upload` environment had no secrets or protection rules when inspected on 2026-09-10. +Release publication now has a dedicated environment so configuring release credentials cannot redirect the model publisher. +The model-catalog workflow retains its existing environment and former account endpoint; model publication and any old-account data migration require their own setup. +Do not copy release credentials into that environment. + +Bucket-scoped Object Read & Write credentials can mutate other objects within their own bucket, including any future model objects in the production bucket. +They do not enforce a `releases/v1/` prefix boundary and must never be distributed to clients. +The publisher itself uses fixed release-record keys. +See [Cloudflare R2 token permissions](https://developers.cloudflare.com/r2/api/tokens/) for the storage capability boundary. + +Users call the public version endpoint without R2 credentials. +The planned Worker accesses private storage through a native binding, which supports both reads and writes. +If enforced read-only storage access is required, configure a separate read-only credential per Worker environment during endpoint implementation. + +## Publication invariants + +Before any write, the publisher inventories every non-private workspace package and requires its version to match the requested canonical stable semver. +It checks exact npm package identity and version, integrity metadata, and tarball availability for all packages. +Requests have a 30-second timeout, npm propagation retries have a ten-minute budget, and storage commands have a one-minute timeout. + +`releases/v1/releases/.json` is created with `If-None-Match: *`. +On a repeated announcement, the existing record must match the verified version, commit, and package metadata. +An interrupted announcement reuses that record's original bytes and timestamp for `releases/v1/latest.json`. +A conflicting immutable record fails without replacing it. + +The latest-pointer update reads the body and ETag from one GET response and uses `If-Match` for replacement or `If-None-Match: *` for first publication. +Lost races retry at most five times, rereading the current version before each attempt. +Equal or older announcements cannot replace a newer pointer. +Malformed existing pointers and storage permission errors fail closed; they are not treated as an absent release. +Provider diagnostics are withheld from errors to avoid exposing credentials or signed request details. + +If npm verification fails, neither record is written. +If the versioned write succeeds but the pointer update fails, rerun the announcement for the same source commit after correcting the failure. +Never delete a production immutable record to force a retry or lower the latest version as ordinary recovery. + +## Controlled regression and staging verification + +Run from the CLI repository root: + +```sh +node --test scripts/publish-release-announcement.test.mjs +npm run check +./test.sh +``` + +The command-level regression launches the actual publisher with isolated manifests representing every public workspace package. +A test-only preload controls registry and tarball responses and rejects other Node network connections. +Default tests intercept the AWS executable as well, so they require no SaaS credentials and write no remote objects. +Before the fixes, this reproduction failed for missing credentials, interrupted-record consistency, and conflicting source commits. + +To deliberately verify real storage, supply the staging key through the environment and run: + +```sh +node scripts/verify-release-publication.mjs --staging +``` + +The staging command fixes the reviewed account and staging bucket in code, uses controlled npm responses, and executes the real AWS CLI for storage. +It never publishes npm packages or runs the release workflow. +The smoke test checks failed verification before first publication, matching record creation, safe repetition, failed verification after publication, conditional advancement, older announcements, and interrupted-announcement recovery. + +The verifier acquires `verification/release-publication.lock` conditionally and refuses to start if the latest pointer or selected version keys already exist. +It uses synthetic versions only in staging, tracks exact keys, and deletes only objects bearing its unique run identity. +Cleanup verifies every fixture key is absent and removes the lock. +If cleanup fails, the lock remains for investigation; inspect its recorded keys and ownership before removing anything. +Do not empty the bucket or delete another run's records. + +`.github/workflows/verify-release-publication.yml` exposes the same verification as a manual-only workflow using the staging environment. +It requires no dependency installation and is excluded from default PR CI. +Dispatch it from `main` once the workflow is merged; adding a workflow on a pull request does not make it dispatchable from the default branch. + +## Verification evidence + +On 2026-09-10, both newly supplied keys successfully listed release records in their own buckets and received `AccessDenied` for the other release bucket and the unrelated `image-converter` bucket. +Both release namespaces were empty before the staging check. +The previous shared key was not installed in GitHub; revoking it remains a separate Cloudflare owner action. +The 14 focused publisher regressions, `npm run check`, `./test.sh`, and actionlint 1.7.12 checks of both changed workflows passed. +No real release was run. + +Live staging verification passed with AWS CLI 1.46.1 and Node 22.19.0 using the dedicated staging key. +The actual publisher ran with controlled npm metadata for all 11 public workspace packages and real R2 requests. +The fixture versions were `0.1789059755304.0` and `0.1789059755304.1`. +Both matching records, byte-preserving repeated publication, unchanged pointers after verification failures, conditional advancement, older-announcement rejection, and interrupted-record recovery passed. +The exact cleanup covered `releases/v1/latest.json`, `releases/v1/releases/0.1789059755304.0.json`, `releases/v1/releases/0.1789059755304.1.json`, and `verification/release-publication.lock`; each was confirmed absent afterward. + +The GitHub environment secret names and timestamps were verified after secure installation on 2026-09-10 at 17:03 UTC. +No secret values were printed, committed, or copied into client or Worker configuration. +This verifies credential configuration and real publisher-to-R2 behavior; a GitHub-hosted staging run remains available after the new manual workflow reaches `main`. + +A final credential-isolation check confirmed both release namespaces remained empty after staging cleanup. diff --git a/scripts/fixtures/release-publication/aws.mjs b/scripts/fixtures/release-publication/aws.mjs new file mode 100644 index 000000000..126f4a39e --- /dev/null +++ b/scripts/fixtures/release-publication/aws.mjs @@ -0,0 +1,39 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { existsSync, readFileSync, writeFileSync } from "node:fs"; + +const args = process.argv.slice(2); +const value = (flag) => args[args.indexOf(flag) + 1]; +const path = process.env.DIFFEX_PUBLICATION_STORAGE; +const state = JSON.parse(readFileSync(path, "utf8")); +const key = value("--key"); +const object = state.objects[key]; +if (state.failure) { + console.error(`An error occurred (${state.failure}) when calling the fixture operation: fixture-secret`); + process.exit(1); +} +function fail(code) { + console.error(`An error occurred (${code}) when calling the fixture operation`); + process.exit(1); +} +if ( + args[0] !== "s3api" || + value("--bucket") !== "fixture-releases" || + value("--endpoint-url") !== "https://storage.example.invalid" +) + fail("AccessDenied"); +state.calls.push(args.slice(0, 2).concat(key)); +writeFileSync(path, JSON.stringify(state)); +const etag = (body) => `"${createHash("sha256").update(body).digest("hex")}"`; +if (args[1] === "head-object" || args[1] === "get-object") { + if (!object) fail("NoSuchKey"); + if (args[1] === "get-object") writeFileSync(args.at(-1), object); + console.log(JSON.stringify({ ETag: etag(object) })); +} else if (args[1] === "put-object") { + if (args.includes("--if-none-match") && object) fail("PreconditionFailed"); + if (args.includes("--if-match") && (!object || etag(object) !== value("--if-match"))) fail("PreconditionFailed"); + if (!existsSync(value("--body"))) fail("MissingBody"); + state.objects[key] = readFileSync(value("--body"), "utf8"); + writeFileSync(path, JSON.stringify(state)); + console.log(JSON.stringify({ ETag: etag(state.objects[key]) })); +} else fail("UnmockedOperation"); diff --git a/scripts/fixtures/release-publication/harness.mjs b/scripts/fixtures/release-publication/harness.mjs new file mode 100644 index 000000000..5b74351cb --- /dev/null +++ b/scripts/fixtures/release-publication/harness.mjs @@ -0,0 +1,74 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { chmodSync, copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { getPublicWorkspacePackages } from "../../release-packages.mjs"; + +const publisher = fileURLToPath(new URL("../../publish-release-announcement.mjs", import.meta.url)); +const registry = fileURLToPath(new URL("./registry.mjs", import.meta.url)); +export function createPublicationFixture({ staging = false } = {}) { + const directory = mkdtempSync(join(tmpdir(), "diffex-publication-test-")); + const statePath = join(directory, "storage.json"); + const fixturePath = join(directory, "registry.json"); + writeFileSync(statePath, JSON.stringify({ objects: {}, calls: [] })); + mkdirSync(join(directory, "bin")); + const aws = join(directory, "bin/aws"); + copyFileSync(fileURLToPath(new URL("./aws.mjs", import.meta.url)), aws); + chmodSync(aws, 0o700); + const packages = getPublicWorkspacePackages(); + const bucket = staging ? "diffex-artifacts-staging" : "fixture-releases"; + const endpoint = staging + ? "https://6b24afa849f8254ab5e6e2d1956239cc.r2.cloudflarestorage.com" + : "https://storage.example.invalid"; + return { + directory, + bucket, + endpoint, + packageCount: packages.length, + state: () => JSON.parse(readFileSync(statePath, "utf8")), + seed: (objects, failure) => writeFileSync(statePath, JSON.stringify({ objects, failure, calls: [] })), + run(version, { failure, credentials = true, commit = "a".repeat(40) } = {}) { + const manifests = packages.map((pkg, index) => { + const target = join(directory, "packages", `package-${index}`); + mkdirSync(target, { recursive: true }); + const manifest = { name: pkg.name, version }; + writeFileSync(join(target, "package.json"), JSON.stringify(manifest)); + return manifest; + }); + writeFileSync(fixturePath, JSON.stringify({ packages: manifests, failure })); + const env = { + PATH: staging ? process.env.PATH : `${join(directory, "bin")}:${process.env.PATH}`, + AWS_ACCESS_KEY_ID: credentials ? (staging ? process.env.AWS_ACCESS_KEY_ID : "fixture-access") : "", + AWS_SECRET_ACCESS_KEY: credentials ? (staging ? process.env.AWS_SECRET_ACCESS_KEY : "fixture-secret") : "", + AWS_SESSION_TOKEN: staging ? (process.env.AWS_SESSION_TOKEN ?? "") : "", + AWS_CONFIG_FILE: join(directory, "no-aws-config"), + AWS_SHARED_CREDENTIALS_FILE: join(directory, "no-aws-credentials"), + AWS_EC2_METADATA_DISABLED: "true", + DIFFEX_PUBLICATION_FIXTURE: fixturePath, + DIFFEX_PUBLICATION_STORAGE: statePath, + }; + const result = spawnSync( + process.execPath, + [ + "--import", + registry, + publisher, + "--version", + version, + "--bucket", + bucket, + "--endpoint", + endpoint, + "--source-commit", + commit, + ], + { cwd: directory, env, encoding: "utf8", timeout: staging ? 180000 : 30000 }, + ); + assert.ifError(result.error); + return result; + }, + cleanup: () => rmSync(directory, { recursive: true, force: true }), + }; +} diff --git a/scripts/fixtures/release-publication/registry.mjs b/scripts/fixtures/release-publication/registry.mjs new file mode 100644 index 000000000..c892ea9e1 --- /dev/null +++ b/scripts/fixtures/release-publication/registry.mjs @@ -0,0 +1,41 @@ +import { readFileSync } from "node:fs"; +import { syncBuiltinESMExports } from "node:module"; +import net from "node:net"; +import tls from "node:tls"; + +// Only the child publisher loads this fixture. Never contact npm or a provider. +const fixture = JSON.parse(readFileSync(process.env.DIFFEX_PUBLICATION_FIXTURE, "utf8")); +const deny = () => { + throw new Error("Unmocked network access in publication fixture"); +}; +net.connect = deny; +net.createConnection = deny; +net.Socket.prototype.connect = deny; +tls.connect = deny; +syncBuiltinESMExports(); +let now = Date.now(); +Date.now = () => { + now += 11 * 60 * 1000; + return now; +}; +globalThis.fetch = async (input, options) => { + const url = new URL(input); + if (url.origin === "https://registry.npmjs.org") { + const pkg = fixture.packages.find((pkg) => url.pathname === `/${encodeURIComponent(pkg.name)}/${pkg.version}`); + if (!pkg || options?.method === "HEAD") throw new Error("Unexpected registry request"); + if (fixture.failure === "package" && pkg.name === fixture.packages.at(-1).name) + return new Response(null, { status: 404 }); + return Response.json({ + name: pkg.name, + version: pkg.version, + dist: { + tarball: `https://tarballs.example.invalid/${encodeURIComponent(pkg.name)}/${pkg.version}.tgz`, + integrity: "sha512-controlled-fixture", + }, + }); + } + if (url.origin === "https://tarballs.example.invalid" && options?.method === "HEAD") { + return new Response(null, { status: fixture.failure === "tarball" ? 503 : 200 }); + } + throw new Error("Unexpected publication fixture request"); +}; diff --git a/scripts/publish-release-announcement.mjs b/scripts/publish-release-announcement.mjs index 6efa512d8..8c581de7f 100644 --- a/scripts/publish-release-announcement.mjs +++ b/scripts/publish-release-announcement.mjs @@ -5,6 +5,7 @@ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { pathToFileURL } from "node:url"; +import { isDeepStrictEqual } from "node:util"; import { getPublicWorkspacePackages } from "./release-packages.mjs"; const RELEASES_PREFIX = "releases/v1"; @@ -12,7 +13,8 @@ const REGISTRY_URL = "https://registry.npmjs.org"; const RETRY_DELAY_MS = 5000; const RETRY_TIMEOUT_MS = 10 * 60 * 1000; const MAX_POINTER_UPDATE_ATTEMPTS = 5; -const STABLE_SEMVER_RE = /^(\d+)\.(\d+)\.(\d+)$/; +const STABLE_SEMVER_RE = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/; +const REQUEST_TIMEOUT_MS = 30_000; function parseArgs(args) { const options = { @@ -30,7 +32,9 @@ function parseArgs(args) { const value = args[++index]; if (!value) throw new Error(`${arg} requires a value`); options[ - { "--bucket": "bucket", "--endpoint": "endpoint", "--source-commit": "sourceCommit", "--version": "version" }[arg] + { "--bucket": "bucket", "--endpoint": "endpoint", "--source-commit": "sourceCommit", "--version": "version" }[ + arg + ] ] = value; } @@ -48,7 +52,10 @@ function sleep(milliseconds) { async function resolvePackageFromNpm(pkg) { const packageUrl = `${REGISTRY_URL}/${encodeURIComponent(pkg.name)}/${pkg.version}`; - const response = await fetch(packageUrl, { headers: { accept: "application/json" } }); + const response = await fetch(packageUrl, { + headers: { accept: "application/json" }, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); if (!response.ok) { throw new Error(`${pkg.name}@${pkg.version}: npm registry returned ${response.status}`); } @@ -58,6 +65,7 @@ async function resolvePackageFromNpm(pkg) { typeof data !== "object" || data === null || Array.isArray(data) || + data.name !== pkg.name || data.version !== pkg.version || typeof data.dist !== "object" || data.dist === null || @@ -68,7 +76,7 @@ async function resolvePackageFromNpm(pkg) { throw new Error(`${pkg.name}@${pkg.version}: npm registry returned invalid metadata`); } - const tarball = await fetch(data.dist.tarball, { method: "HEAD" }); + const tarball = await fetch(data.dist.tarball, { method: "HEAD", signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS) }); if (!tarball.ok) { throw new Error(`${pkg.name}@${pkg.version}: tarball returned ${tarball.status}`); } @@ -97,12 +105,16 @@ async function verifyPackagesAreAvailable(packages) { return results.map((result) => result.value); } - console.log(`Waiting for ${failures.length} Diffex package${failures.length === 1 ? "" : "s"} on npm (attempt ${attempt}):`); + console.log( + `Waiting for ${failures.length} Diffex package${failures.length === 1 ? "" : "s"} on npm (attempt ${attempt}):`, + ); for (const failure of failures) console.log(` ${failure}`); if (Date.now() < deadline) await sleep(RETRY_DELAY_MS); } while (Date.now() < deadline); - throw new Error(`Timed out waiting for Diffex packages to become available from npm:\n${failures.map((failure) => ` ${failure}`).join("\n")}`); + throw new Error( + `Timed out waiting for Diffex packages to become available from npm:\n${failures.map((failure) => ` ${failure}`).join("\n")}`, + ); } function gitSourceCommit() { @@ -116,52 +128,43 @@ function runAws(args, { allowNotFound = false, allowPreconditionFailure = false ...process.env, AWS_DEFAULT_REGION: process.env.AWS_DEFAULT_REGION || "auto", AWS_EC2_METADATA_DISABLED: "true", + AWS_PAGER: "", }, stdio: ["ignore", "pipe", "pipe"], + timeout: 60_000, }); - if (result.error) throw result.error; + if (result.error) throw new Error("AWS CLI could not run or timed out; verify installation and connectivity."); if (result.status === 0) return result.stdout; - const message = `${result.stdout}\n${result.stderr}`.trim(); - if (allowNotFound && /(?:404|NoSuchKey|Not Found)/i.test(message)) return undefined; - if (allowPreconditionFailure && /(?:412|PreconditionFailed|ConditionalRequestConflict)/i.test(message)) { + // Never echo provider output: it can contain credentials, signed URLs, or headers. + const code = /An error occurred \(([^)]+)\)/.exec(result.stderr)?.[1]; + if (allowNotFound && ["404", "NoSuchKey", "NotFound"].includes(code)) return undefined; + if (allowPreconditionFailure && ["412", "PreconditionFailed", "ConditionalRequestConflict"].includes(code)) return undefined; + if (["AccessDenied", "InvalidAccessKeyId", "SignatureDoesNotMatch", "ExpiredToken"].includes(code)) { + throw new Error("R2 rejected publication credentials; verify account, bucket scope, and credential validity."); } - throw new Error(`aws ${args.slice(0, 2).join(" ")} failed:\n${message}`); + throw new Error( + `AWS CLI ${args.slice(0, 2).join(" ")} failed; check storage availability and AWS CLI conditional-write support.`, + ); } -function readLatestRelease(bucket, endpoint, outputPath) { - const head = runAws( - [ - "s3api", - "head-object", - "--bucket", - bucket, - "--key", - `${RELEASES_PREFIX}/latest.json`, - "--endpoint-url", - endpoint, - ], +function readRelease(bucket, endpoint, key, outputPath) { + // Get the ETag and body from one response so conditional writes use one snapshot. + const result = runAws( + ["s3api", "get-object", "--bucket", bucket, "--key", key, "--endpoint-url", endpoint, outputPath], { allowNotFound: true }, ); - if (head === undefined) return undefined; - - const metadata = JSON.parse(head); - if (typeof metadata.ETag !== "string") { - throw new Error("Latest Diffex release marker has no ETag."); + if (result === undefined) return undefined; + const metadata = JSON.parse(result); + if (typeof metadata.ETag !== "string") throw new Error("Diffex release record has no ETag."); + const body = readFileSync(outputPath, "utf8"); + let release; + try { + release = JSON.parse(body); + } catch { + throw new Error("Diffex release record is not valid JSON."); } - runAws([ - "s3api", - "get-object", - "--bucket", - bucket, - "--key", - `${RELEASES_PREFIX}/latest.json`, - "--endpoint-url", - endpoint, - outputPath, - ]); - const release = JSON.parse(readFileSync(outputPath, "utf8")); if ( typeof release !== "object" || release === null || @@ -169,9 +172,9 @@ function readLatestRelease(bucket, endpoint, outputPath) { typeof release.version !== "string" || !STABLE_SEMVER_RE.test(release.version) ) { - throw new Error("Latest Diffex release marker has an invalid version."); + throw new Error("Diffex release record has an invalid version."); } - return { etag: metadata.ETag, version: release.version }; + return { etag: metadata.ETag, version: release.version, release, body }; } function putJson(bucket, endpoint, path, key, cacheControl, condition) { @@ -202,8 +205,9 @@ export function compareReleaseVersions(left, right) { if (!leftMatch || !rightMatch) throw new Error("Release versions must be stable semver versions."); for (const index of [1, 2, 3]) { - const difference = Number(leftMatch[index]) - Number(rightMatch[index]); - if (difference !== 0) return difference; + const leftPart = BigInt(leftMatch[index]); + const rightPart = BigInt(rightMatch[index]); + if (leftPart !== rightPart) return leftPart > rightPart ? 1 : -1; } return 0; } @@ -220,12 +224,20 @@ export async function advanceLatestRelease(version, readLatest, writeLatest) { return { advanced: true, version }; } } - throw new Error(`Could not advance the Diffex release marker to ${version} after ${MAX_POINTER_UPDATE_ATTEMPTS} attempts.`); + throw new Error( + `Could not advance the Diffex release marker to ${version} after ${MAX_POINTER_UPDATE_ATTEMPTS} attempts.`, + ); } async function main() { const options = parseArgs(process.argv.slice(2)); + if (!process.env.AWS_ACCESS_KEY_ID?.trim() || !process.env.AWS_SECRET_ACCESS_KEY?.trim()) { + throw new Error( + "Release publication requires AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY from the protected release environment.", + ); + } const packages = getPublicWorkspacePackages(); + if (packages.length === 0) throw new Error("No public workspace packages found for verification."); for (const pkg of packages) { if (pkg.version !== options.version) { throw new Error(`${pkg.name} is ${pkg.version}; expected ${options.version}`); @@ -243,7 +255,6 @@ async function main() { const temporaryDirectory = mkdtempSync(join(tmpdir(), "diffex-release-announcement-")); try { const releasePath = join(temporaryDirectory, "release.json"); - const latestPath = join(temporaryDirectory, "latest.json"); writeFileSync(releasePath, `${JSON.stringify(release, null, "\t")}\n`); if ( !putJson( @@ -255,18 +266,47 @@ async function main() { { missing: true }, ) ) { - console.log(`Release record ${options.version} already exists.`); + const existing = readRelease( + options.bucket, + options.endpoint, + `${RELEASES_PREFIX}/releases/${options.version}.json`, + join(temporaryDirectory, "existing.json"), + ); + const identity = ({ schemaVersion, version, sourceCommit, packages }) => ({ + schemaVersion, + version, + sourceCommit, + packages: + Array.isArray(packages) && packages.every((pkg) => pkg && typeof pkg.name === "string") + ? [...packages].sort((a, b) => a.name.localeCompare(b.name)) + : undefined, + }); + if ( + !existing || + !isDeepStrictEqual(identity(existing.release), identity(release)) || + typeof existing.release.publishedAt !== "string" || + !Number.isFinite(Date.parse(existing.release.publishedAt)) + ) { + throw new Error("Existing immutable release record conflicts with the verified announcement."); + } + writeFileSync(releasePath, existing.body); + console.log(`Release record ${options.version} already exists and matches verified publication.`); } - writeFileSync(latestPath, `${JSON.stringify(release, null, "\t")}\n`); const result = await advanceLatestRelease( options.version, - () => readLatestRelease(options.bucket, options.endpoint, join(temporaryDirectory, "latest-current.json")), + () => + readRelease( + options.bucket, + options.endpoint, + `${RELEASES_PREFIX}/latest.json`, + join(temporaryDirectory, "latest-current.json"), + ), (condition) => putJson( options.bucket, options.endpoint, - latestPath, + releasePath, `${RELEASES_PREFIX}/latest.json`, "no-store", condition, diff --git a/scripts/publish-release-announcement.test.mjs b/scripts/publish-release-announcement.test.mjs index 1a51146dd..a52d9704b 100644 --- a/scripts/publish-release-announcement.test.mjs +++ b/scripts/publish-release-announcement.test.mjs @@ -1,5 +1,6 @@ import assert from "node:assert/strict"; import test from "node:test"; +import { createPublicationFixture } from "./fixtures/release-publication/harness.mjs"; import { advanceLatestRelease, compareReleaseVersions } from "./publish-release-announcement.mjs"; test("compares stable release versions numerically", () => { @@ -31,9 +32,7 @@ test("retries a lost conditional update and preserves a racing newer marker", as "0.84.0", async () => { readCount++; - return readCount === 1 - ? { etag: '"previous"', version: "0.83.0" } - : { etag: '"newer"', version: "0.85.0" }; + return readCount === 1 ? { etag: '"previous"', version: "0.83.0" } : { etag: '"newer"', version: "0.85.0" }; }, async (condition) => { writeCount++; @@ -60,3 +59,137 @@ test("creates a missing marker with an if-none-match condition", async () => { assert.deepEqual(result, { advanced: true, version: "0.84.0" }); assert.deepEqual(condition, { missing: true }); }); + +// Exercise the exact release-CI command with isolated manifests, npm, and storage. +test("publisher rejects missing credentials before package verification or storage", () => { + const fixture = createPublicationFixture(); + try { + const result = fixture.run("0.0.3", { credentials: false }); + assert.equal(result.status, 1); + assert.match(result.stderr, /AWS_ACCESS_KEY_ID.*AWS_SECRET_ACCESS_KEY/); + assert.doesNotMatch(result.stdout, /available from npm/); + assert.deepEqual(fixture.state().calls, []); + } finally { + fixture.cleanup(); + } +}); + +for (const failure of ["package", "tarball"]) { + test(`publisher leaves both records untouched when ${failure} verification fails`, () => { + const fixture = createPublicationFixture(); + try { + const objects = { "releases/v1/latest.json": JSON.stringify({ version: "0.0.2" }) }; + fixture.seed(objects); + const result = fixture.run("0.0.3", { failure }); + assert.equal(result.status, 1); + assert.match(result.stderr, /Timed out waiting/); + assert.deepEqual(fixture.state().objects, objects); + assert.deepEqual(fixture.state().calls, []); + } finally { + fixture.cleanup(); + } + }); +} + +test("publisher safely creates, repeats, and orders verified announcements", () => { + const fixture = createPublicationFixture(); + try { + let result = fixture.run("0.0.3"); + assert.equal(result.status, 0, result.stderr); + const first = fixture.state().objects; + assert.equal(first["releases/v1/latest.json"], first["releases/v1/releases/0.0.3.json"]); + result = fixture.run("0.0.3"); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(fixture.state().objects, first); + result = fixture.run("0.0.2"); + assert.equal(result.status, 0, result.stderr); + assert.equal(fixture.state().objects["releases/v1/latest.json"], first["releases/v1/latest.json"]); + assert.ok(fixture.state().objects["releases/v1/releases/0.0.2.json"]); + } finally { + fixture.cleanup(); + } +}); + +test("retry after an interrupted announcement reuses the immutable record", () => { + const fixture = createPublicationFixture(); + try { + const first = fixture.run("0.0.3"); + assert.equal(first.status, 0, first.stderr); + const key = "releases/v1/releases/0.0.3.json"; + const record = JSON.parse(fixture.state().objects[key]); + record.publishedAt = "2025-01-01T00:00:00.000Z"; + const body = JSON.stringify(record); + fixture.seed({ [key]: body }); + const retry = fixture.run("0.0.3"); + assert.equal(retry.status, 0, retry.stderr); + assert.equal(fixture.state().objects["releases/v1/latest.json"], body); + } finally { + fixture.cleanup(); + } +}); + +test("publisher rejects conflicting immutable release identity", () => { + const fixture = createPublicationFixture(); + try { + const first = fixture.run("0.0.3"); + assert.equal(first.status, 0, first.stderr); + const objects = fixture.state().objects; + const retry = fixture.run("0.0.3", { commit: "b".repeat(40) }); + assert.equal(retry.status, 1); + assert.match(retry.stderr, /conflict/i); + assert.deepEqual(fixture.state().objects, objects); + } finally { + fixture.cleanup(); + } +}); + +test("publisher redacts storage failures and never treats denied access as a missing record", () => { + const fixture = createPublicationFixture(); + try { + fixture.seed({}, "AccessDenied"); + const result = fixture.run("0.0.3"); + assert.equal(result.status, 1); + assert.match(result.stderr, /R2 rejected publication credentials/); + assert.doesNotMatch(result.stderr + result.stdout, /fixture-secret/); + assert.deepEqual(fixture.state().objects, {}); + } finally { + fixture.cleanup(); + } +}); + +test("publisher fails closed when the existing latest record is malformed", () => { + const fixture = createPublicationFixture(); + try { + fixture.seed({ "releases/v1/latest.json": "malformed fixture-secret" }); + const result = fixture.run("0.0.3"); + assert.equal(result.status, 1); + assert.match(result.stderr, /not valid JSON/); + assert.doesNotMatch(result.stderr + result.stdout, /fixture-secret/); + assert.equal(fixture.state().objects["releases/v1/latest.json"], "malformed fixture-secret"); + } finally { + fixture.cleanup(); + } +}); + +test("pointer contention has a bounded retry limit", async () => { + let attempts = 0; + await assert.rejects( + advanceLatestRelease( + "0.0.3", + async () => undefined, + async () => { + attempts++; + return false; + }, + ), + /after 5 attempts/, + ); + assert.equal(attempts, 5); +}); + +test("version comparison rejects noncanonical versions and avoids numeric precision loss", () => { + for (const value of ["01.2.3", "1.02.3", "1.2.03", "v1.2.3", "1.2.3+build", "1.2.3-beta.1"]) { + assert.throws(() => compareReleaseVersions(value, "1.2.3")); + } + assert.equal(compareReleaseVersions("9007199254740993.0.0", "9007199254740992.0.0"), 1); +}); diff --git a/scripts/verify-release-publication.mjs b/scripts/verify-release-publication.mjs new file mode 100644 index 000000000..e32e3e260 --- /dev/null +++ b/scripts/verify-release-publication.mjs @@ -0,0 +1,135 @@ +#!/usr/bin/env node +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import { readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { createPublicationFixture } from "./fixtures/release-publication/harness.mjs"; + +if (process.argv.length !== 3 || process.argv[2] !== "--staging") { + throw new Error("Use --staging with staging-scoped AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY."); +} +if (!process.env.AWS_ACCESS_KEY_ID || !process.env.AWS_SECRET_ACCESS_KEY) { + throw new Error("Staging verification requires AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY."); +} +const fixture = createPublicationFixture({ staging: true }); +const commit = randomBytes(20).toString("hex"); +const version = `0.${Date.now()}.0`; +const newer = version.replace(/\.0$/, ".1"); +const latestKey = "releases/v1/latest.json"; +const lockKey = "verification/release-publication.lock"; +const keys = [latestKey, `releases/v1/releases/${version}.json`, `releases/v1/releases/${newer}.json`]; +let locked = false; +let ready = false; +const failures = []; + +function aws(operation, args, { allowMissing = false } = {}) { + const result = spawnSync( + "aws", + ["s3api", operation, "--bucket", fixture.bucket, "--endpoint-url", fixture.endpoint, ...args], + { + encoding: "utf8", + timeout: 60000, + env: { ...process.env, AWS_DEFAULT_REGION: "auto", AWS_EC2_METADATA_DISABLED: "true", AWS_PAGER: "" }, + }, + ); + const code = /An error occurred \(([^)]+)\)/.exec(result.stderr ?? "")?.[1]; + if (allowMissing && ["NoSuchKey", "404", "NotFound"].includes(code)) return undefined; + if (result.error || result.status !== 0) + throw new Error( + `Staging ${operation} failed; check credentials, concurrency lock, and conditional-write support.`, + ); + return JSON.parse(result.stdout || "{}"); +} +function read(key) { + const file = join(fixture.directory, "readback.json"); + const metadata = aws("get-object", ["--key", key, file], { allowMissing: true }); + if (!metadata) return undefined; + return { body: readFileSync(file, "utf8"), metadata }; +} +function publish(value, options = {}) { + const result = fixture.run(value, { commit, ...options }); + assert.equal( + result.status, + options.failure ? 1 : 0, + "Unexpected publisher exit status; provider output is withheld.", + ); +} +function removeOwned(key) { + const object = read(key); + if (!object) return; + assert.ok(JSON.parse(object.body).sourceCommit === commit, "Refusing cleanup of an object owned by another writer."); + aws("delete-object", ["--key", key]); + assert.equal(read(key), undefined, "Staging object remains after cleanup."); +} + +try { + const lockFile = join(fixture.directory, "lock.json"); + writeFileSync(lockFile, JSON.stringify({ sourceCommit: commit, keys })); + aws("put-object", ["--key", lockKey, "--body", lockFile, "--if-none-match", "*"]); + locked = true; + for (const key of keys) + assert.ok(read(key) === undefined, "Staging release keys must be absent before verification."); + ready = true; + console.log(`Staging publication fixture: ${version}, ${newer}`); + + for (const failure of ["package", "tarball"]) { + publish(version, { failure }); + for (const key of keys) assert.equal(read(key), undefined); + } + console.log("Missing packages and unavailable tarballs leave absent records unchanged."); + publish(version); + const initial = read(latestKey); + assert.equal(initial.body, read(keys[1]).body); + assert.equal(initial.metadata.CacheControl, "no-store"); + assert.match(initial.metadata.ContentType, /^application\/json/); + assert.equal(JSON.parse(initial.body).packages.length, fixture.packageCount); + console.log("First verified publication creates matching versioned and latest records."); + publish(version); + assert.equal(read(latestKey).body, initial.body); + assert.equal(read(keys[1]).body, initial.body); + for (const failure of ["package", "tarball"]) { + publish(newer, { failure }); + assert.equal(read(latestKey).body, initial.body); + assert.equal(read(keys[2]), undefined); + } + console.log("Repeated publication and failed verification preserve existing record bytes."); + publish(newer); + const advanced = read(latestKey).body; + assert.equal(advanced, read(keys[2]).body); + assert.equal(JSON.parse(advanced).version, newer); + publish(version); + assert.equal(read(latestKey).body, advanced); + console.log("Conditional advancement succeeds and older announcements cannot regress latest."); + removeOwned(latestKey); + publish(newer); + assert.equal(read(latestKey).body, advanced); + console.log("Interrupted publication recovery reuses the immutable record exactly."); +} catch (error) { + failures.push(error); +} finally { + try { + if (locked) { + let cleanupFailed = false; + for (const key of ready ? keys : []) { + try { + removeOwned(key); + } catch (error) { + cleanupFailed = true; + failures.push(error); + } + } + if (cleanupFailed) { + console.error("Staging cleanup failed; retain the lock and inspect this run's keys."); + } else { + removeOwned(lockKey); + console.log("Cleanup verified: fixture keys are absent and the staging lock was removed."); + } + } + } catch (error) { + failures.push(error); + } finally { + fixture.cleanup(); + } +} +if (failures.length) throw new AggregateError(failures, "Staging publication verification failed.");