From e6560ea6e400718349c84f1f4ba117badc67df9e Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Mon, 26 Oct 2026 19:45:37 -0400 Subject: [PATCH 1/4] feat(ai): publish immutable model catalogs - Validate complete catalog exports before publication - Preserve generation provenance and advance R2 indexes conditionally - Bind serialized uploads to current main and separate staging credentials --- .github/workflows/publish-model-catalog.yml | 62 +++-- packages/ai/scripts/generate-models.ts | 6 + packages/ai/scripts/model-data.ts | 48 +--- packages/ai/src/index.ts | 1 + packages/ai/src/model-validation.ts | 73 ++++++ packages/ai/test/catalog-export.test.ts | 74 ++++++ scripts/model-publication-security.test.mjs | 11 + scripts/model-publication-window.mjs | 19 ++ scripts/publish-model-catalog.mjs | 241 +++++++++++--------- scripts/publish-model-catalog.test.mjs | 127 +++++++++++ 10 files changed, 474 insertions(+), 188 deletions(-) create mode 100644 packages/ai/src/model-validation.ts create mode 100644 packages/ai/test/catalog-export.test.ts create mode 100644 scripts/model-publication-window.mjs create mode 100644 scripts/publish-model-catalog.test.mjs diff --git a/.github/workflows/publish-model-catalog.yml b/.github/workflows/publish-model-catalog.yml index 8a0c34143..382c10958 100644 --- a/.github/workflows/publish-model-catalog.yml +++ b/.github/workflows/publish-model-catalog.yml @@ -12,7 +12,7 @@ on: - '.gitignore' - 'package.json' - 'packages/ai/**' - - 'scripts/publish-model-catalog.mjs' + - 'scripts/*model*.mjs' # GitHub schedules use UTC. Run hourly candidates across the CET/CEST # boundaries; the publish job only uploads at 10:17, 12:17, and 14:17 # Europe/Vienna time. @@ -26,11 +26,17 @@ on: default: 'main' type: string publish: - description: 'Upload the generated catalog to production R2' + description: 'Upload the generated catalog to the selected R2 environment' required: true default: false type: boolean + environment: + description: 'Catalog upload environment' + type: choice + options: [staging, production] + default: staging + permissions: contents: read @@ -38,8 +44,10 @@ jobs: generate: if: ${{ github.event_name != 'workflow_run' || (github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push' && github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.head_repository.full_name == github.repository) }} runs-on: ubuntu-latest + outputs: + source_sha: ${{ steps.source.outputs.sha }} env: - SOURCE_REF: ${{ github.event.workflow_run.head_sha || inputs.source_ref || github.sha }} + SOURCE_REF: ${{ needs.generate.outputs.source_sha }} steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -47,6 +55,10 @@ jobs: ref: ${{ env.SOURCE_REF }} persist-credentials: false + - name: Resolve source SHA + id: source + run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + - name: Setup Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: @@ -57,6 +69,8 @@ jobs: run: npm ci --ignore-scripts - name: Generate model catalog JSON + env: + MODEL_CATALOG_SOURCE_SHA: ${{ steps.source.outputs.sha }} run: npm run generate:model-catalog - name: Validate model catalog JSON @@ -74,15 +88,16 @@ jobs: if: ${{ github.ref == 'refs/heads/main' && ((github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.event == 'push' && github.event.workflow_run.head_branch == 'main' && github.event.workflow_run.head_repository.full_name == github.repository) || github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && inputs.publish && inputs.source_ref == 'main')) }} needs: generate runs-on: ubuntu-latest - environment: diffex-model-upload + environment: ${{ inputs.environment == 'staging' && 'diffex-model-upload-staging' || 'diffex-model-upload-production' }} concurrency: - group: publish-model-catalog-r2 - cancel-in-progress: true + group: publish-model-catalog-${{ inputs.environment == 'staging' && 'staging' || 'production' }} + cancel-in-progress: false env: - SOURCE_REF: ${{ github.event.workflow_run.head_sha || inputs.source_ref || github.sha }} + SOURCE_REF: ${{ needs.generate.outputs.source_sha }} AWS_DEFAULT_REGION: auto AWS_EC2_METADATA_DISABLED: 'true' - R2_ENDPOINT: https://67c0d357268b0fca6e0b465bb9d01b84.r2.cloudflarestorage.com + CATALOG_BUCKET: ${{ inputs.environment == 'staging' && 'diffex-artifacts-staging' || 'diffex-artifacts' }} + R2_ENDPOINT: https://6b24afa849f8254ab5e6e2d1956239cc.r2.cloudflarestorage.com steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -109,39 +124,18 @@ jobs: env: EVENT_NAME: ${{ github.event_name }} MANUAL_PUBLISH: ${{ inputs.publish || false }} - run: | - set -euo pipefail - - local_day="$(TZ=Europe/Vienna date +%u)" - local_hour_text="$(TZ=Europe/Vienna date +%H)" - local_hour="$((10#$local_hour_text))" - local_time="$(TZ=Europe/Vienna date '+%Y-%m-%d %H:%M:%S %Z')" - allowed=false - reason="outside the Monday-Friday 10:00-15:00 Europe/Vienna publication window" - - if [[ "$EVENT_NAME" == "workflow_dispatch" && "$MANUAL_PUBLISH" == "true" ]]; then - allowed=true - reason="manual publication" - elif (( local_day <= 5 && local_hour >= 10 && local_hour < 15 )); then - if [[ "$EVENT_NAME" != "schedule" ]] || (( (local_hour - 10) % 2 == 0 )); then - allowed=true - reason="business-hours publication" - else - reason="not a scheduled 10:17, 12:17, or 14:17 Europe/Vienna publication" - fi - fi - - echo "allowed=$allowed" >> "$GITHUB_OUTPUT" - echo "R2 publication allowed: $allowed ($reason; local time: $local_time)" + run: node scripts/model-publication-window.mjs - name: Publish model catalog to R2 if: steps.publication-window.outputs.allowed == 'true' env: + GH_TOKEN: ${{ github.token }} AWS_ACCESS_KEY_ID: ${{ secrets.DIFFEX_ARTIFACTS_R2_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.DIFFEX_ARTIFACTS_R2_SECRET_ACCESS_KEY }} run: | node scripts/publish-model-catalog.mjs \ --input .artifacts/model-catalog \ - --bucket diffex-artifacts \ + --bucket "$CATALOG_BUCKET" \ --endpoint "$R2_ENDPOINT" \ - --source-commit "$(git rev-parse HEAD)" + --source-commit "$SOURCE_REF" \ + --require-main diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index 4e1f8fd4b..4dd6abbba 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -10,6 +10,7 @@ import { CLOUDFLARE_AI_GATEWAY_OPENAI_BASE_URL, CLOUDFLARE_WORKERS_AI_BASE_URL, } from "../src/api/cloudflare.ts"; +import { parseModelCatalog } from "../src/model-validation.ts"; import type { AnthropicMessagesCompat, Api, @@ -2855,6 +2856,7 @@ async function generateModels() { } } + for (const providerId of sortedProviderIds) parseModelCatalog(providerId, jsonProviders[providerId]); const generatedAt = new Date().toISOString(); if (!generatorOptions.jsonOnly) { @@ -2964,6 +2966,10 @@ async function generateModels() { const providerOutputDir = join(generatorOptions.jsonOutputDir, "providers"); rmSync(generatorOptions.jsonOutputDir, { recursive: true, force: true }); mkdirSync(providerOutputDir, { recursive: true }); + writeJson(join(generatorOptions.jsonOutputDir, "generation.json"), { + generatedAt, + sourceCommit: process.env.MODEL_CATALOG_SOURCE_SHA, + }); writeJson(join(generatorOptions.jsonOutputDir, "models.json"), jsonProviders); writeJson(join(generatorOptions.jsonOutputDir, "providers.json"), sortedProviderIds); for (const providerId of sortedProviderIds) { diff --git a/packages/ai/scripts/model-data.ts b/packages/ai/scripts/model-data.ts index f2081817a..96ed93c95 100644 --- a/packages/ai/scripts/model-data.ts +++ b/packages/ai/scripts/model-data.ts @@ -2,6 +2,8 @@ import { createHash } from "node:crypto"; import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; import { join } from "node:path"; +import { validateModelValue } from "../src/model-validation.ts"; + export const MODEL_DATA_SCHEMA_VERSION = 3; export const MODEL_DATA_MANIFEST_FILE = ".manifest.json"; @@ -137,52 +139,6 @@ export function createModelDataManifest( }; } -function validateModelValue( - value: unknown, - providerId: string, - modelId: string, - expectedApi: string, - errors: string[], -): void { - const label = `${providerId}/${modelId}`; - if (!isRecord(value)) { - errors.push(`${label} must be an object`); - return; - } - if (value.id !== modelId) errors.push(`${label} has id ${JSON.stringify(value.id)}, expected ${JSON.stringify(modelId)}`); - if (value.provider !== providerId) { - errors.push(`${label} has provider ${JSON.stringify(value.provider)}, expected ${JSON.stringify(providerId)}`); - } - if (value.api !== expectedApi) { - errors.push(`${label} has api ${JSON.stringify(value.api)}, expected ${JSON.stringify(expectedApi)}`); - } - if (typeof value.name !== "string" || value.name.length === 0) errors.push(`${label} has no model name`); - if (typeof value.baseUrl !== "string") errors.push(`${label} has no baseUrl string`); - if (typeof value.reasoning !== "boolean") errors.push(`${label} has no reasoning boolean`); - if ( - !Array.isArray(value.input) || - value.input.length === 0 || - value.input.some((entry) => entry !== "text" && entry !== "image") - ) { - errors.push(`${label} has invalid input modalities`); - } - if (typeof value.contextWindow !== "number" || !Number.isFinite(value.contextWindow) || value.contextWindow <= 0) { - errors.push(`${label} has invalid contextWindow`); - } - if (typeof value.maxTokens !== "number" || !Number.isFinite(value.maxTokens) || value.maxTokens <= 0) { - errors.push(`${label} has invalid maxTokens`); - } - if (!isRecord(value.cost)) { - errors.push(`${label} has invalid cost metadata`); - } else { - for (const field of ["input", "output", "cacheRead", "cacheWrite"] as const) { - const cost = value.cost[field]; - if (typeof cost !== "number" || !Number.isFinite(cost)) { - errors.push(`${label} has invalid cost.${field}`); - } - } - } -} function throwValidationErrors(errors: string[]): never { const visible = errors.slice(0, 30); diff --git a/packages/ai/src/index.ts b/packages/ai/src/index.ts index b15455080..e0b2493f5 100644 --- a/packages/ai/src/index.ts +++ b/packages/ai/src/index.ts @@ -35,6 +35,7 @@ export type { OAuthSelectPrompt, } from "./compat/extension-oauth-types.ts"; export * from "./images-models.ts"; +export { parseModelCatalog } from "./model-validation.ts"; export * from "./models.ts"; export * from "./models-store.ts"; export * from "./providers/faux.ts"; diff --git a/packages/ai/src/model-validation.ts b/packages/ai/src/model-validation.ts new file mode 100644 index 000000000..f67b96ac2 --- /dev/null +++ b/packages/ai/src/model-validation.ts @@ -0,0 +1,73 @@ +import type { Api, Model } from "./types.ts"; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function validateModelValue( + value: unknown, + providerId: string, + modelId: string, + expectedApi: string, + errors: string[], +): void { + const label = `${providerId}/${modelId}`; + if (!isRecord(value)) { + errors.push(`${label} must be an object`); + return; + } + if (value.id !== modelId) + errors.push(`${label} has id ${JSON.stringify(value.id)}, expected ${JSON.stringify(modelId)}`); + if (value.provider !== providerId) { + errors.push(`${label} has provider ${JSON.stringify(value.provider)}, expected ${JSON.stringify(providerId)}`); + } + if (typeof value.api !== "string" || value.api.length === 0) errors.push(`${label} has no api string`); + if (value.api !== expectedApi) { + errors.push(`${label} has api ${JSON.stringify(value.api)}, expected ${JSON.stringify(expectedApi)}`); + } + if (typeof value.name !== "string" || value.name.length === 0) errors.push(`${label} has no model name`); + if (typeof value.baseUrl !== "string") errors.push(`${label} has no baseUrl string`); + if (typeof value.reasoning !== "boolean") errors.push(`${label} has no reasoning boolean`); + if ( + !Array.isArray(value.input) || + value.input.length === 0 || + value.input.some((entry) => entry !== "text" && entry !== "image") + ) { + errors.push(`${label} has invalid input modalities`); + } + if (typeof value.contextWindow !== "number" || !Number.isFinite(value.contextWindow) || value.contextWindow <= 0) { + errors.push(`${label} has invalid contextWindow`); + } + if (typeof value.maxTokens !== "number" || !Number.isFinite(value.maxTokens) || value.maxTokens <= 0) { + errors.push(`${label} has invalid maxTokens`); + } + if (!isRecord(value.cost)) { + errors.push(`${label} has invalid cost metadata`); + } else { + for (const field of ["input", "output", "cacheRead", "cacheWrite"] as const) { + const cost = value.cost[field]; + if (typeof cost !== "number" || !Number.isFinite(cost)) { + errors.push(`${label} has invalid cost.${field}`); + } + } + } +} + +/** Validate a complete provider shard before accepting any model metadata. */ +export function parseModelCatalog(providerId: string, value: unknown): Model[] { + if (!isRecord(value) || Object.keys(value).length === 0) { + throw new Error(`Invalid model catalog for provider "${providerId}"`); + } + const errors: string[] = []; + for (const [id, model] of Object.entries(value)) { + validateModelValue( + model, + providerId, + id, + isRecord(model) && typeof model.api === "string" ? model.api : "", + errors, + ); + } + if (errors.length) throw new Error(`Invalid model catalog: ${errors.slice(0, 30).join("; ")}`); + return Object.values(value) as Model[]; +} diff --git a/packages/ai/test/catalog-export.test.ts b/packages/ai/test/catalog-export.test.ts new file mode 100644 index 000000000..7069c10f3 --- /dev/null +++ b/packages/ai/test/catalog-export.test.ts @@ -0,0 +1,74 @@ +import { spawnSync } from "node:child_process"; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { describe, expect, it } from "vitest"; + +describe("strict JSON catalog export", () => { + it.each(["valid", "upstream-failure", "malformed-model"])("validates %s before writing output", (scenario) => { + const root = mkdtempSync(join(tmpdir(), "diffex-export-")); + try { + const preload = join(root, "upstream.mjs"); + const catalog = { + "alibaba-token-plan": { + models: Object.fromEntries( + [ + "deepseek-v4-flash-0731", + "deepseek-v4-pro", + "glm-5.2", + "qwen3.6-flash", + "qwen3.7-max", + "qwen3.7-plus", + "qwen3.8-max", + ].map((id) => [id, { id, name: id, tool_call: true }]), + ), + }, + }; + writeFileSync( + preload, + `const scenario = ${JSON.stringify(scenario)}; +const catalog = ${JSON.stringify(catalog)}; +globalThis.fetch = async (input) => { + if (scenario === "upstream-failure") return new Response("unavailable", { status: 503 }); + if (String(input) === "https://models.dev/api.json") return Response.json(catalog); + if (String(input) === "https://openrouter.ai/api/v1/models") return Response.json({ data: [{ + id: "fixture", name: "Fixture", supported_parameters: ["tools"], + pricing: { prompt: scenario === "malformed-model" ? "invalid" : "0.001" } + }] }); + return Response.json({ data: [] }); +}; +`, + ); + const output = join(root, "catalog"); + const result = spawnSync( + process.execPath, + [ + "--import", + pathToFileURL(preload).href, + "scripts/generate-models.ts", + "--strict", + "--json-only", + "--json-output", + output, + ], + { + cwd: fileURLToPath(new URL("..", import.meta.url)), + encoding: "utf8", + timeout: 10_000, + }, + ); + if (scenario === "valid") { + expect(result.status, result.stderr).toBe(0); + const generation = JSON.parse(readFileSync(join(output, "generation.json"), "utf8")); + expect(Number.isFinite(Date.parse(generation.generatedAt))).toBe(true); + } else { + expect(result.status).toBe(1); + expect(result.stderr).toContain(scenario === "upstream-failure" ? "503" : "cost.input"); + expect(existsSync(output)).toBe(false); + } + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/scripts/model-publication-security.test.mjs b/scripts/model-publication-security.test.mjs index 6dedfdeda..40565ea65 100644 --- a/scripts/model-publication-security.test.mjs +++ b/scripts/model-publication-security.test.mjs @@ -44,3 +44,14 @@ test("publication accepts only main for scheduled and manual runs", () => { } assert.equal(allowed("publish", { event_name: "schedule", ref: "refs/heads/feature" }), false); }); + +test("binds artifact and publishing code to one SHA and serializes uploads without cancellation", () => { + assert.equal(workflow.jobs.publish.env.SOURCE_REF, "${{ needs.generate.outputs.source_sha }}"); + assert.equal(workflow.jobs.publish.concurrency["cancel-in-progress"], false); + assert.match(workflow.jobs.publish.environment, /diffex-model-upload-staging/); + assert.match(workflow.jobs.publish.environment, /diffex-model-upload-production/); + assert.match(workflow.jobs.publish.env.R2_ENDPOINT, /6b24afa849f8254ab5e6e2d1956239cc/); + assert.match(workflow.jobs.publish.steps.at(-1).run, /--require-main/); + assert.equal(workflow.jobs.generate.steps.find((step) => step.name === "Generate model catalog JSON").env.MODEL_CATALOG_SOURCE_SHA, + "${{ steps.source.outputs.sha }}"); +}); diff --git a/scripts/model-publication-window.mjs b/scripts/model-publication-window.mjs new file mode 100644 index 000000000..e38aa5da5 --- /dev/null +++ b/scripts/model-publication-window.mjs @@ -0,0 +1,19 @@ +import { appendFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; + +export function publicationAllowed(event, manual, date = new Date()) { + if (event === "workflow_dispatch") return manual; + if (event !== "schedule" && event !== "workflow_run") return false; + const parts = Object.fromEntries(new Intl.DateTimeFormat("en-GB", { + timeZone: "Europe/Vienna", weekday: "short", hour: "2-digit", hourCycle: "h23", + }).formatToParts(date).map(({ type, value }) => [type, value])); + const hour = Number(parts.hour); + return !["Sat", "Sun"].includes(parts.weekday) && hour >= 10 && hour < 15 && + (event !== "schedule" || (hour - 10) % 2 === 0); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const allowed = publicationAllowed(process.env.EVENT_NAME, process.env.MANUAL_PUBLISH === "true"); + appendFileSync(process.env.GITHUB_OUTPUT, `allowed=${allowed}\n`); + console.log(`Catalog publication window: ${allowed}`); +} diff --git a/scripts/publish-model-catalog.mjs b/scripts/publish-model-catalog.mjs index 5efe93d79..038b21b97 100644 --- a/scripts/publish-model-catalog.mjs +++ b/scripts/publish-model-catalog.mjs @@ -12,12 +12,14 @@ import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { spawnSync } from "node:child_process"; import { isDeepStrictEqual } from "node:util"; +import { pathToFileURL } from "node:url"; +import { parseModelCatalog } from "../packages/ai/src/model-validation.ts"; const CATALOG_SCHEMA_VERSION = 1; const CATALOG_PREFIX = `models/v${CATALOG_SCHEMA_VERSION}`; const CATALOG_INDEX_KEY = `${CATALOG_PREFIX}/index.json`; // Bump this only when generated model metadata requires behavior unavailable in older Diffex clients. -const MINIMUM_DIFFEX_VERSION = "0.80.7"; +const MINIMUM_DIFFEX_VERSION = "0.3.3"; const JSON_CONTENT_TYPE = "application/json; charset=utf-8"; const IMMUTABLE_CACHE_CONTROL = "public, max-age=31536000, immutable"; const INDEX_CACHE_CONTROL = "no-store"; @@ -39,6 +41,10 @@ function parseArgs(args) { options.dryRun = true; continue; } + if (arg === "--require-main") { + options.requireMain = true; + continue; + } if (arg === "--input" || arg === "--bucket" || arg === "--endpoint" || arg === "--source-commit") { const value = args[++index]; if (!value) throw new Error(`${arg} requires a value`); @@ -65,13 +71,17 @@ function readJson(path) { return JSON.parse(readFileSync(path, "utf8")); } -function validateBundle(inputDir) { +export function validateBundle(inputDir) { const modelsPath = join(inputDir, "models.json"); const providerIndexPath = join(inputDir, "providers.json"); const providersDir = join(inputDir, "providers"); const modelsBytes = readFileSync(modelsPath); const models = JSON.parse(modelsBytes.toString("utf8")); const providerIds = readJson(providerIndexPath); + const generation = readJson(join(inputDir, "generation.json")); + if (typeof generation.generatedAt !== "string" || !Number.isFinite(Date.parse(generation.generatedAt))) { + throw new Error("Invalid generation timestamp"); + } if (typeof models !== "object" || models === null || Array.isArray(models)) { throw new Error("models.json must contain an object"); @@ -90,6 +100,7 @@ function validateBundle(inputDir) { let modelCount = 0; for (const providerId of providerIds) { + if (!/^[a-z0-9][a-z0-9-]{0,99}$/.test(providerId)) throw new Error(`Invalid provider ID: ${providerId}`); const providerModels = models[providerId]; if (typeof providerModels !== "object" || providerModels === null || Array.isArray(providerModels)) { throw new Error(`Provider catalog must be an object: ${providerId}`); @@ -98,18 +109,7 @@ function validateBundle(inputDir) { if (!isDeepStrictEqual(providerFile, providerModels)) { throw new Error(`Provider shard does not match models.json: ${providerId}`); } - for (const [modelId, model] of Object.entries(providerModels)) { - if ( - typeof model !== "object" || - model === null || - Array.isArray(model) || - model.id !== modelId || - model.provider !== providerId - ) { - throw new Error(`Invalid model entry: ${providerId}/${modelId}`); - } - modelCount++; - } + modelCount += parseModelCatalog(providerId, providerModels).length; } const shardFiles = readdirSync(providersDir).filter((name) => name.endsWith(".json")).sort(); @@ -123,6 +123,8 @@ function validateBundle(inputDir) { const digest = createHash("sha256").update(modelsBytes).digest("hex"); return { + generatedAt: generation.generatedAt, + sourceCommit: generation.sourceCommit, modelsPath, providerIndexPath, providersDir, @@ -139,51 +141,40 @@ function gitSourceCommit() { return result.stdout.trim(); } -function aws(args, { allowNotFound = false } = {}) { - const result = spawnSync("aws", args, { - encoding: "utf8", - env: { - ...process.env, - AWS_DEFAULT_REGION: process.env.AWS_DEFAULT_REGION || "auto", - AWS_EC2_METADATA_DISABLED: "true", +// Use S3 conditional puts, never copy/overwrite immutable revision objects. +export function createS3Store(bucket, endpoint, temporaryDir) { + let sequence = 0; + function command(args, { missing = false, conflict = false } = {}) { + const result = spawnSync("aws", ["s3api", ...args, "--bucket", bucket, "--endpoint-url", endpoint, "--output", "json"], { + encoding: "utf8", + env: { ...process.env, AWS_DEFAULT_REGION: "auto", AWS_EC2_METADATA_DISABLED: "true", AWS_PAGER: "" }, + }); + if (result.error) throw result.error; + if (result.status === 0) return JSON.parse(result.stdout || "{}"); + const message = `${result.stdout}\n${result.stderr}`; + if (missing && /(?:NoSuchKey|\(404\)|Not Found)/i.test(message)) return null; + if (conflict && /(?:PreconditionFailed|ConditionalRequestConflict|\(412\)|\(409\))/i.test(message)) return false; + throw new Error(`R2 ${args[0]} failed: ${message.trim()}`); + } + return { + get(key) { + const path = join(temporaryDir, `get-${sequence++}`); + const metadata = command(["get-object", "--key", key, path], { missing: true }); + if (!metadata) return null; + if (!metadata.ETag) throw new Error(`Missing ETag for ${key}`); + return { bytes: readFileSync(path), etag: metadata.ETag }; }, - }); - if (result.error) throw result.error; - if (result.status === 0) return true; - const message = `${result.stdout}\n${result.stderr}`.trim(); - if (allowNotFound && /(?:404|NoSuchKey|Not Found)/i.test(message)) return false; - throw new Error(`aws ${args.slice(0, 2).join(" ")} failed:\n${message}`); -} - -function downloadIndex(bucket, endpoint, outputPath) { - return aws( - [ - "s3", - "cp", - `s3://${bucket}/${CATALOG_INDEX_KEY}`, - outputPath, - "--endpoint-url", - endpoint, - "--only-show-errors", - ], - { allowNotFound: true }, - ); -} - -function uploadJson(bucket, endpoint, sourcePath, key, cacheControl) { - aws([ - "s3", - "cp", - sourcePath, - `s3://${bucket}/${key}`, - "--endpoint-url", - endpoint, - "--content-type", - JSON_CONTENT_TYPE, - "--cache-control", - cacheControl, - "--only-show-errors", - ]); + put(key, bytes, etag) { + const path = join(temporaryDir, `put-${sequence++}`); + writeFileSync(path, bytes); + return command([ + "put-object", "--key", key, "--body", path, + "--content-type", JSON_CONTENT_TYPE, + "--cache-control", key === CATALOG_INDEX_KEY ? INDEX_CACHE_CONTROL : IMMUTABLE_CACHE_CONTROL, + ...(etag ? ["--if-match", etag] : ["--if-none-match", "*"]), + ], { conflict: true }) !== false; + }, + }; } function validateIndex(index) { @@ -201,12 +192,16 @@ function validateIndex(index) { typeof catalog !== "object" || catalog === null || Array.isArray(catalog) || - typeof catalog.minimumDiffexVersion !== "string" || - typeof catalog.revision !== "string" + !/^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/.test(catalog.minimumDiffexVersion) || + !/^sha256-[a-f0-9]{64}$/.test(catalog.revision) ) { throw new Error(`Existing ${CATALOG_INDEX_KEY} contains an invalid catalog entry`); } } + if (!index.catalogs.some((catalog) => catalog.revision === index.defaultRevision) || + new Set(index.catalogs.map((catalog) => catalog.minimumDiffexVersion)).size !== index.catalogs.length) { + throw new Error("Invalid default revision or duplicate compatibility floors"); + } return index; } @@ -225,7 +220,7 @@ function buildIndex(existingIndex, publication) { minimumDiffexVersion: MINIMUM_DIFFEX_VERSION, revision: publication.revision, sourceCommit: publication.sourceCommit, - publishedAt: new Date().toISOString(), + generatedAt: publication.generatedAt, providerCount: publication.providerCount, modelCount: publication.modelCount, }; @@ -236,73 +231,103 @@ function buildIndex(existingIndex, publication) { return { schemaVersion: CATALOG_SCHEMA_VERSION, defaultRevision: publication.revision, + validatedAt: new Date().toISOString(), catalogs, }; } -async function main() { - const options = parseArgs(process.argv.slice(2)); - const inputDir = resolve(options.input); +function jsonBytes(value) { + return Buffer.from(`${JSON.stringify(value, null, 2)}\n`); +} + +export function assertCurrentSource(sourceCommit) { + if (!/^[a-f0-9]{40}$/.test(sourceCommit) || gitSourceCommit() !== sourceCommit) { + throw new Error("Publisher code and artifact source SHA differ"); + } + const repository = process.env.GITHUB_REPOSITORY; + if (repository !== "diffexai/diffex") throw new Error("Untrusted catalog repository"); + const result = spawnSync("gh", ["api", `repos/${repository}/commits/main`, "--jq", ".sha"], { encoding: "utf8" }); + if (result.status !== 0 || result.stdout.trim() !== sourceCommit) throw new Error("Refusing stale main revision"); +} + +export function publishCatalog(inputDir, store, sourceCommit, checkSource = () => {}) { const bundle = validateBundle(inputDir); + if (!/^[a-f0-9]{40}$/.test(sourceCommit)) throw new Error("Invalid source commit"); + if (bundle.sourceCommit !== undefined && bundle.sourceCommit !== sourceCommit) throw new Error("Artifact source SHA differs"); + checkSource(sourceCommit); + const prefix = `${CATALOG_PREFIX}/revisions/${bundle.revision}`; + const objects = new Map([ + [`${prefix}/models.json`, readFileSync(bundle.modelsPath)], + [`${prefix}/providers.json`, readFileSync(bundle.providerIndexPath)], + ...bundle.providerIds.map((id) => [`${prefix}/providers/${id}.json`, readFileSync(join(bundle.providersDir, `${id}.json`))]), + ]); + const providers = Object.fromEntries(bundle.providerIds.map((id) => { + const key = `${prefix}/providers/${id}.json`; + const bytes = objects.get(key); + return [id, { key, sha256: createHash("sha256").update(bytes).digest("hex"), modelCount: Object.keys(JSON.parse(bytes)).length }]; + })); const publication = { schemaVersion: CATALOG_SCHEMA_VERSION, minimumDiffexVersion: MINIMUM_DIFFEX_VERSION, revision: bundle.revision, - sourceCommit: options.sourceCommit || gitSourceCommit(), + sourceCommit, + generatedAt: bundle.generatedAt, + providerIds: bundle.providerIds, providerCount: bundle.providerCount, modelCount: bundle.modelCount, + providers, }; - writeFileSync(join(inputDir, "publication.json"), `${JSON.stringify(publication, null, 2)}\n`); + const current = store.get(CATALOG_INDEX_KEY); + const index = current ? validateIndex(JSON.parse(current.bytes)) : undefined; + for (const [key, bytes] of objects) { + store.put(key, bytes); // Create-only; another publisher may already own it. + const stored = store.get(key); + if (!stored || !stored.bytes.equals(bytes)) throw new Error(`Revision verification failed: ${key}`); + } + const manifestKey = `${prefix}/publication.json`; + store.put(manifestKey, jsonBytes(publication)); + const storedManifest = store.get(manifestKey); + if (!storedManifest) throw new Error("Missing publication manifest after upload"); + const immutable = JSON.parse(storedManifest.bytes); + // Identical content retains the first successful generation timestamp and source. + if (!/^[a-f0-9]{40}$/.test(immutable.sourceCommit) || + typeof immutable.generatedAt !== "string" || !Number.isFinite(Date.parse(immutable.generatedAt)) || + !isDeepStrictEqual(immutable, { ...publication, sourceCommit: immutable.sourceCommit, generatedAt: immutable.generatedAt })) { + throw new Error("Immutable publication manifest differs"); + } + checkSource(sourceCommit); // Recheck after uploading, immediately before the CAS. + const next = { ...buildIndex(index, immutable), validationSourceCommit: sourceCommit }; + if (!store.put(CATALOG_INDEX_KEY, jsonBytes(next), current?.etag)) { + throw new Error("Catalog index changed concurrently; regenerate from current main and retry"); + } + const verified = store.get(CATALOG_INDEX_KEY); + if (!verified || !verified.bytes.equals(jsonBytes(next))) throw new Error("Catalog index changed after publication"); + return immutable; +} - console.log(JSON.stringify(publication, null, 2)); +function main() { + const options = parseArgs(process.argv.slice(2)); + const inputDir = resolve(options.input); + const bundle = validateBundle(inputDir); if (options.dryRun) { - console.log(`Validated model catalog at ${inputDir}; no objects uploaded.`); + console.log(`Validated ${bundle.modelCount} models at ${bundle.generatedAt}; revision ${bundle.revision}`); return; } - + const sourceCommit = options.sourceCommit || gitSourceCommit(); + if (options.requireMain && bundle.sourceCommit !== sourceCommit) throw new Error("Artifact is not bound to publisher SHA"); const temporaryDir = mkdtempSync(join(tmpdir(), "diffex-model-catalog-")); try { - const currentIndexPath = join(temporaryDir, "index-current.json"); - const hasCurrentIndex = downloadIndex(options.bucket, options.endpoint, currentIndexPath); - const currentIndex = hasCurrentIndex ? validateIndex(readJson(currentIndexPath)) : undefined; - const currentEntry = currentIndex?.catalogs.find( - (catalog) => catalog.minimumDiffexVersion === MINIMUM_DIFFEX_VERSION, - ); - if (currentIndex?.defaultRevision === bundle.revision && currentEntry?.revision === bundle.revision) { - console.log(`Model catalog ${bundle.revision} is already current; no objects uploaded.`); - return; - } - - const revisionPrefix = `${CATALOG_PREFIX}/revisions/${bundle.revision}`; - uploadJson(options.bucket, options.endpoint, bundle.modelsPath, `${revisionPrefix}/models.json`, IMMUTABLE_CACHE_CONTROL); - uploadJson( - options.bucket, - options.endpoint, - bundle.providerIndexPath, - `${revisionPrefix}/providers.json`, - IMMUTABLE_CACHE_CONTROL, - ); - for (const providerId of bundle.providerIds) { - uploadJson( - options.bucket, - options.endpoint, - join(bundle.providersDir, `${providerId}.json`), - `${revisionPrefix}/providers/${providerId}.json`, - IMMUTABLE_CACHE_CONTROL, - ); - } - - const nextIndex = buildIndex(currentIndex, publication); - const nextIndexPath = join(temporaryDir, "index-next.json"); - writeFileSync(nextIndexPath, `${JSON.stringify(nextIndex, null, 2)}\n`); - uploadJson(options.bucket, options.endpoint, nextIndexPath, CATALOG_INDEX_KEY, INDEX_CACHE_CONTROL); - console.log(`Published ${bundle.revision} to s3://${options.bucket}/${revisionPrefix}`); + const publication = publishCatalog(inputDir, createS3Store(options.bucket, options.endpoint, temporaryDir), sourceCommit, + options.requireMain ? assertCurrentSource : undefined); + console.log(`Published and validated ${publication.revision}; generated ${publication.generatedAt}`); } finally { rmSync(temporaryDir, { recursive: true, force: true }); } } -main().catch((error) => { - console.error(error instanceof Error ? error.message : error); - process.exitCode = 1; -}); +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + try { main(); } catch (error) { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + } +} diff --git a/scripts/publish-model-catalog.test.mjs b/scripts/publish-model-catalog.test.mjs new file mode 100644 index 000000000..05d5f92ed --- /dev/null +++ b/scripts/publish-model-catalog.test.mjs @@ -0,0 +1,127 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { publishCatalog, validateBundle } from "./publish-model-catalog.mjs"; +import { publicationAllowed } from "./model-publication-window.mjs"; + +const source = "a".repeat(40); +const indexKey = "models/v1/index.json"; + +export function catalogFixture(root) { + mkdirSync(join(root, "providers"), { recursive: true }); + const models = Object.fromEntries(["anthropic", "openai", "openrouter"].map((provider) => [provider, + Object.fromEntries(Array.from({ length: 170 }, (_, i) => [`model-${i}`, { + id: `model-${i}`, name: `Model ${i}`, api: "openai-completions", provider, + baseUrl: "https://example.test/v1", reasoning: false, input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow: 4096, maxTokens: 1000, + }]))])); + writeFileSync(join(root, "models.json"), JSON.stringify(models)); + writeFileSync(join(root, "providers.json"), JSON.stringify(Object.keys(models))); + writeFileSync(join(root, "generation.json"), JSON.stringify({ generatedAt: "2026-10-26T10:00:00.000Z", sourceCommit: source })); + for (const [id, shard] of Object.entries(models)) writeFileSync(join(root, "providers", `${id}.json`), JSON.stringify(shard)); + return models; +} + +function setup(t) { + const root = mkdtempSync(join(tmpdir(), "diffex-publish-test-")); + t.after(() => rmSync(root, { recursive: true, force: true })); + catalogFixture(root); + const objects = new Map(); + const store = { + get(key) { return objects.get(key) ?? null; }, + put(key, bytes, etag) { + const existing = objects.get(key); + if (etag ? existing?.etag !== etag : existing !== undefined) return false; + objects.set(key, { bytes: Buffer.from(bytes), etag: `"${createHash("sha256").update(bytes).digest("hex")}"` }); + return true; + }, + }; + return { root, objects, store }; +} + +test("publishes complete revisions and preserves immutable metadata on identical retries", (t) => { + const { root, store, objects } = setup(t); + const first = publishCatalog(root, store, source); + const immutable = new Map([...objects].filter(([key]) => key !== indexKey)); + const firstIndex = JSON.parse(objects.get(indexKey).bytes); + assert.equal(first.minimumDiffexVersion, "0.3.3"); + assert.equal(first.providerCount, 3); + assert.equal(first.modelCount, 510); + assert.ok(firstIndex.validatedAt); + writeFileSync(join(root, "generation.json"), JSON.stringify({ generatedAt: "2026-10-27T10:00:00.000Z", sourceCommit: "b".repeat(40) })); + assert.deepEqual(publishCatalog(root, store, "b".repeat(40)), first); + assert.equal(JSON.parse(objects.get(indexKey).bytes).validationSourceCommit, "b".repeat(40)); + assert.deepEqual(new Map([...objects].filter(([key]) => key !== indexKey)), immutable); +}); + +test("interrupted uploads leave the index untouched and can be resumed", (t) => { + const { root, store, objects } = setup(t); + const put = store.put; + let writes = 0; + store.put = (...args) => { if (++writes === 3) throw new Error("interrupted"); return put(...args); }; + assert.throws(() => publishCatalog(root, store, source), /interrupted/); + assert.equal(objects.has(indexKey), false); + store.put = put; + publishCatalog(root, store, source); + assert.ok(objects.has(indexKey)); +}); + +test("rejects conflicting revision bytes and conditional index conflicts", (t) => { + const { root, store, objects } = setup(t); + const revision = validateBundle(root).revision; + store.put(`models/v1/revisions/${revision}/models.json`, Buffer.from("{}")); + assert.throws(() => publishCatalog(root, store, source), /verification failed/); + assert.equal(objects.has(indexKey), false); + objects.clear(); + const put = store.put; + store.put = (key, ...args) => key === indexKey ? false : put(key, ...args); + assert.throws(() => publishCatalog(root, store, source), /changed concurrently/); + assert.equal(objects.has(indexKey), false); +}); + +test("rejects stale sources both before and after upload", (t) => { + const { root, store, objects } = setup(t); + assert.throws(() => publishCatalog(root, store, "b".repeat(40)), /source SHA/); + assert.throws(() => publishCatalog(root, store, source, () => { throw new Error("stale"); }), /stale/); + assert.equal(objects.size, 0); + let checks = 0; + assert.throws(() => publishCatalog(root, store, source, () => { if (++checks === 2) throw new Error("stale"); }), /stale/); + assert.equal(objects.has(indexKey), false); +}); + +test("rejects malformed fields, inconsistent shards, invalid timestamps and small catalogs", (t) => { + const { root } = setup(t); + const models = JSON.parse(readFileSync(join(root, "models.json"))); + models.openai["model-0"].contextWindow = -1; + writeFileSync(join(root, "models.json"), JSON.stringify(models)); + assert.throws(() => validateBundle(root), /shard does not match/); + writeFileSync(join(root, "providers/openai.json"), JSON.stringify(models.openai)); + assert.throws(() => validateBundle(root), /contextWindow/); + catalogFixture(root); + writeFileSync(join(root, "generation.json"), "{}"); + assert.throws(() => validateBundle(root), /timestamp/); + catalogFixture(root); + for (const provider of Object.keys(models)) { + models[provider] = { "model-1": models[provider]["model-1"] }; + writeFileSync(join(root, "providers", `${provider}.json`), JSON.stringify(models[provider])); + } + writeFileSync(join(root, "models.json"), JSON.stringify(models)); + assert.throws(() => validateBundle(root), /expected at least 500/); +}); + +test("publication windows preserve weekday Vienna hours across DST boundaries", () => { + for (const [date, schedule, ci] of [ + ["2026-03-27T09:17:00Z", true, true], ["2026-03-30T08:17:00Z", true, true], + ["2026-10-23T12:17:00Z", true, true], ["2026-10-26T13:17:00Z", true, true], + ["2026-10-26T10:17:00Z", false, true], ["2026-10-26T14:00:00Z", false, false], + ["2026-10-26T08:59:00Z", false, false], ["2026-10-25T09:17:00Z", false, false], + ]) { + assert.equal(publicationAllowed("schedule", false, new Date(date)), schedule, date); + assert.equal(publicationAllowed("workflow_run", false, new Date(date)), ci, date); + } + assert.equal(publicationAllowed("workflow_dispatch", true, new Date("2026-10-25")), true); + assert.equal(publicationAllowed("pull_request", true), false); +}); From 30353beeb977f956ed71f98c10e9a79232383986 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Mon, 26 Oct 2026 19:58:30 -0400 Subject: [PATCH 2/4] fix(coding-agent): isolate catalog caches - Match cache bodies, validators, and refresh timers to URL and client version - Validate downloads and preserve matching caches on failure - Use the Diffex catalog origin and retire catalog identity exceptions --- docs/identity-handoff.md | 6 +- identity-residual-allowlist.md | 5 +- packages/ai/src/models-store.ts | 4 + .../src/core/remote-catalog-provider.ts | 56 +++++----- .../test/remote-catalog-provider.test.ts | 102 +++++++++++++++++- scripts/runtime-identity-allowlist.json | 18 ---- 6 files changed, 131 insertions(+), 60 deletions(-) diff --git a/docs/identity-handoff.md b/docs/identity-handoff.md index 8415e181e..ed4e86558 100644 --- a/docs/identity-handoff.md +++ b/docs/identity-handoff.md @@ -14,13 +14,15 @@ No package, image, release asset, or source archive is published by this change. | Product author and website | DiffexAI, `https://diffex.ai/` | | Legal copyright owner | Luntraa, Inc. | | Release API | `https://api.diffex.ai/api/latest-version`, owned by `diffex-cloud` | -| Radius and catalog services | Approved external integrations recorded in the candidate handoff | +| Model catalog API | `https://api.diffex.ai/api/models/providers/{providerId}`, owned by `diffex-cloud` | +| Radius services | Approved external integrations recorded in the candidate handoff | | Security contact | Existing private access to a Diffex repository owner; no dedicated mailbox or public reporting channel claimed | | Issue authorization | Write/admin permission on the current Diffex repository | Diffex and Luntraa, Inc. are unrelated to Earendil. The former staff-team authorization, security mailbox reference, announcement, and product affiliation wording have been removed. -The owner explicitly retained OpenAI Codex and xAI OAuth in addition to the release audit's Radius, catalog, and provider-attribution integrations. +The owner explicitly retained OpenAI Codex and xAI OAuth in addition to the release audit's Radius and provider-attribution integrations. +The catalog authority is now Diffex; see [the catalog contract](model-catalog.md). The exact residual inventory is [identity-residual-allowlist.md](../identity-residual-allowlist.md). | Workspace | Final package | Publication | diff --git a/identity-residual-allowlist.md b/identity-residual-allowlist.md index 93429e4de..8bab5065a 100644 --- a/identity-residual-allowlist.md +++ b/identity-residual-allowlist.md @@ -25,11 +25,10 @@ Paths below are descriptive coordinates; the full source context is authoritativ | `packages/coding-agent/src/core/provider-attribution.ts:58` | `Pi` | approved integration | Provider attribution retained by the release audit; optional headers default off. | | `packages/coding-agent/src/core/provider-attribution.ts:64` | `pi-coding-agent` | approved integration | Provider attribution retained by the release audit; optional headers default off. | | `packages/coding-agent/src/core/provider-attribution.ts:80` | `pi` | approved integration | OpenCode session/client attribution retained independently of the optional attribution toggle. | -| `packages/coding-agent/src/core/remote-catalog-provider.ts:6` | `pi.dev` | approved integration | Pi-supported model catalogs retained by the release audit. | -| `packages/coding-agent/src/core/remote-catalog-provider.ts:44` | `pi.dev` | approved integration | Pi-supported model catalogs retained by the release audit. | The owner explicitly approved retention of OpenAI Codex and xAI OAuth during this implementation. -The release audit approves Radius gateway/OAuth, Pi-supported model catalogs, and provider attribution. +The release audit approves Radius gateway/OAuth and provider attribution. +Model catalogs use the Diffex-owned API described in [the catalog contract](docs/model-catalog.md); the former catalog exceptions are removed. These entries permit those integrations only; they do not authorize install reporting, update authority, arbitrary traffic to the same host, or new dependencies. The source check covers tracked and unignored package source, examples, configuration, workflow files, operational extensions, dependency metadata, and root identity documents. Tests and their fixtures describe rejected historical inputs and are not shipped runtime configuration. diff --git a/packages/ai/src/models-store.ts b/packages/ai/src/models-store.ts index 96dd8b3b0..674184d1c 100644 --- a/packages/ai/src/models-store.ts +++ b/packages/ai/src/models-store.ts @@ -2,6 +2,10 @@ import type { Api, Model } from "./types.ts"; export interface ModelsStoreEntry { models: readonly Model[]; + /** Exact provider catalog URL that supplied this entry. */ + catalogUrl?: string; + /** Diffex client version used to select its compatibility tier. */ + catalogClientVersion?: string; /** Unix timestamp from the remote catalog's Last-Modified header. */ lastModified?: number; /** Unix timestamp of the last completed remote check. */ diff --git a/packages/coding-agent/src/core/remote-catalog-provider.ts b/packages/coding-agent/src/core/remote-catalog-provider.ts index 7bc33d71c..ef0f43737 100644 --- a/packages/coding-agent/src/core/remote-catalog-provider.ts +++ b/packages/coding-agent/src/core/remote-catalog-provider.ts @@ -1,9 +1,9 @@ -import type { Api, Model, ModelsStoreEntry, Provider } from "@diffexai/diffex-ai"; +import { type Api, type Model, type ModelsStoreEntry, type Provider, parseModelCatalog } from "@diffexai/diffex-ai"; import { VERSION } from "../config.ts"; import { getDiffexUserAgent } from "../utils/diffex-user-agent.ts"; import { fetchWithRetry } from "../utils/management-http.ts"; -const DEFAULT_CATALOG_BASE_URL = "https://pi.dev"; +const DEFAULT_CATALOG_BASE_URL = "https://api.diffex.ai"; export const REMOTE_CATALOG_REFRESH_INTERVAL_MS = 4 * 60 * 60 * 1000; function mergeModels(baseline: readonly Model[], dynamic: readonly Model[]): Model[] { @@ -16,20 +16,6 @@ function mergeModels(baseline: readonly Model[], dynamic: readonly Model[] { - const entries = Array.isArray(value) - ? value - : typeof value === "object" && value !== null && "models" in value && Array.isArray(value.models) - ? value.models - : typeof value === "object" && value !== null - ? Object.values(value) - : undefined; - if (!entries) throw new Error(`Invalid model catalog for provider "${providerId}"`); - return entries - .filter((entry): entry is Model => typeof entry === "object" && entry !== null && "id" in entry) - .map((model) => ({ ...model, provider: providerId })); -} - function remoteModels( entry: ModelsStoreEntry | undefined, localGeneratedAt: number | undefined, @@ -41,19 +27,24 @@ function remoteModels( return entry.models; } -/** Add a persisted pi.dev catalog overlay to a static built-in provider. */ +/** Add a persisted remote catalog overlay to a static built-in provider. */ export function withRemoteCatalog( provider: Provider, catalogBaseUrl: string = DEFAULT_CATALOG_BASE_URL, localGeneratedAt?: number, ): Provider { let dynamicModels: readonly Model[] = []; + const catalogUrl = new URL(`/api/models/providers/${encodeURIComponent(provider.id)}`, catalogBaseUrl).href; + const provenance = { catalogUrl, catalogClientVersion: VERSION }; return { ...provider, getModels: () => mergeModels(provider.getModels(), dynamicModels), refreshModels: async (context) => { - const stored = context.stored; + const stored = + context.stored?.catalogUrl === catalogUrl && context.stored.catalogClientVersion === VERSION + ? context.stored + : undefined; const restored = remoteModels(stored, localGeneratedAt).filter((model) => model.provider === provider.id); if ( !(await context.publish({ @@ -77,44 +68,45 @@ export function withRemoteCatalog( // Only revalidate when a cached body backs the validator, so a 304 can never // leave the overlay empty. const validator = stored?.models.length ? stored.etag : undefined; - const url = new URL(`/api/models/providers/${encodeURIComponent(provider.id)}`, catalogBaseUrl); - const response = await fetchWithRetry(url, { + const response = await fetchWithRetry(catalogUrl, { headers: { accept: "application/json", "User-Agent": getDiffexUserAgent(VERSION), ...(validator ? { "if-none-match": validator } : {}), }, - signal: context.signal, + signal: AbortSignal.any([context.signal, AbortSignal.timeout(15_000)]), }); if (context.signal.aborted) return; const checkedAt = Date.now(); // Unchanged: dynamicModels already holds the stored overlay, so only the // freshness window moves. - if (response.status === 304 && stored) { + if (response.status === 304 && stored && validator) { await context.publish({ persist: { ...stored, checkedAt } }); return; } if (response.status === 404 || response.status === 501) { - await context.publish({ - persist: { - ...(stored ?? { models: [] }), - checkedAt, - lastModified: 0, - etag: undefined, - }, - }); + if (!stored) + await context.publish({ + persist: { + ...provenance, + models: [], + checkedAt, + lastModified: 0, + etag: undefined, + }, + }); return; } if (!response.ok) { // Transient failure: the cached body and its validator stay valid, so keep the // etag and let the next refresh revalidate instead of downloading the catalog. - await context.publish({ persist: { ...(stored ?? { models: [] }), checkedAt } }); throw new Error(`Model catalog request failed for ${provider.id}: ${response.status}`); } - const refreshed = parseCatalog(provider.id, await response.json()); + const refreshed = parseModelCatalog(provider.id, await response.json()); const lastModified = Date.parse(response.headers.get("last-modified") ?? ""); if (context.signal.aborted) return; const entry = { + ...provenance, models: refreshed, checkedAt, lastModified: Number.isNaN(lastModified) ? 0 : lastModified, diff --git a/packages/coding-agent/test/remote-catalog-provider.test.ts b/packages/coding-agent/test/remote-catalog-provider.test.ts index 5ee8e91f3..83e9174a3 100644 --- a/packages/coding-agent/test/remote-catalog-provider.test.ts +++ b/packages/coding-agent/test/remote-catalog-provider.test.ts @@ -1,14 +1,19 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { createModels, createProvider, InMemoryModelsStore, type Model, type ModelsPublication, + type ModelsStore, type Provider, type RefreshModelsContext, } from "@diffexai/diffex-ai"; import { afterEach, describe, expect, it, vi } from "vitest"; import { VERSION } from "../src/config.ts"; +import { FileModelsStore } from "../src/core/models-store.ts"; import { withRemoteCatalog } from "../src/core/remote-catalog-provider.ts"; const neverAbortedSignal = new AbortController().signal; @@ -43,14 +48,14 @@ function testProvider(localGeneratedAt?: number) { }, }, }), - "https://pi.dev", + "https://api.diffex.ai", localGeneratedAt, ); } async function refreshProvider( provider: Provider, - store: InMemoryModelsStore, + store: ModelsStore, overrides: Partial> = {}, ): Promise { const publish = async (publication: ModelsPublication): Promise => { @@ -143,7 +148,7 @@ describe("remote catalog provider", () => { expect(stored?.checkedAt).toBeGreaterThanOrEqual(checkedAt ?? 0); }); - it("drops a stale etag when the overlay becomes unavailable", async () => { + it("preserves a valid matching cache when the overlay becomes unavailable", async () => { const responses = [ new Response(JSON.stringify({ dynamic: model("dynamic") }), { headers: { "content-type": "application/json", etag: '"catalog-1"' }, @@ -157,7 +162,8 @@ describe("remote catalog provider", () => { await refreshProvider(provider, store); await refreshProvider(provider, store, { force: true }); - expect((await store.read(provider.id))?.etag).toBeUndefined(); + expect((await store.read(provider.id))?.etag).toBe('"catalog-1"'); + expect(provider.getModels().map((entry) => entry.id)).toEqual(["static", "dynamic"]); }); it("keeps the etag and overlay after a transient failure", async () => { @@ -229,7 +235,7 @@ describe("remote catalog provider", () => { expect((await store.read(provider.id))?.models.map((entry) => entry.id)).toEqual(["newer"]); }); - it("treats unimplemented pi.dev catalog routes as an unavailable overlay", async () => { + it("treats incompatible catalog routes as an unavailable overlay", async () => { vi.spyOn(globalThis, "fetch").mockResolvedValue(new Response("not implemented", { status: 501 })); const provider = testProvider(); const store = new InMemoryModelsStore(); @@ -238,4 +244,90 @@ describe("remote catalog provider", () => { expect(provider.getModels().map((entry) => entry.id)).toEqual(["static"]); expect(await store.read(provider.id)).toMatchObject({ models: [], checkedAt: expect.any(Number) }); }); + + it.each([ + {}, + { catalogUrl: "https://old.example/api/models/providers/test-provider", catalogClientVersion: VERSION }, + { catalogUrl: "https://api.diffex.ai/api/models/providers/test-provider", catalogClientVersion: "0.0.1" }, + ])("ignores foreign cache models, validators and timers: %j", async (provenance) => { + const store = new InMemoryModelsStore(); + await store.write("test-provider", { + models: [model("foreign")], + checkedAt: Date.now(), + lastModified: Date.now(), + etag: '"foreign"', + ...provenance, + }); + const fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue(Response.json({ fresh: model("fresh") })); + const provider = testProvider(); + await refreshProvider(provider, store, { allowNetwork: false }); + expect(provider.getModels().map((entry) => entry.id)).toEqual(["static"]); + expect(fetchSpy).not.toHaveBeenCalled(); + await refreshProvider(provider, store); + expect(fetchSpy).toHaveBeenCalledOnce(); + expect(fetchSpy.mock.calls[0]?.[1]?.headers).not.toHaveProperty("if-none-match"); + expect(provider.getModels().map((entry) => entry.id)).toEqual(["static", "fresh"]); + expect(await store.read(provider.id)).toMatchObject({ + catalogUrl: "https://api.diffex.ai/api/models/providers/test-provider", + catalogClientVersion: VERSION, + }); + }); + + it.each(["json", "fields", "provider", "key", "network", "timeout", "cancel", "http"])( + "preserves the matching cache after %s failure", + async (failure) => { + const store = new InMemoryModelsStore(); + const fetchSpy = vi + .spyOn(globalThis, "fetch") + .mockResolvedValue(Response.json({ valid: model("valid") }, { headers: { etag: '"valid"' } })); + const provider = testProvider(); + await refreshProvider(provider, store); + const before = await store.read(provider.id); + const controller = new AbortController(); + fetchSpy.mockImplementation(async () => { + if (failure === "network") throw new TypeError("network unavailable"); + if (failure === "timeout") throw new DOMException("Timed out", "TimeoutError"); + if (failure === "cancel") controller.abort(); + if (failure === "http") return new Response(null, { status: 503 }); + if (failure === "json") return new Response("{broken"); + return Response.json({ + invalid: { + ...model(failure === "key" ? "wrong" : "invalid"), + ...(failure === "fields" ? { contextWindow: -1 } : {}), + ...(failure === "provider" ? { provider: "other" } : {}), + }, + }); + }); + const refresh = refreshProvider(provider, store, { force: true, signal: controller.signal }); + if (failure === "cancel") await refresh; + else await expect(refresh).rejects.toThrow(); + expect(await store.read(provider.id)).toEqual(before); + expect(provider.getModels().map((entry) => entry.id)).toEqual(["static", "valid"]); + }, + ); + + it("persists provenance across restart and restores matching models offline", async () => { + const root = mkdtempSync(join(tmpdir(), "diffex-catalog-restart-")); + try { + const path = join(root, "models-store.json"); + const fetchSpy = vi + .spyOn(globalThis, "fetch") + .mockResolvedValue(Response.json({ dynamic: model("dynamic") }, { headers: { etag: '"valid"' } })); + await refreshProvider(testProvider(), new FileModelsStore(path)); + const restarted = testProvider(); + const store = new FileModelsStore(path); + await refreshProvider(restarted, store, { allowNetwork: false }); + await refreshProvider(restarted, store); + expect(fetchSpy).toHaveBeenCalledOnce(); + expect(restarted.getModels().map((entry) => entry.id)).toEqual(["static", "dynamic"]); + const entry = await store.read(restarted.id); + if (!entry) throw new Error("Missing persisted cache"); + await store.write(restarted.id, { ...entry, checkedAt: Date.now() - 4 * 60 * 60 * 1000 }); + fetchSpy.mockResolvedValue(new Response(null, { status: 304 })); + await refreshProvider(restarted, store); + expect(fetchSpy).toHaveBeenCalledTimes(2); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); diff --git a/scripts/runtime-identity-allowlist.json b/scripts/runtime-identity-allowlist.json index 1034d0e08..1f941d1e2 100644 --- a/scripts/runtime-identity-allowlist.json +++ b/scripts/runtime-identity-allowlist.json @@ -108,23 +108,5 @@ ], "category": "approved integration", "reason": "OpenCode session/client attribution retained by the release audit independently of the optional attribution toggle." - }, - { - "path": "packages/coding-agent/src/core/remote-catalog-provider.ts", - "context": "const DEFAULT_CATALOG_BASE_URL = \"https://pi.dev\";", - "identities": [ - "pi.dev" - ], - "category": "approved integration", - "reason": "Pi-supported model catalogs retained by the release audit." - }, - { - "path": "packages/coding-agent/src/core/remote-catalog-provider.ts", - "context": "/** Add a persisted pi.dev catalog overlay to a static built-in provider. */", - "identities": [ - "pi.dev" - ], - "category": "approved integration", - "reason": "Pi-supported model catalogs retained by the release audit." } ] From ea34b434181afe1c79e21d46692a44b47164492d Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Mon, 26 Oct 2026 20:07:03 -0400 Subject: [PATCH 3/4] feat(ai): verify and monitor catalog delivery - Exercise real publication, Worker serving, CLI refresh, and offline recovery with isolated fixtures. - Detect missed generation and production publication independently of content age. - Bind generation and upload to the resolved source revision. --- .github/workflows/monitor-model-catalog.yml | 30 +++++ .github/workflows/publish-model-catalog.yml | 3 +- docs/model-catalog.md | 133 ++++++++++++++++++++ scripts/catalog-local-e2e.mjs | 121 ++++++++++++++++++ scripts/catalog-staging-rehearsal.mjs | 107 ++++++++++++++++ scripts/fixtures/model-catalog.mjs | 18 +++ scripts/model-publication-security.test.mjs | 1 + scripts/monitor-model-publication.mjs | 36 ++++++ scripts/monitor-model-publication.test.mjs | 20 +++ scripts/publish-model-catalog.test.mjs | 33 ++--- 10 files changed, 485 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/monitor-model-catalog.yml create mode 100644 docs/model-catalog.md create mode 100644 scripts/catalog-local-e2e.mjs create mode 100644 scripts/catalog-staging-rehearsal.mjs create mode 100644 scripts/fixtures/model-catalog.mjs create mode 100644 scripts/monitor-model-publication.mjs create mode 100644 scripts/monitor-model-publication.test.mjs diff --git a/.github/workflows/monitor-model-catalog.yml b/.github/workflows/monitor-model-catalog.yml new file mode 100644 index 000000000..cb6206a86 --- /dev/null +++ b/.github/workflows/monitor-model-catalog.yml @@ -0,0 +1,30 @@ +name: Monitor Model Publication + +on: + schedule: + - cron: '37 * * * *' + workflow_dispatch: + +permissions: + contents: read + actions: read + +concurrency: + group: monitor-model-catalog + cancel-in-progress: false + +jobs: + monitor: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: '22' + - name: Detect missing validation and publication runs + env: + GH_TOKEN: ${{ github.token }} + run: node scripts/monitor-model-publication.mjs diff --git a/.github/workflows/publish-model-catalog.yml b/.github/workflows/publish-model-catalog.yml index 382c10958..758114ae6 100644 --- a/.github/workflows/publish-model-catalog.yml +++ b/.github/workflows/publish-model-catalog.yml @@ -1,4 +1,5 @@ name: Publish Model Catalog +run-name: Model catalog (${{ inputs.environment == 'staging' && 'staging' || 'production' }}) on: workflow_run: @@ -47,7 +48,7 @@ jobs: outputs: source_sha: ${{ steps.source.outputs.sha }} env: - SOURCE_REF: ${{ needs.generate.outputs.source_sha }} + SOURCE_REF: ${{ github.event.workflow_run.head_sha || inputs.source_ref || github.sha }} steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/docs/model-catalog.md b/docs/model-catalog.md new file mode 100644 index 000000000..f4cc15322 --- /dev/null +++ b/docs/model-catalog.md @@ -0,0 +1,133 @@ +# Diffex model catalog contract + +The CLI repository owns generation, validation, publication, and persisted client overlays. +`diffex-cloud/services/public-api` owns anonymous serving through `https://api.diffex.ai/api/models/providers/{providerId}`. +No database is involved. +Upstream metadata sources and generator overrides remain unchanged. + +## Storage and publication + +Account `6b24afa849f8254ab5e6e2d1956239cc` hosts private buckets `diffex-artifacts-staging` and `diffex-artifacts`. +The Worker binds the matching bucket as `MODEL_CATALOGS`. +GitHub environments `diffex-model-upload-staging` and `diffex-model-upload-production` each contain bucket-scoped `DIFFEX_ARTIFACTS_R2_ACCESS_KEY_ID` and `DIFFEX_ARTIFACTS_R2_SECRET_ACCESS_KEY` secrets and permit only `main`. +Neither bucket needs public access. + +The strict JSON export includes `models.json`, sorted `providers.json`, `providers/{providerId}.json`, and `generation.json` with `generatedAt` and the workflow's resolved `sourceCommit`. +All model fields pass the bundled-model validator before export. +Publication additionally requires matching provider shards, `anthropic`, `openai`, `openrouter`, and at least 500 models. + +The revision is `sha256-` followed by the SHA-256 of the exact `models.json` bytes. +Immutable objects live under `models/v1/revisions/{revision}/`. +That directory contains the models, provider list, shards, and an immutable `publication.json`: + +```json +{ + "schemaVersion": 1, + "minimumDiffexVersion": "0.3.3", + "revision": "sha256-<64 lowercase hexadecimal characters>", + "sourceCommit": "<40 lowercase hexadecimal characters>", + "generatedAt": "2026-10-26T10:00:00.000Z", + "providerIds": ["anthropic", "openai", "openrouter"], + "providerCount": 3, + "modelCount": 510, + "providers": { + "openai": { + "key": "models/v1/revisions/sha256-/providers/openai.json", + "sha256": "", + "modelCount": 170 + } + } +} +``` + +The example abbreviates the provider map; every listed provider must have an entry, and counts must sum exactly. +Identical-content retries preserve the first manifest's source and timestamp. +The initial compatibility floor is `0.3.3`; raise it only with metadata requiring newer client behavior. +Raising a floor requires a new content revision, since a published manifest cannot be edited. + +The mutable `models/v1/index.json` contains `schemaVersion: 1`, `defaultRevision`, `catalogs`, `validatedAt`, and `validationSourceCommit`. +Each tier has `minimumDiffexVersion`, `revision`, `sourceCommit`, `generatedAt`, `providerCount`, and `modelCount`. +Publication replaces only the matching compatibility floor and retains other tiers. +Every successful publication, including unchanged content, advances validation evidence independently of generation age. + +All revision objects use S3 `PutObject` with `If-None-Match: *` and are downloaded and verified before index advancement. +The publisher verifies an existing immutable manifest instead of replacing it. +The index is updated with `If-Match` against the ETag read before uploads, or `If-None-Match: *` for the initial write. +Conflicts fail the run; regenerate from current main and retry. +Interrupted uploads leave the previously selected revision intact. +R2's [consistency guarantees](https://developers.cloudflare.com/r2/reference/consistency/) allow verification directly through private storage without public caches. + +## Serving and cache provenance + +The only new production route is `https://api.diffex.ai/api/models/providers/*`. +Release discovery stays on its existing Worker route; auth, webhooks, `/v1/*`, and production health routes remain on Render. +Provider IDs must match `[a-z0-9][a-z0-9-]{0,99}` with no extra path segments or encoded separators. +Storage keys come exclusively from the selected, validated publication manifest. + +Anonymous `GET` and `HEAD` select the highest compatibility floor supported by the `diffex/` user-agent token. +A prerelease does not satisfy the same stable-version floor. +Unrecognized or absent Diffex versions select `defaultRevision`. +No compatible tier returns `501`; an unknown provider returns `404`; unsupported methods return `405` with `Allow: GET, HEAD`. +Missing, malformed, oversized, corrupt, or inaccessible storage returns `503` and never exposes internal storage errors. + +Successful responses retain the provider's model-ID-keyed JSON object and return `Content-Type: application/json; charset=utf-8`, `Cache-Control: public, no-cache`, and `Vary: User-Agent`. +The quoted ETag is `"{revision}:{providerId}"`. +Cloudflare may add the weak `W/` prefix when compressing a response; clients retain the received validator and probes compare its opaque value. +`Last-Modified` is the immutable generation timestamp rounded to HTTP-date precision. +`If-None-Match` takes precedence over `If-Modified-Since`, supports weak tags and lists for GET/HEAD revalidation, and returns bodyless `304` on a match. +Every HEAD response is bodyless. +The mutable index is read directly from R2 on each request. + +Client cache entries record the exact provider URL as `catalogUrl` and the selecting CLI version as `catalogClientVersion`. +Only entries matching both values may restore models, suppress refreshes for four hours, or send a cached ETag. +Legacy or mismatched entries fall back to bundled models and refresh unconditionally when network access is allowed. +Newer bundled metadata retains precedence. +Successful bodies are fully validated before persistence; failures and cancellation preserve the last valid matching cache. +`diffex update --models` forces refresh, including rollback discovery. + +## Validation and rollout + +Run focused publisher and client tests and `npm run check` in the CLI repository. +Run `npm run check` in `diffex-cloud/services/public-api`. +The opt-in local integration requires the companion checkout, AWS CLI with conditional-put support, and tmux: + +```sh +DIFFEX_CLOUD_DIR=/path/to/diffex-cloud node scripts/catalog-local-e2e.mjs +``` + +It exercises the actual publisher through local R2's S3 API, the Worker runtime, CLI update and persisted cache, the interactive selector, and offline startup using an isolated agent directory. +It blocks unrelated outbound requests and performs no model inference. + +Coordinate repository changes before enabling scheduled publication. +Rehearse against staging with staging-only credentials and verify fixture cleanup. +From the CLI repository, set `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_DEFAULT_REGION=auto` in an approved environment, then run: + +```sh +node scripts/catalog-staging-rehearsal.mjs --staging +``` + +The rehearsal requires the catalog Worker to be deployed to staging first. +It creates an isolated synthetic revision, checks interrupted uploads and conditional conflicts, exercises the deployed endpoint and isolated CLI, restores the prior index conditionally, and deletes and verifies only its fixture objects. +It refuses to overwrite a concurrent index change and retains recovery evidence if cleanup fails. +Seed production with a validated real catalog, deploy the narrow Worker route, and verify real `200`/`304` responses and unchanged release/Render routes before shipping the client default. +Do not publish synthetic fixture models to production. + +## Monitoring and rollback + +Weekday publication candidates preserve 10:17, 12:17, and 14:17 Europe/Vienna and the existing successful-main-CI window of 10:00 through 14:59. +Manual publication bypasses the window; PRs only validate. +Generation and publishing code use one resolved SHA; the publisher checks current main before upload and immediately before index advancement. +Uploads serialize per environment without cancelling active uploads. + +`monitor-model-catalog.yml` checks successful generation and actual production upload steps separately, with one hour of scheduling grace and weekend/DST handling. +`diffex-cloud`'s existing monitor probes availability, HEAD, ETag/date validators, release discovery, and Render routing. +Old generation timestamps are healthy when content is unchanged. +Both [GitHub scheduled workflows](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#schedule) can be delayed or dropped, so configure an independent external heartbeat for the monitor and confirm failure notifications after merge. +Repository changes alone do not prove scheduled execution or notification delivery. + +Retain immutable revisions for rollback. +Download and preserve the current index bytes and ETag, verify every object referenced by the desired historical index, then replace the index with `If-Match` against the captured current ETag. +Do not force an index update after a conditional conflict. +Alternatively, roll back to a verified Worker deployment while preserving catalog bindings and narrow routes. +Worker rollback does not roll back R2 objects. +Clients discover a catalog rollback on their next refresh or through `diffex update --models`. diff --git a/scripts/catalog-local-e2e.mjs b/scripts/catalog-local-e2e.mjs new file mode 100644 index 000000000..92dd36b70 --- /dev/null +++ b/scripts/catalog-local-e2e.mjs @@ -0,0 +1,121 @@ +#!/usr/bin/env node +// Explicit opt-in integration with the companion checkout and an installed AWS CLI. +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { buildSync } from "esbuild"; +import { catalogFixture } from "./fixtures/model-catalog.mjs"; + +const repo = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const cloud = process.env.DIFFEX_CLOUD_DIR; +if (!cloud) throw new Error("Set DIFFEX_CLOUD_DIR to the companion diffex-cloud checkout"); +const requireCloud = createRequire(join(resolve(cloud), "services/public-api/package.json")); +const { Miniflare, convertV4MiniflareOptions } = requireCloud("miniflare"); +const root = mkdtempSync(join(tmpdir(), "diffex-catalog-e2e-")); +const env = { ...process.env, AWS_ACCESS_KEY_ID: "catalog-local", AWS_SECRET_ACCESS_KEY: "catalog-local-secret", AWS_DEFAULT_REGION: "auto", AWS_EC2_METADATA_DISABLED: "true" }; + +async function run(command, args, extra = {}) { + return new Promise((resolveResult, reject) => { + if (command === "tmux") args = ["-L", `diffex-catalog-${process.pid}`, ...args]; + const child = spawn(command, args, { cwd: repo, env: { ...env, ...extra }, stdio: ["ignore", "pipe", "pipe"] }); + let output = ""; + child.stdout.on("data", (data) => { output += data; }); + child.stderr.on("data", (data) => { output += data; }); + child.on("error", reject); + child.on("exit", (code) => code === 0 ? resolveResult(output) : reject(new Error(`${command} exited ${code}: ${output}`))); + }); +} + +const bundle = buildSync({ entryPoints: [join(cloud, "services/public-api/src/index.ts")], bundle: true, format: "esm", write: false, platform: "browser", target: "es2022" }).outputFiles[0].text; +const mf = new Miniflare(convertV4MiniflareOptions({ modules: true, script: bundle, compatibilityDate: "2026-09-10", host: "127.0.0.1", port: 0, + r2Buckets: { MODEL_CATALOGS: { id: "catalog-fixture", s3Credentials: { accessKeyId: env.AWS_ACCESS_KEY_ID, secretAccessKey: env.AWS_SECRET_ACCESS_KEY } }, RELEASES: "release-fixture" }, + bindings: { RELEASE_SERVING_ENABLED: "true" }, ratelimits: { RELEASE_RATE_LIMITER: { namespace_id: "1", simple: { limit: 600, period: 60 } } }, +})); +try { + const origin = (await mf.ready).origin; + const input = join(root, "catalog"); + catalogFixture(input); + writeFileSync(join(input, "generation.json"), JSON.stringify({ generatedAt: new Date(Date.now() + 60_000).toISOString() })); + const publish = ["scripts/publish-model-catalog.mjs", "--input", input, "--bucket", "catalog-fixture", "--endpoint", `${origin}/cdn-cgi/local/r2/s3`, "--source-commit", "a".repeat(40)]; + console.log(await run(process.execPath, publish)); + const first = await fetch(`${origin}/api/models/providers/openai`, { headers: { "User-Agent": "diffex/0.3.3" } }); + assert.equal(first.status, 200); + const etag = first.headers.get("etag"); + const modified = first.headers.get("last-modified"); + await first.body.cancel(); + console.log(await run(process.execPath, publish)); + const unchanged = await fetch(`${origin}/api/models/providers/openai`, { headers: { "If-None-Match": etag } }); + assert.equal(unchanged.status, 304); + assert.equal(unchanged.headers.get("last-modified"), modified); + const agentDir = join(root, "agent"); + mkdirSync(agentDir); + writeFileSync(join(agentDir, "auth.json"), JSON.stringify({ openai: { type: "api_key", key: "local-fixture-never-used" } })); + const preload = join(root, "route-catalog.mjs"); + writeFileSync(preload, `let original = globalThis.fetch; +const routed = (input, init) => { + const url = new URL(typeof input === "string" || input instanceof URL ? input : input.url); + if (url.origin === "https://api.diffex.ai" && url.pathname.startsWith("/api/models/providers/")) { + return original(${JSON.stringify(origin)} + url.pathname, init); + } + throw new Error("Unexpected outbound request in isolated catalog rehearsal"); +}; +Object.defineProperty(globalThis, "fetch", { configurable: true, get: () => routed, set: (value) => { original = value; } }); +`); + const cliEnv = { DIFFEX_CODING_AGENT_DIR: agentDir, NODE_OPTIONS: `--import=${preload}` }; + const cli = join(repo, "diffex-test.sh"); + console.log(await run(cli, ["--no-env", "update", "--models"], cliEnv)); + const cache = JSON.parse(readFileSync(join(agentDir, "models-store.json"), "utf8")); + assert.equal(cache.openai.models.length, 170); + assert.equal(cache.openai.catalogUrl, "https://api.diffex.ai/api/models/providers/openai"); + assert.equal(cache.openai.etag, etag); + console.log(await run(cli, ["--no-env", "update", "--models"], cliEnv)); + assert.equal(JSON.parse(readFileSync(join(agentDir, "models-store.json"), "utf8")).openai.etag, etag); + // Use tmux to exercise the real interactive model selector without inference. + const session = `diffex-catalog-${process.pid}`; + const tmuxConfig = join(root, "tmux.conf"); + writeFileSync(tmuxConfig, "set -g extended-keys on\nset -g extended-keys-format csi-u\n"); + try { + await run("tmux", ["-f", tmuxConfig, "new-session", "-d", "-s", session, "-x", "100", "-y", "30", "-c", root, "env", `DIFFEX_CODING_AGENT_DIR=${agentDir}`, `NODE_OPTIONS=--import=${preload}`, cli, "--no-env", "--provider", "openai", "--model", "model-0", "--no-session", "--no-extensions", "--no-skills", "--no-prompt-templates"]); + let snapshot = ""; + for (let i = 0; i < 30; i++) { + snapshot = await run("tmux", ["capture-pane", "-t", session, "-p"]); + if (snapshot.includes("model-0") && snapshot.includes("Ask Diffex")) break; + await new Promise((r) => setTimeout(r, 200)); + } + await run("tmux", ["send-keys", "-t", session, "/scoped-models", "Enter"]); + for (let i = 0; i < 30; i++) { + snapshot = await run("tmux", ["capture-pane", "-t", session, "-p"]); + if (snapshot.includes("model-0") && snapshot.includes("Model catalogs refreshed")) break; + await new Promise((r) => setTimeout(r, 200)); + } + assert.match(snapshot, /model-0/); + assert.match(snapshot, /Model catalogs refreshed/); + assert.doesNotMatch(snapshot, /Connection error/); + await run("tmux", ["send-keys", "-t", session, "model-0"]); + for (let i = 0; i < 30; i++) { + snapshot = await run("tmux", ["capture-pane", "-t", session, "-p"]); + if (snapshot.includes("model-0 [openai]")) break; + await new Promise((r) => setTimeout(r, 200)); + } + assert.match(snapshot, /model-0 \[openai\]/); + console.log(snapshot); + await mf.dispose(); + const restarted = await run(cli, ["--no-env", "--list-models", "model-0"], { ...cliEnv, DIFFEX_OFFLINE: "1" }); + assert.match(restarted, /model-0/); + await run("tmux", ["send-keys", "-t", session, "Escape"]); + await run("tmux", ["send-keys", "-t", session, "/scoped-models", "Enter"]); + await new Promise((r) => setTimeout(r, 500)); + const offline = await run("tmux", ["capture-pane", "-t", session, "-p"]); + assert.match(offline, /model-0/); + } finally { + await run("tmux", ["kill-session", "-t", session]); + } + console.log("Local publisher -> R2 -> Worker -> CLI -> interactive selector passed; offline cache preserved."); +} finally { + await mf.dispose(); + rmSync(root, { recursive: true, force: true }); +} diff --git a/scripts/catalog-staging-rehearsal.mjs b/scripts/catalog-staging-rehearsal.mjs new file mode 100644 index 000000000..bc52f1693 --- /dev/null +++ b/scripts/catalog-staging-rehearsal.mjs @@ -0,0 +1,107 @@ +#!/usr/bin/env node +import assert from "node:assert/strict"; +import { randomUUID } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { catalogFixture } from "./fixtures/model-catalog.mjs"; +import { createS3Store, publishCatalog, validateBundle } from "./publish-model-catalog.mjs"; + +if (process.argv.length !== 3 || process.argv[2] !== "--staging") throw new Error("Explicit --staging opt-in required"); +const bucket = "diffex-artifacts-staging"; +const endpoint = "https://6b24afa849f8254ab5e6e2d1956239cc.r2.cloudflarestorage.com"; +const origin = "https://diffex-public-api-staging.yibai14651.workers.dev"; +const indexKey = "models/v1/index.json"; +const root = mkdtempSync(join(tmpdir(), "diffex-catalog-staging-")); +const source = execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim(); +const store = createS3Store(bucket, endpoint, root); +const previous = store.get(indexKey); +if (previous) writeFileSync(join(root, "index-before.json"), previous.bytes); +let fixtureRevision; +let writtenIndex; +let fixtureKeys = []; +try { + const input = join(root, "catalog"); + const models = catalogFixture(input, source); + models.openai["model-0"].name = `Staging rehearsal ${randomUUID()}`; + writeFileSync(join(input, "models.json"), JSON.stringify(models)); + writeFileSync(join(input, "providers/openai.json"), JSON.stringify(models.openai)); + writeFileSync(join(input, "generation.json"), JSON.stringify({ generatedAt: new Date().toISOString(), sourceCommit: source })); + fixtureRevision = validateBundle(input).revision; + const prefix = `models/v1/revisions/${fixtureRevision}`; + fixtureKeys = ["models.json", "providers.json", "publication.json", ...Object.keys(models).map((id) => `providers/${id}.json`)].map((suffix) => `${prefix}/${suffix}`); + for (const key of fixtureKeys) assert.equal(store.get(key), null, "Fixture must not already exist"); + let puts = 0; + assert.throws(() => publishCatalog(input, { ...store, put: (...args) => { + if (++puts === 3) throw new Error("Intentional interrupted fixture upload"); + return store.put(...args); + } }, source), /Intentional interrupted/); + assert.deepEqual(store.get(indexKey)?.bytes, previous?.bytes, "Interrupted upload changed index"); + const publication = publishCatalog(input, store, source); + writtenIndex = store.get(indexKey); + assert.equal(store.put(indexKey, Buffer.from("{}"), '"incorrect-etag"'), false, "R2 must reject stale conditional index writes"); + assert.equal(store.put(`${prefix}/publication.json`, Buffer.from("{}")), false, "R2 must reject immutable overwrites"); + const url = `${origin}/api/models/providers/openai`; + // Synchronous S3 checks can outlast an idle HTTP connection; do not reuse it across those checks. + const first = await fetch(url, { headers: { "User-Agent": "diffex/0.3.3", Connection: "close" }, signal: AbortSignal.timeout(10000) }); + assert.equal(first.status, 200); + assert.deepEqual(await first.json(), models.openai); + assert.equal(first.headers.get("etag")?.replace(/^W\//, ""), `"${fixtureRevision}:openai"`); + assert.equal(first.headers.get("last-modified"), new Date(publication.generatedAt).toUTCString()); + assert.deepEqual(publishCatalog(input, store, source), publication); + writtenIndex = store.get(indexKey); + const revalidated = await fetch(url, { headers: { "If-None-Match": first.headers.get("etag"), Connection: "close" }, signal: AbortSignal.timeout(10000) }); + assert.equal(revalidated.status, 304); + assert.equal(await revalidated.text(), ""); + assert.equal(revalidated.headers.get("last-modified"), first.headers.get("last-modified")); + const head = await fetch(url, { method: "HEAD", headers: { Connection: "close" }, signal: AbortSignal.timeout(10000) }); + assert.equal(head.status, 200); + assert.equal(await head.text(), ""); + const agentDir = join(root, "agent"); + mkdirSync(agentDir); + writeFileSync(join(agentDir, "auth.json"), JSON.stringify({ openai: { type: "api_key", key: "staging-fixture-never-used" } })); + const preload = join(root, "staging-route.mjs"); + writeFileSync(preload, `let original = globalThis.fetch; +const routed = (input, init) => { + const url = new URL(typeof input === "string" || input instanceof URL ? input : input.url); + if (url.origin === "https://api.diffex.ai" && url.pathname.startsWith("/api/models/providers/")) return original(${JSON.stringify(origin)} + url.pathname, init); + throw new Error("Unexpected outbound request in staging rehearsal"); +}; +Object.defineProperty(globalThis, "fetch", { configurable: true, get: () => routed, set: (value) => { original = value; } }); +`); + const cliEnv = { ...process.env, DIFFEX_CODING_AGENT_DIR: agentDir, NODE_OPTIONS: `--import=${preload}` }; + for (let i = 0; i < 2; i++) { + const output = execFileSync("./diffex-test.sh", ["--no-env", "update", "--models"], { env: cliEnv, encoding: "utf8", timeout: 30_000 }); + assert.match(output, /Model catalogs refreshed/); + const cache = JSON.parse(readFileSync(join(agentDir, "models-store.json"), "utf8")); + assert.equal(cache.openai.models.length, 170); + assert.equal(cache.openai.etag.replace(/^W\//, ""), `"${fixtureRevision}:openai"`); + } + const offline = execFileSync("./diffex-test.sh", ["--no-env", "--list-models", "model-0"], { env: { ...cliEnv, DIFFEX_OFFLINE: "1" }, encoding: "utf8", timeout: 30_000 }); + assert.match(offline, /model-0/); + console.log(JSON.stringify({ staging: true, revision: fixtureRevision, interruptedUploadPreservedIndex: true, conditionalConflictsRejected: true, identicalRetryPreservedManifest: true, clientRefreshAndOfflineRecovery: true, statuses: [first.status, revalidated.status, head.status] })); +} finally { + try { + if (writtenIndex) { + if (previous) { + assert.equal(store.put(indexKey, previous.bytes, writtenIndex.etag), true, "Concurrent staging publication: refusing to overwrite index"); + } else { + execFileSync("aws", ["s3api", "delete-object", "--bucket", bucket, "--endpoint-url", endpoint, "--key", indexKey, "--if-match", writtenIndex.etag], { stdio: "pipe" }); + } + } + const current = store.get(indexKey); + if (current && JSON.parse(current.bytes).catalogs.some((tier) => tier.revision === fixtureRevision)) throw new Error("Fixture is still selected; refusing cleanup"); + for (const key of fixtureKeys) { + const object = store.get(key); + if (object) execFileSync("aws", ["s3api", "delete-object", "--bucket", bucket, "--endpoint-url", endpoint, "--key", key, "--if-match", object.etag], { stdio: "pipe" }); + assert.equal(store.get(key), null, `Fixture cleanup failed: ${key}`); + } + assert.deepEqual(store.get(indexKey)?.bytes, previous?.bytes, "Original index was not restored"); + console.log(JSON.stringify({ cleanup: "verified", indexRestored: true, remainingFixtureObjects: 0 })); + } catch (error) { + console.error(`Cleanup did not complete; recovery evidence retained at ${root}`); + throw error; + } + rmSync(root, { recursive: true, force: true }); +} diff --git a/scripts/fixtures/model-catalog.mjs b/scripts/fixtures/model-catalog.mjs new file mode 100644 index 000000000..909342571 --- /dev/null +++ b/scripts/fixtures/model-catalog.mjs @@ -0,0 +1,18 @@ +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +export function catalogFixture(root, source = "a".repeat(40)) { + mkdirSync(join(root, "providers"), { recursive: true }); + const models = Object.fromEntries(["anthropic", "openai", "openrouter"].map((provider) => [provider, + Object.fromEntries(Array.from({ length: 170 }, (_, i) => [`model-${i}`, { + id: `model-${i}`, name: `Model ${i}`, api: "openai-completions", provider, + baseUrl: "https://example.test/v1", reasoning: false, input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow: 4096, maxTokens: 1000, + }]))])); + writeFileSync(join(root, "models.json"), JSON.stringify(models)); + writeFileSync(join(root, "providers.json"), JSON.stringify(Object.keys(models))); + writeFileSync(join(root, "generation.json"), JSON.stringify({ generatedAt: "2026-10-26T10:00:00.000Z", sourceCommit: source })); + for (const [id, shard] of Object.entries(models)) writeFileSync(join(root, "providers", `${id}.json`), JSON.stringify(shard)); + return models; +} + diff --git a/scripts/model-publication-security.test.mjs b/scripts/model-publication-security.test.mjs index 40565ea65..743baa987 100644 --- a/scripts/model-publication-security.test.mjs +++ b/scripts/model-publication-security.test.mjs @@ -46,6 +46,7 @@ test("publication accepts only main for scheduled and manual runs", () => { }); test("binds artifact and publishing code to one SHA and serializes uploads without cancellation", () => { + assert.equal(workflow.jobs.generate.env.SOURCE_REF, "${{ github.event.workflow_run.head_sha || inputs.source_ref || github.sha }}"); assert.equal(workflow.jobs.publish.env.SOURCE_REF, "${{ needs.generate.outputs.source_sha }}"); assert.equal(workflow.jobs.publish.concurrency["cancel-in-progress"], false); assert.match(workflow.jobs.publish.environment, /diffex-model-upload-staging/); diff --git a/scripts/monitor-model-publication.mjs b/scripts/monitor-model-publication.mjs new file mode 100644 index 000000000..92a91eee8 --- /dev/null +++ b/scripts/monitor-model-publication.mjs @@ -0,0 +1,36 @@ +import { execFileSync } from "node:child_process"; +import { pathToFileURL } from "node:url"; + +// Allow one hour for runner delays. Weekends do not make unchanged content stale. +export function expectedPublication(now = new Date()) { + const cutoff = now.getTime() - 60 * 60 * 1000; + for (let time = Math.floor(cutoff / 60000) * 60000; time > cutoff - 4 * 86400000; time -= 60000) { + const parts = Object.fromEntries(new Intl.DateTimeFormat("en-GB", { + timeZone: "Europe/Vienna", weekday: "short", hour: "2-digit", minute: "2-digit", hourCycle: "h23", + }).formatToParts(new Date(time)).map(({ type, value }) => [type, value])); + if (!["Sat", "Sun"].includes(parts.weekday) && ["10", "12", "14"].includes(parts.hour) && parts.minute === "17") return time; + } + throw new Error("Could not determine publication deadline"); +} + +export function checkPublicationRuns(jobs, now = new Date()) { + const expected = expectedPublication(now); + const generated = jobs.filter((job) => job.name === "generate" && job.conclusion === "success").map((job) => Date.parse(job.completed_at)); + const published = jobs.flatMap((job) => job.name === "publish" ? (job.steps ?? []).filter((step) => step.name === "Publish model catalog to R2" && step.conclusion === "success").map((step) => Date.parse(step.completed_at)) : []); + const validation = Math.max(0, ...generated); + const publication = Math.max(0, ...published); + if (validation < expected) throw new Error(`Missed successful catalog generation since ${new Date(expected).toISOString()}`); + if (publication < expected) throw new Error(`Missed successful catalog publication since ${new Date(expected).toISOString()}`); + return { lastSuccessfulGeneration: new Date(validation).toISOString(), lastSuccessfulPublication: new Date(publication).toISOString() }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const api = (path) => JSON.parse(execFileSync("gh", ["api", path], { encoding: "utf8" })); + const runs = api("repos/diffexai/diffex/actions/workflows/publish-model-catalog.yml/runs?branch=main&per_page=100").workflow_runs; + const jobs = runs.filter((run) => run.event !== "pull_request").flatMap((run) => { + const entries = api(`repos/diffexai/diffex/actions/runs/${run.id}/jobs?per_page=100`).jobs; + // Production monitoring must not count a manual staging upload as production health. + return run.display_title === "Model catalog (production)" ? entries : entries.filter((job) => job.name === "generate"); + }); + console.log(JSON.stringify(checkPublicationRuns(jobs))); +} diff --git a/scripts/monitor-model-publication.test.mjs b/scripts/monitor-model-publication.test.mjs new file mode 100644 index 000000000..6d3785145 --- /dev/null +++ b/scripts/monitor-model-publication.test.mjs @@ -0,0 +1,20 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { checkPublicationRuns, expectedPublication } from "./monitor-model-publication.mjs"; + +test("monitors successful generation and publication separately, including unchanged catalogs", () => { + const now = new Date("2026-10-26T11:30:00Z"); + const completed_at = "2026-10-26T09:30:00Z"; + const generated = { name: "generate", conclusion: "success", completed_at }; + const published = { name: "publish", steps: [{ name: "Publish model catalog to R2", conclusion: "success", completed_at }] }; + assert.doesNotThrow(() => checkPublicationRuns([generated, published], now)); + assert.throws(() => checkPublicationRuns([generated], now), /publication/); + assert.throws(() => checkPublicationRuns([published], now), /generation/); + assert.throws(() => checkPublicationRuns([], now), /Missed/); +}); + +test("allows runner delay and handles weekends and DST", () => { + assert.equal(new Date(expectedPublication(new Date("2026-10-26T10:00:00Z"))).toISOString(), "2026-10-23T12:17:00.000Z"); + assert.equal(new Date(expectedPublication(new Date("2026-10-26T10:18:00Z"))).toISOString(), "2026-10-26T09:17:00.000Z"); + assert.equal(new Date(expectedPublication(new Date("2026-03-30T09:18:00Z"))).toISOString(), "2026-03-30T08:17:00.000Z"); +}); diff --git a/scripts/publish-model-catalog.test.mjs b/scripts/publish-model-catalog.test.mjs index 05d5f92ed..6575ef1de 100644 --- a/scripts/publish-model-catalog.test.mjs +++ b/scripts/publish-model-catalog.test.mjs @@ -1,30 +1,16 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; -import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import test from "node:test"; +import { catalogFixture } from "./fixtures/model-catalog.mjs"; import { publishCatalog, validateBundle } from "./publish-model-catalog.mjs"; import { publicationAllowed } from "./model-publication-window.mjs"; const source = "a".repeat(40); const indexKey = "models/v1/index.json"; -export function catalogFixture(root) { - mkdirSync(join(root, "providers"), { recursive: true }); - const models = Object.fromEntries(["anthropic", "openai", "openrouter"].map((provider) => [provider, - Object.fromEntries(Array.from({ length: 170 }, (_, i) => [`model-${i}`, { - id: `model-${i}`, name: `Model ${i}`, api: "openai-completions", provider, - baseUrl: "https://example.test/v1", reasoning: false, input: ["text"], - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow: 4096, maxTokens: 1000, - }]))])); - writeFileSync(join(root, "models.json"), JSON.stringify(models)); - writeFileSync(join(root, "providers.json"), JSON.stringify(Object.keys(models))); - writeFileSync(join(root, "generation.json"), JSON.stringify({ generatedAt: "2026-10-26T10:00:00.000Z", sourceCommit: source })); - for (const [id, shard] of Object.entries(models)) writeFileSync(join(root, "providers", `${id}.json`), JSON.stringify(shard)); - return models; -} - function setup(t) { const root = mkdtempSync(join(tmpdir(), "diffex-publish-test-")); t.after(() => rmSync(root, { recursive: true, force: true })); @@ -69,6 +55,21 @@ test("interrupted uploads leave the index untouched and can be resumed", (t) => assert.ok(objects.has(indexKey)); }); +test("preserves other compatibility tiers when advancing the current floor", (t) => { + const { root, store } = setup(t); + publishCatalog(root, store, source); + const previous = store.get(indexKey); + const index = JSON.parse(previous.bytes); + const older = { ...index.catalogs[0], minimumDiffexVersion: "0.3.2", revision: `sha256-${"b".repeat(64)}` }; + index.catalogs.unshift(older); + assert.equal(store.put(indexKey, Buffer.from(JSON.stringify(index)), previous.etag), true); + const published = publishCatalog(root, store, source); + const next = JSON.parse(store.get(indexKey).bytes); + assert.deepEqual(next.catalogs[0], older); + assert.equal(next.catalogs.length, 2); + assert.equal(next.defaultRevision, published.revision); +}); + test("rejects conflicting revision bytes and conditional index conflicts", (t) => { const { root, store, objects } = setup(t); const revision = validateBundle(root).revision; From d69e4599a9db8a310f8fffce9110922173849799 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Mon, 26 Oct 2026 20:26:02 -0400 Subject: [PATCH 4/4] fix(ai): constrain catalog workflow writes - Emit the publication decision on stdout so the script cannot write an environment-selected file. - Restrict fixture property writes to known providers and cover the CLI regression. --- .github/workflows/publish-model-catalog.yml | 2 +- scripts/model-publication-security.test.mjs | 22 ++++++++++++++++++++- scripts/model-publication-window.mjs | 4 +--- scripts/publish-model-catalog.test.mjs | 2 +- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/.github/workflows/publish-model-catalog.yml b/.github/workflows/publish-model-catalog.yml index 758114ae6..ed0ee628f 100644 --- a/.github/workflows/publish-model-catalog.yml +++ b/.github/workflows/publish-model-catalog.yml @@ -125,7 +125,7 @@ jobs: env: EVENT_NAME: ${{ github.event_name }} MANUAL_PUBLISH: ${{ inputs.publish || false }} - run: node scripts/model-publication-window.mjs + run: node scripts/model-publication-window.mjs >> "$GITHUB_OUTPUT" - name: Publish model catalog to R2 if: steps.publication-window.outputs.allowed == 'true' diff --git a/scripts/model-publication-security.test.mjs b/scripts/model-publication-security.test.mjs index 743baa987..7db24acd5 100644 --- a/scripts/model-publication-security.test.mjs +++ b/scripts/model-publication-security.test.mjs @@ -1,11 +1,31 @@ import assert from "node:assert/strict"; -import { readFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import test from "node:test"; import vm from "node:vm"; import { parse } from "yaml"; const workflow = parse(readFileSync(new URL("../.github/workflows/publish-model-catalog.yml", import.meta.url), "utf8")); +test("publication-window CLI emits only its decision and does not write environment-selected paths", (t) => { + const root = mkdtempSync(join(tmpdir(), "diffex-window-security-")); + t.after(() => rmSync(root, { recursive: true, force: true })); + const target = join(root, "unrelated-file"); + writeFileSync(target, "unchanged\n"); + for (const manual of ["true", "false", "true\nallowed=true"]) { + const result = spawnSync(process.execPath, ["scripts/model-publication-window.mjs"], { + cwd: new URL("../", import.meta.url), + encoding: "utf8", + env: { ...process.env, EVENT_NAME: "workflow_dispatch", MANUAL_PUBLISH: manual, GITHUB_OUTPUT: target }, + }); + assert.equal(result.status, 0, result.stderr); + assert.equal(readFileSync(target, "utf8"), "unchanged\n"); + assert.equal(result.stdout, `allowed=${manual === "true"}\n`); + } +}); + function allowed(job, overrides = {}, inputs = {}) { const github = { repository: "diffexai/diffex", diff --git a/scripts/model-publication-window.mjs b/scripts/model-publication-window.mjs index e38aa5da5..07823e04c 100644 --- a/scripts/model-publication-window.mjs +++ b/scripts/model-publication-window.mjs @@ -1,4 +1,3 @@ -import { appendFileSync } from "node:fs"; import { pathToFileURL } from "node:url"; export function publicationAllowed(event, manual, date = new Date()) { @@ -14,6 +13,5 @@ export function publicationAllowed(event, manual, date = new Date()) { if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { const allowed = publicationAllowed(process.env.EVENT_NAME, process.env.MANUAL_PUBLISH === "true"); - appendFileSync(process.env.GITHUB_OUTPUT, `allowed=${allowed}\n`); - console.log(`Catalog publication window: ${allowed}`); + console.log(`allowed=${allowed}`); } diff --git a/scripts/publish-model-catalog.test.mjs b/scripts/publish-model-catalog.test.mjs index 6575ef1de..1ff62e582 100644 --- a/scripts/publish-model-catalog.test.mjs +++ b/scripts/publish-model-catalog.test.mjs @@ -105,7 +105,7 @@ test("rejects malformed fields, inconsistent shards, invalid timestamps and smal writeFileSync(join(root, "generation.json"), "{}"); assert.throws(() => validateBundle(root), /timestamp/); catalogFixture(root); - for (const provider of Object.keys(models)) { + for (const provider of ["anthropic", "openai", "openrouter"]) { models[provider] = { "model-1": models[provider]["model-1"] }; writeFileSync(join(root, "providers", `${provider}.json`), JSON.stringify(models[provider])); }