From 247e7e9d8940b6b9dd8683e5f328c4afcf40f852 Mon Sep 17 00:00:00 2001 From: Jeffrey Tseng Date: Fri, 2 Oct 2026 20:13:52 -0700 Subject: [PATCH 1/2] fix(ai): stabilize catalogs and stacked CI Use fixed provider inputs in isolated copies of the real model generator so Together catalog removals cannot invalidate metadata regression tests, while retaining strict failure and compatibility coverage. Run the existing CI workflow for pull requests against prerequisite branches so every stacked head receives the full build and test checks. --- .github/workflows/ci.yml | 1 - packages/ai/test/fixtures/model-generation.ts | 59 +++++++ .../ai/test/generate-models-strict.test.ts | 144 ++++------------ packages/ai/test/together-models.test.ts | 163 +++++++++++------- 4 files changed, 202 insertions(+), 165 deletions(-) create mode 100644 packages/ai/test/fixtures/model-generation.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 40f43665f..bb7f6ceed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,6 @@ on: push: branches: [main] pull_request: - branches: [main] concurrency: group: ci-${{ github.ref }} diff --git a/packages/ai/test/fixtures/model-generation.ts b/packages/ai/test/fixtures/model-generation.ts new file mode 100644 index 000000000..d4d371884 --- /dev/null +++ b/packages/ai/test/fixtures/model-generation.ts @@ -0,0 +1,59 @@ +import { type SpawnSyncReturns, spawnSync } from "node:child_process"; +import { cpSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const packageRoot = fileURLToPath(new URL("../..", import.meta.url)); + +export const individualModelIds = [ + "deepseek-v4-flash-0731", + "deepseek-v4-pro", + "glm-5.2", + "qwen3.6-flash", + "qwen3.7-max", + "qwen3.7-plus", + "qwen3.8-max", +] as const; + +export const individualModelCatalog = { + "alibaba-token-plan": { + models: Object.fromEntries(individualModelIds.map((id) => [id, { id, name: id, tool_call: true }])), + }, +}; + +export function withModelGenerationFixture( + catalog: object, + verify: (fixture: { packageRoot: string; generate: () => SpawnSyncReturns }) => void, +): void { + const fixtureRoot = mkdtempSync(join(tmpdir(), "diffex-generate-models-")); + const isolatedPackageRoot = join(fixtureRoot, "package"); + const preloadPath = join(fixtureRoot, "mock-catalog.mjs"); + function generate() { + return spawnSync( + process.execPath, + ["--import", pathToFileURL(preloadPath).href, "scripts/generate-models.ts", "--strict"], + { cwd: isolatedPackageRoot, encoding: "utf8", timeout: 10_000 }, + ); + } + try { + mkdirSync(isolatedPackageRoot); + for (const entry of ["package.json", "scripts", "src"]) { + cpSync(join(packageRoot, entry), join(isolatedPackageRoot, entry), { recursive: true }); + } + writeFileSync( + preloadPath, + `const catalog = ${JSON.stringify(catalog)};\n` + + `const emptyFeeds = new Set(["https://openrouter.ai/api/v1/models", "https://ai-gateway.vercel.sh/v1/models", "https://integrate.api.nvidia.com/v1/models"]);\n` + + `globalThis.fetch = async (input) => {\n` + + ` const url = String(input);\n` + + ` if (url === "https://models.dev/api.json") return new Response(JSON.stringify(catalog), { status: 200 });\n` + + ` if (emptyFeeds.has(url)) return new Response(JSON.stringify({ data: [] }), { status: 200 });\n` + + ` throw new Error(\`Unexpected fetch: \${url}\`);\n` + + `};\n`, + ); + verify({ packageRoot: isolatedPackageRoot, generate }); + } finally { + rmSync(fixtureRoot, { force: true, recursive: true }); + } +} diff --git a/packages/ai/test/generate-models-strict.test.ts b/packages/ai/test/generate-models-strict.test.ts index 3b759885a..fabf82b44 100644 --- a/packages/ai/test/generate-models-strict.test.ts +++ b/packages/ai/test/generate-models-strict.test.ts @@ -1,39 +1,15 @@ -import { spawnSync } from "node:child_process"; -import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; +import { readFileSync } from "node:fs"; import { join } from "node:path"; -import { fileURLToPath, pathToFileURL } from "node:url"; -import { afterEach, describe, expect, it } from "vitest"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { individualModelCatalog, individualModelIds, withModelGenerationFixture } from "./fixtures/model-generation.ts"; const packageRoot = fileURLToPath(new URL("..", import.meta.url)); -const temporaryRoots: string[] = []; - -afterEach(() => { - for (const root of temporaryRoots.splice(0)) rmSync(root, { force: true, recursive: true }); -}); describe("strict model generation", () => { it("preserves Kimi Coding models from the renamed regional catalog", () => { - const fixtureRoot = mkdtempSync(join(tmpdir(), "diffex-generate-kimi-")); - temporaryRoots.push(fixtureRoot); - const isolatedPackageRoot = join(fixtureRoot, "package"); - mkdirSync(isolatedPackageRoot); - for (const entry of ["package.json", "scripts", "src"]) { - cpSync(join(packageRoot, entry), join(isolatedPackageRoot, entry), { recursive: true }); - } - const modelIds = [ - "deepseek-v4-flash-0731", - "deepseek-v4-pro", - "glm-5.2", - "qwen3.6-flash", - "qwen3.7-max", - "qwen3.7-plus", - "qwen3.8-max", - ]; const catalog = { - "alibaba-token-plan": { - models: Object.fromEntries(modelIds.map((id) => [id, { id, name: id, tool_call: true }])), - }, + ...individualModelCatalog, "kimi-code-plan-cn": { models: { "kimi-for-coding": { @@ -46,51 +22,25 @@ describe("strict model generation", () => { }, }, }; - const preloadPath = join(fixtureRoot, "mock-catalog.mjs"); - writeFileSync( - preloadPath, - `const catalog = ${JSON.stringify(catalog)};\n` + - `globalThis.fetch = async (input) => new Response(JSON.stringify(\n` + - ` String(input) === "https://models.dev/api.json" ? catalog : { data: [] }\n` + - `), { status: 200 });\n`, - ); - const result = spawnSync( - process.execPath, - ["--import", pathToFileURL(preloadPath).href, "scripts/generate-models.ts", "--strict"], - { cwd: isolatedPackageRoot, encoding: "utf8", timeout: 10_000 }, - ); - expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); - const models = JSON.parse(readFileSync(join(isolatedPackageRoot, "src/providers/data/kimi-coding.json"), "utf8")); - expect(models["anthropic-messages"]["kimi-for-coding"]).toMatchObject({ - provider: "kimi-coding", - baseUrl: "https://api.kimi.com/coding", - contextWindow: 1048576, - maxTokens: 32768, - compat: { forceAdaptiveThinking: true, allowEmptySignature: true }, + withModelGenerationFixture(catalog, (fixture) => { + const result = fixture.generate(); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const models = JSON.parse( + readFileSync(join(fixture.packageRoot, "src/providers/data/kimi-coding.json"), "utf8"), + ); + expect(models["anthropic-messages"]["kimi-for-coding"]).toMatchObject({ + provider: "kimi-coding", + baseUrl: "https://api.kimi.com/coding", + contextWindow: 1048576, + maxTokens: 32768, + compat: { forceAdaptiveThinking: true, allowEmptySignature: true }, + }); }); }); it("fails before mutating generated data when an Individual model loses tool support", () => { - const fixtureRoot = mkdtempSync(join(tmpdir(), "diffex-generate-models-")); - temporaryRoots.push(fixtureRoot); - const isolatedPackageRoot = join(fixtureRoot, "package"); - mkdirSync(isolatedPackageRoot); - for (const entry of ["package.json", "scripts", "src"]) { - cpSync(join(packageRoot, entry), join(isolatedPackageRoot, entry), { recursive: true }); - } - const preloadPath = join(fixtureRoot, "mock-models-dev.mjs"); - const modelIds = [ - "deepseek-v4-flash-0731", - "deepseek-v4-pro", - "glm-5.2", - "qwen3.6-flash", - "qwen3.7-max", - "qwen3.7-plus", - "qwen3.8-max", - "qwen3.8-max-preview", - ]; const sourceModels = Object.fromEntries( - modelIds.map((id) => [ + [...individualModelIds, "qwen3.8-max-preview"].map((id) => [ id, { id, @@ -100,43 +50,25 @@ describe("strict model generation", () => { ]), ); const catalog = { "alibaba-token-plan": { models: sourceModels } }; - writeFileSync( - preloadPath, - `const catalog = ${JSON.stringify(catalog)};\n` + - `globalThis.fetch = async (input) => {\n` + - ` if (String(input) === "https://models.dev/api.json") {\n` + - ` return new Response(JSON.stringify(catalog), { status: 200 });\n` + - ` }\n` + - ` throw new Error(\`Unexpected fetch: \${String(input)}\`);\n` + - `};\n`, - ); - - const generatedPaths = [ - "src/models.generated.ts", - "src/providers/qwen-token-plan-individual.models.ts", - "src/providers/data/qwen-token-plan-individual.json", - "src/providers/data/.manifest.json", - ]; - const sourceBefore = generatedPaths.map((path) => readFileSync(join(packageRoot, path), "utf8")); - const isolatedBefore = generatedPaths.map((path) => readFileSync(join(isolatedPackageRoot, path), "utf8")); + withModelGenerationFixture(catalog, (fixture) => { + const generatedPaths = [ + "src/models.generated.ts", + "src/providers/qwen-token-plan-individual.models.ts", + "src/providers/data/qwen-token-plan-individual.json", + "src/providers/data/.manifest.json", + ]; + const sourceBefore = generatedPaths.map((path) => readFileSync(join(packageRoot, path), "utf8")); + const isolatedBefore = generatedPaths.map((path) => readFileSync(join(fixture.packageRoot, path), "utf8")); - const result = spawnSync( - process.execPath, - ["--import", pathToFileURL(preloadPath).href, "scripts/generate-models.ts", "--strict"], - { - cwd: isolatedPackageRoot, - encoding: "utf8", - timeout: 10_000, - }, - ); - - expect(result.status).toBe(1); - expect(`${result.stdout}\n${result.stderr}`).toContain( - "qwen-token-plan-individual model IDs do not match (missing: deepseek-v4-flash-0731)", - ); - expect(generatedPaths.map((path) => readFileSync(join(isolatedPackageRoot, path), "utf8"))).toEqual( - isolatedBefore, - ); - expect(generatedPaths.map((path) => readFileSync(join(packageRoot, path), "utf8"))).toEqual(sourceBefore); + const result = fixture.generate(); + expect(result.status).toBe(1); + expect(`${result.stdout}\n${result.stderr}`).toContain( + "qwen-token-plan-individual model IDs do not match (missing: deepseek-v4-flash-0731)", + ); + expect(generatedPaths.map((path) => readFileSync(join(fixture.packageRoot, path), "utf8"))).toEqual( + isolatedBefore, + ); + expect(generatedPaths.map((path) => readFileSync(join(packageRoot, path), "utf8"))).toEqual(sourceBefore); + }); }); }); diff --git a/packages/ai/test/together-models.test.ts b/packages/ai/test/together-models.test.ts index fb28555b9..e7dbf8617 100644 --- a/packages/ai/test/together-models.test.ts +++ b/packages/ai/test/together-models.test.ts @@ -1,6 +1,29 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { getModel } from "../src/compat.ts"; import { findEnvKeys, getEnvApiKey } from "../src/env-api-keys.ts"; +import type { Model } from "../src/types.ts"; +import { individualModelCatalog, withModelGenerationFixture } from "./fixtures/model-generation.ts"; + +const togetherModels = { + "moonshotai/Kimi-K3": { + id: "moonshotai/Kimi-K3", + name: "Kimi K3", + tool_call: true, + reasoning: true, + modalities: { input: ["text", "image"] }, + limit: { context: 1048576, output: 131072 }, + cost: { input: 3, output: 15, cache_read: 0.3 }, + }, + "openai/gpt-oss-120b": { + id: "openai/gpt-oss-120b", + tool_call: true, + reasoning: true, + reasoning_options: [{ type: "effort", values: ["low", "medium", "high"] }], + }, + "deepseek-ai/DeepSeek-V4-Pro": { id: "deepseek-ai/DeepSeek-V4-Pro", tool_call: true, reasoning: true }, + "MiniMaxAI/MiniMax-M2.7": { id: "MiniMaxAI/MiniMax-M2.7", tool_call: true, reasoning: true }, +}; const originalTogetherApiKey = process.env.TOGETHER_API_KEY; @@ -13,68 +36,92 @@ afterEach(() => { }); describe("Together models", () => { - it("registers the default Kimi K3 model via OpenAI-compatible Chat Completions API", () => { - const model = getModel("together", "moonshotai/Kimi-K3"); + it("generates Together metadata and reasoning controls independently of the live catalog", () => { + withModelGenerationFixture({ ...individualModelCatalog, together: { models: togetherModels } }, (fixture) => { + const result = fixture.generate(); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const catalog: Record>> = JSON.parse( + readFileSync(join(fixture.packageRoot, "src/providers/data/together.json"), "utf8"), + ); + const models = catalog["openai-completions"]; + const model = models["moonshotai/Kimi-K3"]; - expect(model).toBeDefined(); - expect(model.api).toBe("openai-completions"); - expect(model.provider).toBe("together"); - expect(model.baseUrl).toBe("https://api.together.ai/v1"); - expect(model.reasoning).toBe(true); - expect(model.thinkingLevelMap).toEqual({ minimal: null, low: null, medium: null }); - expect(model.input).toEqual(["text", "image"]); - expect(model.contextWindow).toBe(1048576); - expect(model.maxTokens).toBe(131072); - expect(model.cost).toEqual({ - input: 3, - output: 15, - cacheRead: 0.3, - cacheWrite: 0, - }); - expect(model.compat).toEqual({ - supportsStore: false, - supportsDeveloperRole: false, - supportsReasoningEffort: false, - maxTokensField: "max_tokens", - thinkingFormat: "together", - supportsStrictMode: false, - supportsLongCacheRetention: false, - }); - }); + expect(model).toBeDefined(); + expect(model.api).toBe("openai-completions"); + expect(model.provider).toBe("together"); + expect(model.baseUrl).toBe("https://api.together.ai/v1"); + expect(model.reasoning).toBe(true); + expect(model.thinkingLevelMap).toEqual({ minimal: null, low: null, medium: null }); + expect(model.input).toEqual(["text", "image"]); + expect(model.contextWindow).toBe(1048576); + expect(model.maxTokens).toBe(131072); + expect(model.cost).toEqual({ + input: 3, + output: 15, + cacheRead: 0.3, + cacheWrite: 0, + }); + expect(model.compat).toEqual({ + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + maxTokensField: "max_tokens", + thinkingFormat: "together", + supportsStrictMode: false, + supportsLongCacheRetention: false, + }); + const gptOss = models["openai/gpt-oss-120b"]; + expect(gptOss.thinkingLevelMap).toEqual({ + off: null, + minimal: null, + low: "low", + medium: "medium", + high: "high", + max: null, + xhigh: null, + }); + expect(gptOss.compat).toMatchObject({ + supportsReasoningEffort: true, + thinkingFormat: "openai", + }); - it("models Together reasoning controls from the Together API surface", () => { - const gptOss = getModel("together", "openai/gpt-oss-120b"); - expect(gptOss.thinkingLevelMap).toEqual({ - off: null, - minimal: null, - low: "low", - medium: "medium", - high: "high", - max: null, - xhigh: null, - }); - expect(gptOss.compat).toMatchObject({ - supportsReasoningEffort: true, - thinkingFormat: "openai", - }); + const deepSeekV4 = models["deepseek-ai/DeepSeek-V4-Pro"]; + expect(deepSeekV4.thinkingLevelMap).toEqual({ + minimal: null, + low: null, + medium: null, + high: "high", + xhigh: null, + }); + expect(deepSeekV4.compat).toMatchObject({ + supportsReasoningEffort: true, + thinkingFormat: "together", + }); - const deepSeekV4 = getModel("together", "deepseek-ai/DeepSeek-V4-Pro"); - expect(deepSeekV4.thinkingLevelMap).toEqual({ - minimal: null, - low: null, - medium: null, - high: "high", - xhigh: null, - }); - expect(deepSeekV4.compat).toMatchObject({ - supportsReasoningEffort: true, - thinkingFormat: "together", + const minimax = models["MiniMaxAI/MiniMax-M2.7"]; + expect(minimax.thinkingLevelMap).toEqual({ off: null, minimal: null, low: null, medium: null }); + expect(minimax.compat?.thinkingFormat).toBeUndefined(); + expect(minimax.compat?.supportsReasoningEffort).toBe(false); }); + }); - const minimax = getModel("together", "MiniMaxAI/MiniMax-M2.7"); - expect(minimax.thinkingLevelMap).toEqual({ off: null, minimal: null, low: null, medium: null }); - expect(minimax.compat?.thinkingFormat).toBeUndefined(); - expect(minimax.compat?.supportsReasoningEffort).toBe(false); + it.each(["absent", "deprecated", "without tools"] as const)("omits a Together model that is %s", (state) => { + const models: Record = { ...togetherModels }; + const deepSeekId = "deepseek-ai/DeepSeek-V4-Pro"; + if (state === "absent") delete models[deepSeekId]; + else models[deepSeekId] = { ...togetherModels[deepSeekId], status: state, tool_call: state !== "without tools" }; + withModelGenerationFixture({ ...individualModelCatalog, together: { models } }, (fixture) => { + const result = fixture.generate(); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const catalog = JSON.parse( + readFileSync(join(fixture.packageRoot, "src/providers/data/together.json"), "utf8"), + ); + expect(catalog["openai-completions"][deepSeekId]).toBeUndefined(); + expect(catalog["openai-completions"]["openai/gpt-oss-120b"]).toMatchObject({ + provider: "together", + compat: { supportsReasoningEffort: true, thinkingFormat: "openai" }, + }); + }); }); it("resolves TOGETHER_API_KEY from the environment", () => { From 18e8620d3e0f8eef4f3a90f8e269d5fe893b0414 Mon Sep 17 00:00:00 2001 From: Jeffrey Tseng Date: Fri, 2 Oct 2026 21:55:58 -0700 Subject: [PATCH 2/2] fix(ci): stabilize lifecycle registry fixtures Prevent real npm integrity checks from failing on idle pooled sockets by explicitly closing and framing owned registry responses. Start bounded cancellation escalation only after the real archive request, retain a finite startup deadline, record observed child completion, and clean up the detached fixture group while preserving strict checksum, installation, and repair assertions. --- scripts/npm-lifecycle.mjs | 90 ++++++++++++++++++++++++++++++--------- 1 file changed, 70 insertions(+), 20 deletions(-) diff --git a/scripts/npm-lifecycle.mjs b/scripts/npm-lifecycle.mjs index 589f005dd..798f7fb13 100644 --- a/scripts/npm-lifecycle.mjs +++ b/scripts/npm-lifecycle.mjs @@ -92,6 +92,9 @@ let interrupt; let tarballRequests = 0; const server = createServer((request, reply) => { const path = decodeURIComponent(request.url); + // A stalled npm event loop can reuse a socket after the fixture's idle deadline. + // Each owned response closes its connection instead of racing pooled socket reuse. + reply.setHeader("connection", "close"); if (path === "/@diffexai/diffex") { const pkg = { ...metadata, @@ -102,20 +105,22 @@ const server = createServer((request, reply) => { .digest("base64")}`, }, }; + const body = JSON.stringify({ + name: metadata.name, + "dist-tags": { latest: metadata.version }, + versions: { [metadata.version]: pkg }, + }); reply.setHeader("content-type", "application/json"); - reply.end( - JSON.stringify({ - name: metadata.name, - "dist-tags": { latest: metadata.version }, - versions: { [metadata.version]: pkg }, - }), - ); + reply.setHeader("content-length", Buffer.byteLength(body)); + reply.end(body); } else if (path === "/diffex.tgz") { tarballRequests++; if (mode === "interrupt") { interrupt?.(); return; } + reply.setHeader("content-type", "application/octet-stream"); + reply.setHeader("content-length", bytes.length); reply.end(bytes); } else { reply.writeHead(404).end(); @@ -215,27 +220,72 @@ try { else { const child = spawn(process.execPath, ["--import", guard, executable, "update", "--self", "--force"], { ...options, + timeout: undefined, detached: true, - stdio: "ignore", + stdio: ["ignore", "inherit", "inherit"], + }); + const closed = new Promise((done, reject) => { + child.once("close", (code, signal) => done({ code, signal })); + child.once("error", reject); }); + function signalGroup(signal) { + if (!child.pid) return false; + try { + process.kill(-child.pid, signal); + return true; + } catch (error) { + if (error.code === "ESRCH") return false; + throw error; + } + } let interrupted = false; - interrupt = () => { - interrupted = true; - process.kill(-child.pid, "SIGTERM"); - }; - const deadline = setTimeout(() => process.kill(-child.pid, "SIGKILL"), 30000); + let cancellationFallbackTriggered = false; + let outcome; + let startupDeadline; + let cancellationDeadline; + const stopped = new Promise((done, reject) => { + closed.then(done, reject); + startupDeadline = setTimeout( + () => reject(new Error("Timed out before the real npm archive download")), + options.timeout, + ); + interrupt = () => { + if (interrupted) return; + try { + assert(signalGroup("SIGTERM"), "Update exited before archive cancellation"); + interrupted = true; + clearTimeout(startupDeadline); + // Bound cancellation after the observed download, independently of npm startup. + cancellationDeadline = setTimeout(() => { + try { + cancellationFallbackTriggered = true; + signalGroup("SIGKILL"); + } catch (error) { + reject(error); + } + }, 5000); + } catch (error) { + reject(error); + } + }; + }); try { - await new Promise((done, reject) => { - child.on("close", done); - child.on("error", reject); - }); + outcome = await stopped; + assert(interrupted, "Interrupted update must reach the real npm archive download"); } finally { - clearTimeout(deadline); + interrupt = undefined; + clearTimeout(startupDeadline); + clearTimeout(cancellationDeadline); + // The detached group belongs only to this fixture, including npm descendants. + signalGroup("SIGKILL"); + await closed.catch(() => {}); } - assert(interrupted, "Interrupted update must reach the real npm archive download"); evidence.commands.push({ command: "diffex update --self --force", - signal: "SIGTERM", + requestedSignal: "SIGTERM", + signal: outcome.signal, + exitCode: outcome.code, + cancellationFallbackTriggered, boundary: "npm archive download", }); }