diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 40f43665f..bb7f6ceed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,6 @@ on: push: branches: [main] pull_request: - branches: [main] concurrency: group: ci-${{ github.ref }} diff --git a/packages/ai/test/fixtures/model-generation.ts b/packages/ai/test/fixtures/model-generation.ts new file mode 100644 index 000000000..d4d371884 --- /dev/null +++ b/packages/ai/test/fixtures/model-generation.ts @@ -0,0 +1,59 @@ +import { type SpawnSyncReturns, spawnSync } from "node:child_process"; +import { cpSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const packageRoot = fileURLToPath(new URL("../..", import.meta.url)); + +export const individualModelIds = [ + "deepseek-v4-flash-0731", + "deepseek-v4-pro", + "glm-5.2", + "qwen3.6-flash", + "qwen3.7-max", + "qwen3.7-plus", + "qwen3.8-max", +] as const; + +export const individualModelCatalog = { + "alibaba-token-plan": { + models: Object.fromEntries(individualModelIds.map((id) => [id, { id, name: id, tool_call: true }])), + }, +}; + +export function withModelGenerationFixture( + catalog: object, + verify: (fixture: { packageRoot: string; generate: () => SpawnSyncReturns }) => void, +): void { + const fixtureRoot = mkdtempSync(join(tmpdir(), "diffex-generate-models-")); + const isolatedPackageRoot = join(fixtureRoot, "package"); + const preloadPath = join(fixtureRoot, "mock-catalog.mjs"); + function generate() { + return spawnSync( + process.execPath, + ["--import", pathToFileURL(preloadPath).href, "scripts/generate-models.ts", "--strict"], + { cwd: isolatedPackageRoot, encoding: "utf8", timeout: 10_000 }, + ); + } + try { + mkdirSync(isolatedPackageRoot); + for (const entry of ["package.json", "scripts", "src"]) { + cpSync(join(packageRoot, entry), join(isolatedPackageRoot, entry), { recursive: true }); + } + writeFileSync( + preloadPath, + `const catalog = ${JSON.stringify(catalog)};\n` + + `const emptyFeeds = new Set(["https://openrouter.ai/api/v1/models", "https://ai-gateway.vercel.sh/v1/models", "https://integrate.api.nvidia.com/v1/models"]);\n` + + `globalThis.fetch = async (input) => {\n` + + ` const url = String(input);\n` + + ` if (url === "https://models.dev/api.json") return new Response(JSON.stringify(catalog), { status: 200 });\n` + + ` if (emptyFeeds.has(url)) return new Response(JSON.stringify({ data: [] }), { status: 200 });\n` + + ` throw new Error(\`Unexpected fetch: \${url}\`);\n` + + `};\n`, + ); + verify({ packageRoot: isolatedPackageRoot, generate }); + } finally { + rmSync(fixtureRoot, { force: true, recursive: true }); + } +} diff --git a/packages/ai/test/generate-models-strict.test.ts b/packages/ai/test/generate-models-strict.test.ts index 3b759885a..fabf82b44 100644 --- a/packages/ai/test/generate-models-strict.test.ts +++ b/packages/ai/test/generate-models-strict.test.ts @@ -1,39 +1,15 @@ -import { spawnSync } from "node:child_process"; -import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; +import { readFileSync } from "node:fs"; import { join } from "node:path"; -import { fileURLToPath, pathToFileURL } from "node:url"; -import { afterEach, describe, expect, it } from "vitest"; +import { fileURLToPath } from "node:url"; +import { describe, expect, it } from "vitest"; +import { individualModelCatalog, individualModelIds, withModelGenerationFixture } from "./fixtures/model-generation.ts"; const packageRoot = fileURLToPath(new URL("..", import.meta.url)); -const temporaryRoots: string[] = []; - -afterEach(() => { - for (const root of temporaryRoots.splice(0)) rmSync(root, { force: true, recursive: true }); -}); describe("strict model generation", () => { it("preserves Kimi Coding models from the renamed regional catalog", () => { - const fixtureRoot = mkdtempSync(join(tmpdir(), "diffex-generate-kimi-")); - temporaryRoots.push(fixtureRoot); - const isolatedPackageRoot = join(fixtureRoot, "package"); - mkdirSync(isolatedPackageRoot); - for (const entry of ["package.json", "scripts", "src"]) { - cpSync(join(packageRoot, entry), join(isolatedPackageRoot, entry), { recursive: true }); - } - const modelIds = [ - "deepseek-v4-flash-0731", - "deepseek-v4-pro", - "glm-5.2", - "qwen3.6-flash", - "qwen3.7-max", - "qwen3.7-plus", - "qwen3.8-max", - ]; const catalog = { - "alibaba-token-plan": { - models: Object.fromEntries(modelIds.map((id) => [id, { id, name: id, tool_call: true }])), - }, + ...individualModelCatalog, "kimi-code-plan-cn": { models: { "kimi-for-coding": { @@ -46,51 +22,25 @@ describe("strict model generation", () => { }, }, }; - const preloadPath = join(fixtureRoot, "mock-catalog.mjs"); - writeFileSync( - preloadPath, - `const catalog = ${JSON.stringify(catalog)};\n` + - `globalThis.fetch = async (input) => new Response(JSON.stringify(\n` + - ` String(input) === "https://models.dev/api.json" ? catalog : { data: [] }\n` + - `), { status: 200 });\n`, - ); - const result = spawnSync( - process.execPath, - ["--import", pathToFileURL(preloadPath).href, "scripts/generate-models.ts", "--strict"], - { cwd: isolatedPackageRoot, encoding: "utf8", timeout: 10_000 }, - ); - expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); - const models = JSON.parse(readFileSync(join(isolatedPackageRoot, "src/providers/data/kimi-coding.json"), "utf8")); - expect(models["anthropic-messages"]["kimi-for-coding"]).toMatchObject({ - provider: "kimi-coding", - baseUrl: "https://api.kimi.com/coding", - contextWindow: 1048576, - maxTokens: 32768, - compat: { forceAdaptiveThinking: true, allowEmptySignature: true }, + withModelGenerationFixture(catalog, (fixture) => { + const result = fixture.generate(); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const models = JSON.parse( + readFileSync(join(fixture.packageRoot, "src/providers/data/kimi-coding.json"), "utf8"), + ); + expect(models["anthropic-messages"]["kimi-for-coding"]).toMatchObject({ + provider: "kimi-coding", + baseUrl: "https://api.kimi.com/coding", + contextWindow: 1048576, + maxTokens: 32768, + compat: { forceAdaptiveThinking: true, allowEmptySignature: true }, + }); }); }); it("fails before mutating generated data when an Individual model loses tool support", () => { - const fixtureRoot = mkdtempSync(join(tmpdir(), "diffex-generate-models-")); - temporaryRoots.push(fixtureRoot); - const isolatedPackageRoot = join(fixtureRoot, "package"); - mkdirSync(isolatedPackageRoot); - for (const entry of ["package.json", "scripts", "src"]) { - cpSync(join(packageRoot, entry), join(isolatedPackageRoot, entry), { recursive: true }); - } - const preloadPath = join(fixtureRoot, "mock-models-dev.mjs"); - const modelIds = [ - "deepseek-v4-flash-0731", - "deepseek-v4-pro", - "glm-5.2", - "qwen3.6-flash", - "qwen3.7-max", - "qwen3.7-plus", - "qwen3.8-max", - "qwen3.8-max-preview", - ]; const sourceModels = Object.fromEntries( - modelIds.map((id) => [ + [...individualModelIds, "qwen3.8-max-preview"].map((id) => [ id, { id, @@ -100,43 +50,25 @@ describe("strict model generation", () => { ]), ); const catalog = { "alibaba-token-plan": { models: sourceModels } }; - writeFileSync( - preloadPath, - `const catalog = ${JSON.stringify(catalog)};\n` + - `globalThis.fetch = async (input) => {\n` + - ` if (String(input) === "https://models.dev/api.json") {\n` + - ` return new Response(JSON.stringify(catalog), { status: 200 });\n` + - ` }\n` + - ` throw new Error(\`Unexpected fetch: \${String(input)}\`);\n` + - `};\n`, - ); - - const generatedPaths = [ - "src/models.generated.ts", - "src/providers/qwen-token-plan-individual.models.ts", - "src/providers/data/qwen-token-plan-individual.json", - "src/providers/data/.manifest.json", - ]; - const sourceBefore = generatedPaths.map((path) => readFileSync(join(packageRoot, path), "utf8")); - const isolatedBefore = generatedPaths.map((path) => readFileSync(join(isolatedPackageRoot, path), "utf8")); + withModelGenerationFixture(catalog, (fixture) => { + const generatedPaths = [ + "src/models.generated.ts", + "src/providers/qwen-token-plan-individual.models.ts", + "src/providers/data/qwen-token-plan-individual.json", + "src/providers/data/.manifest.json", + ]; + const sourceBefore = generatedPaths.map((path) => readFileSync(join(packageRoot, path), "utf8")); + const isolatedBefore = generatedPaths.map((path) => readFileSync(join(fixture.packageRoot, path), "utf8")); - const result = spawnSync( - process.execPath, - ["--import", pathToFileURL(preloadPath).href, "scripts/generate-models.ts", "--strict"], - { - cwd: isolatedPackageRoot, - encoding: "utf8", - timeout: 10_000, - }, - ); - - expect(result.status).toBe(1); - expect(`${result.stdout}\n${result.stderr}`).toContain( - "qwen-token-plan-individual model IDs do not match (missing: deepseek-v4-flash-0731)", - ); - expect(generatedPaths.map((path) => readFileSync(join(isolatedPackageRoot, path), "utf8"))).toEqual( - isolatedBefore, - ); - expect(generatedPaths.map((path) => readFileSync(join(packageRoot, path), "utf8"))).toEqual(sourceBefore); + const result = fixture.generate(); + expect(result.status).toBe(1); + expect(`${result.stdout}\n${result.stderr}`).toContain( + "qwen-token-plan-individual model IDs do not match (missing: deepseek-v4-flash-0731)", + ); + expect(generatedPaths.map((path) => readFileSync(join(fixture.packageRoot, path), "utf8"))).toEqual( + isolatedBefore, + ); + expect(generatedPaths.map((path) => readFileSync(join(packageRoot, path), "utf8"))).toEqual(sourceBefore); + }); }); }); diff --git a/packages/ai/test/together-models.test.ts b/packages/ai/test/together-models.test.ts index fb28555b9..e7dbf8617 100644 --- a/packages/ai/test/together-models.test.ts +++ b/packages/ai/test/together-models.test.ts @@ -1,6 +1,29 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { getModel } from "../src/compat.ts"; import { findEnvKeys, getEnvApiKey } from "../src/env-api-keys.ts"; +import type { Model } from "../src/types.ts"; +import { individualModelCatalog, withModelGenerationFixture } from "./fixtures/model-generation.ts"; + +const togetherModels = { + "moonshotai/Kimi-K3": { + id: "moonshotai/Kimi-K3", + name: "Kimi K3", + tool_call: true, + reasoning: true, + modalities: { input: ["text", "image"] }, + limit: { context: 1048576, output: 131072 }, + cost: { input: 3, output: 15, cache_read: 0.3 }, + }, + "openai/gpt-oss-120b": { + id: "openai/gpt-oss-120b", + tool_call: true, + reasoning: true, + reasoning_options: [{ type: "effort", values: ["low", "medium", "high"] }], + }, + "deepseek-ai/DeepSeek-V4-Pro": { id: "deepseek-ai/DeepSeek-V4-Pro", tool_call: true, reasoning: true }, + "MiniMaxAI/MiniMax-M2.7": { id: "MiniMaxAI/MiniMax-M2.7", tool_call: true, reasoning: true }, +}; const originalTogetherApiKey = process.env.TOGETHER_API_KEY; @@ -13,68 +36,92 @@ afterEach(() => { }); describe("Together models", () => { - it("registers the default Kimi K3 model via OpenAI-compatible Chat Completions API", () => { - const model = getModel("together", "moonshotai/Kimi-K3"); + it("generates Together metadata and reasoning controls independently of the live catalog", () => { + withModelGenerationFixture({ ...individualModelCatalog, together: { models: togetherModels } }, (fixture) => { + const result = fixture.generate(); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const catalog: Record>> = JSON.parse( + readFileSync(join(fixture.packageRoot, "src/providers/data/together.json"), "utf8"), + ); + const models = catalog["openai-completions"]; + const model = models["moonshotai/Kimi-K3"]; - expect(model).toBeDefined(); - expect(model.api).toBe("openai-completions"); - expect(model.provider).toBe("together"); - expect(model.baseUrl).toBe("https://api.together.ai/v1"); - expect(model.reasoning).toBe(true); - expect(model.thinkingLevelMap).toEqual({ minimal: null, low: null, medium: null }); - expect(model.input).toEqual(["text", "image"]); - expect(model.contextWindow).toBe(1048576); - expect(model.maxTokens).toBe(131072); - expect(model.cost).toEqual({ - input: 3, - output: 15, - cacheRead: 0.3, - cacheWrite: 0, - }); - expect(model.compat).toEqual({ - supportsStore: false, - supportsDeveloperRole: false, - supportsReasoningEffort: false, - maxTokensField: "max_tokens", - thinkingFormat: "together", - supportsStrictMode: false, - supportsLongCacheRetention: false, - }); - }); + expect(model).toBeDefined(); + expect(model.api).toBe("openai-completions"); + expect(model.provider).toBe("together"); + expect(model.baseUrl).toBe("https://api.together.ai/v1"); + expect(model.reasoning).toBe(true); + expect(model.thinkingLevelMap).toEqual({ minimal: null, low: null, medium: null }); + expect(model.input).toEqual(["text", "image"]); + expect(model.contextWindow).toBe(1048576); + expect(model.maxTokens).toBe(131072); + expect(model.cost).toEqual({ + input: 3, + output: 15, + cacheRead: 0.3, + cacheWrite: 0, + }); + expect(model.compat).toEqual({ + supportsStore: false, + supportsDeveloperRole: false, + supportsReasoningEffort: false, + maxTokensField: "max_tokens", + thinkingFormat: "together", + supportsStrictMode: false, + supportsLongCacheRetention: false, + }); + const gptOss = models["openai/gpt-oss-120b"]; + expect(gptOss.thinkingLevelMap).toEqual({ + off: null, + minimal: null, + low: "low", + medium: "medium", + high: "high", + max: null, + xhigh: null, + }); + expect(gptOss.compat).toMatchObject({ + supportsReasoningEffort: true, + thinkingFormat: "openai", + }); - it("models Together reasoning controls from the Together API surface", () => { - const gptOss = getModel("together", "openai/gpt-oss-120b"); - expect(gptOss.thinkingLevelMap).toEqual({ - off: null, - minimal: null, - low: "low", - medium: "medium", - high: "high", - max: null, - xhigh: null, - }); - expect(gptOss.compat).toMatchObject({ - supportsReasoningEffort: true, - thinkingFormat: "openai", - }); + const deepSeekV4 = models["deepseek-ai/DeepSeek-V4-Pro"]; + expect(deepSeekV4.thinkingLevelMap).toEqual({ + minimal: null, + low: null, + medium: null, + high: "high", + xhigh: null, + }); + expect(deepSeekV4.compat).toMatchObject({ + supportsReasoningEffort: true, + thinkingFormat: "together", + }); - const deepSeekV4 = getModel("together", "deepseek-ai/DeepSeek-V4-Pro"); - expect(deepSeekV4.thinkingLevelMap).toEqual({ - minimal: null, - low: null, - medium: null, - high: "high", - xhigh: null, - }); - expect(deepSeekV4.compat).toMatchObject({ - supportsReasoningEffort: true, - thinkingFormat: "together", + const minimax = models["MiniMaxAI/MiniMax-M2.7"]; + expect(minimax.thinkingLevelMap).toEqual({ off: null, minimal: null, low: null, medium: null }); + expect(minimax.compat?.thinkingFormat).toBeUndefined(); + expect(minimax.compat?.supportsReasoningEffort).toBe(false); }); + }); - const minimax = getModel("together", "MiniMaxAI/MiniMax-M2.7"); - expect(minimax.thinkingLevelMap).toEqual({ off: null, minimal: null, low: null, medium: null }); - expect(minimax.compat?.thinkingFormat).toBeUndefined(); - expect(minimax.compat?.supportsReasoningEffort).toBe(false); + it.each(["absent", "deprecated", "without tools"] as const)("omits a Together model that is %s", (state) => { + const models: Record = { ...togetherModels }; + const deepSeekId = "deepseek-ai/DeepSeek-V4-Pro"; + if (state === "absent") delete models[deepSeekId]; + else models[deepSeekId] = { ...togetherModels[deepSeekId], status: state, tool_call: state !== "without tools" }; + withModelGenerationFixture({ ...individualModelCatalog, together: { models } }, (fixture) => { + const result = fixture.generate(); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const catalog = JSON.parse( + readFileSync(join(fixture.packageRoot, "src/providers/data/together.json"), "utf8"), + ); + expect(catalog["openai-completions"][deepSeekId]).toBeUndefined(); + expect(catalog["openai-completions"]["openai/gpt-oss-120b"]).toMatchObject({ + provider: "together", + compat: { supportsReasoningEffort: true, thinkingFormat: "openai" }, + }); + }); }); it("resolves TOGETHER_API_KEY from the environment", () => { diff --git a/scripts/npm-lifecycle.mjs b/scripts/npm-lifecycle.mjs index 589f005dd..798f7fb13 100644 --- a/scripts/npm-lifecycle.mjs +++ b/scripts/npm-lifecycle.mjs @@ -92,6 +92,9 @@ let interrupt; let tarballRequests = 0; const server = createServer((request, reply) => { const path = decodeURIComponent(request.url); + // A stalled npm event loop can reuse a socket after the fixture's idle deadline. + // Each owned response closes its connection instead of racing pooled socket reuse. + reply.setHeader("connection", "close"); if (path === "/@diffexai/diffex") { const pkg = { ...metadata, @@ -102,20 +105,22 @@ const server = createServer((request, reply) => { .digest("base64")}`, }, }; + const body = JSON.stringify({ + name: metadata.name, + "dist-tags": { latest: metadata.version }, + versions: { [metadata.version]: pkg }, + }); reply.setHeader("content-type", "application/json"); - reply.end( - JSON.stringify({ - name: metadata.name, - "dist-tags": { latest: metadata.version }, - versions: { [metadata.version]: pkg }, - }), - ); + reply.setHeader("content-length", Buffer.byteLength(body)); + reply.end(body); } else if (path === "/diffex.tgz") { tarballRequests++; if (mode === "interrupt") { interrupt?.(); return; } + reply.setHeader("content-type", "application/octet-stream"); + reply.setHeader("content-length", bytes.length); reply.end(bytes); } else { reply.writeHead(404).end(); @@ -215,27 +220,72 @@ try { else { const child = spawn(process.execPath, ["--import", guard, executable, "update", "--self", "--force"], { ...options, + timeout: undefined, detached: true, - stdio: "ignore", + stdio: ["ignore", "inherit", "inherit"], + }); + const closed = new Promise((done, reject) => { + child.once("close", (code, signal) => done({ code, signal })); + child.once("error", reject); }); + function signalGroup(signal) { + if (!child.pid) return false; + try { + process.kill(-child.pid, signal); + return true; + } catch (error) { + if (error.code === "ESRCH") return false; + throw error; + } + } let interrupted = false; - interrupt = () => { - interrupted = true; - process.kill(-child.pid, "SIGTERM"); - }; - const deadline = setTimeout(() => process.kill(-child.pid, "SIGKILL"), 30000); + let cancellationFallbackTriggered = false; + let outcome; + let startupDeadline; + let cancellationDeadline; + const stopped = new Promise((done, reject) => { + closed.then(done, reject); + startupDeadline = setTimeout( + () => reject(new Error("Timed out before the real npm archive download")), + options.timeout, + ); + interrupt = () => { + if (interrupted) return; + try { + assert(signalGroup("SIGTERM"), "Update exited before archive cancellation"); + interrupted = true; + clearTimeout(startupDeadline); + // Bound cancellation after the observed download, independently of npm startup. + cancellationDeadline = setTimeout(() => { + try { + cancellationFallbackTriggered = true; + signalGroup("SIGKILL"); + } catch (error) { + reject(error); + } + }, 5000); + } catch (error) { + reject(error); + } + }; + }); try { - await new Promise((done, reject) => { - child.on("close", done); - child.on("error", reject); - }); + outcome = await stopped; + assert(interrupted, "Interrupted update must reach the real npm archive download"); } finally { - clearTimeout(deadline); + interrupt = undefined; + clearTimeout(startupDeadline); + clearTimeout(cancellationDeadline); + // The detached group belongs only to this fixture, including npm descendants. + signalGroup("SIGKILL"); + await closed.catch(() => {}); } - assert(interrupted, "Interrupted update must reach the real npm archive download"); evidence.commands.push({ command: "diffex update --self --force", - signal: "SIGTERM", + requestedSignal: "SIGTERM", + signal: outcome.signal, + exitCode: outcome.code, + cancellationFallbackTriggered, boundary: "npm archive download", }); }