diff --git a/bun.lock b/bun.lock index 46b47cd422..a48808397e 100644 --- a/bun.lock +++ b/bun.lock @@ -371,6 +371,7 @@ "@types/turndown": "5.0.5", "@types/yargs": "17.0.33", "@typescript/native-preview": "catalog:", + "ajv": "8.20.0", "drizzle-orm": "catalog:", "playwright-core": "1.59.1", "prettier": "3.6.2", diff --git a/packages/core/src/v1/session.ts b/packages/core/src/v1/session.ts index 34bb729683..21f26d173f 100644 --- a/packages/core/src/v1/session.ts +++ b/packages/core/src/v1/session.ts @@ -484,6 +484,9 @@ export const Assistant = Schema.Struct({ structured: Schema.optional(Schema.Any), variant: Schema.optional(Schema.String), finish: Schema.optional(Schema.String), + // altimate_change start — preserve committed-answer echo diagnostics across Effect message events + metadata: Schema.optional(Schema.Record(Schema.String, Schema.Unknown)), + // altimate_change end }).annotate({ identifier: "AssistantMessage" }) export type Assistant = Omit>, "error"> & { error?: AssistantError diff --git a/packages/opencode/package.json b/packages/opencode/package.json index 722248cde9..764cae76e0 100644 --- a/packages/opencode/package.json +++ b/packages/opencode/package.json @@ -48,6 +48,7 @@ "@types/turndown": "5.0.5", "@types/yargs": "17.0.33", "@typescript/native-preview": "catalog:", + "ajv": "8.20.0", "drizzle-orm": "catalog:", "playwright-core": "1.59.1", "prettier": "3.6.2", diff --git a/packages/opencode/src/acp/event.ts b/packages/opencode/src/acp/event.ts index 7d05fa6ee5..2b02ef6cb5 100644 --- a/packages/opencode/src/acp/event.ts +++ b/packages/opencode/src/acp/event.ts @@ -40,6 +40,9 @@ export class Subscription { private readonly abort = new AbortController() private readonly shellSnapshots = new Map() private readonly toolStarts = new Set() + // altimate_change start — ACP chunks cannot retract provisional compaction text + private readonly summaryMessages = new Set() + // altimate_change end private readonly permission: ACPPermission.Handler private started = false @@ -67,6 +70,15 @@ export class Subscription { async handle(event: Event) { switch (event.type) { + // altimate_change start — recovery clears summary before publishing its authoritative answer + case "message.updated": { + const message = event.properties.info + if (message.role !== "assistant") return + if (message.summary) this.summaryMessages.add(message.id) + else this.summaryMessages.delete(message.id) + return + } + // altimate_change end case "permission.asked": this.permission.handle(event) return @@ -169,6 +181,9 @@ export class Subscription { : await this.fetchPartMetadata(session.id, session.cwd, props.messageID, props.partID) if (metadata?.role !== "assistant") return if (metadata.partType === "text" && props.field === "text" && metadata.ignored !== true) { + // altimate_change start — internal summaries may be removed or replaced after streaming + if (this.summaryMessages.has(props.messageID)) return + // altimate_change end await this.input.connection.sessionUpdate({ sessionId: session.id, update: { @@ -218,6 +233,9 @@ export class Subscription { } private async recordFetchedPart(sessionId: string, message: SessionMessageResponse, part: Part) { + // altimate_change start — cover subscriptions that join after the initial message update + if (message.info.role === "assistant" && message.info.summary) this.summaryMessages.add(message.info.id) + // altimate_change end return await Effect.runPromise( this.input.session.recordPartMetadata({ sessionId, diff --git a/packages/opencode/src/agent/agent.ts b/packages/opencode/src/agent/agent.ts index 7219df198d..feea76341b 100644 --- a/packages/opencode/src/agent/agent.ts +++ b/packages/opencode/src/agent/agent.ts @@ -328,6 +328,7 @@ export const layer = Layer.effect( Permission.fromConfig({ "*": "deny", ...readOnlyDataPermission, + answer_commit: "allow", webfetch: "allow", websearch: "allow", question: "allow", diff --git a/packages/opencode/src/altimate/prompts/analyst.txt b/packages/opencode/src/altimate/prompts/analyst.txt index 71c9006ddd..83a38e519b 100644 --- a/packages/opencode/src/altimate/prompts/analyst.txt +++ b/packages/opencode/src/altimate/prompts/analyst.txt @@ -18,4 +18,6 @@ You are a trainable AI teammate. Check the "Teammate Training" section in your s If the user corrects your behavior, offer to save it as a rule using `training_save`. Use `training_list` to review learned knowledge. Skills: /teach, /train, /training-status. -Finish with a delimited answer: one or two sentences of basis (population, grain, caveats), then the final line `**Answer:** `. The final message is the deliverable: keep working notes in earlier turns, and never list candidates you excluded or alternative answers under other interpretations — pick one and state it. When the request asks for only the answer or a specific format, the final message is exactly that, with no basis. A top-N with a tie at the boundary still returns N concrete rows (note the tie briefly). If unresolved, use `**Answer:** unable to determine — `; never guess. +When the request supplies an output JSON schema (the `StructuredOutput` tool is available), finish with `StructuredOutput` instead of `answer_commit` or a plain-text reply; follow that schema. The text finalization rules below apply when no output schema is supplied. + +Finish by calling `answer_commit` with the single chosen value(s) with one field per item the request asks for, a one-or-two-sentence basis (population, grain, source object/column), or `unresolved` with the reason, then reply with exactly the text it returns and nothing else. Always supply `basis`, even when unresolved; it is never shown when the request asks for only the answer. Never list excluded candidates or alternative answers; a top-N with a tie at the boundary still returns N concrete rows (note the tie briefly in `rows.note`); when the request asks for only the answer or a specific format, set `answer_only`. If `answer_commit` is unavailable, finish with one or two sentences of basis and a final line `**Answer:** `, or `**Answer:** unable to determine — `; when the request asks for only the answer or a specific format, the final message is exactly the requested value(s) or table in the requested format, with no label and no basis (or `unable to determine — ` if unresolved); never guess. diff --git a/packages/opencode/src/altimate/run-budget.ts b/packages/opencode/src/altimate/run-budget.ts index 04f7f46697..59be163076 100644 --- a/packages/opencode/src/altimate/run-budget.ts +++ b/packages/opencode/src/altimate/run-budget.ts @@ -4,6 +4,13 @@ import z from "zod" const generations = new Map void>() +/** A harness deadline stops generation but may still deliver an already committed answer. */ +export class HarnessDeadline extends DOMException { + constructor() { + super("Run time budget hard deadline reached", "AbortError") + } +} + /** Register only the active provider request; disposing cannot remove a newer request. */ export function registerGeneration(sessionID: string, interrupt: () => void): () => void { generations.set(sessionID, interrupt) diff --git a/packages/opencode/src/altimate/tools/answer-commit.ts b/packages/opencode/src/altimate/tools/answer-commit.ts new file mode 100644 index 0000000000..ebf1127ae3 --- /dev/null +++ b/packages/opencode/src/altimate/tools/answer-commit.ts @@ -0,0 +1,287 @@ +import z from "zod" +import { Effect, Option, Schema, SchemaIssue } from "effect" +import type { JSONSchema7 } from "@ai-sdk/provider" +import { Tool } from "../../tool/tool" + +/** Maximum rendered answer length, including Markdown and basis, before the model echoes it. */ +export const MAX_RENDERED_ANSWER_CHARACTERS = 60_000 +export const MAX_BASIS_CHARACTERS = 400 + +const renderedLengthGuidance = `Rendered answer text must not exceed ${MAX_RENDERED_ANSWER_CHARACTERS.toLocaleString("en-US")} characters; aggregate, narrow the result, or write the full table to a file and commit the summary.` + +// Strict-schema callers fill unused optional properties with null, "", [] or {}. +// Only optional properties accept these fillers; required values still validate. +function optional(schema: T) { + return z + .union([schema, z.null(), z.literal(""), z.array(z.never()).length(0), z.object({}).strict()]) + .optional() + .transform((value) => { + if (value === null || value === undefined || (typeof value === "string" && !value.trim())) return undefined + if (Array.isArray(value) && value.length === 0) return undefined + if (typeof value === "object" && value !== null && Object.keys(value).length === 0) return undefined + return value as z.output + }) +} + +const number = z + .number() + .refine( + (value) => { + const digits = value.toExponential().split("e")[0]!.replace(/[-.]/g, "").length + return digits <= 15 && (!Number.isInteger(value) || Number.isSafeInteger(value)) + }, + "Numeric values must have at most 15 significant digits and integers must be safe; pass the exact value as a string", + ) + +const precisionRange = "precision must be an integer from 0 to 10 (inclusive)" +const precision = z.number({ error: precisionRange }).int(precisionRange).min(0, precisionRange).max(10, precisionRange) + +const field = z + .object({ + name: z.string().describe("Field label; use an empty name for an unnamed value"), + value: z + .union([z.string(), number], { error: "Each field value must be one non-empty string or number" }) + .refine((value) => typeof value !== "string" || value.trim().length > 0, "Field value must be non-empty"), + units: optional(z.string()), + precision: optional(precision).describe( + "Decimal places for numeric values: integer from 0 to 10 (inclusive); integers stay integers", + ), + }) + .strict() + +const rows = z + .object({ + columns: z.array(z.string()).min(1, "rows.columns requires at least one column"), + // altimate_change start - preserve the tabular format requested in text mode + format: optional(z.enum(["markdown", "csv", "tsv", "lines"])).describe( + "Tabular output format: markdown (default), csv, tsv, or lines. CSV/TSV include headers and quote delimiters, quotes, and newlines; null cells are empty. Lines omits headers and separates cells with one space for single-column or two-column plain lists.", + ), + // altimate_change end + values: z + .array(z.array(z.union([z.string(), number, z.null()]))) + .min(1, "rows.values requires at least one row") + .describe( + "Concrete result rows; when the request names N, the row count must equal N, even with a tie at the top-N boundary", + ), + precision: optional(z.record(z.string(), precision)).describe( + "Display decimal places by column name: integer from 0 to 10 (inclusive). Numeric cells use fixed decimals, including integers; strings and nulls are unchanged. Omitted columns keep their default formatting.", + ), + note: optional(z.string()).describe( + "Brief result note, including any tie at the top-N boundary; omitted when answer_only is true", + ), + }) + .strict() + .superRefine((rows, ctx) => { + if (rows.precision !== undefined && new Set(rows.columns).size !== rows.columns.length) { + ctx.addIssue({ + code: "custom", + path: ["precision"], + message: "rows.precision requires unique column labels", + }) + } + const unknown = Object.keys(rows.precision ?? {}).filter((key) => !rows.columns.includes(key)) + if (unknown.length) { + ctx.addIssue({ + code: "custom", + path: ["precision"], + message: `Unknown rows.precision keys: ${JSON.stringify(unknown)}; available columns: ${JSON.stringify(rows.columns)}`, + }) + } + rows.values.forEach((row, index) => { + if (row.length !== rows.columns.length) { + ctx.addIssue({ + code: "custom", + path: ["values", index], + message: `Row ${index} must have exactly ${rows.columns.length} cells; received ${row.length}`, + }) + } + }) + }) + +const basisRequired = `basis is always required: supply one or two sentences, at most ${MAX_BASIS_CHARACTERS} characters, naming population, grain, source object/column, and caveats` + +const parameters = z + .object({ + fields: optional( + z.array(field).superRefine((fields, ctx) => { + if (fields.length === 1) return + fields.forEach((field, index) => { + if (field.name !== "" && !field.name.trim()) { + ctx.addIssue({ + code: "custom", + path: [index, "name"], + message: "Field name must be non-empty", + }) + } + }) + }), + ).describe("One entry per item the request asks for, in the order asked."), + rows: optional(rows), + unresolved: optional( + z + .object({ + reason: z.string().refine((value) => value.trim().length > 0, "unresolved.reason must be non-empty"), + }) + .strict(), + ), + basis: z + .string({ error: basisRequired }) + .max(MAX_BASIS_CHARACTERS, `basis must contain at most ${MAX_BASIS_CHARACTERS} characters`) + .refine((value) => value.trim().length > 0, basisRequired) + .describe( + `basis is always required: one or two sentences, at most ${MAX_BASIS_CHARACTERS} characters, naming population, grain, the source object/column that supplies the requested entity or value, and caveats; never rendered with answer_only`, + ), + answer_only: optional(z.boolean()), + readings: optional( + z.array( + z + .object({ + clause: z.string(), + reading: z.string(), + result_summary: z.string(), + decided_by: z.string(), + }) + .strict(), + ), + ).describe("Advisory readings recorded only in metadata, never in the answer text"), + }) + .strict() + .refine( + (value) => [value.fields, value.rows, value.unresolved].filter((shape) => shape !== undefined).length === 1, + "Exactly one of fields, rows, or unresolved is required", + ) + +export type AnswerCommitInput = z.input + +// Tolerate boolean aliases at runtime while advertising the canonical boolean schema. +// Only answer_only is boolean; answer values and table cells must remain untouched. +const parser = z.preprocess((input) => { + if (!input || typeof input !== "object" || Array.isArray(input) || !("answer_only" in input)) return input + const value = typeof input.answer_only === "string" ? input.answer_only.toLowerCase() : input.answer_only + const answer_only = + value === "true" || value === 1 ? true : value === "false" || value === 0 ? false : input.answer_only + return { ...input, answer_only } +}, parameters) + +function validationError(error: z.ZodError): string { + const describe = (issues: z.core.$ZodIssue[], parent: PropertyKey[] = []): string[] => + issues.flatMap((issue) => { + const path = [...parent, ...issue.path] + // The first union member is the actual value schema; the others are fillers. + if (issue.code === "invalid_union" && issue.errors[0]?.length && !issue.message.includes("Each field value")) + return describe(issue.errors[0], path) + return [`${path.join(".") || "answer_commit"}: ${issue.message}`] + }) + const message = describe(error.issues).join("; ").replace(/\s+/g, " ") + const example = { + fields: [{ name: "", value: 2 }], + basis: "Counted all records at record grain using records.id; no exclusions.", + } + return `${message}\nMinimal valid example: \`${JSON.stringify(example)}\`` +} + +function tableCell(value: string | number | null): string { + return String(value) + .replaceAll("\\", "\\\\") + .replaceAll("|", "\\|") + .replace(/\r\n|\r|\n/g, " ") +} + +function limitRenderedAnswer(text: string): string { + if (text.length > MAX_RENDERED_ANSWER_CHARACTERS) throw new Error(renderedLengthGuidance) + return text +} + +/** Deterministically finalize a data-question answer without changing session state. */ +export function renderAnswer(payload: unknown): string { + const parsed = parser.safeParse(payload) + if (!parsed.success) throw new Error(validationError(parsed.error)) + const args = parsed.data + const basis = !args.answer_only && args.basis ? `${args.basis}\n\n` : "" + if (args.fields) { + const single = args.fields.length === 1 + const values = args.fields + .map((field) => { + const value = + typeof field.value === "number" && field.precision !== undefined && !Number.isInteger(field.value) + ? field.value.toFixed(field.precision) + : String(field.value) + const redundant = single && ["", "answer", "result", "value"].includes(field.name.trim().toLowerCase()) + const name = !args.answer_only && field.name && !redundant ? `${field.name}: ` : "" + return `${name}${value}${field.units ? ` ${field.units}` : ""}` + }) + .join("; ") + return limitRenderedAnswer(args.answer_only ? values : `${basis}**Answer:** ${values}`) + } + if (args.rows) { + // altimate_change start - render the requested format before applying the shared size cap + const { columns, values, precision, format = "markdown" } = args.rows + const row = (cells: (string | number | null)[]) => { + if (format === "markdown") return `| ${cells.map(tableCell).join(" | ")} |` + if (format === "lines") return cells.map(String).join(" ") + const delimiter = format === "csv" ? "," : "\t" + return cells + .map((value) => { + const cell = value === null ? "" : String(value) + return cell.includes(delimiter) || /["\r\n]/.test(cell) ? `"${cell.replaceAll('"', '""')}"` : cell + }) + .join(delimiter) + } + const table = [ + ...(format === "lines" ? [] : [row(columns)]), + ...(format === "markdown" ? [`|${columns.map(() => "---").join("|")}|`] : []), + ...values.map((cells) => + row( + cells.map((value, index) => { + const places = + precision && Object.hasOwn(precision, columns[index]!) ? precision[columns[index]!] : undefined + return typeof value === "number" && places !== undefined ? value.toFixed(places) : value + }), + ), + ), + ].join("\n") + // altimate_change end + return limitRenderedAnswer( + args.answer_only ? table : `${basis}**Answer:**\n${table}${args.rows.note ? `\n${args.rows.note}` : ""}`, + ) + } + const unresolved = `unable to determine — ${args.unresolved!.reason}` + return limitRenderedAnswer(args.answer_only ? unresolved : `${basis}**Answer:** ${unresolved}`) +} + +// Keep the renderer and native Effect tool on the same parser, including filler normalization. +const Parameters = Schema.declareConstructor, AnswerCommitInput>()([], () => (input) => { + const parsed = parser.safeParse(input) + return parsed.success + ? Effect.succeed(parsed.data) + : Effect.fail(new SchemaIssue.InvalidValue(Option.some(input), { message: validationError(parsed.error) })) +}) + +export const AnswerCommitTool = Tool.define( + "answer_commit", + Effect.succeed({ + // altimate_change start - expose format selection to the model before the terminal tool call + description: `Deterministically finalize a data-question answer. Call this as the LAST tool call, once the answer is decided and all queries are finished. After it returns, reply with exactly the returned text and nothing else. Supply exactly one of fields, rows, or unresolved. Supply one field per item the request asks for, in the order asked: a request for 'the repository and the most copied file' needs two fields (one named for the repository, one for the file); never collapse a compound request into one field, and never add fields the request did not ask for. Numbers are accepted up to 15 significant digits; any decimal that must be reproduced exactly (more digits, trailing zeros, or currency strings) must be passed as a string. For fields.precision and each rows.precision value, use an integer from 0 to 10 (inclusive). Set rows.precision by column name to display numeric cells with fixed decimal places, including integers; strings are never changed. When the request names a tabular format, set rows.format accordingly: markdown (default), csv, tsv, or lines (one row per line with cells separated by a single space, without headers, for single-column or two-column plain lists). CSV/TSV include headers and quote cells containing the delimiter, quotes, or newlines, doubling embedded quotes; null cells are empty. Always supply \`basis\`; it is never shown when the request asks for only the answer. Supply one or two sentences, at most ${MAX_BASIS_CHARACTERS} characters, naming population, grain, the source object/column that supplies the requested entity or value, and caveats. Resolve policy eligibility separately from metric and entity attribution: a rule about a related object does not change which object supplies the requested entity. Say in \`basis\` which source object and column supply it. When the request names N, rows.values must contain exactly N concrete rows, even with a tie at the top-N boundary; note the tie briefly in rows.note. Use answer_only when the request asks for only the answer or a specific format: the reply is exactly the value(s) or table, with no basis, field names, note, or answer label. An unresolved answer retains its unable-to-determine status. Omit unused optional properties. ${renderedLengthGuidance} This tool is not available for JSON-schema requests; use StructuredOutput instead.`, + // altimate_change end + parameters: Parameters, + formatValidationError: (error: unknown) => (Schema.isSchemaError(error) ? error.message : String(error)), + // Preserve the existing wire schema; Zod and the SDK use different recursive JSON Schema types. + // oxlint-disable-next-line typescript-eslint/no-unsafe-type-assertion + jsonSchema: z.toJSONSchema(parameters, { io: "input" }) as JSONSchema7, + execute: (args: Schema.Schema.Type, ctx: Tool.Context) => + Effect.gen(function* () { + if (ctx.agent !== "analyst") + return yield* Effect.die(new Error("answer_commit is only available to the analyst agent")) + yield* ctx.ask({ permission: "answer_commit", patterns: ["*"], always: ["*"], metadata: {} }) + const output = renderAnswer(args) + return { + title: "Answer committed", + // The processor caps model-facing output but preserves metadata for the final echo. + metadata: { readings: args.readings ?? [], truncated: false, committed_text: output }, + output, + } + }), + }), +) + +export * as AnswerCommit from "./answer-commit" diff --git a/packages/opencode/src/cli/cmd/run-accounting.ts b/packages/opencode/src/cli/cmd/run-accounting.ts index 01e6386811..ecd915bcb3 100644 --- a/packages/opencode/src/cli/cmd/run-accounting.ts +++ b/packages/opencode/src/cli/cmd/run-accounting.ts @@ -33,6 +33,41 @@ export type Termination = { // Timeout classification for why_harness_stopped="timeout" and retry decisions. const TIMEOUT_PATTERN = /\btimed?\s*out\b|\bETIMEDOUT\b|TimeoutError/i +/** Request ownership from the ordered message/request event stream. */ +export function createRequestOwnership() { + let activeRequestID: string | undefined + const users = new Map() + const assistants = new Map() + const seenRequests = new Set() + return { + get activeRequestID() { + return activeRequestID + }, + beginRequest(messageID: string) { + if (seenRequests.has(messageID)) return false + seenRequests.add(messageID) + users.set(messageID, messageID) + activeRequestID = messageID + return true + }, + onUserMessage(messageID: string, parentID?: string) { + // Internal rows have no parentID in their message schema. The prompt loop + // reports their origin separately, before creating the next assistant. + // A persisted real request always retains its own boundary. + if (seenRequests.has(messageID)) return + if (parentID !== undefined) users.set(messageID, users.get(parentID)) + else if (!users.has(messageID)) users.set(messageID, activeRequestID) + }, + onAssistantMessage(info: { id: string; parentID: string }) { + if (assistants.has(info.id)) return + assistants.set(info.id, users.get(info.parentID)) + }, + ownerOf(messageID: string) { + return assistants.get(messageID) + }, + } +} + /** Holds only overflow errors whose trace status depends on later recovery. */ export function createRecoverableOverflowTraceErrors() { let pending: string[] = [] @@ -57,6 +92,9 @@ export function createRecoverableOverflowTraceErrors() { export function create() { const summaryMessages = new Set() let turnCount = 0 + // altimate_change start — track the active request's committed echo turn allowance + let committedAnswer = false + // altimate_change end let lastFinishReason: string | undefined // altimate_change start — upstream_fix: onText and onStepFinish are // independently overwritten by whichever message last emitted a text/finish @@ -111,6 +149,17 @@ export function create() { } return { + // altimate_change start — only a successful commit exempts the request's echo from its turn budget + resetCommittedAnswer() { + committedAnswer = false + }, + onAnswerCommit(status: "completed" | "error") { + committedAnswer = status === "completed" + }, + get committedAnswer() { + return committedAnswer + }, + // altimate_change end /** Record whether an assistant message is actual compaction machinery. */ onAssistantMessage(info: { id: string; agent?: string; summary?: boolean }) { if (info.summary === true) summaryMessages.add(info.id) diff --git a/packages/opencode/src/cli/cmd/run-output.ts b/packages/opencode/src/cli/cmd/run-output.ts index f0fada88b8..8aef8481c4 100644 --- a/packages/opencode/src/cli/cmd/run-output.ts +++ b/packages/opencode/src/cli/cmd/run-output.ts @@ -1,4 +1,4 @@ -// altimate_change — what `altimate-code run --output ` writes. +// altimate_change start — what `altimate-code run --output ` writes. // // The file is the answer, not the transcript. A model often narrates between tool // calls ("Mixed time formats. Let me check all formats.") and that text is superseded @@ -14,10 +14,12 @@ export const NO_TEXT_OUTPUT = "(no text output — tool-only response)" // token when the deliverable is already complete; that acknowledgement is not a new reply. const ACKNOWLEDGEMENT = new RegExp(`^${DONE_TOKEN}\\.?$`, "i") +type CollectedText = { text: string; partID?: string } + export class OutputCollector { - private readonly all: string[] = [] - private final: string[] = [] - private previous: string[] = [] + private all: CollectedText[] = [] + private final: CollectedText[] = [] + private previous: CollectedText[] = [] /** A new model step starts; text before it is no longer the reply. Compaction steps do not count. */ stepStart(compaction = false) { @@ -27,16 +29,24 @@ export class OutputCollector { } /** Completed assistant text. Synthetic/compaction text is kept for the fallback only. */ - text(text: string, opts: { synthetic?: boolean; compaction?: boolean } = {}) { + text(text: string, opts: { synthetic?: boolean; compaction?: boolean; partID?: string } = {}) { const trimmed = text.trim() if (!trimmed) return - this.all.push(trimmed) + const part = { text: trimmed, partID: opts.partID } + this.all.push(part) if (opts.synthetic || opts.compaction) return if (ACKNOWLEDGEMENT.test(trimmed) && this.final.length === 0) { this.final = this.previous return } - this.final.push(trimmed) + this.final.push(part) + } + + /** Removed text must disappear from the reply and every fallback. */ + remove(partID: string) { + this.all = this.all.filter((part) => part.partID !== partID) + this.final = this.final.filter((part) => part.partID !== partID) + this.previous = this.previous.filter((part) => part.partID !== partID) } private readonly notes: string[] = [] @@ -51,7 +61,12 @@ export class OutputCollector { * an answer and would read as one); otherwise everything that was said. */ content(): string { - const chosen = this.final.length ? this.final : this.notes.length ? this.notes : this.all + const chosen = this.final.length + ? this.final.map((part) => part.text) + : this.notes.length + ? this.notes + : this.all.map((part) => part.text) return chosen.join("\n\n") || NO_TEXT_OUTPUT } } +// altimate_change end diff --git a/packages/opencode/src/cli/cmd/run.ts b/packages/opencode/src/cli/cmd/run.ts index 0ec53126dc..db9dd78fd2 100644 --- a/packages/opencode/src/cli/cmd/run.ts +++ b/packages/opencode/src/cli/cmd/run.ts @@ -407,7 +407,7 @@ export const RunCommand = cmd({ // altimate_change start — budget limits for CI/enterprise governance .option("max-turns", { type: "number", - describe: "maximum assistant work turns; one best-effort finalization generation may follow the limit", + describe: "maximum assistant work turns; allows one committed-answer echo, or one best-effort reply after a budget abort", }) // altimate_change start — wall-clock work budget, with a separate finalization allowance .option("time-budget", { @@ -729,6 +729,15 @@ You are speaking to a non-technical business executive. Follow these rules stric // altimate_change start — turn accounting + dual-attribution // termination state for this run (see run-accounting.ts). const accounting = RunAccounting.create() + const requestOwnership = RunAccounting.createRequestOwnership() + // A queued request must not revoke the in-flight echo owned by an earlier request. + const committedRequests = new Set() + const beginRequest = (messageID: string) => { + // The persisted request boundary includes requests from other attached clients. + // Retries retain their ID; internal synthetic/compaction rows do not emit this event. + if (!requestOwnership.beginRequest(messageID)) return + accounting.resetCommittedAnswer() + } // altimate_change end // altimate_change start — idle-done fallback state (run-mode-only by // construction — this exists only in the run command). Thresholds are @@ -845,12 +854,17 @@ You are speaking to a non-technical business executive. Follow these rules stric const abortForTimeBudget = () => { if (!sessionID || timeAbortRequested) return timeAbortRequested = true - timeBudgetAbort = sdk.session.abort({ sessionID }, { signal: AbortSignal.timeout(5_000) }).then((result) => { + // altimate_change start — distinguish harness deadlines from user cancellation on the server + timeBudgetAbort = sdk.session.abort( + { sessionID }, + { signal: AbortSignal.timeout(5_000), headers: { "x-altimate-run-budget": "hard-deadline" } }, + ).then((result) => { if (result.error) throw result.error }).catch((e) => { accounting.onSessionError("TimeBudgetAbortError", String(e)) eventAbort.abort() }) + // altimate_change end } const checkTimeBudget = () => { if (!sessionID || !workActive || timeExhausted()) return @@ -885,12 +899,15 @@ You are speaking to a non-technical business executive. Follow these rules stric // altimate_change start — save the reminder then interrupt only the active model request // The server applies the effort override before saving this user turn, // and defers interruption until executing tools have kept their results. + const messageID = MessageID.ascending() + beginRequest(messageID) // altimate_change end timeBudgetReminder = sdk.session // altimate_change start — carries soft control to both local and attached servers .prompt( { sessionID, + messageID, noReply: true, agent, model: args.model ? Provider.parseModel(args.model) : undefined, @@ -947,6 +964,25 @@ You are speaking to a non-technical business executive. Follow these rules stric if (options?.signal?.aborted) break await runTracing?.observe(event) if (options?.signal?.aborted) break + // altimate_change start — preserve request boundaries and internal continuation ownership on attached streams + const requestEvent = event as { + type: string + properties: { sessionID?: string; messageID?: string; parentID?: string } + } + if ( + requestEvent.type === "session.request.started" && + requestEvent.properties.sessionID === sessionID && + requestEvent.properties.messageID + ) + beginRequest(requestEvent.properties.messageID) + if ( + requestEvent.type === "session.request.continued" && + requestEvent.properties.sessionID === sessionID && + requestEvent.properties.messageID && + requestEvent.properties.parentID + ) + requestOwnership.onUserMessage(requestEvent.properties.messageID, requestEvent.properties.parentID) + // altimate_change end // altimate_change start — shared SDKs omit fork-only run budget events const budgetEvent = event as { type: string; properties: { sessionID?: string; timestamp?: number } } if (budgetEvent.properties.sessionID === sessionID && typeof budgetEvent.properties.timestamp === "number") { @@ -964,12 +1000,24 @@ You are speaking to a non-technical business executive. Follow these rules stric // step-start parts (which carry only messageID/sessionID) can be attributed. // The assistant message row is persisted — and this event published — before // its first step-start part streams, so the lookup is populated in time. + if ( + event.type === "message.updated" && + event.properties.info.role === "user" && + event.properties.info.sessionID === sessionID + ) { + // This provisional owner is replaced by the persisted request boundary + // or the internal continuation's explicit origin before its assistant starts. + requestOwnership.onUserMessage(event.properties.info.id) + } if ( event.type === "message.updated" && event.properties.info.role === "assistant" && event.properties.info.sessionID === sessionID ) { accounting.onAssistantMessage(event.properties.info) + // Keep the request that owned the assistant's creation, even if an older + // tool finishes after another attached client persists a new request. + requestOwnership.onAssistantMessage(event.properties.info) assistantStarted = true // Enrich only from this run's assistant, independently of presentation. const info = event.properties.info @@ -996,12 +1044,29 @@ You are speaking to a non-technical business executive. Follow these rules stric toggles.set("start", true) } + // altimate_change start — discarded streamed text must not survive in --output fallbacks + if (event.type === "message.part.removed" && event.properties.sessionID === sessionID) { + outputParts.remove(event.properties.partID) + } + // altimate_change end if (event.type === "message.part.updated") { const part = event.properties.part if (part.sessionID !== sessionID) continue // altimate_change start — tool steps cannot satisfy the final-answer check if (part.type === "tool" && !accounting.isCompactionStep(part.messageID)) lastToolStep = step + if ( + part.type === "tool" && part.tool === "answer_commit" && + (part.state.status === "completed" || part.state.status === "error") + ) { + const requestID = requestOwnership.ownerOf(part.messageID) + if (requestID !== undefined) { + // Success and failed retries affect only the request that owns this tool. + if (part.state.status === "completed") committedRequests.add(requestID) + else committedRequests.delete(requestID) + if (requestID === requestOwnership.activeRequestID) accounting.onAnswerCommit(part.state.status) + } + } // altimate_change end // altimate_change start — feed every part event through the @@ -1055,9 +1120,11 @@ You are speaking to a non-technical business executive. Follow these rules stric // the owning message's agent is resolved via the message.updated lookup // above, so compacting models are not differentially charged turns. const counted = accounting.onStepStart(part.messageID) + const requestID = requestOwnership.ownerOf(part.messageID) + const committedAnswer = requestID !== undefined && committedRequests.has(requestID) // altimate_change start — the time-budget abort already owns the interrupted work phase if ( - counted && maxTurns && accounting.turnCount > maxTurns && !budgetAborted && + counted && maxTurns && accounting.turnCount > maxTurns && !committedAnswer && !budgetAborted && (!timeExhausted() || options?.turnLimit !== undefined) ) { stopTimeBudget() @@ -1131,6 +1198,7 @@ You are speaking to a non-technical business executive. Follow these rules stric const text = part.text.trim() if (args.output && text) { outputParts.text(text, { + partID: part.id, synthetic: part.synthetic === true, compaction: accounting.isCompactionStep(part.messageID), }) @@ -1529,6 +1597,7 @@ You are speaking to a non-technical business executive. Follow these rules stric // server whether that message landed, and only re-send when it did not. const sendMessageID = MessageID.ascending() const send = () => { + beginRequest(sendMessageID) // altimate_change start — start once at prompt dispatch, never reset on retry or child work startWorkClock() // altimate_change end @@ -1648,6 +1717,7 @@ You are speaking to a non-technical business executive. Follow these rules stric }) let streamFailed = false const messageID = MessageID.ascending() + beginRequest(messageID) const promptPromise = (async (): Promise => { // The previous turn may have published idle just before releasing its // session lock. Retry that narrow race, but only after definitive diff --git a/packages/opencode/src/mcp/index.ts b/packages/opencode/src/mcp/index.ts index 5efa2c0538..da8e75a1ce 100644 --- a/packages/opencode/src/mcp/index.ts +++ b/packages/opencode/src/mcp/index.ts @@ -47,6 +47,9 @@ import { InstanceState } from "@/effect/instance-state" import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process" import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner" import { McpCatalog } from "./catalog" +// altimate_change start — reserved tool IDs must use the permission matcher's platform semantics. +import { Wildcard } from "@/util/wildcard" +// altimate_change end // altimate_change start — persist enabled flag + telemetry import { findAllConfigPaths, listMcpInConfig, addMcpToConfig, readMcpEntryFromDisk } from "./config" import { Global } from "@opencode-ai/core/global" @@ -1246,6 +1249,10 @@ export const layer = Layer.effect( const timeout = requestTimeout(s, clientName, mcpConfig, defaultTimeout) for (const mcpTool of listed) { const key = McpCatalog.sanitize(clientName) + "_" + McpCatalog.sanitize(mcpTool.name) + // altimate_change start — reserve the native finalizer's permission and echo identity, + // including when ALTIMATE_ANSWER_COMMIT=0 disables the native tool. + if (Wildcard.match(key, "answer_commit")) continue + // altimate_change end // attach the original client name for source classification downstream. result[key] = Object.assign(McpCatalog.convertTool(mcpTool, client, timeout), { client: clientName }) } diff --git a/packages/opencode/src/provider/transform.ts b/packages/opencode/src/provider/transform.ts index cf0d48b209..ec7b0407bb 100644 --- a/packages/opencode/src/provider/transform.ts +++ b/packages/opencode/src/provider/transform.ts @@ -1526,6 +1526,13 @@ export namespace ProviderTransform { // altimate_change end export function schema(model: Provider.Model, schema: JSONSchema.BaseSchema | JSONSchema7): JSONSchema7 { + // altimate_change start — omit the tool schema dialect declaration for Anthropic, Bedrock, and Vertex + if (["@ai-sdk/anthropic", "@ai-sdk/amazon-bedrock", "@ai-sdk/google-vertex/anthropic"].includes(model.api.npm)) { + const { $schema, ...rest } = schema + schema = rest + } + // altimate_change end + /* if (["openai", "azure"].includes(providerID)) { if (schema.type === "object" && schema.properties) { diff --git a/packages/opencode/src/server/routes/session.ts b/packages/opencode/src/server/routes/session.ts index dcb9fe93e0..10584cded3 100644 --- a/packages/opencode/src/server/routes/session.ts +++ b/packages/opencode/src/server/routes/session.ts @@ -22,6 +22,7 @@ import { TraceContext } from "@/altimate/observability/trace-context" // altimate_change end // altimate_change start — run-scoped request interruption and effort overrides import { withBudgetControl } from "@/altimate/session/run-budget-control" +import { RunBudget } from "@/altimate/run-budget" // altimate_change end // altimate_change start — upstream_fix: bridge merge wired routes to Effect-TS Permission, but the runtime // asks come from PermissionNext. Use PermissionNext here so this deprecated reply route hits the same @@ -400,7 +401,12 @@ export const SessionRoutes = lazy(() => }), ), async (c) => { - await SessionPrompt.cancel(c.req.valid("param").sessionID) + // altimate_change start — carry harness deadline provenance through the SDK abort endpoint + await SessionPrompt.cancel( + c.req.valid("param").sessionID, + c.req.header("x-altimate-run-budget") === "hard-deadline" ? new RunBudget.HarnessDeadline() : undefined, + ) + // altimate_change end return c.json(true) }, ) diff --git a/packages/opencode/src/session/compaction.ts b/packages/opencode/src/session/compaction.ts index 3388f8d034..d2f9d05cf7 100644 --- a/packages/opencode/src/session/compaction.ts +++ b/packages/opencode/src/session/compaction.ts @@ -1288,6 +1288,10 @@ export namespace SessionCompaction { abort: AbortSignal auto: boolean overflow?: boolean + // altimate_change start - let the prompt loop retain request-scoped answers across compaction + onContinuation?: (messageID: MessageID) => void + recoverCommittedAnswer?: boolean + // altimate_change end // altimate_change start — keep nudge delivery scoped to the active prompt generation nudgeGeneration?: NudgeArbiter.Generation // altimate_change end @@ -1441,6 +1445,9 @@ export namespace SessionCompaction { // altimate_change start — keep nudge delivery scoped to this prompt generation nudgeGeneration: input.nudgeGeneration, // altimate_change end + // altimate_change start - the prompt owns the current request's committed answer + recoverCommittedAnswer: input.recoverCommittedAnswer, + // altimate_change end }) // Allow plugins to inject context or replace compaction prompt const compacting = await Plugin.trigger( @@ -1589,7 +1596,8 @@ When constructing the summary, try to stick to this template: // A "continue" result was previously committed regardless of whether the // summary step produced any text — an empty summary erases history (the // post-compaction amnesia signature). Guard the commit: retry ONCE with - // identical input, then mark the summary message as errored and stop. + // identical input, then stop with a recoverable cause for a committed answer + // or mark the summary message as errored when there is no committed answer. const summaryHasText = () => MessageV2.get({ sessionID: input.sessionID, messageID: msg.id }).parts.some( (part) => part.type === "text" && part.text.trim().length > 0, @@ -1599,16 +1607,31 @@ When constructing the summary, try to stick to this template: log.warn("compaction summary empty, retrying once", { sessionID: input.sessionID }) result = await processor.process(summarizerInput) if (result === "continue" && !summaryHasText()) { - processor.message.error = new NamedError.Unknown({ + const error = new NamedError.Unknown({ message: "Compaction summarizer produced no summary text after retry", }).toObject() - processor.message.finish = "error" + if (input.recoverCommittedAnswer && !input.abort.aborted) { + processor.message.metadata = { + ...processor.message.metadata, + answer_commit_echo_error: error, + } + } else { + processor.message.error = error + processor.message.finish = "error" + } await Session.updateMessage(processor.message) result = "stop" } } // altimate_change end + // altimate_change start - exit without a fatal summary or continuation before the prompt publishes the answer + if (input.recoverCommittedAnswer && processor.message.metadata?.answer_commit_echo_error) { + compactionAttempts.delete(input.sessionID) + return "stop" + } + // altimate_change end + if (result === "compact") { processor.message.error = new MessageV2.ContextOverflowError({ message: replay @@ -1697,6 +1720,9 @@ When constructing the summary, try to stick to this template: }) } // altimate_change end + // altimate_change start - report the persisted replay's identity, independent of its part flags + input.onContinuation?.(replayMsg.id) + // altimate_change end } else { // altimate_change start — the continue message // carries the original format/tools/system/variant, exactly as the replay @@ -1783,6 +1809,9 @@ When constructing the summary, try to stick to this template: end: Date.now(), }, }) + // altimate_change start - report the persisted continuation's identity + input.onContinuation?.(continueMsg.id) + // altimate_change end } } if (processor.message.error) { diff --git a/packages/opencode/src/session/message-v2.ts b/packages/opencode/src/session/message-v2.ts index aea9f5f6f7..7fc1f336a1 100644 --- a/packages/opencode/src/session/message-v2.ts +++ b/packages/opencode/src/session/message-v2.ts @@ -27,6 +27,79 @@ interface FetchDecompressionError extends Error { path: string } +// altimate_change start — keep committed-echo parsing helpers private at module scope +const echoNetworkCodes = new Set([ + "ECONNRESET", + "ECONNREFUSED", + "ETIMEDOUT", + "ESOCKETTIMEDOUT", + "EPIPE", + "ENOTFOUND", + "EAI_AGAIN", + "UND_ERR_SOCKET", + "UND_ERR_CONNECT_TIMEOUT", + "UND_ERR_HEADERS_TIMEOUT", + "UND_ERR_BODY_TIMEOUT", +]) + +const echoPolicyCodes = new Set([ + "policy_violation", + "content_filter", + "content_policy", + "content_policy_violation", + "guardrail_intervened", +]) + +function isEchoPolicyCode(code: string | undefined): boolean { + if (!code) return false + return echoPolicyCodes.has(code.toLowerCase()) || /^(content_policy.*|(safety|moderation)_.+)$/i.test(code) +} + +function responseErrorCode(responseBody: string | undefined, matches: (code: string) => boolean): string | undefined { + if (!responseBody) return undefined + const find = (value: unknown): string | undefined => { + if (!value || typeof value !== "object") return undefined + for (const [key, child] of Object.entries(value)) { + if (key === "code" && typeof child === "string" && matches(child)) return child + const nested = find(child) + if (nested) return nested + } + } + try { + return find(JSON.parse(responseBody)) + } catch { + return undefined + } +} + +function echoPolicyCode(responseBody: string | undefined): string | undefined { + return responseErrorCode(responseBody, isEchoPolicyCode) +} + +/** + * The provider's own error code: `error.code`, the root `code`, or the upstream code that the + * LiteLLM / Altimate Base gateway wraps under `error.provider_specific_fields.error.code`. + * Arbitrary nested diagnostic fields are never consulted. + */ +function providerErrorCode(responseBody: string | undefined): string | undefined { + if (!responseBody) return undefined + try { + const body = JSON.parse(responseBody) as { + code?: unknown + error?: { code?: unknown; provider_specific_fields?: { error?: { code?: unknown } } } + } + const candidates = [body?.error?.provider_specific_fields?.error?.code, body?.error?.code, body?.code] + return candidates.find((code): code is string => typeof code === "string") + } catch { + return undefined + } +} + +function isOverflowCode(code: string | undefined): boolean { + return !!code && ["context_length_exceeded", "content_length_exceeded"].includes(code.toLowerCase()) +} +// altimate_change end + export namespace MessageV2 { export function isMedia(mime: string) { return mime.startsWith("image/") || mime === "application/pdf" @@ -511,6 +584,9 @@ export namespace MessageV2 { structured: z.any().optional(), variant: z.string().optional(), finish: z.string().optional(), + // altimate_change start — retain recovered committed-answer echo failures on the assistant message + metadata: z.record(z.string(), z.unknown()).optional(), + // altimate_change end }).meta({ ref: "AssistantMessage", }) @@ -1143,10 +1219,14 @@ export namespace MessageV2 { }, { cause: e }, ).toObject() - case (e as SystemError)?.code === "ECONNRESET": + // altimate_change start — retain structured network codes for echo recovery + case echoNetworkCodes.has((e as SystemError)?.code ?? ""): + // altimate_change end return new MessageV2.APIError( { - message: "Connection reset by server", + // altimate_change start — preserve the established reset message and other transport details + message: (e as SystemError).code === "ECONNRESET" ? "Connection reset by server" : errorMessage(e), + // altimate_change end isRetryable: true, metadata: { code: (e as SystemError).code ?? "", @@ -1202,7 +1282,25 @@ export namespace MessageV2 { providerID: ctx.providerID, error: e, }) - if (parsed.type === "context_overflow") { + // altimate_change start — structured policy decisions override overflow text, while explicit overflow codes win + // Only the provider's own code decides overflow; a nested diagnostic code must not mask a policy denial. + // An explicit policy code also beats a 413 status: a policy denial must stay fatal, never a recoverable overflow. + const overflowCode = isOverflowCode(providerErrorCode(e.responseBody)) + const policyCode = echoPolicyCode(e.responseBody) + if (policyCode && !overflowCode) { + return new MessageV2.APIError( + { + message: parsed.message, + statusCode: e.statusCode, + isRetryable: parsed.type === "api_error" ? parsed.isRetryable : e.isRetryable, + responseHeaders: parsed.type === "api_error" ? parsed.responseHeaders : e.responseHeaders, + responseBody: parsed.responseBody, + metadata: { ...(parsed.type === "api_error" ? parsed.metadata : undefined), code: policyCode }, + }, + { cause: e }, + ).toObject() + } + if (parsed.type === "context_overflow" || overflowCode) { return new MessageV2.ContextOverflowError( { message: parsed.message, @@ -1211,6 +1309,7 @@ export namespace MessageV2 { { cause: e }, ).toObject() } + // altimate_change end return new MessageV2.APIError( { @@ -1219,7 +1318,11 @@ export namespace MessageV2 { isRetryable: parsed.isRetryable, responseHeaders: parsed.responseHeaders, responseBody: parsed.responseBody, - metadata: parsed.metadata, + // altimate_change start — SDK-wrapped socket failures retain their structural transport code + metadata: echoNetworkCodes.has((e.cause as SystemError)?.code ?? "") + ? { ...parsed.metadata, code: (e.cause as SystemError).code ?? "" } + : parsed.metadata, + // altimate_change end }, { cause: e }, ).toObject() @@ -1262,3 +1365,22 @@ export namespace MessageV2 { } } } + +// altimate_change start — recover committed echoes only from transient transport failures +export function isEchoTransportError(error: MessageV2.Assistant["error"]): boolean { + if (error?.name !== "APIError") return false + const { statusCode, responseBody, metadata, isRetryable } = error.data + // Provider decisions can be wrapped by a gateway with a transient HTTP status. + // Inspect structured codes, never prose in a message or response body. + if (echoPolicyCode(responseBody) || isEchoPolicyCode(metadata?.code)) return false + if (statusCode !== undefined) + return (statusCode >= 500 && statusCode < 600) || statusCode === 429 || statusCode === 408 + const code = metadata?.code + return ( + (code !== undefined && + (echoNetworkCodes.has(code) || + ["ZlibError", "ProviderHeaderTimeoutError", "ProviderStreamIdleTimeoutError"].includes(code))) || + (isRetryable && metadata?.url !== undefined && !responseBody) + ) +} +// altimate_change end diff --git a/packages/opencode/src/session/processor.ts b/packages/opencode/src/session/processor.ts index 0c993ae542..516021239d 100644 --- a/packages/opencode/src/session/processor.ts +++ b/packages/opencode/src/session/processor.ts @@ -1,4 +1,6 @@ -import { MessageV2 } from "./message-v2" +// altimate_change start — module-scope echo transport predicate +import { MessageV2, isEchoTransportError } from "./message-v2" +// altimate_change end import { Log } from "@/util/log" // altimate_change start — client trace propagation import { TraceContext } from "@/altimate/observability/trace-context" @@ -260,6 +262,9 @@ export namespace SessionProcessor { // altimate_change start — keep nudge delivery scoped to the active prompt generation nudgeGeneration?: NudgeArbiter.Generation // altimate_change end + // altimate_change start - recover summarizer failures only when the prompt retains this request's committed answer + recoverCommittedAnswer?: boolean + // altimate_change end }) { // altimate_change start — Map (not plain object) so adversarial ids can // never resolve to inherited Object.prototype members. @@ -337,9 +342,14 @@ export namespace SessionProcessor { return id === undefined ? undefined : toolcalls.get(id) }, // altimate_change end - async process(streamInput: LLM.StreamInput) { + // altimate_change start - opt-in text buffering for committed-answer echoes + async process({ deferText, ...streamInput }: LLM.StreamInput & { deferText?: MessageV2.TextPart[] }) { + // altimate_change end log.info("process") needsCompaction = false + // altimate_change start - let the prompt's exit guard publish a recovered compaction answer + let recoveredCommittedAnswer = false + // altimate_change end // altimate_change start — resolve breaker config + arm state once per step. // ANNOTATE-ONLY by default (mode "annotate"): directives and the hard stop // require mode "armed" AND run mode. Skipped entirely for plan/review-class @@ -1106,7 +1116,9 @@ export namespace SessionProcessor { tokens: usage.tokens, cost: usage.cost, }) - await Session.updateMessage(input.assistantMessage) + // altimate_change start - buffered echoes publish authoritative text before a finished message + if (!deferText || input.assistantMessage.error) await Session.updateMessage(input.assistantMessage) + // altimate_change end // altimate_change start — capture the snapshot diff as the generic, // command-agnostic mutation ground truth (also catches bash-mediated // writes like `sed -i`/heredocs, which emit no edit event). @@ -1189,13 +1201,19 @@ export namespace SessionProcessor { }, metadata: value.providerMetadata, } - await Session.updatePart(currentText) + // altimate_change start - retain the live part through text deltas and completion + if (deferText) deferText.push(currentText) + else await Session.updatePart(currentText) + // altimate_change end break case "text-delta": if (currentText) { currentText.text += value.text if (value.providerMetadata) currentText.metadata = value.providerMetadata + // altimate_change start - unvalidated echo deltas must not reach clients + if (deferText) break + // altimate_change end await Session.updatePartDelta({ sessionID: currentText.sessionID, messageID: currentText.messageID, @@ -1224,7 +1242,9 @@ export namespace SessionProcessor { end: Date.now(), } if (value.providerMetadata) currentText.metadata = value.providerMetadata - await Session.updatePart(currentText) + // altimate_change start - publish the echo only after finish validation + if (!deferText) await Session.updatePart(currentText) + // altimate_change end } currentText = undefined break @@ -1262,7 +1282,20 @@ export namespace SessionProcessor { // altimate_change end }) const error = MessageV2.fromError(e, { providerID: input.model.providerID }) - if (MessageV2.ContextOverflowError.isInstance(error)) { + // altimate_change start — committed echoes and summaries retain recoverable causes without a fatal event + if ( + (deferText || (input.recoverCommittedAnswer && input.assistantMessage.summary)) && + !input.abort.aborted && + input.assistantMessage.finish !== "content-filter" && + (isEchoTransportError(error) || MessageV2.ContextOverflowError.isInstance(error)) + ) { + input.assistantMessage.metadata = { + ...input.assistantMessage.metadata, + answer_commit_echo_error: error, + } + recoveredCommittedAnswer = input.recoverCommittedAnswer === true && input.assistantMessage.summary === true + } else if (MessageV2.ContextOverflowError.isInstance(error)) { + // altimate_change end if ((await Config.get()).compaction?.auto === false && !input.assistantMessage.summary) { // altimate_change start — upstream_fix: honor // compaction.auto=false on reactive provider overflow. @@ -1272,7 +1305,8 @@ export namespace SessionProcessor { sessionID: input.sessionID, error, }) - await SessionStatus.set(input.sessionID, { type: "idle" }) + // The prompt loop publishes idle after flushing deferred text. + if (!deferText) await SessionStatus.set(input.sessionID, { type: "idle" }) // altimate_change end } else { needsCompaction = true @@ -1309,11 +1343,11 @@ export namespace SessionProcessor { }) } // altimate_change end - input.assistantMessage.error = error // altimate_change start — order terminal error before idle publication // Publish the error before idle. The run harness drains this // generation through idle before opening a challenge stream; an // unawaited publish can otherwise arrive on that fresh stream. + input.assistantMessage.error = error await Bus.publish(Session.Event.Error, { sessionID: input.assistantMessage.sessionID, error: input.assistantMessage.error, @@ -1334,7 +1368,8 @@ export namespace SessionProcessor { }) // altimate_change end // altimate_change start — SessionStatus.set became async; await so idle state flushes before exit - await SessionStatus.set(input.sessionID, { type: "idle" }) + // The prompt loop publishes idle after flushing deferred text. + if (!deferText) await SessionStatus.set(input.sessionID, { type: "idle" }) // altimate_change end } // altimate_change start — end ordinary provider/session error handling @@ -1347,7 +1382,7 @@ export namespace SessionProcessor { const end = Date.now() if (currentText) { currentText.time = { start: currentText.time?.start ?? end, end } - await Session.updatePart(currentText) + if (!deferText) await Session.updatePart(currentText) } for (const part of Object.values(reasoningMap)) { part.time = { ...part.time, end } @@ -1398,7 +1433,9 @@ export namespace SessionProcessor { } } input.assistantMessage.time.completed = Date.now() - await Session.updateMessage(input.assistantMessage) + // altimate_change start - the prompt finalizes buffered echoes after publishing their committed text + if (!deferText || input.assistantMessage.error) await Session.updateMessage(input.assistantMessage) + // altimate_change end // altimate_change start — explicit model DONE is the PRIMARY // termination path. A turn that finished with "stop", has no error, and // asserts completion (trailing DONE token per the SessionTermination @@ -1426,6 +1463,7 @@ export namespace SessionProcessor { // altimate_change start — the ordering itself lives in the exported // `resolveFinishOutcome` so the unit gate can exercise the REAL // decision instead of a hand-written copy of it (PR #1171 review). + if (recoveredCommittedAnswer) return "stop" as const return resolveFinishOutcome({ needsCompaction, explicitDone, diff --git a/packages/opencode/src/session/prompt.ts b/packages/opencode/src/session/prompt.ts index 1747e75abb..88c945152a 100644 --- a/packages/opencode/src/session/prompt.ts +++ b/packages/opencode/src/session/prompt.ts @@ -6,7 +6,9 @@ import fs from "fs/promises" import z from "zod" import { Filesystem } from "../util/filesystem" import { SessionID, MessageID, PartID } from "./schema" -import { MessageV2 } from "./message-v2" +// altimate_change start — module-scope echo transport predicate +import { MessageV2, isEchoTransportError } from "./message-v2" +// altimate_change end import { Log } from "../util/log" import { SessionRevert } from "./revert" import { Session } from "." @@ -24,6 +26,11 @@ import { NudgeArbiter } from "./nudge" import { SessionTermination } from "./termination" import { Instance } from "../project/instance" import { Bus } from "../bus" +// altimate_change start - distinguish submitted requests from compaction-created user rows in the stream +import { BusEvent } from "../bus/bus-event" +import { Database, eq, sql } from "../storage/db" +import { MessageTable } from "@opencode-ai/core/session/sql" +// altimate_change end import { ProviderTransform } from "../provider/transform" import { SystemPrompt } from "./system" import { InstructionPrompt } from "./instruction" @@ -111,6 +118,10 @@ import * as OnboardingTelemetry from "@/altimate/telemetry/onboarding" // altima // altimate_change start — keyless public Zen predicate (flat module) import { isPublicZen } from "@/provider/public-zen" // altimate_change end +// altimate_change start — distinguish harness deadlines and reuse their committed answer during finalization +import { RunBudget } from "@/altimate/run-budget" +import * as RunBudgetEffort from "@/altimate/session/run-budget-effort" +// altimate_change end // @ts-ignore globalThis.AI_SDK_LOG_WARNINGS = false @@ -125,6 +136,17 @@ IMPORTANT: const STRUCTURED_OUTPUT_SYSTEM_PROMPT = `IMPORTANT: The user has requested structured output. You MUST use the StructuredOutput tool to provide your final response. Do NOT respond with plain text - you MUST call the StructuredOutput tool with your answer formatted according to the schema.` +// altimate_change start - request identity is independent of compaction message adjacency +const RequestStarted = BusEvent.define( + "session.request.started", + z.object({ sessionID: SessionID.zod, messageID: MessageID.zod }), +) +const RequestContinued = BusEvent.define( + "session.request.continued", + z.object({ sessionID: SessionID.zod, messageID: MessageID.zod, parentID: MessageID.zod }), +) +// altimate_change end + export namespace SessionPrompt { const log = Log.create({ service: "session.prompt" }) @@ -574,7 +596,7 @@ export namespace SessionPrompt { } // altimate_change start — SessionStatus.set became async in v1.4.0; await so idle state actually flushes - export async function cancel(sessionID: SessionID) { + export async function cancel(sessionID: SessionID, reason?: RunBudget.HarnessDeadline) { log.info("cancel", { sessionID }) const s = state() const match = s[sessionID] @@ -587,7 +609,7 @@ export namespace SessionPrompt { // arrive synchronously after cancel() and must start a fresh generation, // never queue its callback on the signal that was just aborted. match.closing = true - match.abort.abort() + match.abort.abort(reason) if (!match.loopOwned) { // shell() also uses start(), but it has no loop disposer when no prompt // callbacks are queued. That owner must restore idle directly. @@ -660,7 +682,7 @@ export namespace SessionPrompt { // altimate_change start — client trace: the turns this generation runs. Released when it ends // (disposed before the generation's own cleanup, so no newer generation exists yet). const tracedTurns = new Set() - let currentTurn: string | undefined + let currentTurn: MessageID | undefined using _traceGeneration = defer(() => TraceContext.release(sessionID, tracedTurns)) // altimate_change end @@ -819,6 +841,187 @@ export namespace SessionPrompt { }) using _unsubToolsChanged = defer(unsubscribeToolsChanged) // altimate_change end + // altimate_change start — retain the exact committed text through automatic compaction + let committedAnswer: { + userID: MessageID + output: string + assistant: Pick + } | undefined + const compactionContinuations = new Set() + // The legacy message table has no explicit sequence column. Its SQLite rowid + // records insertion order and survives updateMessage's data-only upserts. + // Read it with the stored timestamp: caller-provided IDs carry no chronology. + const readMessageOrder = () => new Map( + Database.use((db) => db + .select({ id: MessageTable.id, created: MessageTable.time_created, sequence: sql`rowid` }) + .from(MessageTable) + .where(eq(MessageTable.session_id, sessionID)) + .all()).map((row) => [row.id, row]), + ) + const compareMessageOrder = (order: ReturnType, leftID: MessageID, rightID: MessageID) => { + const left = order.get(leftID) + const right = order.get(rightID) + if (!left || !right) return 0 + return left.created - right.created || left.sequence - right.sequence + } + function* orderedMessageStream(order: ReturnType) { + // stream() already orders by stored timestamp; repair only tied groups, + // retaining lazy history hydration and filterCompacted's summary placement. + let tied: MessageV2.WithParts[] = [] + for (const msg of MessageV2.stream(sessionID)) { + if (tied.length && order.get(tied[0].info.id)?.created !== order.get(msg.info.id)?.created) { + yield* tied.sort((a, b) => compareMessageOrder(order, b.info.id, a.info.id)) + tied = [] + } + tied.push(msg) + } + yield* tied.sort((a, b) => compareMessageOrder(order, b.info.id, a.info.id)) + } + const captureCommittedAnswer = (user: MessageV2.User, assistant: MessageV2.Assistant, parts: MessageV2.Part[]) => { + if (assistant.parentID !== user.id || assistant.summary || !parts.some((part) => part.type === "tool")) return + const commit = parts.findLast((part) => part.type === "tool" && part.tool === "answer_commit") + committedAnswer = + user.agent === "analyst" && + (user.format?.type ?? "text") === "text" && + assistant.agent === "analyst" && + commit?.type === "tool" && + commit.state.status === "completed" + ? { + userID: user.id, + assistant: { + agent: assistant.agent, + mode: assistant.mode, + modelID: assistant.modelID, + providerID: assistant.providerID, + variant: assistant.variant, + path: assistant.path, + }, + // Completed-tool metadata retains the full answer through the per-result output cap. + output: + typeof commit.state.metadata.committed_text === "string" + ? commit.state.metadata.committed_text + : commit.state.output, + } + : undefined + } + const publishText = async (parts: MessageV2.TextPart[]) => { + for (const part of parts) { + await Session.updatePart({ ...part, text: "", time: part.time && { start: part.time.start } }) + await Session.updatePartDelta({ + sessionID, + messageID: part.messageID, + partID: part.id, + field: "text", + delta: part.text, + }) + await Session.updatePart(part) + } + } + const finishCommittedAnswer = async ( + message: MessageV2.Assistant, + deferredText: MessageV2.TextPart[] = [], + error = message.error, + ) => { + // altimate_change start — a work deadline may publish the answer already committed before cancellation + const deadline = abort.reason instanceof RunBudget.HarnessDeadline + if ( + !committedAnswer || (abort.aborted && !deadline) || + compareMessageOrder(readMessageOrder(), message.id, committedAnswer.userID) <= 0 + ) return false + // Keep user aborts and HTTP policy rejections fatal even if a later operation throws another error. + if ( + message.finish === "content-filter" || + [message.error, error].some( + (cause) => + (cause?.name === "MessageAbortedError" && !deadline) || + cause?.name === "ContentFilterError" || + (cause && + ["APIError", "UnknownError", "ProviderAuthError"].includes(cause.name) && + !isEchoTransportError(cause)), + ) + ) + return false + // altimate_change end + // Once the echo has started, this answer still belongs to the earlier request. + // Publish it before the loop continues to any user request queued during generation. + const recoveryCause = error ?? message.metadata?.answer_commit_echo_error + const cause = + recoveryCause ?? + (message.finish !== "stop" ? { finish: message.finish ?? "unknown" } : undefined) + let storedText = MessageV2.parts(message.id).filter((part) => part.type === "text") + if ( + message.summary && message.finish && !message.error && + !message.metadata?.answer_commit_echo_error && + storedText.some((part) => part.text.trim().length > 0) + ) { + // A successful summary remains the compaction boundary. Deliver the + // committed request's answer separately, retaining its original identity. + const order = readMessageOrder() + let created = Date.now() + for (const msg of orderedMessageStream(order)) { + if (compareMessageOrder(order, msg.info.id, committedAnswer.userID) <= 0) break + if ( + msg.info.role === "user" && + !msg.parts.some((part) => part.type === "compaction") && + !compactionContinuations.has(msg.info.id) + ) { + // Walk newest to oldest so the answer precedes the earliest queued request. + created = order.get(msg.info.id)!.created - 1 + } + } + message = await Session.updateMessage({ + ...committedAnswer.assistant, + id: MessageID.ascending(), + parentID: committedAnswer.userID, + sessionID, + role: "assistant", + cost: 0, + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + time: { created }, + }) as MessageV2.Assistant + storedText = [] + deferredText = [] + } + // Only buffered parts are safe to reuse: stored parts may already have + // streamed deltas, and a blank update does not reset client text. + const first = deferredText[0] + const part: MessageV2.TextPart = { + id: first?.id ?? PartID.ascending(), + sessionID, + messageID: message.id, + type: "text", + text: committedAnswer.output, + time: { start: first?.time?.start ?? message.time.created, end: Date.now() }, + } + for (const old of storedText) { + if (old.id !== part.id) await Session.removePart({ sessionID, messageID: message.id, partID: old.id }) + } + if (cause) message.metadata = { ...message.metadata, answer_commit_echo_error: cause } + // altimate_change start — reuse delivered commits even when the deadline arrives during publication + message.metadata = { ...message.metadata, answer_commit_finalized: true } + // altimate_change end + delete message.error + // A failed summariser becomes the original request's answer, so clients must no longer hide it as compaction machinery. + if (message.summary) { + delete message.summary + message.parentID = committedAnswer.userID + message.agent = "analyst" + message.mode = "analyst" + await Session.updateMessage({ ...message, finish: undefined, time: { created: message.time.created } }) + } + // Persist the authoritative text before making this message terminal on resume. + await publishText([part]) + message.finish = message.finish === "other" && !recoveryCause ? "other" : "stop" + message.time.completed = Math.max(message.time.created, Date.now()) + await Session.updateMessage(message) + committedAnswer = undefined + return true + } + // Recovery may finish an earlier request while this generation still has queued work. + recovery: while (true) { + let loopFailure: { error: unknown } | undefined + try { + // altimate_change end while (true) { // altimate_change start — SessionStatus.set became async in v1.4.0; await so busy state flushes before LLM call await SessionStatus.set(sessionID, { type: "busy" }) @@ -829,7 +1032,8 @@ export namespace SessionPrompt { // compaction marker, hydrate the stream once and pass that unfiltered // history to the ledger. Previously filterCompacted read the recent view // and process() synchronously hydrated the entire session a second time. - const messageStream = MessageV2.stream(sessionID) + const messageOrder = readMessageOrder() + const messageStream = orderedMessageStream(messageOrder) const firstMessage = messageStream.next() const newestIsCompaction = !firstMessage.done && firstMessage.value.parts.some((part) => part.type === "compaction") @@ -866,6 +1070,35 @@ export namespace SessionPrompt { } if (!lastUser) throw new Error("No user message found in stream. This should never happen.") + // altimate_change start - only compaction-created continuations belong to the same request + const commitRequest = msgs.findLast( + (msg) => msg.info.role === "user" && !msg.parts.some((part) => part.type === "compaction"), + ) + if ( + committedAnswer && + msgs.some( + (msg) => + msg.info.role === "user" && + compareMessageOrder(messageOrder, msg.info.id, committedAnswer!.userID) > 0 && + !msg.parts.some((part) => part.type === "compaction") && + !compactionContinuations.has(msg.info.id), + ) + ) { + // A request queued after the commit still receives its own response; + // publish the completed answer for the earlier request first. + // Reload persisted history so the queued request includes the recovered answer. + if (lastAssistant && (await finishCommittedAnswer(lastAssistant))) continue + committedAnswer = undefined + } + if (commitRequest?.info.role === "user" && lastAssistant) { + // Text-only echoes and compaction summaries retain the commit until delivery or a new request. + captureCommittedAnswer( + commitRequest.info, + lastAssistant, + msgs.find((msg) => msg.info.id === lastAssistant.id)?.parts ?? [], + ) + } + // altimate_change end // altimate_change start — client trace for this step. A user message carrying a compaction // part was created by this loop: it continues the turn that triggered it. Any other user // message is a turn of its own, traced or not. @@ -873,8 +1106,10 @@ export namespace SessionPrompt { const isCompaction = msgs.some( (msg) => msg.info.id === stepUser.id && msg.parts.some((part) => part.type === "compaction"), ) - if (isCompaction && currentTurn && !TraceContext.forMessage(stepUser.id)) { - TraceContext.inherit(stepUser.id, currentTurn) + if (isCompaction && currentTurn) { + // Request ownership must survive even when no client trace is active. + await Bus.publish(RequestContinued, { sessionID, messageID: stepUser.id, parentID: currentTurn }) + if (!TraceContext.forMessage(stepUser.id)) TraceContext.inherit(stepUser.id, currentTurn) } else if (!isCompaction) { currentTurn = stepUser.id } @@ -884,6 +1119,22 @@ export namespace SessionPrompt { // altimate_change start — always track the current agent name so early breaks still report it if (lastUser.agent) sessionAgentName = lastUser.agent // altimate_change end + // altimate_change start — budget finalization acknowledges a delivered commit without a model turn + const finalizationParts = msgs.find((msg) => msg.info.id === lastUser.id)?.parts + if ( + RunBudgetEffort.toolsDisabled(sessionID) && + lastAssistant?.metadata?.answer_commit_finalized === true && + lastAssistant.finish === "stop" && !lastAssistant.error && + lastUser.agent === "analyst" && (lastUser.format?.type ?? "text") === "text" && + finalizationParts?.length === 1 && + finalizationParts[0].type === "text" && finalizationParts[0].synthetic && + finalizationParts[0].text === SessionTermination.REPLY_AFTER_BUDGET && + !msgs.some((msg) => + msg.info.role === "user" && msg.info.id !== lastUser.id && + compareMessageOrder(messageOrder, msg.info.id, lastAssistant!.id) > 0, + ) + ) break + // altimate_change end // altimate_change start — upstream_fix: a provider can finish with // "stop" while still emitting tool parts; those tools must be replayed // into the next loop instead of terminating the session. @@ -900,7 +1151,7 @@ export namespace SessionPrompt { lastAssistant?.finish && !["tool-calls", "unknown"].includes(lastAssistant.finish) && !lastAssistantHasToolParts && - lastUser.id < lastAssistant.id + lastAssistant.parentID === lastUser.id ) { log.info("exiting loop", { sessionID }) break @@ -1113,6 +1364,7 @@ export namespace SessionPrompt { await Session.updateMessage(summaryUserMsg) // altimate_change start — this loop-written message continues the step's turn (client trace) TraceContext.inherit(summaryUserMsg.id, lastUser.id) + await Bus.publish(RequestContinued, { sessionID, messageID: summaryUserMsg.id, parentID: lastUser.id }) // altimate_change end await Session.updatePart({ id: PartID.ascending(), @@ -1135,6 +1387,9 @@ export namespace SessionPrompt { return false }) // altimate_change end + // altimate_change start — publish the recorded continuation origin before its next assistant + let continuationID: MessageID | undefined + // altimate_change end const result = await SessionCompaction.process({ messages: msgs, parentID: lastUser.id, @@ -1142,6 +1397,13 @@ export namespace SessionPrompt { sessionID, auto: task.auto, overflow: task.overflow, + // altimate_change start - distinguish automatic continuations from user-authored synthetic parts + onContinuation: (messageID) => { + compactionContinuations.add(messageID) + continuationID = messageID + }, + recoverCommittedAnswer: committedAnswer !== undefined, + // altimate_change end // altimate_change start — keep nudge delivery scoped to this prompt generation nudgeGeneration, // altimate_change end @@ -1152,6 +1414,11 @@ export namespace SessionPrompt { learnDelivery: learnDelivery ? lessonState : undefined, // altimate_change end }) + // altimate_change start — attached run clients cannot read the compactor's in-memory origin + if (continuationID) { + await Bus.publish(RequestContinued, { sessionID, messageID: continuationID, parentID: lastUser.id }) + } + // altimate_change end // altimate_change start — treat any non-"continue" result as stop: an // undefined/unknown result must never fall through to `continue`, which // re-enters compaction on the same unresolved marker and busy-loops. @@ -1466,6 +1733,9 @@ export namespace SessionPrompt { // Inject StructuredOutput tool if JSON schema mode enabled if (lastUser.format?.type === "json_schema") { + // altimate_change start - schema-mode requests finish through StructuredOutput, never answer_commit + delete tools.answer_commit + // altimate_change end tools["StructuredOutput"] = createStructuredOutputTool({ schema: lastUser.format.schema, onSuccess(output) { @@ -1593,7 +1863,9 @@ export namespace SessionPrompt { // Ephemerally wrap queued user messages with a reminder to stay on track if (step > 1 && lastFinished) { for (const msg of msgs) { - if (msg.info.role !== "user" || msg.info.id <= lastFinished.id) continue + // altimate_change start - queued request order is independent of caller-supplied IDs + if (msg.info.role !== "user" || compareMessageOrder(messageOrder, msg.info.id, lastFinished.id) <= 0) continue + // altimate_change end for (const part of msg.parts) { if (part.type !== "text" || part.ignored || part.synthetic) continue if (!part.text.trim()) continue @@ -1688,6 +1960,11 @@ export namespace SessionPrompt { if (format.type === "json_schema") { system.push(STRUCTURED_OUTPUT_SYSTEM_PROMPT) } + // altimate_change start - every successful commit makes the next generation an echo, including after compaction + const echoCommittedAnswer = + format.type === "text" && agent.name === "analyst" && committedAnswer !== undefined + const deferredText: MessageV2.TextPart[] = [] + // altimate_change end // altimate_change start - trace system prompt once per loop() call. // The system prompt is functionally identical across steps within a single @@ -1723,6 +2000,50 @@ export namespace SessionPrompt { } // altimate_change end + // altimate_change start - keep past commits as context without letting historical tool stubs re-advertise them in schema mode + const modelHistory = + format.type === "json_schema" + ? msgs.map((msg) => ({ + ...msg, + parts: msg.parts.map((part): MessageV2.Part => { + if (part.type !== "tool" || part.tool !== "answer_commit") return part + let text = "answer_commit did not complete." + if (part.state.status === "error") text = part.state.error + if (part.state.status === "completed") { + const mask = part.state.metadata?.observation_mask + text = part.state.time.compacted + ? typeof mask === "string" && mask.length > 0 + ? mask + : "[Old tool result content cleared]" + : part.state.output + } + return { id: part.id, sessionID: part.sessionID, messageID: part.messageID, type: "text", text } + }), + })) + : msgs + // altimate_change end + // altimate_change start - recheck persisted requests after asynchronous history conversion, before sending an echo + const modelMessages = await MessageV2.toModelMessages(modelHistory, model) + if (echoCommittedAnswer && committedAnswer) { + const order = readMessageOrder() + let newerUserRequest = false + for (const msg of MessageV2.stream(sessionID)) { + if ( + msg.info.role === "user" && + compareMessageOrder(order, msg.info.id, committedAnswer.userID) > 0 && + !msg.parts.some((part) => part.type === "compaction") && + !compactionContinuations.has(msg.info.id) + ) { + newerUserRequest = true + break + } + } + if (newerUserRequest) { + await finishCommittedAnswer(processor.message) + continue + } + } + // altimate_change end const result = await processor.process({ user: lastUser, // altimate_change start — correlate request-boundary snapshots with this assistant message @@ -1733,21 +2054,59 @@ export namespace SessionPrompt { sessionID, system, messages: [ - ...(await MessageV2.toModelMessages(msgs, model)), - ...(isLastStep + // altimate_change start - use the history converted before the fresh-user check + ...modelMessages, + // altimate_change end + // altimate_change start - a committed answer uses the next normal generation even before the step limit + ...(echoCommittedAnswer ? [ { - role: "assistant" as const, - content: MAX_STEPS, + // A user turn: Bedrock's Converse API rejects a conversation that ends with an + // assistant message ("conversation must end with a user message"), and every + // provider accepts an instruction as the final user turn. + role: "user" as const, + content: "Tools are disabled. Reply with exactly the text returned by answer_commit and nothing else.", }, ] - : []), + : isLastStep + ? [ + { + role: "assistant" as const, + content: MAX_STEPS, + }, + ] + : []), + // altimate_change end ], - tools, + // altimate_change start - the final echo cannot spend another step on tools + tools: echoCommittedAnswer ? {} : tools, + deferText: echoCommittedAnswer ? deferredText : undefined, + // altimate_change end model, - toolChoice: format.type === "json_schema" ? "required" : undefined, + // altimate_change start - a user output schema still takes precedence over text finalization + toolChoice: echoCommittedAnswer ? "none" : format.type === "json_schema" ? "required" : undefined, + // altimate_change end }) + // altimate_change start - finalize buffered echoes before publishing any text to clients + const stepParts = MessageV2.parts(processor.message.id) + // Capture before post-processing can fail or request compaction. + captureCommittedAnswer(lastUser, processor.message, stepParts) + if (echoCommittedAnswer && result === "compact") { + processor.message.metadata = { + ...processor.message.metadata, + answer_commit_echo_error: processor.message.metadata?.answer_commit_echo_error ?? { + result, + finish: processor.message.finish ?? "unknown", + }, + } + } + const echoFinished = + echoCommittedAnswer && (await finishCommittedAnswer(processor.message, deferredText)) + // Abort and content-filter exits retain their original buffered text and timing. + if (!echoFinished) await publishText(deferredText) + // A finalized echo already has its answer, including when usage requested compaction. + // altimate_change end // If structured output was captured, save it and exit immediately // This takes priority because the StructuredOutput tool was called successfully if (structuredOutput !== undefined) { @@ -1776,7 +2135,6 @@ export namespace SessionPrompt { sessionTotalCost += processor.message.cost ?? 0 const t = processor.message.tokens sessionTotalTokens += t.input + t.output + t.reasoning + t.cache.read + t.cache.write - const stepParts = await MessageV2.parts(processor.message.id) toolCallCount += stepParts.filter((p) => p.type === "tool").length if (processor.message.error) sessionHadError = true // altimate_change start — capture last message error for agent_outcome reason @@ -2016,6 +2374,7 @@ export namespace SessionPrompt { } as MessageV2.Info) // This loop-written message continues the step's turn (client trace). TraceContext.inherit(syntheticMessageID, lastUser.id) + await Bus.publish(RequestContinued, { sessionID, messageID: syntheticMessageID, parentID: lastUser.id }) // Append the validator body as a text part on the new user message. await Session.updatePart({ @@ -2069,6 +2428,9 @@ export namespace SessionPrompt { } // altimate_change end + // altimate_change start - use the normal stop check for finished echoes, preserving any newly queued request + if (echoFinished) continue + // altimate_change end if (result === "stop") break if (result === "compact") { // altimate_change start — track compaction count @@ -2088,6 +2450,39 @@ export namespace SessionPrompt { } continue } + // altimate_change start - one request-exit guard covers compaction stops and failures outside the processor + } catch (error) { + loopFailure = { error } + } + let recovered = false + const recoveredUserID = committedAnswer?.userID + if (committedAnswer) { + const exiting = [...orderedMessageStream(readMessageOrder())].find((msg) => msg.info.role === "assistant") + if (exiting?.info.role === "assistant") { + recovered = await finishCommittedAnswer( + exiting.info, + [], + loopFailure ? MessageV2.fromError(loopFailure.error, { providerID: exiting.info.providerID }) : exiting.info.error, + ) + } + } + if (loopFailure && !recovered) throw loopFailure.error + if (recovered && recoveredUserID) { + // Publishing can itself yield to a queued prompt. Re-read persisted order + // before resolving this generation's callers with the recovered answer. + const order = readMessageOrder() + for (const msg of orderedMessageStream(order)) { + if (compareMessageOrder(order, msg.info.id, recoveredUserID) <= 0) break + if ( + msg.info.role === "user" && + !msg.parts.some((part) => part.type === "compaction") && + !compactionContinuations.has(msg.info.id) + ) continue recovery + } + } + break + } + // altimate_change end // altimate_change start — the generation-scoped disposer publishes the // sole normal idle transition after removing this loop from state. Abort // and processor-error paths may already be idle; the disposer detects that. @@ -2203,7 +2598,7 @@ export namespace SessionPrompt { await Telemetry.shutdown() // altimate_change end // altimate_change start — resolve callbacks from this prompt generation only - for await (const item of MessageV2.stream(sessionID)) { + for (const item of orderedMessageStream(readMessageOrder())) { if (item.info.role === "user") continue // Resolve only callers queued on THIS loop generation. A cancelled loop // may finish unwinding after a replacement generation has already begun; @@ -3007,6 +3402,9 @@ export namespace SessionPrompt { for (const part of parts) { await Session.updatePart(part) } + // altimate_change start - internal compaction continuations do not pass through this request boundary + await Bus.publish(RequestStarted, { sessionID: input.sessionID, messageID: info.id }) + // altimate_change end return { info, diff --git a/packages/opencode/src/tool/registry.ts b/packages/opencode/src/tool/registry.ts index a33b2a2e32..5d16f19629 100644 --- a/packages/opencode/src/tool/registry.ts +++ b/packages/opencode/src/tool/registry.ts @@ -33,6 +33,9 @@ import { WebSearchTool } from "./websearch" import { CodeSearchTool } from "./codesearch" import { Flag } from "@/flag/flag" import { Log } from "@/util/log" +// altimate_change start — reserved tool IDs must use the permission matcher's platform semantics. +import { Wildcard } from "@/util/wildcard" +// altimate_change end import { LspTool } from "./lsp" // altimate_change start — bridge legacy plugin tools to the Effect Tool API (v1.17.9) import { Effect } from "effect" @@ -58,6 +61,7 @@ import { pathToFileURL } from "url" import { SqlExecuteTool } from "../altimate/tools/sql-execute" import { SqlMaterializeTool } from "../altimate/tools/sql-materialize" import { SqlCheckTool } from "../altimate/tools/sql-check" +import { AnswerCommit } from "../altimate/tools/answer-commit" import { SchemaInspectTool } from "../altimate/tools/schema-inspect" import { SqlAnalyzeTool } from "../altimate/tools/sql-analyze" import { SqlOptimizeTool } from "../altimate/tools/sql-optimize" @@ -508,8 +512,9 @@ export namespace ToolRegistry { ) // altimate_change end // altimate_change end - // altimate_change start — include plugin-provided custom tools - return [...resolved, ...custom, ...pluginCustom] + // altimate_change start — reserve the finalizer ID from file/plugin tools even when + // its native candidate is disabled or this catalogue is used without an analyst. + return [...resolved, ...custom, ...pluginCustom].filter((tool) => !Wildcard.match(tool.id, "answer_commit")) // altimate_change end } @@ -546,6 +551,13 @@ export namespace ToolRegistry { // engine only; the older datamate route is not offered there. const hidden = await hiddenToolIds() // altimate_change end + // altimate_change start — analyst-only finalization is a gated candidate, never a tool_lookup catalogue entry. + // A configured agent named analyst intentionally retains access. External tools cannot + // use this reserved ID: all() excludes file/plugin collisions and MCP.tools excludes MCP collisions. + if (agent?.name === "analyst" && process.env.ALTIMATE_ANSWER_COMMIT !== "0") { + tools.push(await AppRuntime.runPromise(AnswerCommit.AnswerCommitTool)) + } + // altimate_change end const result = await Promise.all( tools .filter((t) => { diff --git a/packages/opencode/test/agent/answer-commit.test.ts b/packages/opencode/test/agent/answer-commit.test.ts new file mode 100644 index 0000000000..0aa2987bf1 --- /dev/null +++ b/packages/opencode/test/agent/answer-commit.test.ts @@ -0,0 +1,1674 @@ +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test" +import { createOpenAICompatible } from "@ai-sdk/openai-compatible" +import { LayerNode } from "@opencode-ai/core/effect/layer-node" +import { SessionProjector } from "@opencode-ai/core/session/projector" +import path from "node:path" +import { ProjectV2 } from "@opencode-ai/core/project" +import { Effect, Layer } from "effect" +import { Agent } from "../../src/agent/agent" +import { renderAnswer } from "../../src/altimate/tools/answer-commit" +import { TraceContext } from "../../src/altimate/observability/trace-context" +import { ToolLookupTool } from "../../src/altimate/tools/tool-lookup" +import { GlobalBus, type GlobalEvent } from "../../src/bus/global" +import { OutputCollector } from "../../src/cli/cmd/run-output" +import { MCP } from "../../src/mcp" +import { Permission } from "../../src/permission" +import { Plugin } from "../../src/plugin" +import { Provider } from "../../src/provider/provider" +import { Session } from "../../src/session/session" +import { Session as LegacySession } from "../../src/session" +import { SessionPrompt } from "../../src/session/prompt" +import { SessionRoutes } from "../../src/server/routes/session" +import { SessionTermination } from "../../src/session/termination" +import { SessionCompaction } from "../../src/session/compaction" +import { MessageV2, isEchoTransportError } from "../../src/session/message-v2" +import { SessionSummary } from "../../src/session/summary" +// altimate_change start — error provenance fixtures for committed-answer echoes +import { APICallError } from "ai" +import { SessionRetry } from "../../src/session/retry" +import { Database, eq, sql } from "../../src/storage/db" +import { MessageTable } from "@opencode-ai/core/session/sql" +import { Snapshot } from "../../src/snapshot" +import * as RunAccounting from "../../src/cli/cmd/run-accounting" +import { ACPEvent } from "../../src/acp/event" +import { ACPSession } from "../../src/acp/session" +import type { Event as ClientEvent, OpencodeClient, SessionMessageResponse } from "@opencode-ai/sdk/v2" +// altimate_change end +import { RuntimeFlags } from "../../src/effect/runtime-flags" +import { ModelID, ProviderID } from "../../src/provider/schema" +import { MessageID, PartID, SessionID } from "../../src/session/schema" +import { SessionTools } from "../../src/session/tools" +import { ToolRegistry } from "../../src/tool/registry" +import { Truncate } from "../../src/tool/truncate" +import { Token } from "../../src/util/token" +import { initTool } from "../altimate/tool-fixture" +import { TestInstance } from "../fixture/fixture" +import { testEffect } from "../lib/effect" +import { reply } from "../lib/llm-server" +import { withLegacyInstance } from "../session/legacy-instance" + +const model = { providerID: ProviderID.opencode, modelID: ModelID.make("test") } +const it = testEffect(Layer.mergeAll(Agent.defaultLayer, ToolRegistry.layer)) +const promptTests = testEffect( + LayerNode.buildLayer(LayerNode.group([SessionPrompt.node, Session.node, Agent.node, SessionProjector.node]), { + replacements: [ + LayerNode.replace( + MCP.node, + Layer.mock(MCP.Service, { + tools: () => Effect.succeed({}), + snapshot: () => Effect.succeed({ tools: {}, meta: undefined }), + listMeta: () => Effect.succeed(undefined), + entry: () => Effect.succeed(undefined), + }), + ), + LayerNode.replace( + SessionSummary.node, + Layer.mock(SessionSummary.Service, { + summarize: () => Effect.void, + diff: () => Effect.succeed([]), + computeDiff: () => Effect.succeed([]), + }), + ), + LayerNode.replace(RuntimeFlags.node, RuntimeFlags.layer({ experimentalEventSystem: true })), + ], + }), +) +const promptModel = { providerID: ProviderID.make("test"), modelID: ModelID.make("test-model") } +const echoInstruction = "Tools are disabled. Reply with exactly the text returned by answer_commit and nothing else." + +let previousToggle: string | undefined + +beforeEach(() => { + previousToggle = process.env.ALTIMATE_ANSWER_COMMIT + delete process.env.ALTIMATE_ANSWER_COMMIT +}) + +afterEach(() => { + if (previousToggle === undefined) delete process.env.ALTIMATE_ANSWER_COMMIT + else process.env.ALTIMATE_ANSWER_COMMIT = previousToggle +}) + +describe("answer_commit agent scope", () => { + test("single-instance answer_commit cases use scoped Effect fixtures", async () => { + const source = await Bun.file(import.meta.path).text() + expect(source).not.toMatch(/\bInstance\.provide\s*\(/) + expect(source).not.toMatch(/\bawait\s+tmpdir\s*\(/) + }) + + it.instance("analyst permits and receives answer_commit", () => + Effect.gen(function* () { + const agents = yield* Agent.Service + const registry = yield* ToolRegistry.Service + const analyst = yield* agents.get("analyst") + expect(Permission.evaluate("answer_commit", "*", analyst.permission).action).toBe("allow") + const tools = yield* registry.tools({ ...model, agent: analyst }) + expect(tools.filter((tool) => tool.id === "answer_commit")).toHaveLength(1) + }), + ) + + it.instance( + "a custom agent named analyst receives answer_commit by design", + () => + Effect.gen(function* () { + const agents = yield* Agent.Service + const registry = yield* ToolRegistry.Service + const analyst = yield* agents.get("analyst") + expect(analyst.description).toBe("Configured analyst") + expect(analyst.prompt).toBe("Use the configured analyst instructions.") + // Configuration customizes the native analyst; access intentionally follows its name. + expect(Permission.evaluate("answer_commit", "*", analyst.permission).action).toBe("allow") + const tools = yield* registry.tools({ ...model, agent: analyst }) + expect(tools.filter((tool) => tool.id === "answer_commit")).toHaveLength(1) + }), + { + config: { + agent: { + analyst: { + description: "Configured analyst", + prompt: "Use the configured analyst instructions.", + }, + }, + }, + }, + ) + + it.instance("session resolver selects native answer_commit over a plugin tool with the same id", () => + withLegacyInstance( + Effect.gen(function* () { + const tmp = yield* TestInstance + const agents = yield* Agent.Service + const analyst = yield* agents.get("analyst") + let pluginCalls = 0 + const plugin = Layer.mock(Plugin.Service, { + list: () => + Effect.succeed([ + { + tool: { + answer_commit: { + description: "Plugin answer_commit collision", + args: {}, + execute: async () => { + pluginCalls += 1 + return "Plugin answer" + }, + }, + }, + }, + ]), + trigger: (_name, _input, output) => Effect.succeed(output), + }) + const resolverModel: Provider.Model = { + id: model.modelID, + providerID: model.providerID, + api: { id: model.modelID, url: "https://example.com", npm: "@ai-sdk/openai" }, + name: "Test model", + capabilities: { + temperature: true, + reasoning: false, + attachment: false, + toolcall: true, + interleaved: false, + input: { text: true, audio: false, image: false, video: false, pdf: false }, + output: { text: true, audio: false, image: false, video: false, pdf: false }, + }, + cost: { input: 0, output: 0, cache: { read: 0, write: 0 } }, + limit: { context: 128000, output: 4096 }, + status: "active", + options: {}, + headers: {}, + release_date: "2026-01-01", + } + const sessionID = SessionID.make("ses_answer_commit_collision") + const messageID = MessageID.make("msg_answer_commit_collision") + const tools = yield* SessionTools.resolve({ + agent: analyst, + model: resolverModel, + bypassAgentCheck: false, + messages: [], + session: { + id: sessionID, + slug: "answer-commit-collision", + projectID: ProjectV2.ID.make("project"), + directory: tmp.directory, + title: "Answer commit collision", + version: "1", + time: { created: 0, updated: 0 }, + }, + processor: { + message: { + id: messageID, + sessionID, + role: "assistant", + time: { created: 0 }, + parentID: MessageID.make("msg_user"), + modelID: model.modelID, + providerID: model.providerID, + mode: "analyst", + agent: "analyst", + path: { cwd: tmp.directory, root: tmp.directory }, + cost: 0, + tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } }, + }, + updateToolCall: () => Effect.succeed(undefined), + completeToolCall: () => Effect.void, + }, + promptOps: { + resolvePromptParts: () => Effect.succeed([]), + prompt: () => Effect.die("unexpected prompt"), + cancel: () => Effect.void, + }, + }).pipe( + Effect.provide( + Layer.mergeAll( + plugin, + ToolRegistry.layer.pipe(Layer.provide(plugin)), + Layer.mock(Permission.Service, { ask: () => Effect.void }), + Layer.mock(MCP.Service, { tools: () => Effect.succeed({}) }), + Layer.mock(Truncate.Service, {}), + ), + ), + ) + const result = yield* Effect.promise(() => + Promise.resolve( + tools.answer_commit.execute!( + { + fields: [{ name: "Revenue", value: 42 }], + basis: "Revenue sums orders.amount across all orders, with one row per order.", + }, + { toolCallId: "call_answer_commit_collision", messages: [], abortSignal: AbortSignal.any([]) }, + ), + ), + ) + expect(result).toMatchObject({ + title: "Answer committed", + output: "Revenue sums orders.amount across all orders, with one row per order.\n\n**Answer:** Revenue: 42", + }) + expect(pluginCalls).toBe(0) + }), + ), + ) + + it.instance( + "builder, plan, data-explorer and custom agents never receive answer_commit", + () => + Effect.gen(function* () { + const agents = yield* Agent.Service + const registry = yield* ToolRegistry.Service + for (const name of ["builder", "build", "plan", "data-explorer", "custom"]) { + const agent = yield* agents.get(name) + expect(agent).toBeDefined() + const tools = yield* registry.tools({ ...model, agent }) + expect(tools.map((tool) => tool.id)).not.toContain("answer_commit") + } + expect((yield* registry.tools(model)).map((tool) => tool.id)).not.toContain("answer_commit") + }), + { + config: { + permission: { "*": "allow", answer_commit: "allow" }, + agent: { + custom: { + description: "A custom agent", + mode: "primary", + permission: { "*": "allow", answer_commit: "allow" }, + }, + }, + }, + }, + ) + + it.instance("tool_lookup does not expose answer_commit to builder", () => + withLegacyInstance( + Effect.gen(function* () { + const registry = yield* ToolRegistry.Service + const tool = yield* Effect.promise(() => initTool(ToolLookupTool)) + const ctx = { + sessionID: SessionID.make("ses_answer_commit"), + messageID: MessageID.make("msg_answer_commit"), + callID: "call_answer_commit_lookup", + agent: "builder", + abort: AbortSignal.any([]), + messages: [], + metadata: () => {}, + ask: async () => {}, + } + const missing = yield* Effect.promise(() => tool.execute({ tool_name: "answer_commit" }, ctx)) + expect(missing.title).toBe("Tool not found") + const listing = yield* Effect.promise(() => tool.execute({ tool_name: "__missing_tool" }, ctx)) + expect(listing.output).not.toContain("answer_commit") + expect((yield* registry.allInfos()).map((info) => info.id)).not.toContain("answer_commit") + expect(yield* registry.ids()).not.toContain("answer_commit") + }), + ), + ) + + it.instance("ALTIMATE_ANSWER_COMMIT=0 removes the analyst candidate and can be toggled back on", () => + Effect.gen(function* () { + const agents = yield* Agent.Service + const registry = yield* ToolRegistry.Service + const analyst = yield* agents.get("analyst") + process.env.ALTIMATE_ANSWER_COMMIT = "0" + expect((yield* registry.tools({ ...model, agent: analyst })).map((tool) => tool.id)).not.toContain( + "answer_commit", + ) + process.env.ALTIMATE_ANSWER_COMMIT = "1" + expect((yield* registry.tools({ ...model, agent: analyst })).map((tool) => tool.id)).toContain("answer_commit") + }), + ) + + it.instance("answer_commit is reserved from plugin and file tools even when the native tool is disabled", () => + withLegacyInstance( + Effect.gen(function* () { + const tmp = yield* TestInstance + const agents = yield* Agent.Service + const analyst = yield* agents.get("analyst") + const builder = yield* agents.get("builder") + const filename = path.join(tmp.directory, "answer.ts") + yield* Effect.promise(() => + Bun.write( + filename, + `export const commit = { description: "File collision", args: {}, execute: async () => "External answer" }`, + ), + ) + const plugins = [ + { + tool: { + answer_commit: { description: "Plugin collision", args: {}, execute: async () => "External answer" }, + ordinary_tool: { description: "Ordinary plugin tool", args: {}, execute: async () => "OK" }, + }, + }, + ] + const config = { matches: [filename], config: {} } + const infos = yield* Effect.promise(() => ToolRegistry.allInfos(plugins, config)) + expect(infos.map((tool) => tool.id)).not.toContain("answer_commit") + expect(infos.map((tool) => tool.id)).toContain("ordinary_tool") + for (const toggle of ["0", "1"]) { + process.env.ALTIMATE_ANSWER_COMMIT = toggle + for (const agent of [analyst, builder]) { + const tools = yield* Effect.promise(() => ToolRegistry.tools(model, agent, plugins, undefined, config)) + expect(tools.filter((tool) => tool.id === "answer_commit")).toHaveLength( + toggle === "1" && agent.name === "analyst" ? 1 : 0, + ) + expect(tools.map((tool) => tool.id)).toContain("ordinary_tool") + } + } + const platform = Object.getOwnPropertyDescriptor(process, "platform")! + Object.defineProperty(process, "platform", { ...platform, value: "win32" }) + try { + const variants = [{ tool: { Answer_Commit: plugins[0].tool.answer_commit } }] + const infos = yield* Effect.promise(() => ToolRegistry.allInfos(variants, config)) + expect(Permission.evaluate("Answer_Commit", "*", analyst.permission).action).toBe("allow") + expect(infos.map((tool) => tool.id)).not.toContain("Answer_Commit") + } finally { + Object.defineProperty(process, "platform", platform) + } + }), + ), + ) +}) + +// altimate_change start — committed-answer echo transport and delivery regressions +// Replay immutable bus events through the real ACP handler with the message state +// available at each event, rather than letting later recovery rewrite its lookup. +const acpText = (events: ClientEvent[], sessionID: string) => Effect.gen(function* () { + const session = yield* ACPSession.Service + yield* session.create({ id: sessionID, cwd: process.cwd() }) + const messages = new Map() + const chunks: string[] = [] + const subscription = new ACPEvent.Subscription({ + session, + sdk: { session: { message: async ({ messageID }: { messageID: string }) => ({ data: messages.get(messageID) }) } } as unknown as OpencodeClient, + connection: { + sessionUpdate: async ({ update }) => { + if (update.sessionUpdate === "agent_message_chunk" && update.content.type === "text") chunks.push(update.content.text) + }, + }, + }) + for (const event of events) { + if (event.type === "message.updated") { + const info = event.properties.info + messages.set(info.id, { info, parts: messages.get(info.id)?.parts ?? [] }) + } + if (event.type === "message.part.updated") { + const part = event.properties.part + const message = messages.get(part.messageID) + if (message) message.parts = [...message.parts.filter((old) => old.id !== part.id), part] + } + if (event.type === "message.part.removed") { + const message = messages.get(event.properties.messageID) + if (message) message.parts = message.parts.filter((part) => part.id !== event.properties.partID) + } + yield* Effect.promise(() => subscription.handle(event)) + } + return chunks +}).pipe(Effect.provide(ACPSession.defaultLayer)) + +for (const location of ["reply", "previous", "compaction fallback"] as const) { + test(`run output drops removed text from the ${location} before delivering a committed answer`, () => { + const output = new OutputCollector() + output.text("Summary so far: ", { partID: "partial", compaction: location === "compaction fallback" }) + if (location === "previous") output.stepStart() + output.remove("partial") + expect(output.content()).toBe("(no text output — tool-only response)") + if (location === "previous") { + output.text("DONE", { partID: "acknowledgement" }) + expect(output.content()).toBe("DONE") + } + output.text("42", { partID: "committed" }) + expect(output.content()).toBe("42") + }) +} + +for (const scenario of [ + "text", + "echo-length", + "echo-mismatch", + "echo-no-text", + "echo-write-order", + "echo-transport-error", + "echo-socket-reset", + "echo-policy-denied", + "echo-guardrail-denied", + "echo-guardrail-overflow-text", + "echo-forbidden", + "echo-snapshot-error", + "echo-overflow-enabled", + "echo-reactive-compaction", + "compaction-error", + "compaction-error-queued-user", + "compaction-queued-user", + "overflow-code-compaction", + "compaction-streamed-error", + "compaction-empty-summary", + "compaction-empty-summary-queued-user", + "compaction-empty-summary-streamed", + "compaction-empty-summary-without-commit", + "compaction-forbidden", + "compaction-guardrail-denied", + "compaction-aborted", + "compaction-content-filter", + "echo-overflow-disabled", + "echo-preparation-error", + "echo-aborted", + "echo-harness-deadline", + "echo-user-cancelled", + "echo-budget-finalization", + "echo-budget-finalization-publishing", + "echo-content-filter", + "echo-http-content-filter", + "echo-other", + "echo-exact-length", + "echo-capped", + "echo-under-cap", + "json_schema", + "json_schema-history", + "new-user", + "new-user-after-commit-reversed-ids", + "queued-user", + "queued-user-reversed-ids", + "user-during-echo", + "user-during-mismatched-echo", + "user-during-echo-reversed-ids", + "user-during-echo-reversed-ids-same-time", + "failed", + "failed-after-success", + "sibling-tool", +] as const) { + const structured = scenario.startsWith("json_schema") + const priorCommit = scenario === "json_schema-history" + const budgetFinalization = scenario.startsWith("echo-budget-finalization") + const tinyCap = scenario === "echo-capped" || scenario === "echo-under-cap" + const compacting = scenario.startsWith("compaction-") + const emptySummary = scenario.startsWith("compaction-empty-summary") + const streamedSummary = scenario === "compaction-streamed-error" || scenario === "compaction-empty-summary-streamed" + const userDuringEcho = scenario.startsWith("user-during-") + const queuedUser = scenario === "queued-user" || scenario === "queued-user-reversed-ids" + const userAfterCommit = scenario === "new-user-after-commit-reversed-ids" + const userAfterCompaction = scenario === "compaction-queued-user" + const userDuringFailedCompaction = scenario === "compaction-error-queued-user" || scenario === "compaction-empty-summary-queued-user" + const overflowCodeCompaction = scenario === "overflow-code-compaction" + const summaryText = "Retained summary: counted 42 records using records.id." + const reversedIDs = scenario.includes("reversed-ids") + const name = { + text: "text requests offer answer_commit and immediately echo a successful commit without a step limit", + "echo-length": "a length-limited echo is replaced with the stored committed output verbatim", + "echo-mismatch": "a mismatched echo delivers the committed output exactly once to run output and ACP", + "echo-no-text": "a stop echo without text publishes the committed output as a completed timed text part", + "echo-write-order": "a committed echo persists its authoritative text before any finished message update", + "echo-transport-error": "a 503 echo transport failure publishes the committed output without fatal events", + "echo-socket-reset": "a socket reset during echo publishes the committed output without fatal events", + "echo-policy-denied": "a 400 structured policy denial during echo emits fatal message and session errors", + "echo-guardrail-denied": "a 503 orchestration guardrail denial during echo remains fatal", + "echo-guardrail-overflow-text": "a 503 guardrail denial with overflow text during echo stays fatal without publishing or compacting", + "echo-forbidden": "a 403 echo failure retains its fatal error without publishing the committed output", + "echo-snapshot-error": "a local snapshot failure before the echo request retains its fatal UnknownError", + "echo-overflow-enabled": "a context overflow during echo publishes the committed output without fatal events", + "echo-reactive-compaction": "an echo triggering reactive compaction publishes only the committed text without a continuation", + "compaction-error": "a 503 compaction summariser failure after a commit publishes the committed output without fatal events", + "compaction-error-queued-user": "a user queued during a 503 compaction failure receives tools and its own reply after the original committed answer", + "compaction-queued-user": "a queued user after successful compaction preserves the summary and receives tools after a new committed-answer message", + "overflow-code-compaction": "a 413 content_length_exceeded response triggers compaction and resumes with tools", + "compaction-streamed-error": "a streamed partial compaction summary followed by a 503 delivers only the committed answer to ACP and run", + "compaction-empty-summary": "two empty compaction summaries after a commit publish the committed output without fatal events", + "compaction-empty-summary-queued-user": "a user queued during two empty compaction summaries receives tools and its own reply after the original committed answer", + "compaction-empty-summary-streamed": "two empty compaction summaries after streamed partial text deliver only the committed answer to ACP and run", + "compaction-empty-summary-without-commit": "two empty compaction summaries without a commit retain the existing UnknownError", + "compaction-forbidden": "a 403 compaction summariser failure after a commit remains fatal", + "compaction-guardrail-denied": "a 503 orchestration guardrail denial during compaction after a commit remains fatal", + "compaction-aborted": "an aborted compaction after a commit retains its fatal error without publishing the committed output", + "compaction-content-filter": "a content-filtered compaction after a commit retains its fatal error without publishing the committed output", + "echo-overflow-disabled": "an unrecoverable context overflow after a commit publishes the committed output and records the cause", + "echo-preparation-error": "a local echo preparation failure after a commit is not recovered", + "echo-aborted": "an aborted echo retains its fatal error without publishing the committed output", + "echo-harness-deadline": "a harness deadline abort during echo publishes the stored committed text exactly once", + "echo-user-cancelled": "a user cancellation during echo does not finalize the stored committed answer", + "echo-budget-finalization": "budget finalization after a hard abort reuses the committed answer without starting a model turn", + "echo-budget-finalization-publishing": "budget finalization reuses the committed answer when the hard deadline arrives during publication", + "echo-content-filter": "a content-filtered echo retains its fatal error and original text", + "echo-http-content-filter": "an HTTP content-filtered echo retains its fatal error without publishing the committed output", + "echo-other": "an echo with an other finish keeps other and publishes the committed output", + "echo-exact-length": "an exact echo with a length finish ends normally with stop", + "echo-capped": "a dispatch-capped commit preserves the full rendered answer in the final assistant text", + "echo-under-cap": "a commit below the dispatch cap preserves the rendered answer in the final assistant text", + json_schema: "json_schema requests omit answer_commit and never enter echo mode", + "json_schema-history": "json_schema requests preserve masked commit history without re-advertising answer_commit", + "new-user": "a new user message resets a successful commit before the next step", + "new-user-after-commit-reversed-ids": "a later caller-supplied ID sorting before the committed request is detected at the next loop snapshot", + "queued-user": "a user message persisted after the snapshot keeps tools enabled and prevents stale echo", + "queued-user-reversed-ids": "a later caller-supplied ID sorting before the committed request is detected after the snapshot", + "user-during-echo": "a user message persisted during echo generation is answered on the next step", + "user-during-mismatched-echo": "a user message persisted during a mismatched echo preserves the committed answer and is answered in the same prompt call", + "user-during-echo-reversed-ids": "a later caller-supplied ID sorting before the committed request preserves its echo and keeps tools available", + "user-during-echo-reversed-ids-same-time": "equal request creation times use persisted sequence for caller-supplied IDs sorting backwards", + failed: "a failed commit leaves tools available for repair", + "failed-after-success": "a successful commit followed by validation failure leaves tools available for repair", + "sibling-tool": "a successful commit enters echo mode even when a sibling tool follows it", + }[scenario] + promptTests.instance( + name, + () => + withLegacyInstance( + Effect.gen(function* () { + const prompt = yield* SessionPrompt.Service + const sessions = yield* Session.Service + const agents = yield* Agent.Service + expect((yield* agents.get("analyst")).steps).toBeUndefined() + const chat = yield* sessions.create({ title: "Answer commit finalization" }) + const requests: { tools?: { function: { name: string } }[]; messages: unknown[]; tool_choice?: string }[] = [] + const args = { + ...(scenario === "echo-capped" + ? { + rows: { + columns: ["Region", "Revenue"], + values: Array.from({ length: 30 }, (_, i) => [`Region ${i + 1}`, (i + 1) * 100]), + }, + } + : { fields: [{ name: "", value: 42 }] }), + answer_only: true, + basis: "Counted all records at record grain using records.id.", + } + const renderedAnswer = renderAnswer(args) + const originalUserID = reversedIDs ? MessageID.make(`msg_zzzz_original_${chat.id}`) : undefined + let queuedUserID: MessageID | undefined + const queuedReplies: Promise[] = [] + const insertUser = async (created = Date.now()) => { + if (reversedIDs) { + const original = MessageV2.get({ sessionID: chat.id, messageID: originalUserID! }) + const createdSpy = scenario === "user-during-echo-reversed-ids-same-time" + ? spyOn(Date, "now").mockReturnValue(original.info.time.created) + : undefined + try { + const next = await SessionPrompt.prompt({ + sessionID: chat.id, + messageID: MessageID.make(`msg_aaaa_later_${chat.id}`), + agent: "analyst", + model: promptModel, + noReply: true, + parts: [{ type: "text", text: "A new question: how many sources?" }], + }) + expect(next.info.id < original.info.id).toBe(true) + if (createdSpy) expect(next.info.time.created).toBe(original.info.time.created) + queuedUserID = next.info.id + } finally { + createdSpy?.mockRestore() + } + return + } + const userID = MessageID.ascending() + await LegacySession.updateMessage({ + id: userID, + sessionID: chat.id, + role: "user", + agent: "analyst", + model: promptModel, + time: { created }, + }) + await LegacySession.updatePart({ + id: PartID.ascending(), + messageID: userID, + sessionID: chat.id, + type: "text", + text: "A new question: how many sources?", + }) + queuedUserID = userID + } + const language = createOpenAICompatible({ + name: "test", + baseURL: "https://unused.invalid/v1", + fetch: Object.assign( + async (_url: string | URL | Request, init?: RequestInit) => { + if (typeof init?.body !== "string") throw new Error("expected a JSON request body") + requests.push(JSON.parse(init.body)) + if (requests.length > (userDuringFailedCompaction ? emptySummary ? 4 : 3 : priorCommit ? 4 : userDuringEcho || emptySummary || userAfterCompaction || overflowCodeCompaction ? 3 : 2)) + throw new Error("unexpected extra generation") + const step = priorCommit && requests.length > 2 ? requests.length - 2 : requests.length + if (step === 2 && userDuringFailedCompaction) { + const queued = await SessionPrompt.prompt({ + sessionID: chat.id, + agent: "analyst", + model: promptModel, + noReply: true, + parts: [{ type: "text", text: "A new question: how many sources?" }], + }) + queuedUserID = queued.info.id + // Register both waiting callers after persistence and before the + // summariser responds, without racing asynchronous prompt setup. + queuedReplies.push(SessionPrompt.loop({ sessionID: chat.id }), SessionPrompt.loop({ sessionID: chat.id })) + } + if (step === 1 && overflowCodeCompaction) { + return new Response(JSON.stringify({ error: { code: "content_length_exceeded", message: "Request entity too large" } }), { + status: 413, + headers: { "Content-Type": "application/json" }, + }) + } + if ( + step === 2 && + [ + "echo-transport-error", "echo-policy-denied", "echo-guardrail-denied", "echo-guardrail-overflow-text", "echo-forbidden", "echo-http-content-filter", + "compaction-error", "compaction-error-queued-user", "compaction-forbidden", "compaction-guardrail-denied", "compaction-content-filter", "echo-overflow-disabled", "echo-overflow-enabled", + ].includes(scenario) + ) { + const error = + scenario === "echo-policy-denied" + ? { code: "policy_violation", message: "Request denied" } + : scenario === "echo-guardrail-overflow-text" + ? { code: "guardrail_intervened", message: "maximum context length exceeded" } + : scenario === "echo-http-content-filter" || scenario === "compaction-content-filter" + ? { code: "content_filter", message: "Echo request blocked by content policy" } + : scenario === "echo-overflow-disabled" || scenario === "echo-overflow-enabled" + ? { code: "context_length_exceeded", message: "maximum context length exceeded" } + : { message: compacting ? "Compaction summariser rejected the request" : "Echo transport rejected the request" } + const guardrail = scenario === "echo-guardrail-denied" || scenario === "compaction-guardrail-denied" + return new Response(JSON.stringify(guardrail + ? { error, orchestrationError: { code: "guardrail_intervened" } } + : { error }), { + status: scenario === "echo-transport-error" || scenario === "compaction-error" || scenario === "compaction-error-queued-user" || guardrail || scenario === "echo-guardrail-overflow-text" + ? 503 + : scenario === "echo-forbidden" || scenario === "compaction-forbidden" ? 403 : 400, + headers: { "Content-Type": "application/json" }, + }) + } + if (step === 2 && scenario === "echo-socket-reset") { + throw Object.assign(new Error("socket reset"), { code: "ECONNRESET" }) + } + if (step === 2 && (scenario === "echo-aborted" || scenario === "compaction-aborted")) { + throw new DOMException("Echo request aborted", "AbortError") + } + if (step === 2 && ["echo-harness-deadline", "echo-user-cancelled", "echo-budget-finalization"].includes(scenario)) { + const response = await SessionRoutes().request(`/${chat.id}/abort`, { + method: "POST", + headers: scenario === "echo-user-cancelled" ? {} : { "x-altimate-run-budget": "hard-deadline" }, + }) + expect(response.status).toBe(200) + expect(init?.signal?.aborted).toBe(true) + throw init!.signal!.reason + } + if ( + (scenario === "new-user" && requests.length === 1) || + (userDuringEcho && requests.length === 2) + ) { + await insertUser() + } + const response = + step === 1 + ? // The structured case deliberately attempts a stale commit call to verify it cannot activate echo mode. + scenario === "sibling-tool" + ? reply().tool("answer_commit", args).tool("tool_lookup", { tool_name: "__missing_tool" }).item() + : scenario === "failed-after-success" + ? reply().tool("answer_commit", args).tool("answer_commit", {}).item() + : scenario === "compaction-empty-summary-without-commit" + ? reply().tool("tool_lookup", { tool_name: "__missing_tool" }).usage({ input: 95000, output: 1 }).item() + : reply() + .tool("answer_commit", scenario === "failed" ? {} : args) + .usage({ input: compacting ? 95000 : 0, output: 1 }) + .item() + : step === 2 && (userAfterCompaction || overflowCodeCompaction) + ? reply().text(summaryText).stop().item() + : userDuringFailedCompaction && step === (emptySummary ? 4 : 3) + ? reply().text("3").stop().item() + : streamedSummary + ? reply().text("Summary so far: ").stop().item() + : emptySummary || scenario === "echo-no-text" + ? (emptySummary ? reply().text("") : reply()).stop().item() + : structured && (!priorCommit || requests.length > 2) + ? reply().tool("StructuredOutput", { count: 42 }).item() + : reply() + .text( + scenario === "new-user" || queuedUser || userAfterCommit || step === 3 + ? "3" + : scenario === "echo-length" + ? "4" + : scenario === "echo-mismatch" || scenario === "echo-other" || scenario === "echo-reactive-compaction" || + scenario === "user-during-mismatched-echo" || reversedIDs + ? "The answer is 42." + : "42", + ) + .usage({ input: scenario === "echo-reactive-compaction" ? 95000 : 0, output: 1 }) + .stop() + .item() + if (response.type !== "sse") throw new Error("expected SSE response") + const chunks = [...response.head, ...response.tail] + if (step === 1 && scenario === "failed-after-success") { + // OpenAI-compatible streams identify distinct calls by index, including calls to the same tool. + let index = -1 + for (const chunk of chunks) { + const event = chunk as { + choices?: { delta?: { tool_calls?: { id?: string; index: number }[] } }[] + } + for (const choice of event.choices ?? []) { + for (const call of choice.delta?.tool_calls ?? []) { + if (call.id) index++ + call.index = index + } + } + } + } + if ( + step === 2 && + (scenario === "echo-length" || + scenario === "echo-exact-length" || + scenario === "echo-other" || + scenario === "echo-content-filter") + ) { + for (const chunk of chunks) { + const event = chunk as { choices?: { finish_reason?: string }[] } + for (const choice of event.choices ?? []) { + if (choice.finish_reason === "stop") + choice.finish_reason = + scenario === "echo-content-filter" ? "content_filter" : scenario === "echo-other" ? "other" : "length" + } + } + } + return new Response( + chunks.map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("") + "data: [DONE]\n\n", + { headers: { "Content-Type": "text/event-stream" } }, + ) + }, + { preconnect: () => undefined }, + ), + }).chatModel("test-model") + const doStream = language.doStream.bind(language) + const streamSpy = scenario === "compaction-streamed-error" + ? spyOn(language, "doStream").mockImplementation(async (options) => { + const response = await doStream(options) + if (requests.length !== 2) return response + let failed = false + return { ...response, stream: response.stream.pipeThrough(new TransformStream({ + transform(value, controller) { + if (failed) return + controller.enqueue(value) + if (value.type !== "text-delta") return + failed = true + controller.enqueue({ type: "error", error: new APICallError({ + message: "Compaction summariser failed after partial text", + url: "https://unused.invalid/v1/chat/completions", + requestBodyValues: {}, + statusCode: 503, + isRetryable: true, + }) }) + }, + })) } + }) + : undefined + // Completion hooks can discard streamed text, leaving an empty summary + // on both attempts even though clients already saw its deltas. + const trigger = Plugin.trigger + const completedTextIDs: PartID[] = [] + const textHookSpy = spyOn(Plugin, "trigger").mockImplementation(async (...input) => { + if (input[0] === "experimental.text.complete") + completedTextIDs.push(PartID.make((input[1] as { partID: string }).partID)) + const result = await trigger(...input) + if (scenario === "compaction-empty-summary-streamed" && input[0] === "experimental.text.complete" && requests.length > 1) + return { ...result, text: "" } + return result + }) + const languageSpy = spyOn(Provider, "getLanguage").mockResolvedValue(language) + const updatePart = LegacySession.updatePart + let completedSummary: MessageV2.WithParts | undefined + const processCompaction = SessionCompaction.process + const compactionSpy = userAfterCompaction + ? spyOn(SessionCompaction, "process").mockImplementation(Object.assign(async (...input: Parameters) => { + const result = await processCompaction(...input) + expect(result).toBe("continue") + completedSummary = structuredClone([...MessageV2.stream(chat.id)].find( + (message) => message.info.role === "assistant" && message.info.summary && message.info.finish && !message.info.error, + )) + expect(completedSummary).toBeDefined() + // Leave room to recover the answer after the summary and before this request. + await insertUser(Math.max(Date.now(), completedSummary!.info.time.created + 1)) + return result + }, processCompaction)) + : undefined + let publicationAborts = 0 + const commitSpy = userAfterCommit || scenario === "echo-budget-finalization-publishing" + ? spyOn(LegacySession, "updatePart").mockImplementation(Object.assign(async (part: MessageV2.Part) => { + const result = await updatePart(part) + if (userAfterCommit && part.type === "tool" && part.tool === "answer_commit" && part.state.status === "completed" && !queuedUserID) + await insertUser() + if (scenario === "echo-budget-finalization-publishing" && requests.length === 2 && part.type === "text" && part.time?.end) { + publicationAborts++ + const response = await SessionRoutes().request(`/${chat.id}/abort`, { + method: "POST", + headers: { "x-altimate-run-budget": "hard-deadline" }, + }) + expect(response.status).toBe(200) + } + return result + }, updatePart)) + : undefined + let echoPrepared = false + const track = Snapshot.track + const snapshotSpy = spyOn(Snapshot, "track").mockImplementation(async () => { + if (scenario === "echo-snapshot-error" && echoPrepared) throw new Error("Snapshot tracking failed") + return track() + }) + const retrySpy = scenario === "compaction-error" || scenario === "compaction-error-queued-user" || scenario === "echo-guardrail-denied" || scenario === "echo-guardrail-overflow-text" || scenario === "compaction-guardrail-denied" + ? spyOn(SessionRetry, "retryable").mockReturnValue(undefined) + : undefined + const finishedEchoText: string[][] = [] + const updateMessage = LegacySession.updateMessage + const finishSpy = scenario === "echo-write-order" + ? spyOn(LegacySession, "updateMessage").mockImplementation(Object.assign(async (message: MessageV2.Info) => { + if (message.role === "assistant" && requests.length === 2 && message.finish) { + finishedEchoText.push(MessageV2.parts(message.id) + .filter((part) => part.type === "text" && part.time?.end !== undefined) + .map((part) => (part as MessageV2.TextPart).text)) + } + return updateMessage(message) + }, updateMessage)) + : undefined + const toModelMessages = MessageV2.toModelMessages + const historySpy = spyOn(MessageV2, "toModelMessages").mockImplementation(async (...input) => { + if (scenario === "echo-preparation-error" && requests.length === 1) + throw new Error("Echo preparation failed") + if (requests.length === 1) echoPrepared = true + const messages = await toModelMessages(...input) + if (queuedUser && requests.length === 1 && !queuedUserID) await insertUser() + return messages + }) + const output = new OutputCollector() + const accounting = RunAccounting.create() + const compactionTextIDs: MessageID[] = [] + const textUpdates: MessageV2.TextPart[] = [] + const textPartIDs = new Set() + const textDeltas: { messageID: MessageID; partID: PartID; text: string }[] = [] + const removedTextIDs: PartID[] = [] + const clientEvents: ClientEvent[] = [] + const textEvents: { messageID: MessageID; type: string; text: string }[] = [] + const sessionErrors: unknown[] = [] + let compactions = 0 + const assistantUpdates: MessageV2.Assistant[] = [] + const requestStarts: string[] = [] + const collect = ({ payload: event }: GlobalEvent) => { + const request = event as { type: string; properties: { sessionID?: string; messageID?: string } } + if ( + request.type === "session.request.started" && + request.properties.sessionID === chat.id && + request.properties.messageID + ) + requestStarts.push(request.properties.messageID) + if (event.type === "session.error" && event.properties.sessionID === chat.id) { + sessionErrors.push(event.properties.error) + accounting.onSessionError(event.properties.error?.name) + } + if (event.type === "session.compacted" && event.properties.sessionID === chat.id) { + compactions++ + accounting.onCompactionRecovered() + } + if (event.type === MessageV2.Event.Updated.type) { + const message = event.properties.info + if (message.sessionID === chat.id && message.role === "assistant") { + clientEvents.push(structuredClone(event) as unknown as ClientEvent) + assistantUpdates.push(structuredClone(message)) + accounting.onAssistantMessage(message) + } + } + if (event.type === MessageV2.Event.PartUpdated.type) { + const part = event.properties.part as MessageV2.Part + if (part.sessionID !== chat.id) return + if (part.type === "step-start") output.stepStart(accounting.isCompactionStep(part.messageID)) + if (part.type !== "text") return + clientEvents.push(structuredClone(event) as unknown as ClientEvent) + // Snapshot the published value: later mutations must not rewrite what a client observed. + textUpdates.push(structuredClone(part)) + textEvents.push({ messageID: part.messageID, type: part.time?.end ? "completion" : "start", text: part.text }) + textPartIDs.add(part.id) + // Match run.ts: only completed text parts are printed and collected for --output. + if (part.time?.end) { + const compaction = accounting.isCompactionStep(part.messageID) + if (compaction) compactionTextIDs.push(part.messageID) + output.text(part.text.trim(), { partID: part.id, synthetic: part.synthetic === true, compaction }) + } + } + if ( + event.type === MessageV2.Event.PartDelta.type && + event.properties.sessionID === chat.id && + event.properties.field === "text" && + textPartIDs.has(event.properties.partID) + ) { + // ACP forwards deltas for known text parts and ignores later text updates. + clientEvents.push(structuredClone(event) as unknown as ClientEvent) + textDeltas.push({ messageID: event.properties.messageID, partID: event.properties.partID, text: event.properties.delta }) + textEvents.push({ messageID: event.properties.messageID, type: "delta", text: event.properties.delta }) + } + if (event.type === MessageV2.Event.PartRemoved.type && event.properties.sessionID === chat.id) { + clientEvents.push(structuredClone(event) as unknown as ClientEvent) + removedTextIDs.push(event.properties.partID) + output.remove(event.properties.partID) + } + } + GlobalBus.on("event", collect) + try { + if (priorCommit) { + yield* prompt.prompt({ + sessionID: chat.id, + agent: "analyst", + model: promptModel, + parts: [{ type: "text", text: "How many records? Reply with only the count." }], + }) + const history = yield* sessions.messages({ sessionID: chat.id }) + const commit = history + .flatMap((message) => message.parts) + .find((part) => part.type === "tool" && part.tool === "answer_commit") + if (commit?.type !== "tool" || commit.state.status !== "completed") + throw new Error("missing prior commit") + const state = commit.state + yield* Effect.promise(() => + LegacySession.updatePart({ + ...commit, + state: { + ...state, + output: "UNMASKED_COMMIT_OUTPUT", + metadata: { ...state.metadata, observation_mask: "Previous committed answer was masked." }, + time: { ...state.time, compacted: Date.now() }, + }, + }), + ) + } + const pending = prompt.prompt({ + sessionID: chat.id, + ...(originalUserID ? { messageID: originalUserID } : {}), + agent: "analyst", + model: promptModel, + parts: [{ type: "text", text: "How many records? Reply with only the count." }], + ...(structured + ? { + format: { + type: "json_schema" as const, + retryCount: 0, + schema: { + type: "object", + properties: { count: { type: "number" } }, + required: ["count"], + additionalProperties: false, + }, + }, + } + : {}), + }) + if (scenario === "echo-preparation-error") { + const outcome = yield* Effect.exit(pending) + expect(outcome._tag).toBe("Failure") + expect(requests).toHaveLength(1) + expect(output.content()).toBe("(no text output — tool-only response)") + return + } + const result = yield* pending + if (result.info.role === "assistant") accounting.onPromptResult(result.info) + if (budgetFinalization || ["echo-harness-deadline", "echo-user-cancelled"].includes(scenario)) { + if (scenario === "echo-budget-finalization-publishing") expect(publicationAborts).toBe(1) + if (scenario === "echo-user-cancelled") { + expect(result.info).toHaveProperty("error.name", "MessageAbortedError") + expect(result.parts.filter((part) => part.type === "text")).toEqual([]) + expect(textUpdates.filter((part) => part.time?.end)).toEqual([]) + expect(textDeltas).toEqual([]) + expect(requests).toHaveLength(2) + return + } + if (budgetFinalization) { + const messageID = MessageID.ascending() + const response = yield* Effect.promise(() => Promise.resolve(SessionRoutes().request(`/${chat.id}/message`, { + method: "POST", + headers: { + "content-type": "application/json", + "x-altimate-run-budget": "finalization", + traceparent: "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01", + }, + body: JSON.stringify({ + messageID, + agent: "analyst", + model: promptModel, + parts: [{ type: "text", text: SessionTermination.REPLY_AFTER_BUDGET, synthetic: true }], + }), + }))) + expect(response.status).toBe(200) + const finalized = yield* Effect.promise(() => response.json()) + expect(requests).toHaveLength(2) + expect(TraceContext.forMessage(messageID)).toBeUndefined() + expect(finalized.info.id).toBe(result.info.id) + expect(finalized.parts.filter((part: MessageV2.Part) => part.type === "text").map((part: MessageV2.TextPart) => part.text)) + .toEqual([renderedAnswer]) + } + expect(result.info).not.toHaveProperty("error") + expect(result.info).toHaveProperty("finish", "stop") + expect(result.parts.filter((part) => part.type === "text").map((part) => part.text)).toEqual([renderedAnswer]) + expect(textUpdates.filter((part) => part.time?.end).map((part) => part.text)).toEqual([renderedAnswer]) + expect(textDeltas.map((part) => part.text)).toEqual([renderedAnswer]) + expect(output.content()).toBe(renderedAnswer) + expect(requests).toHaveLength(2) + expect(requests[1].messages.at(-1)).toEqual({ role: "user", content: echoInstruction }) + const history = yield* sessions.messages({ sessionID: chat.id }) + const commit = history.flatMap((message) => message.parts) + .find((part) => part.type === "tool" && part.tool === "answer_commit") + expect(commit).toHaveProperty("state.metadata.committed_text", renderedAnswer) + return + } + if (userDuringFailedCompaction) { + if (!queuedUserID) throw new Error("missing queued user request") + const callbacks = yield* Effect.promise(() => Promise.all(queuedReplies)) + expect(callbacks).toHaveLength(2) + for (const reply of [result, ...callbacks]) { + expect(reply.info).toHaveProperty("parentID", queuedUserID) + expect(reply.info).toHaveProperty("finish", "stop") + expect(reply.info).not.toHaveProperty("error") + expect(reply.parts.filter((part) => part.type === "text").map((part) => part.text)).toEqual(["3"]) + expect(reply.info.id).toBe(result.info.id) + } + expect(requests).toHaveLength(emptySummary ? 4 : 3) + const summaries = requests.slice(1, emptySummary ? 3 : 2) + for (const request of summaries) { + expect(JSON.stringify(request.messages)).toContain("Provide a detailed prompt for continuing") + expect(request.tools ?? []).toEqual([]) + } + if (emptySummary) expect(summaries[1].messages).toEqual(summaries[0].messages) + const resumed = requests.at(-1)! + expect((resumed.tools ?? []).map((tool) => tool.function.name)).toContain("answer_commit") + expect(JSON.stringify(resumed.messages)).not.toContain(echoInstruction) + expect(JSON.stringify(resumed.messages)).toContain("A new question: how many sources?") + const history = yield* sessions.messages({ sessionID: chat.id }) + const commit = history.find((message) => message.parts.some((part) => part.type === "tool" && part.tool === "answer_commit")) + if (commit?.info.role !== "assistant") throw new Error("missing committed request") + const answers = history.filter((message) => message.info.role === "assistant" && + message.parts.some((part) => part.type === "text" && part.text === renderedAnswer)) + expect(answers).toHaveLength(1) + const answer = answers[0] + expect(answer.info).toMatchObject({ parentID: commit.info.parentID, finish: "stop" }) + expect(answer.info).not.toHaveProperty("summary", true) + expect(answer.info).not.toHaveProperty("error") + expect(answer.info).toHaveProperty("metadata.answer_commit_echo_error.data.message", emptySummary + ? "Compaction summarizer produced no summary text after retry" + : "Compaction summariser rejected the request") + const ordered = Database.use((db) => db + .select({ id: MessageTable.id }) + .from(MessageTable) + .where(eq(MessageTable.session_id, chat.id)) + .orderBy(MessageTable.time_created, sql`rowid`) + .all()) + const answerIndex = ordered.findIndex((message) => message.id === answer.info.id) + expect(answerIndex).toBeGreaterThanOrEqual(0) + expect(ordered.findIndex((message) => message.id === queuedUserID)).toBeGreaterThan(answerIndex) + const modelHistory = resumed.messages as { role: string; content: unknown }[] + const answerTurn = modelHistory.findIndex((message) => message.role === "assistant" && message.content === renderedAnswer) + const queuedTurn = modelHistory.findIndex((message) => message.role === "user" && + JSON.stringify(message.content).includes("A new question: how many sources?")) + expect(answerTurn).toBeGreaterThanOrEqual(0) + expect(queuedTurn).toBeGreaterThan(answerTurn) + expect(modelHistory.findLastIndex((message) => message.role === "user")).toBe(queuedTurn) + expect(textEvents.filter((event) => event.messageID === answer.info.id)).toEqual([ + { messageID: answer.info.id, type: "start", text: "" }, + { messageID: answer.info.id, type: "delta", text: renderedAnswer }, + { messageID: answer.info.id, type: "completion", text: renderedAnswer }, + ]) + // Run prints completed, non-compaction text; ACP consumes the same + // immutable event sequence through its real subscription handler. + expect(textUpdates.filter((part) => part.time?.end && !compactionTextIDs.includes(part.messageID)) + .map((part) => part.text)).toEqual([renderedAnswer, "3"]) + expect(output.content()).toBe("3") + expect(yield* acpText(clientEvents, chat.id)).toEqual([renderedAnswer, "3"]) + expect(sessionErrors).toEqual([]) + expect(assistantUpdates.filter((message) => message.error)).toEqual([]) + expect(accounting.fatal).toBe(false) + expect(requestStarts).toEqual([commit.info.parentID, queuedUserID]) + return + } + if (emptySummary) { + expect(requests).toHaveLength(3) + expect(JSON.stringify(requests[1].messages)).toContain("Provide a detailed prompt for continuing") + expect(requests[2].messages).toEqual(requests[1].messages) + expect(requests.slice(1).map((request) => request.tools ?? [])).toEqual([[], []]) + if (scenario === "compaction-empty-summary-without-commit") { + expect(result.info).toHaveProperty("error.name", "UnknownError") + expect(result.info).toHaveProperty( + "error.data.message", + "Compaction summarizer produced no summary text after retry", + ) + expect(result.info).toHaveProperty("finish", "error") + expect(result.info).not.toHaveProperty("metadata.answer_commit_echo_error") + expect(result.parts.filter((part) => part.type === "text")).toEqual([]) + expect(accounting.fatal).toBe(true) + const history = yield* sessions.messages({ sessionID: chat.id }) + expect(history.flatMap((message) => message.parts).filter( + (part) => part.type === "tool" && part.tool === "answer_commit", + )).toEqual([]) + return + } + } + if ( + scenario === "echo-snapshot-error" || + scenario === "echo-policy-denied" || + scenario === "echo-guardrail-denied" || + scenario === "echo-guardrail-overflow-text" || + scenario === "echo-forbidden" || + scenario === "echo-aborted" || + scenario === "echo-content-filter" || + scenario === "echo-http-content-filter" || + scenario === "compaction-aborted" || + scenario === "compaction-forbidden" || + scenario === "compaction-guardrail-denied" || + scenario === "compaction-content-filter" + ) { + expect(result.info).toHaveProperty( + "error.name", + scenario === "echo-snapshot-error" + ? "UnknownError" + : scenario === "echo-aborted" || scenario === "compaction-aborted" + ? "MessageAbortedError" + : scenario === "echo-http-content-filter" || scenario === "compaction-content-filter" || + scenario === "echo-policy-denied" || scenario === "echo-guardrail-denied" || scenario === "echo-guardrail-overflow-text" || scenario === "echo-forbidden" || + scenario === "compaction-forbidden" || scenario === "compaction-guardrail-denied" + ? "APIError" + : "ContentFilterError", + ) + expect(result.info).not.toHaveProperty("metadata.answer_commit_echo_error") + expect(result.info).not.toHaveProperty("finish", "stop") + expect(requests).toHaveLength(scenario === "echo-snapshot-error" ? 1 : 2) + expect(sessionErrors).toHaveLength(1) + expect(accounting.fatal).toBe(true) + expect(assistantUpdates.some((message) => message.id === result.info.id && message.error)).toBe(true) + if (scenario === "echo-guardrail-overflow-text") { + expect(compactions).toBe(0) + const history = yield* sessions.messages({ sessionID: chat.id }) + expect(history.flatMap((message) => message.parts).filter((part) => part.type === "compaction")).toEqual([]) + expect(textUpdates.filter((part) => part.time?.end)).toEqual([]) + expect(yield* acpText(clientEvents, chat.id)).toEqual([]) + } + if (scenario !== "echo-content-filter") { + expect(result.parts.filter((part) => part.type === "text")).toEqual([]) + expect(output.content()).toBe("(no text output — tool-only response)") + expect(textDeltas.filter((part) => part.messageID === result.info.id)).toEqual([]) + } else { + expect(result.parts).toContainEqual(expect.objectContaining({ type: "text", text: "42" })) + } + return + } + expect(result.info).not.toHaveProperty("error") + if (userAfterCompaction || overflowCodeCompaction) { + expect(compactions).toBe(1) + expect(requests).toHaveLength(3) + expect(JSON.stringify(requests[1].messages)).toContain("Provide a detailed prompt for continuing") + expect(requests[1].tools ?? []).toEqual([]) + expect((requests[2].tools ?? []).map((tool) => tool.function.name)).toContain("answer_commit") + expect(JSON.stringify(requests[2].messages)).not.toContain(echoInstruction) + expect(JSON.stringify(requests[2].messages)).toContain(summaryText) + const history = [...MessageV2.stream(chat.id)].reverse() + const summary = history.find((message) => message.info.role === "assistant" && message.info.summary) + if (!summary) throw new Error("missing successful compaction summary") + expect(summary.info).toHaveProperty("finish", "stop") + expect(summary.info).not.toHaveProperty("error") + expect(summary.parts.filter((part) => part.type === "text").map((part) => part.text)).toEqual([summaryText]) + const filtered = MessageV2.filterCompacted(MessageV2.stream(chat.id)) + expect(filtered[0].parts.some((part) => part.type === "compaction")).toBe(true) + expect(filtered[1].info.id).toBe(summary.info.id) + expect(accounting.fatal).toBe(false) + expect(result.parts.filter((part) => part.type === "text").map((part) => part.text)).toEqual(["3"]) + if (overflowCodeCompaction) { + expect(sessionErrors).toEqual([expect.objectContaining({ name: "ContextOverflowError" })]) + expect(history.flatMap((message) => message.parts)).toContainEqual(expect.objectContaining({ + type: "compaction", auto: true, overflow: true, + })) + return + } + expect(sessionErrors).toEqual([]) + if (!completedSummary) throw new Error("missing saved compaction summary") + expect(summary).toEqual(completedSummary) + const commit = history.find((message) => message.parts.some((part) => part.type === "tool" && part.tool === "answer_commit")) + if (commit?.info.role !== "assistant") throw new Error("missing committed request") + const answers = history.filter((message) => message.info.role === "assistant" && + message.parts.some((part) => part.type === "text" && part.text === renderedAnswer)) + expect(answers).toHaveLength(1) + const answer = answers[0] + if (answer.info.role !== "assistant") throw new Error("missing recovered committed answer") + expect(answer.info.id).not.toBe(summary.info.id) + expect(answer.info.id).not.toBe(commit.info.id) + expect(answer.info).toMatchObject({ + parentID: commit.info.parentID, + agent: commit.info.agent, + mode: commit.info.mode, + modelID: commit.info.modelID, + providerID: commit.info.providerID, + finish: "stop", + }) + expect(answer.info).not.toHaveProperty("summary") + expect(answer.info).not.toHaveProperty("error") + expect(answer.info).not.toHaveProperty("metadata.answer_commit_echo_error") + const ordered = Database.use((db) => db + .select({ id: MessageTable.id, created: MessageTable.time_created }) + .from(MessageTable) + .where(eq(MessageTable.session_id, chat.id)) + .orderBy(MessageTable.time_created, sql`rowid`) + .all()) + const summaryIndex = ordered.findIndex((message) => message.id === summary.info.id) + const answerIndex = ordered.findIndex((message) => message.id === answer.info.id) + const queuedIndex = ordered.findIndex((message) => message.id === queuedUserID) + expect(summaryIndex).toBeGreaterThanOrEqual(0) + expect(answerIndex).toBeGreaterThan(summaryIndex) + expect(queuedIndex).toBeGreaterThan(answerIndex) + expect(ordered[answerIndex].created).toBe(ordered[queuedIndex].created - 1) + expect(answer.info.time.completed).toBeGreaterThanOrEqual(answer.info.time.created) + const modelHistory = requests[2].messages as { role: string; content: unknown }[] + const answerTurn = modelHistory.findIndex((message) => message.role === "assistant" && message.content === renderedAnswer) + const queuedTurn = modelHistory.findIndex((message) => message.role === "user" && + JSON.stringify(message.content).includes("A new question: how many sources?")) + expect(answerTurn).toBeGreaterThanOrEqual(0) + expect(queuedTurn).toBeGreaterThan(answerTurn) + expect(modelHistory.findLastIndex((message) => message.role === "user")).toBe(queuedTurn) + expect(textEvents.filter((event) => event.messageID === answer.info.id)).toEqual([ + { messageID: answer.info.id, type: "start", text: "" }, + { messageID: answer.info.id, type: "delta", text: renderedAnswer }, + { messageID: answer.info.id, type: "completion", text: renderedAnswer }, + ]) + expect(yield* acpText(clientEvents, chat.id)).toEqual([renderedAnswer, "3"]) + expect(compactionTextIDs).not.toContain(answer.info.id) + expect(removedTextIDs).toEqual([]) + expect(queuedUserID).toBeDefined() + expect(result.info).toHaveProperty("parentID", queuedUserID) + expect(JSON.stringify(requests[2].messages)).toContain("A new question: how many sources?") + return + } + if (streamedSummary) { + const chunks = yield* acpText(clientEvents, chat.id) + expect(chunks).toEqual([renderedAnswer]) + expect(output.content()).toBe(renderedAnswer) + const partials = textDeltas.filter((part) => part.text.includes("Summary so far")) + expect(partials).toHaveLength(emptySummary ? 2 : 1) + const parts = result.parts.filter((part) => part.type === "text") + expect(parts.map((part) => part.text)).toEqual([renderedAnswer]) + for (const partial of partials) { + expect(removedTextIDs).toContain(partial.partID) + expect(parts[0].id).not.toBe(partial.partID) + } + expect(result.info).toHaveProperty("finish", "stop") + expect(result.info).not.toHaveProperty("summary", true) + expect(result.info).toHaveProperty("metadata.answer_commit_echo_error.data.message", emptySummary + ? "Compaction summarizer produced no summary text after retry" + : "Compaction summariser failed after partial text") + if (!emptySummary) expect(result.info).toHaveProperty("metadata.answer_commit_echo_error.data.statusCode", 503) + expect(sessionErrors).toEqual([]) + expect(accounting.fatal).toBe(false) + expect(requests).toHaveLength(emptySummary ? 3 : 2) + expect(JSON.stringify(requests[1].messages)).toContain("Provide a detailed prompt for continuing") + const persisted = (yield* sessions.messages({ sessionID: chat.id })).find((message) => message.info.id === result.info.id) + expect(persisted?.parts.filter((part) => part.type === "text").map((part) => part.text)).toEqual([renderedAnswer]) + return + } + if (scenario === "echo-reactive-compaction") { + expect(result.info).toHaveProperty("finish", "stop") + expect(result.info).toHaveProperty("metadata.answer_commit_echo_error.result", "compact") + expect(result.parts.filter((part) => part.type === "text").map((part) => part.text)).toEqual([renderedAnswer]) + expect(textUpdates.filter((part) => part.time?.end).map((part) => part.text)).toEqual([renderedAnswer]) + expect(textDeltas.map((part) => part.text)).toEqual([renderedAnswer]) + expect(output.content()).toBe(renderedAnswer) + expect(requests).toHaveLength(2) + expect(JSON.stringify(requests[1].messages)).toContain(echoInstruction) + const history = yield* sessions.messages({ sessionID: chat.id }) + expect(history.flatMap((message) => message.parts).filter((part) => part.type === "compaction")).toEqual([]) + const users = history.filter((message) => message.info.role === "user") + expect(users).toHaveLength(1) + expect(requestStarts).toEqual([users[0].info.id]) + expect(history.find((message) => message.info.id === result.info.id)?.info) + .toHaveProperty("metadata.answer_commit_echo_error.result", "compact") + return + } + if ( + scenario === "compaction-error" || scenario === "compaction-empty-summary" || + scenario === "echo-overflow-disabled" || scenario === "echo-overflow-enabled" + ) { + const cause = + scenario === "compaction-error" + ? "Compaction summariser rejected the request" + : emptySummary + ? "Compaction summarizer produced no summary text after retry" + : "maximum context length exceeded" + expect(result.info).toHaveProperty("metadata.answer_commit_echo_error.data.message", cause) + if (emptySummary) expect(result.info).toHaveProperty("metadata.answer_commit_echo_error.name", "UnknownError") + expect(result.info).toHaveProperty("finish", "stop") + expect(result.info).not.toHaveProperty("summary", true) + expect(result.parts.filter((part) => part.type === "text").map((part) => part.text)).toEqual([renderedAnswer]) + expect(textUpdates.filter((part) => part.messageID === result.info.id).map((part) => part.text)).toEqual([ + "", renderedAnswer, + ]) + expect(textDeltas.filter((part) => part.messageID === result.info.id).map((part) => part.text)).toEqual([ + renderedAnswer, + ]) + expect(output.content()).toBe(renderedAnswer) + expect(requests).toHaveLength(emptySummary ? 3 : 2) + expect(sessionErrors).toEqual([]) + expect(assistantUpdates.filter((message) => message.error)).toEqual([]) + expect(accounting.fatal).toBe(false) + expect(compactionTextIDs).not.toContain(result.info.id) + const history = yield* sessions.messages({ sessionID: chat.id }) + if (compacting) { + expect(JSON.stringify(requests[1].messages)).toContain("Provide a detailed prompt for continuing") + expect(JSON.stringify(requests[1].messages)).not.toContain(echoInstruction) + // Only the original request and the compaction marker exist: no continuation was created. + expect(history.filter((message) => message.info.role === "user")).toHaveLength(2) + const request = history.find( + (message) => message.info.role === "user" && !message.parts.some((part) => part.type === "compaction"), + ) + if (!request) throw new Error("missing original request") + expect(requestStarts).toEqual([request.info.id]) + } + const persisted = history.find((message) => message.info.id === result.info.id) + expect(persisted?.info).toHaveProperty("metadata.answer_commit_echo_error.data.message", cause) + expect(persisted?.info).not.toHaveProperty("error") + return + } + if (scenario === "echo-transport-error" || scenario === "echo-socket-reset") { + const cause = scenario === "echo-socket-reset" + ? "Cannot connect to API: socket reset" + : "Echo transport rejected the request" + expect(sessionErrors).toEqual([]) + expect(assistantUpdates.filter((message) => message.error)).toEqual([]) + expect(result.info).toHaveProperty( + "metadata.answer_commit_echo_error.data.message", + cause, + ) + const persisted = (yield* sessions.messages({ sessionID: chat.id })).find( + (message) => message.info.id === result.info.id, + ) + expect(persisted?.info).toHaveProperty( + "metadata.answer_commit_echo_error.data.message", + cause, + ) + expect(persisted?.info).not.toHaveProperty("error") + } + expect(requests).toHaveLength(priorCommit ? 4 : userDuringEcho ? 3 : 2) + const tools = (index: number) => (requests[index].tools ?? []).map((tool) => tool.function.name) + if (structured) { + for (const request of priorCommit ? requests.slice(2) : requests) { + expect((request.tools ?? []).map((tool) => tool.function.name)).not.toContain("answer_commit") + expect((request.tools ?? []).map((tool) => tool.function.name)).toContain("StructuredOutput") + expect(request.tool_choice).toBe("required") + expect(JSON.stringify(request.messages)).not.toContain(echoInstruction) + if (priorCommit) { + expect(JSON.stringify(request.messages)).toContain("Previous committed answer was masked.") + expect(JSON.stringify(request.messages)).not.toContain("UNMASKED_COMMIT_OUTPUT") + } + } + expect(result.info).toHaveProperty("structured", { count: 42 }) + } else { + expect(tools(0)).toContain("answer_commit") + const messages = yield* sessions.messages({ sessionID: chat.id }) + const commit = messages + .flatMap((message) => message.parts) + .find((part) => part.type === "tool" && part.tool === "answer_commit") + expect(commit?.type === "tool" && commit.state.status).toBe(scenario === "failed" ? "error" : "completed") + if (scenario === "failed-after-success") { + expect( + messages + .flatMap((message) => message.parts) + .filter((part) => part.type === "tool" && part.tool === "answer_commit") + .map((part) => part.type === "tool" && part.state.status), + ).toEqual(["completed", "error"]) + } + if (scenario === "text" || scenario === "sibling-tool" || scenario.startsWith("echo-")) { + if (commit?.type !== "tool" || commit.state.status !== "completed") throw new Error("missing commit") + expect(tools(1)).toEqual([]) + expect(JSON.stringify(requests[1].messages)).toContain(echoInstruction) + // The echo instruction must be the final user turn: Bedrock's Converse API rejects + // a conversation that ends with an assistant message. + const last = requests[1].messages.at(-1) as { role?: string; content?: unknown } + expect(last.role).toBe("user") + expect(JSON.stringify(last.content)).toContain(echoInstruction) + if (tinyCap) { + expect(Token.estimate(commit.state.output)).toBeLessThanOrEqual(100) + if (scenario === "echo-capped") { + expect(Token.estimate(renderedAnswer)).toBeGreaterThan(100) + expect(commit.state.output).not.toBe(renderedAnswer) + expect(commit.state.output).toContain("truncated before dispatch") + } else { + expect(commit.state.output).toBe(renderedAnswer) + } + expect(requests[1].messages).toContainEqual( + expect.objectContaining({ content: expect.stringContaining(commit.state.output) }), + ) + expect(last.content).toBe(echoInstruction) + } + const expectedText = renderedAnswer + expect(result.info).toHaveProperty("finish", scenario === "echo-other" ? "other" : "stop") + if (scenario === "echo-other") expect(result.info).toHaveProperty("metadata.answer_commit_echo_error.finish", "other") + if (scenario === "echo-other") expect(accounting.fatal).toBe(true) + if (scenario === "echo-write-order") { + expect(finishedEchoText.length).toBeGreaterThan(0) + expect(finishedEchoText.every((texts) => texts.length === 1 && texts[0] === renderedAnswer)).toBe(true) + } + expect(result.parts).toContainEqual(expect.objectContaining({ type: "text", text: expectedText })) + if (scenario === "echo-no-text") { + const part = result.parts.find((part) => part.type === "text") + if (part?.type !== "text" || !part.time) throw new Error("missing completed echo text") + expect(typeof part.time.start).toBe("number") + expect(typeof part.time.end).toBe("number") + expect(part.time.end).toBeGreaterThanOrEqual(part.time.start) + } + expect( + result.parts + .filter((part) => part.type === "text") + .map((part) => part.text) + .join(""), + ).toBe(expectedText) + const finalUpdates = textUpdates.filter((part) => part.messageID === result.info.id) + if (scenario === "text" || scenario === "echo-mismatch" || scenario === "echo-write-order") { + // The ordinary echo reuses its buffered part, with one delivery. + expect(completedTextIDs).toEqual([finalUpdates[0].id]) + expect(finalUpdates.map((part) => part.id)).toEqual([completedTextIDs[0], completedTextIDs[0]]) + expect(removedTextIDs).toEqual([]) + expect(yield* acpText(clientEvents, chat.id)).toEqual([expectedText]) + } + expect(finalUpdates.filter((part) => part.time?.end).map((part) => part.text)).toEqual([expectedText]) + expect(output.content()).toBe(expectedText) + expect( + textDeltas + .filter((part) => part.messageID === result.info.id) + .map((part) => part.text), + ).toEqual([expectedText]) + if (scenario === "echo-capped") expect(commit.state.metadata.committed_text).toBe(renderedAnswer) + if (scenario === "text" || scenario === "echo-transport-error" || scenario === "echo-other") { + // A correct echo publishes only its initial and completed text parts. + expect(finalUpdates).toHaveLength(2) + } + } else if (userDuringEcho) { + if (commit?.type !== "tool" || commit.state.status !== "completed") throw new Error("missing commit") + const commitMessage = messages.find((message) => message.info.id === commit.messageID) + if (commitMessage?.info.role !== "assistant") throw new Error("missing commit message") + const committedUserID = commitMessage.info.parentID + const echo = messages.find( + (message) => message.info.role === "assistant" && + message.info.parentID === committedUserID && message.info.finish === "stop", + ) + if (!echo) throw new Error("missing completed echo for the original request") + expect(echo.info).not.toHaveProperty("error") + expect(echo.parts.filter((part) => part.type === "text").map((part) => part.text)).toEqual([renderedAnswer]) + expect(textEvents.filter((event) => event.messageID === echo.info.id)).toEqual([ + { messageID: echo.info.id, type: "start", text: "" }, + { messageID: echo.info.id, type: "delta", text: renderedAnswer }, + { messageID: echo.info.id, type: "completion", text: renderedAnswer }, + ]) + expect(tools(1)).toEqual([]) + expect(JSON.stringify(requests[1].messages)).toContain(echoInstruction) + expect(tools(2)).toContain("answer_commit") + expect(JSON.stringify(requests[2].messages)).not.toContain(echoInstruction) + expect(JSON.stringify(requests[2].messages)).toContain("A new question: how many sources?") + expect(queuedUserID).toBeDefined() + expect(result.info).toHaveProperty("parentID", queuedUserID) + expect(result.info).toHaveProperty("finish", "stop") + expect(result.parts.filter((part) => part.type === "text").map((part) => part.text)).toEqual(["3"]) + } else { + expect(tools(1)).toContain("answer_commit") + expect(JSON.stringify(requests[1].messages)).not.toContain(echoInstruction) + if (queuedUser || userAfterCommit) { + expect(queuedUserID).toBeDefined() + expect(JSON.stringify(requests[1].messages)).toContain("A new question: how many sources?") + expect(result.info).toHaveProperty("parentID", queuedUserID) + expect(result.parts.filter((part) => part.type === "text").map((part) => part.text)).toEqual(["3"]) + if (reversedIDs) { + const originalAnswer = messages.find( + (message) => message.info.role === "assistant" && + message.info.parentID === originalUserID && message.info.finish === "stop", + ) + expect(originalAnswer?.parts.filter((part) => part.type === "text").map((part) => part.text)) + .toEqual([renderedAnswer]) + expect(textUpdates.filter((part) => part.messageID === originalAnswer?.info.id && part.time?.end) + .map((part) => part.text)).toEqual([renderedAnswer]) + } + } + } + } + } finally { + GlobalBus.off("event", collect) + commitSpy?.mockRestore() + compactionSpy?.mockRestore() + finishSpy?.mockRestore() + retrySpy?.mockRestore() + snapshotSpy.mockRestore() + historySpy.mockRestore() + languageSpy.mockRestore() + streamSpy?.mockRestore() + textHookSpy?.mockRestore() + } + }), + ), + { + config: { + snapshot: false, + learn: { enabled: false }, + ...(scenario === "echo-overflow-disabled" ? { compaction: { auto: false } } : {}), + ...(tinyCap ? { tool_output: { dispatch_max_tokens: 100 } } : {}), + agent: { analyst: { permission: { StructuredOutput: "allow" } } }, + provider: { + test: { + name: "Test", + npm: "@ai-sdk/openai-compatible", + env: [], + models: { + "test-model": { + name: "Test Model", + tool_call: true, + limit: { context: 100000, output: 10000 }, + }, + }, + options: { apiKey: "test-key", baseURL: "https://unused.invalid/v1" }, + }, + }, + }, + }, + 30_000, + ) +} + +describe("committed-answer echo error classification", () => { + for (const statusCode of [408, 429, 500, 503, 599]) { + test(`HTTP ${statusCode} is a recoverable echo transport failure`, () => { + const error = new MessageV2.APIError({ message: "Request failed", statusCode, isRetryable: false }).toObject() + expect(isEchoTransportError(error)).toBe(true) + }) + } + for (const statusCode of [400, 401, 403, 404]) { + test(`HTTP ${statusCode} is never a recoverable echo transport failure`, () => { + const error = new MessageV2.APIError({ message: "Request failed", statusCode, isRetryable: true }).toObject() + expect(isEchoTransportError(error)).toBe(false) + }) + } + for (const code of [ + "policy_violation", "content_filter", "content_policy", "content_policy_violation", + "guardrail_intervened", "safety_rejection", "moderation_blocked", + ]) { + test(`structured ${code} overrides an otherwise recoverable HTTP status`, () => { + const error = new MessageV2.APIError({ + message: "Request denied", + statusCode: 503, + isRetryable: true, + responseBody: JSON.stringify({ error: { code } }), + }).toObject() + expect(isEchoTransportError(error)).toBe(false) + }) + } + test("a policy code beyond the response-body limit survives API error normalization", () => { + const error = MessageV2.fromError(new APICallError({ + message: "Request denied", + url: "https://example.com/v1/chat/completions", + requestBodyValues: {}, + statusCode: 503, + isRetryable: true, + responseBody: JSON.stringify({ details: "x".repeat(5000), error: { code: "policy_violation" } }), + }), { providerID: promptModel.providerID }) + expect(error.name).toBe("APIError") + expect(isEchoTransportError(error)).toBe(false) + }) + test("a structured policy code never overrides an explicit context overflow code", () => { + const error = MessageV2.fromError(new APICallError({ + message: "maximum context length exceeded", + url: "https://example.com/v1/chat/completions", + requestBodyValues: {}, + statusCode: 400, + isRetryable: false, + responseBody: JSON.stringify({ + error: { code: "context_length_exceeded", message: "maximum context length exceeded" }, + orchestrationError: { code: "policy_violation" }, + }), + }), { providerID: promptModel.providerID }) + expect(error.name).toBe("ContextOverflowError") + expect(isEchoTransportError(error)).toBe(false) + }) + test("a 503 orchestration guardrail code survives normalization and vetoes echo recovery", () => { + const error = MessageV2.fromError(new APICallError({ + message: "Request denied", + url: "https://example.com/v1/chat/completions", + requestBodyValues: {}, + statusCode: 503, + isRetryable: true, + responseBody: JSON.stringify({ orchestrationError: { code: "guardrail_intervened" } }), + }), { providerID: promptModel.providerID }) + expect(error.name).toBe("APIError") + expect(isEchoTransportError(error)).toBe(false) + }) + test("a 413 content_length_exceeded code remains a context overflow", () => { + const error = MessageV2.fromError(new APICallError({ + message: "Request entity too large", + url: "https://example.com/v1/chat/completions", + requestBodyValues: {}, + statusCode: 413, + isRetryable: false, + responseBody: JSON.stringify({ error: { code: "content_length_exceeded" } }), + }), { providerID: promptModel.providerID }) + expect(error.name).toBe("ContextOverflowError") + expect(isEchoTransportError(error)).toBe(false) + }) + test("a wrapped socket reset without a status remains recoverable even when the SDK disables retries", () => { + const error = MessageV2.fromError(new APICallError({ + message: "Connection interrupted", + url: "https://example.com/v1/chat/completions", + requestBodyValues: {}, + isRetryable: false, + cause: Object.assign(new Error("socket reset"), { code: "ECONNRESET" }), + }), { providerID: promptModel.providerID }) + expect(error.name).toBe("APIError") + expect(isEchoTransportError(error)).toBe(true) + }) + test("statusless socket and watchdog failures are recoverable but local failures and auth are not", () => { + for (const code of ["ECONNRESET", "ETIMEDOUT", "ProviderStreamIdleTimeoutError"]) { + const error = new MessageV2.APIError({ + message: "Transport interrupted", + isRetryable: false, + metadata: { code }, + }).toObject() + expect(isEchoTransportError(error)).toBe(true) + } + expect(isEchoTransportError( + new MessageV2.APIError({ message: "Unclassified failure", isRetryable: false }).toObject(), + )).toBe(false) + expect(isEchoTransportError( + MessageV2.fromError(new Error("Snapshot tracking failed"), { providerID: promptModel.providerID }), + )).toBe(false) + expect(isEchoTransportError( + new MessageV2.AuthError({ message: "Authentication failed", providerID: promptModel.providerID }).toObject(), + )).toBe(false) + }) +}) +// altimate_change end + +test("analyst prompt gives StructuredOutput precedence only when an output JSON schema is supplied", async () => { + const prompt = await Bun.file(new URL("../../src/altimate/prompts/analyst.txt", import.meta.url)).text() + expect(prompt).toContain( + "When the request supplies an output JSON schema (the `StructuredOutput` tool is available), finish with `StructuredOutput` instead of `answer_commit` or a plain-text reply; follow that schema. The text finalization rules below apply when no output schema is supplied.", + ) + expect(prompt).not.toContain("When a JSON schema is requested") +}) diff --git a/packages/opencode/test/altimate/sql-validation-e2e.test.ts b/packages/opencode/test/altimate/sql-validation-e2e.test.ts index 62b1c22453..9e61a73cab 100644 --- a/packages/opencode/test/altimate/sql-validation-e2e.test.ts +++ b/packages/opencode/test/altimate/sql-validation-e2e.test.ts @@ -93,7 +93,7 @@ describe("Tool name consistency in prompts", () => { fn: async () => { const analyst = await Agent.get("analyst") expect(analyst).toBeDefined() - for (const name of ["sql_analyze", "sql_materialize", "task", "skill", "read"]) { + for (const name of ["sql_analyze", "sql_materialize", "answer_commit", "task", "skill", "read"]) { expect(analyst!.prompt).toContain(name) } expect(analyst!.prompt).not.toContain("altimate_core_validate") @@ -101,6 +101,34 @@ describe("Tool name consistency in prompts", () => { }) }) + test("analyst prompt requires deterministic answer finalization", async () => { + const prompt = await Bun.file(path.join(import.meta.dir, "../../src/altimate/prompts/analyst.txt")).text() + const paragraph = prompt.trim().split("\n\n").at(-1) + // altimate_change start - require one field for each requested answer item + expect(paragraph).toContain( + "Finish by calling `answer_commit` with the single chosen value(s) with one field per item the request asks for, a one-or-two-sentence basis (population, grain, source object/column), or `unresolved` with the reason, then reply with exactly the text it returns and nothing else.", + ) + // altimate_change end + expect(paragraph).toContain("Always supply `basis`, even when unresolved") + expect(paragraph).toContain("it is never shown when the request asks for only the answer") + expect(paragraph).toContain("Never list excluded candidates or alternative answers") + expect(paragraph).toContain("when the request asks for only the answer or a specific format, set `answer_only`") + }) + + test("analyst prompt preserves the top-N boundary-tie contract", async () => { + const prompt = await Bun.file(path.join(import.meta.dir, "../../src/altimate/prompts/analyst.txt")).text() + expect(prompt.trim().split("\n\n").at(-1)).toContain( + "a top-N with a tie at the boundary still returns N concrete rows (note the tie briefly in `rows.note`)", + ) + }) + + test("analyst prompt unavailable-tool fallback omits the answer_only label and basis", async () => { + const prompt = await Bun.file(path.join(import.meta.dir, "../../src/altimate/prompts/analyst.txt")).text() + expect(prompt.trim().split("\n\n").at(-1)).toContain( + "If `answer_commit` is unavailable, finish with one or two sentences of basis and a final line `**Answer:** `, or `**Answer:** unable to determine — `; when the request asks for only the answer or a specific format, the final message is exactly the requested value(s) or table in the requested format, with no label and no basis (or `unable to determine — ` if unresolved); never guess.", + ) + }) + test("builder prompt contains pre-execution protocol with correct tool names", async () => { await using tmp = await tmpdir() await Instance.provide({ diff --git a/packages/opencode/test/altimate/tools/answer-commit.test.ts b/packages/opencode/test/altimate/tools/answer-commit.test.ts new file mode 100644 index 0000000000..0d371de6c6 --- /dev/null +++ b/packages/opencode/test/altimate/tools/answer-commit.test.ts @@ -0,0 +1,1071 @@ +import { describe, expect, test } from "bun:test" +import Ajv2020 from "ajv/dist/2020" +import { rejects } from "node:assert/strict" +import { Effect } from "effect" +import { AnswerCommit } from "../../../src/altimate/tools/answer-commit" +import * as answerCommitModule from "../../../src/altimate/tools/answer-commit" +import { MessageID, SessionID } from "../../../src/session/schema" +import { ToolJsonSchema } from "../../../src/tool/json-schema" +import { initTool, toolInfo } from "../tool-fixture" + +test("answer_commit exposes the namespace projection", () => { + expect(answerCommitModule).toHaveProperty("AnswerCommit.AnswerCommitTool", AnswerCommit.AnswerCommitTool) + expect(answerCommitModule).toHaveProperty("AnswerCommit.renderAnswer", AnswerCommit.renderAnswer) +}) + +describe("answer_commit boolean tolerance", () => { + test.each([true, "true", "TRUE", "TrUe", 1])("answer_only %s suppresses basis and labels", async (answer_only) => { + const payload = { + answer_only, + basis: "Counted records at record grain using records.id.", + fields: [{ name: "Count", value: 2 }], + } + expect(AnswerCommit.renderAnswer(payload)).toBe("2") + const tool = await initTool(AnswerCommit.AnswerCommitTool) + expect((await tool.execute(payload, context)).output).toBe("2") + }) + + test.each([false, "false", "FALSE", "FaLsE", 0])("answer_only %s renders the basis", async (answer_only) => { + const basis = "Counted records at record grain using records.id." + const payload = { answer_only, basis, fields: [{ name: "Count", value: 2 }] } + expect(AnswerCommit.renderAnswer(payload)).toBe(`${basis}\n\n**Answer:** Count: 2`) + const tool = await initTool(AnswerCommit.AnswerCommitTool) + expect((await tool.execute(payload, context)).output).toBe(`${basis}\n\n**Answer:** Count: 2`) + }) + + test.each(["yes", "no", "1", "0", 2, -1])("rejects invalid answer_only %s", async (answer_only) => { + const payload = { + answer_only, + basis: "Counted records at record grain using records.id.", + fields: [{ name: "", value: 2 }], + } + expect(() => AnswerCommit.renderAnswer(payload)).toThrow(/answer_only.*boolean/i) + const tool = await initTool(AnswerCommit.AnswerCommitTool) + await rejects(tool.execute(payload, context), /answer_only.*boolean/i) + }) + + test("keeps the advertised answer_only value boolean", async () => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + expect(tool.jsonSchema?.properties?.answer_only).toMatchObject({ anyOf: [{ type: "boolean" }, {}, {}, {}, {}] }) + }) +}) + +describe("answer_commit required basis", () => { + test.each([undefined, false, "false", 0, null, "", [], {}, true, "true", "TRUE", "TrUe", 1].map((value) => [value]))( + "answer_only %j requires basis for every answer shape", + async (answer_only) => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + for (const shape of [ + { fields: [{ name: "", value: 2 }] }, + { rows: { columns: ["Count"], values: [[2]] } }, + { unresolved: { reason: "No matching records." } }, + ]) { + const payload = { ...shape, answer_only } + const error = /basis.*required.*population.*grain.*source object\/column.*caveats/i + expect(() => AnswerCommit.renderAnswer(payload)).toThrow(error) + await rejects(tool.execute(payload, context), error) + } + }, + ) + + test.each([undefined, null, "", " \t\r\n ", [], {}].map((value) => [value]))( + "basis filler %j cannot replace a required basis", + async (basis) => { + const payload = { fields: [{ name: "", value: 2 }], basis } + expect(() => AnswerCommit.renderAnswer(payload)).toThrow(/basis.*required/i) + const tool = await initTool(AnswerCommit.AnswerCommitTool) + await rejects(tool.execute(payload, context), /basis.*required/i) + }, + ) + + test.each([undefined, null, "", " \t\r\n ", [], {}].map((value) => [value]))( + "answer_only rejects absent basis %j", + async (basis) => { + const payload = { answer_only: "true", fields: [{ name: "Count", value: 2 }], basis } + expect(() => AnswerCommit.renderAnswer(payload)).toThrow(/basis.*required/i) + const tool = await initTool(AnswerCommit.AnswerCommitTool) + await rejects(tool.execute(payload, context), /basis.*required/i) + }, + ) + + test("description and schema explain the basis requirement and attribution source", async () => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + expect(tool.description).toContain( + "Always supply `basis`; it is never shown when the request asks for only the answer", + ) + expect(tool.description).toContain( + "one or two sentences, at most 400 characters, naming population, grain, the source object/column that supplies the requested entity or value, and caveats", + ) + expect(tool.description).toContain( + "Resolve policy eligibility separately from metric and entity attribution: a rule about a related object does not change which object supplies the requested entity. Say in `basis` which source object and column supply it.", + ) + expect(structuredClone(tool.jsonSchema?.properties?.basis)).toMatchObject({ + type: "string", + description: expect.stringMatching( + /always required.*one or two sentences, at most 400 characters.*population.*grain.*source object\/column.*caveats.*never rendered with answer_only/i, + ), + }) + expect(tool.jsonSchema?.required).toContain("basis") + }) +}) + +describe("answer_commit validation repair", () => { + test.each([ + ["invalid boolean", { answer_only: "yes", fields: [{ name: "", value: 2 }] }, /answer_only/], + ["missing basis", { fields: [{ name: "", value: 2 }] }, /basis/], + [ + "answer_only fields missing basis", + { answer_only: true, fields: [{ name: "Id", value: "requested-id" }] }, + /basis/, + ], + [ + "answer_only rows missing basis", + { answer_only: true, rows: { columns: ["Id"], values: [["requested-id"]] } }, + /basis/, + ], + [ + "answer_only unresolved missing basis", + { answer_only: true, unresolved: { reason: "No matching records." } }, + /basis/, + ], + ["missing value", { fields: [{ name: "Count" }] }, /fields.*value/], + ["missing shape", { basis: "Counted records at record grain using records.id." }, /fields, rows, or unresolved/], + ] as [string, unknown, RegExp][])( + "%s returns a one-line error and a valid repair example", + async (_name, payload, expected) => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + for (const execute of [() => AnswerCommit.renderAnswer(payload), () => tool.execute(payload, context)]) { + let message = "" + try { + await execute() + } catch (error) { + message = error instanceof Error ? error.message : String(error) + } + const lines = message.split("\n") + expect(lines[0]).toMatch(expected) + expect(lines[1]).toStartWith("Minimal valid example: ") + const exampleText = lines[1]!.match(/^Minimal valid example: `(.*)`\.?$/)?.[1] + expect(exampleText).toBeDefined() + const example = JSON.parse(exampleText!) + expect(AnswerCommit.renderAnswer(example)).toBeString() + expect((await tool.execute(example, context)).output).toBe(AnswerCommit.renderAnswer(example)) + } + }, + ) +}) + +test("answer_commit executes synchronously with native Effect permissions", async () => { + const info = await toolInfo(AnswerCommit.AnswerCommitTool) + const tool = await Effect.runPromise(info.init()) + const asks: unknown[] = [] + const result = Effect.runSync( + tool.execute( + { fields: [{ name: "Count", value: 2 }], basis: "Counted records at record grain using records.id." }, + { + ...context, + sessionID: SessionID.make("ses_test"), + messageID: MessageID.make("msg_test"), + metadata: () => Effect.void, + ask: (input) => Effect.sync(() => void asks.push(input)), + }, + ), + ) + expect(result.output).toBe("Counted records at record grain using records.id.\n\n**Answer:** Count: 2") + expect(asks).toEqual([{ permission: "answer_commit", patterns: ["*"], always: ["*"], metadata: {} }]) +}) + +const recordBasis = "Used all records at record grain from records.value." + +// altimate_change start - preserve the requested tabular output format +describe("answer_commit row formats", () => { + test.each([ + { + name: "csv quotes commas, quotes and newlines and includes headers", + rows: { + format: "csv", + columns: ['Name, "label"', "Value"], + values: [ + ['West, "Co"', null], + ["line\r\nbreak", " 001.20|\\ "], + ], + }, + expected: '"Name, ""label""",Value\n"West, ""Co""",\n"line\r\nbreak", 001.20|\\ ', + }, + { + name: "tsv quotes tabs, quotes and newlines and includes headers", + rows: { + format: "tsv", + columns: ["Name\tlabel", "Value"], + values: [ + ["West\tCo", null], + ['a"b', "line\nbreak"], + ["line\rbreak", "comma, unchanged"], + ], + }, + expected: '"Name\tlabel"\tValue\n"West\tCo"\t\n"a""b"\t"line\nbreak"\n"line\rbreak"\tcomma, unchanged', + }, + { + name: "lines renders a single-column list without headers", + rows: { format: "lines", columns: ["Name"], values: [["West"], ["East"]] }, + expected: "West\nEast", + }, + { + name: "lines separates two-column cells with one space", + rows: { + format: "lines", + columns: ["Name", "Count"], + values: [ + ["West|\\", 5], + ["East", "001.20"], + ["Missing", null], + ], + precision: { Count: 2 }, + }, + expected: "West|\\ 5.00\nEast 001.20\nMissing null", + }, + { + name: "csv applies precision to numeric cells and preserves strings", + rows: { + format: "csv", + columns: ["Count", "Rate", "Unspecified"], + values: [ + [5, 1.234, 2.5], + ["005.00", "1.2300", null], + ], + precision: { Count: 2, Rate: 1 }, + }, + expected: "Count,Rate,Unspecified\n5.00,1.2,2.5\n005.00,1.2300,", + }, + { + name: "default remains markdown", + rows: { + columns: ["Name", "Count"], + values: [ + ["West|\\", 5], + ["East", null], + ], + }, + expected: "| Name | Count |\n|---|---|\n| West\\|\\\\ | 5 |\n| East | null |", + }, + { + name: "explicit markdown retains table rendering", + rows: { format: "markdown", columns: ["Count"], values: [[5]], precision: { Count: 2 } }, + expected: "| Count |\n|---|\n| 5.00 |", + }, + ])("$name", async ({ rows, expected }) => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + for (const answer_only of [true, false]) { + const payload = { basis: recordBasis, answer_only, rows: { ...rows, note: "Result note." } } + const output = answer_only ? expected : `${recordBasis}\n\n**Answer:**\n${expected}\nResult note.` + expect(AnswerCommit.renderAnswer(payload)).toBe(output) + expect((await tool.execute(payload, context)).output).toBe(output) + } + }) + + test("description and schema advertise the tabular formats", async () => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + expect(tool.description).toMatch(/when the request names a tabular format, set rows.format accordingly/i) + expect(structuredClone(tool.jsonSchema?.properties?.rows)).toMatchObject({ + anyOf: expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + format: expect.objectContaining({ + anyOf: expect.arrayContaining([{ type: "string", enum: ["markdown", "csv", "tsv", "lines"] }]), + description: expect.stringMatching(/markdown.*default.*csv.*tsv.*lines/i), + }), + }), + }), + ]), + }) + }) + + test.each(["csv", "tsv", "lines"])("%s retains the rendered-size cap", async (format) => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + const payload = { + basis: recordBasis, + answer_only: true, + rows: { format, columns: ["Value"], values: [['"'.repeat(format === "lines" ? 60_001 : 30_000)]] }, + } + const error = /60,000 characters.*aggregate.*narrow/i + expect(() => AnswerCommit.renderAnswer(payload)).toThrow(error) + await rejects(tool.execute(payload, context), error) + }) +}) +// altimate_change end + +describe("answer_commit numeric safety", () => { + test("description explains numeric precision and exact decimal strings", async () => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + expect(tool.description).toContain( + "Numbers are accepted up to 15 significant digits; any decimal that must be reproduced exactly (more digits, trailing zeros, or currency strings) must be passed as a string.", + ) + }) + + test.each(["9007199254740993", "-9007199254740993", "1e30"])("rejects unsafe field number %s", async (value) => { + const payload = JSON.parse(`{"fields":[{"name":"Count","value":${value}}]}`) + expect(() => AnswerCommit.renderAnswer(payload)).toThrow(/fields.*value.*safe.*string/i) + const tool = await initTool(AnswerCommit.AnswerCommitTool) + await rejects(tool.execute(payload, context), /fields.*value.*safe.*string/i) + }) + + test.each(["9007199254740993", "-9007199254740993", "1e30"])("rejects unsafe table number %s", async (value) => { + const payload = JSON.parse(`{"rows":{"columns":["Count"],"values":[[${value}]]}}`) + expect(() => AnswerCommit.renderAnswer(payload)).toThrow(/rows.*values.*safe.*string/i) + const tool = await initTool(AnswerCommit.AnswerCommitTool) + await rejects(tool.execute(payload, context), /rows.*values.*safe.*string/i) + }) + + test.each(["1234567890123456.7", "9007199254740991.1", "-1234567890123456.7", "0.1234567890123456"])( + "rejects precision-losing parsed number %s in fields and rows", + async (literal) => { + const value = JSON.parse(literal) + const tool = await initTool(AnswerCommit.AnswerCommitTool) + for (const shape of [ + { fields: [{ name: "", value }] }, + { rows: { columns: ["Value"], values: [[value]] } }, + ]) { + const payload = { basis: recordBasis, ...shape } + const error = /15 significant digits.*exact value as a string/i + expect(() => AnswerCommit.renderAnswer(payload)).toThrow(error) + await rejects(tool.execute(payload, context), error) + } + }, + ) + + test.each([ + 999999999999999, + -999999999999999, + 46919.15, + 12, + 0.1, + -3.5e-7, + 1.25, + "9007199254740993", + "-9007199254740993", + "1234567890123456.7", + "9007199254740991.1", + ])( + "preserves safe numbers and numeric strings: %s", + async (value) => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + const fields = { basis: recordBasis, fields: [{ name: "", value }] } + const rows = { basis: recordBasis, rows: { columns: ["Count"], values: [[value]] } } + expect(AnswerCommit.renderAnswer(fields)).toBe(`${recordBasis}\n\n**Answer:** ${value}`) + expect((await tool.execute(fields, context)).output).toBe(`${recordBasis}\n\n**Answer:** ${value}`) + expect(AnswerCommit.renderAnswer(rows)).toBe(`${recordBasis}\n\n**Answer:**\n| Count |\n|---|\n| ${value} |`) + expect((await tool.execute(rows, context)).output).toBe( + `${recordBasis}\n\n**Answer:**\n| Count |\n|---|\n| ${value} |`, + ) + }, + ) +}) + +describe("answer_commit display precision", () => { + test("description and field and row schemas state the allowed precision range", async () => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + const range = /integer from 0 to 10 \(inclusive\)/i + expect(tool.description).toMatch(range) + expect(structuredClone(tool.jsonSchema?.properties?.fields)).toMatchObject({ + anyOf: expect.arrayContaining([ + expect.objectContaining({ + items: expect.objectContaining({ + properties: expect.objectContaining({ + precision: expect.objectContaining({ description: expect.stringMatching(range) }), + }), + }), + }), + ]), + }) + expect(structuredClone(tool.jsonSchema?.properties?.rows)).toMatchObject({ + anyOf: expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + precision: expect.objectContaining({ description: expect.stringMatching(range) }), + }), + }), + ]), + }) + }) + + test.each([-1, 11, 14, 1.5, "2", true])("invalid precision %j names the allowed range", async (precision) => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + for (const shape of [ + { fields: [{ name: "", value: 1.5, precision }] }, + { rows: { columns: ["Value"], values: [[5]], precision: { Value: precision } } }, + ]) { + const payload = { basis: recordBasis, ...shape } + const error = /precision.*integer from 0 to 10 \(inclusive\)/i + expect(() => AnswerCommit.renderAnswer(payload)).toThrow(error) + await rejects(tool.execute(payload, context), error) + } + }) + + test.each([ + ["numeric integers use fixed decimals", 5, 2, "5.00"], + ["without precision uses the shortest numeric form", 5, undefined, "5"], + ["numeric strings retain trailing zeros", "5.00", 0, "5.00"], + ["numeric strings retain leading zeros", "001.20", 10, "001.20"], + ["text is unchanged", "unchanged", 2, "unchanged"], + ["null is unchanged", null, 2, "null"], + ["zero precision rounds numeric cells", 1.6, 0, "2"], + ["maximum precision includes ten decimals", 5, 10, "5.0000000000"], + ] as [string, string | number | null, number | undefined, string][])( + "rows %s", + async (_name, value, precision, rendered) => { + const payload = { + basis: recordBasis, + rows: { + columns: ["Value"], + values: [[value]], + ...(precision === undefined ? {} : { precision: { Value: precision } }), + }, + } + const expected = `${recordBasis}\n\n**Answer:**\n| Value |\n|---|\n| ${rendered} |` + expect(AnswerCommit.renderAnswer(payload)).toBe(expected) + const tool = await initTool(AnswerCommit.AnswerCommitTool) + expect((await tool.execute(payload, context)).output).toBe(expected) + }, + ) + + test("row precision applies independently by column name", async () => { + const payload = { + basis: recordBasis, + answer_only: true, + rows: { + columns: ["Quantity", "Rate", "Unspecified"], + values: [ + [5, 1.234, 5], + ["5.00", null, "001.20"], + ], + precision: { Rate: 1, Quantity: 2 }, + }, + } + const expected = "| Quantity | Rate | Unspecified |\n|---|---|---|\n| 5.00 | 1.2 | 5 |\n| 5.00 | null | 001.20 |" + expect(AnswerCommit.renderAnswer(payload)).toBe(expected) + const tool = await initTool(AnswerCommit.AnswerCommitTool) + expect((await tool.execute(payload, context)).output).toBe(expected) + }) + + test.each([{ revenue: 2 }, { revenue: 2, Missing: 1 }])( + "row precision rejects unknown column names %j and lists available columns", + async (precision) => { + const payload = { + basis: recordBasis, + rows: { columns: ["Revenue"], values: [[5]], precision }, + } + const error = `Unknown rows.precision keys: ${JSON.stringify(Object.keys(precision))}; available columns: ["Revenue"]` + expect(() => AnswerCommit.renderAnswer(payload)).toThrow(error) + const tool = await initTool(AnswerCommit.AnswerCommitTool) + await rejects(tool.execute(payload, context), (cause) => cause instanceof Error && cause.message.includes(error)) + }, + ) + + test("row precision requires unique column labels", async () => { + const payload = { + basis: recordBasis, + rows: { columns: ["Amount", "Amount"], values: [[5, 1.234]], precision: { Amount: 2 } }, + } + const error = /precision requires unique column labels/i + expect(() => AnswerCommit.renderAnswer(payload)).toThrow(error) + const tool = await initTool(AnswerCommit.AnswerCommitTool) + await rejects(tool.execute(payload, context), error) + }) + + test("duplicate row column labels render unchanged without precision", async () => { + const payload = { + basis: recordBasis, + answer_only: true, + rows: { columns: ["Amount", "Amount"], values: [[5, 1.234]] }, + } + const expected = "| Amount | Amount |\n|---|---|\n| 5 | 1.234 |" + expect(AnswerCommit.renderAnswer(payload)).toBe(expected) + const tool = await initTool(AnswerCommit.AnswerCommitTool) + expect((await tool.execute(payload, context)).output).toBe(expected) + }) + + test("row precision accepts exact column names", async () => { + const payload = { + basis: recordBasis, + answer_only: true, + rows: { columns: ["Revenue"], values: [[5]], precision: { Revenue: 2 } }, + } + const expected = "| Revenue |\n|---|\n| 5.00 |" + expect(AnswerCommit.renderAnswer(payload)).toBe(expected) + const tool = await initTool(AnswerCommit.AnswerCommitTool) + expect((await tool.execute(payload, context)).output).toBe(expected) + }) +}) + +describe("answer_commit rendered length limit", () => { + const cap = 60_000 + const error = /60,000 characters.*aggregate.*narrow.*write the full table to a file.*commit the summary/i + + test("rejects rows whose rendered text exceeds the cap", async () => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + const payload = { + basis: recordBasis, + answer_only: true, + rows: { columns: ["Value"], values: [["x".repeat(cap)]] }, + } + expect(() => AnswerCommit.renderAnswer(payload)).toThrow(error) + await rejects(tool.execute(payload, context), error) + }) + + test.each([cap - 1, cap])("accepts rows with %i rendered characters", async (length) => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + const overhead = "| Value |\n|---|\n| |".length + const payload = { + basis: recordBasis, + answer_only: true, + rows: { columns: ["Value"], values: [["x".repeat(length - overhead)]] }, + } + const rendered = AnswerCommit.renderAnswer(payload) + expect(rendered.length).toBe(length) + expect((await tool.execute(payload, context)).output).toBe(rendered) + }) + + test("counts escaped table cells and rendered basis toward the cap", async () => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + const payload = { + basis: recordBasis, + rows: { columns: ["Value"], values: [["|".repeat(cap / 2)]] }, + } + expect(() => AnswerCommit.renderAnswer(payload)).toThrow(error) + await rejects(tool.execute(payload, context), error) + }) + + test("applies the rendered cap to fields and unresolved answers", async () => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + for (const shape of [ + { fields: [{ name: "", value: "x".repeat(cap) }] }, + { unresolved: { reason: "x".repeat(cap) } }, + ]) { + const payload = { basis: recordBasis, ...shape } + expect(() => AnswerCommit.renderAnswer(payload)).toThrow(error) + await rejects(tool.execute(payload, context), error) + } + }) + + test("documents the rendered character cap and recovery in the tool description", async () => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + expect(tool.description).toMatch(error) + }) +}) + +describe("renderAnswer golden text", () => { + const cases: [string, unknown, string][] = [ + [ + "single unnamed field", + { basis: recordBasis, fields: [{ name: "", value: "West" }] }, + `${recordBasis}\n\n**Answer:** West`, + ], + [ + "single named field", + { basis: recordBasis, fields: [{ name: "Region", value: "West" }] }, + `${recordBasis}\n\n**Answer:** Region: West`, + ], + [ + "single empty name with units", + { basis: recordBasis, fields: [{ name: "", value: "42", units: "widgets" }] }, + `${recordBasis}\n\n**Answer:** 42 widgets`, + ], + [ + "single whitespace-only name with units", + { basis: recordBasis, fields: [{ name: " \t\r\n ", value: "42", units: "widgets" }] }, + `${recordBasis}\n\n**Answer:** 42 widgets`, + ], + [ + "single Answer name with units", + { basis: recordBasis, fields: [{ name: "Answer", value: "42", units: "widgets" }] }, + `${recordBasis}\n\n**Answer:** 42 widgets`, + ], + [ + "single padded mixed-case answer name with units", + { basis: recordBasis, fields: [{ name: " aNsWeR ", value: "42", units: "widgets" }] }, + `${recordBasis}\n\n**Answer:** 42 widgets`, + ], + [ + "single uppercase result name with units", + { basis: recordBasis, fields: [{ name: "RESULT", value: "42", units: "widgets" }] }, + `${recordBasis}\n\n**Answer:** 42 widgets`, + ], + [ + "single padded mixed-case value name with units", + { basis: recordBasis, fields: [{ name: " vAlUe ", value: "42", units: "widgets" }] }, + `${recordBasis}\n\n**Answer:** 42 widgets`, + ], + [ + "single real name with units keeps its label", + { basis: recordBasis, fields: [{ name: "Widget count", value: "42", units: "widgets" }] }, + `${recordBasis}\n\n**Answer:** Widget count: 42 widgets`, + ], + [ + "multiple fields keep generic labels", + { + basis: recordBasis, + fields: [ + { name: "Answer", value: "42", units: "widgets" }, + { name: "RESULT", value: 2 }, + { name: " vAlUe ", value: 3 }, + { name: "", value: 4 }, + ], + }, + `${recordBasis}\n\n**Answer:** Answer: 42 widgets; RESULT: 2; vAlUe : 3; 4`, + ], + [ + "basis and units", + { basis: "Summed completed orders.", fields: [{ name: "Revenue", value: 42, units: "USD" }] }, + "Summed completed orders.\n\n**Answer:** Revenue: 42 USD", + ], + [ + "multiple fields", + { + basis: recordBasis, + fields: [ + { name: "Orders", value: 3 }, + { name: "Revenue", value: 12.5, units: "USD" }, + ], + }, + `${recordBasis}\n\n**Answer:** Orders: 3; Revenue: 12.5 USD`, + ], + [ + "fixed numeric precision", + { basis: recordBasis, fields: [{ name: "Rate", value: 1.2, precision: 3 }] }, + `${recordBasis}\n\n**Answer:** Rate: 1.200`, + ], + [ + "zero precision", + { basis: recordBasis, fields: [{ name: "Rate", value: 1.6, precision: 0 }] }, + `${recordBasis}\n\n**Answer:** Rate: 2`, + ], + [ + "maximum precision", + { basis: recordBasis, fields: [{ name: "", value: 0.5, precision: 10 }] }, + `${recordBasis}\n\n**Answer:** 0.5000000000`, + ], + [ + "integers stay integers", + { basis: recordBasis, fields: [{ name: "", value: 7, precision: 2 }] }, + `${recordBasis}\n\n**Answer:** 7`, + ], + [ + "zero is a value", + { basis: recordBasis, fields: [{ name: "", value: 0, precision: 2 }] }, + `${recordBasis}\n\n**Answer:** 0`, + ], + [ + "precision leaves text intact", + { basis: recordBasis, fields: [{ name: "", value: "001.20", precision: 1 }] }, + `${recordBasis}\n\n**Answer:** 001.20`, + ], + [ + "answer_only single field", + { + basis: "Summed all orders at order grain using orders.amount.", + answer_only: true, + fields: [{ name: "Revenue", value: 1.2, units: "USD", precision: 2 }], + }, + "1.20 USD", + ], + [ + "answer_only multiple fields", + { + answer_only: true, + basis: "Counted all orders at order grain using orders.id and summed orders.amount for revenue.", + fields: [ + { name: "Orders", value: 3 }, + { name: "Revenue", value: 12, units: "USD" }, + ], + }, + "3; 12 USD", + ], + [ + "rows with basis", + { + basis: recordBasis, + rows: { + columns: ["Region", "Orders"], + values: [ + ["West", 3], + ["East", 2], + ], + }, + }, + `${recordBasis}\n\n**Answer:**\n| Region | Orders |\n|---|---|\n| West | 3 |\n| East | 2 |`, + ], + [ + "rows with basis and note", + { + basis: "Grouped completed orders.", + rows: { columns: ["Region", "Orders"], values: [["West", 3]], note: "Includes all regions." }, + }, + "Grouped completed orders.\n\n**Answer:**\n| Region | Orders |\n|---|---|\n| West | 3 |\nIncludes all regions.", + ], + [ + "answer_only rows omit basis and note", + { + answer_only: true, + basis: "Grouped completed orders.", + rows: { columns: ["Orders"], values: [[3]], note: "This note is omitted." }, + }, + "| Orders |\n|---|\n| 3 |", + ], + [ + "null and empty table cells", + { basis: recordBasis, rows: { columns: ["Value"], values: [[null], [""]] } }, + `${recordBasis}\n\n**Answer:**\n| Value |\n|---|\n| null |\n| |`, + ], + [ + "table cells preserve their boundaries", + { basis: recordBasis, rows: { columns: ["A|B"], values: [["first\nsecond|third"]] } }, + `${recordBasis}\n\n**Answer:**\n| A\\|B |\n|---|\n| first second\\|third |`, + ], + [ + "table headers and cells escape backslashes before pipes", + { + basis: recordBasis, + rows: { columns: [String.raw`A\|B`, String.raw`C\\|D`], values: [[String.raw`a\|b`, String.raw`c\\|d`]] }, + }, + [ + recordBasis, + "", + "**Answer:**", + String.raw`| A\\\|B | C\\\\\|D |`, + "|---|---|", + String.raw`| a\\\|b | c\\\\\|d |`, + ].join("\n"), + ], + [ + "table headers and cells flatten all line endings to spaces", + { basis: recordBasis, rows: { columns: ["A\r\nB\rC\nD"], values: [["a\r\nb\rc\nd"]] } }, + `${recordBasis}\n\n**Answer:**\n| A B C D |\n|---|\n| a b c d |`, + ], + [ + "unresolved with basis", + { basis: "Checked the requested period.", unresolved: { reason: "No matching rows." } }, + "Checked the requested period.\n\n**Answer:** unable to determine — No matching rows.", + ], + [ + "answer_only unresolved retains its status and hides the supplied basis", + { answer_only: true, basis: recordBasis, unresolved: { reason: "No matching rows." } }, + "unable to determine — No matching rows.", + ], + [ + "two sentences and a decimal", + { basis: "The rate is 1.25. There are two orders!", fields: [{ name: "", value: 2 }] }, + "The rate is 1.25. There are two orders!\n\n**Answer:** 2", + ], + [ + "unpunctuated basis", + { basis: "From completed orders", fields: [{ name: "", value: 2 }] }, + "From completed orders\n\n**Answer:** 2", + ], + [ + "unused object and string fillers", + { + fields: [{ name: "", value: 2, units: "", precision: "" }], + rows: {}, + unresolved: "", + basis: recordBasis, + readings: [], + answer_only: "", + }, + `${recordBasis}\n\n**Answer:** 2`, + ], + [ + "unused array fillers", + { + answer_only: true, + fields: [], + rows: { columns: ["Count"], values: [[2]], note: "" }, + unresolved: {}, + basis: recordBasis, + readings: "", + }, + "| Count |\n|---|\n| 2 |", + ], + [ + "whitespace-only units are absent", + { basis: recordBasis, fields: [{ name: "", value: 2, units: " \t\r\n " }] }, + `${recordBasis}\n\n**Answer:** 2`, + ], + [ + "whitespace-only note is absent", + { basis: recordBasis, rows: { columns: ["Count"], values: [[2]], note: " \t\r\n " } }, + `${recordBasis}\n\n**Answer:**\n| Count |\n|---|\n| 2 |`, + ], + [ + "advisory readings do not render", + { + basis: recordBasis, + fields: [{ name: "", value: 2 }], + readings: [{ clause: "month", reading: "calendar month", result_summary: "2", decided_by: "request" }], + }, + `${recordBasis}\n\n**Answer:** 2`, + ], + ] + + test.each(cases)("%s", async (_name, payload, expected) => { + expect(AnswerCommit.renderAnswer(payload)).toBe(expected) + const tool = await initTool(AnswerCommit.AnswerCommitTool) + expect((await tool.execute(payload, context)).output).toBe(expected) + }) +}) + +describe("renderAnswer strict-schema null fillers", () => { + test.each(["precision", "units"])("null %s is absent", (key) => { + expect(AnswerCommit.renderAnswer({ basis: recordBasis, fields: [{ name: "", value: 1.25, [key]: null }] })).toBe( + `${recordBasis}\n\n**Answer:** 1.25`, + ) + }) + + test.each(["readings", "answer_only"])("null %s is absent", (key) => { + expect(AnswerCommit.renderAnswer({ basis: recordBasis, fields: [{ name: "Count", value: 2 }], [key]: null })).toBe( + `${recordBasis}\n\n**Answer:** Count: 2`, + ) + }) + + test("null note is absent", () => { + expect( + AnswerCommit.renderAnswer({ basis: recordBasis, rows: { columns: ["Count"], values: [[2]], note: null } }), + ).toBe(`${recordBasis}\n\n**Answer:**\n| Count |\n|---|\n| 2 |`) + }) +}) + +describe("renderAnswer basis length bound", () => { + test("accepts a two-sentence basis containing U.S. and e.g.", async () => { + const basis = "Counted U.S. orders at order grain from orders.id. Excluded cancelled orders, e.g. pending refunds." + const payload = { basis, fields: [{ name: "", value: 2 }] } + const expected = `${basis}\n\n**Answer:** 2` + expect(AnswerCommit.renderAnswer(payload)).toBe(expected) + const tool = await initTool(AnswerCommit.AnswerCommitTool) + expect((await tool.execute(payload, context)).output).toBe(expected) + }) + + test.each(["First. Second? Third!", "one.\ntwo.\nthree.", "One.\n\nTwo.\n\nThree."])( + "accepts punctuation within the length bound: %s", + (basis) => { + expect(AnswerCommit.renderAnswer({ basis, fields: [{ name: "", value: 2 }] })).toBe(`${basis}\n\n**Answer:** 2`) + }, + ) + + test("accepts exactly 400 basis characters", async () => { + expect(answerCommitModule.MAX_BASIS_CHARACTERS).toBe(400) + const basis = "x".repeat(400) + const payload = { basis, fields: [{ name: "", value: 2 }] } + const expected = `${basis}\n\n**Answer:** 2` + expect(AnswerCommit.renderAnswer(payload)).toBe(expected) + const tool = await initTool(AnswerCommit.AnswerCommitTool) + expect((await tool.execute(payload, context)).output).toBe(expected) + }) + + test.each(["x".repeat(401), ` ${"x".repeat(399)} `])( + "rejects 401 raw basis characters with the named bound", + async (basis) => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + for (const answer_only of [false, true]) { + const payload = { basis, answer_only, fields: [{ name: "", value: 2 }] } + expect(() => AnswerCommit.renderAnswer(payload)).toThrow(/basis.*400 characters/i) + await rejects(tool.execute(payload, context), /basis.*400 characters/i) + } + }, + ) +}) + +describe("renderAnswer structural validation", () => { + test("optional array fillers accept only empty arrays", async () => { + expect( + AnswerCommit.renderAnswer({ + answer_only: true, + fields: [{ name: "", value: 2, precision: [], units: [] }], + basis: recordBasis, + }), + ).toBe("2") + for (const filler of [[null], [0], [""], [{}], [[]]]) { + expect(() => AnswerCommit.renderAnswer({ fields: [{ name: "", value: 2, precision: filler }] })).toThrow( + /precision/i, + ) + expect(() => AnswerCommit.renderAnswer({ fields: [{ name: "", value: 2, units: filler }] })).toThrow(/units/i) + expect(() => AnswerCommit.renderAnswer({ fields: [{ name: "", value: 2 }], basis: filler })).toThrow(/basis/i) + } + + // Empty tuples parse the same fillers but emit invalid draft 2020-12 prefixItems. + const info = await toolInfo(AnswerCommit.AnswerCommitTool) + const tool = await Effect.runPromise(info.init()) + const ajv = new Ajv2020({ strict: false, validateFormats: false }) + expect(ajv.validateSchema(ToolJsonSchema.fromTool({ id: info.id, ...tool }))).toBe(true) + }) + + test.each([ + ["empty value", { fields: [{ name: "Count", value: "" }] }, /fields.*value.*non-empty/i], + ["blank value", { fields: [{ name: "Count", value: " " }] }, /fields.*value.*non-empty/i], + ["whitespace-only value", { fields: [{ name: "Count", value: " \t\r\n " }] }, /fields.*value.*non-empty/i], + [ + "whitespace-only name in multiple fields", + { + fields: [ + { name: " \t\r\n ", value: 2 }, + { name: "Count", value: 3 }, + ], + }, + /fields.*name.*non-empty/i, + ], + ["multiple values in one field", { fields: [{ name: "Count", value: [1, 2] }] }, /fields.*value.*one/i], + ["missing value", { fields: [{ name: "Count" }] }, /fields.*value/i], + ["null field value", { fields: [{ name: "Count", value: null }] }, /fields.*value/i], + ["ragged rows", { rows: { columns: ["A", "B"], values: [[1, 2], [3]] } }, /rows.*1.*2.*cells/i], + ["no columns", { rows: { columns: [], values: [[1]] } }, /rows.*columns/i], + ["no rows", { rows: { columns: ["A"], values: [] } }, /rows.*values/i], + ["negative precision", { fields: [{ name: "", value: 1.5, precision: -1 }] }, /precision/i], + ["excessive precision", { fields: [{ name: "", value: 1.5, precision: 11 }] }, /precision/i], + ["fractional precision", { fields: [{ name: "", value: 1.5, precision: 1.5 }] }, /precision/i], + ["empty reason", { unresolved: { reason: "" } }, /unresolved.*reason/i], + ["no answer shape", { basis: recordBasis }, /exactly one of fields, rows, or unresolved/i], + [ + "only fillers", + { basis: recordBasis, fields: [], rows: {}, unresolved: "" }, + /exactly one of fields, rows, or unresolved/i, + ], + [ + "fields and rows", + { basis: recordBasis, fields: [{ name: "", value: 1 }], rows: { columns: ["A"], values: [[1]] } }, + /exactly one of fields, rows, or unresolved/i, + ], + [ + "fields and unresolved", + { basis: recordBasis, fields: [{ name: "", value: 1 }], unresolved: { reason: "Missing" } }, + /exactly one of fields, rows, or unresolved/i, + ], + [ + "rows and unresolved", + { basis: recordBasis, rows: { columns: ["A"], values: [[1]] }, unresolved: { reason: "Missing" } }, + /exactly one of fields, rows, or unresolved/i, + ], + ] as [string, unknown, RegExp][])("rejects %s", (_name, payload, error) => { + expect(() => AnswerCommit.renderAnswer(payload)).toThrow(error) + }) +}) + +const context = { + sessionID: "test-session", + messageID: "test-message", + agent: "analyst", + abort: new AbortController().signal, + messages: [], +} + +test("answer_commit returns only the rendered answer and advisory metadata", async () => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + const readings = [{ clause: "month", reading: "calendar month", result_summary: "2", decided_by: "request" }] + const asks: unknown[] = [] + const result = await tool.execute( + { fields: [{ name: "", value: 2 }], rows: {}, unresolved: "", basis: recordBasis, readings }, + { + ...context, + ask: (input: unknown) => { + asks.push(input) + }, + }, + ) + expect(result.output).toBe(`${recordBasis}\n\n**Answer:** 2`) + expect(result.metadata.readings).toEqual(readings) + expect(asks).toEqual([{ permission: "answer_commit", patterns: ["*"], always: ["*"], metadata: {} }]) +}) + +test("answer_commit exposes a serializable object input schema", async () => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + expect(tool.jsonSchema?.type).toBe("object") + expect(Object.keys(tool.jsonSchema?.properties ?? {})).toEqual([ + "fields", + "rows", + "unresolved", + "basis", + "answer_only", + "readings", + ]) +}) + +test("answer_commit description defines final-call timing and exact reply", async () => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + expect(tool.description).toContain("LAST tool call") + expect(tool.description).toContain("once the answer is decided") + expect(tool.description).toContain("reply with exactly the returned text and nothing else") +}) + +// altimate_change start - preserve every requested item in compound answers +test("answer_commit description and fields schema require one field per requested item", async () => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + expect(tool.description).toContain("one field per item the request asks for") + expect(tool.description).toContain("never collapse a compound request") + expect(structuredClone(tool.jsonSchema?.properties?.fields)).toMatchObject({ + description: "One entry per item the request asks for, in the order asked.", + }) +}) +// altimate_change end + +test("answer_commit description defines answer_only and unused properties", async () => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + expect(tool.description).toContain("only the answer or a specific format") + expect(tool.description).toContain("exactly the value(s) or table, with no basis, field names, note, or answer label") + expect(tool.description).toContain("Omit unused optional properties") +}) + +test("answer_commit description preserves the top-N boundary-tie contract", async () => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + expect(tool.description).toContain("When the request names N, rows.values must contain exactly N concrete rows") + expect(tool.description).toContain("even with a tie at the top-N boundary; note the tie briefly in rows.note") +}) + +test("answer_commit rows schema documents the requested count and boundary-tie note", async () => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + // Bun's asymmetric matchers mutate nested values; preserve the shared tool schema. + expect(structuredClone(tool.jsonSchema?.properties?.rows)).toMatchObject({ + anyOf: expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + values: expect.objectContaining({ + description: + "Concrete result rows; when the request names N, the row count must equal N, even with a tie at the top-N boundary", + }), + note: expect.objectContaining({ + description: "Brief result note, including any tie at the top-N boundary; omitted when answer_only is true", + }), + }), + }), + ]), + }) +}) + +test("answer_commit description excludes JSON-schema requests", async () => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + expect(tool.description).toContain("not available for JSON-schema requests; use StructuredOutput instead") +}) + +test("answer_commit accepts null optional fields through the tool wrapper", async () => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + const result = await tool.execute( + { + fields: [{ name: "", value: 1.25, precision: null, units: null }], + basis: recordBasis, + readings: null, + answer_only: null, + }, + context, + ) + expect(result.output).toBe(`${recordBasis}\n\n**Answer:** 1.25`) + expect(result.metadata.readings).toEqual([]) +}) + +test("answer_commit names invalid field values through the tool wrapper", async () => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + await expect(tool.execute({ fields: [{ name: "Count", value: "" }] }, context)).rejects.toThrow( + /fields.*value.*non-empty/i, + ) +}) + +test("answer_commit refuses direct execution by another agent", async () => { + const tool = await initTool(AnswerCommit.AnswerCommitTool) + await expect( + tool.execute({ basis: recordBasis, fields: [{ name: "", value: 2 }] }, { ...context, agent: "builder" }), + ).rejects.toThrow(/only.*analyst/i) +}) diff --git a/packages/opencode/test/altimate/tools/tool-schema-anthropic.test.ts b/packages/opencode/test/altimate/tools/tool-schema-anthropic.test.ts new file mode 100644 index 0000000000..0c8a9c25cc --- /dev/null +++ b/packages/opencode/test/altimate/tools/tool-schema-anthropic.test.ts @@ -0,0 +1,235 @@ +import { afterEach, describe, expect, test } from "bun:test" +import Ajv2020 from "ajv/dist/2020" +import path from "node:path" +import { Effect } from "effect" +import { LayerNode } from "@opencode-ai/core/effect/layer-node" +import { Agent } from "@/agent/agent" +import { Config } from "@/config/config" +import { RuntimeFlags } from "@/effect/runtime-flags" +import { PermissionNext } from "@/permission/next" +import type { Provider } from "@/provider/provider" +import { ModelID, ProviderID } from "@/provider/schema" +import { ProviderTransform } from "@/provider/transform" +import { ToolJsonSchema } from "@/tool/json-schema" +import { ToolRegistry } from "@/tool/registry" +import { TestConfig } from "../../fixture/config" +import { disposeAllInstances } from "../../fixture/fixture" +import { testEffect } from "../../lib/effect" + +const fixture = (enableParallel: boolean) => + testEffect( + LayerNode.buildLayer(LayerNode.group([ToolRegistry.node, Agent.node]), { + replacements: [ + LayerNode.replace(Config.node, TestConfig.layer()), + LayerNode.replace(RuntimeFlags.node, RuntimeFlags.layer({ enableParallel })), + ], + }), + ) + +afterEach(disposeAllInstances) + +const ajv = new Ajv2020({ allErrors: true, strict: false, validateFormats: false }) +const formats = new Set([ + "date-time", + "date", + "time", + "duration", + "email", + "idn-email", + "hostname", + "idn-hostname", + "ipv4", + "ipv6", + "uri", + "uri-reference", + "iri", + "iri-reference", + "uuid", + "uri-template", + "json-pointer", + "relative-json-pointer", + "regex", +]) + +function isObject(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value) +} + +function schemaErrors(schema: object | boolean): string[] { + const errors = ajv.validateSchema(schema) ? [] : [ajv.errorsText(ajv.errors, { separator: "\n" })] + function visit(value: unknown, location = "#") { + if (!isObject(value)) return + for (const keyword of ["$schema", "nullable"]) { + if (keyword in value) errors.push(`${location}/${keyword} is not supported by Anthropic`) + } + for (const keyword of ["exclusiveMinimum", "exclusiveMaximum"]) { + if (typeof value[keyword] === "boolean") errors.push(`${location}/${keyword} must be numeric`) + } + if (typeof value.format === "string" && !formats.has(value.format)) { + errors.push(`${location}/format is not a standard format: ${value.format}`) + } + if ( + "additionalProperties" in value && + typeof value.additionalProperties !== "boolean" && + !isObject(value.additionalProperties) + ) { + errors.push(`${location}/additionalProperties must be boolean or a schema`) + } + // Walk schema positions only: property names and enum/default values may contain these keywords. + for (const keyword of ["properties", "patternProperties", "$defs", "definitions", "dependentSchemas"]) { + if (!isObject(value[keyword])) continue + for (const [name, child] of Object.entries(value[keyword])) visit(child, `${location}/${keyword}/${name}`) + } + for (const keyword of ["allOf", "anyOf", "oneOf", "prefixItems"]) { + if (!Array.isArray(value[keyword])) continue + value[keyword].forEach((child, index) => visit(child, `${location}/${keyword}/${index}`)) + } + for (const keyword of [ + "items", + "contains", + "additionalProperties", + "unevaluatedProperties", + "unevaluatedItems", + "propertyNames", + "not", + "if", + "then", + "else", + "contentSchema", + ]) { + visit(value[keyword], `${location}/${keyword}`) + } + } + visit(schema) + return errors +} + +function model(providerID: ProviderID): Provider.Model { + const id = providerID === ProviderID.amazonBedrock ? "global.anthropic.claude-opus-5-5" : "claude-opus-5-5" + return { + id: ModelID.make(id), + providerID, + api: { id, npm: `@ai-sdk/${providerID}`, url: "https://example.invalid" }, + name: id, + capabilities: { + temperature: true, + reasoning: true, + attachment: false, + toolcall: true, + input: { text: true, audio: false, image: false, video: false, pdf: false }, + output: { text: true, audio: false, image: false, video: false, pdf: false }, + interleaved: false, + }, + cost: { input: 0, output: 0, cache: { read: 0, write: 0 } }, + limit: { context: 200_000, output: 32_000 }, + status: "active", + release_date: "2026-01-01", + options: {}, + headers: {}, + } +} + +describe("Anthropic tool JSON schemas", () => { + for (const enableParallel of [false, true]) { + const it = fixture(enableParallel) + for (const providerID of [ProviderID.anthropic, ProviderID.amazonBedrock]) { + for (const name of ["analyst", "builder"]) { + it.instance( + `${providerID}, search ${enableParallel ? "enabled" : "disabled"}: every registered ${name} tool satisfies draft 2020-12 and Anthropic restrictions`, + () => + Effect.gen(function* () { + const registry = yield* ToolRegistry.Service + const agents = yield* Agent.Service + const agent = yield* agents.get(name) + if (!agent) throw new Error(`${name} agent not found`) + const target = model(providerID) + const tools = yield* registry.tools({ providerID, modelID: target.id, agent }) + expect(tools.map((tool) => tool.id)).toEqual(expect.arrayContaining(["sql_check", "sql_execute", "read"])) + expect(tools.some((tool) => tool.id === "answer_commit")).toBe(name === "analyst") + + // Match prompt.ts / session/tools.ts conversion. Both SDK providers forward this schema unchanged. + // Also retain the legacy LLM.stream permission-filtered insertion order for offline diagnostics. + const disabled = PermissionNext.disabled( + tools.map((tool) => tool.id), + agent.permission, + ) + let requestIndex = 0 + const schemas = tools.map((tool, registryIndex) => ({ + name: tool.id, + registryIndex, + requestIndex: tool.id === "invalid" || disabled.has(tool.id) ? null : requestIndex++, + schema: JSON.parse(JSON.stringify(ProviderTransform.schema(target, ToolJsonSchema.fromTool(tool)))), + })) + const dump = process.env.TOOL_SCHEMA_DUMP_DIR + if (dump) { + yield* Effect.promise(() => + Bun.write( + path.join(dump, `${providerID}-${name}${enableParallel ? "-search" : ""}.json`), + JSON.stringify(schemas, null, 2), + ), + ) + } + // Validate every registered tool, including ones hidden by this agent's permissions. + const errors = schemas.flatMap((tool) => + schemaErrors(tool.schema).map((error) => `${tool.name}: ${error}`), + ) + expect(errors).toEqual([]) + }), + ) + } + } + } + + test.each(["@ai-sdk/anthropic", "@ai-sdk/amazon-bedrock", "@ai-sdk/google-vertex/anthropic"])( + "%s removes only the root dialect declaration without mutating the input", + (npm) => { + const target = model(ProviderID.anthropic) + target.api.npm = npm + const expected = { + type: "object" as const, + properties: { + $schema: { type: "string" as const }, + value: { type: "number" as const, exclusiveMinimum: 0 }, + empty: { type: "array" as const, items: {}, minItems: 0, maxItems: 0 }, + }, + required: ["value"], + additionalProperties: false, + } + const input = { $schema: "https://json-schema.org/draft/2020-12/schema", ...expected } + const original = structuredClone(input) + expect(ProviderTransform.schema(target, input)).toEqual(expected) + expect(input).toEqual(original) + }, + ) + + test("rejects empty tuple prefixItems and nested Anthropic-incompatible keywords", () => { + for (const fragment of [ + { type: "array", prefixItems: [] }, + { $schema: "https://json-schema.org/draft/2020-12/schema" }, + { nullable: true }, + { type: "number", exclusiveMinimum: true }, + { type: "number", exclusiveMaximum: false }, + { type: "string", format: "custom-format" }, + { type: "object", additionalProperties: [] }, + ]) { + expect(schemaErrors({ type: "object", properties: { value: { anyOf: [fragment] } } }).length).toBeGreaterThan(0) + } + }) + + test("allows type arrays, compositions, standard formats, and keyword-like data", () => { + expect( + schemaErrors({ + type: "object", + properties: { + nullable: { type: ["string", "null"] }, + $schema: { type: "string", format: "uri" }, + value: { anyOf: [{ type: "number", exclusiveMinimum: 0 }, { type: "null" }] }, + choice: { oneOf: [{ const: 1 }, { const: 2 }] }, + payload: { enum: [{ nullable: true, $schema: "literal data" }] }, + labels: { type: "object", additionalProperties: { type: "string" } }, + }, + additionalProperties: false, + }), + ).toEqual([]) + }) +}) diff --git a/packages/opencode/test/cli/help/__snapshots__/help-snapshots.test.ts.snap b/packages/opencode/test/cli/help/__snapshots__/help-snapshots.test.ts.snap index ac7390182b..7f8dbbce66 100644 --- a/packages/opencode/test/cli/help/__snapshots__/help-snapshots.test.ts.snap +++ b/packages/opencode/test/cli/help/__snapshots__/help-snapshots.test.ts.snap @@ -81,6 +81,7 @@ Options: [string]" `; +// altimate_change start — document the committed-answer echo allowance in run help exports[`opencode CLI help-text snapshots every documented command emits stable help text: opencode run --help 1`] = ` "altimate-code run [message..] @@ -134,14 +135,15 @@ Options: statement (1-indexed) [number] --trace enable session tracing (default: true, disable with --no-trace) [boolean] [default: true] - --max-turns maximum assistant work turns; one best-effort finalization - generation may follow the limit [number] + --max-turns maximum assistant work turns; allows one committed-answer + echo, or one best-effort reply after a budget abort [number] --time-budget maximum work time in seconds (env: ALTIMATE_RUN_TIME_BUDGET_S); bounded finalization may follow [number] --dangerously-skip-permissions auto-approve permissions that are not explicitly denied (dangerous!) [boolean] [default: false]" `; +// altimate_change end exports[`opencode CLI help-text snapshots every documented command emits stable help text: opencode debug --help 1`] = ` "altimate-code debug diff --git a/packages/opencode/test/cli/run/budget-finalization.test.ts b/packages/opencode/test/cli/run/budget-finalization.test.ts index 7be5c3652c..fe3f3dc97c 100644 --- a/packages/opencode/test/cli/run/budget-finalization.test.ts +++ b/packages/opencode/test/cli/run/budget-finalization.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from "bun:test" -import { Effect } from "effect" +import { Effect, Fiber } from "effect" +import { createOpencodeClient } from "@opencode-ai/sdk/v2" import fs from "node:fs/promises" import path from "node:path" import { RunAccounting } from "@/cli/cmd/run-accounting" @@ -18,6 +19,24 @@ function exhausted() { } describe("budget finalization accounting", () => { + // altimate_change start — a failed answer_commit retry revokes the prior echo turn allowance + test("a successful answer_commit grants the committed echo exemption", () => { + const accounting = RunAccounting.create() + expect(accounting.committedAnswer).toBe(false) + accounting.onAnswerCommit("completed") + expect(accounting.committedAnswer).toBe(true) + accounting.resetCommittedAnswer() + expect(accounting.committedAnswer).toBe(false) + }) + + test("a failed answer_commit retry clears the committed echo exemption", () => { + const accounting = RunAccounting.create() + accounting.onAnswerCommit("completed") + expect(accounting.committedAnswer).toBe(true) + accounting.onAnswerCommit("error") + expect(accounting.committedAnswer).toBe(false) + }) + // altimate_change end // altimate_change start — distinguish recovered wall-clock limits from turn limits and real failures test("a successful time-budget finalization clears fatal status but keeps budget attribution", () => { const accounting = RunAccounting.create() @@ -199,6 +218,541 @@ function isFinalization(body: Record) { } describe("opencode run: budget finalization subprocess", () => { + // altimate_change start — attached turn limits use persisted request events without session fetches + for (const scenario of ["ordinary", "shared-request", "failed-commit-retry"] as const) { + const sharedRequest = scenario === "shared-request" + const failedRetry = scenario === "failed-commit-retry" + cliIt.concurrent( + sharedRequest + ? "a new persisted request on --attach --session clears the committed echo exemption" + : failedRetry + ? "a successful answer_commit followed by a failed retry revokes the committed echo exemption" + : "an ordinary over-budget step requests abort without awaiting any session fetch", + ({ opencode }) => + Effect.gen(function* () { + const sessionID = "ses_budget_attached" + let stream: ReadableStreamDefaultController + let aborts = 0 + let sessionFetches = 0 + let prompts = 0 + let subscriptionReady = Promise.withResolvers() + let finishPrompt: (value: Response) => void = () => {} + const encoder = new TextEncoder() + const publish = (type: string, properties: object) => + stream.enqueue(encoder.encode(`data: ${JSON.stringify({ type, properties })}\n\n`)) + const assistant = (messageID: string, parentID: string) => ({ + id: messageID, + sessionID, + role: "assistant", + parentID, + agent: "build", + modelID: "test-model", + providerID: "test", + time: { created: Date.now() }, + }) + const start = (messageID: string, parentID: string) => { + publish("message.updated", { info: assistant(messageID, parentID) }) + publish("message.part.updated", { + part: { id: `${messageID}_start`, messageID, sessionID, type: "step-start" }, + }) + } + const server = yield* Effect.acquireRelease( + Effect.sync(() => + Bun.serve({ + hostname: "127.0.0.1", + port: 0, + idleTimeout: 0, + async fetch(request) { + const route = new URL(request.url).pathname + if (route === "/provider") + return Response.json({ all: [{ id: "test", models: { "test-model": {} } }], connected: ["test"] }) + if (route === "/config") return Response.json({ share: "disabled", tracing: { enabled: false } }) + if (route === "/event") + return new Response( + new ReadableStream({ + start(controller) { + stream = controller + subscriptionReady.resolve() + publish("server.connected", {}) + }, + }), + { + headers: { "content-type": "text/event-stream" }, + }, + ) + if (route === `/session/${sessionID}` && request.method === "GET") { + sessionFetches++ + // This never settles: a read on the step-start path would prevent the abort. + return new Promise(() => {}) + } + if (route === `/session/${sessionID}/abort`) { + aborts++ + const error = { name: "MessageAbortedError", data: { message: "aborted by turn limit" } } + publish("session.error", { sessionID, error }) + publish("session.status", { sessionID, status: { type: "idle" } }) + finishPrompt(Response.json({ info: { error } })) + return Response.json(true) + } + if (route === `/session/${sessionID}/message` && request.method === "POST") { + const body = (await request.json()) as { messageID: string } + // The SDK connects SSE lazily, including during finalization. + // Wait for this prompt's subscriber before publishing its events. + const ready = subscriptionReady + await ready.promise + subscriptionReady = Promise.withResolvers() + const parentID = body.messageID + const messageID = `${parentID}zz_assistant` + prompts++ + publish("session.status", { sessionID, status: { type: "busy" } }) + start(messageID, parentID) + if (prompts > 1) { + publish("message.part.updated", { + part: { + id: `${messageID}_text`, + messageID, + sessionID, + type: "text", + text: "Budget finalization completed.", + time: { start: 1, end: 2 }, + }, + }) + publish("message.part.updated", { + part: { + id: `${messageID}_finish`, + messageID, + sessionID, + type: "step-finish", + reason: "stop", + }, + }) + publish("session.status", { sessionID, status: { type: "idle" } }) + return Response.json({ info: { finish: "stop" } }) + } + if (sharedRequest || failedRetry) { + publish("message.part.updated", { + part: { + id: `${messageID}_commit`, + messageID, + sessionID, + type: "tool", + tool: "answer_commit", + state: { + status: "completed", + input: {}, + output: "Original committed answer.", + title: "Committed", + }, + }, + }) + } + if (failedRetry) { + publish("message.part.updated", { + part: { + id: `${messageID}_commit_retry`, + messageID, + sessionID, + type: "tool", + tool: "answer_commit", + state: { status: "error", input: {}, error: "The retry failed validation." }, + }, + }) + } + if (sharedRequest) { + // The original echo remains exempt, including synthetic compaction continuation rows. + const continuationID = `${parentID}_continuation` + publish("message.updated", { info: { id: continuationID, sessionID, role: "user" } }) + publish("message.part.updated", { + part: { + id: `${continuationID}_text`, + messageID: continuationID, + sessionID, + type: "text", + synthetic: true, + text: "Continue the compacted request.", + }, + }) + start(`${parentID}_echo`, continuationID) + const nextRequestID = "msg_000_shared_request" + publish("message.updated", { info: { id: nextRequestID, sessionID, role: "user" } }) + publish("message.part.updated", { + part: { + id: `${nextRequestID}_text`, + messageID: nextRequestID, + sessionID, + type: "text", + text: "A new user request.", + }, + }) + // The request boundary is emitted after the real user and its parts are persisted. + publish("session.request.started", { sessionID, messageID: nextRequestID }) + // A delayed update from the old request must not re-arm its exemption. + // Client-supplied user IDs need not sort after every earlier assistant ID. + publish("message.updated", { info: assistant(messageID, parentID) }) + publish("message.part.updated", { + part: { + id: `${messageID}_commit`, + messageID, + sessionID, + type: "tool", + tool: "answer_commit", + state: { + status: "completed", + input: {}, + output: "Stale committed answer.", + title: "Committed", + }, + }, + }) + // Its parent predates this subscription, so even the first observed + // update of this older assistant cannot belong to the new request. + const unseenAssistantID = `${parentID}zzz_unseen_assistant` + publish("message.updated", { info: assistant(unseenAssistantID, "msg_older_unobserved_parent") }) + publish("message.part.updated", { + part: { + id: `${unseenAssistantID}_commit`, + messageID: unseenAssistantID, + sessionID, + type: "tool", + tool: "answer_commit", + state: { + status: "completed", + input: {}, + output: "An earlier unobserved request committed.", + title: "Committed", + }, + }, + }) + start(`${nextRequestID}_assistant`, nextRequestID) + } else { + start(`${parentID}_second`, parentID) + } + return new Promise((resolve) => { + finishPrompt = resolve + }) + } + return Response.json({ error: `Unexpected route: ${request.method} ${route}` }, { status: 404 }) + }, + }), + ), + (server) => Effect.sync(() => server.stop(true)), + ) + const result = yield* opencode.run("Start the work.", { + format: "json", + extraArgs: ["--attach", server.url.toString(), "--session", sessionID, "--max-turns", "1", "--no-trace"], + timeoutMs: 20_000, + bunRun: true, + }) + opencode.expectExit(result, 1) + expect(aborts).toBe(1) + expect(sessionFetches).toBe(0) + expect(prompts).toBe(2) + const events = opencode.parseJsonEvents(result.stdout) + if (failedRetry) { + expect(events.filter((event) => event.type === "tool_use").map((event) => event.part)).toEqual([ + expect.objectContaining({ tool: "answer_commit", state: expect.objectContaining({ status: "completed" }) }), + expect.objectContaining({ tool: "answer_commit", state: expect.objectContaining({ status: "error" }) }), + ]) + } + const budgetIndex = events.findIndex((event) => event.type === "budget_exhausted") + expect(budgetIndex).toBeGreaterThan(-1) + expect(events.slice(0, budgetIndex).filter((event) => event.type === "step_start")).toHaveLength( + sharedRequest ? 2 : 1, + ) + expect( + events.filter((event) => event.type === "text").map((event) => (event.part as { text: string }).text), + ).toEqual(["Budget finalization completed."]) + }), + 30_000, + ) + } + // altimate_change end + // altimate_change start — a completed answer gets one tool-free echo beyond the work-turn limit + cliIt.concurrent( + "a request queued before the committed echo step-start preserves its output and is refused for the exhausted budget", + ({ llm, opencode }) => + Effect.gen(function* () { + const committed = "Committed revenue before the queued request: 42" + const echoDispatched = Promise.withResolvers() + const releaseEcho = Promise.withResolvers() + let generations = 0 + const provider = yield* Effect.acquireRelease( + Effect.sync(() => Bun.serve({ + hostname: "127.0.0.1", + port: 0, + idleTimeout: 0, + async fetch(request) { + const body = await request.text() + if (!body.includes("Generate a title for this conversation") && ++generations === 2) { + // Hold even the response headers: the provider cannot emit step-start yet. + echoDispatched.resolve() + await releaseEcho.promise + } + return fetch(`${llm.url}/chat/completions`, { + method: "POST", + headers: { "content-type": "application/json" }, + body, + signal: request.signal, + }) + }, + })), + (server) => Effect.sync(() => { + releaseEcho.resolve() + server.stop(true) + }), + ) + yield* llm.tool("answer_commit", { + fields: [{ name: "", value: committed }], + answer_only: true, + basis: "Summed orders.amount over all orders at order grain.", + }) + yield* llm.text("The provider echo must be replaced by the stored commit.") + yield* llm.pushMatch(({ body }) => !isFinalization(body), reply().reason("The queued request must be refused.").hang()) + yield* llm.textMatch(({ body }) => isFinalization(body), "No budget remains for the queued request.") + const server = yield* opencode.serve({ + env: { + ALTIMATE_ANSWER_COMMIT: "1", + // An attached run cannot resolve --agent locally, so the server defaults to the analyst. + OPENCODE_CONFIG_CONTENT: JSON.stringify({ ...testProviderConfig(`${provider.url}v1`), default_agent: "analyst" }), + }, + bunRun: true, + }) + const sdk = createOpencodeClient({ baseUrl: server.url, throwOnError: true }) + const session = yield* Effect.promise(() => sdk.session.create({ title: "Queued echo budget" })) + const sessionID = session.data!.id + const run = yield* opencode.run("Commit the revenue answer.", { + format: "json", + extraArgs: ["--attach", server.url, "--session", sessionID, "--max-turns", "1", "--no-trace"], + env: { ALTIMATE_ANSWER_COMMIT: "1" }, + timeoutMs: 60_000, + bunRun: true, + }).pipe(Effect.forkChild) + yield* Effect.promise(() => echoDispatched.promise).pipe(Effect.timeout("30 seconds")) + // noReply persists the foreign request and publishes its boundary; the running loop will pick it up. + const queued = yield* Effect.promise(() => sdk.session.prompt({ + sessionID, + agent: "analyst", + model: { providerID: "test", modelID: "test-model" }, + noReply: true, + parts: [{ type: "text", text: "Now answer a different question." }], + })) + releaseEcho.resolve() + const result = yield* Fiber.join(run) + opencode.expectExit(result, 1) + const events = opencode.parseJsonEvents(result.stdout) + const budgetIndex = events.findIndex((event) => event.type === "budget_exhausted") + expect(budgetIndex).toBeGreaterThan(-1) + const beforeBudget = events.slice(0, budgetIndex) + expect(beforeBudget.filter((event) => event.type === "step_start")).toHaveLength(2) + expect(beforeBudget.filter((event) => event.type === "text").map((event) => (event.part as { text: string }).text)) + .toEqual([committed]) + expect(events.filter((event) => event.type === "budget_exhausted")).toHaveLength(1) + const messages = yield* Effect.promise(() => sdk.session.messages({ sessionID })) + const echo = messages.data!.find((message) => message.parts.some((part) => part.type === "text" && part.text === committed))! + expect(echo.info).toMatchObject({ role: "assistant", finish: "stop" }) + expect(echo.info).not.toHaveProperty("parentID", queued.data!.info.id) + expect(messages.data!.map((message) => message.info)).toContainEqual(expect.objectContaining({ + role: "assistant", + parentID: queued.data!.info.id, + error: expect.objectContaining({ name: "MessageAbortedError" }), + })) + }), + 90_000, + ) + // altimate_change start — compaction continuations keep their originating request's echo allowance + // These exercise the same ownership helper as run.ts without requiring a listening socket. + for (const queued of ["none", "before the marker", "during the summary"] as const) { + test(`proactive compaction preserves committed echo ownership with a queued request: ${queued}`, () => { + const ownership = RunAccounting.createRequestOwnership() + const accounting = RunAccounting.create() + const committedRequests = new Set() + ownership.beginRequest("original") + ownership.onAssistantMessage({ id: "commit", parentID: "original" }) + accounting.onStepStart("commit") + committedRequests.add(ownership.ownerOf("commit")!) + const queue = () => { + ownership.onUserMessage("queued") + ownership.beginRequest("queued") + } + if (queued === "before the marker") queue() + ownership.onUserMessage("marker") + ownership.onUserMessage("marker", "original") + ownership.onAssistantMessage({ id: "summary", parentID: "marker" }) + accounting.onAssistantMessage({ id: "summary", summary: true }) + expect(accounting.onStepStart("summary")).toBe(false) + if (queued === "during the summary") queue() + + // SessionCompaction persists this synthetic user before reporting its origin. + ownership.onUserMessage("continuation") + ownership.onUserMessage("continuation", "marker") + // Another compaction must preserve the original ID, not the intervening user ID. + ownership.onUserMessage("second-marker", "continuation") + ownership.onUserMessage("second-continuation", "second-marker") + ownership.onAssistantMessage({ id: "echo", parentID: "second-continuation" }) + expect(ownership.ownerOf("echo")).toBe("original") + expect(accounting.onStepStart("echo")).toBe(true) + expect(accounting.turnCount).toBe(2) + expect(committedRequests.has(ownership.ownerOf("echo")!)).toBe(true) + + if (queued !== "none") { + ownership.onAssistantMessage({ id: "queued-work", parentID: "queued" }) + // Late updates from the old request cannot transfer its exemption to the new one. + ownership.onAssistantMessage({ id: "commit", parentID: "original" }) + expect(ownership.ownerOf("queued-work")).toBe("queued") + expect(committedRequests.has(ownership.ownerOf("queued-work")!)).toBe(false) + } + }) + } + + test("an ordinary request compacted after a committed echo keeps its own over-budget ownership", () => { + const ownership = RunAccounting.createRequestOwnership() + ownership.beginRequest("committed") + ownership.onAssistantMessage({ id: "old-echo", parentID: "committed" }) + ownership.onUserMessage("ordinary") + ownership.beginRequest("ordinary") + ownership.onUserMessage("ordinary", "committed") + // The new request can compact before creating its first assistant. + ownership.onUserMessage("marker", "ordinary") + ownership.onUserMessage("continuation", "marker") + ownership.onAssistantMessage({ id: "ordinary-work", parentID: "continuation" }) + expect(ownership.ownerOf("ordinary-work")).toBe("ordinary") + expect(ownership.ownerOf("ordinary-work")).not.toBe(ownership.ownerOf("old-echo")) + // A continuation with an unobserved parent cannot borrow the active request. + ownership.onUserMessage("unknown", "unobserved") + ownership.onAssistantMessage({ id: "unknown-work", parentID: "unknown" }) + expect(ownership.ownerOf("unknown-work")).toBeUndefined() + }) + // altimate_change end + cliIt.concurrent( + "answer_commit on --max-turns 1 publishes the committed output on its extra echo turn", + ({ llm, opencode }) => + Effect.gen(function* () { + const committed = "Committed revenue: 42" + yield* llm.tool("answer_commit", { + fields: [{ name: "", value: committed }], + answer_only: true, + basis: "Summed orders.amount over all orders at order grain.", + }) + yield* llm.text("A different provider echo must be replaced.") + // The old budget check aborts the echo and requests this fallback instead. + yield* llm.textMatch(({ body }) => isFinalization(body), "The echo was aborted by the turn limit.") + + const result = yield* opencode.run("Commit the revenue answer.", { + agent: "analyst", + format: "json", + extraArgs: ["--max-turns", "1"], + env: { ALTIMATE_ANSWER_COMMIT: "1" }, + timeoutMs: 60_000, + bunRun: true, + }) + opencode.expectExit(result, 0) + const events = opencode.parseJsonEvents(result.stdout) + expect(events.filter((event) => event.type === "budget_exhausted")).toHaveLength(0) + expect(events.filter((event) => event.type === "silent_turn_reply")).toHaveLength(0) + expect(events.filter((event) => event.type === "step_start")).toHaveLength(2) + expect(events.filter((event) => event.type === "tool_use").map((event) => event.part)).toContainEqual( + expect.objectContaining({ tool: "answer_commit", state: expect.objectContaining({ status: "completed" }) }), + ) + expect(events.filter((event) => event.type === "text").map((event) => (event.part as { text: string }).text)).toEqual([ + committed, + ]) + expect(events.filter((event) => event.type === "termination")).toMatchObject([ + { why_model_stopped: "stop", why_harness_stopped: "none" }, + ]) + const requests = (yield* llm.inputs).filter( + (body) => !JSON.stringify(body).includes("Generate a title for this conversation"), + ) + expect(requests).toHaveLength(2) + expect(requests[1]!.tools ?? []).toEqual([]) + expect(requests.filter(isFinalization)).toHaveLength(0) + }), + 90_000, + ) + cliIt.concurrent( + "answer_commit on --max-turns 1 publishes the committed output after an echo context overflow", + ({ llm, opencode }) => + Effect.gen(function* () { + const committed = "Committed revenue after overflow: 42" + yield* llm.tool("answer_commit", { + fields: [{ name: "", value: committed }], + answer_only: true, + basis: "Summed orders.amount over all orders at order grain.", + }) + // A streaming overflow begins the echo step, whose committed text can finish the request. + // The compatible provider exposes the error's message to the processor. + yield* llm.push({ + type: "sse", + head: [], + tail: [{ error: { message: JSON.stringify({ type: "error", error: { code: "context_length_exceeded" } }) } }], + }) + + const result = yield* opencode.run("Commit the revenue answer.", { + agent: "analyst", + format: "json", + extraArgs: ["--max-turns", "1"], + env: { ALTIMATE_ANSWER_COMMIT: "1" }, + timeoutMs: 60_000, + bunRun: true, + }) + opencode.expectExit(result, 0) + const events = opencode.parseJsonEvents(result.stdout) + expect(events.filter((event) => event.type === "budget_exhausted")).toHaveLength(0) + expect(events.filter((event) => event.type === "silent_turn_reply")).toHaveLength(0) + expect(events.filter((event) => event.type === "error")).toHaveLength(0) + expect(events.filter((event) => event.type === "step_start")).toHaveLength(2) + const texts = events + .filter((event) => event.type === "text") + .map((event) => (event.part as { text: string }).text) + expect(texts).toEqual([committed]) + expect(events.filter((event) => event.type === "termination")).toMatchObject([ + // The echo overflowed, so the model never produced a stop; the harness delivered the committed text. + { why_model_stopped: "tool-call", why_harness_stopped: "none" }, + ]) + const requests = (yield* llm.inputs).filter( + (body) => !JSON.stringify(body).includes("Generate a title for this conversation"), + ) + expect(requests).toHaveLength(2) + expect(requests[1]!.tools ?? []).toEqual([]) + expect(requests.filter(isFinalization)).toHaveLength(0) + }), + 90_000, + ) + cliIt.concurrent( + "a soft-time reminder clears the committed answer turn exemption for its new request", + ({ llm, opencode }) => + Effect.gen(function* () { + yield* llm.tool("answer_commit", { + fields: [{ name: "", value: "The original committed answer." }], + answer_only: true, + basis: "Summed orders.amount over all orders at order grain.", + }) + yield* llm.push(reply().reason("Waiting to publish the committed echo.").hang()) + yield* llm.textMatch(({ body }) => isFinalization(body), "The new request exhausted its turn budget.") + yield* llm.pushMatch(({ body }) => !isFinalization(body), reply().reason("Working on the new request.").hang()) + + const result = yield* opencode.run("Commit the revenue answer.", { + agent: "analyst", + format: "json", + extraArgs: ["--max-turns", "1", "--time-budget", "10"], + env: { ALTIMATE_ANSWER_COMMIT: "1" }, + timeoutMs: 60_000, + bunRun: true, + }) + opencode.expectExit(result, 1) + const events = opencode.parseJsonEvents(result.stdout) + expect(events.filter((event) => event.type === "time_budget_warning")).toHaveLength(1) + expect(events.filter((event) => event.type === "budget_exhausted")).toMatchObject([{ limit: 1 }]) + const budgetIndex = events.findIndex((event) => event.type === "budget_exhausted") + expect(events.slice(0, budgetIndex).filter((event) => event.type === "step_start")).toHaveLength(2) + // The commit belongs to the first request and is delivered before the reminder request is answered. + expect(events.filter((event) => event.type === "text").map((event) => (event.part as { text: string }).text)).toEqual([ + "The original committed answer.", + "The new request exhausted its turn budget.", + ]) + expect((yield* llm.inputs).filter(isFinalization)).toHaveLength(1) + }), + 90_000, + ) + // altimate_change end // altimate_change start — exhausted budgets must disable execution even if the model calls a tool for (const budget of ["turn", "time"] as const) { cliIt.concurrent( @@ -311,6 +865,11 @@ describe("opencode run: budget finalization subprocess", () => { const events = opencode.parseJsonEvents(result.stdout) expect(events.filter((event) => event.type === "budget_exhausted")).toHaveLength(1) expect(events.filter((event) => event.type === "silent_turn_reply")).toHaveLength(1) + // altimate_change start — a non-commit still stops before the second work step is emitted + const budgetIndex = events.findIndex((event) => event.type === "budget_exhausted") + expect(events[budgetIndex]).toMatchObject({ limit: 1 }) + expect(events.slice(0, budgetIndex).filter((event) => event.type === "step_start")).toHaveLength(1) + // altimate_change end expect(events.filter((event) => event.type === "termination")).toMatchObject([ { why_model_stopped: "stop", why_harness_stopped: "budget-exhausted", done_reason: "none" }, ]) diff --git a/packages/opencode/test/cli/run/queued-time-budget.test.ts b/packages/opencode/test/cli/run/queued-time-budget.test.ts index a028b32551..99558790d2 100644 --- a/packages/opencode/test/cli/run/queued-time-budget.test.ts +++ b/packages/opencode/test/cli/run/queued-time-budget.test.ts @@ -33,6 +33,37 @@ test("hard deadline aborts a queued session without a post-subscription busy eve }) expect(calls).toEqual(["ses_queued"]) }) + +test("hard deadline abort carries the harness deadline marker", async () => { + const source = await fs.readFile(new URL("../../../src/cli/cmd/run.ts", import.meta.url), "utf8") + const start = source.indexOf(" const abortForTimeBudget = () => {") + const end = source.indexOf(" const checkTimeBudget =", start) + expect(start).toBeGreaterThan(0) + let abortOptions: { signal: AbortSignal; headers?: Record } | undefined + const setup = new Function( + "sdk", + ` + const sessionID = "ses_deadline_marker" + let timeAbortRequested = false + let timeBudgetAbort + const accounting = { onSessionError() {} } + const eventAbort = new AbortController() + ${source.slice(start, end)} + abortForTimeBudget() + return timeBudgetAbort + `, + ) + await setup({ + session: { + abort: async (_: unknown, options: NonNullable) => { + abortOptions = options + return {} + }, + }, + }) + expect(abortOptions?.headers?.["x-altimate-run-budget"]).toBe("hard-deadline") + expect(abortOptions?.signal.aborted).toBe(false) +}) // altimate_change end // altimate_change start — the request lifetime also ends when queued work never becomes busy @@ -51,6 +82,7 @@ for (const command of [undefined, "inspect"]) { const files = [], audienceSystem = undefined const eventAbort = new AbortController() const startWorkClock = () => {} + const beginRequest = () => {} const budgetHeaders = () => ({}) const Telemetry = { startupReady() {} } ${source.slice(start, end)} diff --git a/packages/opencode/test/mcp/lifecycle.test.ts b/packages/opencode/test/mcp/lifecycle.test.ts index 77974ddd01..93eca27cac 100644 --- a/packages/opencode/test/mcp/lifecycle.test.ts +++ b/packages/opencode/test/mcp/lifecycle.test.ts @@ -1358,6 +1358,57 @@ it.instance( }, ) +// altimate_change start — external tools cannot inherit the native finalizer's permission. +for (const toggle of ["1", "0"]) { + it.instance(`MCP reserves answer_commit with ALTIMATE_ANSWER_COMMIT=${toggle}`, () => + MCP.Service.use((mcp: MCPNS.Interface) => + Effect.gen(function* () { + const previous = process.env.ALTIMATE_ANSWER_COMMIT + process.env.ALTIMATE_ANSWER_COMMIT = toggle + try { + lastCreatedClientName = "answer" + const state = getOrCreateClientState("answer") + state.tools = [ + { name: "commit", inputSchema: { type: "object", properties: {} } }, + { name: "lookup", inputSchema: { type: "object", properties: {} } }, + ] + yield* mcp.add("answer", { type: "local", command: ["echo", "test"] }) + expect(Object.keys(yield* mcp.tools())).toEqual(["answer_lookup"]) + expect(Object.keys((yield* mcp.snapshot("answer")).tools)).toEqual(["answer_lookup"]) + expect(state.requestCalls).toBe(0) + } finally { + if (previous === undefined) delete process.env.ALTIMATE_ANSWER_COMMIT + else process.env.ALTIMATE_ANSWER_COMMIT = previous + } + }), + ), + ) +} +// altimate_change end + +// altimate_change start — Windows permission matching treats tool IDs case-insensitively. +it.instance("MCP reserves case variants of answer_commit on Windows", () => + MCP.Service.use((mcp: MCPNS.Interface) => + Effect.gen(function* () { + lastCreatedClientName = "answer" + getOrCreateClientState("answer").tools = [ + { name: "Commit", inputSchema: { type: "object", properties: {} } }, + { name: "lookup", inputSchema: { type: "object", properties: {} } }, + ] + yield* mcp.add("answer", { type: "local", command: ["echo", "test"] }) + const platform = Object.getOwnPropertyDescriptor(process, "platform")! + Object.defineProperty(process, "platform", { ...platform, value: "win32" }) + try { + expect(Object.keys(yield* mcp.tools())).toEqual(["answer_lookup"]) + expect(Object.keys((yield* mcp.snapshot("answer")).tools)).toEqual(["answer_lookup"]) + } finally { + Object.defineProperty(process, "platform", platform) + } + }), + ), +) +// altimate_change end + // ======================================================================== // Test: transport leak — local stdio timeout (#19168) // ======================================================================== diff --git a/packages/opencode/test/session/message-v2.test.ts b/packages/opencode/test/session/message-v2.test.ts index 5e4372b1ad..802e1aac51 100644 --- a/packages/opencode/test/session/message-v2.test.ts +++ b/packages/opencode/test/session/message-v2.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from "bun:test" import { SessionV1 } from "@opencode-ai/core/v1/session" import { APICallError } from "ai" -import { MessageV2 } from "../../src/session/message-v2" +import { MessageV2, isEchoTransportError } from "../../src/session/message-v2" import { SessionRetry } from "../../src/session/retry" import { ProviderError } from "../../src/provider/error" import { ProviderTransform } from "@/provider/transform" @@ -1531,6 +1531,132 @@ describe("session.message-v2.fromError", () => { expect(SessionV1.ContextOverflowError.isInstance(result)).toBe(true) }) + for (const statusCode of [400, 503]) { + for (const code of [ + "guardrail_intervened", "content_filter", "content_policy", "content_policy_violation", + "content_policy_blocked", "policy_violation", "safety_rejection", "moderation_blocked", + ]) { + test(`HTTP ${statusCode} structured ${code} overrides overflow text`, () => { + const result = MessageV2.fromError(new APICallError({ + message: "maximum context length exceeded", + url: "https://example.com", + requestBodyValues: {}, + statusCode, + isRetryable: statusCode === 503, + responseBody: JSON.stringify({ error: { code, message: "maximum context length exceeded" } }), + }), { providerID }) + + expect(result.name).toBe("APIError") + expect((result.data as MessageV2.APIError["data"]).metadata?.code).toBe(code) + expect(isEchoTransportError(result)).toBe(false) + }) + } + + test(`HTTP ${statusCode} nested orchestration guardrail overrides overflow text`, () => { + const result = MessageV2.fromError(new APICallError({ + message: "maximum context length exceeded", + url: "https://example.com", + requestBodyValues: {}, + statusCode, + isRetryable: statusCode === 503, + responseBody: JSON.stringify({ + error: { message: "maximum context length exceeded" }, + details: { orchestrationError: { code: "guardrail_intervened" } }, + }), + }), { providerID }) + + expect(result.name).toBe("APIError") + expect((result.data as MessageV2.APIError["data"]).metadata?.code).toBe("guardrail_intervened") + expect(isEchoTransportError(result)).toBe(false) + }) + } + + for (const code of ["context_length_exceeded", "content_length_exceeded"]) { + test(`an upstream ${code} wrapped by the gateway under provider_specific_fields is still an overflow`, () => { + const result = MessageV2.fromError(new APICallError({ + message: "Request failed", + url: "https://example.com", + requestBodyValues: {}, + statusCode: 400, + isRetryable: false, + responseBody: JSON.stringify({ + error: { + code: "invalid_request_error", + message: "upstream request failed", + provider_specific_fields: { error: { code, message: "maximum context length exceeded" } }, + }, + }), + }), { providerID }) + + expect(result.name).toBe("ContextOverflowError") + }) + } + + for (const code of ["context_length_exceeded", "content_length_exceeded"]) { + test(`a nested diagnostic ${code} never masks a top-level policy denial`, () => { + const result = MessageV2.fromError(new APICallError({ + message: "Request failed", + url: "https://example.com", + requestBodyValues: {}, + statusCode: 400, + isRetryable: false, + responseBody: JSON.stringify({ + error: { code: "policy_violation", message: "maximum context length exceeded" }, + details: { request: { code } }, + }), + }), { providerID }) + + expect(result.name).toBe("APIError") + expect(isEchoTransportError(result)).toBe(false) + expect((result as any).data.metadata.code).toBe("policy_violation") + }) + } + + for (const code of ["context_length_exceeded", "content_length_exceeded"]) { + test(`explicit ${code} remains overflow despite a nested policy code`, () => { + const result = MessageV2.fromError(new APICallError({ + message: "Request failed", + url: "https://example.com", + requestBodyValues: {}, + statusCode: 400, + isRetryable: false, + responseBody: JSON.stringify({ + error: { code }, + orchestrationError: { code: "guardrail_intervened" }, + }), + }), { providerID }) + + expect(result.name).toBe("ContextOverflowError") + }) + } + + test("an explicit policy code on HTTP 413 stays a policy error, not an overflow", () => { + const result = MessageV2.fromError(new APICallError({ + message: "Request failed", + url: "https://example.com", + requestBodyValues: {}, + statusCode: 413, + isRetryable: false, + responseBody: JSON.stringify({ error: { code: "policy_violation" } }), + }), { providerID }) + + expect(result.name).toBe("APIError") + expect(result).toHaveProperty("data.metadata.code", "policy_violation") + }) + + test("HTTP 413 without a policy code is still an overflow", () => { + const result = MessageV2.fromError(new APICallError({ + message: "Request failed", + url: "https://example.com", + requestBodyValues: {}, + statusCode: 413, + isRetryable: false, + responseBody: JSON.stringify({ error: { message: "request entity too large" } }), + }), { providerID }) + + expect(result.name).toBe("ContextOverflowError") + }) + test("does not classify 429 no body as context overflow", () => { const result = MessageV2.fromError( new APICallError({ diff --git a/packages/opencode/test/session/prompt.test.ts b/packages/opencode/test/session/prompt.test.ts index 971a0138eb..e21f963371 100644 --- a/packages/opencode/test/session/prompt.test.ts +++ b/packages/opencode/test/session/prompt.test.ts @@ -980,6 +980,204 @@ it.instance("learn file hooks append glob and anchor matches once and respect th // Loop semantics +// altimate_change start - an answer committed before the step limit must still be echoed +for (const mode of [ + "committed", + "failed", + "structured", + "compacted", + "compacted-usage", + "compacted-unlimited", + "compacted-new-request", + "compacted-synthetic-request", + "compacted-overflow", +] as const) { + const compacts = mode.startsWith("compacted") + const newRequest = mode === "compacted-new-request" || mode === "compacted-synthetic-request" + const scenario = { + committed: "the exact final echo", + failed: "the failed-call limit", + structured: "the user output schema", + compacted: "the exact rows after automatic compaction", + "compacted-usage": "the exact rows after usage-triggered compaction", + "compacted-unlimited": "the exact rows after compaction without a step limit", + "compacted-new-request": "the newer user request after compaction", + "compacted-synthetic-request": "the newer synthetic user request after compaction", + "compacted-overflow": "the exact rows after compaction and directly recovers an echo overflow", + }[mode] + noLLMServer.instance( + `answer_commit preserves ${scenario}`, + () => + Effect.gen(function* () { + const { prompt, chat, sessions } = yield* boot() + const requests: Record[] = [] + const requestCount = compacts ? 3 : 2 + const rows = Array.from({ length: 60 }, (_, index) => [`${index}: ${"x".repeat(500)}`]) + const answer = compacts ? ["| value |", "|---|", ...rows.map(([value]) => `| ${value} |`)].join("\n") : "42" + const args = + mode === "failed" + ? {} + : compacts + ? { + rows: { columns: ["value"], values: rows }, + answer_only: true, + basis: "Selected all records at record grain using records.value.", + } + : { + fields: [{ name: "", value: 42 }], + answer_only: true, + basis: "Counted all records at record grain using records.id.", + } + const language = createOpenAICompatible({ + name: "test", + baseURL: "https://unused.invalid/v1", + fetch: Object.assign( + async (_url: string | URL | Request, init?: RequestInit) => { + if (typeof init?.body !== "string") throw new Error("expected a JSON request body") + const request = JSON.parse(init.body) + requests.push(request) + if (requests.length > requestCount) throw new Error("answer_commit exceeded its generation budget") + if (mode === "compacted-overflow" && requests.length === 3) { + return new Response(JSON.stringify({ error: { message: "request entity too large" } }), { + status: 413, + headers: { "Content-Type": "application/json" }, + }) + } + if (newRequest && requests.length === 2) { + const userID = MessageID.ascending() + await LegacySession.updateMessage({ + id: userID, + sessionID: chat.id, + role: "user", + agent: "analyst", + model: promptRef, + time: { created: Date.now() }, + }) + await LegacySession.updatePart({ + id: PartID.ascending(), + messageID: userID, + sessionID: chat.id, + type: "text", + synthetic: mode === "compacted-synthetic-request", + text: "A new question arrived during compaction; do not repeat the previous answer.", + }) + } + const limited = JSON.stringify(request.messages).includes("CRITICAL - MAXIMUM STEPS REACHED") + const response = + requests.length === 1 + ? reply() + .tool( + mode === "structured" ? "tool_lookup" : "answer_commit", + mode === "structured" ? { tool_name: "__missing_tool" } : args, + ) + .usage({ input: mode === "compacted-usage" ? 80_000 : compacts ? 70_000 : 1, output: 1 }) + .item() + : compacts && requests.length === 2 + ? reply().text("The rows were committed. Continue with the final reply.").stop().item() + : mode === "structured" + ? reply().tool("StructuredOutput", { count: 42 }).item() + : reply() + .text(limited ? "Maximum steps reached." : compacts ? "The rows were compacted away." : answer) + .stop() + .item() + if (response.type !== "sse") throw new Error("expected SSE response") + return new Response( + [...response.head, ...response.tail].map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("") + + "data: [DONE]\n\n", + { headers: { "Content-Type": "text/event-stream" } }, + ) + }, + { preconnect: () => undefined }, + ), + }).chatModel("test-model") + const languageSpy = spyOn(ProviderSvc, "getLanguage").mockResolvedValue(language) + try { + const result = yield* prompt.prompt({ + sessionID: chat.id, + agent: "analyst", + model: promptRef, + parts: [{ type: "text", text: "How many rows? Reply with only the count." }], + ...(mode === "structured" + ? { + format: { + type: "json_schema" as const, + retryCount: 0, + schema: { + type: "object", + properties: { count: { type: "number" } }, + required: ["count"], + additionalProperties: false, + }, + }, + } + : {}), + }) + expect(result.info).not.toHaveProperty("error") + expect(requests).toHaveLength(requestCount) + const messages = yield* sessions.messages({ sessionID: chat.id }) + if (mode === "compacted-overflow") { + expect(result.info).toHaveProperty("finish", "stop") + expect(result.info).toHaveProperty("metadata.answer_commit_echo_error.name", "ContextOverflowError") + const persisted = messages.find((message) => message.info.id === result.info.id) + expect(persisted?.info).not.toHaveProperty("error") + expect(persisted?.info).toHaveProperty("metadata.answer_commit_echo_error.name", "ContextOverflowError") + expect(messages.filter((message) => message.info.role === "assistant" && message.info.summary)).toHaveLength(1) + } + const commit = messages + .flatMap((message) => message.parts) + .find((part): part is MessageV2.ToolPart => part.type === "tool" && part.tool === "answer_commit") + expect(commit?.state.status).toBe(mode === "structured" ? undefined : mode === "failed" ? "error" : "completed") + if (mode === "committed" || (compacts && !newRequest)) { + const finalRequest = requests.at(-1)! + expect(JSON.stringify(finalRequest.messages)).not.toContain("CRITICAL - MAXIMUM STEPS REACHED") + expect(finalRequest.tools ?? []).toHaveLength(0) + expect(requestText(requestMessages(finalRequest).at(-1)!.content)).toBe( + "Tools are disabled. Reply with exactly the text returned by answer_commit and nothing else.", + ) + if (compacts) { + expect(messages.some((message) => message.info.role === "assistant" && message.info.summary)).toBe(true) + expect( + requestMessages(finalRequest).some((message) => requestText(message.content).includes(answer)), + ).toBe(false) + expect(commit?.state.status === "completed" && commit.state.metadata.truncated).toBe(false) + } + expect( + result.parts + .filter((part) => part.type === "text") + .map((part) => part.text) + .join(""), + ).toBe(answer) + } else if (mode === "failed" || newRequest) { + expect(JSON.stringify(requests.at(-1)!.messages)).toContain("CRITICAL - MAXIMUM STEPS REACHED") + expect(result.parts).toContainEqual( + expect.objectContaining({ type: "text", text: "Maximum steps reached." }), + ) + } else { + for (const request of requests) { + expect(JSON.stringify(request.tools)).not.toContain('"name":"answer_commit"') + } + expect(requests[1].tool_choice).toBe("required") + expect(result.info).toHaveProperty("structured", { count: 42 }) + } + } finally { + languageSpy.mockRestore() + } + }), + { + config: { + ...cfg, + snapshot: false, + learn: { enabled: false }, + agent: { + analyst: { steps: mode === "compacted-unlimited" ? undefined : 2, permission: { StructuredOutput: "allow" } }, + }, + }, + }, + 30_000, + ) +} +// altimate_change end + noLLMServer.instance( "loop exits immediately when last assistant has stop finish", () => @@ -1098,6 +1296,72 @@ it.instance("loop surfaces content-filter finishes as session errors", () => ) }), ) +// altimate_change start - provider length codes must retain the automatic compaction path +noLLMServer.instance( + "HTTP 413 content_length_exceeded stays a context overflow and runs compaction without a commit", + () => + Effect.gen(function* () { + const { prompt, chat, sessions } = yield* boot() + const requests: Record[] = [] + const summaryText = "The user requested a count. Continue after compacting the conversation." + const language = createOpenAICompatible({ + name: "test", + baseURL: "https://unused.invalid/v1", + fetch: Object.assign( + async (_url: string | URL | Request, init?: RequestInit) => { + if (typeof init?.body !== "string") throw new Error("expected a JSON request body") + requests.push(JSON.parse(init.body)) + if (requests.length > 3) throw new Error("unexpected request after overflow recovery") + if (requests.length === 1) { + return new Response( + JSON.stringify({ error: { code: "content_length_exceeded", message: "request entity too large" } }), + { status: 413, headers: { "Content-Type": "application/json" } }, + ) + } + const response = reply().text(requests.length === 2 ? summaryText : "42").stop().item() + if (response.type !== "sse") throw new Error("expected SSE response") + return new Response( + [...response.head, ...response.tail].map((chunk) => `data: ${JSON.stringify(chunk)}\n\n`).join("") + + "data: [DONE]\n\n", + { headers: { "Content-Type": "text/event-stream" } }, + ) + }, + { preconnect: () => undefined }, + ), + }).chatModel("test-model") + const languageSpy = spyOn(ProviderSvc, "getLanguage").mockResolvedValue(language) + const classify = spyOn(MessageV2, "fromError") + try { + const result = yield* prompt.prompt({ + sessionID: chat.id, + agent: "build", + model: promptRef, + parts: [{ type: "text", text: "How many rows?" }], + }) + expect(classify.mock.results.map((result) => result.value)).toContainEqual( + expect.objectContaining({ name: "ContextOverflowError" }), + ) + expect(result.info).not.toHaveProperty("error") + expect(result.info).toHaveProperty("finish", "stop") + expect(result.parts.filter((part) => part.type === "text").map((part) => part.text).join("")).toBe("42") + expect(requests).toHaveLength(3) + expect(JSON.stringify(requests[2].messages)).toContain(summaryText) + const messages = yield* sessions.messages({ sessionID: chat.id }) + expect(messages.filter((message) => message.info.role === "assistant" && message.info.summary)).toHaveLength(1) + expect(messages.flatMap((message) => message.parts)).toContainEqual( + expect.objectContaining({ type: "compaction", overflow: true }), + ) + expect(messages.flatMap((message) => message.parts).some((part) => part.type === "tool" && part.tool === "answer_commit")).toBe(false) + } finally { + classify.mockRestore() + languageSpy.mockRestore() + } + }), + { config: { ...cfg, snapshot: false, learn: { enabled: false }, compaction: { auto: true } } }, + 30_000, +) +// altimate_change end + it.instance("loop stops provider overflow instead of auto-compacting when disabled", () => Effect.gen(function* () { const { llm } = yield* useServerConfig((url) => ({