diff --git a/docs/sdk-conformance.md b/docs/sdk-conformance.md index 7d525960..a1077c3d 100644 --- a/docs/sdk-conformance.md +++ b/docs/sdk-conformance.md @@ -19,7 +19,7 @@ repository, not this SDK workspace. Paths below refer to that product repository | SDK field (TS / Py) | Wire attribute | Notes | | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `type: "span"` | `gen_ai.operation.name = "function"` | Always explicit: a propagated `gen_ai.conversation.id` would otherwise trip the ingest's `hasGenAi` → `"llm"` heuristic | +| `type: "span"` | `gen_ai.operation.name = "function"` | Always explicit, so the ingest never guesses it from other `gen_ai.*` keys, such as a caller-set `gen_ai.agent.name` | | `type: "generation"` | `gen_ai.operation.name = "chat"` | | | `type: "tool"` | `gen_ai.operation.name = "execute_tool"` | | | `type: "agent"` | `gen_ai.operation.name = "invoke_agent"` | | @@ -54,6 +54,25 @@ repository, not this SDK workspace. Paths below refer to that product repository | the item run | `gen_ai.conversation.id = "::"` | one session and one trace for each run; replaces an ambient or process session, and the run drops the active span | | the root span of an item run | name `experiment `, `gen_ai.operation.name = "function"` | `input` is the item input; `output` is the task result; gated by capture flags | +`error.type` and `exception.type` are the error's class name, with two exceptions that both SDKs +apply the same way. An error with an HTTP status (400–599) and a code that is not only digits records the code, +such as openai's `insufficient_quota` on a `RateLimitError`. A generic provider error class +(TypeScript `Error`, `APIError`, `ApiError`, `AI_APICallError`, `AI_StreamProviderError`; Python +`Exception`, `APIError`, `APIStatusError`, `ClientError`, `ServerError`, `EventStreamError`) records +its code, else its HTTP status as a string (`429`), else the provider's error type +(`overloaded_error` from an Anthropic stream; Python also reads botocore's `response.Error.Code`, +with its first letter in upper case, so a Bedrock stream's `modelStreamErrorException` records +`ModelStreamErrorException` as in TypeScript). TypeScript records the AI SDK's `AI_RetryError` as +its last error. A thrown value that is not an Error keeps the fallback it had before: `Error` in the +core SDKs, `tool_error` or `error` in the AI SDK integration. + +A failure that a provider reports in a response body or a stream event, not as an HTTP error, is +recorded with the same rule: the integrations build it as a plain `Error` (TypeScript) or +`Exception` (Python) with the provider's `code` and a `type` that names the failure. Thus a +`response.failed` with `rate_limit_exceeded` records `rate_limit_exceeded`, one without a code +records `response.failed`, an OpenRouter numeric code records it as an HTTP status (`502`), and a +failed Google video operation records the `google.rpc.Code` name of its status (`RESOURCE_EXHAUSTED`). + `SpanHandle.serializeContent(value, key)` / `SpanHandle.serialize_content(value, key)` returns what `input` / `output` would emit for `value` under `key`: mask → JSON stringify → truncate. It returns `undefined` / `None` when the mask throws, the value cannot be serialized, or there is no diff --git a/packages/ai/src/batch.ts b/packages/ai/src/batch.ts index 6b955a34..fa0ad8f0 100644 --- a/packages/ai/src/batch.ts +++ b/packages/ai/src/batch.ts @@ -100,6 +100,7 @@ function wrapBatch( if (failed) { attempt(() => { const failure = failureRecord(config, error, "Batch request failed"); + attempt(() => span.setAttribute("error.type", failure.errorType)); attempt(() => span.setStatus({ code: SpanStatusCode.ERROR, message: failure.message })); attempt(() => span.recordException(failure.exception)); }); @@ -237,6 +238,7 @@ export const wrapGetBatchResults = ( if (failed) { attempt(() => { const failure = failureRecord(config, error, "Batch request failed"); + attempt(() => span.setAttribute("error.type", failure.errorType)); attempt(() => span.setStatus({ code: SpanStatusCode.ERROR, message: failure.message })); attempt(() => span.recordException(failure.exception)); }); diff --git a/packages/ai/src/integration_v6.ts b/packages/ai/src/integration_v6.ts index c68208cc..4f7f31c2 100644 --- a/packages/ai/src/integration_v6.ts +++ b/packages/ai/src/integration_v6.ts @@ -10,6 +10,7 @@ import { } from "@opentelemetry/api"; import { createGenerationEmitter, + errorTypeOf, type GenerationEmitter, type GenerationEmitterOverrides, jsonAttr, @@ -443,7 +444,7 @@ function createV6Hooks(config: ResolvedConfig, emitter: GenerationEmitter) { e.error instanceof Error ? e.error.message : unknownErrorMessage(e.error), ) ?? "Tool execution failed"; - const errorType = e.error instanceof Error ? e.error.name : "tool_error"; + const errorType = errorTypeOf(e.error, "tool_error"); span.setStatus({ code: SpanStatusCode.ERROR }); span.setAttribute("error.type", errorType); span.addEvent("exception", { diff --git a/packages/ai/src/integration_v7.ts b/packages/ai/src/integration_v7.ts index c7f0f1fc..2798ed7d 100644 --- a/packages/ai/src/integration_v7.ts +++ b/packages/ai/src/integration_v7.ts @@ -10,6 +10,7 @@ import { } from "@opentelemetry/api"; import { createGenerationEmitter, + errorTypeOf, type GenerationEmitter, type GenerationEmitterOverrides, jsonAttr, @@ -557,7 +558,7 @@ function createV7Hooks( fallback = "Tool execution failed", ): ProviderToolFailure => { try { - if (value instanceof Error && value.name.length > 0) errorType = value.name; + errorType = errorTypeOf(value, errorType); } catch { return { errorType, message: fallback }; } diff --git a/packages/ai/src/media.ts b/packages/ai/src/media.ts index 07542b50..c0c6c98a 100644 --- a/packages/ai/src/media.ts +++ b/packages/ai/src/media.ts @@ -158,6 +158,7 @@ function wrapMedia( if (failed) { attempt(() => { const failure = failureRecord(config, error, "Media generation failed"); + attempt(() => span.setAttribute("error.type", failure.errorType)); attempt(() => span.setStatus({ code: SpanStatusCode.ERROR, message: failure.message })); attempt(() => span.recordException(failure.exception)); }); diff --git a/packages/ai/src/shared.ts b/packages/ai/src/shared.ts index 88f4b714..3d5d6fb8 100644 --- a/packages/ai/src/shared.ts +++ b/packages/ai/src/shared.ts @@ -1,5 +1,5 @@ import type { Exception } from "@opentelemetry/api"; -import { jsonAttr } from "@telemetry-dev/otel"; +import { errorTypeOf, jsonAttr } from "@telemetry-dev/otel"; import type { ResolvedConfig, TelemetryDevOptions } from "./config.ts"; @@ -59,15 +59,18 @@ export function contentAttr( } /** - * What a failed wrapper call records as its span status message and exception. Without a mask - * that is the error itself; with one it is the masked message, or `fallback` when the mask omits - * it or throws, and no stack trace, which repeats the message. + * What a failed wrapper call records as its span's `error.type`, status message, and exception. + * `errorType` follows errorTypeOf and is also the exception's type. Without a mask the message is + * the error itself; with one it is the masked message, or `fallback` when the mask omits it or + * throws, and no stack trace, which repeats the message. */ export function failureRecord( config: Pick, error: unknown, fallback: string, -): { message: string; exception: Exception } { +): { errorType: string; message: string; exception: Exception } { + const errorType = errorTypeOf(error); + if (!config.mask) { let message = fallback; @@ -77,19 +80,23 @@ export function failureRecord( // An error whose string conversion throws is still recorded, with the fallback text. } - return { message, exception: error instanceof Error ? error : message }; + let exception: Exception = message; + + try { + if (error instanceof Error) { + exception = { name: errorType, message: error.message, stack: error.stack }; + } + } catch { + // A throwing message or stack getter leaves the status message as the exception. + } + + return { errorType, message, exception }; } - let name: string | undefined; let detail: string | undefined; try { - if (error instanceof Error) { - name = typeof error.name === "string" && error.name.length > 0 ? error.name : undefined; - detail = error.message; - } else { - detail = unknownErrorMessage(error); - } + detail = error instanceof Error ? error.message : unknownErrorMessage(error); } catch { detail = undefined; } @@ -97,7 +104,7 @@ export function failureRecord( const message = (detail ? contentAttr(config, "exception.message", detail) : undefined) || fallback; - return { message, exception: { name: name ?? "Error", message } }; + return { errorType, message, exception: { name: errorType, message } }; } /** Reports to onError without throwing, including for thrown values String() cannot convert. */ diff --git a/packages/ai/tests/integration_v7.test.ts b/packages/ai/tests/integration_v7.test.ts index cc12bf12..ea2c56ed 100644 --- a/packages/ai/tests/integration_v7.test.ts +++ b/packages/ai/tests/integration_v7.test.ts @@ -17,6 +17,7 @@ import { TraceIdRatioBasedSampler, } from "@opentelemetry/sdk-trace-base"; import { AlsContextManager, sessionSpanContext } from "@telemetry-dev/otel"; +import { APICallError, RetryError, StreamProviderError } from "ai"; import { AsyncLocalStorage } from "node:async_hooks"; import { expect, test, vi } from "vitest"; @@ -2751,6 +2752,60 @@ test("onError marks root and open step failed and a later onEnd for the same cal expect(step.attributes["gen_ai.request.model"]).toBe("gpt-4o"); }); +const apiCallError = (statusCode: number, data: unknown) => + new APICallError({ + message: "failed", + url: "https://api.example.com", + requestBodyValues: {}, + statusCode, + data, + }); + +test.each([ + { + name: "a retried quota error gives the provider's code", + error: new RetryError({ + message: "Failed after 3 attempts", + reason: "maxRetriesExceeded", + errors: [ + apiCallError(429, { error: { type: "insufficient_quota", code: "insufficient_quota" } }), + ], + }), + expected: "insufficient_quota", + }, + { + name: "an API call error without a code gives its HTTP status", + error: apiCallError(529, { type: "error", error: { type: "overloaded_error" } }), + expected: "529", + }, + { + name: "a stream error without a status gives the provider's type", + error: new StreamProviderError({ + message: "Overloaded", + type: "overloaded_error", + isRetryable: true, + }), + expected: "overloaded_error", + }, +])("onError: $name", async ({ error, expected }) => { + const { spanBatches, overrides } = makeCapture(); + const integ = telemetryDev(baseOptions, overrides); + const callId = "call-provider-error"; + + integ.onStart?.({ + callId, + operationId: "ai.generateText", + provider: "openai", + modelId: "gpt-4o", + }); + integ.onStepStart?.({ callId, provider: "openai", modelId: "gpt-4o", stepNumber: 0 }); + await integ.onError?.({ callId, error }); + + const spans = spanBatches[0]!; + expect(spans.find((s) => s.kind === SpanKind.CLIENT)!.attributes["error.type"]).toBe(expected); + expect(spans.find((s) => s.kind === SpanKind.INTERNAL)!.attributes["error.type"]).toBe(expected); +}); + test("onAbort ends the open step, keeps root UNSET, and emits generation.summary aborted", async () => { const { spanBatches, overrides } = makeCapture(); const integ = telemetryDev(baseOptions, overrides); diff --git a/packages/ai/tests/wrappers.test.ts b/packages/ai/tests/wrappers.test.ts index ff7c3714..6e146aca 100644 --- a/packages/ai/tests/wrappers.test.ts +++ b/packages/ai/tests/wrappers.test.ts @@ -627,6 +627,7 @@ test("failures whose string conversion throws are still recorded", async () => { const exception = span.events.find((event) => event.name === "exception"); expect(span.status).toEqual({ code: SpanStatusCode.ERROR, message: expected[index] }); + expect(span.attributes["error.type"]).toBe("RangeError"); expect(exception?.attributes).toMatchObject({ "exception.type": "RangeError", "exception.message": "disk full", diff --git a/packages/anthropic/tests/anthropic.test.ts b/packages/anthropic/tests/anthropic.test.ts index cc98fb60..20d30379 100644 --- a/packages/anthropic/tests/anthropic.test.ts +++ b/packages/anthropic/tests/anthropic.test.ts @@ -690,6 +690,26 @@ test("messages.create streaming preserves events and records aggregated text usa ).toBe(true); }); +test("an error event mid-stream records the provider's error type", async () => { + const spans = setupSpans(); + const overloaded = { + type: "error", + error: { type: "overloaded_error", message: "Overloaded" }, + }; + const fake = createFakeFetch(namedSseResponse([streamEvents()[0]!, overloaded])); + + const stream = await clientWith(fake.fetch).messages.create({ + model: "claude-sonnet-4-6", + max_tokens: 64, + messages: [{ role: "user", content: "Say hello" }], + stream: true, + }); + + // The SDK throws its generic APIError, whose class names no cause. + await expect(collectStream(stream)).rejects.toMatchObject({ type: "overloaded_error" }); + expect((await exportedSpan(spans)).attributes["error.type"]).toBe("overloaded_error"); +}); + test("stream capture retains no output when output capture is disabled", async () => { const spans = setupSpans(undefined, false); const capture = vi.spyOn(telemetrySdk, "boundedCaptureDetails"); diff --git a/packages/google-genai/src/index.ts b/packages/google-genai/src/index.ts index cf6afca0..bf548c6e 100644 --- a/packages/google-genai/src/index.ts +++ b/packages/google-genai/src/index.ts @@ -903,10 +903,45 @@ function mediaResponseFields(response: unknown, outputType: "image" | "video"): responseId: readString(record.responseId) ?? operationName, output: uris.length > 0 ? uris.map((uri) => ({ type: outputType, uri })) : undefined, attributes, - error: record.error, + error: operationError(record.error), }; } +// google.rpc.Code names by number. The error of a long-running operation is a google.rpc.Status, +// which carries only the number, such as 8 for RESOURCE_EXHAUSTED. +const RPC_CODE_NAMES = [ + "OK", + "CANCELLED", + "UNKNOWN", + "INVALID_ARGUMENT", + "DEADLINE_EXCEEDED", + "NOT_FOUND", + "ALREADY_EXISTS", + "PERMISSION_DENIED", + "RESOURCE_EXHAUSTED", + "FAILED_PRECONDITION", + "ABORTED", + "OUT_OF_RANGE", + "UNIMPLEMENTED", + "INTERNAL", + "UNAVAILABLE", + "DATA_LOSS", + "UNAUTHENTICATED", +]; + +// errorTypeOf records the code name in error.type, else "operation.failed". +function operationError(value: unknown): Error | undefined { + if (value === undefined || value === null) return undefined; + + const status = asRecord(value); + const code = readNumber(status?.code); + + return Object.assign(new Error(readString(status?.message) ?? "video operation failed"), { + code: code === undefined ? undefined : RPC_CODE_NAMES[code], + type: "operation.failed", + }); +} + function imageResponseFields(response: unknown): SpanFields { return mediaResponseFields(response, "image"); } diff --git a/packages/google-genai/tests/google-genai.test.ts b/packages/google-genai/tests/google-genai.test.ts index fc678dcd..fc2b568a 100644 --- a/packages/google-genai/tests/google-genai.test.ts +++ b/packages/google-genai/tests/google-genai.test.ts @@ -2948,7 +2948,8 @@ test("API error 400 rethrows ApiError and records error span", async () => { ).rejects.toMatchObject({ name: "ApiError", status: 400 }); const span = await exportedSpan(spans); expect(span.status.code).toBe(SPAN_STATUS_ERROR); - expect(span.attributes["error.type"]).toBe("ApiError"); + // ApiError names no cause, so the HTTP status stands in for it. + expect(span.attributes["error.type"]).toBe("400"); }); test("non-Error unary failures end spans and preserve the thrown values", async () => { @@ -3451,6 +3452,28 @@ test("generateVideos traces submit and tracked polling idempotently", async () = expect(exported.every((span) => span.attributes["gen_ai.output.type"] === "video")).toBe(true); }); +test("a failed video operation records its google.rpc code name as error.type", async () => { + const spans = setupSpans(); + const operation = { name: "operations/video-quota", done: false }; + const client = { + models: { generateVideos: (_params: unknown) => Promise.resolve(operation) }, + operations: { + get: (_params: unknown) => + Promise.resolve({ name: operation.name, done: true, error: { code: 8, message: "quota" } }), + }, + }; + + wrapGoogleGenAI(client); + const submitted = await client.models.generateVideos({ model: "veo", prompt: "ocean" }); + await client.operations.get({ operation: submitted }); + + const span = (await finishedSpans(spans, 2))[1]!; + const exception = span.events.find((event) => event.name === "exception"); + expect(span.status.code).toBe(SPAN_STATUS_ERROR); + expect(span.attributes["error.type"]).toBe("RESOURCE_EXHAUSTED"); + expect(exception?.attributes?.["exception.message"]).toBe("quota"); +}); + test("video operation tracking is identity-only and client-local", async () => { const spans = setupSpans(); const operationName = "operations/shared"; diff --git a/packages/openai/src/index.ts b/packages/openai/src/index.ts index 7c38ed0f..8f8d1090 100644 --- a/packages/openai/src/index.ts +++ b/packages/openai/src/index.ts @@ -331,12 +331,27 @@ function responsesResponse( return fields; } +// A failure that OpenAI reports in a response body or a stream event, not as an HTTP error. It +// carries the provider's code, and its kind for when there is no code, so that errorTypeOf records +// the cause in error.type instead of "Error". +function providerFailure( + message: string, + code: string | undefined, + kind: string | undefined, +): Error { + return Object.assign(new Error(message), { code, type: kind }); +} + function responseFailedError(response: T): Error { const error = asRecord(asRecord(response)?.error); const code = readString(error?.code); const message = readString(error?.message); - return new Error(["response.failed", code, message].filter(Boolean).join(": ")); + return providerFailure( + ["response.failed", code, message].filter(Boolean).join(": "), + code, + "response.failed", + ); } function responseStreamError(event: T): Error { @@ -344,7 +359,11 @@ function responseStreamError(event: T): Error { const code = readString(e?.code); const message = readString(e?.message); - return new Error(["response.error", code, message].filter(Boolean).join(": ")); + return providerFailure( + ["response.error", code, message].filter(Boolean).join(": "), + code, + "response.error", + ); } function responsesEventFailure(event: T): Error | undefined { @@ -363,7 +382,9 @@ function responsesEventFailure(event: T): Error | undefined { ? responseFailedError(asRecord(event)?.response) : responseStreamError(event); } catch { - return new Error(type === "response.failed" ? "response.failed" : "response.error"); + const kind = type === "response.failed" ? "response.failed" : "response.error"; + + return providerFailure(kind, undefined, kind); } } @@ -492,7 +513,11 @@ function videoResponse(response: T): SpanFields { finishReason: status, error: status === "failed" - ? asError(readString(asRecord(r.error)?.message) ?? "video failed") + ? providerFailure( + readString(asRecord(r.error)?.message) ?? "video failed", + readString(asRecord(r.error)?.code), + "video.failed", + ) : undefined, output: { status, @@ -1991,6 +2016,20 @@ function createObservedMediaStream( e.type === "image_edit.completed" || e.type === "image_edit.failed")); + if ( + operation === "images" && + (e.type === "image_generation.failed" || e.type === "image_edit.failed") + ) { + const failure = asRecord(e.error); + const code = readString(failure?.code) ?? readString(e.code); + const message = + readString(failure?.message) ?? + readString(e.message) ?? + `OpenAI media stream ended with ${e.type}`; + + error = providerFailure(code ? `${code}: ${message}` : message, code, e.type); + } + if (operation === "transcriptions" && e.type === "transcript.text.delta") { sawTranscriptDelta = true; } @@ -2944,8 +2983,10 @@ export function wrapOpenAIRealtime( if (trace) { finishTrace(trace, { - error: asError( + error: providerFailure( [readString(error?.code), readString(error?.message)].filter(Boolean).join(": "), + readString(error?.code), + readString(error?.type), ), }); } @@ -3028,9 +3069,11 @@ export function wrapOpenAIRealtime( : undefined, error: status === "failed" - ? asError( + ? providerFailure( readString(asRecord(statusDetails?.error)?.message) ?? "Realtime response failed", + readString(asRecord(statusDetails?.error)?.code), + readString(asRecord(statusDetails?.error)?.type) ?? "response.failed", ) : undefined, }, diff --git a/packages/openai/tests/openai.test.ts b/packages/openai/tests/openai.test.ts index 06d244ae..b41101f9 100644 --- a/packages/openai/tests/openai.test.ts +++ b/packages/openai/tests/openai.test.ts @@ -477,6 +477,37 @@ test("wrapped chat completion APIPromise keeps parse helper and promise identity ]); }); +test.each([ + // A RateLimitError whose code says the account is out of quota, not rate limited. + { status: 429, code: "insufficient_quota", expected: "insufficient_quota" }, + { status: 400, code: "context_length_exceeded", expected: "context_length_exceeded" }, + // Without a code, the class name is specific enough (Python parity: type(e).__name__). + { status: 500, code: null, expected: "InternalServerError" }, + // A numeric code, such as Z.ai's 1302 rate limit, names less than the class. + { status: 429, code: "1302", expected: "RateLimitError" }, +])( + "a $status error with code $code records error.type $expected", + async ({ status, code, expected }) => { + const spans = setupSpans(); + const body = { error: { message: "failed", type: "error", code } }; + const fake = createFakeFetch( + new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }), + ); + + await expect( + clientWith(fake.fetch).chat.completions.create({ + model: "gpt-x", + messages: [{ role: "user", content: "Fail" }], + }), + ).rejects.toMatchObject({ status }); + + expect((await exportedSpan(spans)).attributes["error.type"]).toBe(expected); + }, +); + test("chat completions record one error span when the OpenAI API returns 4xx", async () => { const spans = setupSpans(); const fake = createFakeFetch(jsonErrorResponse(400, "bad model")); @@ -492,11 +523,10 @@ test("chat completions record one error span when the OpenAI API returns 4xx", a expect(span.status.code).toBe(SPAN_STATUS_ERROR); expect(span.attributes["gen_ai.operation.name"]).toBe("chat"); expect(span.attributes["gen_ai.request.model"]).toBe("gpt-bad"); - // The SDK falls back to the constructor name when err.name is the generic "Error" - // (openai's APIError subclasses never override name) — Python parity: type(e).__name__. - expect(span.attributes["error.type"]).toBe("BadRequestError"); + // The error code from the response body names the failure. + expect(span.attributes["error.type"]).toBe("bad_request"); const exception = span.events.find((event) => event.name === "exception"); - expect(exception?.attributes?.["exception.type"]).toBe("BadRequestError"); + expect(exception?.attributes?.["exception.type"]).toBe("bad_request"); expect(exception?.attributes?.["exception.message"]).toContain("bad model"); expect(exception?.attributes?.["log.severity_number"]).toBe(17); }); @@ -3227,9 +3257,9 @@ test("responses create failed body records error while completed body stays OK", const failed = finished.find((span) => span.attributes["gen_ai.response.id"] === "resp_failed"); const success = finished.find((span) => span.attributes["gen_ai.response.id"] === "resp_ok"); expect(failed?.status.code).toBe(SPAN_STATUS_ERROR); - expect(failed?.attributes["error.type"]).toBe("Error"); + expect(failed?.attributes["error.type"]).toBe("server_error"); const exception = failed?.events.find((event) => event.name === "exception"); - expect(exception?.attributes?.["exception.type"]).toBe("Error"); + expect(exception?.attributes?.["exception.type"]).toBe("server_error"); expect(exception?.attributes?.["exception.message"]).toBe( "response.failed: server_error: model exploded", ); @@ -3986,9 +4016,9 @@ test("responses streams record error status from bare error events", async () => expect(events).toEqual([errorEvent]); const span = await exportedSpan(spans); expect(span.status.code).toBe(SPAN_STATUS_ERROR); - expect(span.attributes["error.type"]).toBe("Error"); + expect(span.attributes["error.type"]).toBe("rate_limit_exceeded"); const exception = span.events.find((event) => event.name === "exception"); - expect(exception?.attributes?.["exception.type"]).toBe("Error"); + expect(exception?.attributes?.["exception.type"]).toBe("rate_limit_exceeded"); expect(exception?.attributes?.["exception.message"]).toBe( "response.error: rate_limit_exceeded: try later", ); @@ -4040,7 +4070,7 @@ test("responses streams record error status from response.failed terminal events expect(span.attributes["gen_ai.response.finish_reasons"]).toEqual(["failed"]); expect(jsonAttr(span, "gen_ai.output.messages")).toEqual([]); const exception = span.events.find((event) => event.name === "exception"); - expect(exception?.attributes?.["exception.type"]).toBe("Error"); + expect(exception?.attributes?.["exception.type"]).toBe("server_error"); expect(exception?.attributes?.["exception.message"]).toBe( "response.failed: server_error: stream failed", ); @@ -4776,6 +4806,33 @@ test.each([true, false])( }, ); +test("a failed image stream records the provider's code as error.type", async () => { + const spans = setupSpans(); + const failed = { type: "image_generation.failed", code: "provider_error", message: "failed" }; + const source = new Stream(async function* () { + yield failed; + }, new AbortController()); + const client = wrapOpenAI({ + chat: { completions: { create() {} } }, + responses: { create() {} }, + embeddings: { create() {} }, + images: { generate: async (_params: unknown) => source }, + }); + + const stream = await client.images!.generate({ + model: "gpt-image-1", + prompt: "otter", + stream: true, + }); + expect(await collectStream(stream as Stream)).toEqual([failed]); + + const span = await exportedSpan(spans); + const exception = span.events.find((event) => event.name === "exception"); + expect(span.status.code).toBe(SPAN_STATUS_ERROR); + expect(span.attributes["error.type"]).toBe("provider_error"); + expect(exception?.attributes?.["exception.message"]).toBe("provider_error: failed"); +}); + test("endpoint modality fills aggregate image and transcription output usage", async () => { const spans = setupSpans(); @@ -5531,6 +5588,43 @@ test("realtime caps reconstructed output before a trace timeout", async () => { wrapped.close(); }); +test("a failed realtime response records the provider's code as error.type", async () => { + const spans = setupSpans(); + const listeners = new Map void>>(); + const sent: Array<{ response?: object }> = []; + const wrapped = wrapOpenAIRealtime({ + send(event: unknown) { + sent.push(event as { response?: object }); + }, + close() {}, + on(event: string, listener: (...args: unknown[]) => void) { + listeners.set(event, [...(listeners.get(event) ?? []), listener]); + }, + off() {}, + }); + const emit = (value: unknown) => listeners.get("event")?.forEach((listener) => listener(value)); + + wrapped.send({ type: "response.create" }); + emit({ type: "response.created", response: { id: "failed", ...sent[0]?.response } }); + emit({ + type: "response.done", + response: { + id: "failed", + status: "failed", + status_details: { + type: "failed", + error: { type: "server_error", code: "rate_limit_exceeded", message: "slow down" }, + }, + output: [], + }, + }); + + const span = await exportedSpan(spans); + expect(span.status.code).toBe(SPAN_STATUS_ERROR); + expect(span.attributes["error.type"]).toBe("rate_limit_exceeded"); + wrapped.close(); +}); + test("realtime terminal output supersedes truncated deltas", async () => { const spans = setupSpans(); const listeners = new Map void>>(); @@ -5868,6 +5962,7 @@ test("realtime wrapper keeps recoverable API errors scoped and handles remote cl const failed = await finishedSpans(spans, 2); expect(failed[1]?.status.code).toBe(SPAN_STATUS_ERROR); + expect(failed[1]?.attributes["error.type"]).toBe("invalid_response"); wrapped.send({ type: "response.create" }); const sentEvent = sent.at(-1) as { event_id?: unknown } | undefined; diff --git a/packages/openrouter/src/index.ts b/packages/openrouter/src/index.ts index ad9775aa..f5c1d819 100644 --- a/packages/openrouter/src/index.ts +++ b/packages/openrouter/src/index.ts @@ -535,20 +535,42 @@ function responsesResponse(response: T, includeOutput = true): SpanFields { return fields; } +// A failure that OpenRouter reports in a response body or a stream event, not as an HTTP error. +// errorTypeOf records its string code, else its numeric code (an HTTP status, such as 429), else +// its kind, in error.type instead of "Error". +function providerFailure( + prefix: string, + code: number | string | undefined, + message: string | undefined, + kind: string, +): Error { + return Object.assign(new Error([prefix, code, message].filter(Boolean).join(": ")), { + code: typeof code === "string" ? code : undefined, + status: typeof code === "number" ? code : undefined, + type: kind, + }); +} + function responseFailedError(response: T): Error { const error = asRecord(asRecord(response)?.error); - const code = readNumber(error?.code) ?? readString(error?.code); - const message = readString(error?.message); - return new Error(["response.failed", code, message].filter(Boolean).join(": ")); + return providerFailure( + "response.failed", + readNumber(error?.code) ?? readString(error?.code), + readString(error?.message), + "response.failed", + ); } function responseStreamError(event: T): Error { const e = asRecord(event); - const code = readNumber(e?.code) ?? readString(e?.code); - const message = readString(e?.message); - return new Error(["response.error", code, message].filter(Boolean).join(": ")); + return providerFailure( + "response.error", + readNumber(e?.code) ?? readString(e?.code), + readString(e?.message), + "response.error", + ); } function embeddingsRequest(request: JsonRecord): RequestMapping { @@ -771,7 +793,7 @@ function recordChatChunk(chunk: T, states: Map, budg return { ...update, - error: new Error(["chat stream error", code, message].filter(Boolean).join(": ")), + error: providerFailure("chat stream error", code, message, "stream error"), }; } diff --git a/packages/openrouter/tests/openrouter.test.ts b/packages/openrouter/tests/openrouter.test.ts index c1a74f8d..6b43d623 100644 --- a/packages/openrouter/tests/openrouter.test.ts +++ b/packages/openrouter/tests/openrouter.test.ts @@ -1072,7 +1072,7 @@ test("chat stream chunk errors end one error span with partial output", async () await reader.cancel(); expect(spans.getFinishedSpans()).toHaveLength(1); expect(span.status.code).toBe(SPAN_STATUS_ERROR); - expect(span.attributes["error.type"]).toBe("Error"); + expect(span.attributes["error.type"]).toBe("502"); expect(jsonAttr(span, "gen_ai.output.messages")).toEqual([ { role: "assistant", content: "Partial" }, ]); @@ -2673,7 +2673,7 @@ test("non-streaming failed Responses record a useful error", async () => { const span = await exportedSpan(spans); expect(span.status.code).toBe(SPAN_STATUS_ERROR); - expect(span.attributes["error.type"]).toBe("Error"); + expect(span.attributes["error.type"]).toBe("server_error"); expect(span.events[0]?.attributes?.["exception.message"]).toContain( "response.failed: server_error: provider failed", ); @@ -2706,6 +2706,7 @@ test("failed Responses stream events keep numeric provider error codes", async ( const span = await exportedSpan(spans); expect(span.status.code).toBe(SPAN_STATUS_ERROR); + expect(span.attributes["error.type"]).toBe("502"); expect(span.events[0]?.attributes?.["exception.message"]).toBe( "response.failed: 502: provider failed", ); diff --git a/packages/otel/src/errors.ts b/packages/otel/src/errors.ts new file mode 100644 index 00000000..1723d29e --- /dev/null +++ b/packages/otel/src/errors.ts @@ -0,0 +1,83 @@ +// Error classes that provider SDKs throw for any failed request, whose name says nothing about the +// cause: openai's and Anthropic's APIError (a stream's error event), @google/genai's ApiError, and +// the AI SDK's APICallError and StreamProviderError. Plain Error is here for errors that carry a +// code, such as the failures that the integrations build from a response body with `code`, +// `status`, and `type` fields. The Python SDK keeps the same rule in `_error_type`; change both +// together. +const GENERIC_ERROR_NAMES = new Set([ + "Error", + "APIError", + "ApiError", + "AI_APICallError", + "AI_StreamProviderError", +]); + +function record(value: unknown): Record | undefined { + return typeof value === "object" && value !== null + ? (value as Record) + : undefined; +} + +function text(value: unknown): string | undefined { + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +// Some providers send numeric codes, such as Z.ai's "1302" for a rate limit, and openai's Python SDK +// turns OpenRouter's numeric code into a string. A number names no cause the way a class name does. +function errorCode(value: unknown): string | undefined { + const code = text(value); + + return code === undefined || /^\d+$/.test(code) ? undefined : code; +} + +function httpStatus(value: unknown): number | undefined { + return typeof value === "number" && Number.isInteger(value) && value >= 400 && value <= 599 + ? value + : undefined; +} + +function className(error: Error): string { + // Many SDK error classes (e.g. openai's BadRequestError) never override the inherited "Error" + // name, so the constructor name is the specific one (Python parity: type(error).__name__). + return error.name !== "Error" && error.name.length > 0 + ? error.name + : (error.constructor?.name ?? "Error") || "Error"; +} + +/** + * The `error.type` of a failure: the provider's error code when the error carries one, so that a + * rate limit, an exhausted quota, an overflowing context, and an overloaded provider read apart. + * + * - An error with an HTTP status and a code that is not only digits (openai's APIError: + * `insufficient_quota`, `context_length_exceeded`) gives the code. + * - A generic provider error class (see GENERIC_ERROR_NAMES) gives its code, else its HTTP status + * (`429`), else the provider's error type (Anthropic's `overloaded_error` from a stream). + * - The AI SDK's RetryError gives the type of the last error it retried. + * - Any other error gives its class name; a value that is not an Error gives `fallback`. + */ +export function errorTypeOf(error: unknown, fallback = "Error"): string { + if (!(error instanceof Error)) return fallback; + + try { + const fields = error as Error & Record; + + if (fields.name === "AI_RetryError" && fields.lastError instanceof Error) { + return errorTypeOf(fields.lastError, fallback); + } + + const name = className(error); + const data = record(record(fields.data)?.error); + const code = errorCode(fields.code) ?? errorCode(data?.code); + const status = httpStatus(fields.status) ?? httpStatus(fields.statusCode); + + if (code !== undefined && status !== undefined) return code; + if (!GENERIC_ERROR_NAMES.has(name)) return name; + if (code !== undefined) return code; + if (status !== undefined) return String(status); + + return text(fields.type) ?? text(data?.type) ?? name; + } catch { + // A getter on a third-party error can throw; recording the error must not. + return fallback; + } +} diff --git a/packages/otel/src/index.ts b/packages/otel/src/index.ts index 8c87c910..26b8eec3 100644 --- a/packages/otel/src/index.ts +++ b/packages/otel/src/index.ts @@ -33,6 +33,8 @@ export { export { diag, reportError, setLogLevel } from "./debug.ts"; +export { errorTypeOf } from "./errors.ts"; + export { createGenerationEmitter, type GenerationEmitter, diff --git a/packages/sdk/src/attrs.ts b/packages/sdk/src/attrs.ts index 6dbf34f5..8c4af184 100644 --- a/packages/sdk/src/attrs.ts +++ b/packages/sdk/src/attrs.ts @@ -7,8 +7,8 @@ import { type CaptureConfig, prepareCaptureValue, truncate } from "./capture.ts" export type SpanType = "span" | "generation" | "tool" | "agent" | "embedding"; // Operation names are taken verbatim by the ingest's deriveOperation. Plain spans always get an -// explicit "function": without one, the propagated gen_ai.conversation.id would trip the ingest's -// hasGenAi heuristic and misclassify them as "llm". +// explicit "function": without one, the ingest guesses "llm" from other gen_ai.* keys on the span, +// such as a gen_ai.agent.name the caller set. export const SPAN_TYPE_OPERATIONS = { span: "function", generation: "chat", diff --git a/packages/sdk/src/span.ts b/packages/sdk/src/span.ts index 877f74c0..975c0fc3 100644 --- a/packages/sdk/src/span.ts +++ b/packages/sdk/src/span.ts @@ -21,6 +21,7 @@ import { import { activeContext, DURATION_BUCKETS, + errorTypeOf, omitUndefined, OUTPUT_CHUNK_HISTOGRAM, type OutputChunkHistogram, @@ -159,15 +160,7 @@ function resolveParent(parent: ParentRef | undefined): Context { function applyError(span: Span, error: unknown, capture: FieldCaptureConfig): void { const err = error instanceof Error ? error : undefined; - - // Prefer the specific subclass name (Python parity: type(error).__name__) — many SDK - // error classes (e.g. openai's BadRequestError) never override the inherited "Error" name. - const errorType = - err === undefined - ? "Error" - : err.name !== "Error" && err.name.length > 0 - ? err.name - : (err.constructor?.name ?? "Error") || "Error"; + const errorType = errorTypeOf(error); span.setStatus({ code: SpanStatusCode.ERROR }); span.setAttribute("error.type", errorType); diff --git a/packages/tanstack-ai/src/errors.ts b/packages/tanstack-ai/src/errors.ts index b114fc77..4ebd2916 100644 --- a/packages/tanstack-ai/src/errors.ts +++ b/packages/tanstack-ai/src/errors.ts @@ -1,3 +1,5 @@ +import { errorTypeOf } from "@telemetry-dev/otel"; + export function errorDetails(error: unknown, fallbackType = "Error") { const details = { type: fallbackType, message: "Unknown error" }; @@ -7,7 +9,8 @@ export function errorDetails(error: unknown, fallbackType = "Error") { if (error !== null && typeof error === "object") { const { name, message } = error as { name?: unknown; message?: unknown }; - if (typeof name === "string" && name.length > 0) details.type = name; + if (error instanceof Error) details.type = errorTypeOf(error, fallbackType); + else if (typeof name === "string" && name.length > 0) details.type = name; if (typeof message === "string") details.message = message; } diff --git a/sdks/python-anthropic/tests/test_anthropic.py b/sdks/python-anthropic/tests/test_anthropic.py index 60f113bf..22aaa908 100644 --- a/sdks/python-anthropic/tests/test_anthropic.py +++ b/sdks/python-anthropic/tests/test_anthropic.py @@ -1227,6 +1227,22 @@ def mask(value: Any, _context: telemetry_dev.MaskContext) -> Any: assert a["telemetry.dev.capture.truncated"] is True +def test_error_event_mid_stream_records_the_provider_error_type(memory: SimpleNamespace) -> None: + overloaded = {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}} + client = wrapped_sync_client( + lambda _request: named_sse_response([stream_events()[0], overloaded]) + ) + stream = client.messages.create( + model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES, stream=True + ) + + # The SDK raises its generic APIStatusError, with the 200 status of the stream. + with pytest.raises(anthropic.APIStatusError): + list(stream) + + assert attrs(only_span(memory))["error.type"] == "overloaded_error" + + def test_mid_stream_read_error_records_error_span(memory: SimpleNamespace) -> None: def handler(request: httpx.Request) -> httpx.Response: return failing_sync_sse_response(stream_events()[:3], "sync stream broke") diff --git a/sdks/python-bedrock/tests/test_bedrock.py b/sdks/python-bedrock/tests/test_bedrock.py index 764f3dc4..19e1de8c 100644 --- a/sdks/python-bedrock/tests/test_bedrock.py +++ b/sdks/python-bedrock/tests/test_bedrock.py @@ -506,7 +506,7 @@ def test_converse_stream_accumulates_and_errors( with pytest.raises(EventStreamError): list(response["stream"]) error_span = memory.span_exporter.get_finished_spans()[1] - assert error_span.attributes["error.type"] == "EventStreamError" + assert error_span.attributes["error.type"] == "ModelStreamErrorException" assert error_span.attributes["aws.error.code"] == "modelStreamErrorException" assert error_span.attributes["gen_ai.response.id"] == "stream-error-2" assert error_span.attributes["aws.http.status_code"] == 424 diff --git a/sdks/python-google-genai/src/telemetry_dev_google_genai/__init__.py b/sdks/python-google-genai/src/telemetry_dev_google_genai/__init__.py index 9362a3c3..5eaebd61 100644 --- a/sdks/python-google-genai/src/telemetry_dev_google_genai/__init__.py +++ b/sdks/python-google-genai/src/telemetry_dev_google_genai/__init__.py @@ -468,15 +468,49 @@ def mapper(model: Any, contents: Any, config: Any) -> tuple[str, dict[str, Any]] return mapper +# google.rpc.Code names by number. The error of a long-running operation is a google.rpc.Status, +# which carries only the number, such as 8 for RESOURCE_EXHAUSTED. +_RPC_CODE_NAMES = ( + "OK", + "CANCELLED", + "UNKNOWN", + "INVALID_ARGUMENT", + "DEADLINE_EXCEEDED", + "NOT_FOUND", + "ALREADY_EXISTS", + "PERMISSION_DENIED", + "RESOURCE_EXHAUSTED", + "FAILED_PRECONDITION", + "ABORTED", + "OUT_OF_RANGE", + "UNIMPLEMENTED", + "INTERNAL", + "UNAVAILABLE", + "DATA_LOSS", + "UNAUTHENTICATED", +) + + +def _operation_error(status: Any) -> Exception: + # The core SDK records the code name in error.type, else "operation.failed", because a plain + # Exception names no cause; see _error_type in telemetry_dev/_spans.py. + code = _field(status, "code") + error = Exception(_string(_field(status, "message")) or "video operation failed") + vars(error).update( + code=_RPC_CODE_NAMES[code] + if isinstance(code, int) and not isinstance(code, bool) and 0 <= code < len(_RPC_CODE_NAMES) + else None, + type="operation.failed", + ) + return error + + def _media_response_fields(response: Any, output_type: str | None = None) -> dict[str, Any]: operation_name = _string(_field(response, "name")) operation_error = _field(response, "error") - error_message = _string(_field(operation_error, "message")) fields: dict[str, Any] = { "response_id": _string(_field(response, "id")) or operation_name, - "error": RuntimeError(error_message or "video operation failed") - if operation_error - else None, + "error": _operation_error(operation_error) if operation_error else None, } attrs: dict[str, Any] = {} done = _field(response, "done") diff --git a/sdks/python-google-genai/tests/test_google_genai.py b/sdks/python-google-genai/tests/test_google_genai.py index d6929f55..0f506e0b 100644 --- a/sdks/python-google-genai/tests/test_google_genai.py +++ b/sdks/python-google-genai/tests/test_google_genai.py @@ -670,6 +670,8 @@ async def run() -> Any: span = only_span(memory) assert span.status.status_code is StatusCode.ERROR assert attrs(span)["google_genai.response.operation_done"] is True + assert attrs(span)["error.type"] == "INTERNAL" + assert dict(span.events[0].attributes or {})["exception.message"] == "generation failed" SUPPORT_PROMPT = telemetry_dev.ChatPrompt( @@ -2341,9 +2343,9 @@ async def run() -> None: @pytest.mark.parametrize("async_mode", [False, True], ids=["sync", "async"]) def test_api_error_records_error_and_reraises(memory: SimpleNamespace, async_mode: bool) -> None: - error = errors.APIError( - 400, - {"error": {"message": "bad", "status": "INVALID_ARGUMENT", "code": 400}}, + error = errors.ClientError( + 429, + {"error": {"message": "slow down", "status": "RESOURCE_EXHAUSTED", "code": 429}}, ) client, _transport = client_with_transport([error]) wrapped = wrap_google_genai(client) @@ -2361,6 +2363,8 @@ def test_api_error_records_error_and_reraises(memory: SimpleNamespace, async_mod span = only_span(memory) assert span.status.status_code == StatusCode.ERROR + # ClientError names no cause, so the HTTP status stands in for it. + assert attrs(span)["error.type"] == "429" @pytest.mark.parametrize("async_mode", [False, True], ids=["sync", "async"]) diff --git a/sdks/python-litellm/src/telemetry_dev_litellm/__init__.py b/sdks/python-litellm/src/telemetry_dev_litellm/__init__.py index 5d99c665..5c15dc97 100644 --- a/sdks/python-litellm/src/telemetry_dev_litellm/__init__.py +++ b/sdks/python-litellm/src/telemetry_dev_litellm/__init__.py @@ -751,12 +751,21 @@ def _rerank_response(response: Any) -> dict[str, Any]: } -def _response_error(response: Any) -> RuntimeError | None: +def _provider_failure(message: str, code: str | None, kind: str) -> Exception: + # A failure that the provider reports in a response body or a stream event, not as an HTTP + # error. The core SDK records its `code` in error.type, else its `type`, because a plain + # Exception names no cause; see _error_type in telemetry_dev/_spans.py. + error = Exception(message) + vars(error).update(code=code, type=kind) + return error + + +def _response_error(response: Any) -> Exception | None: if _string(_field(response, "status")) != "failed" and _field(response, "error") is None: return None raw_error = _field(response, "error") message = _string(_field(raw_error, "message")) or "LiteLLM Responses API request failed" - return RuntimeError(message) + return _provider_failure(message, _string(_field(raw_error, "code")), "response.failed") def _responses_response(response: Any) -> dict[str, Any]: @@ -1908,7 +1917,7 @@ def _record(self, event: Any, received_at: float) -> None: code = _string(_field(raw_error, "code")) or _string(_field(event, "code")) if code is not None: message = f"{message} ({code})" - self._finish(error=RuntimeError(message)) + self._finish(error=_provider_failure(message, code, "response.error")) def _finish(self, response: Any = None, error: BaseException | None = None) -> None: if self._finished: diff --git a/sdks/python-litellm/tests/test_litellm.py b/sdks/python-litellm/tests/test_litellm.py index 6770e154..c575801e 100644 --- a/sdks/python-litellm/tests/test_litellm.py +++ b/sdks/python-litellm/tests/test_litellm.py @@ -3950,10 +3950,10 @@ def fake_responses(input: Any, model: str, stream: bool = False, **kwargs: Any) failed_span, error_span = finished_spans(memory) assert failed_span.status.status_code is StatusCode.ERROR - assert attrs(failed_span)["error.type"] == "RuntimeError" + assert attrs(failed_span)["error.type"] == "response.failed" assert attrs(failed_span)["gen_ai.response.status"] == "failed" assert error_span.status.status_code is StatusCode.ERROR - assert attrs(error_span)["error.type"] == "RuntimeError" + assert attrs(error_span)["error.type"] == "transport_error" exception = next(event for event in error_span.events if event.name == "exception") assert exception.attributes is not None assert exception.attributes["exception.message"] == "stream disconnected (transport_error)" diff --git a/sdks/python-openai/src/telemetry_dev_openai/__init__.py b/sdks/python-openai/src/telemetry_dev_openai/__init__.py index d633dfa1..0de76428 100644 --- a/sdks/python-openai/src/telemetry_dev_openai/__init__.py +++ b/sdks/python-openai/src/telemetry_dev_openai/__init__.py @@ -1550,34 +1550,22 @@ def _response_event_has_output(event: Any) -> bool: return isinstance(delta, str) and bool(delta) -def _response_failed_error(response: Any) -> RuntimeError: +def _response_failed_error(response: Any) -> Exception: error = _field(response, "error") - if error is None: - return RuntimeError("response.failed") - code = _string(_field(error, "code")) - message = _string(_field(error, "message")) - if code and message: - return RuntimeError(f"response.failed: {code}: {message}") - if code: - return RuntimeError(f"response.failed: {code}") - if message: - return RuntimeError(f"response.failed: {message}") - return RuntimeError("response.failed") - - -def _response_stream_error(event: Any) -> RuntimeError: + code = _string(_field(error, "code")) if error is not None else None + message = _string(_field(error, "message")) if error is not None else None + text = ": ".join(part for part in ("response.failed", code, message) if part) + return _provider_failure(text, code, "response.failed") + + +def _response_stream_error(event: Any) -> Exception: code = _string(_field(event, "code")) message = _string(_field(event, "message")) - if code and message: - return RuntimeError(f"response.error: {code}: {message}") - if code: - return RuntimeError(f"response.error: {code}") - if message: - return RuntimeError(f"response.error: {message}") - return RuntimeError("response.error") + text = ": ".join(part for part in ("response.error", code, message) if part) + return _provider_failure(text, code, "response.error") -def _responses_event_failure(event: Any) -> RuntimeError | None: +def _responses_event_failure(event: Any) -> Exception | None: try: event_type = _field(event, "type") except Exception: @@ -1586,15 +1574,25 @@ def _responses_event_failure(event: Any) -> RuntimeError | None: try: return _response_failed_error(_field(event, "response")) except Exception: - return RuntimeError("response.failed") + return _provider_failure("response.failed", None, "response.failed") if event_type == "error": try: return _response_stream_error(event) except Exception: - return RuntimeError("response.error") + return _provider_failure("response.error", None, "response.error") return None +def _provider_failure(message: str, code: str | int | None, kind: str | None) -> Exception: + # A failure that the provider reports in a response body or a stream event, not as an HTTP + # error. The core SDK records its `code` in error.type (a number counts as an HTTP status, such + # as 429), else its `type`, because a plain Exception names no cause; see _error_type in + # telemetry_dev/_spans.py. + error = Exception(message) + vars(error).update(code=code, type=kind) + return error + + def _hook_response_close(inner: Any, finish: Callable[[], None]) -> None: """End the span when the transport response is closed behind our back. @@ -2433,7 +2431,7 @@ def _media_stream_error(event: Any, event_type: str) -> BaseException | None: or f"OpenAI media stream ended with {event_type}" ) code = _string(_field(raw_error, "code")) or _string(_field(event, "code")) - return RuntimeError(f"{code}: {message}" if code else message) + return _provider_failure(f"{code}: {message}" if code else message, code, event_type) def _media_event_failure(event: Any) -> BaseException | None: @@ -2447,7 +2445,9 @@ def _media_event_failure(event: Any) -> BaseException | None: return _media_stream_error(event, event_type) except Exception: if event_type.endswith(".failed"): - return RuntimeError(f"OpenAI media stream ended with {event_type}") + return _provider_failure( + f"OpenAI media stream ended with {event_type}", None, event_type + ) return None diff --git a/sdks/python-openai/tests/test_openai.py b/sdks/python-openai/tests/test_openai.py index fc1ea943..263e8a7d 100644 --- a/sdks/python-openai/tests/test_openai.py +++ b/sdks/python-openai/tests/test_openai.py @@ -552,6 +552,31 @@ def handler(_request: httpx.Request) -> httpx.Response: assert attrs(span)["gen_ai.usage.image.output_tokens"] == 9 +@pytest.mark.parametrize( + ("status", "code", "expected"), + [ + # A RateLimitError whose code says the account is out of quota, not rate limited. + (429, "insufficient_quota", "insufficient_quota"), + (400, "context_length_exceeded", "context_length_exceeded"), + # Without a code, the class name is specific enough. + (500, None, "InternalServerError"), + # A numeric code, such as Z.ai's 1302 rate limit or OpenRouter's 429, names less. + (429, "1302", "RateLimitError"), + (429, 429, "RateLimitError"), + ], +) +def test_api_error_records_the_provider_error_code( + memory: SimpleNamespace, status: int, code: str | int | None, expected: str +) -> None: + body = {"error": {"message": "failed", "type": "error", "code": code}} + client = wrap_openai(sync_client(lambda _request: httpx.Response(status, json=body))) + + with pytest.raises(openai.APIStatusError): + client.chat.completions.create(model="gpt-x", messages=cast(Any, CHAT_MESSAGES)) + + assert attrs(only_span(memory))["error.type"] == expected + + @pytest.mark.parametrize("completed", [True, False]) def test_image_stream_is_flagged_only_without_a_terminal_event( memory: SimpleNamespace, completed: bool @@ -1227,7 +1252,7 @@ async def handler(_request: httpx.Request) -> httpx.Response: assert received == ["image_generation.failed"] span = only_span(memory) assert span.status.status_code == StatusCode.ERROR - assert attrs(span)["error.type"] == "RuntimeError" + assert attrs(span)["error.type"] == "provider_error" assert dict(span.events[0].attributes or {})["exception.message"] == ( "provider_error: provider failed" ) @@ -4773,7 +4798,7 @@ def handler(request: httpx.Request) -> httpx.Response: span = only_span(memory) assert span.status.status_code == StatusCode.ERROR a = attrs(span) - assert a["error.type"] == "RuntimeError" + assert a["error.type"] == "server_error" assert a["gen_ai.response.id"] == "resp_failed" events = list(span.events) assert len(events) == 1 @@ -5100,7 +5125,7 @@ async def handler(request: httpx.Request) -> httpx.Response: assert completed_attrs["gen_ai.usage.output_tokens"] == 4 assert completed_attrs["gen_ai.usage.total_tokens"] == 17 assert failed_span.status.status_code == StatusCode.ERROR - assert attrs(failed_span)["error.type"] == "RuntimeError" + assert attrs(failed_span)["error.type"] == "server_error" @pytest.mark.parametrize("async_mode", [False, True]) diff --git a/sdks/python-openrouter/src/telemetry_dev_openrouter/__init__.py b/sdks/python-openrouter/src/telemetry_dev_openrouter/__init__.py index 443dd60f..e4ed2e4a 100644 --- a/sdks/python-openrouter/src/telemetry_dev_openrouter/__init__.py +++ b/sdks/python-openrouter/src/telemetry_dev_openrouter/__init__.py @@ -303,44 +303,56 @@ def _responses_usage(raw: Any) -> dict[str, int | float] | None: ) -def _response_failed_error(response: Any) -> RuntimeError: +def _failure_code(value: Any) -> str | int | None: + # OpenRouter's numeric codes are HTTP statuses, which the core SDK records as one. + if isinstance(value, int) and not isinstance(value, bool): + return value + return value if isinstance(value, str) else None + + +def _provider_failure(message: str, code: str | int | None, kind: str | None) -> Exception: + # A failure that OpenRouter reports in a response body or a stream event, not as an HTTP + # error. The core SDK records its `code` in error.type (a number counts as an HTTP status, such + # as 429), else its `type`, because a plain Exception names no cause; see _error_type in + # telemetry_dev/_spans.py. + error = Exception(message) + vars(error).update(code=code, type=kind) + return error + + +def _response_failed_error(response: Any) -> Exception: error = _field(response, "error") if error is None: - return RuntimeError("response.failed") - code = _error_code(_field(error, "code")) + return _provider_failure("response.failed", None, "response.failed") + raw_code = _field(error, "code") + code = _error_code(raw_code) message = _string(_field(error, "message")) - if code and message: - return RuntimeError(f"response.failed: {code}: {message}") - if code: - return RuntimeError(f"response.failed: {code}") - if message: - return RuntimeError(f"response.failed: {message}") - return RuntimeError("response.failed") + text = ": ".join(part for part in ("response.failed", code, message) if part) + return _provider_failure(text, _failure_code(raw_code), "response.failed") -def _response_stream_error(event: Any) -> RuntimeError: - code = _error_code(_field(event, "code")) +def _response_stream_error(event: Any) -> Exception: + raw_code = _field(event, "code") + code = _error_code(raw_code) message = _string(_field(event, "message")) - if code and message: - return RuntimeError(f"response.error: {code}: {message}") - if code: - return RuntimeError(f"response.error: {code}") - if message: - return RuntimeError(f"response.error: {message}") - return RuntimeError("response.error") + text = ": ".join(part for part in ("response.error", code, message) if part) + return _provider_failure(text, _failure_code(raw_code), "response.error") -def _chunk_error(chunk: Any) -> RuntimeError | None: +def _chunk_error(chunk: Any) -> Exception | None: error = _field(chunk, "error") if error is None: return None - code = _error_code(_field(error, "code")) + raw_code = _field(error, "code") + code = _error_code(raw_code) message = _string(_field(error, "message")) if code and message: - return RuntimeError(f"stream error {code}: {message}") - if message: - return RuntimeError(f"stream error: {message}") - return RuntimeError(f"stream error {code}" if code else "stream error") + text = f"stream error {code}: {message}" + elif message: + text = f"stream error: {message}" + else: + text = f"stream error {code}" if code else "stream error" + return _provider_failure(text, _failure_code(raw_code), "stream error") def _responses_response( diff --git a/sdks/python-openrouter/tests/test_openrouter.py b/sdks/python-openrouter/tests/test_openrouter.py index 7c63e182..32689d2a 100644 --- a/sdks/python-openrouter/tests/test_openrouter.py +++ b/sdks/python-openrouter/tests/test_openrouter.py @@ -1146,12 +1146,12 @@ def handler(request: httpx.Request) -> httpx.Response: span = only_span(memory) a = attrs(span) assert span.status.status_code == StatusCode.ERROR - assert a["error.type"] == "RuntimeError" + assert a["error.type"] == "502" events_recorded = list(span.events) assert len(events_recorded) == 1 assert events_recorded[0].name == "exception" event_attrs = dict(events_recorded[0].attributes or {}) - assert event_attrs["exception.type"] == "RuntimeError" + assert event_attrs["exception.type"] == "502" assert event_attrs["exception.message"] == "stream error 502: upstream blew up" assert json.loads(str(a["gen_ai.output.messages"])) == [{"role": "assistant", "content": "Hel"}] @@ -1427,7 +1427,7 @@ def handler(request: httpx.Request) -> httpx.Response: span = only_span(memory) a = attrs(span) assert span.status.status_code == StatusCode.ERROR - assert a["error.type"] == "RuntimeError" + assert a["error.type"] == "server_error" assert list(cast(Any, a["gen_ai.response.finish_reasons"])) == ["failed"] @@ -1488,12 +1488,12 @@ def handler(request: httpx.Request) -> httpx.Response: span = only_span(memory) assert span.status.status_code == StatusCode.ERROR - assert attrs(span)["error.type"] == "RuntimeError" + assert attrs(span)["error.type"] == str(code) events_recorded = list(span.events) assert len(events_recorded) == 1 assert events_recorded[0].name == "exception" event_attrs = dict(events_recorded[0].attributes or {}) - assert event_attrs["exception.type"] == "RuntimeError" + assert event_attrs["exception.type"] == str(code) assert event_attrs["exception.message"] == f"response.failed: {code}: model blew up" @@ -1516,7 +1516,7 @@ def handler(request: httpx.Request) -> httpx.Response: events_recorded = list(span.events) assert len(events_recorded) == 1 event_attrs = dict(events_recorded[0].attributes or {}) - assert event_attrs["exception.type"] == "RuntimeError" + assert event_attrs["exception.type"] == "502" assert event_attrs["exception.message"] == "response.error: 502: provider disconnected" diff --git a/sdks/python/src/telemetry_dev/_semconv.py b/sdks/python/src/telemetry_dev/_semconv.py index 7d3041b3..cfb79b75 100644 --- a/sdks/python/src/telemetry_dev/_semconv.py +++ b/sdks/python/src/telemetry_dev/_semconv.py @@ -7,8 +7,8 @@ SpanType = Literal["span", "generation", "tool", "agent", "embedding"] LogLevel = Literal["debug", "info", "warn", "warning", "error"] -# Plain spans ALWAYS map to "function" — an operation-less span carrying a propagated -# gen_ai.conversation.id would otherwise be classified as "llm" by the ingest heuristic. +# Plain spans ALWAYS map to "function": without one, the ingest guesses "llm" from other gen_ai.* +# keys on the span, such as a gen_ai.agent.name the caller set. SPAN_TYPE_TO_OPERATION: dict[str, str] = { "span": "function", "generation": "chat", diff --git a/sdks/python/src/telemetry_dev/_spans.py b/sdks/python/src/telemetry_dev/_spans.py index 7bb2215c..334d7ca4 100644 --- a/sdks/python/src/telemetry_dev/_spans.py +++ b/sdks/python/src/telemetry_dev/_spans.py @@ -10,7 +10,7 @@ from contextvars import Token from dataclasses import dataclass from datetime import datetime -from typing import Any, Literal, TypedDict +from typing import Any, Literal, TypedDict, cast from weakref import WeakKeyDictionary, WeakValueDictionary from opentelemetry import context as otel_context @@ -151,10 +151,102 @@ def _to_ns(value: TimeInput) -> int | None: return int(value) +# Error classes that provider SDKs raise for any failed request, whose name says nothing about the +# cause: openai's and anthropic's APIStatusError (anthropic's for a stream's error event), and +# google-genai's APIError, ClientError, and ServerError. botocore's ClientError, and its +# EventStreamError for a Bedrock stream's exception, carry their code in `response`. Plain +# Exception is here for errors that carry a code, such as the failures that the integrations build +# from a response body with `code` and `type` attributes. The TypeScript SDK keeps the same rule in +# `errorTypeOf` (packages/otel/src/errors.ts); change both together. +_GENERIC_ERROR_NAMES = frozenset( + {"Exception", "APIError", "APIStatusError", "ClientError", "ServerError", "EventStreamError"} +) + + +def _text(value: object) -> str | None: + return value if isinstance(value, str) and value else None + + +def _error_code(value: object) -> str | None: + # Some providers send numeric codes, such as Z.ai's "1302" for a rate limit, and openai's SDK + # turns OpenRouter's numeric code into a string. A number names no cause the way a class does. + code = _text(value) + return None if code is None or code.isdigit() else code + + +def _http_status(value: object) -> int | None: + return ( + value + if isinstance(value, int) and not isinstance(value, bool) and 400 <= value <= 599 + else None + ) + + +def _nested(value: object, *keys: str) -> object: + for key in keys: + if not isinstance(value, Mapping): + return None + value = cast("Mapping[str, object]", value).get(key) + return value + + +def _attribute(error: BaseException, name: str) -> object: + # A property of a third-party exception can raise anything; recording the error must not. + try: + return getattr(error, name, None) + except Exception: + return None + + +def _error_type(error: BaseException) -> str: + """The error.type of a failure: the provider's error code when the error carries one. + + - An error with an HTTP status and a code that is not only digits (openai's + `insufficient_quota`, `context_length_exceeded`) gives the code. + - A generic provider error class gives its code, else its HTTP status (`429`), else the + provider's error type (anthropic's `overloaded_error` from a stream). + - Any other error gives its class name. + """ + name = type(error).__name__ + body = _attribute(error, "body") + code = _error_code(_attribute(error, "code")) or _error_code(_nested(body, "error", "code")) + status = next( + ( + status + for status in ( + _http_status(_attribute(error, attribute)) + for attribute in ("status_code", "code", "status") + ) + if status is not None + ), + None, + ) + if code is not None and status is not None: + return code + if name not in _GENERIC_ERROR_NAMES: + return name + if code is not None: + return code + if status is not None: + return str(status) + aws_code = _text(_nested(_attribute(error, "response"), "Error", "Code")) + if aws_code is not None: + # botocore names a Bedrock stream's exception by its event, such as + # modelStreamErrorException; the TypeScript SDK records its class, + # ModelStreamErrorException. + aws_code = aws_code[0].upper() + aws_code[1:] + return ( + _text(_attribute(error, "type")) + or _text(_nested(body, "error", "type")) + or aws_code + or name + ) + + def _record_error_on_span( client: Client, span: Span, state: _SpanState, error: BaseException ) -> None: - error_type = type(error).__name__ + error_type = _error_type(error) span.set_attribute(ATTR_ERROR_TYPE, error_type) span.set_status(Status(StatusCode.ERROR)) attributes: dict[str, otel_types.AttributeValue] = {