From 7d7407423bc13695a05279b3d97eb3a765e1add5 Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Tue, 10 Nov 2026 18:55:49 +0000 Subject: [PATCH 1/2] feat(sdk): record the provider's error code in error.type A failed model call recorded its error class, which often names no cause: openai's RateLimitError is also an exhausted quota, Anthropic's APIError from a stream's error event, @google/genai's ApiError, and the AI SDK's APICallError and RetryError say nothing about why the call failed. Both SDKs now record, in error.type and exception.type: - the error's code when it has an HTTP status and a code that is not only digits (insufficient_quota, context_length_exceeded); - for a generic provider error class, its code, else its HTTP status (429), else the provider's error type (overloaded_error); - for the AI SDK's RetryError, the type of the last error it retried; - otherwise the class name, as before. The core SDK, the AI SDK and TanStack AI integrations, and the Python SDK share the rule. The comments and the conformance doc no longer say a propagated gen_ai.conversation.id makes the ingest guess "llm"; the ingest stops doing that. --- docs/sdk-conformance.md | 12 ++- packages/ai/src/integration_v6.ts | 3 +- packages/ai/src/integration_v7.ts | 3 +- packages/ai/tests/integration_v7.test.ts | 55 ++++++++++++ packages/anthropic/tests/anthropic.test.ts | 20 +++++ .../google-genai/tests/google-genai.test.ts | 3 +- packages/openai/tests/openai.test.ts | 38 +++++++- packages/otel/src/errors.ts | 81 +++++++++++++++++ packages/otel/src/index.ts | 2 + packages/sdk/src/attrs.ts | 4 +- packages/sdk/src/span.ts | 11 +-- packages/tanstack-ai/src/errors.ts | 5 +- sdks/python-anthropic/tests/test_anthropic.py | 16 ++++ .../tests/test_google_genai.py | 8 +- sdks/python-openai/tests/test_openai.py | 25 ++++++ sdks/python/src/telemetry_dev/_semconv.py | 4 +- sdks/python/src/telemetry_dev/_spans.py | 88 ++++++++++++++++++- 17 files changed, 351 insertions(+), 27 deletions(-) create mode 100644 packages/otel/src/errors.ts diff --git a/docs/sdk-conformance.md b/docs/sdk-conformance.md index 7d525960..4786fcec 100644 --- a/docs/sdk-conformance.md +++ b/docs/sdk-conformance.md @@ -19,7 +19,7 @@ repository, not this SDK workspace. Paths below refer to that product repository | SDK field (TS / Py) | Wire attribute | Notes | | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `type: "span"` | `gen_ai.operation.name = "function"` | Always explicit: a propagated `gen_ai.conversation.id` would otherwise trip the ingest's `hasGenAi` → `"llm"` heuristic | +| `type: "span"` | `gen_ai.operation.name = "function"` | Always explicit, so the ingest never guesses it from other `gen_ai.*` keys, such as a caller-set `gen_ai.agent.name` | | `type: "generation"` | `gen_ai.operation.name = "chat"` | | | `type: "tool"` | `gen_ai.operation.name = "execute_tool"` | | | `type: "agent"` | `gen_ai.operation.name = "invoke_agent"` | | @@ -54,6 +54,16 @@ repository, not this SDK workspace. Paths below refer to that product repository | the item run | `gen_ai.conversation.id = "::"` | one session and one trace for each run; replaces an ambient or process session, and the run drops the active span | | the root span of an item run | name `experiment `, `gen_ai.operation.name = "function"` | `input` is the item input; `output` is the task result; gated by capture flags | +`error.type` and `exception.type` are the error's class name, with two exceptions that both SDKs +apply the same way. An error with an HTTP status (400–599) and a code that is not only digits records the code, +such as openai's `insufficient_quota` on a `RateLimitError`. A generic provider error class +(TypeScript `Error`, `APIError`, `ApiError`, `AI_APICallError`, `AI_StreamProviderError`; Python +`Exception`, `APIError`, `APIStatusError`, `ClientError`, `ServerError`) records its code, else its +HTTP status as a string (`429`), else the provider's error type (`overloaded_error` from an +Anthropic stream; Python also reads botocore's `response.Error.Code`). TypeScript records the AI +SDK's `AI_RetryError` as its last error. A thrown value that is not an Error keeps the fallback it +had before: `Error` in the core SDKs, `tool_error` or `error` in the AI SDK integration. + `SpanHandle.serializeContent(value, key)` / `SpanHandle.serialize_content(value, key)` returns what `input` / `output` would emit for `value` under `key`: mask → JSON stringify → truncate. It returns `undefined` / `None` when the mask throws, the value cannot be serialized, or there is no diff --git a/packages/ai/src/integration_v6.ts b/packages/ai/src/integration_v6.ts index c68208cc..4f7f31c2 100644 --- a/packages/ai/src/integration_v6.ts +++ b/packages/ai/src/integration_v6.ts @@ -10,6 +10,7 @@ import { } from "@opentelemetry/api"; import { createGenerationEmitter, + errorTypeOf, type GenerationEmitter, type GenerationEmitterOverrides, jsonAttr, @@ -443,7 +444,7 @@ function createV6Hooks(config: ResolvedConfig, emitter: GenerationEmitter) { e.error instanceof Error ? e.error.message : unknownErrorMessage(e.error), ) ?? "Tool execution failed"; - const errorType = e.error instanceof Error ? e.error.name : "tool_error"; + const errorType = errorTypeOf(e.error, "tool_error"); span.setStatus({ code: SpanStatusCode.ERROR }); span.setAttribute("error.type", errorType); span.addEvent("exception", { diff --git a/packages/ai/src/integration_v7.ts b/packages/ai/src/integration_v7.ts index c7f0f1fc..2798ed7d 100644 --- a/packages/ai/src/integration_v7.ts +++ b/packages/ai/src/integration_v7.ts @@ -10,6 +10,7 @@ import { } from "@opentelemetry/api"; import { createGenerationEmitter, + errorTypeOf, type GenerationEmitter, type GenerationEmitterOverrides, jsonAttr, @@ -557,7 +558,7 @@ function createV7Hooks( fallback = "Tool execution failed", ): ProviderToolFailure => { try { - if (value instanceof Error && value.name.length > 0) errorType = value.name; + errorType = errorTypeOf(value, errorType); } catch { return { errorType, message: fallback }; } diff --git a/packages/ai/tests/integration_v7.test.ts b/packages/ai/tests/integration_v7.test.ts index cc12bf12..ea2c56ed 100644 --- a/packages/ai/tests/integration_v7.test.ts +++ b/packages/ai/tests/integration_v7.test.ts @@ -17,6 +17,7 @@ import { TraceIdRatioBasedSampler, } from "@opentelemetry/sdk-trace-base"; import { AlsContextManager, sessionSpanContext } from "@telemetry-dev/otel"; +import { APICallError, RetryError, StreamProviderError } from "ai"; import { AsyncLocalStorage } from "node:async_hooks"; import { expect, test, vi } from "vitest"; @@ -2751,6 +2752,60 @@ test("onError marks root and open step failed and a later onEnd for the same cal expect(step.attributes["gen_ai.request.model"]).toBe("gpt-4o"); }); +const apiCallError = (statusCode: number, data: unknown) => + new APICallError({ + message: "failed", + url: "https://api.example.com", + requestBodyValues: {}, + statusCode, + data, + }); + +test.each([ + { + name: "a retried quota error gives the provider's code", + error: new RetryError({ + message: "Failed after 3 attempts", + reason: "maxRetriesExceeded", + errors: [ + apiCallError(429, { error: { type: "insufficient_quota", code: "insufficient_quota" } }), + ], + }), + expected: "insufficient_quota", + }, + { + name: "an API call error without a code gives its HTTP status", + error: apiCallError(529, { type: "error", error: { type: "overloaded_error" } }), + expected: "529", + }, + { + name: "a stream error without a status gives the provider's type", + error: new StreamProviderError({ + message: "Overloaded", + type: "overloaded_error", + isRetryable: true, + }), + expected: "overloaded_error", + }, +])("onError: $name", async ({ error, expected }) => { + const { spanBatches, overrides } = makeCapture(); + const integ = telemetryDev(baseOptions, overrides); + const callId = "call-provider-error"; + + integ.onStart?.({ + callId, + operationId: "ai.generateText", + provider: "openai", + modelId: "gpt-4o", + }); + integ.onStepStart?.({ callId, provider: "openai", modelId: "gpt-4o", stepNumber: 0 }); + await integ.onError?.({ callId, error }); + + const spans = spanBatches[0]!; + expect(spans.find((s) => s.kind === SpanKind.CLIENT)!.attributes["error.type"]).toBe(expected); + expect(spans.find((s) => s.kind === SpanKind.INTERNAL)!.attributes["error.type"]).toBe(expected); +}); + test("onAbort ends the open step, keeps root UNSET, and emits generation.summary aborted", async () => { const { spanBatches, overrides } = makeCapture(); const integ = telemetryDev(baseOptions, overrides); diff --git a/packages/anthropic/tests/anthropic.test.ts b/packages/anthropic/tests/anthropic.test.ts index cc98fb60..20d30379 100644 --- a/packages/anthropic/tests/anthropic.test.ts +++ b/packages/anthropic/tests/anthropic.test.ts @@ -690,6 +690,26 @@ test("messages.create streaming preserves events and records aggregated text usa ).toBe(true); }); +test("an error event mid-stream records the provider's error type", async () => { + const spans = setupSpans(); + const overloaded = { + type: "error", + error: { type: "overloaded_error", message: "Overloaded" }, + }; + const fake = createFakeFetch(namedSseResponse([streamEvents()[0]!, overloaded])); + + const stream = await clientWith(fake.fetch).messages.create({ + model: "claude-sonnet-4-6", + max_tokens: 64, + messages: [{ role: "user", content: "Say hello" }], + stream: true, + }); + + // The SDK throws its generic APIError, whose class names no cause. + await expect(collectStream(stream)).rejects.toMatchObject({ type: "overloaded_error" }); + expect((await exportedSpan(spans)).attributes["error.type"]).toBe("overloaded_error"); +}); + test("stream capture retains no output when output capture is disabled", async () => { const spans = setupSpans(undefined, false); const capture = vi.spyOn(telemetrySdk, "boundedCaptureDetails"); diff --git a/packages/google-genai/tests/google-genai.test.ts b/packages/google-genai/tests/google-genai.test.ts index fc678dcd..7db8b66d 100644 --- a/packages/google-genai/tests/google-genai.test.ts +++ b/packages/google-genai/tests/google-genai.test.ts @@ -2948,7 +2948,8 @@ test("API error 400 rethrows ApiError and records error span", async () => { ).rejects.toMatchObject({ name: "ApiError", status: 400 }); const span = await exportedSpan(spans); expect(span.status.code).toBe(SPAN_STATUS_ERROR); - expect(span.attributes["error.type"]).toBe("ApiError"); + // ApiError names no cause, so the HTTP status stands in for it. + expect(span.attributes["error.type"]).toBe("400"); }); test("non-Error unary failures end spans and preserve the thrown values", async () => { diff --git a/packages/openai/tests/openai.test.ts b/packages/openai/tests/openai.test.ts index 06d244ae..1550d393 100644 --- a/packages/openai/tests/openai.test.ts +++ b/packages/openai/tests/openai.test.ts @@ -477,6 +477,37 @@ test("wrapped chat completion APIPromise keeps parse helper and promise identity ]); }); +test.each([ + // A RateLimitError whose code says the account is out of quota, not rate limited. + { status: 429, code: "insufficient_quota", expected: "insufficient_quota" }, + { status: 400, code: "context_length_exceeded", expected: "context_length_exceeded" }, + // Without a code, the class name is specific enough (Python parity: type(e).__name__). + { status: 500, code: null, expected: "InternalServerError" }, + // A numeric code, such as Z.ai's 1302 rate limit, names less than the class. + { status: 429, code: "1302", expected: "RateLimitError" }, +])( + "a $status error with code $code records error.type $expected", + async ({ status, code, expected }) => { + const spans = setupSpans(); + const body = { error: { message: "failed", type: "error", code } }; + const fake = createFakeFetch( + new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }), + ); + + await expect( + clientWith(fake.fetch).chat.completions.create({ + model: "gpt-x", + messages: [{ role: "user", content: "Fail" }], + }), + ).rejects.toMatchObject({ status }); + + expect((await exportedSpan(spans)).attributes["error.type"]).toBe(expected); + }, +); + test("chat completions record one error span when the OpenAI API returns 4xx", async () => { const spans = setupSpans(); const fake = createFakeFetch(jsonErrorResponse(400, "bad model")); @@ -492,11 +523,10 @@ test("chat completions record one error span when the OpenAI API returns 4xx", a expect(span.status.code).toBe(SPAN_STATUS_ERROR); expect(span.attributes["gen_ai.operation.name"]).toBe("chat"); expect(span.attributes["gen_ai.request.model"]).toBe("gpt-bad"); - // The SDK falls back to the constructor name when err.name is the generic "Error" - // (openai's APIError subclasses never override name) — Python parity: type(e).__name__. - expect(span.attributes["error.type"]).toBe("BadRequestError"); + // The error code from the response body names the failure. + expect(span.attributes["error.type"]).toBe("bad_request"); const exception = span.events.find((event) => event.name === "exception"); - expect(exception?.attributes?.["exception.type"]).toBe("BadRequestError"); + expect(exception?.attributes?.["exception.type"]).toBe("bad_request"); expect(exception?.attributes?.["exception.message"]).toContain("bad model"); expect(exception?.attributes?.["log.severity_number"]).toBe(17); }); diff --git a/packages/otel/src/errors.ts b/packages/otel/src/errors.ts new file mode 100644 index 00000000..d059ed32 --- /dev/null +++ b/packages/otel/src/errors.ts @@ -0,0 +1,81 @@ +// Error classes that provider SDKs throw for any failed request, whose name says nothing about the +// cause: openai's and Anthropic's APIError (a stream's error event), @google/genai's ApiError, and +// the AI SDK's APICallError and StreamProviderError. Plain Error is here for errors that carry a +// code. The Python SDK keeps the same rule in `_error_type`; change both together. +const GENERIC_ERROR_NAMES = new Set([ + "Error", + "APIError", + "ApiError", + "AI_APICallError", + "AI_StreamProviderError", +]); + +function record(value: unknown): Record | undefined { + return typeof value === "object" && value !== null + ? (value as Record) + : undefined; +} + +function text(value: unknown): string | undefined { + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +// Some providers send numeric codes, such as Z.ai's "1302" for a rate limit, and openai's Python SDK +// turns OpenRouter's numeric code into a string. A number names no cause the way a class name does. +function errorCode(value: unknown): string | undefined { + const code = text(value); + + return code === undefined || /^\d+$/.test(code) ? undefined : code; +} + +function httpStatus(value: unknown): number | undefined { + return typeof value === "number" && Number.isInteger(value) && value >= 400 && value <= 599 + ? value + : undefined; +} + +function className(error: Error): string { + // Many SDK error classes (e.g. openai's BadRequestError) never override the inherited "Error" + // name, so the constructor name is the specific one (Python parity: type(error).__name__). + return error.name !== "Error" && error.name.length > 0 + ? error.name + : (error.constructor?.name ?? "Error") || "Error"; +} + +/** + * The `error.type` of a failure: the provider's error code when the error carries one, so that a + * rate limit, an exhausted quota, an overflowing context, and an overloaded provider read apart. + * + * - An error with an HTTP status and a code that is not only digits (openai's APIError: + * `insufficient_quota`, `context_length_exceeded`) gives the code. + * - A generic provider error class (see GENERIC_ERROR_NAMES) gives its code, else its HTTP status + * (`429`), else the provider's error type (Anthropic's `overloaded_error` from a stream). + * - The AI SDK's RetryError gives the type of the last error it retried. + * - Any other error gives its class name; a value that is not an Error gives `fallback`. + */ +export function errorTypeOf(error: unknown, fallback = "Error"): string { + if (!(error instanceof Error)) return fallback; + + try { + const fields = error as Error & Record; + + if (fields.name === "AI_RetryError" && fields.lastError instanceof Error) { + return errorTypeOf(fields.lastError, fallback); + } + + const name = className(error); + const data = record(record(fields.data)?.error); + const code = errorCode(fields.code) ?? errorCode(data?.code); + const status = httpStatus(fields.status) ?? httpStatus(fields.statusCode); + + if (code !== undefined && status !== undefined) return code; + if (!GENERIC_ERROR_NAMES.has(name)) return name; + if (code !== undefined) return code; + if (status !== undefined) return String(status); + + return text(fields.type) ?? text(data?.type) ?? name; + } catch { + // A getter on a third-party error can throw; recording the error must not. + return fallback; + } +} diff --git a/packages/otel/src/index.ts b/packages/otel/src/index.ts index 8c87c910..26b8eec3 100644 --- a/packages/otel/src/index.ts +++ b/packages/otel/src/index.ts @@ -33,6 +33,8 @@ export { export { diag, reportError, setLogLevel } from "./debug.ts"; +export { errorTypeOf } from "./errors.ts"; + export { createGenerationEmitter, type GenerationEmitter, diff --git a/packages/sdk/src/attrs.ts b/packages/sdk/src/attrs.ts index 6dbf34f5..8c4af184 100644 --- a/packages/sdk/src/attrs.ts +++ b/packages/sdk/src/attrs.ts @@ -7,8 +7,8 @@ import { type CaptureConfig, prepareCaptureValue, truncate } from "./capture.ts" export type SpanType = "span" | "generation" | "tool" | "agent" | "embedding"; // Operation names are taken verbatim by the ingest's deriveOperation. Plain spans always get an -// explicit "function": without one, the propagated gen_ai.conversation.id would trip the ingest's -// hasGenAi heuristic and misclassify them as "llm". +// explicit "function": without one, the ingest guesses "llm" from other gen_ai.* keys on the span, +// such as a gen_ai.agent.name the caller set. export const SPAN_TYPE_OPERATIONS = { span: "function", generation: "chat", diff --git a/packages/sdk/src/span.ts b/packages/sdk/src/span.ts index 877f74c0..975c0fc3 100644 --- a/packages/sdk/src/span.ts +++ b/packages/sdk/src/span.ts @@ -21,6 +21,7 @@ import { import { activeContext, DURATION_BUCKETS, + errorTypeOf, omitUndefined, OUTPUT_CHUNK_HISTOGRAM, type OutputChunkHistogram, @@ -159,15 +160,7 @@ function resolveParent(parent: ParentRef | undefined): Context { function applyError(span: Span, error: unknown, capture: FieldCaptureConfig): void { const err = error instanceof Error ? error : undefined; - - // Prefer the specific subclass name (Python parity: type(error).__name__) — many SDK - // error classes (e.g. openai's BadRequestError) never override the inherited "Error" name. - const errorType = - err === undefined - ? "Error" - : err.name !== "Error" && err.name.length > 0 - ? err.name - : (err.constructor?.name ?? "Error") || "Error"; + const errorType = errorTypeOf(error); span.setStatus({ code: SpanStatusCode.ERROR }); span.setAttribute("error.type", errorType); diff --git a/packages/tanstack-ai/src/errors.ts b/packages/tanstack-ai/src/errors.ts index b114fc77..4ebd2916 100644 --- a/packages/tanstack-ai/src/errors.ts +++ b/packages/tanstack-ai/src/errors.ts @@ -1,3 +1,5 @@ +import { errorTypeOf } from "@telemetry-dev/otel"; + export function errorDetails(error: unknown, fallbackType = "Error") { const details = { type: fallbackType, message: "Unknown error" }; @@ -7,7 +9,8 @@ export function errorDetails(error: unknown, fallbackType = "Error") { if (error !== null && typeof error === "object") { const { name, message } = error as { name?: unknown; message?: unknown }; - if (typeof name === "string" && name.length > 0) details.type = name; + if (error instanceof Error) details.type = errorTypeOf(error, fallbackType); + else if (typeof name === "string" && name.length > 0) details.type = name; if (typeof message === "string") details.message = message; } diff --git a/sdks/python-anthropic/tests/test_anthropic.py b/sdks/python-anthropic/tests/test_anthropic.py index 79856847..8405ba9d 100644 --- a/sdks/python-anthropic/tests/test_anthropic.py +++ b/sdks/python-anthropic/tests/test_anthropic.py @@ -1227,6 +1227,22 @@ def mask(value: Any, _context: telemetry_dev.MaskContext) -> Any: assert a["telemetry.dev.capture.truncated"] is True +def test_error_event_mid_stream_records_the_provider_error_type(memory: SimpleNamespace) -> None: + overloaded = {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}} + client = wrapped_sync_client( + lambda _request: named_sse_response([stream_events()[0], overloaded]) + ) + stream = client.messages.create( + model="claude-sonnet-4-6", max_tokens=64, messages=MESSAGES, stream=True + ) + + # The SDK raises its generic APIStatusError, with the 200 status of the stream. + with pytest.raises(anthropic.APIStatusError): + list(stream) + + assert attrs(only_span(memory))["error.type"] == "overloaded_error" + + def test_mid_stream_read_error_records_error_span(memory: SimpleNamespace) -> None: def handler(request: httpx.Request) -> httpx.Response: return failing_sync_sse_response(stream_events()[:3], "sync stream broke") diff --git a/sdks/python-google-genai/tests/test_google_genai.py b/sdks/python-google-genai/tests/test_google_genai.py index d6929f55..335620bc 100644 --- a/sdks/python-google-genai/tests/test_google_genai.py +++ b/sdks/python-google-genai/tests/test_google_genai.py @@ -2341,9 +2341,9 @@ async def run() -> None: @pytest.mark.parametrize("async_mode", [False, True], ids=["sync", "async"]) def test_api_error_records_error_and_reraises(memory: SimpleNamespace, async_mode: bool) -> None: - error = errors.APIError( - 400, - {"error": {"message": "bad", "status": "INVALID_ARGUMENT", "code": 400}}, + error = errors.ClientError( + 429, + {"error": {"message": "slow down", "status": "RESOURCE_EXHAUSTED", "code": 429}}, ) client, _transport = client_with_transport([error]) wrapped = wrap_google_genai(client) @@ -2361,6 +2361,8 @@ def test_api_error_records_error_and_reraises(memory: SimpleNamespace, async_mod span = only_span(memory) assert span.status.status_code == StatusCode.ERROR + # ClientError names no cause, so the HTTP status stands in for it. + assert attrs(span)["error.type"] == "429" @pytest.mark.parametrize("async_mode", [False, True], ids=["sync", "async"]) diff --git a/sdks/python-openai/tests/test_openai.py b/sdks/python-openai/tests/test_openai.py index fc1ea943..93fc77bc 100644 --- a/sdks/python-openai/tests/test_openai.py +++ b/sdks/python-openai/tests/test_openai.py @@ -552,6 +552,31 @@ def handler(_request: httpx.Request) -> httpx.Response: assert attrs(span)["gen_ai.usage.image.output_tokens"] == 9 +@pytest.mark.parametrize( + ("status", "code", "expected"), + [ + # A RateLimitError whose code says the account is out of quota, not rate limited. + (429, "insufficient_quota", "insufficient_quota"), + (400, "context_length_exceeded", "context_length_exceeded"), + # Without a code, the class name is specific enough. + (500, None, "InternalServerError"), + # A numeric code, such as Z.ai's 1302 rate limit or OpenRouter's 429, names less. + (429, "1302", "RateLimitError"), + (429, 429, "RateLimitError"), + ], +) +def test_api_error_records_the_provider_error_code( + memory: SimpleNamespace, status: int, code: str | int | None, expected: str +) -> None: + body = {"error": {"message": "failed", "type": "error", "code": code}} + client = wrap_openai(sync_client(lambda _request: httpx.Response(status, json=body))) + + with pytest.raises(openai.APIStatusError): + client.chat.completions.create(model="gpt-x", messages=cast(Any, CHAT_MESSAGES)) + + assert attrs(only_span(memory))["error.type"] == expected + + @pytest.mark.parametrize("completed", [True, False]) def test_image_stream_is_flagged_only_without_a_terminal_event( memory: SimpleNamespace, completed: bool diff --git a/sdks/python/src/telemetry_dev/_semconv.py b/sdks/python/src/telemetry_dev/_semconv.py index 7d3041b3..cfb79b75 100644 --- a/sdks/python/src/telemetry_dev/_semconv.py +++ b/sdks/python/src/telemetry_dev/_semconv.py @@ -7,8 +7,8 @@ SpanType = Literal["span", "generation", "tool", "agent", "embedding"] LogLevel = Literal["debug", "info", "warn", "warning", "error"] -# Plain spans ALWAYS map to "function" — an operation-less span carrying a propagated -# gen_ai.conversation.id would otherwise be classified as "llm" by the ingest heuristic. +# Plain spans ALWAYS map to "function": without one, the ingest guesses "llm" from other gen_ai.* +# keys on the span, such as a gen_ai.agent.name the caller set. SPAN_TYPE_TO_OPERATION: dict[str, str] = { "span": "function", "generation": "chat", diff --git a/sdks/python/src/telemetry_dev/_spans.py b/sdks/python/src/telemetry_dev/_spans.py index 7bb2215c..852efab4 100644 --- a/sdks/python/src/telemetry_dev/_spans.py +++ b/sdks/python/src/telemetry_dev/_spans.py @@ -10,7 +10,7 @@ from contextvars import Token from dataclasses import dataclass from datetime import datetime -from typing import Any, Literal, TypedDict +from typing import Any, Literal, TypedDict, cast from weakref import WeakKeyDictionary, WeakValueDictionary from opentelemetry import context as otel_context @@ -151,10 +151,94 @@ def _to_ns(value: TimeInput) -> int | None: return int(value) +# Error classes that provider SDKs raise for any failed request, whose name says nothing about the +# cause: openai's and anthropic's APIStatusError (anthropic's for a stream's error event), and +# google-genai's APIError, ClientError, and ServerError. botocore's ClientError carries its code +# in `response`. Plain Exception is here for errors that carry a code. The TypeScript SDK keeps +# the same rule in `errorTypeOf` (packages/otel/src/errors.ts); change both together. +_GENERIC_ERROR_NAMES = frozenset( + {"Exception", "APIError", "APIStatusError", "ClientError", "ServerError"} +) + + +def _text(value: object) -> str | None: + return value if isinstance(value, str) and value else None + + +def _error_code(value: object) -> str | None: + # Some providers send numeric codes, such as Z.ai's "1302" for a rate limit, and openai's SDK + # turns OpenRouter's numeric code into a string. A number names no cause the way a class does. + code = _text(value) + return None if code is None or code.isdigit() else code + + +def _http_status(value: object) -> int | None: + return ( + value + if isinstance(value, int) and not isinstance(value, bool) and 400 <= value <= 599 + else None + ) + + +def _nested(value: object, *keys: str) -> object: + for key in keys: + if not isinstance(value, Mapping): + return None + value = cast("Mapping[str, object]", value).get(key) + return value + + +def _attribute(error: BaseException, name: str) -> object: + # A property of a third-party exception can raise anything; recording the error must not. + try: + return getattr(error, name, None) + except Exception: + return None + + +def _error_type(error: BaseException) -> str: + """The error.type of a failure: the provider's error code when the error carries one. + + - An error with an HTTP status and a code that is not only digits (openai's + `insufficient_quota`, `context_length_exceeded`) gives the code. + - A generic provider error class gives its code, else its HTTP status (`429`), else the + provider's error type (anthropic's `overloaded_error` from a stream). + - Any other error gives its class name. + """ + name = type(error).__name__ + body = _attribute(error, "body") + code = _error_code(_attribute(error, "code")) or _error_code(_nested(body, "error", "code")) + status = next( + ( + status + for status in ( + _http_status(_attribute(error, attribute)) + for attribute in ("status_code", "code", "status") + ) + if status is not None + ), + None, + ) + if code is not None and status is not None: + return code + if name not in _GENERIC_ERROR_NAMES: + return name + if code is not None: + return code + if status is not None: + return str(status) + return ( + _text(_attribute(error, "type")) + or _text(_nested(body, "error", "type")) + or _text(_nested(_attribute(error, "response"), "Error", "Code")) + or name + ) + + def _record_error_on_span( client: Client, span: Span, state: _SpanState, error: BaseException ) -> None: - error_type = type(error).__name__ + error_type = _error_type(error) span.set_attribute(ATTR_ERROR_TYPE, error_type) span.set_status(Status(StatusCode.ERROR)) attributes: dict[str, otel_types.AttributeValue] = { From 465d983ff755fafffe440d301e1fd8a98639a25b Mon Sep 17 00:00:00 2001 From: Aikins Laryea Date: Tue, 10 Nov 2026 21:08:29 +0000 Subject: [PATCH 2/2] feat(sdk): record the code of failures reported in a response body A failure that a provider reports in a response body or a stream event, not as an HTTP error, recorded error.type Error or RuntimeError. This covered openai and OpenRouter response.failed and stream error events, OpenRouter chat stream errors, openai Realtime errors and failed image streams, LiteLLM Responses failures, and failed Google video operations. The integrations now build these failures with the provider's code and a type that names the failure, so error.type is the code (such as rate_limit_exceeded), an OpenRouter numeric code as an HTTP status (502), the google.rpc.Code name of a video operation (RESOURCE_EXHAUSTED), or else the failure's kind (response.failed). A TypeScript image stream that ends with a failed event now marks its span failed, as in Python. The AI SDK media and batch spans now set error.type with the same rule. In Python, a Bedrock stream's EventStreamError records the exception name, ModelStreamErrorException, as the TypeScript SDK does. --- docs/sdk-conformance.md | 19 +++-- packages/ai/src/batch.ts | 2 + packages/ai/src/media.ts | 1 + packages/ai/src/shared.ts | 35 +++++---- packages/ai/tests/wrappers.test.ts | 1 + packages/google-genai/src/index.ts | 37 ++++++++- .../google-genai/tests/google-genai.test.ts | 22 ++++++ packages/openai/src/index.ts | 55 ++++++++++++-- packages/openai/tests/openai.test.ts | 75 +++++++++++++++++-- packages/openrouter/src/index.ts | 36 +++++++-- packages/openrouter/tests/openrouter.test.ts | 5 +- packages/otel/src/errors.ts | 4 +- sdks/python-bedrock/tests/test_bedrock.py | 2 +- .../telemetry_dev_google_genai/__init__.py | 42 ++++++++++- .../tests/test_google_genai.py | 2 + .../src/telemetry_dev_litellm/__init__.py | 15 +++- sdks/python-litellm/tests/test_litellm.py | 4 +- .../src/telemetry_dev_openai/__init__.py | 54 ++++++------- sdks/python-openai/tests/test_openai.py | 6 +- .../src/telemetry_dev_openrouter/__init__.py | 62 ++++++++------- .../tests/test_openrouter.py | 12 +-- sdks/python/src/telemetry_dev/_spans.py | 18 +++-- 22 files changed, 392 insertions(+), 117 deletions(-) diff --git a/docs/sdk-conformance.md b/docs/sdk-conformance.md index 4786fcec..a1077c3d 100644 --- a/docs/sdk-conformance.md +++ b/docs/sdk-conformance.md @@ -58,11 +58,20 @@ repository, not this SDK workspace. Paths below refer to that product repository apply the same way. An error with an HTTP status (400–599) and a code that is not only digits records the code, such as openai's `insufficient_quota` on a `RateLimitError`. A generic provider error class (TypeScript `Error`, `APIError`, `ApiError`, `AI_APICallError`, `AI_StreamProviderError`; Python -`Exception`, `APIError`, `APIStatusError`, `ClientError`, `ServerError`) records its code, else its -HTTP status as a string (`429`), else the provider's error type (`overloaded_error` from an -Anthropic stream; Python also reads botocore's `response.Error.Code`). TypeScript records the AI -SDK's `AI_RetryError` as its last error. A thrown value that is not an Error keeps the fallback it -had before: `Error` in the core SDKs, `tool_error` or `error` in the AI SDK integration. +`Exception`, `APIError`, `APIStatusError`, `ClientError`, `ServerError`, `EventStreamError`) records +its code, else its HTTP status as a string (`429`), else the provider's error type +(`overloaded_error` from an Anthropic stream; Python also reads botocore's `response.Error.Code`, +with its first letter in upper case, so a Bedrock stream's `modelStreamErrorException` records +`ModelStreamErrorException` as in TypeScript). TypeScript records the AI SDK's `AI_RetryError` as +its last error. A thrown value that is not an Error keeps the fallback it had before: `Error` in the +core SDKs, `tool_error` or `error` in the AI SDK integration. + +A failure that a provider reports in a response body or a stream event, not as an HTTP error, is +recorded with the same rule: the integrations build it as a plain `Error` (TypeScript) or +`Exception` (Python) with the provider's `code` and a `type` that names the failure. Thus a +`response.failed` with `rate_limit_exceeded` records `rate_limit_exceeded`, one without a code +records `response.failed`, an OpenRouter numeric code records it as an HTTP status (`502`), and a +failed Google video operation records the `google.rpc.Code` name of its status (`RESOURCE_EXHAUSTED`). `SpanHandle.serializeContent(value, key)` / `SpanHandle.serialize_content(value, key)` returns what `input` / `output` would emit for `value` under `key`: mask → JSON stringify → truncate. It diff --git a/packages/ai/src/batch.ts b/packages/ai/src/batch.ts index 6b955a34..fa0ad8f0 100644 --- a/packages/ai/src/batch.ts +++ b/packages/ai/src/batch.ts @@ -100,6 +100,7 @@ function wrapBatch( if (failed) { attempt(() => { const failure = failureRecord(config, error, "Batch request failed"); + attempt(() => span.setAttribute("error.type", failure.errorType)); attempt(() => span.setStatus({ code: SpanStatusCode.ERROR, message: failure.message })); attempt(() => span.recordException(failure.exception)); }); @@ -237,6 +238,7 @@ export const wrapGetBatchResults = ( if (failed) { attempt(() => { const failure = failureRecord(config, error, "Batch request failed"); + attempt(() => span.setAttribute("error.type", failure.errorType)); attempt(() => span.setStatus({ code: SpanStatusCode.ERROR, message: failure.message })); attempt(() => span.recordException(failure.exception)); }); diff --git a/packages/ai/src/media.ts b/packages/ai/src/media.ts index 07542b50..c0c6c98a 100644 --- a/packages/ai/src/media.ts +++ b/packages/ai/src/media.ts @@ -158,6 +158,7 @@ function wrapMedia( if (failed) { attempt(() => { const failure = failureRecord(config, error, "Media generation failed"); + attempt(() => span.setAttribute("error.type", failure.errorType)); attempt(() => span.setStatus({ code: SpanStatusCode.ERROR, message: failure.message })); attempt(() => span.recordException(failure.exception)); }); diff --git a/packages/ai/src/shared.ts b/packages/ai/src/shared.ts index 88f4b714..3d5d6fb8 100644 --- a/packages/ai/src/shared.ts +++ b/packages/ai/src/shared.ts @@ -1,5 +1,5 @@ import type { Exception } from "@opentelemetry/api"; -import { jsonAttr } from "@telemetry-dev/otel"; +import { errorTypeOf, jsonAttr } from "@telemetry-dev/otel"; import type { ResolvedConfig, TelemetryDevOptions } from "./config.ts"; @@ -59,15 +59,18 @@ export function contentAttr( } /** - * What a failed wrapper call records as its span status message and exception. Without a mask - * that is the error itself; with one it is the masked message, or `fallback` when the mask omits - * it or throws, and no stack trace, which repeats the message. + * What a failed wrapper call records as its span's `error.type`, status message, and exception. + * `errorType` follows errorTypeOf and is also the exception's type. Without a mask the message is + * the error itself; with one it is the masked message, or `fallback` when the mask omits it or + * throws, and no stack trace, which repeats the message. */ export function failureRecord( config: Pick, error: unknown, fallback: string, -): { message: string; exception: Exception } { +): { errorType: string; message: string; exception: Exception } { + const errorType = errorTypeOf(error); + if (!config.mask) { let message = fallback; @@ -77,19 +80,23 @@ export function failureRecord( // An error whose string conversion throws is still recorded, with the fallback text. } - return { message, exception: error instanceof Error ? error : message }; + let exception: Exception = message; + + try { + if (error instanceof Error) { + exception = { name: errorType, message: error.message, stack: error.stack }; + } + } catch { + // A throwing message or stack getter leaves the status message as the exception. + } + + return { errorType, message, exception }; } - let name: string | undefined; let detail: string | undefined; try { - if (error instanceof Error) { - name = typeof error.name === "string" && error.name.length > 0 ? error.name : undefined; - detail = error.message; - } else { - detail = unknownErrorMessage(error); - } + detail = error instanceof Error ? error.message : unknownErrorMessage(error); } catch { detail = undefined; } @@ -97,7 +104,7 @@ export function failureRecord( const message = (detail ? contentAttr(config, "exception.message", detail) : undefined) || fallback; - return { message, exception: { name: name ?? "Error", message } }; + return { errorType, message, exception: { name: errorType, message } }; } /** Reports to onError without throwing, including for thrown values String() cannot convert. */ diff --git a/packages/ai/tests/wrappers.test.ts b/packages/ai/tests/wrappers.test.ts index ff7c3714..6e146aca 100644 --- a/packages/ai/tests/wrappers.test.ts +++ b/packages/ai/tests/wrappers.test.ts @@ -627,6 +627,7 @@ test("failures whose string conversion throws are still recorded", async () => { const exception = span.events.find((event) => event.name === "exception"); expect(span.status).toEqual({ code: SpanStatusCode.ERROR, message: expected[index] }); + expect(span.attributes["error.type"]).toBe("RangeError"); expect(exception?.attributes).toMatchObject({ "exception.type": "RangeError", "exception.message": "disk full", diff --git a/packages/google-genai/src/index.ts b/packages/google-genai/src/index.ts index cf6afca0..bf548c6e 100644 --- a/packages/google-genai/src/index.ts +++ b/packages/google-genai/src/index.ts @@ -903,10 +903,45 @@ function mediaResponseFields(response: unknown, outputType: "image" | "video"): responseId: readString(record.responseId) ?? operationName, output: uris.length > 0 ? uris.map((uri) => ({ type: outputType, uri })) : undefined, attributes, - error: record.error, + error: operationError(record.error), }; } +// google.rpc.Code names by number. The error of a long-running operation is a google.rpc.Status, +// which carries only the number, such as 8 for RESOURCE_EXHAUSTED. +const RPC_CODE_NAMES = [ + "OK", + "CANCELLED", + "UNKNOWN", + "INVALID_ARGUMENT", + "DEADLINE_EXCEEDED", + "NOT_FOUND", + "ALREADY_EXISTS", + "PERMISSION_DENIED", + "RESOURCE_EXHAUSTED", + "FAILED_PRECONDITION", + "ABORTED", + "OUT_OF_RANGE", + "UNIMPLEMENTED", + "INTERNAL", + "UNAVAILABLE", + "DATA_LOSS", + "UNAUTHENTICATED", +]; + +// errorTypeOf records the code name in error.type, else "operation.failed". +function operationError(value: unknown): Error | undefined { + if (value === undefined || value === null) return undefined; + + const status = asRecord(value); + const code = readNumber(status?.code); + + return Object.assign(new Error(readString(status?.message) ?? "video operation failed"), { + code: code === undefined ? undefined : RPC_CODE_NAMES[code], + type: "operation.failed", + }); +} + function imageResponseFields(response: unknown): SpanFields { return mediaResponseFields(response, "image"); } diff --git a/packages/google-genai/tests/google-genai.test.ts b/packages/google-genai/tests/google-genai.test.ts index 7db8b66d..fc2b568a 100644 --- a/packages/google-genai/tests/google-genai.test.ts +++ b/packages/google-genai/tests/google-genai.test.ts @@ -3452,6 +3452,28 @@ test("generateVideos traces submit and tracked polling idempotently", async () = expect(exported.every((span) => span.attributes["gen_ai.output.type"] === "video")).toBe(true); }); +test("a failed video operation records its google.rpc code name as error.type", async () => { + const spans = setupSpans(); + const operation = { name: "operations/video-quota", done: false }; + const client = { + models: { generateVideos: (_params: unknown) => Promise.resolve(operation) }, + operations: { + get: (_params: unknown) => + Promise.resolve({ name: operation.name, done: true, error: { code: 8, message: "quota" } }), + }, + }; + + wrapGoogleGenAI(client); + const submitted = await client.models.generateVideos({ model: "veo", prompt: "ocean" }); + await client.operations.get({ operation: submitted }); + + const span = (await finishedSpans(spans, 2))[1]!; + const exception = span.events.find((event) => event.name === "exception"); + expect(span.status.code).toBe(SPAN_STATUS_ERROR); + expect(span.attributes["error.type"]).toBe("RESOURCE_EXHAUSTED"); + expect(exception?.attributes?.["exception.message"]).toBe("quota"); +}); + test("video operation tracking is identity-only and client-local", async () => { const spans = setupSpans(); const operationName = "operations/shared"; diff --git a/packages/openai/src/index.ts b/packages/openai/src/index.ts index 7c38ed0f..8f8d1090 100644 --- a/packages/openai/src/index.ts +++ b/packages/openai/src/index.ts @@ -331,12 +331,27 @@ function responsesResponse( return fields; } +// A failure that OpenAI reports in a response body or a stream event, not as an HTTP error. It +// carries the provider's code, and its kind for when there is no code, so that errorTypeOf records +// the cause in error.type instead of "Error". +function providerFailure( + message: string, + code: string | undefined, + kind: string | undefined, +): Error { + return Object.assign(new Error(message), { code, type: kind }); +} + function responseFailedError(response: T): Error { const error = asRecord(asRecord(response)?.error); const code = readString(error?.code); const message = readString(error?.message); - return new Error(["response.failed", code, message].filter(Boolean).join(": ")); + return providerFailure( + ["response.failed", code, message].filter(Boolean).join(": "), + code, + "response.failed", + ); } function responseStreamError(event: T): Error { @@ -344,7 +359,11 @@ function responseStreamError(event: T): Error { const code = readString(e?.code); const message = readString(e?.message); - return new Error(["response.error", code, message].filter(Boolean).join(": ")); + return providerFailure( + ["response.error", code, message].filter(Boolean).join(": "), + code, + "response.error", + ); } function responsesEventFailure(event: T): Error | undefined { @@ -363,7 +382,9 @@ function responsesEventFailure(event: T): Error | undefined { ? responseFailedError(asRecord(event)?.response) : responseStreamError(event); } catch { - return new Error(type === "response.failed" ? "response.failed" : "response.error"); + const kind = type === "response.failed" ? "response.failed" : "response.error"; + + return providerFailure(kind, undefined, kind); } } @@ -492,7 +513,11 @@ function videoResponse(response: T): SpanFields { finishReason: status, error: status === "failed" - ? asError(readString(asRecord(r.error)?.message) ?? "video failed") + ? providerFailure( + readString(asRecord(r.error)?.message) ?? "video failed", + readString(asRecord(r.error)?.code), + "video.failed", + ) : undefined, output: { status, @@ -1991,6 +2016,20 @@ function createObservedMediaStream( e.type === "image_edit.completed" || e.type === "image_edit.failed")); + if ( + operation === "images" && + (e.type === "image_generation.failed" || e.type === "image_edit.failed") + ) { + const failure = asRecord(e.error); + const code = readString(failure?.code) ?? readString(e.code); + const message = + readString(failure?.message) ?? + readString(e.message) ?? + `OpenAI media stream ended with ${e.type}`; + + error = providerFailure(code ? `${code}: ${message}` : message, code, e.type); + } + if (operation === "transcriptions" && e.type === "transcript.text.delta") { sawTranscriptDelta = true; } @@ -2944,8 +2983,10 @@ export function wrapOpenAIRealtime( if (trace) { finishTrace(trace, { - error: asError( + error: providerFailure( [readString(error?.code), readString(error?.message)].filter(Boolean).join(": "), + readString(error?.code), + readString(error?.type), ), }); } @@ -3028,9 +3069,11 @@ export function wrapOpenAIRealtime( : undefined, error: status === "failed" - ? asError( + ? providerFailure( readString(asRecord(statusDetails?.error)?.message) ?? "Realtime response failed", + readString(asRecord(statusDetails?.error)?.code), + readString(asRecord(statusDetails?.error)?.type) ?? "response.failed", ) : undefined, }, diff --git a/packages/openai/tests/openai.test.ts b/packages/openai/tests/openai.test.ts index 1550d393..b41101f9 100644 --- a/packages/openai/tests/openai.test.ts +++ b/packages/openai/tests/openai.test.ts @@ -3257,9 +3257,9 @@ test("responses create failed body records error while completed body stays OK", const failed = finished.find((span) => span.attributes["gen_ai.response.id"] === "resp_failed"); const success = finished.find((span) => span.attributes["gen_ai.response.id"] === "resp_ok"); expect(failed?.status.code).toBe(SPAN_STATUS_ERROR); - expect(failed?.attributes["error.type"]).toBe("Error"); + expect(failed?.attributes["error.type"]).toBe("server_error"); const exception = failed?.events.find((event) => event.name === "exception"); - expect(exception?.attributes?.["exception.type"]).toBe("Error"); + expect(exception?.attributes?.["exception.type"]).toBe("server_error"); expect(exception?.attributes?.["exception.message"]).toBe( "response.failed: server_error: model exploded", ); @@ -4016,9 +4016,9 @@ test("responses streams record error status from bare error events", async () => expect(events).toEqual([errorEvent]); const span = await exportedSpan(spans); expect(span.status.code).toBe(SPAN_STATUS_ERROR); - expect(span.attributes["error.type"]).toBe("Error"); + expect(span.attributes["error.type"]).toBe("rate_limit_exceeded"); const exception = span.events.find((event) => event.name === "exception"); - expect(exception?.attributes?.["exception.type"]).toBe("Error"); + expect(exception?.attributes?.["exception.type"]).toBe("rate_limit_exceeded"); expect(exception?.attributes?.["exception.message"]).toBe( "response.error: rate_limit_exceeded: try later", ); @@ -4070,7 +4070,7 @@ test("responses streams record error status from response.failed terminal events expect(span.attributes["gen_ai.response.finish_reasons"]).toEqual(["failed"]); expect(jsonAttr(span, "gen_ai.output.messages")).toEqual([]); const exception = span.events.find((event) => event.name === "exception"); - expect(exception?.attributes?.["exception.type"]).toBe("Error"); + expect(exception?.attributes?.["exception.type"]).toBe("server_error"); expect(exception?.attributes?.["exception.message"]).toBe( "response.failed: server_error: stream failed", ); @@ -4806,6 +4806,33 @@ test.each([true, false])( }, ); +test("a failed image stream records the provider's code as error.type", async () => { + const spans = setupSpans(); + const failed = { type: "image_generation.failed", code: "provider_error", message: "failed" }; + const source = new Stream(async function* () { + yield failed; + }, new AbortController()); + const client = wrapOpenAI({ + chat: { completions: { create() {} } }, + responses: { create() {} }, + embeddings: { create() {} }, + images: { generate: async (_params: unknown) => source }, + }); + + const stream = await client.images!.generate({ + model: "gpt-image-1", + prompt: "otter", + stream: true, + }); + expect(await collectStream(stream as Stream)).toEqual([failed]); + + const span = await exportedSpan(spans); + const exception = span.events.find((event) => event.name === "exception"); + expect(span.status.code).toBe(SPAN_STATUS_ERROR); + expect(span.attributes["error.type"]).toBe("provider_error"); + expect(exception?.attributes?.["exception.message"]).toBe("provider_error: failed"); +}); + test("endpoint modality fills aggregate image and transcription output usage", async () => { const spans = setupSpans(); @@ -5561,6 +5588,43 @@ test("realtime caps reconstructed output before a trace timeout", async () => { wrapped.close(); }); +test("a failed realtime response records the provider's code as error.type", async () => { + const spans = setupSpans(); + const listeners = new Map void>>(); + const sent: Array<{ response?: object }> = []; + const wrapped = wrapOpenAIRealtime({ + send(event: unknown) { + sent.push(event as { response?: object }); + }, + close() {}, + on(event: string, listener: (...args: unknown[]) => void) { + listeners.set(event, [...(listeners.get(event) ?? []), listener]); + }, + off() {}, + }); + const emit = (value: unknown) => listeners.get("event")?.forEach((listener) => listener(value)); + + wrapped.send({ type: "response.create" }); + emit({ type: "response.created", response: { id: "failed", ...sent[0]?.response } }); + emit({ + type: "response.done", + response: { + id: "failed", + status: "failed", + status_details: { + type: "failed", + error: { type: "server_error", code: "rate_limit_exceeded", message: "slow down" }, + }, + output: [], + }, + }); + + const span = await exportedSpan(spans); + expect(span.status.code).toBe(SPAN_STATUS_ERROR); + expect(span.attributes["error.type"]).toBe("rate_limit_exceeded"); + wrapped.close(); +}); + test("realtime terminal output supersedes truncated deltas", async () => { const spans = setupSpans(); const listeners = new Map void>>(); @@ -5898,6 +5962,7 @@ test("realtime wrapper keeps recoverable API errors scoped and handles remote cl const failed = await finishedSpans(spans, 2); expect(failed[1]?.status.code).toBe(SPAN_STATUS_ERROR); + expect(failed[1]?.attributes["error.type"]).toBe("invalid_response"); wrapped.send({ type: "response.create" }); const sentEvent = sent.at(-1) as { event_id?: unknown } | undefined; diff --git a/packages/openrouter/src/index.ts b/packages/openrouter/src/index.ts index ad9775aa..f5c1d819 100644 --- a/packages/openrouter/src/index.ts +++ b/packages/openrouter/src/index.ts @@ -535,20 +535,42 @@ function responsesResponse(response: T, includeOutput = true): SpanFields { return fields; } +// A failure that OpenRouter reports in a response body or a stream event, not as an HTTP error. +// errorTypeOf records its string code, else its numeric code (an HTTP status, such as 429), else +// its kind, in error.type instead of "Error". +function providerFailure( + prefix: string, + code: number | string | undefined, + message: string | undefined, + kind: string, +): Error { + return Object.assign(new Error([prefix, code, message].filter(Boolean).join(": ")), { + code: typeof code === "string" ? code : undefined, + status: typeof code === "number" ? code : undefined, + type: kind, + }); +} + function responseFailedError(response: T): Error { const error = asRecord(asRecord(response)?.error); - const code = readNumber(error?.code) ?? readString(error?.code); - const message = readString(error?.message); - return new Error(["response.failed", code, message].filter(Boolean).join(": ")); + return providerFailure( + "response.failed", + readNumber(error?.code) ?? readString(error?.code), + readString(error?.message), + "response.failed", + ); } function responseStreamError(event: T): Error { const e = asRecord(event); - const code = readNumber(e?.code) ?? readString(e?.code); - const message = readString(e?.message); - return new Error(["response.error", code, message].filter(Boolean).join(": ")); + return providerFailure( + "response.error", + readNumber(e?.code) ?? readString(e?.code), + readString(e?.message), + "response.error", + ); } function embeddingsRequest(request: JsonRecord): RequestMapping { @@ -771,7 +793,7 @@ function recordChatChunk(chunk: T, states: Map, budg return { ...update, - error: new Error(["chat stream error", code, message].filter(Boolean).join(": ")), + error: providerFailure("chat stream error", code, message, "stream error"), }; } diff --git a/packages/openrouter/tests/openrouter.test.ts b/packages/openrouter/tests/openrouter.test.ts index c1a74f8d..6b43d623 100644 --- a/packages/openrouter/tests/openrouter.test.ts +++ b/packages/openrouter/tests/openrouter.test.ts @@ -1072,7 +1072,7 @@ test("chat stream chunk errors end one error span with partial output", async () await reader.cancel(); expect(spans.getFinishedSpans()).toHaveLength(1); expect(span.status.code).toBe(SPAN_STATUS_ERROR); - expect(span.attributes["error.type"]).toBe("Error"); + expect(span.attributes["error.type"]).toBe("502"); expect(jsonAttr(span, "gen_ai.output.messages")).toEqual([ { role: "assistant", content: "Partial" }, ]); @@ -2673,7 +2673,7 @@ test("non-streaming failed Responses record a useful error", async () => { const span = await exportedSpan(spans); expect(span.status.code).toBe(SPAN_STATUS_ERROR); - expect(span.attributes["error.type"]).toBe("Error"); + expect(span.attributes["error.type"]).toBe("server_error"); expect(span.events[0]?.attributes?.["exception.message"]).toContain( "response.failed: server_error: provider failed", ); @@ -2706,6 +2706,7 @@ test("failed Responses stream events keep numeric provider error codes", async ( const span = await exportedSpan(spans); expect(span.status.code).toBe(SPAN_STATUS_ERROR); + expect(span.attributes["error.type"]).toBe("502"); expect(span.events[0]?.attributes?.["exception.message"]).toBe( "response.failed: 502: provider failed", ); diff --git a/packages/otel/src/errors.ts b/packages/otel/src/errors.ts index d059ed32..1723d29e 100644 --- a/packages/otel/src/errors.ts +++ b/packages/otel/src/errors.ts @@ -1,7 +1,9 @@ // Error classes that provider SDKs throw for any failed request, whose name says nothing about the // cause: openai's and Anthropic's APIError (a stream's error event), @google/genai's ApiError, and // the AI SDK's APICallError and StreamProviderError. Plain Error is here for errors that carry a -// code. The Python SDK keeps the same rule in `_error_type`; change both together. +// code, such as the failures that the integrations build from a response body with `code`, +// `status`, and `type` fields. The Python SDK keeps the same rule in `_error_type`; change both +// together. const GENERIC_ERROR_NAMES = new Set([ "Error", "APIError", diff --git a/sdks/python-bedrock/tests/test_bedrock.py b/sdks/python-bedrock/tests/test_bedrock.py index 764f3dc4..19e1de8c 100644 --- a/sdks/python-bedrock/tests/test_bedrock.py +++ b/sdks/python-bedrock/tests/test_bedrock.py @@ -506,7 +506,7 @@ def test_converse_stream_accumulates_and_errors( with pytest.raises(EventStreamError): list(response["stream"]) error_span = memory.span_exporter.get_finished_spans()[1] - assert error_span.attributes["error.type"] == "EventStreamError" + assert error_span.attributes["error.type"] == "ModelStreamErrorException" assert error_span.attributes["aws.error.code"] == "modelStreamErrorException" assert error_span.attributes["gen_ai.response.id"] == "stream-error-2" assert error_span.attributes["aws.http.status_code"] == 424 diff --git a/sdks/python-google-genai/src/telemetry_dev_google_genai/__init__.py b/sdks/python-google-genai/src/telemetry_dev_google_genai/__init__.py index 9362a3c3..5eaebd61 100644 --- a/sdks/python-google-genai/src/telemetry_dev_google_genai/__init__.py +++ b/sdks/python-google-genai/src/telemetry_dev_google_genai/__init__.py @@ -468,15 +468,49 @@ def mapper(model: Any, contents: Any, config: Any) -> tuple[str, dict[str, Any]] return mapper +# google.rpc.Code names by number. The error of a long-running operation is a google.rpc.Status, +# which carries only the number, such as 8 for RESOURCE_EXHAUSTED. +_RPC_CODE_NAMES = ( + "OK", + "CANCELLED", + "UNKNOWN", + "INVALID_ARGUMENT", + "DEADLINE_EXCEEDED", + "NOT_FOUND", + "ALREADY_EXISTS", + "PERMISSION_DENIED", + "RESOURCE_EXHAUSTED", + "FAILED_PRECONDITION", + "ABORTED", + "OUT_OF_RANGE", + "UNIMPLEMENTED", + "INTERNAL", + "UNAVAILABLE", + "DATA_LOSS", + "UNAUTHENTICATED", +) + + +def _operation_error(status: Any) -> Exception: + # The core SDK records the code name in error.type, else "operation.failed", because a plain + # Exception names no cause; see _error_type in telemetry_dev/_spans.py. + code = _field(status, "code") + error = Exception(_string(_field(status, "message")) or "video operation failed") + vars(error).update( + code=_RPC_CODE_NAMES[code] + if isinstance(code, int) and not isinstance(code, bool) and 0 <= code < len(_RPC_CODE_NAMES) + else None, + type="operation.failed", + ) + return error + + def _media_response_fields(response: Any, output_type: str | None = None) -> dict[str, Any]: operation_name = _string(_field(response, "name")) operation_error = _field(response, "error") - error_message = _string(_field(operation_error, "message")) fields: dict[str, Any] = { "response_id": _string(_field(response, "id")) or operation_name, - "error": RuntimeError(error_message or "video operation failed") - if operation_error - else None, + "error": _operation_error(operation_error) if operation_error else None, } attrs: dict[str, Any] = {} done = _field(response, "done") diff --git a/sdks/python-google-genai/tests/test_google_genai.py b/sdks/python-google-genai/tests/test_google_genai.py index 335620bc..0f506e0b 100644 --- a/sdks/python-google-genai/tests/test_google_genai.py +++ b/sdks/python-google-genai/tests/test_google_genai.py @@ -670,6 +670,8 @@ async def run() -> Any: span = only_span(memory) assert span.status.status_code is StatusCode.ERROR assert attrs(span)["google_genai.response.operation_done"] is True + assert attrs(span)["error.type"] == "INTERNAL" + assert dict(span.events[0].attributes or {})["exception.message"] == "generation failed" SUPPORT_PROMPT = telemetry_dev.ChatPrompt( diff --git a/sdks/python-litellm/src/telemetry_dev_litellm/__init__.py b/sdks/python-litellm/src/telemetry_dev_litellm/__init__.py index 5d99c665..5c15dc97 100644 --- a/sdks/python-litellm/src/telemetry_dev_litellm/__init__.py +++ b/sdks/python-litellm/src/telemetry_dev_litellm/__init__.py @@ -751,12 +751,21 @@ def _rerank_response(response: Any) -> dict[str, Any]: } -def _response_error(response: Any) -> RuntimeError | None: +def _provider_failure(message: str, code: str | None, kind: str) -> Exception: + # A failure that the provider reports in a response body or a stream event, not as an HTTP + # error. The core SDK records its `code` in error.type, else its `type`, because a plain + # Exception names no cause; see _error_type in telemetry_dev/_spans.py. + error = Exception(message) + vars(error).update(code=code, type=kind) + return error + + +def _response_error(response: Any) -> Exception | None: if _string(_field(response, "status")) != "failed" and _field(response, "error") is None: return None raw_error = _field(response, "error") message = _string(_field(raw_error, "message")) or "LiteLLM Responses API request failed" - return RuntimeError(message) + return _provider_failure(message, _string(_field(raw_error, "code")), "response.failed") def _responses_response(response: Any) -> dict[str, Any]: @@ -1908,7 +1917,7 @@ def _record(self, event: Any, received_at: float) -> None: code = _string(_field(raw_error, "code")) or _string(_field(event, "code")) if code is not None: message = f"{message} ({code})" - self._finish(error=RuntimeError(message)) + self._finish(error=_provider_failure(message, code, "response.error")) def _finish(self, response: Any = None, error: BaseException | None = None) -> None: if self._finished: diff --git a/sdks/python-litellm/tests/test_litellm.py b/sdks/python-litellm/tests/test_litellm.py index 6770e154..c575801e 100644 --- a/sdks/python-litellm/tests/test_litellm.py +++ b/sdks/python-litellm/tests/test_litellm.py @@ -3950,10 +3950,10 @@ def fake_responses(input: Any, model: str, stream: bool = False, **kwargs: Any) failed_span, error_span = finished_spans(memory) assert failed_span.status.status_code is StatusCode.ERROR - assert attrs(failed_span)["error.type"] == "RuntimeError" + assert attrs(failed_span)["error.type"] == "response.failed" assert attrs(failed_span)["gen_ai.response.status"] == "failed" assert error_span.status.status_code is StatusCode.ERROR - assert attrs(error_span)["error.type"] == "RuntimeError" + assert attrs(error_span)["error.type"] == "transport_error" exception = next(event for event in error_span.events if event.name == "exception") assert exception.attributes is not None assert exception.attributes["exception.message"] == "stream disconnected (transport_error)" diff --git a/sdks/python-openai/src/telemetry_dev_openai/__init__.py b/sdks/python-openai/src/telemetry_dev_openai/__init__.py index d633dfa1..0de76428 100644 --- a/sdks/python-openai/src/telemetry_dev_openai/__init__.py +++ b/sdks/python-openai/src/telemetry_dev_openai/__init__.py @@ -1550,34 +1550,22 @@ def _response_event_has_output(event: Any) -> bool: return isinstance(delta, str) and bool(delta) -def _response_failed_error(response: Any) -> RuntimeError: +def _response_failed_error(response: Any) -> Exception: error = _field(response, "error") - if error is None: - return RuntimeError("response.failed") - code = _string(_field(error, "code")) - message = _string(_field(error, "message")) - if code and message: - return RuntimeError(f"response.failed: {code}: {message}") - if code: - return RuntimeError(f"response.failed: {code}") - if message: - return RuntimeError(f"response.failed: {message}") - return RuntimeError("response.failed") - - -def _response_stream_error(event: Any) -> RuntimeError: + code = _string(_field(error, "code")) if error is not None else None + message = _string(_field(error, "message")) if error is not None else None + text = ": ".join(part for part in ("response.failed", code, message) if part) + return _provider_failure(text, code, "response.failed") + + +def _response_stream_error(event: Any) -> Exception: code = _string(_field(event, "code")) message = _string(_field(event, "message")) - if code and message: - return RuntimeError(f"response.error: {code}: {message}") - if code: - return RuntimeError(f"response.error: {code}") - if message: - return RuntimeError(f"response.error: {message}") - return RuntimeError("response.error") + text = ": ".join(part for part in ("response.error", code, message) if part) + return _provider_failure(text, code, "response.error") -def _responses_event_failure(event: Any) -> RuntimeError | None: +def _responses_event_failure(event: Any) -> Exception | None: try: event_type = _field(event, "type") except Exception: @@ -1586,15 +1574,25 @@ def _responses_event_failure(event: Any) -> RuntimeError | None: try: return _response_failed_error(_field(event, "response")) except Exception: - return RuntimeError("response.failed") + return _provider_failure("response.failed", None, "response.failed") if event_type == "error": try: return _response_stream_error(event) except Exception: - return RuntimeError("response.error") + return _provider_failure("response.error", None, "response.error") return None +def _provider_failure(message: str, code: str | int | None, kind: str | None) -> Exception: + # A failure that the provider reports in a response body or a stream event, not as an HTTP + # error. The core SDK records its `code` in error.type (a number counts as an HTTP status, such + # as 429), else its `type`, because a plain Exception names no cause; see _error_type in + # telemetry_dev/_spans.py. + error = Exception(message) + vars(error).update(code=code, type=kind) + return error + + def _hook_response_close(inner: Any, finish: Callable[[], None]) -> None: """End the span when the transport response is closed behind our back. @@ -2433,7 +2431,7 @@ def _media_stream_error(event: Any, event_type: str) -> BaseException | None: or f"OpenAI media stream ended with {event_type}" ) code = _string(_field(raw_error, "code")) or _string(_field(event, "code")) - return RuntimeError(f"{code}: {message}" if code else message) + return _provider_failure(f"{code}: {message}" if code else message, code, event_type) def _media_event_failure(event: Any) -> BaseException | None: @@ -2447,7 +2445,9 @@ def _media_event_failure(event: Any) -> BaseException | None: return _media_stream_error(event, event_type) except Exception: if event_type.endswith(".failed"): - return RuntimeError(f"OpenAI media stream ended with {event_type}") + return _provider_failure( + f"OpenAI media stream ended with {event_type}", None, event_type + ) return None diff --git a/sdks/python-openai/tests/test_openai.py b/sdks/python-openai/tests/test_openai.py index 93fc77bc..263e8a7d 100644 --- a/sdks/python-openai/tests/test_openai.py +++ b/sdks/python-openai/tests/test_openai.py @@ -1252,7 +1252,7 @@ async def handler(_request: httpx.Request) -> httpx.Response: assert received == ["image_generation.failed"] span = only_span(memory) assert span.status.status_code == StatusCode.ERROR - assert attrs(span)["error.type"] == "RuntimeError" + assert attrs(span)["error.type"] == "provider_error" assert dict(span.events[0].attributes or {})["exception.message"] == ( "provider_error: provider failed" ) @@ -4798,7 +4798,7 @@ def handler(request: httpx.Request) -> httpx.Response: span = only_span(memory) assert span.status.status_code == StatusCode.ERROR a = attrs(span) - assert a["error.type"] == "RuntimeError" + assert a["error.type"] == "server_error" assert a["gen_ai.response.id"] == "resp_failed" events = list(span.events) assert len(events) == 1 @@ -5125,7 +5125,7 @@ async def handler(request: httpx.Request) -> httpx.Response: assert completed_attrs["gen_ai.usage.output_tokens"] == 4 assert completed_attrs["gen_ai.usage.total_tokens"] == 17 assert failed_span.status.status_code == StatusCode.ERROR - assert attrs(failed_span)["error.type"] == "RuntimeError" + assert attrs(failed_span)["error.type"] == "server_error" @pytest.mark.parametrize("async_mode", [False, True]) diff --git a/sdks/python-openrouter/src/telemetry_dev_openrouter/__init__.py b/sdks/python-openrouter/src/telemetry_dev_openrouter/__init__.py index 443dd60f..e4ed2e4a 100644 --- a/sdks/python-openrouter/src/telemetry_dev_openrouter/__init__.py +++ b/sdks/python-openrouter/src/telemetry_dev_openrouter/__init__.py @@ -303,44 +303,56 @@ def _responses_usage(raw: Any) -> dict[str, int | float] | None: ) -def _response_failed_error(response: Any) -> RuntimeError: +def _failure_code(value: Any) -> str | int | None: + # OpenRouter's numeric codes are HTTP statuses, which the core SDK records as one. + if isinstance(value, int) and not isinstance(value, bool): + return value + return value if isinstance(value, str) else None + + +def _provider_failure(message: str, code: str | int | None, kind: str | None) -> Exception: + # A failure that OpenRouter reports in a response body or a stream event, not as an HTTP + # error. The core SDK records its `code` in error.type (a number counts as an HTTP status, such + # as 429), else its `type`, because a plain Exception names no cause; see _error_type in + # telemetry_dev/_spans.py. + error = Exception(message) + vars(error).update(code=code, type=kind) + return error + + +def _response_failed_error(response: Any) -> Exception: error = _field(response, "error") if error is None: - return RuntimeError("response.failed") - code = _error_code(_field(error, "code")) + return _provider_failure("response.failed", None, "response.failed") + raw_code = _field(error, "code") + code = _error_code(raw_code) message = _string(_field(error, "message")) - if code and message: - return RuntimeError(f"response.failed: {code}: {message}") - if code: - return RuntimeError(f"response.failed: {code}") - if message: - return RuntimeError(f"response.failed: {message}") - return RuntimeError("response.failed") + text = ": ".join(part for part in ("response.failed", code, message) if part) + return _provider_failure(text, _failure_code(raw_code), "response.failed") -def _response_stream_error(event: Any) -> RuntimeError: - code = _error_code(_field(event, "code")) +def _response_stream_error(event: Any) -> Exception: + raw_code = _field(event, "code") + code = _error_code(raw_code) message = _string(_field(event, "message")) - if code and message: - return RuntimeError(f"response.error: {code}: {message}") - if code: - return RuntimeError(f"response.error: {code}") - if message: - return RuntimeError(f"response.error: {message}") - return RuntimeError("response.error") + text = ": ".join(part for part in ("response.error", code, message) if part) + return _provider_failure(text, _failure_code(raw_code), "response.error") -def _chunk_error(chunk: Any) -> RuntimeError | None: +def _chunk_error(chunk: Any) -> Exception | None: error = _field(chunk, "error") if error is None: return None - code = _error_code(_field(error, "code")) + raw_code = _field(error, "code") + code = _error_code(raw_code) message = _string(_field(error, "message")) if code and message: - return RuntimeError(f"stream error {code}: {message}") - if message: - return RuntimeError(f"stream error: {message}") - return RuntimeError(f"stream error {code}" if code else "stream error") + text = f"stream error {code}: {message}" + elif message: + text = f"stream error: {message}" + else: + text = f"stream error {code}" if code else "stream error" + return _provider_failure(text, _failure_code(raw_code), "stream error") def _responses_response( diff --git a/sdks/python-openrouter/tests/test_openrouter.py b/sdks/python-openrouter/tests/test_openrouter.py index 7c63e182..32689d2a 100644 --- a/sdks/python-openrouter/tests/test_openrouter.py +++ b/sdks/python-openrouter/tests/test_openrouter.py @@ -1146,12 +1146,12 @@ def handler(request: httpx.Request) -> httpx.Response: span = only_span(memory) a = attrs(span) assert span.status.status_code == StatusCode.ERROR - assert a["error.type"] == "RuntimeError" + assert a["error.type"] == "502" events_recorded = list(span.events) assert len(events_recorded) == 1 assert events_recorded[0].name == "exception" event_attrs = dict(events_recorded[0].attributes or {}) - assert event_attrs["exception.type"] == "RuntimeError" + assert event_attrs["exception.type"] == "502" assert event_attrs["exception.message"] == "stream error 502: upstream blew up" assert json.loads(str(a["gen_ai.output.messages"])) == [{"role": "assistant", "content": "Hel"}] @@ -1427,7 +1427,7 @@ def handler(request: httpx.Request) -> httpx.Response: span = only_span(memory) a = attrs(span) assert span.status.status_code == StatusCode.ERROR - assert a["error.type"] == "RuntimeError" + assert a["error.type"] == "server_error" assert list(cast(Any, a["gen_ai.response.finish_reasons"])) == ["failed"] @@ -1488,12 +1488,12 @@ def handler(request: httpx.Request) -> httpx.Response: span = only_span(memory) assert span.status.status_code == StatusCode.ERROR - assert attrs(span)["error.type"] == "RuntimeError" + assert attrs(span)["error.type"] == str(code) events_recorded = list(span.events) assert len(events_recorded) == 1 assert events_recorded[0].name == "exception" event_attrs = dict(events_recorded[0].attributes or {}) - assert event_attrs["exception.type"] == "RuntimeError" + assert event_attrs["exception.type"] == str(code) assert event_attrs["exception.message"] == f"response.failed: {code}: model blew up" @@ -1516,7 +1516,7 @@ def handler(request: httpx.Request) -> httpx.Response: events_recorded = list(span.events) assert len(events_recorded) == 1 event_attrs = dict(events_recorded[0].attributes or {}) - assert event_attrs["exception.type"] == "RuntimeError" + assert event_attrs["exception.type"] == "502" assert event_attrs["exception.message"] == "response.error: 502: provider disconnected" diff --git a/sdks/python/src/telemetry_dev/_spans.py b/sdks/python/src/telemetry_dev/_spans.py index 852efab4..334d7ca4 100644 --- a/sdks/python/src/telemetry_dev/_spans.py +++ b/sdks/python/src/telemetry_dev/_spans.py @@ -153,11 +153,13 @@ def _to_ns(value: TimeInput) -> int | None: # Error classes that provider SDKs raise for any failed request, whose name says nothing about the # cause: openai's and anthropic's APIStatusError (anthropic's for a stream's error event), and -# google-genai's APIError, ClientError, and ServerError. botocore's ClientError carries its code -# in `response`. Plain Exception is here for errors that carry a code. The TypeScript SDK keeps -# the same rule in `errorTypeOf` (packages/otel/src/errors.ts); change both together. +# google-genai's APIError, ClientError, and ServerError. botocore's ClientError, and its +# EventStreamError for a Bedrock stream's exception, carry their code in `response`. Plain +# Exception is here for errors that carry a code, such as the failures that the integrations build +# from a response body with `code` and `type` attributes. The TypeScript SDK keeps the same rule in +# `errorTypeOf` (packages/otel/src/errors.ts); change both together. _GENERIC_ERROR_NAMES = frozenset( - {"Exception", "APIError", "APIStatusError", "ClientError", "ServerError"} + {"Exception", "APIError", "APIStatusError", "ClientError", "ServerError", "EventStreamError"} ) @@ -227,10 +229,16 @@ def _error_type(error: BaseException) -> str: return code if status is not None: return str(status) + aws_code = _text(_nested(_attribute(error, "response"), "Error", "Code")) + if aws_code is not None: + # botocore names a Bedrock stream's exception by its event, such as + # modelStreamErrorException; the TypeScript SDK records its class, + # ModelStreamErrorException. + aws_code = aws_code[0].upper() + aws_code[1:] return ( _text(_attribute(error, "type")) or _text(_nested(body, "error", "type")) - or _text(_nested(_attribute(error, "response"), "Error", "Code")) + or aws_code or name )