From 6460e8d8714b32696484584a50a86da8e775ff7a Mon Sep 17 00:00:00 2001 From: "ci.datadog-api-spec" Date: Fri, 30 Oct 2026 21:21:29 +0000 Subject: [PATCH] Regenerate client from commit aa7d0d7 of spec repo --- .generator/schemas/v2/openapi.yaml | 36 ++++++++++++----- .../llm_obs_custom_eval_config_eval_scope.py | 7 +++- ...custom_eval_config_integration_provider.py | 39 ++++++++++++++----- ...obs_custom_eval_config_llm_judge_config.py | 6 +-- ...llm_obs_custom_eval_config_llm_provider.py | 4 +- ...llm_obs_custom_eval_config_parsing_type.py | 5 ++- .../llm_obs_custom_eval_config_target.py | 2 +- 7 files changed, 70 insertions(+), 29 deletions(-) diff --git a/.generator/schemas/v2/openapi.yaml b/.generator/schemas/v2/openapi.yaml index 8a63ddc4ce..48e5fc2063 100644 --- a/.generator/schemas/v2/openapi.yaml +++ b/.generator/schemas/v2/openapi.yaml @@ -70997,16 +70997,18 @@ components: - attributes type: object LLMObsCustomEvalConfigEvalScope: - description: Scope at which to evaluate spans. + description: Scope at which to evaluate spans. `trace_fan_out` requires `llm_judge_config.target_query`. enum: - span - trace + - trace_fan_out - session example: "span" type: string x-enum-varnames: - SPAN - TRACE + - TRACE_FAN_OUT - SESSION LLMObsCustomEvalConfigInferenceParams: description: LLM inference parameters for a custom evaluator. @@ -71043,14 +71045,20 @@ components: type: number type: object LLMObsCustomEvalConfigIntegrationProvider: - description: Name of the LLM integration provider. + description: >- + Name of the LLM integration provider. Use `deterministic` for a code evaluator, which runs `llm_judge_config.user_specified_json_post_processing_function` instead of calling an LLM. `insights_sponsored` is returned for evaluators that Datadog manages and cannot be set. enum: - openai - - amazon-bedrock + - amazon_bedrock - anthropic - - azure-openai - - vertex-ai - - llm-proxy + - azure_openai + - vertex_ai + - llmproxy + - deterministic + - typesafe + - datadog + - datadog_ai_gateway + - insights_sponsored example: "openai" type: string x-enum-varnames: @@ -71060,6 +71068,11 @@ components: - AZURE_OPENAI - VERTEX_AI - LLM_PROXY + - DETERMINISTIC + - TYPESAFE + - DATADOG + - DATADOG_AI_GATEWAY + - INSIGHTS_SPONSORED LLMObsCustomEvalConfigLLMJudgeConfig: description: LLM judge configuration for a custom evaluator. properties: @@ -71084,13 +71097,13 @@ components: type: boolean output_schema: additionalProperties: {} - description: JSON schema describing the expected output format of the LLM judge. + description: JSON schema describing the expected output format of the LLM judge. Omit for a `deterministic` evaluator. nullable: true type: object parsing_type: $ref: "#/components/schemas/LLMObsCustomEvalConfigParsingType" prompt_template: - description: List of messages forming the LLM judge prompt template. + description: List of messages forming the LLM judge prompt template. Omit for a `deterministic` evaluator. items: $ref: "#/components/schemas/LLMObsCustomEvalConfigPromptMessage" type: array @@ -71100,7 +71113,8 @@ components: nullable: true type: string user_specified_json_post_processing_function: - description: User-provided function applied to post-process the JSON output of the LLM judge. + description: >- + User-provided JavaScript function applied to post-process the JSON output of the LLM judge. For a `deterministic` evaluator it is the evaluator itself: it receives the evaluated span (span scope), `{spans}` (trace scope), or `{session_id, traces}` (session scope), and its return value sets the result. A boolean, number, string or object becomes a boolean, score, categorical or JSON metric; `null` skips the evaluation; and `{value, assessment, reasoning, metadata}` also sets pass or fail, reasoning, and metadata. nullable: true type: string required: @@ -71118,7 +71132,7 @@ components: integration_provider: $ref: "#/components/schemas/LLMObsCustomEvalConfigIntegrationProvider" model_name: - description: Name of the LLM model. + description: Name of the LLM model. Use `javascript` for a `deterministic` evaluator. example: "gpt-4o" type: string vertex_ai: @@ -71141,12 +71155,14 @@ components: - structured_output - json - keyword_search + - typed_questions example: "structured_output" type: string x-enum-varnames: - STRUCTURED_OUTPUT - JSON - KEYWORD_SEARCH + - TYPED_QUESTIONS LLMObsCustomEvalConfigPromptContent: description: A content block within a prompt message. properties: diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_eval_scope.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_eval_scope.py index fcbe5cf913..2447361c72 100644 --- a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_eval_scope.py +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_eval_scope.py @@ -14,19 +14,21 @@ class LLMObsCustomEvalConfigEvalScope(ModelSimple): """ - Scope at which to evaluate spans. + Scope at which to evaluate spans. `trace_fan_out` requires `llm_judge_config.target_query`. - :param value: Must be one of ["span", "trace", "session"]. + :param value: Must be one of ["span", "trace", "trace_fan_out", "session"]. :type value: str """ allowed_values = { "span", "trace", + "trace_fan_out", "session", } SPAN: ClassVar["LLMObsCustomEvalConfigEvalScope"] TRACE: ClassVar["LLMObsCustomEvalConfigEvalScope"] + TRACE_FAN_OUT: ClassVar["LLMObsCustomEvalConfigEvalScope"] SESSION: ClassVar["LLMObsCustomEvalConfigEvalScope"] @cached_property @@ -38,4 +40,5 @@ def openapi_types(_): LLMObsCustomEvalConfigEvalScope.SPAN = LLMObsCustomEvalConfigEvalScope("span") LLMObsCustomEvalConfigEvalScope.TRACE = LLMObsCustomEvalConfigEvalScope("trace") +LLMObsCustomEvalConfigEvalScope.TRACE_FAN_OUT = LLMObsCustomEvalConfigEvalScope("trace_fan_out") LLMObsCustomEvalConfigEvalScope.SESSION = LLMObsCustomEvalConfigEvalScope("session") diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_integration_provider.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_integration_provider.py index 0b8c063981..0a3bbd79fd 100644 --- a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_integration_provider.py +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_integration_provider.py @@ -14,19 +14,24 @@ class LLMObsCustomEvalConfigIntegrationProvider(ModelSimple): """ - Name of the LLM integration provider. + Name of the LLM integration provider. Use `deterministic` for a code evaluator, which runs `llm_judge_config.user_specified_json_post_processing_function` instead of calling an LLM. `insights_sponsored` is returned for evaluators that Datadog manages and cannot be set. - :param value: Must be one of ["openai", "amazon-bedrock", "anthropic", "azure-openai", "vertex-ai", "llm-proxy"]. + :param value: Must be one of ["openai", "amazon_bedrock", "anthropic", "azure_openai", "vertex_ai", "llmproxy", "deterministic", "typesafe", "datadog", "datadog_ai_gateway", "insights_sponsored"]. :type value: str """ allowed_values = { "openai", - "amazon-bedrock", + "amazon_bedrock", "anthropic", - "azure-openai", - "vertex-ai", - "llm-proxy", + "azure_openai", + "vertex_ai", + "llmproxy", + "deterministic", + "typesafe", + "datadog", + "datadog_ai_gateway", + "insights_sponsored", } OPENAI: ClassVar["LLMObsCustomEvalConfigIntegrationProvider"] AMAZON_BEDROCK: ClassVar["LLMObsCustomEvalConfigIntegrationProvider"] @@ -34,6 +39,11 @@ class LLMObsCustomEvalConfigIntegrationProvider(ModelSimple): AZURE_OPENAI: ClassVar["LLMObsCustomEvalConfigIntegrationProvider"] VERTEX_AI: ClassVar["LLMObsCustomEvalConfigIntegrationProvider"] LLM_PROXY: ClassVar["LLMObsCustomEvalConfigIntegrationProvider"] + DETERMINISTIC: ClassVar["LLMObsCustomEvalConfigIntegrationProvider"] + TYPESAFE: ClassVar["LLMObsCustomEvalConfigIntegrationProvider"] + DATADOG: ClassVar["LLMObsCustomEvalConfigIntegrationProvider"] + DATADOG_AI_GATEWAY: ClassVar["LLMObsCustomEvalConfigIntegrationProvider"] + INSIGHTS_SPONSORED: ClassVar["LLMObsCustomEvalConfigIntegrationProvider"] @cached_property def openapi_types(_): @@ -43,8 +53,17 @@ def openapi_types(_): LLMObsCustomEvalConfigIntegrationProvider.OPENAI = LLMObsCustomEvalConfigIntegrationProvider("openai") -LLMObsCustomEvalConfigIntegrationProvider.AMAZON_BEDROCK = LLMObsCustomEvalConfigIntegrationProvider("amazon-bedrock") +LLMObsCustomEvalConfigIntegrationProvider.AMAZON_BEDROCK = LLMObsCustomEvalConfigIntegrationProvider("amazon_bedrock") LLMObsCustomEvalConfigIntegrationProvider.ANTHROPIC = LLMObsCustomEvalConfigIntegrationProvider("anthropic") -LLMObsCustomEvalConfigIntegrationProvider.AZURE_OPENAI = LLMObsCustomEvalConfigIntegrationProvider("azure-openai") -LLMObsCustomEvalConfigIntegrationProvider.VERTEX_AI = LLMObsCustomEvalConfigIntegrationProvider("vertex-ai") -LLMObsCustomEvalConfigIntegrationProvider.LLM_PROXY = LLMObsCustomEvalConfigIntegrationProvider("llm-proxy") +LLMObsCustomEvalConfigIntegrationProvider.AZURE_OPENAI = LLMObsCustomEvalConfigIntegrationProvider("azure_openai") +LLMObsCustomEvalConfigIntegrationProvider.VERTEX_AI = LLMObsCustomEvalConfigIntegrationProvider("vertex_ai") +LLMObsCustomEvalConfigIntegrationProvider.LLM_PROXY = LLMObsCustomEvalConfigIntegrationProvider("llmproxy") +LLMObsCustomEvalConfigIntegrationProvider.DETERMINISTIC = LLMObsCustomEvalConfigIntegrationProvider("deterministic") +LLMObsCustomEvalConfigIntegrationProvider.TYPESAFE = LLMObsCustomEvalConfigIntegrationProvider("typesafe") +LLMObsCustomEvalConfigIntegrationProvider.DATADOG = LLMObsCustomEvalConfigIntegrationProvider("datadog") +LLMObsCustomEvalConfigIntegrationProvider.DATADOG_AI_GATEWAY = LLMObsCustomEvalConfigIntegrationProvider( + "datadog_ai_gateway" +) +LLMObsCustomEvalConfigIntegrationProvider.INSIGHTS_SPONSORED = LLMObsCustomEvalConfigIntegrationProvider( + "insights_sponsored" +) diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_llm_judge_config.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_llm_judge_config.py index cfda678d1f..92270bd179 100644 --- a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_llm_judge_config.py +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_llm_judge_config.py @@ -120,19 +120,19 @@ def __init__( :param modified_library_prompt_template: Whether the library prompt template was modified. :type modified_library_prompt_template: bool, none_type, optional - :param output_schema: JSON schema describing the expected output format of the LLM judge. + :param output_schema: JSON schema describing the expected output format of the LLM judge. Omit for a ``deterministic`` evaluator. :type output_schema: {str: (bool, date, datetime, dict, float, int, list, str, UUID, none_type,)}, none_type, optional :param parsing_type: Output parsing type for a custom LLM judge evaluator. :type parsing_type: LLMObsCustomEvalConfigParsingType, optional - :param prompt_template: List of messages forming the LLM judge prompt template. + :param prompt_template: List of messages forming the LLM judge prompt template. Omit for a ``deterministic`` evaluator. :type prompt_template: [LLMObsCustomEvalConfigPromptMessage], optional :param target_query: Query used to extract the target value to evaluate. :type target_query: str, none_type, optional - :param user_specified_json_post_processing_function: User-provided function applied to post-process the JSON output of the LLM judge. + :param user_specified_json_post_processing_function: User-provided JavaScript function applied to post-process the JSON output of the LLM judge. For a ``deterministic`` evaluator it is the evaluator itself: it receives the evaluated span (span scope), ``{spans}`` (trace scope), or ``{session_id, traces}`` (session scope), and its return value sets the result. A boolean, number, string or object becomes a boolean, score, categorical or JSON metric; ``null`` skips the evaluation; and ``{value, assessment, reasoning, metadata}`` also sets pass or fail, reasoning, and metadata. :type user_specified_json_post_processing_function: str, none_type, optional """ if assessment_criteria is not unset: diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_llm_provider.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_llm_provider.py index b6f59d1be8..a803c8a37f 100644 --- a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_llm_provider.py +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_llm_provider.py @@ -72,10 +72,10 @@ def __init__( :param integration_account_id: Integration account identifier. :type integration_account_id: str, optional - :param integration_provider: Name of the LLM integration provider. + :param integration_provider: Name of the LLM integration provider. Use ``deterministic`` for a code evaluator, which runs ``llm_judge_config.user_specified_json_post_processing_function`` instead of calling an LLM. ``insights_sponsored`` is returned for evaluators that Datadog manages and cannot be set. :type integration_provider: LLMObsCustomEvalConfigIntegrationProvider, optional - :param model_name: Name of the LLM model. + :param model_name: Name of the LLM model. Use ``javascript`` for a ``deterministic`` evaluator. :type model_name: str, optional :param vertex_ai: Google Vertex AI-specific options for LLM provider configuration. diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_parsing_type.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_parsing_type.py index 0a30fbef8c..effdd96320 100644 --- a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_parsing_type.py +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_parsing_type.py @@ -16,7 +16,7 @@ class LLMObsCustomEvalConfigParsingType(ModelSimple): """ Output parsing type for a custom LLM judge evaluator. - :param value: Must be one of ["structured_output", "json", "keyword_search"]. + :param value: Must be one of ["structured_output", "json", "keyword_search", "typed_questions"]. :type value: str """ @@ -24,10 +24,12 @@ class LLMObsCustomEvalConfigParsingType(ModelSimple): "structured_output", "json", "keyword_search", + "typed_questions", } STRUCTURED_OUTPUT: ClassVar["LLMObsCustomEvalConfigParsingType"] JSON: ClassVar["LLMObsCustomEvalConfigParsingType"] KEYWORD_SEARCH: ClassVar["LLMObsCustomEvalConfigParsingType"] + TYPED_QUESTIONS: ClassVar["LLMObsCustomEvalConfigParsingType"] @cached_property def openapi_types(_): @@ -39,3 +41,4 @@ def openapi_types(_): LLMObsCustomEvalConfigParsingType.STRUCTURED_OUTPUT = LLMObsCustomEvalConfigParsingType("structured_output") LLMObsCustomEvalConfigParsingType.JSON = LLMObsCustomEvalConfigParsingType("json") LLMObsCustomEvalConfigParsingType.KEYWORD_SEARCH = LLMObsCustomEvalConfigParsingType("keyword_search") +LLMObsCustomEvalConfigParsingType.TYPED_QUESTIONS = LLMObsCustomEvalConfigParsingType("typed_questions") diff --git a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_target.py b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_target.py index c289909867..e52dfc1ba5 100644 --- a/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_target.py +++ b/src/datadog_api_client/v2/model/llm_obs_custom_eval_config_target.py @@ -64,7 +64,7 @@ def __init__( :param enabled: Whether the evaluator is active for the target application. :type enabled: bool - :param eval_scope: Scope at which to evaluate spans. + :param eval_scope: Scope at which to evaluate spans. ``trace_fan_out`` requires ``llm_judge_config.target_query``. :type eval_scope: LLMObsCustomEvalConfigEvalScope, optional :param experiment_project_ids: Experiment project IDs this evaluator is scoped to.