Fix LLM Obs custom evaluator provider names and add deterministic provider

.generator/schemas/v2/openapi.yaml

@@ -70997,16 +70997,18 @@ components:
         - attributes
       type: object
     LLMObsCustomEvalConfigEvalScope:
-      description: Scope at which to evaluate spans.
+      description: Scope at which to evaluate spans. `trace_fan_out` requires `llm_judge_config.target_query`.
       enum:
         - span
         - trace
+        - trace_fan_out
         - session
       example: "span"
       type: string
       x-enum-varnames:
         - SPAN
         - TRACE
+        - TRACE_FAN_OUT
         - SESSION
     LLMObsCustomEvalConfigInferenceParams:
       description: LLM inference parameters for a custom evaluator.
@@ -71043,14 +71045,20 @@ components:
           type: number
       type: object
     LLMObsCustomEvalConfigIntegrationProvider:
-      description: Name of the LLM integration provider.
+      description: >-
+        Name of the LLM integration provider. Use `deterministic` for a code evaluator, which runs `llm_judge_config.user_specified_json_post_processing_function` instead of calling an LLM. `insights_sponsored` is returned for evaluators that Datadog manages and cannot be set.
       enum:
         - openai
-        - amazon-bedrock
+        - amazon_bedrock
         - anthropic
-        - azure-openai
-        - vertex-ai
-        - llm-proxy
+        - azure_openai
+        - vertex_ai
+        - llmproxy
+        - deterministic
+        - typesafe
+        - datadog
+        - datadog_ai_gateway
+        - insights_sponsored
       example: "openai"
       type: string
       x-enum-varnames:
@@ -71060,6 +71068,11 @@ components:
         - AZURE_OPENAI
         - VERTEX_AI
         - LLM_PROXY
+        - DETERMINISTIC
+        - TYPESAFE
+        - DATADOG
+        - DATADOG_AI_GATEWAY
+        - INSIGHTS_SPONSORED
     LLMObsCustomEvalConfigLLMJudgeConfig:
       description: LLM judge configuration for a custom evaluator.
       properties:
@@ -71084,13 +71097,13 @@ components:
           type: boolean
         output_schema:
           additionalProperties: {}
-          description: JSON schema describing the expected output format of the LLM judge.
+          description: JSON schema describing the expected output format of the LLM judge. Omit for a `deterministic` evaluator.
           nullable: true
           type: object
         parsing_type:
           $ref: "#/components/schemas/LLMObsCustomEvalConfigParsingType"
         prompt_template:
-          description: List of messages forming the LLM judge prompt template.
+          description: List of messages forming the LLM judge prompt template. Omit for a `deterministic` evaluator.
           items:
             $ref: "#/components/schemas/LLMObsCustomEvalConfigPromptMessage"
           type: array
@@ -71100,7 +71113,8 @@ components:
           nullable: true
           type: string
         user_specified_json_post_processing_function:
-          description: User-provided function applied to post-process the JSON output of the LLM judge.
+          description: >-
+            User-provided JavaScript function applied to post-process the JSON output of the LLM judge. For a `deterministic` evaluator it is the evaluator itself: it receives the evaluated span (span scope), `{spans}` (trace scope), or `{session_id, traces}` (session scope), and its return value sets the result. A boolean, number, string or object becomes a boolean, score, categorical or JSON metric; `null` skips the evaluation; and `{value, assessment, reasoning, metadata}` also sets pass or fail, reasoning, and metadata.
           nullable: true
           type: string
       required:
@@ -71118,7 +71132,7 @@ components:
         integration_provider:
           $ref: "#/components/schemas/LLMObsCustomEvalConfigIntegrationProvider"
         model_name:
-          description: Name of the LLM model.
+          description: Name of the LLM model. Use `javascript` for a `deterministic` evaluator.
           example: "gpt-4o"
           type: string
         vertex_ai:
@@ -71141,12 +71155,14 @@ components:
         - structured_output
         - json
         - keyword_search
+        - typed_questions
       example: "structured_output"
       type: string
       x-enum-varnames:
         - STRUCTURED_OUTPUT
         - JSON
         - KEYWORD_SEARCH
+        - TYPED_QUESTIONS
     LLMObsCustomEvalConfigPromptContent:
       description: A content block within a prompt message.
       properties:

src/datadog_api_client/v2/model/llm_obs_custom_eval_config_eval_scope.py

@@ -14,19 +14,21 @@
 
 class LLMObsCustomEvalConfigEvalScope(ModelSimple):
     """
-    Scope at which to evaluate spans.
+    Scope at which to evaluate spans. `trace_fan_out` requires `llm_judge_config.target_query`.
 
-    :param value: Must be one of ["span", "trace", "session"].
+    :param value: Must be one of ["span", "trace", "trace_fan_out", "session"].
     :type value: str
     """
 
     allowed_values = {
         "span",
         "trace",
+        "trace_fan_out",
         "session",
     }
     SPAN: ClassVar["LLMObsCustomEvalConfigEvalScope"]
     TRACE: ClassVar["LLMObsCustomEvalConfigEvalScope"]
+    TRACE_FAN_OUT: ClassVar["LLMObsCustomEvalConfigEvalScope"]
     SESSION: ClassVar["LLMObsCustomEvalConfigEvalScope"]
 
     @cached_property
@@ -38,4 +40,5 @@ def openapi_types(_):
 
 LLMObsCustomEvalConfigEvalScope.SPAN = LLMObsCustomEvalConfigEvalScope("span")
 LLMObsCustomEvalConfigEvalScope.TRACE = LLMObsCustomEvalConfigEvalScope("trace")
+LLMObsCustomEvalConfigEvalScope.TRACE_FAN_OUT = LLMObsCustomEvalConfigEvalScope("trace_fan_out")
 LLMObsCustomEvalConfigEvalScope.SESSION = LLMObsCustomEvalConfigEvalScope("session")

src/datadog_api_client/v2/model/llm_obs_custom_eval_config_integration_provider.py

@@ -14,26 +14,36 @@
 
 class LLMObsCustomEvalConfigIntegrationProvider(ModelSimple):
     """
-    Name of the LLM integration provider.
+    Name of the LLM integration provider. Use `deterministic` for a code evaluator, which runs `llm_judge_config.user_specified_json_post_processing_function` instead of calling an LLM. `insights_sponsored` is returned for evaluators that Datadog manages and cannot be set.
 
-    :param value: Must be one of ["openai", "amazon-bedrock", "anthropic", "azure-openai", "vertex-ai", "llm-proxy"].
+    :param value: Must be one of ["openai", "amazon_bedrock", "anthropic", "azure_openai", "vertex_ai", "llmproxy", "deterministic", "typesafe", "datadog", "datadog_ai_gateway", "insights_sponsored"].
     :type value: str
     """
 
     allowed_values = {
         "openai",
-        "amazon-bedrock",
+        "amazon_bedrock",
         "anthropic",
-        "azure-openai",
-        "vertex-ai",
-        "llm-proxy",
+        "azure_openai",
+        "vertex_ai",
+        "llmproxy",
+        "deterministic",
+        "typesafe",
+        "datadog",
+        "datadog_ai_gateway",
+        "insights_sponsored",
     }
     OPENAI: ClassVar["LLMObsCustomEvalConfigIntegrationProvider"]
     AMAZON_BEDROCK: ClassVar["LLMObsCustomEvalConfigIntegrationProvider"]
     ANTHROPIC: ClassVar["LLMObsCustomEvalConfigIntegrationProvider"]
     AZURE_OPENAI: ClassVar["LLMObsCustomEvalConfigIntegrationProvider"]
     VERTEX_AI: ClassVar["LLMObsCustomEvalConfigIntegrationProvider"]
     LLM_PROXY: ClassVar["LLMObsCustomEvalConfigIntegrationProvider"]
+    DETERMINISTIC: ClassVar["LLMObsCustomEvalConfigIntegrationProvider"]
+    TYPESAFE: ClassVar["LLMObsCustomEvalConfigIntegrationProvider"]
+    DATADOG: ClassVar["LLMObsCustomEvalConfigIntegrationProvider"]
+    DATADOG_AI_GATEWAY: ClassVar["LLMObsCustomEvalConfigIntegrationProvider"]
+    INSIGHTS_SPONSORED: ClassVar["LLMObsCustomEvalConfigIntegrationProvider"]
 
     @cached_property
     def openapi_types(_):
@@ -43,8 +53,17 @@ def openapi_types(_):
 
 
 LLMObsCustomEvalConfigIntegrationProvider.OPENAI = LLMObsCustomEvalConfigIntegrationProvider("openai")
-LLMObsCustomEvalConfigIntegrationProvider.AMAZON_BEDROCK = LLMObsCustomEvalConfigIntegrationProvider("amazon-bedrock")
+LLMObsCustomEvalConfigIntegrationProvider.AMAZON_BEDROCK = LLMObsCustomEvalConfigIntegrationProvider("amazon_bedrock")
 LLMObsCustomEvalConfigIntegrationProvider.ANTHROPIC = LLMObsCustomEvalConfigIntegrationProvider("anthropic")
-LLMObsCustomEvalConfigIntegrationProvider.AZURE_OPENAI = LLMObsCustomEvalConfigIntegrationProvider("azure-openai")
-LLMObsCustomEvalConfigIntegrationProvider.VERTEX_AI = LLMObsCustomEvalConfigIntegrationProvider("vertex-ai")
-LLMObsCustomEvalConfigIntegrationProvider.LLM_PROXY = LLMObsCustomEvalConfigIntegrationProvider("llm-proxy")
+LLMObsCustomEvalConfigIntegrationProvider.AZURE_OPENAI = LLMObsCustomEvalConfigIntegrationProvider("azure_openai")
+LLMObsCustomEvalConfigIntegrationProvider.VERTEX_AI = LLMObsCustomEvalConfigIntegrationProvider("vertex_ai")
+LLMObsCustomEvalConfigIntegrationProvider.LLM_PROXY = LLMObsCustomEvalConfigIntegrationProvider("llmproxy")
+LLMObsCustomEvalConfigIntegrationProvider.DETERMINISTIC = LLMObsCustomEvalConfigIntegrationProvider("deterministic")
+LLMObsCustomEvalConfigIntegrationProvider.TYPESAFE = LLMObsCustomEvalConfigIntegrationProvider("typesafe")
+LLMObsCustomEvalConfigIntegrationProvider.DATADOG = LLMObsCustomEvalConfigIntegrationProvider("datadog")
+LLMObsCustomEvalConfigIntegrationProvider.DATADOG_AI_GATEWAY = LLMObsCustomEvalConfigIntegrationProvider(
+    "datadog_ai_gateway"
+)
+LLMObsCustomEvalConfigIntegrationProvider.INSIGHTS_SPONSORED = LLMObsCustomEvalConfigIntegrationProvider(
+    "insights_sponsored"
+)

src/datadog_api_client/v2/model/llm_obs_custom_eval_config_llm_judge_config.py

@@ -120,19 +120,19 @@ def __init__(
         :param modified_library_prompt_template: Whether the library prompt template was modified.
         :type modified_library_prompt_template: bool, none_type, optional
 
-        :param output_schema: JSON schema describing the expected output format of the LLM judge.
+        :param output_schema: JSON schema describing the expected output format of the LLM judge. Omit for a ``deterministic`` evaluator.
         :type output_schema: {str: (bool, date, datetime, dict, float, int, list, str, UUID, none_type,)}, none_type, optional
 
         :param parsing_type: Output parsing type for a custom LLM judge evaluator.
         :type parsing_type: LLMObsCustomEvalConfigParsingType, optional
 
-        :param prompt_template: List of messages forming the LLM judge prompt template.
+        :param prompt_template: List of messages forming the LLM judge prompt template. Omit for a ``deterministic`` evaluator.
         :type prompt_template: [LLMObsCustomEvalConfigPromptMessage], optional
 
         :param target_query: Query used to extract the target value to evaluate.
         :type target_query: str, none_type, optional
 
-        :param user_specified_json_post_processing_function: User-provided function applied to post-process the JSON output of the LLM judge.
+        :param user_specified_json_post_processing_function: User-provided JavaScript function applied to post-process the JSON output of the LLM judge. For a ``deterministic`` evaluator it is the evaluator itself: it receives the evaluated span (span scope), ``{spans}`` (trace scope), or ``{session_id, traces}`` (session scope), and its return value sets the result. A boolean, number, string or object becomes a boolean, score, categorical or JSON metric; ``null`` skips the evaluation; and ``{value, assessment, reasoning, metadata}`` also sets pass or fail, reasoning, and metadata.
         :type user_specified_json_post_processing_function: str, none_type, optional
         """
         if assessment_criteria is not unset:

src/datadog_api_client/v2/model/llm_obs_custom_eval_config_llm_provider.py

@@ -72,10 +72,10 @@ def __init__(
         :param integration_account_id: Integration account identifier.
         :type integration_account_id: str, optional
 
-        :param integration_provider: Name of the LLM integration provider.
+        :param integration_provider: Name of the LLM integration provider. Use ``deterministic`` for a code evaluator, which runs ``llm_judge_config.user_specified_json_post_processing_function`` instead of calling an LLM. ``insights_sponsored`` is returned for evaluators that Datadog manages and cannot be set.
         :type integration_provider: LLMObsCustomEvalConfigIntegrationProvider, optional
 
-        :param model_name: Name of the LLM model.
+        :param model_name: Name of the LLM model. Use ``javascript`` for a ``deterministic`` evaluator.
         :type model_name: str, optional
 
         :param vertex_ai: Google Vertex AI-specific options for LLM provider configuration.

src/datadog_api_client/v2/model/llm_obs_custom_eval_config_parsing_type.py

@@ -16,18 +16,20 @@ class LLMObsCustomEvalConfigParsingType(ModelSimple):
     """
     Output parsing type for a custom LLM judge evaluator.
 
-    :param value: Must be one of ["structured_output", "json", "keyword_search"].
+    :param value: Must be one of ["structured_output", "json", "keyword_search", "typed_questions"].
     :type value: str
     """
 
     allowed_values = {
         "structured_output",
         "json",
         "keyword_search",
+        "typed_questions",
     }
     STRUCTURED_OUTPUT: ClassVar["LLMObsCustomEvalConfigParsingType"]
     JSON: ClassVar["LLMObsCustomEvalConfigParsingType"]
     KEYWORD_SEARCH: ClassVar["LLMObsCustomEvalConfigParsingType"]
+    TYPED_QUESTIONS: ClassVar["LLMObsCustomEvalConfigParsingType"]
 
     @cached_property
     def openapi_types(_):
@@ -39,3 +41,4 @@ def openapi_types(_):
 LLMObsCustomEvalConfigParsingType.STRUCTURED_OUTPUT = LLMObsCustomEvalConfigParsingType("structured_output")
 LLMObsCustomEvalConfigParsingType.JSON = LLMObsCustomEvalConfigParsingType("json")
 LLMObsCustomEvalConfigParsingType.KEYWORD_SEARCH = LLMObsCustomEvalConfigParsingType("keyword_search")
+LLMObsCustomEvalConfigParsingType.TYPED_QUESTIONS = LLMObsCustomEvalConfigParsingType("typed_questions")

src/datadog_api_client/v2/model/llm_obs_custom_eval_config_target.py

@@ -64,7 +64,7 @@ def __init__(
         :param enabled: Whether the evaluator is active for the target application.
         :type enabled: bool
 
-        :param eval_scope: Scope at which to evaluate spans.
+        :param eval_scope: Scope at which to evaluate spans. ``trace_fan_out`` requires ``llm_judge_config.target_query``.
         :type eval_scope: LLMObsCustomEvalConfigEvalScope, optional
 
         :param experiment_project_ids: Experiment project IDs this evaluator is scoped to.