Fix LLM Obs custom evaluator provider names and add deterministic provider

.generator/schemas/v2/openapi.yaml

@@ -70997,16 +70997,18 @@ components:
         - attributes
       type: object
     LLMObsCustomEvalConfigEvalScope:
-      description: Scope at which to evaluate spans.
+      description: Scope at which to evaluate spans. `trace_fan_out` requires `llm_judge_config.target_query`.
       enum:
         - span
         - trace
+        - trace_fan_out
         - session
       example: "span"
       type: string
       x-enum-varnames:
         - SPAN
         - TRACE
+        - TRACE_FAN_OUT
         - SESSION
     LLMObsCustomEvalConfigInferenceParams:
       description: LLM inference parameters for a custom evaluator.
@@ -71043,14 +71045,20 @@ components:
           type: number
       type: object
     LLMObsCustomEvalConfigIntegrationProvider:
-      description: Name of the LLM integration provider.
+      description: >-
+        Name of the LLM integration provider. Use `deterministic` for a code evaluator, which runs `llm_judge_config.user_specified_json_post_processing_function` instead of calling an LLM. `insights_sponsored` is returned for evaluators that Datadog manages and cannot be set.
       enum:
         - openai
-        - amazon-bedrock
+        - amazon_bedrock
         - anthropic
-        - azure-openai
-        - vertex-ai
-        - llm-proxy
+        - azure_openai
+        - vertex_ai
+        - llmproxy
+        - deterministic
+        - typesafe
+        - datadog
+        - datadog_ai_gateway
+        - insights_sponsored
       example: "openai"
       type: string
       x-enum-varnames:
@@ -71060,6 +71068,11 @@ components:
         - AZURE_OPENAI
         - VERTEX_AI
         - LLM_PROXY
+        - DETERMINISTIC
+        - TYPESAFE
+        - DATADOG
+        - DATADOG_AI_GATEWAY
+        - INSIGHTS_SPONSORED
     LLMObsCustomEvalConfigLLMJudgeConfig:
       description: LLM judge configuration for a custom evaluator.
       properties:
@@ -71084,13 +71097,13 @@ components:
           type: boolean
         output_schema:
           additionalProperties: {}
-          description: JSON schema describing the expected output format of the LLM judge.
+          description: JSON schema describing the expected output format of the LLM judge. Omit for a `deterministic` evaluator.
           nullable: true
           type: object
         parsing_type:
           $ref: "#/components/schemas/LLMObsCustomEvalConfigParsingType"
         prompt_template:
-          description: List of messages forming the LLM judge prompt template.
+          description: List of messages forming the LLM judge prompt template. Omit for a `deterministic` evaluator.
           items:
             $ref: "#/components/schemas/LLMObsCustomEvalConfigPromptMessage"
           type: array
@@ -71100,7 +71113,8 @@ components:
           nullable: true
           type: string
         user_specified_json_post_processing_function:
-          description: User-provided function applied to post-process the JSON output of the LLM judge.
+          description: >-
+            User-provided JavaScript function applied to post-process the JSON output of the LLM judge. For a `deterministic` evaluator it is the evaluator itself: it receives the evaluated span (span scope), `{spans}` (trace scope), or `{session_id, traces}` (session scope), and its return value sets the result. A boolean, number, string or object becomes a boolean, score, categorical or JSON metric; `null` skips the evaluation; and `{value, assessment, reasoning, metadata}` also sets pass or fail, reasoning, and metadata.
           nullable: true
           type: string
       required:
@@ -71118,7 +71132,7 @@ components:
         integration_provider:
           $ref: "#/components/schemas/LLMObsCustomEvalConfigIntegrationProvider"
         model_name:
-          description: Name of the LLM model.
+          description: Name of the LLM model. Use `javascript` for a `deterministic` evaluator.
           example: "gpt-4o"
           type: string
         vertex_ai:
@@ -71141,12 +71155,14 @@ components:
         - structured_output
         - json
         - keyword_search
+        - typed_questions
       example: "structured_output"
       type: string
       x-enum-varnames:
         - STRUCTURED_OUTPUT
         - JSON
         - KEYWORD_SEARCH
+        - TYPED_QUESTIONS
     LLMObsCustomEvalConfigPromptContent:
       description: A content block within a prompt message.
       properties:

src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigEvalScope.java

@@ -18,18 +18,23 @@
 import java.util.HashSet;
 import java.util.Set;
 
-/** Scope at which to evaluate spans. */
+/**
+ * Scope at which to evaluate spans. <code>trace_fan_out</code> requires <code>
+ * llm_judge_config.target_query</code>.
+ */
 @JsonSerialize(
     using = LLMObsCustomEvalConfigEvalScope.LLMObsCustomEvalConfigEvalScopeSerializer.class)
 public class LLMObsCustomEvalConfigEvalScope extends ModelEnum<String> {
 
   private static final Set<String> allowedValues =
-      new HashSet<String>(Arrays.asList("span", "trace", "session"));
+      new HashSet<String>(Arrays.asList("span", "trace", "trace_fan_out", "session"));
 
   public static final LLMObsCustomEvalConfigEvalScope SPAN =
       new LLMObsCustomEvalConfigEvalScope("span");
   public static final LLMObsCustomEvalConfigEvalScope TRACE =
       new LLMObsCustomEvalConfigEvalScope("trace");
+  public static final LLMObsCustomEvalConfigEvalScope TRACE_FAN_OUT =
+      new LLMObsCustomEvalConfigEvalScope("trace_fan_out");
   public static final LLMObsCustomEvalConfigEvalScope SESSION =
       new LLMObsCustomEvalConfigEvalScope("session");
 

src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigIntegrationProvider.java

@@ -18,7 +18,12 @@
 import java.util.HashSet;
 import java.util.Set;
 
-/** Name of the LLM integration provider. */
+/**
+ * Name of the LLM integration provider. Use <code>deterministic</code> for a code evaluator, which
+ * runs <code>llm_judge_config.user_specified_json_post_processing_function</code> instead of
+ * calling an LLM. <code>insights_sponsored</code> is returned for evaluators that Datadog manages
+ * and cannot be set.
+ */
 @JsonSerialize(
     using =
         LLMObsCustomEvalConfigIntegrationProvider
@@ -28,20 +33,40 @@ public class LLMObsCustomEvalConfigIntegrationProvider extends ModelEnum<String>
   private static final Set<String> allowedValues =
       new HashSet<String>(
           Arrays.asList(
-              "openai", "amazon-bedrock", "anthropic", "azure-openai", "vertex-ai", "llm-proxy"));
+              "openai",
+              "amazon_bedrock",
+              "anthropic",
+              "azure_openai",
+              "vertex_ai",
+              "llmproxy",
+              "deterministic",
+              "typesafe",
+              "datadog",
+              "datadog_ai_gateway",
+              "insights_sponsored"));
 
   public static final LLMObsCustomEvalConfigIntegrationProvider OPENAI =
       new LLMObsCustomEvalConfigIntegrationProvider("openai");
   public static final LLMObsCustomEvalConfigIntegrationProvider AMAZON_BEDROCK =
-      new LLMObsCustomEvalConfigIntegrationProvider("amazon-bedrock");
+      new LLMObsCustomEvalConfigIntegrationProvider("amazon_bedrock");
   public static final LLMObsCustomEvalConfigIntegrationProvider ANTHROPIC =
       new LLMObsCustomEvalConfigIntegrationProvider("anthropic");
   public static final LLMObsCustomEvalConfigIntegrationProvider AZURE_OPENAI =
-      new LLMObsCustomEvalConfigIntegrationProvider("azure-openai");
+      new LLMObsCustomEvalConfigIntegrationProvider("azure_openai");
   public static final LLMObsCustomEvalConfigIntegrationProvider VERTEX_AI =
-      new LLMObsCustomEvalConfigIntegrationProvider("vertex-ai");
+      new LLMObsCustomEvalConfigIntegrationProvider("vertex_ai");
   public static final LLMObsCustomEvalConfigIntegrationProvider LLM_PROXY =
-      new LLMObsCustomEvalConfigIntegrationProvider("llm-proxy");
+      new LLMObsCustomEvalConfigIntegrationProvider("llmproxy");
+  public static final LLMObsCustomEvalConfigIntegrationProvider DETERMINISTIC =
+      new LLMObsCustomEvalConfigIntegrationProvider("deterministic");
+  public static final LLMObsCustomEvalConfigIntegrationProvider TYPESAFE =
+      new LLMObsCustomEvalConfigIntegrationProvider("typesafe");
+  public static final LLMObsCustomEvalConfigIntegrationProvider DATADOG =
+      new LLMObsCustomEvalConfigIntegrationProvider("datadog");
+  public static final LLMObsCustomEvalConfigIntegrationProvider DATADOG_AI_GATEWAY =
+      new LLMObsCustomEvalConfigIntegrationProvider("datadog_ai_gateway");
+  public static final LLMObsCustomEvalConfigIntegrationProvider INSIGHTS_SPONSORED =
+      new LLMObsCustomEvalConfigIntegrationProvider("insights_sponsored");
 
   LLMObsCustomEvalConfigIntegrationProvider(String value) {
     super(value, allowedValues);

src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigLLMJudgeConfig.java

@@ -251,7 +251,8 @@ public LLMObsCustomEvalConfigLLMJudgeConfig putOutputSchemaItem(
   }
 
   /**
-   * JSON schema describing the expected output format of the LLM judge.
+   * JSON schema describing the expected output format of the LLM judge. Omit for a <code>
+   * deterministic</code> evaluator.
    *
    * @return outputSchema
    */
@@ -324,7 +325,8 @@ public LLMObsCustomEvalConfigLLMJudgeConfig addPromptTemplateItem(
   }
 
   /**
-   * List of messages forming the LLM judge prompt template.
+   * List of messages forming the LLM judge prompt template. Omit for a <code>deterministic</code>
+   * evaluator.
    *
    * @return promptTemplate
    */
@@ -383,7 +385,13 @@ public LLMObsCustomEvalConfigLLMJudgeConfig userSpecifiedJsonPostProcessingFunct
   }
 
   /**
-   * User-provided function applied to post-process the JSON output of the LLM judge.
+   * User-provided JavaScript function applied to post-process the JSON output of the LLM judge. For
+   * a <code>deterministic</code> evaluator it is the evaluator itself: it receives the evaluated
+   * span (span scope), <code>{spans}</code> (trace scope), or <code>{session_id, traces}</code>
+   * (session scope), and its return value sets the result. A boolean, number, string or object
+   * becomes a boolean, score, categorical or JSON metric; <code>null</code> skips the evaluation;
+   * and <code>{value, assessment, reasoning, metadata}</code> also sets pass or fail, reasoning,
+   * and metadata.
    *
    * @return userSpecifiedJsonPostProcessingFunction
    */

src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigLLMProvider.java

@@ -97,7 +97,10 @@ public LLMObsCustomEvalConfigLLMProvider integrationProvider(
   }
 
   /**
-   * Name of the LLM integration provider.
+   * Name of the LLM integration provider. Use <code>deterministic</code> for a code evaluator,
+   * which runs <code>llm_judge_config.user_specified_json_post_processing_function</code> instead
+   * of calling an LLM. <code>insights_sponsored</code> is returned for evaluators that Datadog
+   * manages and cannot be set.
    *
    * @return integrationProvider
    */
@@ -122,7 +125,7 @@ public LLMObsCustomEvalConfigLLMProvider modelName(String modelName) {
   }
 
   /**
-   * Name of the LLM model.
+   * Name of the LLM model. Use <code>javascript</code> for a <code>deterministic</code> evaluator.
    *
    * @return modelName
    */

src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigParsingType.java

@@ -24,14 +24,17 @@
 public class LLMObsCustomEvalConfigParsingType extends ModelEnum<String> {
 
   private static final Set<String> allowedValues =
-      new HashSet<String>(Arrays.asList("structured_output", "json", "keyword_search"));
+      new HashSet<String>(
+          Arrays.asList("structured_output", "json", "keyword_search", "typed_questions"));
 
   public static final LLMObsCustomEvalConfigParsingType STRUCTURED_OUTPUT =
       new LLMObsCustomEvalConfigParsingType("structured_output");
   public static final LLMObsCustomEvalConfigParsingType JSON =
       new LLMObsCustomEvalConfigParsingType("json");
   public static final LLMObsCustomEvalConfigParsingType KEYWORD_SEARCH =
       new LLMObsCustomEvalConfigParsingType("keyword_search");
+  public static final LLMObsCustomEvalConfigParsingType TYPED_QUESTIONS =
+      new LLMObsCustomEvalConfigParsingType("typed_questions");
 
   LLMObsCustomEvalConfigParsingType(String value) {
     super(value, allowedValues);

src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigTarget.java

@@ -113,7 +113,8 @@ public LLMObsCustomEvalConfigTarget evalScope(LLMObsCustomEvalConfigEvalScope ev
   }
 
   /**
-   * Scope at which to evaluate spans.
+   * Scope at which to evaluate spans. <code>trace_fan_out</code> requires <code>
+   * llm_judge_config.target_query</code>.
    *
    * @return evalScope
    */