diff --git a/.generator/schemas/v2/openapi.yaml b/.generator/schemas/v2/openapi.yaml
index 8a63ddc4ce8..48e5fc20636 100644
--- a/.generator/schemas/v2/openapi.yaml
+++ b/.generator/schemas/v2/openapi.yaml
@@ -70997,16 +70997,18 @@ components:
- attributes
type: object
LLMObsCustomEvalConfigEvalScope:
- description: Scope at which to evaluate spans.
+ description: Scope at which to evaluate spans. `trace_fan_out` requires `llm_judge_config.target_query`.
enum:
- span
- trace
+ - trace_fan_out
- session
example: "span"
type: string
x-enum-varnames:
- SPAN
- TRACE
+ - TRACE_FAN_OUT
- SESSION
LLMObsCustomEvalConfigInferenceParams:
description: LLM inference parameters for a custom evaluator.
@@ -71043,14 +71045,20 @@ components:
type: number
type: object
LLMObsCustomEvalConfigIntegrationProvider:
- description: Name of the LLM integration provider.
+ description: >-
+ Name of the LLM integration provider. Use `deterministic` for a code evaluator, which runs `llm_judge_config.user_specified_json_post_processing_function` instead of calling an LLM. `insights_sponsored` is returned for evaluators that Datadog manages and cannot be set.
enum:
- openai
- - amazon-bedrock
+ - amazon_bedrock
- anthropic
- - azure-openai
- - vertex-ai
- - llm-proxy
+ - azure_openai
+ - vertex_ai
+ - llmproxy
+ - deterministic
+ - typesafe
+ - datadog
+ - datadog_ai_gateway
+ - insights_sponsored
example: "openai"
type: string
x-enum-varnames:
@@ -71060,6 +71068,11 @@ components:
- AZURE_OPENAI
- VERTEX_AI
- LLM_PROXY
+ - DETERMINISTIC
+ - TYPESAFE
+ - DATADOG
+ - DATADOG_AI_GATEWAY
+ - INSIGHTS_SPONSORED
LLMObsCustomEvalConfigLLMJudgeConfig:
description: LLM judge configuration for a custom evaluator.
properties:
@@ -71084,13 +71097,13 @@ components:
type: boolean
output_schema:
additionalProperties: {}
- description: JSON schema describing the expected output format of the LLM judge.
+ description: JSON schema describing the expected output format of the LLM judge. Omit for a `deterministic` evaluator.
nullable: true
type: object
parsing_type:
$ref: "#/components/schemas/LLMObsCustomEvalConfigParsingType"
prompt_template:
- description: List of messages forming the LLM judge prompt template.
+ description: List of messages forming the LLM judge prompt template. Omit for a `deterministic` evaluator.
items:
$ref: "#/components/schemas/LLMObsCustomEvalConfigPromptMessage"
type: array
@@ -71100,7 +71113,8 @@ components:
nullable: true
type: string
user_specified_json_post_processing_function:
- description: User-provided function applied to post-process the JSON output of the LLM judge.
+ description: >-
+ User-provided JavaScript function applied to post-process the JSON output of the LLM judge. For a `deterministic` evaluator it is the evaluator itself: it receives the evaluated span (span scope), `{spans}` (trace scope), or `{session_id, traces}` (session scope), and its return value sets the result. A boolean, number, string or object becomes a boolean, score, categorical or JSON metric; `null` skips the evaluation; and `{value, assessment, reasoning, metadata}` also sets pass or fail, reasoning, and metadata.
nullable: true
type: string
required:
@@ -71118,7 +71132,7 @@ components:
integration_provider:
$ref: "#/components/schemas/LLMObsCustomEvalConfigIntegrationProvider"
model_name:
- description: Name of the LLM model.
+ description: Name of the LLM model. Use `javascript` for a `deterministic` evaluator.
example: "gpt-4o"
type: string
vertex_ai:
@@ -71141,12 +71155,14 @@ components:
- structured_output
- json
- keyword_search
+ - typed_questions
example: "structured_output"
type: string
x-enum-varnames:
- STRUCTURED_OUTPUT
- JSON
- KEYWORD_SEARCH
+ - TYPED_QUESTIONS
LLMObsCustomEvalConfigPromptContent:
description: A content block within a prompt message.
properties:
diff --git a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigEvalScope.java b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigEvalScope.java
index 28f40873cca..5e58038a818 100644
--- a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigEvalScope.java
+++ b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigEvalScope.java
@@ -18,18 +18,23 @@
import java.util.HashSet;
import java.util.Set;
-/** Scope at which to evaluate spans. */
+/**
+ * Scope at which to evaluate spans. trace_fan_out requires
+ * llm_judge_config.target_query.
+ */
@JsonSerialize(
using = LLMObsCustomEvalConfigEvalScope.LLMObsCustomEvalConfigEvalScopeSerializer.class)
public class LLMObsCustomEvalConfigEvalScope extends ModelEnum {
private static final Set allowedValues =
- new HashSet(Arrays.asList("span", "trace", "session"));
+ new HashSet(Arrays.asList("span", "trace", "trace_fan_out", "session"));
public static final LLMObsCustomEvalConfigEvalScope SPAN =
new LLMObsCustomEvalConfigEvalScope("span");
public static final LLMObsCustomEvalConfigEvalScope TRACE =
new LLMObsCustomEvalConfigEvalScope("trace");
+ public static final LLMObsCustomEvalConfigEvalScope TRACE_FAN_OUT =
+ new LLMObsCustomEvalConfigEvalScope("trace_fan_out");
public static final LLMObsCustomEvalConfigEvalScope SESSION =
new LLMObsCustomEvalConfigEvalScope("session");
diff --git a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigIntegrationProvider.java b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigIntegrationProvider.java
index 97de6497a5d..497ccdaa477 100644
--- a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigIntegrationProvider.java
+++ b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigIntegrationProvider.java
@@ -18,7 +18,12 @@
import java.util.HashSet;
import java.util.Set;
-/** Name of the LLM integration provider. */
+/**
+ * Name of the LLM integration provider. Use deterministic for a code evaluator, which
+ * runs llm_judge_config.user_specified_json_post_processing_function instead of
+ * calling an LLM. insights_sponsored is returned for evaluators that Datadog manages
+ * and cannot be set.
+ */
@JsonSerialize(
using =
LLMObsCustomEvalConfigIntegrationProvider
@@ -28,20 +33,40 @@ public class LLMObsCustomEvalConfigIntegrationProvider extends ModelEnum
private static final Set allowedValues =
new HashSet(
Arrays.asList(
- "openai", "amazon-bedrock", "anthropic", "azure-openai", "vertex-ai", "llm-proxy"));
+ "openai",
+ "amazon_bedrock",
+ "anthropic",
+ "azure_openai",
+ "vertex_ai",
+ "llmproxy",
+ "deterministic",
+ "typesafe",
+ "datadog",
+ "datadog_ai_gateway",
+ "insights_sponsored"));
public static final LLMObsCustomEvalConfigIntegrationProvider OPENAI =
new LLMObsCustomEvalConfigIntegrationProvider("openai");
public static final LLMObsCustomEvalConfigIntegrationProvider AMAZON_BEDROCK =
- new LLMObsCustomEvalConfigIntegrationProvider("amazon-bedrock");
+ new LLMObsCustomEvalConfigIntegrationProvider("amazon_bedrock");
public static final LLMObsCustomEvalConfigIntegrationProvider ANTHROPIC =
new LLMObsCustomEvalConfigIntegrationProvider("anthropic");
public static final LLMObsCustomEvalConfigIntegrationProvider AZURE_OPENAI =
- new LLMObsCustomEvalConfigIntegrationProvider("azure-openai");
+ new LLMObsCustomEvalConfigIntegrationProvider("azure_openai");
public static final LLMObsCustomEvalConfigIntegrationProvider VERTEX_AI =
- new LLMObsCustomEvalConfigIntegrationProvider("vertex-ai");
+ new LLMObsCustomEvalConfigIntegrationProvider("vertex_ai");
public static final LLMObsCustomEvalConfigIntegrationProvider LLM_PROXY =
- new LLMObsCustomEvalConfigIntegrationProvider("llm-proxy");
+ new LLMObsCustomEvalConfigIntegrationProvider("llmproxy");
+ public static final LLMObsCustomEvalConfigIntegrationProvider DETERMINISTIC =
+ new LLMObsCustomEvalConfigIntegrationProvider("deterministic");
+ public static final LLMObsCustomEvalConfigIntegrationProvider TYPESAFE =
+ new LLMObsCustomEvalConfigIntegrationProvider("typesafe");
+ public static final LLMObsCustomEvalConfigIntegrationProvider DATADOG =
+ new LLMObsCustomEvalConfigIntegrationProvider("datadog");
+ public static final LLMObsCustomEvalConfigIntegrationProvider DATADOG_AI_GATEWAY =
+ new LLMObsCustomEvalConfigIntegrationProvider("datadog_ai_gateway");
+ public static final LLMObsCustomEvalConfigIntegrationProvider INSIGHTS_SPONSORED =
+ new LLMObsCustomEvalConfigIntegrationProvider("insights_sponsored");
LLMObsCustomEvalConfigIntegrationProvider(String value) {
super(value, allowedValues);
diff --git a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigLLMJudgeConfig.java b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigLLMJudgeConfig.java
index 1462aa0afcf..41f661add91 100644
--- a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigLLMJudgeConfig.java
+++ b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigLLMJudgeConfig.java
@@ -251,7 +251,8 @@ public LLMObsCustomEvalConfigLLMJudgeConfig putOutputSchemaItem(
}
/**
- * JSON schema describing the expected output format of the LLM judge.
+ * JSON schema describing the expected output format of the LLM judge. Omit for a
+ * deterministic evaluator.
*
* @return outputSchema
*/
@@ -324,7 +325,8 @@ public LLMObsCustomEvalConfigLLMJudgeConfig addPromptTemplateItem(
}
/**
- * List of messages forming the LLM judge prompt template.
+ * List of messages forming the LLM judge prompt template. Omit for a deterministic
+ * evaluator.
*
* @return promptTemplate
*/
@@ -383,7 +385,13 @@ public LLMObsCustomEvalConfigLLMJudgeConfig userSpecifiedJsonPostProcessingFunct
}
/**
- * User-provided function applied to post-process the JSON output of the LLM judge.
+ * User-provided JavaScript function applied to post-process the JSON output of the LLM judge. For
+ * a deterministic evaluator it is the evaluator itself: it receives the evaluated
+ * span (span scope), {spans} (trace scope), or {session_id, traces}
+ * (session scope), and its return value sets the result. A boolean, number, string or object
+ * becomes a boolean, score, categorical or JSON metric; null skips the evaluation;
+ * and {value, assessment, reasoning, metadata} also sets pass or fail, reasoning,
+ * and metadata.
*
* @return userSpecifiedJsonPostProcessingFunction
*/
diff --git a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigLLMProvider.java b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigLLMProvider.java
index 0c0b4e736b9..54489bd53f0 100644
--- a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigLLMProvider.java
+++ b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigLLMProvider.java
@@ -97,7 +97,10 @@ public LLMObsCustomEvalConfigLLMProvider integrationProvider(
}
/**
- * Name of the LLM integration provider.
+ * Name of the LLM integration provider. Use deterministic for a code evaluator,
+ * which runs llm_judge_config.user_specified_json_post_processing_function instead
+ * of calling an LLM. insights_sponsored is returned for evaluators that Datadog
+ * manages and cannot be set.
*
* @return integrationProvider
*/
@@ -122,7 +125,7 @@ public LLMObsCustomEvalConfigLLMProvider modelName(String modelName) {
}
/**
- * Name of the LLM model.
+ * Name of the LLM model. Use javascript for a deterministic evaluator.
*
* @return modelName
*/
diff --git a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigParsingType.java b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigParsingType.java
index 02480aeccf3..e374be3f4b6 100644
--- a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigParsingType.java
+++ b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigParsingType.java
@@ -24,7 +24,8 @@
public class LLMObsCustomEvalConfigParsingType extends ModelEnum {
private static final Set allowedValues =
- new HashSet(Arrays.asList("structured_output", "json", "keyword_search"));
+ new HashSet(
+ Arrays.asList("structured_output", "json", "keyword_search", "typed_questions"));
public static final LLMObsCustomEvalConfigParsingType STRUCTURED_OUTPUT =
new LLMObsCustomEvalConfigParsingType("structured_output");
@@ -32,6 +33,8 @@ public class LLMObsCustomEvalConfigParsingType extends ModelEnum {
new LLMObsCustomEvalConfigParsingType("json");
public static final LLMObsCustomEvalConfigParsingType KEYWORD_SEARCH =
new LLMObsCustomEvalConfigParsingType("keyword_search");
+ public static final LLMObsCustomEvalConfigParsingType TYPED_QUESTIONS =
+ new LLMObsCustomEvalConfigParsingType("typed_questions");
LLMObsCustomEvalConfigParsingType(String value) {
super(value, allowedValues);
diff --git a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigTarget.java b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigTarget.java
index 879f3c68dbe..2f3af4d6c08 100644
--- a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigTarget.java
+++ b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigTarget.java
@@ -113,7 +113,8 @@ public LLMObsCustomEvalConfigTarget evalScope(LLMObsCustomEvalConfigEvalScope ev
}
/**
- * Scope at which to evaluate spans.
+ * Scope at which to evaluate spans. trace_fan_out requires
+ * llm_judge_config.target_query.
*
* @return evalScope
*/