diff --git a/.generator/schemas/v2/openapi.yaml b/.generator/schemas/v2/openapi.yaml index 8a63ddc4ce8..48e5fc20636 100644 --- a/.generator/schemas/v2/openapi.yaml +++ b/.generator/schemas/v2/openapi.yaml @@ -70997,16 +70997,18 @@ components: - attributes type: object LLMObsCustomEvalConfigEvalScope: - description: Scope at which to evaluate spans. + description: Scope at which to evaluate spans. `trace_fan_out` requires `llm_judge_config.target_query`. enum: - span - trace + - trace_fan_out - session example: "span" type: string x-enum-varnames: - SPAN - TRACE + - TRACE_FAN_OUT - SESSION LLMObsCustomEvalConfigInferenceParams: description: LLM inference parameters for a custom evaluator. @@ -71043,14 +71045,20 @@ components: type: number type: object LLMObsCustomEvalConfigIntegrationProvider: - description: Name of the LLM integration provider. + description: >- + Name of the LLM integration provider. Use `deterministic` for a code evaluator, which runs `llm_judge_config.user_specified_json_post_processing_function` instead of calling an LLM. `insights_sponsored` is returned for evaluators that Datadog manages and cannot be set. enum: - openai - - amazon-bedrock + - amazon_bedrock - anthropic - - azure-openai - - vertex-ai - - llm-proxy + - azure_openai + - vertex_ai + - llmproxy + - deterministic + - typesafe + - datadog + - datadog_ai_gateway + - insights_sponsored example: "openai" type: string x-enum-varnames: @@ -71060,6 +71068,11 @@ components: - AZURE_OPENAI - VERTEX_AI - LLM_PROXY + - DETERMINISTIC + - TYPESAFE + - DATADOG + - DATADOG_AI_GATEWAY + - INSIGHTS_SPONSORED LLMObsCustomEvalConfigLLMJudgeConfig: description: LLM judge configuration for a custom evaluator. properties: @@ -71084,13 +71097,13 @@ components: type: boolean output_schema: additionalProperties: {} - description: JSON schema describing the expected output format of the LLM judge. + description: JSON schema describing the expected output format of the LLM judge. Omit for a `deterministic` evaluator. nullable: true type: object parsing_type: $ref: "#/components/schemas/LLMObsCustomEvalConfigParsingType" prompt_template: - description: List of messages forming the LLM judge prompt template. + description: List of messages forming the LLM judge prompt template. Omit for a `deterministic` evaluator. items: $ref: "#/components/schemas/LLMObsCustomEvalConfigPromptMessage" type: array @@ -71100,7 +71113,8 @@ components: nullable: true type: string user_specified_json_post_processing_function: - description: User-provided function applied to post-process the JSON output of the LLM judge. + description: >- + User-provided JavaScript function applied to post-process the JSON output of the LLM judge. For a `deterministic` evaluator it is the evaluator itself: it receives the evaluated span (span scope), `{spans}` (trace scope), or `{session_id, traces}` (session scope), and its return value sets the result. A boolean, number, string or object becomes a boolean, score, categorical or JSON metric; `null` skips the evaluation; and `{value, assessment, reasoning, metadata}` also sets pass or fail, reasoning, and metadata. nullable: true type: string required: @@ -71118,7 +71132,7 @@ components: integration_provider: $ref: "#/components/schemas/LLMObsCustomEvalConfigIntegrationProvider" model_name: - description: Name of the LLM model. + description: Name of the LLM model. Use `javascript` for a `deterministic` evaluator. example: "gpt-4o" type: string vertex_ai: @@ -71141,12 +71155,14 @@ components: - structured_output - json - keyword_search + - typed_questions example: "structured_output" type: string x-enum-varnames: - STRUCTURED_OUTPUT - JSON - KEYWORD_SEARCH + - TYPED_QUESTIONS LLMObsCustomEvalConfigPromptContent: description: A content block within a prompt message. properties: diff --git a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigEvalScope.java b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigEvalScope.java index 28f40873cca..5e58038a818 100644 --- a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigEvalScope.java +++ b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigEvalScope.java @@ -18,18 +18,23 @@ import java.util.HashSet; import java.util.Set; -/** Scope at which to evaluate spans. */ +/** + * Scope at which to evaluate spans. trace_fan_out requires + * llm_judge_config.target_query. + */ @JsonSerialize( using = LLMObsCustomEvalConfigEvalScope.LLMObsCustomEvalConfigEvalScopeSerializer.class) public class LLMObsCustomEvalConfigEvalScope extends ModelEnum { private static final Set allowedValues = - new HashSet(Arrays.asList("span", "trace", "session")); + new HashSet(Arrays.asList("span", "trace", "trace_fan_out", "session")); public static final LLMObsCustomEvalConfigEvalScope SPAN = new LLMObsCustomEvalConfigEvalScope("span"); public static final LLMObsCustomEvalConfigEvalScope TRACE = new LLMObsCustomEvalConfigEvalScope("trace"); + public static final LLMObsCustomEvalConfigEvalScope TRACE_FAN_OUT = + new LLMObsCustomEvalConfigEvalScope("trace_fan_out"); public static final LLMObsCustomEvalConfigEvalScope SESSION = new LLMObsCustomEvalConfigEvalScope("session"); diff --git a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigIntegrationProvider.java b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigIntegrationProvider.java index 97de6497a5d..497ccdaa477 100644 --- a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigIntegrationProvider.java +++ b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigIntegrationProvider.java @@ -18,7 +18,12 @@ import java.util.HashSet; import java.util.Set; -/** Name of the LLM integration provider. */ +/** + * Name of the LLM integration provider. Use deterministic for a code evaluator, which + * runs llm_judge_config.user_specified_json_post_processing_function instead of + * calling an LLM. insights_sponsored is returned for evaluators that Datadog manages + * and cannot be set. + */ @JsonSerialize( using = LLMObsCustomEvalConfigIntegrationProvider @@ -28,20 +33,40 @@ public class LLMObsCustomEvalConfigIntegrationProvider extends ModelEnum private static final Set allowedValues = new HashSet( Arrays.asList( - "openai", "amazon-bedrock", "anthropic", "azure-openai", "vertex-ai", "llm-proxy")); + "openai", + "amazon_bedrock", + "anthropic", + "azure_openai", + "vertex_ai", + "llmproxy", + "deterministic", + "typesafe", + "datadog", + "datadog_ai_gateway", + "insights_sponsored")); public static final LLMObsCustomEvalConfigIntegrationProvider OPENAI = new LLMObsCustomEvalConfigIntegrationProvider("openai"); public static final LLMObsCustomEvalConfigIntegrationProvider AMAZON_BEDROCK = - new LLMObsCustomEvalConfigIntegrationProvider("amazon-bedrock"); + new LLMObsCustomEvalConfigIntegrationProvider("amazon_bedrock"); public static final LLMObsCustomEvalConfigIntegrationProvider ANTHROPIC = new LLMObsCustomEvalConfigIntegrationProvider("anthropic"); public static final LLMObsCustomEvalConfigIntegrationProvider AZURE_OPENAI = - new LLMObsCustomEvalConfigIntegrationProvider("azure-openai"); + new LLMObsCustomEvalConfigIntegrationProvider("azure_openai"); public static final LLMObsCustomEvalConfigIntegrationProvider VERTEX_AI = - new LLMObsCustomEvalConfigIntegrationProvider("vertex-ai"); + new LLMObsCustomEvalConfigIntegrationProvider("vertex_ai"); public static final LLMObsCustomEvalConfigIntegrationProvider LLM_PROXY = - new LLMObsCustomEvalConfigIntegrationProvider("llm-proxy"); + new LLMObsCustomEvalConfigIntegrationProvider("llmproxy"); + public static final LLMObsCustomEvalConfigIntegrationProvider DETERMINISTIC = + new LLMObsCustomEvalConfigIntegrationProvider("deterministic"); + public static final LLMObsCustomEvalConfigIntegrationProvider TYPESAFE = + new LLMObsCustomEvalConfigIntegrationProvider("typesafe"); + public static final LLMObsCustomEvalConfigIntegrationProvider DATADOG = + new LLMObsCustomEvalConfigIntegrationProvider("datadog"); + public static final LLMObsCustomEvalConfigIntegrationProvider DATADOG_AI_GATEWAY = + new LLMObsCustomEvalConfigIntegrationProvider("datadog_ai_gateway"); + public static final LLMObsCustomEvalConfigIntegrationProvider INSIGHTS_SPONSORED = + new LLMObsCustomEvalConfigIntegrationProvider("insights_sponsored"); LLMObsCustomEvalConfigIntegrationProvider(String value) { super(value, allowedValues); diff --git a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigLLMJudgeConfig.java b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigLLMJudgeConfig.java index 1462aa0afcf..41f661add91 100644 --- a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigLLMJudgeConfig.java +++ b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigLLMJudgeConfig.java @@ -251,7 +251,8 @@ public LLMObsCustomEvalConfigLLMJudgeConfig putOutputSchemaItem( } /** - * JSON schema describing the expected output format of the LLM judge. + * JSON schema describing the expected output format of the LLM judge. Omit for a + * deterministic evaluator. * * @return outputSchema */ @@ -324,7 +325,8 @@ public LLMObsCustomEvalConfigLLMJudgeConfig addPromptTemplateItem( } /** - * List of messages forming the LLM judge prompt template. + * List of messages forming the LLM judge prompt template. Omit for a deterministic + * evaluator. * * @return promptTemplate */ @@ -383,7 +385,13 @@ public LLMObsCustomEvalConfigLLMJudgeConfig userSpecifiedJsonPostProcessingFunct } /** - * User-provided function applied to post-process the JSON output of the LLM judge. + * User-provided JavaScript function applied to post-process the JSON output of the LLM judge. For + * a deterministic evaluator it is the evaluator itself: it receives the evaluated + * span (span scope), {spans} (trace scope), or {session_id, traces} + * (session scope), and its return value sets the result. A boolean, number, string or object + * becomes a boolean, score, categorical or JSON metric; null skips the evaluation; + * and {value, assessment, reasoning, metadata} also sets pass or fail, reasoning, + * and metadata. * * @return userSpecifiedJsonPostProcessingFunction */ diff --git a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigLLMProvider.java b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigLLMProvider.java index 0c0b4e736b9..54489bd53f0 100644 --- a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigLLMProvider.java +++ b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigLLMProvider.java @@ -97,7 +97,10 @@ public LLMObsCustomEvalConfigLLMProvider integrationProvider( } /** - * Name of the LLM integration provider. + * Name of the LLM integration provider. Use deterministic for a code evaluator, + * which runs llm_judge_config.user_specified_json_post_processing_function instead + * of calling an LLM. insights_sponsored is returned for evaluators that Datadog + * manages and cannot be set. * * @return integrationProvider */ @@ -122,7 +125,7 @@ public LLMObsCustomEvalConfigLLMProvider modelName(String modelName) { } /** - * Name of the LLM model. + * Name of the LLM model. Use javascript for a deterministic evaluator. * * @return modelName */ diff --git a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigParsingType.java b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigParsingType.java index 02480aeccf3..e374be3f4b6 100644 --- a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigParsingType.java +++ b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigParsingType.java @@ -24,7 +24,8 @@ public class LLMObsCustomEvalConfigParsingType extends ModelEnum { private static final Set allowedValues = - new HashSet(Arrays.asList("structured_output", "json", "keyword_search")); + new HashSet( + Arrays.asList("structured_output", "json", "keyword_search", "typed_questions")); public static final LLMObsCustomEvalConfigParsingType STRUCTURED_OUTPUT = new LLMObsCustomEvalConfigParsingType("structured_output"); @@ -32,6 +33,8 @@ public class LLMObsCustomEvalConfigParsingType extends ModelEnum { new LLMObsCustomEvalConfigParsingType("json"); public static final LLMObsCustomEvalConfigParsingType KEYWORD_SEARCH = new LLMObsCustomEvalConfigParsingType("keyword_search"); + public static final LLMObsCustomEvalConfigParsingType TYPED_QUESTIONS = + new LLMObsCustomEvalConfigParsingType("typed_questions"); LLMObsCustomEvalConfigParsingType(String value) { super(value, allowedValues); diff --git a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigTarget.java b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigTarget.java index 879f3c68dbe..2f3af4d6c08 100644 --- a/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigTarget.java +++ b/src/main/java/com/datadog/api/client/v2/model/LLMObsCustomEvalConfigTarget.java @@ -113,7 +113,8 @@ public LLMObsCustomEvalConfigTarget evalScope(LLMObsCustomEvalConfigEvalScope ev } /** - * Scope at which to evaluate spans. + * Scope at which to evaluate spans. trace_fan_out requires + * llm_judge_config.target_query. * * @return evalScope */