diff --git a/.generator/schemas/v2/openapi.yaml b/.generator/schemas/v2/openapi.yaml index 8a63ddc4ce88..48e5fc20636b 100644 --- a/.generator/schemas/v2/openapi.yaml +++ b/.generator/schemas/v2/openapi.yaml @@ -70997,16 +70997,18 @@ components: - attributes type: object LLMObsCustomEvalConfigEvalScope: - description: Scope at which to evaluate spans. + description: Scope at which to evaluate spans. `trace_fan_out` requires `llm_judge_config.target_query`. enum: - span - trace + - trace_fan_out - session example: "span" type: string x-enum-varnames: - SPAN - TRACE + - TRACE_FAN_OUT - SESSION LLMObsCustomEvalConfigInferenceParams: description: LLM inference parameters for a custom evaluator. @@ -71043,14 +71045,20 @@ components: type: number type: object LLMObsCustomEvalConfigIntegrationProvider: - description: Name of the LLM integration provider. + description: >- + Name of the LLM integration provider. Use `deterministic` for a code evaluator, which runs `llm_judge_config.user_specified_json_post_processing_function` instead of calling an LLM. `insights_sponsored` is returned for evaluators that Datadog manages and cannot be set. enum: - openai - - amazon-bedrock + - amazon_bedrock - anthropic - - azure-openai - - vertex-ai - - llm-proxy + - azure_openai + - vertex_ai + - llmproxy + - deterministic + - typesafe + - datadog + - datadog_ai_gateway + - insights_sponsored example: "openai" type: string x-enum-varnames: @@ -71060,6 +71068,11 @@ components: - AZURE_OPENAI - VERTEX_AI - LLM_PROXY + - DETERMINISTIC + - TYPESAFE + - DATADOG + - DATADOG_AI_GATEWAY + - INSIGHTS_SPONSORED LLMObsCustomEvalConfigLLMJudgeConfig: description: LLM judge configuration for a custom evaluator. properties: @@ -71084,13 +71097,13 @@ components: type: boolean output_schema: additionalProperties: {} - description: JSON schema describing the expected output format of the LLM judge. + description: JSON schema describing the expected output format of the LLM judge. Omit for a `deterministic` evaluator. nullable: true type: object parsing_type: $ref: "#/components/schemas/LLMObsCustomEvalConfigParsingType" prompt_template: - description: List of messages forming the LLM judge prompt template. + description: List of messages forming the LLM judge prompt template. Omit for a `deterministic` evaluator. items: $ref: "#/components/schemas/LLMObsCustomEvalConfigPromptMessage" type: array @@ -71100,7 +71113,8 @@ components: nullable: true type: string user_specified_json_post_processing_function: - description: User-provided function applied to post-process the JSON output of the LLM judge. + description: >- + User-provided JavaScript function applied to post-process the JSON output of the LLM judge. For a `deterministic` evaluator it is the evaluator itself: it receives the evaluated span (span scope), `{spans}` (trace scope), or `{session_id, traces}` (session scope), and its return value sets the result. A boolean, number, string or object becomes a boolean, score, categorical or JSON metric; `null` skips the evaluation; and `{value, assessment, reasoning, metadata}` also sets pass or fail, reasoning, and metadata. nullable: true type: string required: @@ -71118,7 +71132,7 @@ components: integration_provider: $ref: "#/components/schemas/LLMObsCustomEvalConfigIntegrationProvider" model_name: - description: Name of the LLM model. + description: Name of the LLM model. Use `javascript` for a `deterministic` evaluator. example: "gpt-4o" type: string vertex_ai: @@ -71141,12 +71155,14 @@ components: - structured_output - json - keyword_search + - typed_questions example: "structured_output" type: string x-enum-varnames: - STRUCTURED_OUTPUT - JSON - KEYWORD_SEARCH + - TYPED_QUESTIONS LLMObsCustomEvalConfigPromptContent: description: A content block within a prompt message. properties: diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_eval_scope.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_eval_scope.rb index cc980bd70b46..34242c9b3957 100644 --- a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_eval_scope.rb +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_eval_scope.rb @@ -17,12 +17,13 @@ require 'time' module DatadogAPIClient::V2 - # Scope at which to evaluate spans. + # Scope at which to evaluate spans. `trace_fan_out` requires `llm_judge_config.target_query`. class LLMObsCustomEvalConfigEvalScope include BaseEnumModel SPAN = "span".freeze TRACE = "trace".freeze + TRACE_FAN_OUT = "trace_fan_out".freeze SESSION = "session".freeze end end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_integration_provider.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_integration_provider.rb index 865b9a0a748e..83e7dc40ffbe 100644 --- a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_integration_provider.rb +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_integration_provider.rb @@ -17,15 +17,20 @@ require 'time' module DatadogAPIClient::V2 - # Name of the LLM integration provider. + # Name of the LLM integration provider. Use `deterministic` for a code evaluator, which runs `llm_judge_config.user_specified_json_post_processing_function` instead of calling an LLM. `insights_sponsored` is returned for evaluators that Datadog manages and cannot be set. class LLMObsCustomEvalConfigIntegrationProvider include BaseEnumModel OPENAI = "openai".freeze - AMAZON_BEDROCK = "amazon-bedrock".freeze + AMAZON_BEDROCK = "amazon_bedrock".freeze ANTHROPIC = "anthropic".freeze - AZURE_OPENAI = "azure-openai".freeze - VERTEX_AI = "vertex-ai".freeze - LLM_PROXY = "llm-proxy".freeze + AZURE_OPENAI = "azure_openai".freeze + VERTEX_AI = "vertex_ai".freeze + LLM_PROXY = "llmproxy".freeze + DETERMINISTIC = "deterministic".freeze + TYPESAFE = "typesafe".freeze + DATADOG = "datadog".freeze + DATADOG_AI_GATEWAY = "datadog_ai_gateway".freeze + INSIGHTS_SPONSORED = "insights_sponsored".freeze end end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_llm_judge_config.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_llm_judge_config.rb index c0bac64f8d5c..c60372e4afed 100644 --- a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_llm_judge_config.rb +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_llm_judge_config.rb @@ -36,19 +36,19 @@ class LLMObsCustomEvalConfigLLMJudgeConfig # Whether the library prompt template was modified. attr_accessor :modified_library_prompt_template - # JSON schema describing the expected output format of the LLM judge. + # JSON schema describing the expected output format of the LLM judge. Omit for a `deterministic` evaluator. attr_accessor :output_schema # Output parsing type for a custom LLM judge evaluator. attr_accessor :parsing_type - # List of messages forming the LLM judge prompt template. + # List of messages forming the LLM judge prompt template. Omit for a `deterministic` evaluator. attr_accessor :prompt_template # Query used to extract the target value to evaluate. attr_accessor :target_query - # User-provided function applied to post-process the JSON output of the LLM judge. + # User-provided JavaScript function applied to post-process the JSON output of the LLM judge. For a `deterministic` evaluator it is the evaluator itself: it receives the evaluated span (span scope), `{spans}` (trace scope), or `{session_id, traces}` (session scope), and its return value sets the result. A boolean, number, string or object becomes a boolean, score, categorical or JSON metric; `null` skips the evaluation; and `{value, assessment, reasoning, metadata}` also sets pass or fail, reasoning, and metadata. attr_accessor :user_specified_json_post_processing_function attr_accessor :additional_properties diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_llm_provider.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_llm_provider.rb index 02db64c79e0c..8a15554a51e4 100644 --- a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_llm_provider.rb +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_llm_provider.rb @@ -27,10 +27,10 @@ class LLMObsCustomEvalConfigLLMProvider # Integration account identifier. attr_accessor :integration_account_id - # Name of the LLM integration provider. + # Name of the LLM integration provider. Use `deterministic` for a code evaluator, which runs `llm_judge_config.user_specified_json_post_processing_function` instead of calling an LLM. `insights_sponsored` is returned for evaluators that Datadog manages and cannot be set. attr_accessor :integration_provider - # Name of the LLM model. + # Name of the LLM model. Use `javascript` for a `deterministic` evaluator. attr_accessor :model_name # Google Vertex AI-specific options for LLM provider configuration. diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_parsing_type.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_parsing_type.rb index 733e51567317..329259f652fa 100644 --- a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_parsing_type.rb +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_parsing_type.rb @@ -24,5 +24,6 @@ class LLMObsCustomEvalConfigParsingType STRUCTURED_OUTPUT = "structured_output".freeze JSON = "json".freeze KEYWORD_SEARCH = "keyword_search".freeze + TYPED_QUESTIONS = "typed_questions".freeze end end diff --git a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_target.rb b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_target.rb index 10792ac1de14..980a5699bc72 100644 --- a/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_target.rb +++ b/lib/datadog_api_client/v2/models/llm_obs_custom_eval_config_target.rb @@ -27,7 +27,7 @@ class LLMObsCustomEvalConfigTarget # Whether the evaluator is active for the target application. attr_reader :enabled - # Scope at which to evaluate spans. + # Scope at which to evaluate spans. `trace_fan_out` requires `llm_judge_config.target_query`. attr_accessor :eval_scope # Experiment project IDs this evaluator is scoped to.