From 1e19a1634b81f20516d9c2147f6bd0315269c123 Mon Sep 17 00:00:00 2001 From: Adriana Knight Date: Thu, 17 Sep 2026 11:53:15 -0400 Subject: [PATCH 1/5] Support passing in datasets via SDK --- .../evaluations/events.py | 4 +- .../evaluations/module.py | 149 ++- .../evaluations/runner.py | 176 +++- .../evaluations/types.py | 12 +- packages/client/tests/test_evaluations_run.py | 918 ++++++++++++++++++ 5 files changed, 1186 insertions(+), 73 deletions(-) diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/events.py b/packages/client/src/launchdarkly_ai_server/evaluations/events.py index 4098e61b..79be8c0b 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/events.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/events.py @@ -37,15 +37,15 @@ class CriterionEventPayload: evaluation_id: str evaluation_run_id: str run_id: str - dataset_id: str row_index: int criterion_type: str kind: CriterionEventKind event_id: str emitted_at: str evaluation_key: str - dataset_key: str status: CriterionStatus + dataset_id: str | None = None + dataset_key: str | None = None started_at: str evaluated_at: str latency_ms: int diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index 80d22ece..12017c0f 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -2,6 +2,7 @@ import asyncio import inspect +import json import logging import math import os @@ -22,10 +23,17 @@ EvalHandler, EvaluationsRunner, ToolImplementation, + _normalize_inline_rows, _provides_for, _segment, ) -from .types import EvalRunResult, GenerationConfig, RunSummary +from .types import ( + DatasetRef, + DatasetRow, + EvalRunResult, + GenerationConfig, + RunSummary, +) logger = logging.getLogger(__name__) @@ -54,6 +62,15 @@ def _can_emit_events(client: Any) -> bool: ) +def _require_json_serializable(value: Any, description: str) -> None: + try: + json.dumps(value, allow_nan=False) + except (TypeError, ValueError) as error: + raise EvaluationsError( + f"{description} must be JSON-serializable: {error}" + ) from error + + def _is_terminal_summary(summary: RunSummary) -> bool: accounted_rows = summary.passed_rows + summary.failed_rows + summary.error_rows return ( @@ -96,7 +113,8 @@ async def run( *, project_key: str, key: str, - dataset: str, + dataset: str | None = None, + rows: list[DatasetRow] | None = None, handler: EvalHandler, generation: GenerationConfig, tools: Mapping[str, ToolImplementation] | None = None, @@ -109,11 +127,16 @@ async def run( """ Create and run an evaluation in the caller's process. - Each dataset row is generated with ``handler``; every entry in - ``criteria`` — LaunchDarkly :class:`Judge` references and local - deterministic :class:`Scorer` functions — is then run against each - generated row, and one evaluation event is emitted per - ``(row, criterion)`` result. + Rows come from exactly one of two sources. Pass ``dataset`` — the *key* + of a LaunchDarkly-hosted dataset — to have the harness read its rows, + or pass ``rows`` to supply them from code, in which case no dataset is + read or referenced at all. Supplying both, or neither, is an error + raised before any network I/O. + + Each row is generated with ``handler``; every entry in ``criteria`` — + LaunchDarkly :class:`Judge` references and local deterministic + :class:`Scorer` functions — is then run against each generated row, and + one evaluation event is emitted per ``(row, criterion)`` result. A :class:`Judge` is an independent AI Config and may be served by a different provider or mode than ``generation``. ``handler`` runs a judge @@ -135,6 +158,7 @@ async def run( project_key=project_key, key=key, dataset=dataset, + rows=rows, handler=handler, generation=generation, concurrency=concurrency, @@ -160,12 +184,17 @@ async def run( resolved_judges = await self._runner._resolve_judges( project_key, ld_judges, handler, run_judge_handlers ) - dataset_ref = await asyncio.to_thread( - self._runner._fetch_dataset, project_key, dataset - ) - rows = await asyncio.to_thread( - self._runner._get_dataset_rows, project_key, dataset - ) + dataset_ref: DatasetRef | None + if dataset is not None: + dataset_ref = await asyncio.to_thread( + self._runner._fetch_dataset, project_key, dataset + ) + run_rows = await asyncio.to_thread( + self._runner._get_dataset_rows, project_key, dataset + ) + else: + dataset_ref = None + run_rows = await asyncio.to_thread(_normalize_inline_rows, rows or []) evaluation = await asyncio.to_thread( self._runner._create_evaluation, project_key, @@ -178,11 +207,11 @@ async def run( self._runner._create_evaluation_run, project_key, evaluation.id, - dataset_ref.id, + dataset_ref.id if dataset_ref is not None else None, ) config = self._runner._build_handler_config(generation, resolved_tools) results = await self._runner._run_rows( - rows, + run_rows, handler, config, run_tools, @@ -348,12 +377,83 @@ def _validate_judge_handlers(judge_handlers: list[EvalHandler]) -> None: "judge's provider and mode." ) + @staticmethod + def _validate_rows(rows: list[DatasetRow]) -> None: + """ + Check caller-supplied rows before any record exists. + + Stricter than the hosted path on purpose. ``_row_from_api_item`` coerces + bad server data rather than failing a run already in flight; a bad inline + row is a caller bug, and every check here runs with zero requests issued + — where the hosted empty-dataset rule only fires after two GETs. + """ + if not rows: + raise EvaluationsError("rows must not be empty") + seen: dict[int, int] = {} + for index, row in enumerate(rows): + if not isinstance(row, DatasetRow): + raise EvaluationsError( + f"rows[{index}] must be a DatasetRow, got {type(row).__name__}" + ) + row_index = row.row_index + # bool is an int subclass, so True would otherwise pass as 1 and go + # on the wire as `rowIndex: true`, which ingest drops. + if ( + isinstance(row_index, bool) + or not isinstance(row_index, int) + or row_index < 0 + ): + raise EvaluationsError( + f"rows[{index}].row_index must be a non-negative integer, " + f"got {row_index!r}" + ) + if row_index in seen: + raise EvaluationsError( + f"rows[{index}].row_index {row_index} duplicates " + f"rows[{seen[row_index]}]. LaunchDarkly identifies an inline " + "row by (run, row_index), so duplicates collapse into one " + "stored row and the run never accounts for every row." + ) + seen[row_index] = index + for name, value in ( + ("input", row.input), + ("expected_output", row.expected_output), + ): + # A hosted row renders a non-string as None (it is the API's data + # to tolerate); inline, that would silently run the whole + # evaluation on empty inputs at full generation cost. + if value is not None and not isinstance(value, str): + raise EvaluationsError( + f"rows[{index}].{name} must be a string or None, got " + f"{type(value).__name__}" + ) + for field_name, mapping_value in ( + ("variables", row.variables), + ("metadata", row.metadata), + ): + if mapping_value is None: + continue + if not isinstance(mapping_value, Mapping): + raise EvaluationsError( + f"rows[{index}].{field_name} must be a mapping, got " + f"{type(mapping_value).__name__}" + ) + # Serializability is checked on the copy, not the caller's + # container, because the copy is what reaches the wire -- any + # Mapping is accepted here and normalized to a dict. The copy is + # shallow in both places, so a nested mapping the serializer + # cannot encode is correctly still an error. + _require_json_serializable( + dict(mapping_value), f"rows[{index}].{field_name}" + ) + @staticmethod def _validate_run_args( *, project_key: str, key: str, - dataset: str, + dataset: str | None, + rows: list[DatasetRow] | None, handler: EvalHandler, generation: GenerationConfig, concurrency: int, @@ -363,10 +463,23 @@ def _validate_run_args( for name, value in ( ("project_key", project_key), ("key", key), - ("dataset", dataset), ): - if not value.strip(): + if not isinstance(value, str) or not value.strip(): raise EvaluationsError(f"{name} must not be blank") + if dataset is None and rows is None: + raise EvaluationsError( + "one of dataset or rows is required: pass dataset to read a " + "LaunchDarkly-hosted dataset, or rows to supply them from code" + ) + if dataset is not None and rows is not None: + raise EvaluationsError( + "dataset and rows are mutually exclusive: pass exactly one" + ) + if dataset is not None: + if not isinstance(dataset, str) or not dataset.strip(): + raise EvaluationsError("dataset must not be blank") + else: + EvaluationsModule._validate_rows(rows or []) if not callable(handler): raise EvaluationsError("handler must be callable") provider = generation.get("provider") diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index 164ea57d..b4a3bad9 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -178,6 +178,102 @@ def _required_string(data: Mapping[str, Any], key: str, description: str) -> str return value +def _render_row( + *, + row_index: int, + input_value: str | None, + expected_value: str | None, + variables: dict[str, Any], + metadata: dict[str, Any] | None, +) -> DatasetRow: + """ + Render one row's templates and inject the rendered values into its variables. + + The single render step behind both row sources -- an LD-hosted dataset and + rows the caller supplied to ``run(rows=[...])``. Forking it would fork every + downstream assertion about ``variables``, since ``_judge_variables`` and a + criterion's ``ground_truth_context`` both read the injected keys. + + ``input`` and ``expected_output`` are the exact injected key names, and are + a cross-language contract: dataset-authored templates reference + ``{{input}}`` and ``{{expected_output}}``, so neither is camelCased. + + ``variables`` and ``metadata`` must already be copies owned by the caller of + this function -- both front-ends copy before calling, so rendering never + writes back into an object the SDK's caller still holds. + """ + rendered_input = ( + parse_template(input_value, variables) if input_value is not None else None + ) + rendered_expected = ( + parse_template(expected_value, variables) + if expected_value is not None + else None + ) + variables["input"] = rendered_input + variables["expected_output"] = rendered_expected + return DatasetRow( + row_index=row_index, + input=rendered_input, + expected_output=rendered_expected, + variables=variables, + metadata=metadata, + ) + + +def _row_from_api_item(item: Mapping[str, Any]) -> DatasetRow: + """ + Normalize one row of an LD-hosted dataset page. + + This front-end *coerces* bad values rather than rejecting them: the data is + the API's, not the caller's, so a missing or non-string ``input`` renders as + ``None`` instead of failing a run that is already in flight. Inline rows are + held to the stricter standard in ``_validate_run_args``. + """ + row_index = item.get("rowIndex") + if not isinstance(row_index, int): + raise EvaluationsError("A dataset row is missing its integer rowIndex") + variables_value = item.get("variables") + input_value = item.get("input") + expected_value = item.get("expectedOutput") + metadata_value = item.get("metadata") + return _render_row( + row_index=row_index, + input_value=input_value if isinstance(input_value, str) else None, + expected_value=expected_value if isinstance(expected_value, str) else None, + variables=dict(variables_value) if isinstance(variables_value, Mapping) else {}, + metadata=dict(metadata_value) if isinstance(metadata_value, Mapping) else None, + ) + + +def _normalize_inline_rows(rows: list[DatasetRow]) -> list[DatasetRow]: + """ + Normalize rows the caller supplied to ``run(rows=[...])``. + + Shape and serializability were already checked by ``_validate_run_args``, + so this front-end only copies and renders. Both copies are load-bearing: + + * ``dict(row.variables)`` because ``parse_template`` resolves a placeholder + via ``isinstance(value, dict)`` -- hand it a ``MappingProxyType``, a + ``ChainMap``, or any other ``Mapping`` and *every* placeholder is left + literal, sending raw mustache text to the model and the judge prompt. + * fresh ``DatasetRow`` objects because the dataclass is mutable and the + caller still holds theirs. Writing the injected keys back would mean a CI + script that runs the same row list twice resolves ``{{input}}`` against + the first run's already-rendered value. + """ + return [ + _render_row( + row_index=row.row_index, + input_value=row.input, + expected_value=row.expected_output, + variables=dict(row.variables), + metadata=dict(row.metadata) if row.metadata is not None else None, + ) + for row in rows + ] + + class ConcurrencyController: """Owns row-worker permits.""" @@ -372,45 +468,8 @@ def _get_dataset_rows(self, project_key: str, dataset_key: str) -> list[DatasetR if not items: break for item_value in items: - item = _mapping(item_value, description="dataset row") - row_index = item.get("rowIndex") - if not isinstance(row_index, int): - raise EvaluationsError( - "A dataset row is missing its integer rowIndex" - ) - variables_value = item.get("variables") - variables = ( - dict(variables_value) - if isinstance(variables_value, Mapping) - else {} - ) - input_value = item.get("input") - expected_value = item.get("expectedOutput") - rendered_input = ( - parse_template(input_value, variables) - if isinstance(input_value, str) - else None - ) - rendered_expected = ( - parse_template(expected_value, variables) - if isinstance(expected_value, str) - else None - ) - variables["input"] = rendered_input - variables["expected_output"] = rendered_expected - metadata_value = item.get("metadata") rows.append( - DatasetRow( - row_index=row_index, - input=rendered_input, - expected_output=rendered_expected, - variables=variables, - metadata=( - dict(metadata_value) - if isinstance(metadata_value, Mapping) - else None - ), - ) + _row_from_api_item(_mapping(item_value, description="dataset row")) ) offset += len(items) if not rows: @@ -468,16 +527,17 @@ def _create_evaluation_run( self, project_key: str, evaluation_id: str, - dataset_id: str, + dataset_id: str | None = None, ) -> EvaluationRunRef: path = ( f"projects/{_segment(project_key)}/evaluations/" f"{_segment(evaluation_id)}/runs" ) - body: dict[str, Any] = { - "source": "api", - "datasetId": dataset_id, - } + # No row count in either mode: with criteria a row is not accounted for + # by a single result, so what completes a row is derived server-side. + body: dict[str, Any] = {"source": "api"} + if dataset_id is not None: + body["datasetId"] = dataset_id raw = _mapping( self._api.post(path, body=body), description="evaluation run", @@ -596,7 +656,7 @@ def _emit_generation_events( project_key: str, evaluation: EvaluationRef, evaluation_run: EvaluationRunRef, - dataset: DatasetRef, + dataset: DatasetRef | None, results: list[dict[str, Any]], ) -> None: """Queue one LD custom event for each executed dataset row.""" @@ -610,14 +670,15 @@ def _emit_generation_events( }, ) for result in results: - identity = { + identity: dict[str, Any] = { "projectKey": project_key, "evaluationId": evaluation.id, "evaluationRunId": evaluation_run.id, "runId": evaluation_run.id, - "datasetId": dataset.id, "rowIndex": result["row_index"], } + if dataset is not None: + identity["datasetId"] = dataset.id event_id = hashlib.sha256( json.dumps(identity, sort_keys=True, separators=(",", ":")).encode() ).hexdigest() @@ -655,12 +716,22 @@ def _emit_generation_events( "emittedAt": emitted_at, "evaluationKey": evaluation.key, "evaluationVersion": evaluation.version, - "datasetKey": dataset.key, "status": result["status"], "startedAt": result["started_at"], "generatedAt": result["generated_at"], "latencyMs": result["latency_ms"], } + if dataset is not None: + payload["datasetKey"] = dataset.key + else: + if result.get("input") is not None: + payload["input"] = result["input"] + if result.get("expected_output") is not None: + payload["expectedOutput"] = result["expected_output"] + if result.get("variables"): + payload["variables"] = result["variables"] + if result.get("metadata"): + payload["metadata"] = result["metadata"] if generated["output"] is not None: payload["output"] = generated["output"] if generated["error"] is not None: @@ -940,7 +1011,7 @@ def _emit_evaluation_events( project_key: str, evaluation: EvaluationRef, evaluation_run: EvaluationRunRef, - dataset: DatasetRef, + dataset: DatasetRef | None, results: list[dict[str, Any]], ) -> None: context = to_ld_context( @@ -959,15 +1030,16 @@ def _emit_evaluation_events( # before it -- so every result is attempted and the failures are # raised together once the loop is done. try: - identity = { + identity: dict[str, Any] = { "projectKey": project_key, "evaluationId": evaluation.id, "evaluationRunId": evaluation_run.id, "runId": evaluation_run.id, - "datasetId": dataset.id, "rowIndex": result["row_index"], "criterionType": result["criterion_type"], } + if dataset is not None: + identity["datasetId"] = dataset.id event_id = hashlib.sha256( json.dumps(identity, sort_keys=True, separators=(",", ":")).encode() ).hexdigest() @@ -993,14 +1065,14 @@ def _emit_evaluation_events( "evaluation_id": evaluation.id, "evaluation_run_id": evaluation_run.id, "run_id": evaluation_run.id, - "dataset_id": dataset.id, + "dataset_id": dataset.id if dataset else None, "row_index": result["row_index"], "criterion_type": result["criterion_type"], "event_id": event_id, "emitted_at": emitted_at, "evaluation_key": evaluation.key, "evaluation_version": evaluation.version, - "dataset_key": dataset.key, + "dataset_key": dataset.key if dataset else None, "status": CriterionStatus(result["status"]), "started_at": result["started_at"], "evaluated_at": result["evaluated_at"], diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/types.py b/packages/client/src/launchdarkly_ai_server/evaluations/types.py index f397859b..ff8279e1 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/types.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/types.py @@ -48,7 +48,17 @@ class DatasetRef: @dataclass class DatasetRow: - """A rendered dataset row ready for handler invocation and ingest.""" + """ + One row of an evaluation dataset. + + Serves both directions. As an *input* to ``run(rows=[...])`` the caller sets + ``input`` and ``expected_output`` to unrendered ``{{variable}}`` templates, + and owns ``row_index`` -- LaunchDarkly keys an inline row off + ``(run, row_index)``, so the values must be unique across the list. As an + *output* -- the row a ``Scorer`` receives -- the templates have been rendered + and ``variables`` has been augmented with the rendered ``input`` and + ``expected_output``. A run never mutates the instances a caller passed in. + """ row_index: int input: str | None = None diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index c7ca278b..be9672d5 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -1,8 +1,11 @@ from __future__ import annotations +import asyncio +import hashlib import json from collections.abc import Callable from datetime import datetime +from types import MappingProxyType from typing import Any from unittest.mock import AsyncMock, MagicMock @@ -2249,3 +2252,918 @@ async def handler( assert result.passed is True assert max_in_flight == 2 + + +def inline_run_transport( + *, + with_tool: bool = False, + summary: dict[str, Any] | None = None, + evaluation_version: int = 3, +) -> SequencedTransport: + responses = [] + if with_tool: + responses.append( + response( + 200, + { + "key": "lookup_order", + "version": 7, + "description": "Look up an order", + "schema": {"type": "object"}, + }, + ) + ) + responses.extend( + [ + response( + 201, + { + "id": "evaluation-id", + "name": "support-qa", + "version": evaluation_version, + }, + ), + response( + 201, + {"id": "run-id", "evaluationId": "evaluation-id", "state": "PENDING"}, + ), + response( + 200, + summary + or { + "statusCounts": { + "total": 1, + "passed": 1, + "failed": 0, + "error": 0, + "pending": 0, + } + }, + ), + ] + ) + return SequencedTransport(responses) + + +async def echo_handler( + config: dict[str, Any], + user_input: str | None, + tool_handlers: dict[str, Callable[..., Any]], + variables: dict[str, Any], +) -> dict[str, Any]: + return { + "output": f"generated: {user_input}", + "usage": {"input_tokens": 10, "output_tokens": 4}, + } + + +def digest(identity: dict[str, Any]) -> str: + return hashlib.sha256( + json.dumps(identity, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + + +@pytest.mark.asyncio +async def test_inline_rows_run_skips_dataset_requests_entirely( + stub_sdk_client: MagicMock, +) -> None: + transport = inline_run_transport(with_tool=True) + evals = init_evaluations( + api_token="token", + sdk_key="sdk-key", + ui_base_uri="https://ui.example.com", + transport=transport, + ) + + result = await evals.run( + project_key="proj", + key="support-qa", + rows=[DatasetRow(row_index=0, input="Order A19")], + handler=successful_handler, + tools={"lookup_order": lookup_order}, + generation={ + "provider": "OpenAI", + "model": "gpt-4o", + "parameters": {"temperature": 0.2}, + "instructions": "Help the user.", + }, + ) + + assert [request["method"] for request in transport.requests] == [ + "GET", + "POST", + "POST", + "GET", + ] + # The transport raises on a request past the end of its list, so a surviving + # _fetch_dataset would already have failed -- but it tolerates surplus + # responses, so assert the absence outright. + assert not any("/datasets" in request["url"] for request in transport.requests) + assert transport.requests[0]["url"].endswith( + "/api/v2/projects/proj/ai-tools/lookup_order" + ) + # The evaluation body is unchanged by the row source. + assert transport.requests[1]["body"] == { + "name": "support-qa", + "generationProvider": "OpenAI", + "generationModel": "gpt-4o", + "parameters": {"temperature": 0.2}, + "messages": [{"role": "system", "content": "Help the user."}], + "tools": [{"key": "lookup_order", "version": 7}], + } + # Equality, not a subset check: no datasetId, and no row count either. + assert transport.requests[2]["body"] == {"source": "api"} + assert result.passed is True + assert result.run_id == "run-id" + assert result.url == ( + "https://ui.example.com/projects/proj/ai/evaluations/evaluation-id/runs/run-id" + ) + + +@pytest.mark.asyncio +async def test_inline_rows_without_tools_issues_no_get_before_creation() -> None: + transport = inline_run_transport() + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + + result = await evals.run( + project_key="proj", + key="support-qa", + rows=[DatasetRow(row_index=0, input="Order A19")], + handler=echo_handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + assert [request["method"] for request in transport.requests] == [ + "POST", + "POST", + "GET", + ] + assert result.passed is True + + +@pytest.mark.asyncio +async def test_inline_generation_event_carries_row_data_and_no_dataset_identifiers( + stub_sdk_client: MagicMock, +) -> None: + transport = inline_run_transport() + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + + await evals.run( + project_key="proj", + key="support-qa", + rows=[ + DatasetRow( + row_index=4, + input="Order {{order_id}}", + expected_output="Found {{order_id}}", + variables={"order_id": "A19"}, + metadata={"suite": "orders"}, + ) + ], + handler=echo_handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + event = stub_sdk_client.track.call_args_list[0].args[2] + assert "datasetId" not in event + assert "datasetKey" not in event + # The row the dataset would otherwise own, rendered. + assert event["input"] == "Order A19" + assert event["expectedOutput"] == "Found A19" + assert event["variables"] == { + "order_id": "A19", + "input": "Order A19", + "expected_output": "Found A19", + } + assert event["metadata"] == {"suite": "orders"} + # Everything else about the event is unchanged. + assert event["projectKey"] == "proj" + assert event["rowIndex"] == 4 + assert event["evaluationKey"] == "support-qa" + assert event["evaluationVersion"] == 3 + assert event["status"] == "COMPLETE" + assert event["output"] == "generated: Order A19" + assert event["usage"] == {"inputTokens": 10, "outputTokens": 4} + assert len(event["eventId"]) == len(event["contentHash"]) == 64 + + +@pytest.mark.asyncio +async def test_inline_generation_event_id_covers_five_identity_fields( + stub_sdk_client: MagicMock, +) -> None: + transport = inline_run_transport() + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + + await evals.run( + project_key="proj", + key="support-qa", + rows=[DatasetRow(row_index=4, input="Order A19")], + handler=echo_handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + event = stub_sdk_client.track.call_args_list[0].args[2] + assert event["eventId"] == digest( + { + "projectKey": "proj", + "evaluationId": "evaluation-id", + "evaluationRunId": "run-id", + "runId": "run-id", + "rowIndex": 4, + } + ) + # A null-valued datasetId is not the same thing as an absent one. + assert event["eventId"] != digest( + { + "projectKey": "proj", + "evaluationId": "evaluation-id", + "evaluationRunId": "run-id", + "runId": "run-id", + "rowIndex": 4, + "datasetId": None, + } + ) + + +@pytest.mark.asyncio +async def test_inline_content_hash_ignores_row_data( + stub_sdk_client: MagicMock, +) -> None: + hashes = [] + for variables, metadata in ( + ({"a": "1"}, {"suite": "one"}), + ({"b": "2"}, {"suite": "two"}), + ): + transport = inline_run_transport() + evals = init_evaluations( + api_token="token", sdk_key="sdk-key", transport=transport + ) + client = MagicMock() + client.flush = AsyncMock() + with pytest.MonkeyPatch.context() as patch: + patch.setattr( + "launchdarkly_ai_server.evaluations.module.init_client", + AsyncMock(return_value=client), + ) + await evals.run( + project_key="proj", + key="support-qa", + rows=[ + DatasetRow( + row_index=4, + input="fixed", + variables=variables, + metadata=metadata, + ) + ], + handler=echo_handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + hashes.append(client.track.call_args_list[0].args[2]["contentHash"]) + + assert hashes[0] == hashes[1] + + +@pytest.mark.asyncio +async def test_inline_criterion_event_omits_dataset_and_row_data( + stub_sdk_client: MagicMock, +) -> None: + transport = inline_run_transport() + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + + def check_refund(row: DatasetRow, output: Any) -> bool: + assert row.row_index == 42 + assert row.input == "Ticket A" + return "refund" in str(output) + + await evals.run( + project_key="proj", + key="support-qa", + rows=[ + DatasetRow( + row_index=42, + input="Ticket {{id}}", + expected_output="refund row", + variables={"id": "A"}, + metadata={"suite": "orders"}, + ) + ], + handler=refund_handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + criteria=[Scorer(name="refund-exists", fn=check_refund)], + ) + + events = [call.args[2] for call in stub_sdk_client.track.call_args_list] + scorer_event = next(event for event in events if event.get("kind") == "scorer") + assert "datasetId" not in scorer_event + assert "datasetKey" not in scorer_event + assert {"input", "expectedOutput", "variables", "metadata", "output"}.isdisjoint( + scorer_event + ) + assert scorer_event["criterionType"] == "refund-exists" + assert scorer_event["rowIndex"] == 42 + assert scorer_event["score"] == 1 + assert scorer_event["eventId"] == digest( + { + "projectKey": "proj", + "evaluationId": "evaluation-id", + "evaluationRunId": "run-id", + "runId": "run-id", + "rowIndex": 42, + "criterionType": "refund-exists", + } + ) + # The generation event for the same row still carries the row data. + generation_event = next( + event for event in events if event.get("status") and "kind" not in event + ) + assert generation_event["input"] == "Ticket A" + + +async def refund_handler(*args: object) -> dict[str, Any]: + return {"output": "refund exists"} + + +@pytest.mark.asyncio +async def test_inline_rows_render_templates_and_inject_variables() -> None: + transport = inline_run_transport() + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + seen: dict[str, Any] = {} + + async def capture( + config: dict[str, Any], + user_input: str | None, + tool_handlers: dict[str, Callable[..., Any]], + variables: dict[str, Any], + ) -> dict[str, Any]: + seen["user_input"] = user_input + seen["variables"] = variables + return {"output": "ok"} + + def scorer(row: DatasetRow, output: Any) -> float: + seen["row"] = row + return 1.0 + + await evals.run( + project_key="proj", + key="support-qa", + rows=[ + DatasetRow( + row_index=0, + input="Order {{order_id}}", + expected_output="Found {{order_id}}", + variables={"order_id": "A19"}, + ) + ], + handler=capture, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + criteria=[Scorer(name="always", fn=scorer)], + ) + + assert seen["user_input"] == "Order A19" + assert seen["variables"] == { + "order_id": "A19", + "input": "Order A19", + "expected_output": "Found A19", + } + assert seen["row"].input == "Order A19" + assert seen["row"].expected_output == "Found A19" + + +@pytest.mark.asyncio +async def test_inline_unresolved_placeholder_is_left_literal() -> None: + transport = inline_run_transport() + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + seen: dict[str, Any] = {} + + async def capture( + config: dict[str, Any], + user_input: str | None, + tool_handlers: dict[str, Callable[..., Any]], + variables: dict[str, Any], + ) -> dict[str, Any]: + seen["user_input"] = user_input + return {"output": "ok"} + + await evals.run( + project_key="proj", + key="support-qa", + rows=[DatasetRow(row_index=0, input="Order {{missing}}")], + handler=capture, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + assert seen["user_input"] == "Order {{missing}}" + + +@pytest.mark.asyncio +async def test_inline_row_without_input_or_expected_output_injects_none( + stub_sdk_client: MagicMock, +) -> None: + transport = inline_run_transport() + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + seen: dict[str, Any] = {} + + async def capture( + config: dict[str, Any], + user_input: str | None, + tool_handlers: dict[str, Callable[..., Any]], + variables: dict[str, Any], + ) -> dict[str, Any]: + seen["user_input"] = user_input + seen["variables"] = variables + return {"output": "ok"} + + await evals.run( + project_key="proj", + key="support-qa", + rows=[DatasetRow(row_index=0)], + handler=capture, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + assert seen["user_input"] is None + assert seen["variables"]["input"] is None + assert seen["variables"]["expected_output"] is None + # Null-valued fields are omitted from the payload rather than serialized. + event = stub_sdk_client.track.call_args_list[0].args[2] + assert "input" not in event + assert "expectedOutput" not in event + assert "metadata" not in event + + +@pytest.mark.asyncio +async def test_inline_variables_accept_a_non_dict_mapping() -> None: + transport = inline_run_transport() + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + seen: dict[str, Any] = {} + + async def capture( + config: dict[str, Any], + user_input: str | None, + tool_handlers: dict[str, Callable[..., Any]], + variables: dict[str, Any], + ) -> dict[str, Any]: + seen["user_input"] = user_input + return {"output": "ok"} + + await evals.run( + project_key="proj", + key="support-qa", + rows=[ + DatasetRow( + row_index=0, + input="Order {{order_id}}", + variables=MappingProxyType({"order_id": "A19"}), # type: ignore[arg-type] + ) + ], + handler=capture, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + assert seen["user_input"] == "Order A19" + + +@pytest.mark.asyncio +async def test_inline_rows_are_not_mutated_by_the_run() -> None: + transport = inline_run_transport() + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + variables = {"order_id": "A19"} + metadata = {"suite": "orders"} + rows = [ + DatasetRow( + row_index=0, + input="Order {{order_id}}", + expected_output="Found {{order_id}}", + variables=variables, + metadata=metadata, + ) + ] + + def mutating_scorer(row: DatasetRow, output: Any) -> float: + row.variables["injected-by-scorer"] = True + if row.metadata is not None: + row.metadata["injected-by-scorer"] = True + return 1.0 + + await evals.run( + project_key="proj", + key="support-qa", + rows=rows, + handler=echo_handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + criteria=[Scorer(name="mutates", fn=mutating_scorer)], + ) + + assert rows[0].input == "Order {{order_id}}" + assert rows[0].expected_output == "Found {{order_id}}" + assert rows[0].variables == {"order_id": "A19"} + assert rows[0].metadata == {"suite": "orders"} + assert variables == {"order_id": "A19"} + assert metadata == {"suite": "orders"} + + +@pytest.mark.asyncio +async def test_inline_rows_can_be_reused_across_two_runs() -> None: + rows = [ + DatasetRow( + row_index=0, input="Order {{order_id}}", variables={"order_id": "A19"} + ) + ] + rendered = [] + for _ in range(2): + transport = inline_run_transport() + evals = init_evaluations( + api_token="token", sdk_key="sdk-key", transport=transport + ) + client = MagicMock() + client.flush = AsyncMock() + with pytest.MonkeyPatch.context() as patch: + patch.setattr( + "launchdarkly_ai_server.evaluations.module.init_client", + AsyncMock(return_value=client), + ) + await evals.run( + project_key="proj", + key="support-qa", + rows=rows, + handler=echo_handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + rendered.append(client.track.call_args_list[0].args[2]["input"]) + + assert rendered == ["Order A19", "Order A19"] + + +@pytest.mark.asyncio +async def test_inline_metadata_mapping_is_copied_before_the_scorer_sees_it() -> None: + transport = inline_run_transport() + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + metadata = MappingProxyType({"suite": "orders"}) + seen: dict[str, Any] = {} + + def scorer(row: DatasetRow, output: Any) -> float: + seen["metadata"] = row.metadata + return 1.0 + + await evals.run( + project_key="proj", + key="support-qa", + rows=[ + DatasetRow(row_index=0, input="x", metadata=metadata), # type: ignore[arg-type] + ], + handler=echo_handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + criteria=[Scorer(name="always", fn=scorer)], + ) + + assert seen["metadata"] == {"suite": "orders"} + assert isinstance(seen["metadata"], dict) + + +@pytest.mark.asyncio +async def test_inline_error_row_emits_row_data_without_dataset_identifiers( + stub_sdk_client: MagicMock, +) -> None: + transport = inline_run_transport( + summary={ + "statusCounts": { + "total": 1, + "passed": 0, + "failed": 0, + "error": 1, + "pending": 0, + } + } + ) + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + + async def failing(*args: object) -> dict[str, Any]: + raise RuntimeError("provider exploded") + + result = await evals.run( + project_key="proj", + key="support-qa", + rows=[ + DatasetRow( + row_index=4, + input="Order {{order_id}}", + variables={"order_id": "A19"}, + metadata={"suite": "orders"}, + ) + ], + handler=failing, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + assert result.passed is False + event = stub_sdk_client.track.call_args_list[0].args[2] + assert event["status"] == "ERROR" + assert event["error"] == { + "code": 5001, + "message": "handler raised: provider exploded", + } + assert event["errorMessage"] == "handler raised: provider exploded" + assert "datasetId" not in event + assert "datasetKey" not in event + assert event["input"] == "Order A19" + assert event["metadata"] == {"suite": "orders"} + + +@pytest.mark.asyncio +async def test_inline_rows_respect_the_concurrency_bound() -> None: + transport = inline_run_transport( + summary={ + "statusCounts": { + "total": 5, + "passed": 5, + "failed": 0, + "error": 0, + "pending": 0, + } + } + ) + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + in_flight = 0 + max_in_flight = 0 + + async def handler(*args: object) -> dict[str, Any]: + nonlocal in_flight, max_in_flight + in_flight += 1 + max_in_flight = max(max_in_flight, in_flight) + await asyncio.sleep(0.01) + in_flight -= 1 + return {"output": "ok"} + + result = await evals.run( + project_key="proj", + key="support-qa", + rows=[DatasetRow(row_index=index, input="x") for index in range(5)], + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + concurrency=2, + ) + + assert result.passed is True + assert max_in_flight == 2 + + +@pytest.mark.asyncio +async def test_inline_rows_run_with_judge_emits_criterion_event_without_dataset_id( + monkeypatch: pytest.MonkeyPatch, + stub_sdk_client: MagicMock, +) -> None: + from launchdarkly_ai_server import parse_template + + accuracy_judge_variation(monkeypatch) + transport = inline_run_transport() + evals = init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + judge_prompts: list[str] = [] + + async def handler( + config: dict[str, Any], + user_input: str | None, + tool_handlers: dict[str, Callable[..., Any]], + variables: dict[str, Any], + ) -> dict[str, Any]: + instructions = config.get("instructions", "") + if "Judge" in instructions: + judge_prompts.append(parse_template(instructions, variables)) + return {"output": '{"score": 1, "reasoning": "accurate"}'} + return {"output": "generated"} + + await evals.run( + project_key="proj", + key="support-qa", + rows=[ + DatasetRow( + row_index=7, + input="Question {{id}}", + expected_output="Answer {{id}}", + variables={"id": "A"}, + ) + ], + handler=handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + criteria=[Judge(key="$ld:ai:judge:accuracy")], + ) + + # The judge prompt renders against the inline row's injected variables. + assert judge_prompts == ["Judge generated against Answer A"] + events = [call.args[2] for call in stub_sdk_client.track.call_args_list] + judge_event = next(event for event in events if event.get("kind") == "judge") + assert "datasetId" not in judge_event + assert "datasetKey" not in judge_event + assert judge_event["judgeKey"] == "$ld:ai:judge:accuracy" + assert judge_event["variationKey"] == "default" + assert judge_event["version"] == 12 + assert judge_event["score"] == 1 + assert judge_event["reason"] == "accurate" + +def no_request_evals(transport: SequencedTransport) -> Any: + return init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) + + +@pytest.mark.asyncio +async def test_run_requires_dataset_or_rows() -> None: + transport = SequencedTransport([]) + evals = no_request_evals(transport) + + with pytest.raises(EvaluationsError) as error: + await evals.run( + project_key="proj", + key="support-qa", + handler=echo_handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + assert "dataset" in str(error.value) + assert "rows" in str(error.value) + assert transport.requests == [] + + +@pytest.mark.asyncio +async def test_run_rejects_both_dataset_and_rows() -> None: + transport = SequencedTransport([]) + evals = no_request_evals(transport) + + with pytest.raises(EvaluationsError, match="mutually exclusive"): + await evals.run( + project_key="proj", + key="support-qa", + dataset="golden", + rows=[DatasetRow(row_index=0, input="x")], + handler=echo_handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + assert transport.requests == [] + + +@pytest.mark.asyncio +async def test_run_rejects_blank_dataset_when_rows_omitted() -> None: + transport = SequencedTransport([]) + evals = no_request_evals(transport) + + with pytest.raises(EvaluationsError, match="dataset must not be blank"): + await evals.run( + project_key="proj", + key="support-qa", + dataset=" ", + handler=echo_handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + assert transport.requests == [] + + +@pytest.mark.asyncio +async def test_run_rejects_empty_rows_list() -> None: + transport = SequencedTransport([]) + evals = no_request_evals(transport) + + with pytest.raises(EvaluationsError, match="rows must not be empty"): + await evals.run( + project_key="proj", + key="support-qa", + rows=[], + handler=echo_handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + assert transport.requests == [] + + +@pytest.mark.asyncio +async def test_run_rejects_non_dataset_row_entries() -> None: + transport = SequencedTransport([]) + evals = no_request_evals(transport) + + with pytest.raises(EvaluationsError, match=r"rows\[0\] must be a DatasetRow"): + await evals.run( + project_key="proj", + key="support-qa", + rows=[{"row_index": 0, "input": "x"}], # type: ignore[list-item] + handler=echo_handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + assert transport.requests == [] + + +@pytest.mark.parametrize("row_index", [-1, 1.5, "0", True]) +@pytest.mark.asyncio +async def test_run_rejects_invalid_row_index(row_index: Any) -> None: + transport = SequencedTransport([]) + evals = no_request_evals(transport) + + with pytest.raises( + EvaluationsError, match=r"rows\[0\].row_index must be a non-negative integer" + ): + await evals.run( + project_key="proj", + key="support-qa", + rows=[DatasetRow(row_index=row_index, input="x")], + handler=echo_handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + assert transport.requests == [] + + +@pytest.mark.asyncio +async def test_run_rejects_duplicate_row_index() -> None: + """ + Load-bearing, not tidiness: ingest keys an inline row off (run, rowIndex) + alone, so a duplicate collapses two rows into one stored row and the run can + never account for its total -- surfacing only as a polling timeout. + """ + transport = SequencedTransport([]) + evals = no_request_evals(transport) + + with pytest.raises(EvaluationsError, match=r"rows\[1\].row_index 3 duplicates"): + await evals.run( + project_key="proj", + key="support-qa", + rows=[ + DatasetRow(row_index=3, input="first"), + DatasetRow(row_index=3, input="second"), + ], + handler=echo_handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + assert transport.requests == [] + + +@pytest.mark.parametrize( + "bad_value", + [ + datetime(2026, 9, 17), + {1, 2}, + float("nan"), + float("inf"), + object(), + ], +) +@pytest.mark.parametrize("field", ["variables", "metadata"]) +@pytest.mark.asyncio +async def test_run_rejects_unserializable_row_values( + field: str, bad_value: Any +) -> None: + transport = SequencedTransport([]) + evals = no_request_evals(transport) + row = DatasetRow(row_index=0, input="x", **{field: {"bad": bad_value}}) + + with pytest.raises( + EvaluationsError, match=rf"rows\[0\].{field} must be JSON-serializable" + ): + await evals.run( + project_key="proj", + key="support-qa", + rows=[row], + handler=echo_handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + assert transport.requests == [] + + +@pytest.mark.parametrize("field", ["variables", "metadata"]) +@pytest.mark.asyncio +async def test_run_rejects_non_mapping_row_values(field: str) -> None: + transport = SequencedTransport([]) + evals = no_request_evals(transport) + row = DatasetRow(row_index=0, input="x", **{field: ["not", "a", "mapping"]}) + + with pytest.raises(EvaluationsError, match=rf"rows\[0\].{field} must be a mapping"): + await evals.run( + project_key="proj", + key="support-qa", + rows=[row], + handler=echo_handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + assert transport.requests == [] + + +@pytest.mark.parametrize("field", ["input", "expected_output"]) +@pytest.mark.asyncio +async def test_run_rejects_non_string_inline_input(field: str) -> None: + transport = SequencedTransport([]) + evals = no_request_evals(transport) + row = DatasetRow(row_index=0, **{field: 42}) + + with pytest.raises( + EvaluationsError, match=rf"rows\[0\].{field} must be a string or None" + ): + await evals.run( + project_key="proj", + key="support-qa", + rows=[row], + handler=echo_handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + assert transport.requests == [] From 8c5198a04d5054fd789db43d12ec1109d9dcee43 Mon Sep 17 00:00:00 2001 From: Adriana Knight Date: Thu, 17 Sep 2026 11:53:25 -0400 Subject: [PATCH 2/5] Docs updates --- packages/ai/README.md | 2 ++ packages/client/README.md | 33 +++++++++++++++++++++++++++++++-- packages/client/agents.md | 12 +++++++++++- 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/packages/ai/README.md b/packages/ai/README.md index b1b47a70..262fb217 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -73,6 +73,8 @@ result = await evals.run( ) ``` +Pass `rows=[DatasetRow(...)]` in place of `dataset` to evaluate rows supplied from code instead of an LD-hosted dataset; `DatasetRow` is re-exported here too, and exactly one of the two arguments is required. + `LD_API_TOKEN` is required. Configure `LD_SDK_KEY` — or initialize your own client with `init_client(client=...)` — to emit one `$ld:ai:offline-evals:generation` event per generated row, plus one `$ld:ai:offline-evals:criterion` event per `(row, criterion)` when `criteria` are supplied, through the standard SDK event transport. The SDK reports scores; LaunchDarkly rules on them at ingest. A judge served by a different provider than `generation` needs a handler for it in `judge_handlers`. Use `LD_API_BASE_URI` for staging or local management API traffic; it is separate from the SDK delivery setting `LD_BASE_URI`. Evaluation-run links use the explicit `ui_base_uri` option or `LD_UI_BASE_URI` (for example, `https://ld-stg.launchdarkly.com` in staging), defaulting to `https://app.launchdarkly.com`. See the [core evaluations guide](../client/README.md#run-an-evaluation-from-code). --- diff --git a/packages/client/README.md b/packages/client/README.md index 1e50a5ec..560b677f 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -46,7 +46,7 @@ No code changes are required — `init_client()` detects the packages at runtime ### Run an evaluation from code -The evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, and invokes your handler once per row. Rows can then be scored by LaunchDarkly judges and local scorer functions; see [Score rows with judges and scorers](#score-rows-with-judges-and-scorers). Each success or error queues a `$ld:ai:offline-evals:generation` custom event containing the evaluation, run, dataset, and row identifiers plus output or error (`errorMessage` is included for `ERROR` rows), nested `usage.inputTokens`/`usage.outputTokens`, timing, and stable hashes. Dataset-owned input, expected output, metadata, and variables are not duplicated in the event. Each queued event is logged at `INFO` on the `launchdarkly_ai_server.evaluations.runner` logger with its RFC3339 UTC `emittedAt` timestamp and stable `eventId`, making it possible to compare SDK emission time with ClickHouse arrival time once that logger is enabled. The same `emittedAt` value is included in the event payload. Events are flushed before the summary is fetched and the call returns; handlers are never rerun to retry event delivery. Pass/fail is derived from LaunchDarkly's run summary. +The evaluations harness takes a dataset, creates a new evaluation and API-source run, and invokes your handler once per row. Rows come from exactly one of two places: pass `dataset` to read an LD-hosted dataset by key, or pass `rows` to supply them from code (see [Supply dataset rows from code](#supply-dataset-rows-from-code)). Rows can then be scored by LaunchDarkly judges and local scorer functions; see [Score rows with judges and scorers](#score-rows-with-judges-and-scorers). Each success or error queues a `$ld:ai:offline-evals:generation` custom event containing the evaluation, run, and row identifiers plus output or error (`errorMessage` is included for `ERROR` rows), nested `usage.inputTokens`/`usage.outputTokens`, timing, and stable hashes. When rows come from an LD-hosted dataset the event carries the dataset identifiers and omits the row's own input, expected output, metadata, and variables, because the dataset already holds them; when you pass `rows` inline the reverse is true — LaunchDarkly has no copy, so the generation event carries `input`, `expectedOutput`, `variables`, and `metadata` and no dataset identifiers. The criterion event never carries row data in either mode. Each queued event is logged at `INFO` on the `launchdarkly_ai_server.evaluations.runner` logger with its RFC3339 UTC `emittedAt` timestamp and stable `eventId`, making it possible to compare SDK emission time with ClickHouse arrival time once that logger is enabled. The same `emittedAt` value is included in the event payload. Events are flushed before the summary is fetched and the call returns; handlers are never rerun to retry event delivery. Pass/fail is derived from LaunchDarkly's run summary. Result links use `ui_base_uri`, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; this is independent of `LD_API_BASE_URI`. After flushing generation events, the harness polls the run summary endpoint until passed + failed + error rows fully account for a nonzero total with no pending rows, polling every `poll_interval_seconds` (default 2s) up to `poll_timeout_seconds` (default 180s); pass either to `run()` to widen both for large datasets. The summary endpoint does not return run state, so `RunSummary` exposes row counts only. A run passes only when the completed summary has no failed, error, or pending rows. `failed_rows` counts rows whose criteria were scored and did not meet their threshold, so a gate that ignored it would exit 0 on a run where every row failed its judge. Evaluation keys must be unique because every call creates a new evaluation with `POST`. @@ -80,6 +80,35 @@ sys.exit(asyncio.run(main())) `project_key` is supplied per run rather than during initialization. `generation.instructions` is shorthand for one system message; use `generation.messages` instead for a full message list, but do not supply both. The harness never retries a handler invocation because doing so could repeat tool side effects. Its retries apply only to LaunchDarkly management API requests. +### Supply dataset rows from code + +Pass `rows` instead of `dataset` to evaluate rows you already have — a CI fixture list, a JSON file, a generated corpus — without creating a hosted dataset first. The run then reads no dataset and issues no dataset requests. + +```python +from launchdarkly_ai_server import DatasetRow, init_evaluations + +result = await init_evaluations().run( + project_key="my-project", + key="support-qa-2026-08-20", + rows=[ + DatasetRow( + row_index=0, + input="Where is order {{order_id}}?", + expected_output="Order {{order_id}} shipped on the 3rd.", + variables={"order_id": "A19"}, + metadata={"suite": "orders"}, + ), + DatasetRow(row_index=1, input="How do I get a refund?"), + ], + handler=create_openai_messages_handler(), + generation={"provider": "OpenAI", "model": "gpt-4o"}, +) +``` + +Exactly one of `dataset` and `rows` is required; supplying both, or neither, raises before any request is made. `input` and `expected_output` are `{{variable}}` templates rendered against the row's `variables` exactly as a hosted dataset's are, and a placeholder with no matching variable is left literal — so data that legitimately contains braces survives. The `DatasetRow` objects you pass are never mutated, so the same list is safe to reuse across runs. + +You own `row_index`. LaunchDarkly identifies an inline row by `(run, row_index)`, so the values must be unique within the list; a duplicate would collapse two rows into one stored row and the run could never account for every row. `run()` rejects duplicates, negative or non-integer indices, non-string `input`/`expected_output`, and `variables`/`metadata` whose contents the event transport could not serialize (a date, a set, a non-finite number, a bare object) — all before any record exists. That last check is worth knowing about: the SDK's event transport serializes on a background thread and cannot report a failure back to `run()`, so an unencodable value would otherwise become silently lost events and then a polling timeout with nothing to explain it. + Generation and criterion events are the only path by which row results reach LaunchDarkly, so `init_evaluations()` raises rather than creating a run that can never complete unless it can resolve an event transport: either an SDK key (`sdk_key` or `LD_SDK_KEY`) or a client already initialized through `init_client(client=...)`. Bringing your own client lets a process emit evaluation events without an SDK key in scope. Every generated row is emitted and flushed unconditionally; no feature flag gates event publishing. The harness then polls the summary endpoint until row accounting shows processing is complete. ### Score rows with judges and scorers @@ -112,7 +141,7 @@ result = await init_evaluations().run( ) ``` -`Scorer.fn` receives the `DatasetRow` the output was generated from plus the generated output, may be sync or async, and must return a bool or a number from 0 to 1; booleans become 1.0 or 0.0. `Judge.threshold` defaults to 0.5 and `Scorer.threshold` to 1.0 — a perfect score, which is what a boolean scorer wants — and both accept an optional `pass_rate_threshold`. Judge keys and scorer names share one `criterionType` namespace and must be unique within a run, case-insensitively, because that name is part of each result's deterministic event identity. `Judge.ground_truth_context` overrides what the judge is graded against when the dataset row's expected output is not it. +`Scorer.fn` receives the `DatasetRow` the output was generated from plus the generated output, may be sync or async, and must return a bool or a number from 0 to 1; booleans become 1.0 or 0.0. That row carries the *rendered* `input` and `expected_output`, and its `variables` has been augmented with those two rendered values under the keys `input` and `expected_output` — the same shape whichever source the row came from. `Judge.threshold` defaults to 0.5 and `Scorer.threshold` to 1.0 — a perfect score, which is what a boolean scorer wants — and both accept an optional `pass_rate_threshold`. Judge keys and scorer names share one `criterionType` namespace and must be unique within a run, case-insensitively, because that name is part of each result's deterministic event identity. `Judge.ground_truth_context` overrides what the judge is graded against when the dataset row's expected output is not it. **The SDK reports scores and never rules on them.** LaunchDarkly derives each row's verdict at ingest by comparing the score against the criterion's stored threshold and success direction, so pass/fail policy is one server-side implementation that applies to every SDK version and to runs already recorded. A judge's direction lives on its AI Config and is injected server-side, keeping the one input a verdict turns on server-attested; a `Scorer` has no LaunchDarkly-side config to read, so it declares its own `success_direction` (default `"higher_is_better"` — set `"lower_is_better"` for a scorer that counts something unwanted, like a regex hit count). diff --git a/packages/client/agents.md b/packages/client/agents.md index cb117045..4ee81752 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -131,7 +131,9 @@ Handlers may return any of these — the client normalizes them before emitting `init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK delivery and may point at a relay proxy. Evaluation-run links use the separate `ui_base_uri` option, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; do not derive their host from `LD_API_BASE_URI`. An event transport is resolved in `init_evaluations()`, which raises before any network I/O when it finds neither an SDK key (`sdk_key` or `LD_SDK_KEY`) nor an already-initialized event-capable client: generation events are the only ingest path for row results, so a run without a transport could never complete. The lifecycle module's bring-your-own-client path (`init_client(client=...)`) therefore satisfies the check on its own, and `run()` reuses that singleton through `_resolve_client`; `run()` raises if the client disappears before it emits. Both polling arguments reject NaN, which would otherwise never compare past a deadline and hang the run. The harness always queues one `$ld:ai:offline-evals:generation` custom event per row through the standard SDK event transport and flushes before returning. No feature flag gates event emission. The harness polls the run summary endpoint until a nonzero `total_rows` has `pending_rows == 0` and `passed + failed + error` rows accounting for the total, polling every `poll_interval_seconds` (default 2s) until `poll_timeout_seconds` (default 180s); both are `run()` arguments so large datasets can widen them. The summary endpoint does not return run state, so `RunSummary` exposes row counts only. -`await EvaluationsModule.run(...)` takes `project_key` per call. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST` and a run with `source="api"`, so its key must be unique. The harness directly invokes the supplied handler once per row and never retries it — event delivery is never a reason to rerun a handler because that would repeat tool side effects; retries apply only to management API requests. A 429 is replayed for any method, but 5xx responses and transport failures are replayed only for `GET`/`HEAD`, so an evaluation or run `POST` that the server may already have applied is never duplicated. Management API calls run in a worker thread (`asyncio.to_thread`) because the client is synchronous; the caller's event loop stays free. Generation events go through the already-initialized SDK client when the application has one — `init_client` is idempotent, so an existing singleton wins and the evaluations SDK key is ignored with a warning. Dataset-owned `input`, `expected_output`, `metadata`, and `variables` are deliberately excluded from the event payload. The harness flushes events, polls the run summary endpoint until row accounting is complete (`total_rows > 0`, `pending_rows == 0`, and `passed + failed + error == total_rows`), and raises a timeout once `poll_timeout_seconds` elapses if the backend never reaches one. `RunSummary` includes row counts only, and `EvalRunResult.passed` is true only when error and pending row counts are both zero. +`await EvaluationsModule.run(...)` takes `project_key` per call, and exactly one of `dataset` (an LD-hosted dataset key) or `rows` (rows supplied from code). Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST` and a run with `source="api"`, so its key must be unique. The harness directly invokes the supplied handler once per row and never retries it — event delivery is never a reason to rerun a handler because that would repeat tool side effects; retries apply only to management API requests. A 429 is replayed for any method, but 5xx responses and transport failures are replayed only for `GET`/`HEAD`, so an evaluation or run `POST` that the server may already have applied is never duplicated. Management API calls run in a worker thread (`asyncio.to_thread`) because the client is synchronous; the caller's event loop stays free. Generation events go through the already-initialized SDK client when the application has one — `init_client` is idempotent, so an existing singleton wins and the evaluations SDK key is ignored with a warning. + +Which row source a run uses changes exactly three things. A `rows=` run reads no dataset, so it issues neither dataset GET; its run-creation body is exactly `{"source": "api"}` with `datasetId` omitted rather than nulled; and `datasetId`/`datasetKey` drop off both event payloads, which shortens the generation event's identity set from six fields to five and therefore changes its `eventId` (ingest keys such a row off `(run, rowIndex)` alone and never reads `eventId`). Because LaunchDarkly holds no copy of an inline row, the generation event carries its `input`, `expectedOutput`, `variables`, and `metadata` — the one case where those are not excluded. For a `dataset=` run they stay excluded because the dataset owns them, and the criterion event excludes them in both modes. Rendering and the `input`/`expected_output` variable injection go through one shared helper (`_render_row`) for both sources, with `_row_from_api_item` coercing bad server data and `_validate_rows` rejecting bad caller data outright; keep that split rather than unifying it. The harness flushes events, polls the run summary endpoint until row accounting is complete (`total_rows > 0`, `pending_rows == 0`, and `passed + failed + error == total_rows`), and raises a timeout once `poll_timeout_seconds` elapses if the backend never reaches one. `RunSummary` includes row counts only, and `EvalRunResult.passed` is true only when error and pending row counts are both zero. --- @@ -250,6 +252,14 @@ When `enabled` is `False`, `config` is always `None`. When `enabled` is `True` b `execute_and_track` expects the handler to return a plain `dict` with at least `output` and `usage` keys. Do not return a custom class — `parse_usage` and the telemetry pipeline both access dict keys. +### 3. Letting an unserializable value into an evaluation event payload + +An SDK event buffer is drained and serialized on a background thread, so a value the JSON encoder cannot encode is not reported back to the caller — the events are simply lost, and the run ends in a polling timeout with nothing to explain it. This is only reachable through `run(rows=[...])`, where `variables`/`metadata` hold arbitrary caller objects, which is why `_validate_rows` serialization-checks them up front with `allow_nan=False` and refuses to coerce. Do not relax that into a `default=str` rescue: silently stringifying a caller's value changes what a judge renders and what LaunchDarkly stores, and is unrecoverable once the row is persisted. + +### 4. Rendering an evaluation row in place + +`DatasetRow` is a mutable dataclass and, for `run(rows=[...])`, the instances belong to the caller. Rendering writes the injected `input`/`expected_output` keys into `variables`, so doing it in place would make a second run over the same list resolve `{{input}}` against the first run's already-rendered value — a CI retry silently evaluating different data. `_normalize_inline_rows` builds fresh rows with fresh variable maps; keep it that way. The `dict()` copy there is also load-bearing for a second reason: `parse_template` resolves placeholders via `isinstance(value, dict)`, so any other `Mapping` would leave every placeholder literal and send raw mustache text to the model. + --- ## Adding a New Export From 031ae6845f4301a9bf707a6ef407f566f672f243 Mon Sep 17 00:00:00 2001 From: Adriana Knight Date: Thu, 17 Sep 2026 12:07:09 -0400 Subject: [PATCH 3/5] Lint test file --- packages/client/tests/test_evaluations_run.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index be9672d5..cce06296 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -2960,6 +2960,7 @@ async def handler( assert judge_event["score"] == 1 assert judge_event["reason"] == "accurate" + def no_request_evals(transport: SequencedTransport) -> Any: return init_evaluations(api_token="token", sdk_key="sdk-key", transport=transport) From 6748eaa04c9b0bdf21a54bc1abfb0e5b26ec3f7d Mon Sep 17 00:00:00 2001 From: Adriana Knight Date: Fri, 18 Sep 2026 13:47:06 -0400 Subject: [PATCH 4/5] enforce limits from client side --- packages/client/README.md | 2 + packages/client/agents.md | 2 +- .../evaluations/module.py | 55 ++++++++-- packages/client/tests/test_evaluations_run.py | 102 ++++++++++++++++++ 4 files changed, 150 insertions(+), 11 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index 560b677f..2d175b04 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -109,6 +109,8 @@ Exactly one of `dataset` and `rows` is required; supplying both, or neither, rai You own `row_index`. LaunchDarkly identifies an inline row by `(run, row_index)`, so the values must be unique within the list; a duplicate would collapse two rows into one stored row and the run could never account for every row. `run()` rejects duplicates, negative or non-integer indices, non-string `input`/`expected_output`, and `variables`/`metadata` whose contents the event transport could not serialize (a date, a set, a non-finite number, a bare object) — all before any record exists. That last check is worth knowing about: the SDK's event transport serializes on a background thread and cannot report a failure back to `run()`, so an unencodable value would otherwise become silently lost events and then a polling timeout with nothing to explain it. +Inline rows are also bounded: at most 10,000 rows per run, with all DatasetRow fields each at most 1 MiB (1,048,576 bytes) once UTF-8 encoded — the encoded length, not the character count, because that is what travels in the event. Both are checked before any record exists, so an oversized dataset costs no API requests and no generation calls. Neither bound applies to a hosted `dataset`: LaunchDarkly holds those rows already, and a caller could not fix an oversized one from their own process. Pass `dataset` instead of `rows` for anything larger. + Generation and criterion events are the only path by which row results reach LaunchDarkly, so `init_evaluations()` raises rather than creating a run that can never complete unless it can resolve an event transport: either an SDK key (`sdk_key` or `LD_SDK_KEY`) or a client already initialized through `init_client(client=...)`. Bringing your own client lets a process emit evaluation events without an SDK key in scope. Every generated row is emitted and flushed unconditionally; no feature flag gates event publishing. The harness then polls the summary endpoint until row accounting shows processing is complete. ### Score rows with judges and scorers diff --git a/packages/client/agents.md b/packages/client/agents.md index 4ee81752..91580007 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -133,7 +133,7 @@ Handlers may return any of these — the client normalizes them before emitting `await EvaluationsModule.run(...)` takes `project_key` per call, and exactly one of `dataset` (an LD-hosted dataset key) or `rows` (rows supplied from code). Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST` and a run with `source="api"`, so its key must be unique. The harness directly invokes the supplied handler once per row and never retries it — event delivery is never a reason to rerun a handler because that would repeat tool side effects; retries apply only to management API requests. A 429 is replayed for any method, but 5xx responses and transport failures are replayed only for `GET`/`HEAD`, so an evaluation or run `POST` that the server may already have applied is never duplicated. Management API calls run in a worker thread (`asyncio.to_thread`) because the client is synchronous; the caller's event loop stays free. Generation events go through the already-initialized SDK client when the application has one — `init_client` is idempotent, so an existing singleton wins and the evaluations SDK key is ignored with a warning. -Which row source a run uses changes exactly three things. A `rows=` run reads no dataset, so it issues neither dataset GET; its run-creation body is exactly `{"source": "api"}` with `datasetId` omitted rather than nulled; and `datasetId`/`datasetKey` drop off both event payloads, which shortens the generation event's identity set from six fields to five and therefore changes its `eventId` (ingest keys such a row off `(run, rowIndex)` alone and never reads `eventId`). Because LaunchDarkly holds no copy of an inline row, the generation event carries its `input`, `expectedOutput`, `variables`, and `metadata` — the one case where those are not excluded. For a `dataset=` run they stay excluded because the dataset owns them, and the criterion event excludes them in both modes. Rendering and the `input`/`expected_output` variable injection go through one shared helper (`_render_row`) for both sources, with `_row_from_api_item` coercing bad server data and `_validate_rows` rejecting bad caller data outright; keep that split rather than unifying it. The harness flushes events, polls the run summary endpoint until row accounting is complete (`total_rows > 0`, `pending_rows == 0`, and `passed + failed + error == total_rows`), and raises a timeout once `poll_timeout_seconds` elapses if the backend never reaches one. `RunSummary` includes row counts only, and `EvalRunResult.passed` is true only when error and pending row counts are both zero. +Which row source a run uses changes exactly three things. A `rows=` run reads no dataset, so it issues neither dataset GET; its run-creation body is exactly `{"source": "api"}` with `datasetId` omitted rather than nulled; and `datasetId`/`datasetKey` drop off both event payloads, which shortens the generation event's identity set from six fields to five and therefore changes its `eventId` (ingest keys such a row off `(run, rowIndex)` alone and never reads `eventId`). Because LaunchDarkly holds no copy of an inline row, the generation event carries its `input`, `expectedOutput`, `variables`, and `metadata` — the one case where those are not excluded. For a `dataset=` run they stay excluded because the dataset owns them, and the criterion event excludes them in both modes. Rendering and the `input`/`expected_output` variable injection go through one shared helper (`_render_row`) for both sources, with `_row_from_api_item` coercing bad server data and `_validate_rows` rejecting bad caller data outright; keep that split rather than unifying it. The `MAX_ROWS` and `MAX_INLINE_TEXT_BYTES` ceilings live on the `_validate_rows` side of that split for the same reason: an inline row is only bounded because it travels inside its own generation event, whereas a hosted dataset is LaunchDarkly's to bound and a caller could not shrink one from their process — enforcing the caps there would fail a run over data the caller cannot reach. `MAX_INLINE_TEXT_BYTES` is measured on the UTF-8 encoding of all fields individually, not on `len()` and not on the row as a whole. The harness flushes events, polls the run summary endpoint until row accounting is complete (`total_rows > 0`, `pending_rows == 0`, and `passed + failed + error == total_rows`), and raises a timeout once `poll_timeout_seconds` elapses if the backend never reaches one. `RunSummary` includes row counts only, and `EvalRunResult.passed` is true only when error and pending row counts are both zero. --- diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index 12017c0f..8c56dbba 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -40,6 +40,8 @@ DEFAULT_UI_BASE_URI = "https://app.launchdarkly.com" SUMMARY_POLL_INTERVAL_SECONDS = 2.0 SUMMARY_POLL_TIMEOUT_SECONDS = 180.0 +MAX_INLINE_TEXT_BYTES = 1_048_576 +MAX_ROWS = 10000 def _env(name: str) -> str | None: @@ -131,7 +133,10 @@ async def run( of a LaunchDarkly-hosted dataset — to have the harness read its rows, or pass ``rows`` to supply them from code, in which case no dataset is read or referenced at all. Supplying both, or neither, is an error - raised before any network I/O. + raised before any network I/O. Inline rows are capped at ``MAX_ROWS``, + and each row field is capped at ``MAX_INLINE_TEXT_BYTES`` encoded bytes + (a mapping field is measured as its JSON form); both are checked before + any request is issued. Each row is generated with ``handler``; every entry in ``criteria`` — LaunchDarkly :class:`Judge` references and local deterministic @@ -385,10 +390,21 @@ def _validate_rows(rows: list[DatasetRow]) -> None: Stricter than the hosted path on purpose. ``_row_from_api_item`` coerces bad server data rather than failing a run already in flight; a bad inline row is a caller bug, and every check here runs with zero requests issued + (the ``MAX_ROWS`` and ``MAX_INLINE_TEXT_BYTES`` ceilings included: a hosted + dataset is LaunchDarkly's to bound, and a caller could not fix an oversized + one from their own process anyway) — where the hosted empty-dataset rule only fires after two GETs. """ if not rows: raise EvaluationsError("rows must not be empty") + # Ahead of the per-row loop so a runaway list fails on one len() rather than + # after a serializability check per row. + if len(rows) > MAX_ROWS: + raise EvaluationsError( + f"rows has {len(rows)} entries, over the {MAX_ROWS} row limit for " + "inline rows. Upload the dataset to LaunchDarkly and pass dataset= " + "instead." + ) seen: dict[int, int] = {} for index, row in enumerate(rows): if not isinstance(row, DatasetRow): @@ -419,14 +435,26 @@ def _validate_rows(rows: list[DatasetRow]) -> None: ("input", row.input), ("expected_output", row.expected_output), ): + if value is None: + continue # A hosted row renders a non-string as None (it is the API's data # to tolerate); inline, that would silently run the whole # evaluation on empty inputs at full generation cost. - if value is not None and not isinstance(value, str): + if not isinstance(value, str): raise EvaluationsError( f"rows[{index}].{name} must be a string or None, got " f"{type(value).__name__}" ) + # Encoded length, not len(): the field travels as UTF-8 JSON inside + # the generation event, so a shorter string of non-ASCII text can + # still be over the limit that ingest applies. + size = len(value.encode("utf-8")) + if size > MAX_INLINE_TEXT_BYTES: + raise EvaluationsError( + f"rows[{index}].{name} is {size} bytes, over the " + f"{MAX_INLINE_TEXT_BYTES} byte limit for an inline row " + "field." + ) for field_name, mapping_value in ( ("variables", row.variables), ("metadata", row.metadata), @@ -438,14 +466,21 @@ def _validate_rows(rows: list[DatasetRow]) -> None: f"rows[{index}].{field_name} must be a mapping, got " f"{type(mapping_value).__name__}" ) - # Serializability is checked on the copy, not the caller's - # container, because the copy is what reaches the wire -- any - # Mapping is accepted here and normalized to a dict. The copy is - # shallow in both places, so a nested mapping the serializer - # cannot encode is correctly still an error. - _require_json_serializable( - dict(mapping_value), f"rows[{index}].{field_name}" - ) + # Serializability and size are both checked on the copy, not the + # caller's container, because the copy is what reaches the wire -- + # any Mapping is accepted here and normalized to a dict, and the + # serializer only encodes dict. The copy is shallow in both + # places, so a nested mapping the serializer cannot encode is + # correctly still an error. + normalized = dict(mapping_value) + _require_json_serializable(normalized, f"rows[{index}].{field_name}") + size = len(json.dumps(normalized, ensure_ascii=False).encode("utf-8")) + if size > MAX_INLINE_TEXT_BYTES: + raise EvaluationsError( + f"rows[{index}].{field_name} is {size} bytes, over the " + f"{MAX_INLINE_TEXT_BYTES} byte limit for an inline row " + "field." + ) @staticmethod def _validate_run_args( diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index cce06296..61e1dd00 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -20,6 +20,11 @@ Scorer, init_evaluations, ) +from launchdarkly_ai_server.evaluations.module import ( + MAX_INLINE_TEXT_BYTES, + MAX_ROWS, + EvaluationsModule, +) @pytest.fixture(autouse=True) @@ -3168,3 +3173,100 @@ async def test_run_rejects_non_string_inline_input(field: str) -> None: ) assert transport.requests == [] + + +@pytest.mark.asyncio +async def test_run_rejects_more_rows_than_the_limit() -> None: + transport = SequencedTransport([]) + evals = no_request_evals(transport) + rows = [DatasetRow(row_index=index, input="x") for index in range(MAX_ROWS + 1)] + + with pytest.raises( + EvaluationsError, match=rf"rows has {MAX_ROWS + 1} entries, over the {MAX_ROWS}" + ): + await evals.run( + project_key="proj", + key="support-qa", + rows=rows, + handler=echo_handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + assert transport.requests == [] + + +def _sized_field_value(field: str, text: str) -> Any: + return {"k": text} if field in {"variables", "metadata"} else text + + +def _sized_field_text(field: str, total_bytes: int) -> str: + """ASCII text that makes ``field`` encode to exactly ``total_bytes``.""" + if field not in {"variables", "metadata"}: + return "a" * total_bytes + # Derived rather than hardcoded so the wrapper in sized_field_value stays + # free to change: the mapping's braces, quotes and key are measured too. + overhead = len(json.dumps(_sized_field_value(field, ""), ensure_ascii=False)) + return "a" * (total_bytes - overhead) + + +@pytest.mark.parametrize("field", ["input", "expected_output", "variables", "metadata"]) +@pytest.mark.asyncio +async def test_run_rejects_oversized_inline_text(field: str) -> None: + transport = SequencedTransport([]) + evals = no_request_evals(transport) + oversize = MAX_INLINE_TEXT_BYTES + 1 + row = DatasetRow( + row_index=0, + **{field: _sized_field_value(field, _sized_field_text(field, oversize))}, + ) + + with pytest.raises( + EvaluationsError, + match=rf"rows\[0\].{field} is {oversize} bytes, over the " + rf"{MAX_INLINE_TEXT_BYTES} byte limit", + ): + await evals.run( + project_key="proj", + key="support-qa", + rows=[row], + handler=echo_handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + assert transport.requests == [] + + +@pytest.mark.parametrize("field", ["input", "expected_output", "variables", "metadata"]) +@pytest.mark.asyncio +async def test_run_rejects_oversized_multibyte_inline_text(field: str) -> None: + transport = SequencedTransport([]) + evals = no_request_evals(transport) + # Two bytes per character, so half as many characters as the byte limit. The + # mapping wrapper only adds ASCII, so a mapping field is over on the same text. + text = "é" * (MAX_INLINE_TEXT_BYTES // 2 + 1) + assert len(text) < MAX_INLINE_TEXT_BYTES + row = DatasetRow(row_index=0, **{field: _sized_field_value(field, text)}) + + with pytest.raises( + EvaluationsError, match=rf"rows\[0\].{field} is \d+ bytes, over the" + ): + await evals.run( + project_key="proj", + key="support-qa", + rows=[row], + handler=echo_handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + assert transport.requests == [] + + +@pytest.mark.parametrize("field", ["input", "expected_output", "variables", "metadata"]) +def test_validate_rows_accepts_the_limits_exactly(field: str) -> None: + EvaluationsModule._validate_rows( + [DatasetRow(row_index=index, input="x") for index in range(MAX_ROWS)] + ) + at_limit = _sized_field_value( + field, _sized_field_text(field, MAX_INLINE_TEXT_BYTES) + ) + EvaluationsModule._validate_rows([DatasetRow(row_index=0, **{field: at_limit})]) From d9971cd5696a4995b369e64555e0590d019de390 Mon Sep 17 00:00:00 2001 From: Adriana Knight Date: Fri, 18 Sep 2026 16:55:04 -0400 Subject: [PATCH 5/5] check rendered row size --- packages/client/agents.md | 2 +- .../evaluations/module.py | 102 +++++++++++++----- packages/client/tests/test_evaluations_run.py | 101 +++++++++++++++-- 3 files changed, 169 insertions(+), 36 deletions(-) diff --git a/packages/client/agents.md b/packages/client/agents.md index 91580007..a877dc19 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -133,7 +133,7 @@ Handlers may return any of these — the client normalizes them before emitting `await EvaluationsModule.run(...)` takes `project_key` per call, and exactly one of `dataset` (an LD-hosted dataset key) or `rows` (rows supplied from code). Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; only `run()` is public. Each call creates a new evaluation with `POST` and a run with `source="api"`, so its key must be unique. The harness directly invokes the supplied handler once per row and never retries it — event delivery is never a reason to rerun a handler because that would repeat tool side effects; retries apply only to management API requests. A 429 is replayed for any method, but 5xx responses and transport failures are replayed only for `GET`/`HEAD`, so an evaluation or run `POST` that the server may already have applied is never duplicated. Management API calls run in a worker thread (`asyncio.to_thread`) because the client is synchronous; the caller's event loop stays free. Generation events go through the already-initialized SDK client when the application has one — `init_client` is idempotent, so an existing singleton wins and the evaluations SDK key is ignored with a warning. -Which row source a run uses changes exactly three things. A `rows=` run reads no dataset, so it issues neither dataset GET; its run-creation body is exactly `{"source": "api"}` with `datasetId` omitted rather than nulled; and `datasetId`/`datasetKey` drop off both event payloads, which shortens the generation event's identity set from six fields to five and therefore changes its `eventId` (ingest keys such a row off `(run, rowIndex)` alone and never reads `eventId`). Because LaunchDarkly holds no copy of an inline row, the generation event carries its `input`, `expectedOutput`, `variables`, and `metadata` — the one case where those are not excluded. For a `dataset=` run they stay excluded because the dataset owns them, and the criterion event excludes them in both modes. Rendering and the `input`/`expected_output` variable injection go through one shared helper (`_render_row`) for both sources, with `_row_from_api_item` coercing bad server data and `_validate_rows` rejecting bad caller data outright; keep that split rather than unifying it. The `MAX_ROWS` and `MAX_INLINE_TEXT_BYTES` ceilings live on the `_validate_rows` side of that split for the same reason: an inline row is only bounded because it travels inside its own generation event, whereas a hosted dataset is LaunchDarkly's to bound and a caller could not shrink one from their process — enforcing the caps there would fail a run over data the caller cannot reach. `MAX_INLINE_TEXT_BYTES` is measured on the UTF-8 encoding of all fields individually, not on `len()` and not on the row as a whole. The harness flushes events, polls the run summary endpoint until row accounting is complete (`total_rows > 0`, `pending_rows == 0`, and `passed + failed + error == total_rows`), and raises a timeout once `poll_timeout_seconds` elapses if the backend never reaches one. `RunSummary` includes row counts only, and `EvalRunResult.passed` is true only when error and pending row counts are both zero. +Which row source a run uses changes exactly three things. A `rows=` run reads no dataset, so it issues neither dataset GET; its run-creation body is exactly `{"source": "api"}` with `datasetId` omitted rather than nulled; and `datasetId`/`datasetKey` drop off both event payloads, which shortens the generation event's identity set from six fields to five and therefore changes its `eventId` (ingest keys such a row off `(run, rowIndex)` alone and never reads `eventId`). Because LaunchDarkly holds no copy of an inline row, the generation event carries its `input`, `expectedOutput`, `variables`, and `metadata` — the one case where those are not excluded. For a `dataset=` run they stay excluded because the dataset owns them, and the criterion event excludes them in both modes. Rendering and the `input`/`expected_output` variable injection go through one shared helper (`_render_row`) for both sources, with `_row_from_api_item` coercing bad server data and `_validate_rows` rejecting bad caller data outright; keep that split rather than unifying it. The `MAX_ROWS` and `MAX_INLINE_TEXT_BYTES` ceilings are inline-only for the same reason: an inline row is only bounded because it travels inside its own generation event, whereas a hosted dataset is LaunchDarkly's to bound and a caller could not shrink one from their process — enforcing the caps there would fail a run over data the caller cannot reach. `MAX_INLINE_TEXT_BYTES` is measured on the UTF-8 encoding of all fields individually, not on `len()` and not on the row as a whole, and on the *rendered* row rather than the caller's: expanding a `{{...}}` placeholder can grow `input` or `expected_output` past the cap, and the injected `input`/`expected_output` keys always grow `variables`, so a row measured before rendering can pass the cap and still produce an event that ingest rejects on the SDK's background flush thread, where nothing can report it and the run only shows up as a polling timeout. That is why `MAX_ROWS` and the shape checks sit in `_validate_rows` while the byte cap sits in `_prepare_inline_rows`, which renders first; `run()` calls it ahead of all I/O so both still report with zero requests issued, and it is the single render per run — the rows it returns are what `_run_rows` and the generation events both use. The harness flushes events, polls the run summary endpoint until row accounting is complete (`total_rows > 0`, `pending_rows == 0`, and `passed + failed + error == total_rows`), and raises a timeout once `poll_timeout_seconds` elapses if the backend never reaches one. `RunSummary` includes row counts only, and `EvalRunResult.passed` is true only when error and pending row counts are both zero. --- diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index 8c56dbba..0dab8f16 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -73,6 +73,48 @@ def _require_json_serializable(value: Any, description: str) -> None: ) from error +def _require_within_inline_limit(size: int, index: int, field_name: str) -> None: + if size <= MAX_INLINE_TEXT_BYTES: + return + raise EvaluationsError( + f"rows[{index}].{field_name} is {size} bytes rendered, over the " + f"{MAX_INLINE_TEXT_BYTES} byte limit for an inline row field. The size " + "is measured on the rendered value the generation event carries: " + "{{...}} placeholders are expanded, and variables additionally carries " + "the rendered input and expected_output." + ) + + +def _check_rendered_row_size(row: DatasetRow, index: int) -> None: + """ + Enforce the inline byte cap on the values the generation event will carry. + + Measured after ``_render_row`` rather than on the caller's row, because the + two differ in size in both directions: expanding a ``{{...}}`` placeholder + can grow ``input`` or ``expected_output`` past the cap, and the injected + ``input``/``expected_output`` keys always grow ``variables``. Measuring the + unrendered row would pass a row whose event is oversized -- and an + oversized event is rejected on the SDK's background flush thread, where + nothing can report it back, so the run ends in a polling timeout with no + cause. Callers therefore need headroom under the cap, not equality with it. + """ + for name, value in ( + ("input", row.input), + ("expected_output", row.expected_output), + ): + if value is None: + continue + _require_within_inline_limit(len(value.encode("utf-8")), index, name) + for field_name, mapping_value in ( + ("variables", row.variables), + ("metadata", row.metadata), + ): + if mapping_value is None: + continue + size = len(json.dumps(dict(mapping_value), ensure_ascii=False).encode("utf-8")) + _require_within_inline_limit(size, index, field_name) + + def _is_terminal_summary(summary: RunSummary) -> bool: accounted_rows = summary.passed_rows + summary.failed_rows + summary.error_rows return ( @@ -135,8 +177,11 @@ async def run( read or referenced at all. Supplying both, or neither, is an error raised before any network I/O. Inline rows are capped at ``MAX_ROWS``, and each row field is capped at ``MAX_INLINE_TEXT_BYTES`` encoded bytes - (a mapping field is measured as its JSON form); both are checked before - any request is issued. + *as the generation event will carry it* — templates expanded, and + ``variables`` holding the injected ``input``/``expected_output`` — so a + field needs headroom under the cap rather than exactly the cap. A + mapping field is measured as its JSON form. Both caps are checked + before any request is issued. Each row is generated with ``handler``; every entry in ``criteria`` — LaunchDarkly :class:`Judge` references and local deterministic @@ -178,6 +223,11 @@ async def run( ld_judges = [ criterion for criterion in run_criteria if isinstance(criterion, Judge) ] + inline_rows: list[DatasetRow] = ( + [] + if dataset is not None + else await asyncio.to_thread(self._prepare_inline_rows, rows or []) + ) client = await self._resolve_client() # The management API client is synchronous; running it in a worker thread @@ -199,7 +249,7 @@ async def run( ) else: dataset_ref = None - run_rows = await asyncio.to_thread(_normalize_inline_rows, rows or []) + run_rows = inline_rows evaluation = await asyncio.to_thread( self._runner._create_evaluation, project_key, @@ -382,6 +432,13 @@ def _validate_judge_handlers(judge_handlers: list[EvalHandler]) -> None: "judge's provider and mode." ) + @staticmethod + def _prepare_inline_rows(rows: list[DatasetRow]) -> list[DatasetRow]: + prepared = _normalize_inline_rows(rows) + for index, row in enumerate(prepared): + _check_rendered_row_size(row, index) + return prepared + @staticmethod def _validate_rows(rows: list[DatasetRow]) -> None: """ @@ -390,10 +447,15 @@ def _validate_rows(rows: list[DatasetRow]) -> None: Stricter than the hosted path on purpose. ``_row_from_api_item`` coerces bad server data rather than failing a run already in flight; a bad inline row is a caller bug, and every check here runs with zero requests issued - (the ``MAX_ROWS`` and ``MAX_INLINE_TEXT_BYTES`` ceilings included: a hosted - dataset is LaunchDarkly's to bound, and a caller could not fix an oversized - one from their own process anyway) + (the ``MAX_ROWS`` ceiling included: a hosted dataset is LaunchDarkly's to + bound, and a caller could not shrink an oversized one from their own + process anyway) — where the hosted empty-dataset rule only fires after two GETs. + + Shape only. The ``MAX_INLINE_TEXT_BYTES`` ceiling is inline-only for the + same reason ``MAX_ROWS`` is, but has to be measured on the rendered row + rather than this one; ``_prepare_inline_rows`` applies it, still with + zero requests issued. """ if not rows: raise EvaluationsError("rows must not be empty") @@ -445,16 +507,6 @@ def _validate_rows(rows: list[DatasetRow]) -> None: f"rows[{index}].{name} must be a string or None, got " f"{type(value).__name__}" ) - # Encoded length, not len(): the field travels as UTF-8 JSON inside - # the generation event, so a shorter string of non-ASCII text can - # still be over the limit that ingest applies. - size = len(value.encode("utf-8")) - if size > MAX_INLINE_TEXT_BYTES: - raise EvaluationsError( - f"rows[{index}].{name} is {size} bytes, over the " - f"{MAX_INLINE_TEXT_BYTES} byte limit for an inline row " - "field." - ) for field_name, mapping_value in ( ("variables", row.variables), ("metadata", row.metadata), @@ -466,21 +518,15 @@ def _validate_rows(rows: list[DatasetRow]) -> None: f"rows[{index}].{field_name} must be a mapping, got " f"{type(mapping_value).__name__}" ) - # Serializability and size are both checked on the copy, not the - # caller's container, because the copy is what reaches the wire -- - # any Mapping is accepted here and normalized to a dict, and the + # Serializability is checked on the copy, not the caller's + # container, because the copy is what reaches the wire -- any + # Mapping is accepted here and normalized to a dict, and the # serializer only encodes dict. The copy is shallow in both # places, so a nested mapping the serializer cannot encode is # correctly still an error. - normalized = dict(mapping_value) - _require_json_serializable(normalized, f"rows[{index}].{field_name}") - size = len(json.dumps(normalized, ensure_ascii=False).encode("utf-8")) - if size > MAX_INLINE_TEXT_BYTES: - raise EvaluationsError( - f"rows[{index}].{field_name} is {size} bytes, over the " - f"{MAX_INLINE_TEXT_BYTES} byte limit for an inline row " - "field." - ) + _require_json_serializable( + dict(mapping_value), f"rows[{index}].{field_name}" + ) @staticmethod def _validate_run_args( diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 61e1dd00..a52f5656 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -3220,9 +3220,13 @@ async def test_run_rejects_oversized_inline_text(field: str) -> None: **{field: _sized_field_value(field, _sized_field_text(field, oversize))}, ) + # variables is measured with the injected input and expected_output keys + # present, so its rendered size is larger than the caller's own mapping. + expected_bytes = r"\d+" if field == "variables" else str(oversize) + with pytest.raises( EvaluationsError, - match=rf"rows\[0\].{field} is {oversize} bytes, over the " + match=rf"rows\[0\].{field} is {expected_bytes} bytes rendered, over the " rf"{MAX_INLINE_TEXT_BYTES} byte limit", ): await evals.run( @@ -3248,7 +3252,7 @@ async def test_run_rejects_oversized_multibyte_inline_text(field: str) -> None: row = DatasetRow(row_index=0, **{field: _sized_field_value(field, text)}) with pytest.raises( - EvaluationsError, match=rf"rows\[0\].{field} is \d+ bytes, over the" + EvaluationsError, match=rf"rows\[0\].{field} is \d+ bytes rendered, over the" ): await evals.run( project_key="proj", @@ -3261,12 +3265,95 @@ async def test_run_rejects_oversized_multibyte_inline_text(field: str) -> None: assert transport.requests == [] -@pytest.mark.parametrize("field", ["input", "expected_output", "variables", "metadata"]) -def test_validate_rows_accepts_the_limits_exactly(field: str) -> None: +def test_inline_rows_accept_the_rendered_limits_exactly() -> None: EvaluationsModule._validate_rows( [DatasetRow(row_index=index, input="x") for index in range(MAX_ROWS)] ) - at_limit = _sized_field_value( - field, _sized_field_text(field, MAX_INLINE_TEXT_BYTES) + + # metadata is carried through untouched, so it may sit exactly at the cap. + metadata_text = _sized_field_text("metadata", MAX_INLINE_TEXT_BYTES) + EvaluationsModule._prepare_inline_rows( + [DatasetRow(row_index=0, metadata={"k": metadata_text})] + ) + + # variables is measured with the injected input and expected_output keys, so + # the caller's own entries have to leave room for them -- None here, as a row + # this close to the cap has neither. + injected_nulls = len(json.dumps({"k": "", "input": None, "expected_output": None})) + EvaluationsModule._prepare_inline_rows( + [ + DatasetRow( + row_index=0, + variables={"k": "a" * (MAX_INLINE_TEXT_BYTES - injected_nulls)}, + ) + ] + ) + + # input and expected_output are each measured twice: as themselves, and again + # inside variables, which rendering injects both of them into. The injected + # copies are the binding constraint -- the two strings share one cap, so + # neither field can reach the cap on its own. + room = MAX_INLINE_TEXT_BYTES - len(json.dumps({"input": "", "expected_output": ""})) + at_limit = DatasetRow( + row_index=0, + input="a" * (room // 2), + expected_output="a" * (room - room // 2), ) - EvaluationsModule._validate_rows([DatasetRow(row_index=0, **{field: at_limit})]) + EvaluationsModule._prepare_inline_rows([at_limit]) + + one_over = DatasetRow( + row_index=0, + input=f"{at_limit.input}a", + expected_output=at_limit.expected_output, + ) + with pytest.raises(EvaluationsError, match=r"rows\[0\].variables is \d+ bytes"): + EvaluationsModule._prepare_inline_rows([one_over]) + + +@pytest.mark.asyncio +async def test_run_rejects_inline_input_that_renders_over_the_limit() -> None: + transport = SequencedTransport([]) + evals = no_request_evals(transport) + half = "a" * (MAX_INLINE_TEXT_BYTES // 2 + 1) + row = DatasetRow(row_index=0, input="{{half}}{{half}}", variables={"half": half}) + assert len(row.input.encode("utf-8")) < MAX_INLINE_TEXT_BYTES + assert len(json.dumps(row.variables).encode("utf-8")) < MAX_INLINE_TEXT_BYTES + + with pytest.raises( + EvaluationsError, + match=rf"rows\[0\].input is {len(half) * 2} bytes rendered, over the " + rf"{MAX_INLINE_TEXT_BYTES} byte limit", + ): + await evals.run( + project_key="proj", + key="support-qa", + rows=[row], + handler=echo_handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + assert transport.requests == [] + + +@pytest.mark.asyncio +async def test_run_rejects_inline_variables_the_injected_keys_push_over() -> None: + transport = SequencedTransport([]) + evals = no_request_evals(transport) + text = "a" * (MAX_INLINE_TEXT_BYTES - len(json.dumps({"k": ""}))) + row = DatasetRow(row_index=0, variables={"k": text}) + assert len(json.dumps(row.variables).encode("utf-8")) == MAX_INLINE_TEXT_BYTES + + with pytest.raises( + EvaluationsError, + match=rf"rows\[0\].variables is \d+ bytes rendered, over the " + rf"{MAX_INLINE_TEXT_BYTES} byte limit", + ): + await evals.run( + project_key="proj", + key="support-qa", + rows=[row], + handler=echo_handler, + generation={"provider": "OpenAI", "model": "gpt-4o"}, + ) + + assert transport.requests == []