From a540bcd185eb8594089074ae81ec6d9cd1397b38 Mon Sep 17 00:00:00 2001 From: janmatzek Date: Mon, 12 Oct 2026 09:09:08 +0200 Subject: [PATCH] feat(gooddata-eval): add --tag to put custom tags on Langfuse traces Repeatable --tag adds custom tags after the automatic ones; blank tags are rejected, duplicates dropped. jira: GDAI-2568 risk: low --- .../src/gooddata_eval/cli/agentic_runner.py | 11 +++ .../src/gooddata_eval/cli/main.py | 22 +++++ .../gooddata_eval/core/agentic/_langfuse.py | 9 +- .../src/gooddata_eval/core/config.py | 2 + .../gooddata_eval/core/langfuse/experiment.py | 9 ++ .../src/gooddata_eval/core/langfuse/sink.py | 24 +++-- .../tests/test_agentic_observe_experiment.py | 22 +++++ .../tests/test_agentic_runner.py | 12 +++ packages/gooddata-eval/tests/test_cli.py | 91 ++++++++++++++++++- .../gooddata-eval/tests/test_langfuse_sink.py | 28 ++++++ 10 files changed, 217 insertions(+), 13 deletions(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index 1ead2d118..5102d9602 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -38,6 +38,7 @@ class _LfKw(TypedDict, total=False): model_version_override: str | None reasoning_effort: ReasoningEffort | None submit_trace_link: SubmitTraceLink + run_metadata_extra: dict[str, Any] | None AGENTIC_TEST_KINDS = frozenset( @@ -152,11 +153,13 @@ def _dispatch_agentic( agent_id: str | None = None, submit_trace_link: SubmitTraceLink = run_trace_link_inline, gate: EvalGate = DEFAULT_GATE, + run_metadata_extra: dict[str, Any] | None = None, ) -> AgenticEvalOutcome: """Call the appropriate evaluate_agentic_* function for the item's test_kind. `gate` reaches every kind except those in UNGATED_AGENTIC_TEST_KINDS, which have no K to gate over; the CLI refuses --gate power for a dataset containing one. + `run_metadata_extra` reaches every kind unchanged. Every evaluate_agentic_* function returns an AgenticEvalOutcome (reasoning_steps, conversation_id, response_id, detail) on success and attaches the same four attributes @@ -172,6 +175,7 @@ def _dispatch_agentic( "model_version_override": model_version_override, "reasoning_effort": reasoning_effort, "submit_trace_link": submit_trace_link, + "run_metadata_extra": run_metadata_extra, } if kind in ("vis_agentic", "agentic_visualization"): @@ -400,6 +404,7 @@ def run_agentic_items( agent_id: str | None = None, concurrency: int = 1, gate: EvalGate = DEFAULT_GATE, + tags: tuple[str, ...] = (), ) -> EvalReport: """Run agentic items through evaluate_agentic_* and return an EvalReport. @@ -408,9 +413,14 @@ def run_agentic_items( from the parallel ones -- a metric being created and dropped mid-run would otherwise be visible to a catalog-reading item running alongside it. + ``tags`` are added to every Langfuse trace gd-eval writes for these items, after the + automatic ones. They reach Langfuse whenever trace linking runs, with or without + ``use_langfuse``. + Results are collected in dataset order regardless of completion order. """ langfuse = make_langfuse_client() if use_langfuse else None + run_metadata_extra: dict[str, Any] | None = {"tags": list(tags)} if tags else None report = EvalReport(model=model_version, gate=normalize_gate(gate)) total = len(items) @@ -452,6 +462,7 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport: agent_id, submit_trace_link=linker.submit, gate=gate, + run_metadata_extra=run_metadata_extra, ) if isinstance(outcome, AgenticEvalOutcome): reasoning_steps = outcome.reasoning_steps diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/main.py b/packages/gooddata-eval/src/gooddata_eval/cli/main.py index a724a8bac..8d90b12ce 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/main.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/main.py @@ -183,6 +183,17 @@ def _build_parser() -> argparse.ArgumentParser: action="store_true", help="Log scores and traces to Langfuse (requires --langfuse-dataset and LANGFUSE_* env vars).", ) + run.add_argument( + "--tag", + dest="tags", + action="append", + default=[], + type=_parse_tag, + metavar="TAG", + help="Custom tag for every Langfuse trace gd-eval writes, next to the automatic ones. " + "Repeat for several tags. Applied wherever traces are written: single-turn kinds only " + "with --langfuse, agentic kinds whenever LANGFUSE_* env vars are set.", + ) run.add_argument( "--agent-id", dest="agent_id", @@ -377,6 +388,14 @@ def _parse_model_arg(val: str) -> tuple[str | None, str]: return None, val +def _parse_tag(val: str) -> str: + """Strip one --tag value and reject a blank one, which Langfuse would store as an empty tag.""" + tag = val.strip() + if not tag: + raise argparse.ArgumentTypeError("tag must not be blank") + return tag + + def _make_progress_callbacks(console: Console): """Build (on_item_start, on_run_done, on_item_done) callbacks that stream progress. @@ -549,6 +568,7 @@ def _run(config: RunConfig) -> int: model_id=resolved.model_id, provider_type=resolved.provider_type, reasoning_effort=config.reasoning_effort, + tags=config.tags, ) def on_langfuse_item_done( @@ -574,6 +594,7 @@ def on_langfuse_item_done( model_version=resolved.model_id, reasoning_effort=config.reasoning_effort, use_langfuse=config.log_to_langfuse, + tags=config.tags, run_ts=run_ts, on_item_start=on_item_start, on_item_done=on_item_done, @@ -728,6 +749,7 @@ def main(argv: list[str] | None = None) -> int: agent_id=args.agent_id or os.environ.get("GD_EVAL_AGENT_ID"), turn_timeout_s=args.turn_timeout, item_timeout_s=args.item_timeout, + tags=tuple(dict.fromkeys(args.tags)), ) return _run(config) except ( diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py index f6e9ff5b5..9ff89b28f 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py @@ -21,6 +21,7 @@ ExperimentRun, ScoreTarget, build_experiment_root_span, + trace_tags, ) # Part of this module's public surface: external callers import both names from here. @@ -302,7 +303,10 @@ def _experiment_root_span( """gd-eval's own root span for one (dataset item, run) -- the whole experiment item.""" start, end = _span_window(trace, window) session_id = conversation_id or getattr(trace, "session_id", None) - tags = tuple(tag for tag in ("gd-eval", run_metadata.get("testing_framework")) if tag) + custom_tags = run_metadata.get("tags") or () + if isinstance(custom_tags, str): + custom_tags = (custom_tags,) + tags = trace_tags("gd-eval", run_metadata.get("testing_framework"), *custom_tags) # One predicate for both the name and the input, so a falsy question cannot name the span # after the item while the input still says "question". has_input = item_input is not None @@ -514,7 +518,8 @@ def build_run_context( model_version_override: Explicit model-version tag; when set it wins over the workspace lookup. run_metadata_extra: Optional extra key/values added to the dataset-run metadata - (e.g. a testing-framework tag or a CI run id for scoping). Default None keeps + (e.g. a testing-framework tag or a CI run id for scoping). A `tags` list is + also added to the trace tags, after the automatic ones. Default None keeps behavior unchanged. The SDK-derived model_version is applied last and cannot be overwritten by this dict. reasoning_effort: Effort the run requested, stamped into both the run name and diff --git a/packages/gooddata-eval/src/gooddata_eval/core/config.py b/packages/gooddata-eval/src/gooddata_eval/core/config.py index b665b74be..4d9032831 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/config.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/config.py @@ -97,3 +97,5 @@ class RunConfig: item_timeout_s: float | None = None """Wall-clock cap per item across all its turns; None keeps GOODDATA_EVAL_CHAT_ITEM_TIMEOUT_S.""" gate: EvalGate = DEFAULT_GATE + tags: tuple[str, ...] = () + """Custom Langfuse trace tags, added after the automatic ones.""" diff --git a/packages/gooddata-eval/src/gooddata_eval/core/langfuse/experiment.py b/packages/gooddata-eval/src/gooddata_eval/core/langfuse/experiment.py index b3e9745d6..a5fb6414d 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/langfuse/experiment.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/langfuse/experiment.py @@ -44,6 +44,15 @@ def experiment_id_for(run_name: str) -> str: return str(uuid.uuid5(_EXPERIMENT_NAMESPACE, run_name)) +def trace_tags(*tags: str | None) -> tuple[str, ...]: + """The non-empty `tags` in first-seen order, each once. + + Callers list the automatic tags before the user's own, so a custom tag that repeats an + automatic one keeps the automatic tag's position. + """ + return tuple(dict.fromkeys(tag for tag in tags if tag)) + + @dataclass(frozen=True) class ExperimentRun: name: str diff --git a/packages/gooddata-eval/src/gooddata_eval/core/langfuse/sink.py b/packages/gooddata-eval/src/gooddata_eval/core/langfuse/sink.py index 49fc47fcc..35367fe58 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/langfuse/sink.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/langfuse/sink.py @@ -9,7 +9,12 @@ from typing import TYPE_CHECKING, Any from gooddata_eval.core.langfuse.client import HttpxLangfuseClient -from gooddata_eval.core.langfuse.experiment import ExperimentItem, ExperimentRun, build_experiment_root_span +from gooddata_eval.core.langfuse.experiment import ( + ExperimentItem, + ExperimentRun, + build_experiment_root_span, + trace_tags, +) _MAX_LATENCY_S = 60.0 _QUALITY_WEIGHT = 0.6 @@ -59,6 +64,7 @@ def __init__( provider_type: str = "", reasoning_effort: ReasoningEffort | None = None, *, + tags: tuple[str, ...] = (), transport: httpx.BaseTransport | None = None, ): self._dataset_name = dataset_name @@ -66,6 +72,7 @@ def __init__( self._model_id = model_id self._provider_type = provider_type self._reasoning_effort = reasoning_effort + self._tags = tags self._client = HttpxLangfuseClient(timeout=10.0, transport=transport) self._warned_unlinkable = False @@ -112,15 +119,12 @@ def _build_span(self, report: ItemReport, dataset_item_id: str, run: ExperimentR start = end - timedelta(seconds=report.avg_latency_s) # "gd-eval" leads, exactly as on the agentic path, so one tag filter in Langfuse # finds both single-shot and agentic traces this package wrote. - tags = tuple( - t - for t in ( - "gd-eval", - report.test_kind, - self._provider_type, - f"effort-{self._reasoning_effort.lower()}" if self._reasoning_effort else None, - ) - if t + tags = trace_tags( + "gd-eval", + report.test_kind, + self._provider_type, + f"effort-{self._reasoning_effort.lower()}" if self._reasoning_effort else None, + *self._tags, ) return build_experiment_root_span( run, diff --git a/packages/gooddata-eval/tests/test_agentic_observe_experiment.py b/packages/gooddata-eval/tests/test_agentic_observe_experiment.py index 94e80f6f4..10bae7aab 100644 --- a/packages/gooddata-eval/tests/test_agentic_observe_experiment.py +++ b/packages/gooddata-eval/tests/test_agentic_observe_experiment.py @@ -213,6 +213,28 @@ def test_a_score_target_writes_to_both_the_gen_ai_trace_and_the_experiment_span( assert [body["value"] for body in bodies] == [1.0, 1.0] +@pytest.mark.parametrize( + ("run_metadata", "expected_tags"), + [ + ({"model_version": "gpt"}, ["gd-eval"]), + ({"tags": ["acme", "nightly"]}, ["gd-eval", "acme", "nightly"]), + ( + {"testing_framework": "tavern-e2e", "tags": ["acme", "gd-eval", "tavern-e2e", "acme"]}, + ["gd-eval", "tavern-e2e", "acme"], + ), + # A library caller passing one tag as a bare string gets that tag, not its characters. + ({"tags": "acme"}, ["gd-eval", "acme"]), + ], +) +def test_custom_run_tags_follow_the_automatic_ones(rec, run_metadata, expected_tags): + trace = _gen_ai_trace() + + with observe(rec.client, trace.id, "item-1", "run0", run_metadata, trace=trace): + pass + + assert _attrs(rec.spans()[0])["langfuse.trace.tags"] == expected_tags + + def test_a_refused_gen_ai_score_still_reaches_the_experiment_span(): """One destination rejecting a score must not cost the other one its copy.""" bodies: list[dict] = [] diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index d62ed4f16..a47fa66ae 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -968,3 +968,15 @@ def test_run_agentic_items_surfaces_failed_runs_on_a_partial_pass(): report = run_agentic_items([_item()], host="http://host", token="tok", workspace_id="ws1", run_ts="2026-01-01") assert report.items[0].pass_at_k is True assert report.items[0].failed_runs == [_A_FAILED_RUN] + + +@pytest.mark.parametrize(("tags", "expected_extra"), [((), None), (("acme", "nightly"), {"tags": ["acme", "nightly"]})]) +def test_run_agentic_items_passes_custom_tags_as_run_metadata(tags, expected_extra): + with patch( + "gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill", + return_value=AgenticEvalOutcome(detail={"alert_created": True}), + ) as mock_eval: + run_agentic_items( + [_item()], host="http://host", token="tok", workspace_id="ws1", run_ts="2026-01-01", tags=tags + ) + assert mock_eval.call_args.kwargs["run_metadata_extra"] == expected_extra diff --git a/packages/gooddata-eval/tests/test_cli.py b/packages/gooddata-eval/tests/test_cli.py index 1d23293bf..69fb13caa 100644 --- a/packages/gooddata-eval/tests/test_cli.py +++ b/packages/gooddata-eval/tests/test_cli.py @@ -38,6 +38,24 @@ def test_parse_args_agent_id_defaults_to_none(): assert args.agent_id is None +def test_parse_args_tag_is_repeatable(): + args = cli_main.parse_args( + ["run", "--host", "h", "--workspace", "w", "--dataset", "d", "--tag", "acme", "--tag", "nightly"] + ) + assert args.tags == ["acme", "nightly"] + + +def test_parse_args_tags_default_to_empty(): + args = cli_main.parse_args(["run", "--host", "h", "--workspace", "w", "--dataset", "d"]) + assert args.tags == [] + + +@pytest.mark.parametrize("blank", ["", " "]) +def test_parse_args_rejects_a_blank_tag(blank): + with pytest.raises(SystemExit): + cli_main.parse_args(["run", "--host", "h", "--workspace", "w", "--dataset", "d", "--tag", blank]) + + def test_cli_run_end_to_end(monkeypatch, tmp_path, fixtures_dir): # Stub connection + model activation + chat backend so no network is needed. monkeypatch.setattr(cli_main, "resolve_connection", lambda host, token, profile: ("https://h", "tok")) @@ -372,7 +390,7 @@ def close(self): ... langfuse_calls: list = [] class _FakeSink: - def __init__(self, dataset_name, run_name, model_id="", provider_type="", reasoning_effort=None): ... + def __init__(self, dataset_name, run_name, model_id="", provider_type="", reasoning_effort=None, tags=()): ... def log_item(self, report, *, dataset_item_id): langfuse_calls.append(dataset_item_id) @@ -880,6 +898,77 @@ def _fake_run_agentic(items, **kwargs): assert captured["concurrency"] == 3 +def test_cli_passes_custom_tags_to_both_langfuse_paths(monkeypatch): + monkeypatch.setattr(cli_main, "resolve_connection", lambda host, token, profile: ("https://h", "tok")) + + class _FakeController: + def __init__(self, *a, **k): ... + def get_active(self): + return ActiveLlmProvider(provider_id="prov", default_model_id="gpt-5.2") + + def resolve_and_activate(self, requested, provider=None): + return ResolvedModel(provider_id="prov", model_id="gpt-5.2", switched=False, provider_name="P") + + def restore(self, original): ... + def close(self): ... + + monkeypatch.setattr(cli_main, "WorkspaceModelController", _FakeController) + monkeypatch.setattr(cli_main, "ChatClient", lambda **k: object()) + monkeypatch.setattr( + cli_main, + "_load_dataset", + lambda config: [ + DatasetItem( + id="q1", dataset_name="d", test_kind="agentic_general_question", question="q", expected_output="a" + ) + ], + ) + monkeypatch.setattr(cli_main, "run_items", lambda items, backend, **kw: EvalReport(model="gpt-5.2")) + + sink_kwargs: dict = {} + + class _FakeSink: + def __init__(self, **kwargs): + sink_kwargs.update(kwargs) + + def log_item(self, report, *, dataset_item_id): ... + + monkeypatch.setattr(cli_main, "LangfuseSink", _FakeSink) + + agentic_kwargs: dict = {} + + def _fake_run_agentic(items, **kwargs): + agentic_kwargs.update(kwargs) + return EvalReport(model="gpt-5.2") + + monkeypatch.setattr(cli_main, "run_agentic_items", _fake_run_agentic) + + exit_code = cli_main.main( + [ + "run", + "--host", + "https://h", + "--token", + "tok", + "--workspace", + "ws1", + "--langfuse-dataset", + "my_dataset", + "--langfuse", + "--tag", + "acme", + "--tag", + " nightly ", + "--tag", + "acme", + ] + ) + + assert exit_code == 0 + assert sink_kwargs["tags"] == ("acme", "nightly") + assert agentic_kwargs["tags"] == ("acme", "nightly") + + def test_cli_timers_flag_enables_timer_output(monkeypatch): # --timers is the discoverable front door for GD_EVAL_TIMERS. The env var is the # mechanism because the [timer] call sites sit four layers below the CLI, inside the diff --git a/packages/gooddata-eval/tests/test_langfuse_sink.py b/packages/gooddata-eval/tests/test_langfuse_sink.py index 5101903fc..986f4ec4a 100644 --- a/packages/gooddata-eval/tests/test_langfuse_sink.py +++ b/packages/gooddata-eval/tests/test_langfuse_sink.py @@ -249,3 +249,31 @@ def recording_handler(request: httpx.Request) -> httpx.Response: assert not any(r.url.path == "/api/public/scores" for r in calls) err = capsys.readouterr().err assert "span export failed" in err.lower() + + +def _tags_from(requests: list[httpx.Request]) -> list[str]: + attrs = {a["key"]: a["value"] for a in _span_from(requests)["attributes"]} + return [v["stringValue"] for v in attrs["langfuse.trace.tags"]["arrayValue"]["values"]] + + +def test_langfuse_sink_tags_without_custom_tags(monkeypatch): + requests: list[httpx.Request] = [] + sink = _make_sink(monkeypatch, _known_item_handler(requests), provider_type="OPENAI", reasoning_effort="LOW") + + sink.log_item(_passing_report(), dataset_item_id="item-1") + + assert _tags_from(requests) == ["gd-eval", "visualization", "OPENAI", "effort-low"] + + +def test_langfuse_sink_appends_custom_tags_after_the_automatic_ones(monkeypatch): + requests: list[httpx.Request] = [] + sink = _make_sink( + monkeypatch, + _known_item_handler(requests), + provider_type="OPENAI", + tags=("acme", "gd-eval", "nightly", "acme"), + ) + + sink.log_item(_passing_report(), dataset_item_id="item-1") + + assert _tags_from(requests) == ["gd-eval", "visualization", "OPENAI", "acme", "nightly"]