feat(gooddata-eval): add --tag to put custom tags on Langfuse traces

packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py

@@ -38,6 +38,7 @@ class _LfKw(TypedDict, total=False):
     model_version_override: str | None
     reasoning_effort: ReasoningEffort | None
     submit_trace_link: SubmitTraceLink
+    run_metadata_extra: dict[str, Any] | None
 
 
 AGENTIC_TEST_KINDS = frozenset(
@@ -152,11 +153,13 @@ def _dispatch_agentic(
     agent_id: str | None = None,
     submit_trace_link: SubmitTraceLink = run_trace_link_inline,
     gate: EvalGate = DEFAULT_GATE,
+    run_metadata_extra: dict[str, Any] | None = None,
 ) -> AgenticEvalOutcome:
     """Call the appropriate evaluate_agentic_* function for the item's test_kind.
 
     `gate` reaches every kind except those in UNGATED_AGENTIC_TEST_KINDS, which have no K
     to gate over; the CLI refuses --gate power for a dataset containing one.
+    `run_metadata_extra` reaches every kind unchanged.
 
     Every evaluate_agentic_* function returns an AgenticEvalOutcome (reasoning_steps,
     conversation_id, response_id, detail) on success and attaches the same four attributes
@@ -172,6 +175,7 @@ def _dispatch_agentic(
         "model_version_override": model_version_override,
         "reasoning_effort": reasoning_effort,
         "submit_trace_link": submit_trace_link,
+        "run_metadata_extra": run_metadata_extra,
     }
 
     if kind in ("vis_agentic", "agentic_visualization"):
@@ -400,6 +404,7 @@ def run_agentic_items(
     agent_id: str | None = None,
     concurrency: int = 1,
     gate: EvalGate = DEFAULT_GATE,
+    tags: tuple[str, ...] = (),
 ) -> EvalReport:
     """Run agentic items through evaluate_agentic_* and return an EvalReport.
 
@@ -408,9 +413,14 @@ def run_agentic_items(
     from the parallel ones -- a metric being created and dropped mid-run would otherwise be
     visible to a catalog-reading item running alongside it.
 
+    ``tags`` are added to every Langfuse trace gd-eval writes for these items, after the
+    automatic ones. They reach Langfuse whenever trace linking runs, with or without
+    ``use_langfuse``.
+
     Results are collected in dataset order regardless of completion order.
     """
     langfuse = make_langfuse_client() if use_langfuse else None
+    run_metadata_extra: dict[str, Any] | None = {"tags": list(tags)} if tags else None
 
     report = EvalReport(model=model_version, gate=normalize_gate(gate))
     total = len(items)
@@ -452,6 +462,7 @@ def _process_item(index: int, item: DatasetItem) -> ItemReport:
                 agent_id,
                 submit_trace_link=linker.submit,
                 gate=gate,
+                run_metadata_extra=run_metadata_extra,
             )
             if isinstance(outcome, AgenticEvalOutcome):
                 reasoning_steps = outcome.reasoning_steps

packages/gooddata-eval/src/gooddata_eval/cli/main.py

@@ -183,6 +183,17 @@ def _build_parser() -> argparse.ArgumentParser:
         action="store_true",
         help="Log scores and traces to Langfuse (requires --langfuse-dataset and LANGFUSE_* env vars).",
     )
+    run.add_argument(
+        "--tag",
+        dest="tags",
+        action="append",
+        default=[],
+        type=_parse_tag,
+        metavar="TAG",
+        help="Custom tag for every Langfuse trace gd-eval writes, next to the automatic ones. "
+        "Repeat for several tags. Applied wherever traces are written: single-turn kinds only "
+        "with --langfuse, agentic kinds whenever LANGFUSE_* env vars are set.",
+    )
     run.add_argument(
         "--agent-id",
         dest="agent_id",
@@ -377,6 +388,14 @@ def _parse_model_arg(val: str) -> tuple[str | None, str]:
     return None, val
 
 
+def _parse_tag(val: str) -> str:
+    """Strip one --tag value and reject a blank one, which Langfuse would store as an empty tag."""
+    tag = val.strip()
+    if not tag:
+        raise argparse.ArgumentTypeError("tag must not be blank")
+    return tag
+
+
 def _make_progress_callbacks(console: Console):
     """Build (on_item_start, on_run_done, on_item_done) callbacks that stream progress.
 
@@ -549,6 +568,7 @@ def _run(config: RunConfig) -> int:
                     model_id=resolved.model_id,
                     provider_type=resolved.provider_type,
                     reasoning_effort=config.reasoning_effort,
+                    tags=config.tags,
                 )
 
                 def on_langfuse_item_done(
@@ -574,6 +594,7 @@ def on_langfuse_item_done(
                     model_version=resolved.model_id,
                     reasoning_effort=config.reasoning_effort,
                     use_langfuse=config.log_to_langfuse,
+                    tags=config.tags,
                     run_ts=run_ts,
                     on_item_start=on_item_start,
                     on_item_done=on_item_done,
@@ -728,6 +749,7 @@ def main(argv: list[str] | None = None) -> int:
             agent_id=args.agent_id or os.environ.get("GD_EVAL_AGENT_ID"),
             turn_timeout_s=args.turn_timeout,
             item_timeout_s=args.item_timeout,
+            tags=tuple(dict.fromkeys(args.tags)),
         )
         return _run(config)
     except (

packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py

@@ -21,6 +21,7 @@
     ExperimentRun,
     ScoreTarget,
     build_experiment_root_span,
+    trace_tags,
 )
 
 # Part of this module's public surface: external callers import both names from here.
@@ -302,7 +303,10 @@ def _experiment_root_span(
     """gd-eval's own root span for one (dataset item, run) -- the whole experiment item."""
     start, end = _span_window(trace, window)
     session_id = conversation_id or getattr(trace, "session_id", None)
-    tags = tuple(tag for tag in ("gd-eval", run_metadata.get("testing_framework")) if tag)
+    custom_tags = run_metadata.get("tags") or ()
+    if isinstance(custom_tags, str):
+        custom_tags = (custom_tags,)
+    tags = trace_tags("gd-eval", run_metadata.get("testing_framework"), *custom_tags)
     # One predicate for both the name and the input, so a falsy question cannot name the span
     # after the item while the input still says "question".
     has_input = item_input is not None
@@ -514,7 +518,8 @@ def build_run_context(
         model_version_override: Explicit model-version tag; when set it wins over the
             workspace lookup.
         run_metadata_extra: Optional extra key/values added to the dataset-run metadata
-            (e.g. a testing-framework tag or a CI run id for scoping). Default None keeps
+            (e.g. a testing-framework tag or a CI run id for scoping). A `tags` list is
+            also added to the trace tags, after the automatic ones. Default None keeps
             behavior unchanged. The SDK-derived model_version is applied last and cannot
             be overwritten by this dict.
         reasoning_effort: Effort the run requested, stamped into both the run name and

packages/gooddata-eval/src/gooddata_eval/core/config.py

@@ -97,3 +97,5 @@ class RunConfig:
     item_timeout_s: float | None = None
     """Wall-clock cap per item across all its turns; None keeps GOODDATA_EVAL_CHAT_ITEM_TIMEOUT_S."""
     gate: EvalGate = DEFAULT_GATE
+    tags: tuple[str, ...] = ()
+    """Custom Langfuse trace tags, added after the automatic ones."""

packages/gooddata-eval/src/gooddata_eval/core/langfuse/experiment.py

@@ -44,6 +44,15 @@ def experiment_id_for(run_name: str) -> str:
     return str(uuid.uuid5(_EXPERIMENT_NAMESPACE, run_name))
 
 
+def trace_tags(*tags: str | None) -> tuple[str, ...]:
+    """The non-empty `tags` in first-seen order, each once.
+
+    Callers list the automatic tags before the user's own, so a custom tag that repeats an
+    automatic one keeps the automatic tag's position.
+    """
+    return tuple(dict.fromkeys(tag for tag in tags if tag))
+
+
 @dataclass(frozen=True)
 class ExperimentRun:
     name: str

packages/gooddata-eval/src/gooddata_eval/core/langfuse/sink.py

@@ -9,7 +9,12 @@
 from typing import TYPE_CHECKING, Any
 
 from gooddata_eval.core.langfuse.client import HttpxLangfuseClient
-from gooddata_eval.core.langfuse.experiment import ExperimentItem, ExperimentRun, build_experiment_root_span
+from gooddata_eval.core.langfuse.experiment import (
+    ExperimentItem,
+    ExperimentRun,
+    build_experiment_root_span,
+    trace_tags,
+)
 
 _MAX_LATENCY_S = 60.0
 _QUALITY_WEIGHT = 0.6
@@ -59,13 +64,15 @@ def __init__(
         provider_type: str = "",
         reasoning_effort: ReasoningEffort | None = None,
         *,
+        tags: tuple[str, ...] = (),
         transport: httpx.BaseTransport | None = None,
     ):
         self._dataset_name = dataset_name
         self._run_name = run_name
         self._model_id = model_id
         self._provider_type = provider_type
         self._reasoning_effort = reasoning_effort
+        self._tags = tags
         self._client = HttpxLangfuseClient(timeout=10.0, transport=transport)
         self._warned_unlinkable = False
 
@@ -112,15 +119,12 @@ def _build_span(self, report: ItemReport, dataset_item_id: str, run: ExperimentR
         start = end - timedelta(seconds=report.avg_latency_s)
         # "gd-eval" leads, exactly as on the agentic path, so one tag filter in Langfuse
         # finds both single-shot and agentic traces this package wrote.
-        tags = tuple(
-            t
-            for t in (
-                "gd-eval",
-                report.test_kind,
-                self._provider_type,
-                f"effort-{self._reasoning_effort.lower()}" if self._reasoning_effort else None,
-            )
-            if t
+        tags = trace_tags(
+            "gd-eval",
+            report.test_kind,
+            self._provider_type,
+            f"effort-{self._reasoning_effort.lower()}" if self._reasoning_effort else None,
+            *self._tags,
         )
         return build_experiment_root_span(
             run,

packages/gooddata-eval/tests/test_agentic_observe_experiment.py

@@ -213,6 +213,28 @@ def test_a_score_target_writes_to_both_the_gen_ai_trace_and_the_experiment_span(
     assert [body["value"] for body in bodies] == [1.0, 1.0]
 
 
+@pytest.mark.parametrize(
+    ("run_metadata", "expected_tags"),
+    [
+        ({"model_version": "gpt"}, ["gd-eval"]),
+        ({"tags": ["acme", "nightly"]}, ["gd-eval", "acme", "nightly"]),
+        (
+            {"testing_framework": "tavern-e2e", "tags": ["acme", "gd-eval", "tavern-e2e", "acme"]},
+            ["gd-eval", "tavern-e2e", "acme"],
+        ),
+        # A library caller passing one tag as a bare string gets that tag, not its characters.
+        ({"tags": "acme"}, ["gd-eval", "acme"]),
+    ],
+)
+def test_custom_run_tags_follow_the_automatic_ones(rec, run_metadata, expected_tags):
+    trace = _gen_ai_trace()
+
+    with observe(rec.client, trace.id, "item-1", "run0", run_metadata, trace=trace):
+        pass
+
+    assert _attrs(rec.spans()[0])["langfuse.trace.tags"] == expected_tags
+
+
 def test_a_refused_gen_ai_score_still_reaches_the_experiment_span():
     """One destination rejecting a score must not cost the other one its copy."""
     bodies: list[dict] = []

packages/gooddata-eval/tests/test_agentic_runner.py

@@ -968,3 +968,15 @@ def test_run_agentic_items_surfaces_failed_runs_on_a_partial_pass():
         report = run_agentic_items([_item()], host="http://host", token="tok", workspace_id="ws1", run_ts="2026-01-01")
     assert report.items[0].pass_at_k is True
     assert report.items[0].failed_runs == [_A_FAILED_RUN]
+
+
+@pytest.mark.parametrize(("tags", "expected_extra"), [((), None), (("acme", "nightly"), {"tags": ["acme", "nightly"]})])
+def test_run_agentic_items_passes_custom_tags_as_run_metadata(tags, expected_extra):
+    with patch(
+        "gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill",
+        return_value=AgenticEvalOutcome(detail={"alert_created": True}),
+    ) as mock_eval:
+        run_agentic_items(
+            [_item()], host="http://host", token="tok", workspace_id="ws1", run_ts="2026-01-01", tags=tags
+        )
+    assert mock_eval.call_args.kwargs["run_metadata_extra"] == expected_extra

packages/gooddata-eval/tests/test_cli.py

@@ -38,6 +38,24 @@ def test_parse_args_agent_id_defaults_to_none():
     assert args.agent_id is None
 
 
+def test_parse_args_tag_is_repeatable():
+    args = cli_main.parse_args(
+        ["run", "--host", "h", "--workspace", "w", "--dataset", "d", "--tag", "acme", "--tag", "nightly"]
+    )
+    assert args.tags == ["acme", "nightly"]
+
+
+def test_parse_args_tags_default_to_empty():
+    args = cli_main.parse_args(["run", "--host", "h", "--workspace", "w", "--dataset", "d"])
+    assert args.tags == []
+
+
+@pytest.mark.parametrize("blank", ["", "   "])
+def test_parse_args_rejects_a_blank_tag(blank):
+    with pytest.raises(SystemExit):
+        cli_main.parse_args(["run", "--host", "h", "--workspace", "w", "--dataset", "d", "--tag", blank])
+
+
 def test_cli_run_end_to_end(monkeypatch, tmp_path, fixtures_dir):
     # Stub connection + model activation + chat backend so no network is needed.
     monkeypatch.setattr(cli_main, "resolve_connection", lambda host, token, profile: ("https://h", "tok"))
@@ -372,7 +390,7 @@ def close(self): ...
     langfuse_calls: list = []
 
     class _FakeSink:
-        def __init__(self, dataset_name, run_name, model_id="", provider_type="", reasoning_effort=None): ...
+        def __init__(self, dataset_name, run_name, model_id="", provider_type="", reasoning_effort=None, tags=()): ...
         def log_item(self, report, *, dataset_item_id):
             langfuse_calls.append(dataset_item_id)
 
@@ -880,6 +898,77 @@ def _fake_run_agentic(items, **kwargs):
     assert captured["concurrency"] == 3
 
 
+def test_cli_passes_custom_tags_to_both_langfuse_paths(monkeypatch):
+    monkeypatch.setattr(cli_main, "resolve_connection", lambda host, token, profile: ("https://h", "tok"))
+
+    class _FakeController:
+        def __init__(self, *a, **k): ...
+        def get_active(self):
+            return ActiveLlmProvider(provider_id="prov", default_model_id="gpt-5.2")
+
+        def resolve_and_activate(self, requested, provider=None):
+            return ResolvedModel(provider_id="prov", model_id="gpt-5.2", switched=False, provider_name="P")
+
+        def restore(self, original): ...
+        def close(self): ...
+
+    monkeypatch.setattr(cli_main, "WorkspaceModelController", _FakeController)
+    monkeypatch.setattr(cli_main, "ChatClient", lambda **k: object())
+    monkeypatch.setattr(
+        cli_main,
+        "_load_dataset",
+        lambda config: [
+            DatasetItem(
+                id="q1", dataset_name="d", test_kind="agentic_general_question", question="q", expected_output="a"
+            )
+        ],
+    )
+    monkeypatch.setattr(cli_main, "run_items", lambda items, backend, **kw: EvalReport(model="gpt-5.2"))
+
+    sink_kwargs: dict = {}
+
+    class _FakeSink:
+        def __init__(self, **kwargs):
+            sink_kwargs.update(kwargs)
+
+        def log_item(self, report, *, dataset_item_id): ...
+
+    monkeypatch.setattr(cli_main, "LangfuseSink", _FakeSink)
+
+    agentic_kwargs: dict = {}
+
+    def _fake_run_agentic(items, **kwargs):
+        agentic_kwargs.update(kwargs)
+        return EvalReport(model="gpt-5.2")
+
+    monkeypatch.setattr(cli_main, "run_agentic_items", _fake_run_agentic)
+
+    exit_code = cli_main.main(
+        [
+            "run",
+            "--host",
+            "https://h",
+            "--token",
+            "tok",
+            "--workspace",
+            "ws1",
+            "--langfuse-dataset",
+            "my_dataset",
+            "--langfuse",
+            "--tag",
+            "acme",
+            "--tag",
+            " nightly ",
+            "--tag",
+            "acme",
+        ]
+    )
+
+    assert exit_code == 0
+    assert sink_kwargs["tags"] == ("acme", "nightly")
+    assert agentic_kwargs["tags"] == ("acme", "nightly")
+
+
 def test_cli_timers_flag_enables_timer_output(monkeypatch):
     # --timers is the discoverable front door for GD_EVAL_TIMERS. The env var is the
     # mechanism because the [timer] call sites sit four layers below the CLI, inside the