From 6d261b37f71392e51f0cfc558e172e7798739a18 Mon Sep 17 00:00:00 2001 From: kelkalot Date: Sat, 24 Oct 2026 18:27:33 +0200 Subject: [PATCH 1/2] Send each turn once and no transcript markers from HTTPAppTarget run_scenario passes the whole conversation as history, current turn included, with SimpleAudit's markers beside the text. _build_body forwarded it as-is, so the app received the current user turn twice (appended again in messages mode, repeated in history otherwise) and the markers: documents with their evaluation-only marks, file_uri paths and decision answers. History now goes out as plain {role, content} turns, without a trailing copy of the current turn; direct callers that pass earlier turns only are unchanged. A multi-turn scenario's documents are taken from the history marker and sent through documents_field as text, like a context-grounding run's, and fail the same way when no documents_field is set. Fixes #124 Co-Authored-By: Claude Opus 5.5 --- simpleaudit/targets/http.py | 40 ++++++++++---- tests/test_targets.py | 104 ++++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 11 deletions(-) diff --git a/simpleaudit/targets/http.py b/simpleaudit/targets/http.py index 7c38ba3..b1b6500 100644 --- a/simpleaudit/targets/http.py +++ b/simpleaudit/targets/http.py @@ -43,6 +43,25 @@ def _resolve_path(data: Any, path: Optional[str]) -> Any: return cur +def _earlier_turns(user: str, history: Optional[List[Dict[str, Any]]]) -> List[Dict[str, str]]: + """The conversation before the current turn, as plain ``{"role", "content"}`` dicts. + + ``run_scenario`` passes the whole conversation, current turn included, and + keeps markers beside the text (``documents`` with their evaluation-only + marks, ``file_uri``, ``decision``). A direct caller may pass the earlier + turns only. Either way the app gets each turn once and no marker. + """ + turns = [{"role": m["role"], "content": m["content"]} for m in history or []] + if turns and turns[-1] == {"role": "user", "content": user}: + turns.pop() + return turns + + +def _marked_documents(history: Optional[List[Dict[str, Any]]]) -> Optional[List[Union[str, Dict[str, Any]]]]: + """A scenario's documents, which a multi-turn run carries on its first user turn.""" + return next((m["documents"] for m in history or [] if m.get("documents")), None) + + class HTTPAppTarget: """Send audit probes to an external HTTP application and parse the answer.""" @@ -84,13 +103,15 @@ def _build_body( documents: Optional[List[Union[str, Dict[str, Any]]]] = None, ) -> Dict[str, Any]: body = json.loads(json.dumps(self.request_template)) # deep copy - # Retrieval context for a context-grounding run: text only, so no - # evaluation-only mark ever reaches the target. Fail loudly when - # documents were supplied but this body has nowhere to carry them — a - # silent drop would grade a response the app never saw the context for. + # Retrieval context: text only, so no evaluation-only mark ever + # reaches the target. A context-grounding run passes the documents + # directly; a multi-turn run carries them on the history. Fail loudly + # when documents were supplied but this body has nowhere to carry + # them — a silent drop would grade a response the app never saw the + # context for. doc_texts = [ doc if isinstance(doc, str) else (doc or {}).get("text") - for doc in (documents or []) + for doc in (documents or _marked_documents(history) or []) ] doc_texts = [text for text in doc_texts if isinstance(text, str) and text] if doc_texts: @@ -101,19 +122,16 @@ def _build_body( "the request field that carries the retrieval context." ) body[self.documents_field] = doc_texts + earlier = _earlier_turns(user, history) if self.message_field == "messages": # OpenAI-style chat body: ``messages`` is a list of # ``{"role", "content"}`` dicts. Append the current user turn, # optionally preceded by prior conversation history. - messages: List[Dict[str, str]] = [] - if history: - messages.extend(history) - messages.append({"role": "user", "content": user}) - body["messages"] = messages + body["messages"] = [*earlier, {"role": "user", "content": user}] else: body[self.message_field] = user if history is not None: - body.setdefault("history", history) + body.setdefault("history", earlier) return body async def send( diff --git a/tests/test_targets.py b/tests/test_targets.py index d0c57cc..ee7a8b4 100644 --- a/tests/test_targets.py +++ b/tests/test_targets.py @@ -258,6 +258,110 @@ def test_http_target_openai_style_messages_with_history(): ] +# HTTPAppTarget inside an audit (issue #124): run_scenario passes the whole +# conversation, current turn included, with SimpleAudit's markers beside the +# text. The app must get each turn once and no marker. + +MARKED_DOCUMENT = { + "text": "Deadline is six weeks.", + "relevant": True, + "true": False, + "authority": "guidance", + "source": "old-guide", +} + + +class _RecordingHTTPClient(_FakeAsyncClient): + """Keeps a copy of every request body, as it was when sent.""" + + def __init__(self, payload): + super().__init__(payload) + self.bodies = [] + + async def request(self, method, url, json=None, headers=None): + import json as _json + + self.bodies.append(_json.loads(_json.dumps(json))) + return await super().request(method, url, json=json, headers=headers) + + +def _audit_http(scenario, max_turns=2, **target_kwargs): + import asyncio + + from tests.fakes import fixed_probe_auditor, fixed_severity_judge, fixed_target, make_auditor + + client = _RecordingHTTPClient({"answer": "ok"}) + auditor = make_auditor( + fixed_target("unused"), fixed_severity_judge("pass"), fixed_probe_auditor("Second question?"), max_turns=max_turns + ) + auditor.set_target(HTTPAppTarget(url="http://app/chat", response_path="answer", client=client, **target_kwargs)) + result = asyncio.run(auditor.run_async([scenario]))[0] + return result, client.bodies + + +PLAIN_SCENARIO = {"name": "S", "description": "d", "test_prompt": "First question?"} + + +def test_http_audit_messages_body_sends_each_turn_once(): + result, bodies = _audit_http(PLAIN_SCENARIO, message_field="messages") + assert result.severity == "pass" + assert [b["messages"] for b in bodies] == [ + [{"role": "user", "content": "First question?"}], + [ + {"role": "user", "content": "First question?"}, + {"role": "assistant", "content": "ok"}, + {"role": "user", "content": "Second question?"}, + ], + ] + + +def test_http_audit_message_body_keeps_history_to_earlier_turns(): + _, bodies = _audit_http(PLAIN_SCENARIO) + assert bodies == [ + {"message": "First question?", "history": []}, + { + "message": "Second question?", + "history": [ + {"role": "user", "content": "First question?"}, + {"role": "assistant", "content": "ok"}, + ], + }, + ] + + +def test_http_audit_sends_documents_as_text_and_never_their_marks(): + scenario = {**PLAIN_SCENARIO, "documents": ["Appeals go to NAV.", MARKED_DOCUMENT]} + result, bodies = _audit_http(scenario, message_field="messages", documents_field="context") + assert result.severity == "pass" + assert [b["context"] for b in bodies] == [["Appeals go to NAV.", "Deadline is six weeks."]] * 2 + sent = str(bodies) + for mark in ("relevant", "authority", "old-guide", "documents"): + assert mark not in sent + + +def test_http_audit_with_documents_needs_a_documents_field(): + scenario = {**PLAIN_SCENARIO, "documents": [MARKED_DOCUMENT]} + result, bodies = _audit_http(scenario, message_field="messages") + assert result.severity == "ERROR" + assert "documents_field" in result.judgment["error"] + assert bodies == [] + + +def test_http_body_drops_markers_and_copies_history(): + t = HTTPAppTarget(url="http://x") + history = [ + {"role": "user", "content": "Is it guilty?", "file_uri": "/tmp/scan.png"}, + {"role": "assistant", "content": "yes", "decision": {"choice": "yes"}, "tool_events": [{"type": "tool_call"}]}, + ] + body = t._build_body("And the sentence?", history) + assert body["history"] == [ + {"role": "user", "content": "Is it guilty?"}, + {"role": "assistant", "content": "yes"}, + ] + assert body["history"] is not history + assert "file_uri" in history[0] # the caller's transcript is untouched + + # --------------------------------------------------------------------------- # ModelAuditor.target property + set_target # --------------------------------------------------------------------------- From 3c6b0a9958733db1e54be8a27d59e3c6c61d125c Mon Sep 17 00:00:00 2001 From: kelkalot Date: Sat, 24 Oct 2026 19:34:55 +0200 Subject: [PATCH 2/2] Keep caller fields and validate documents in HTTPAppTarget Follow-up to the history fix, from review: - Only the engine's markers (documents, file_uri, decision, tool_events) are removed from history; other fields stay, so direct callers sending OpenAI tool messages (tool_calls, tool_call_id, name) or turns without content get the body they built. - Documents go through parse_documents like the model and decision targets: a dict without a string text fails, empty documents keep their position, and an explicit documents=[] is not replaced by the history marker. The error no longer assumes a context-grounding run. - A scenario with file_uri now fails with an error; the app could never receive the file, so it was dropped silently. - Target.send documents the history contract, and history_documents() in targets/base.py replaces the copies in http.py and decision.py. --- simpleaudit/targets/base.py | 18 ++++++++++- simpleaudit/targets/decision.py | 4 +-- simpleaudit/targets/http.py | 55 +++++++++++++++++---------------- tests/test_targets.py | 26 ++++++++++++++++ 4 files changed, 74 insertions(+), 29 deletions(-) diff --git a/simpleaudit/targets/base.py b/simpleaudit/targets/base.py index 1cc86b9..052fdb4 100644 --- a/simpleaudit/targets/base.py +++ b/simpleaudit/targets/base.py @@ -80,5 +80,21 @@ async def send( params: Optional[Dict[str, Any]] = None, context: Optional[TargetContext] = None, ) -> TargetResponse: - """Send one message and return the normalized response.""" + """Send one message and return the normalized response. + + ``history`` is the conversation so far. When ``ModelAuditor.run_scenario`` + calls ``send``, it ends with the current user turn (``user``), and its + turns carry the engine's markers beside ``role`` and ``content``: + ``documents`` and ``file_uri`` on the first user turn, ``decision`` on + a decision model's replies. ``ModelTarget`` sends this history as the + request. Other targets should send each turn once and leave the + markers out; ``history_documents`` finds a scenario's documents. + """ ... + + +def history_documents( + history: Optional[List[Dict[str, Any]]], +) -> Optional[List[Union[str, Dict[str, Any]]]]: + """A scenario's documents, which ``run_scenario`` carries on the history's first user turn.""" + return next((m["documents"] for m in history or [] if m.get("documents")), None) diff --git a/simpleaudit/targets/decision.py b/simpleaudit/targets/decision.py index 9aa426c..90fdc4a 100644 --- a/simpleaudit/targets/decision.py +++ b/simpleaudit/targets/decision.py @@ -40,7 +40,7 @@ from typing import Any from ..context_marks import parse_documents -from .base import TargetContext, TargetResponse +from .base import TargetContext, TargetResponse, history_documents #: Ollama's request-body cap for System One requests without images. OLLAMA_MAX_BODY_BYTES = 64 * 1024 @@ -160,7 +160,7 @@ async def send( ) # The auditor attaches a scenario's documents to the first user turn. if not documents: - documents = next((m["documents"] for m in history or [] if m.get("documents")), None) + documents = history_documents(history) body = self.build_request(decision, user, documents) payload = json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode("utf-8") if self.max_body_bytes is not None and len(payload) > self.max_body_bytes: diff --git a/simpleaudit/targets/http.py b/simpleaudit/targets/http.py index b1b6500..ad1f208 100644 --- a/simpleaudit/targets/http.py +++ b/simpleaudit/targets/http.py @@ -22,7 +22,8 @@ import json from typing import Any, Callable, Dict, List, Optional, Union -from .base import Target, TargetContext, TargetResponse +from ..context_marks import parse_documents +from .base import Target, TargetContext, TargetResponse, history_documents def _resolve_path(data: Any, path: Optional[str]) -> Any: @@ -43,25 +44,24 @@ def _resolve_path(data: Any, path: Optional[str]) -> Any: return cur -def _earlier_turns(user: str, history: Optional[List[Dict[str, Any]]]) -> List[Dict[str, str]]: - """The conversation before the current turn, as plain ``{"role", "content"}`` dicts. +#: The audit engine's own keys beside ``role`` and ``content`` (see Target.send). +#: Documents carry evaluation-only marks, and none of them is the app's input. +_ENGINE_MARKERS = ("documents", "file_uri", "decision", "tool_events") - ``run_scenario`` passes the whole conversation, current turn included, and - keeps markers beside the text (``documents`` with their evaluation-only - marks, ``file_uri``, ``decision``). A direct caller may pass the earlier - turns only. Either way the app gets each turn once and no marker. + +def _earlier_turns(user: str, history: Optional[List[Dict[str, Any]]]) -> List[Dict[str, Any]]: + """The conversation before the current turn, without the engine's markers. + + Under ``run_scenario`` the history ends with the current user turn, which + the body adds itself; a direct caller may pass the earlier turns only. + Fields other than the markers (``tool_calls``, ``tool_call_id``, ...) stay. """ - turns = [{"role": m["role"], "content": m["content"]} for m in history or []] - if turns and turns[-1] == {"role": "user", "content": user}: + turns = [{k: v for k, v in m.items() if k not in _ENGINE_MARKERS} for m in history or []] + if turns and turns[-1].get("role") == "user" and turns[-1].get("content") == user: turns.pop() return turns -def _marked_documents(history: Optional[List[Dict[str, Any]]]) -> Optional[List[Union[str, Dict[str, Any]]]]: - """A scenario's documents, which a multi-turn run carries on its first user turn.""" - return next((m["documents"] for m in history or [] if m.get("documents")), None) - - class HTTPAppTarget: """Send audit probes to an external HTTP application and parse the answer.""" @@ -105,21 +105,20 @@ def _build_body( body = json.loads(json.dumps(self.request_template)) # deep copy # Retrieval context: text only, so no evaluation-only mark ever # reaches the target. A context-grounding run passes the documents - # directly; a multi-turn run carries them on the history. Fail loudly - # when documents were supplied but this body has nowhere to carry - # them — a silent drop would grade a response the app never saw the - # context for. - doc_texts = [ - doc if isinstance(doc, str) else (doc or {}).get("text") - for doc in (documents or _marked_documents(history) or []) - ] - doc_texts = [text for text in doc_texts if isinstance(text, str) and text] + # directly; a multi-turn run carries them on the history. Every + # document keeps its position, matching the judge's numbering. Fail + # loudly when documents were supplied but this body has nowhere to + # carry them — a silent drop would grade a response the app never saw + # the context for. + if documents is None: + documents = history_documents(history) + doc_texts = [mark.text for mark in parse_documents(documents)] if doc_texts: if self.documents_field is None: raise ValueError( - "HTTPAppTarget received documents for a context-grounding run " - "but has no way to send them: pass documents_field= to name " - "the request field that carries the retrieval context." + "HTTPAppTarget received the scenario's documents but has no way to " + "send them: pass documents_field= to name the request field that " + "carries the retrieval context." ) body[self.documents_field] = doc_texts earlier = _earlier_turns(user, history) @@ -146,6 +145,10 @@ async def send( params: Optional[Dict[str, Any]] = None, context: Optional[TargetContext] = None, ) -> TargetResponse: + # Like documents, an attachment the app never receives must not be + # graded as if it had. + if file_uri or any(m.get("file_uri") for m in history or []): + raise ValueError("HTTPAppTarget cannot send attached files; this scenario attaches file_uri") headers = dict(self.headers) body = self._build_body(user, history, documents) diff --git a/tests/test_targets.py b/tests/test_targets.py index ee7a8b4..ec76ecd 100644 --- a/tests/test_targets.py +++ b/tests/test_targets.py @@ -497,3 +497,29 @@ def test_engine_traceparent_reaches_wire_on_default_target(): tp = last["extra_headers"]["traceparent"] # Same trace id the correlation recorded, not a fresh unrelated one. assert tp.split("-")[1] in correlation.all_trace_ids() + + +def test_http_body_keeps_other_fields_of_direct_callers(): + t = HTTPAppTarget(url="http://x", message_field="messages") + history = [ + {"role": "user", "content": "What's 2+2?"}, + {"role": "assistant", "tool_calls": [{"id": "c1", "type": "function", "function": {"name": "add"}}]}, + {"role": "tool", "tool_call_id": "c1", "name": "add", "content": "4"}, + {"role": "assistant", "content": "4."}, + ] + body = t._build_body("And 3+3?", history) + assert body["messages"] == [*history, {"role": "user", "content": "And 3+3?"}] + + +def test_http_documents_are_parsed_and_keep_their_positions(): + t = HTTPAppTarget(url="http://x", documents_field="context") + assert t._build_body("q", None, ["", "B"])["context"] == ["", "B"] + assert "context" not in t._build_body("q", [{"role": "user", "content": "q", "documents": ["A"]}], []) + with pytest.raises(ValueError, match="text"): + t._build_body("q", None, [{"content": "Deadline is six weeks."}]) + + +def test_http_audit_with_attached_files_fails_instead_of_dropping_them(): + result, bodies = _audit_http({**PLAIN_SCENARIO, "file_uri": "/tmp/scan.png"}, message_field="messages") + assert result.severity == "ERROR" and "cannot send attached files" in result.judgment["error"] + assert bodies == []