HTTPAppTarget: send each turn once and no transcript markers (fixes #124)

simpleaudit/targets/base.py

@@ -80,5 +80,21 @@ async def send(
         params: Optional[Dict[str, Any]] = None,
         context: Optional[TargetContext] = None,
     ) -> TargetResponse:
-        """Send one message and return the normalized response."""
+        """Send one message and return the normalized response.
+
+        ``history`` is the conversation so far. When ``ModelAuditor.run_scenario``
+        calls ``send``, it ends with the current user turn (``user``), and its
+        turns carry the engine's markers beside ``role`` and ``content``:
+        ``documents`` and ``file_uri`` on the first user turn, ``decision`` on
+        a decision model's replies. ``ModelTarget`` sends this history as the
+        request. Other targets should send each turn once and leave the
+        markers out; ``history_documents`` finds a scenario's documents.
+        """
         ...
+
+
+def history_documents(
+    history: Optional[List[Dict[str, Any]]],
+) -> Optional[List[Union[str, Dict[str, Any]]]]:
+    """A scenario's documents, which ``run_scenario`` carries on the history's first user turn."""
+    return next((m["documents"] for m in history or [] if m.get("documents")), None)

simpleaudit/targets/decision.py

@@ -40,7 +40,7 @@
 from typing import Any
 
 from ..context_marks import parse_documents
-from .base import TargetContext, TargetResponse
+from .base import TargetContext, TargetResponse, history_documents
 
 #: Ollama's request-body cap for System One requests without images.
 OLLAMA_MAX_BODY_BYTES = 64 * 1024
@@ -160,7 +160,7 @@ async def send(
             )
         # The auditor attaches a scenario's documents to the first user turn.
         if not documents:
-            documents = next((m["documents"] for m in history or [] if m.get("documents")), None)
+            documents = history_documents(history)
         body = self.build_request(decision, user, documents)
         payload = json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
         if self.max_body_bytes is not None and len(payload) > self.max_body_bytes:

simpleaudit/targets/http.py

@@ -22,7 +22,8 @@
 import json
 from typing import Any, Callable, Dict, List, Optional, Union
 
-from .base import Target, TargetContext, TargetResponse
+from ..context_marks import parse_documents
+from .base import Target, TargetContext, TargetResponse, history_documents
 
 
 def _resolve_path(data: Any, path: Optional[str]) -> Any:
@@ -43,6 +44,24 @@ def _resolve_path(data: Any, path: Optional[str]) -> Any:
     return cur
 
 
+#: The audit engine's own keys beside ``role`` and ``content`` (see Target.send).
+#: Documents carry evaluation-only marks, and none of them is the app's input.
+_ENGINE_MARKERS = ("documents", "file_uri", "decision", "tool_events")
+
+
+def _earlier_turns(user: str, history: Optional[List[Dict[str, Any]]]) -> List[Dict[str, Any]]:
+    """The conversation before the current turn, without the engine's markers.
+
+    Under ``run_scenario`` the history ends with the current user turn, which
+    the body adds itself; a direct caller may pass the earlier turns only.
+    Fields other than the markers (``tool_calls``, ``tool_call_id``, ...) stay.
+    """
+    turns = [{k: v for k, v in m.items() if k not in _ENGINE_MARKERS} for m in history or []]
+    if turns and turns[-1].get("role") == "user" and turns[-1].get("content") == user:
+        turns.pop()
+    return turns
+
+
 class HTTPAppTarget:
     """Send audit probes to an external HTTP application and parse the answer."""
 
@@ -84,36 +103,34 @@ def _build_body(
         documents: Optional[List[Union[str, Dict[str, Any]]]] = None,
     ) -> Dict[str, Any]:
         body = json.loads(json.dumps(self.request_template))  # deep copy
-        # Retrieval context for a context-grounding run: text only, so no
-        # evaluation-only mark ever reaches the target. Fail loudly when
-        # documents were supplied but this body has nowhere to carry them — a
-        # silent drop would grade a response the app never saw the context for.
-        doc_texts = [
-            doc if isinstance(doc, str) else (doc or {}).get("text")
-            for doc in (documents or [])
-        ]
-        doc_texts = [text for text in doc_texts if isinstance(text, str) and text]
+        # Retrieval context: text only, so no evaluation-only mark ever
+        # reaches the target. A context-grounding run passes the documents
+        # directly; a multi-turn run carries them on the history. Every
+        # document keeps its position, matching the judge's numbering. Fail
+        # loudly when documents were supplied but this body has nowhere to
+        # carry them — a silent drop would grade a response the app never saw
+        # the context for.
+        if documents is None:
+            documents = history_documents(history)
+        doc_texts = [mark.text for mark in parse_documents(documents)]
         if doc_texts:
             if self.documents_field is None:
                 raise ValueError(
-                    "HTTPAppTarget received documents for a context-grounding run "
-                    "but has no way to send them: pass documents_field= to name "
-                    "the request field that carries the retrieval context."
+                    "HTTPAppTarget received the scenario's documents but has no way to "
+                    "send them: pass documents_field= to name the request field that "
+                    "carries the retrieval context."
                 )
             body[self.documents_field] = doc_texts
+        earlier = _earlier_turns(user, history)
         if self.message_field == "messages":
             # OpenAI-style chat body: ``messages`` is a list of
             # ``{"role", "content"}`` dicts. Append the current user turn,
             # optionally preceded by prior conversation history.
-            messages: List[Dict[str, str]] = []
-            if history:
-                messages.extend(history)
-            messages.append({"role": "user", "content": user})
-            body["messages"] = messages
+            body["messages"] = [*earlier, {"role": "user", "content": user}]
         else:
             body[self.message_field] = user
             if history is not None:
-                body.setdefault("history", history)
+                body.setdefault("history", earlier)
         return body
 
     async def send(
@@ -128,6 +145,10 @@ async def send(
         params: Optional[Dict[str, Any]] = None,
         context: Optional[TargetContext] = None,
     ) -> TargetResponse:
+        # Like documents, an attachment the app never receives must not be
+        # graded as if it had.
+        if file_uri or any(m.get("file_uri") for m in history or []):
+            raise ValueError("HTTPAppTarget cannot send attached files; this scenario attaches file_uri")
         headers = dict(self.headers)
         body = self._build_body(user, history, documents)
 

tests/test_targets.py

@@ -258,6 +258,110 @@ def test_http_target_openai_style_messages_with_history():
     ]
 
 
+# HTTPAppTarget inside an audit (issue #124): run_scenario passes the whole
+# conversation, current turn included, with SimpleAudit's markers beside the
+# text. The app must get each turn once and no marker.
+
+MARKED_DOCUMENT = {
+    "text": "Deadline is six weeks.",
+    "relevant": True,
+    "true": False,
+    "authority": "guidance",
+    "source": "old-guide",
+}
+
+
+class _RecordingHTTPClient(_FakeAsyncClient):
+    """Keeps a copy of every request body, as it was when sent."""
+
+    def __init__(self, payload):
+        super().__init__(payload)
+        self.bodies = []
+
+    async def request(self, method, url, json=None, headers=None):
+        import json as _json
+
+        self.bodies.append(_json.loads(_json.dumps(json)))
+        return await super().request(method, url, json=json, headers=headers)
+
+
+def _audit_http(scenario, max_turns=2, **target_kwargs):
+    import asyncio
+
+    from tests.fakes import fixed_probe_auditor, fixed_severity_judge, fixed_target, make_auditor
+
+    client = _RecordingHTTPClient({"answer": "ok"})
+    auditor = make_auditor(
+        fixed_target("unused"), fixed_severity_judge("pass"), fixed_probe_auditor("Second question?"), max_turns=max_turns
+    )
+    auditor.set_target(HTTPAppTarget(url="http://app/chat", response_path="answer", client=client, **target_kwargs))
+    result = asyncio.run(auditor.run_async([scenario]))[0]
+    return result, client.bodies
+
+
+PLAIN_SCENARIO = {"name": "S", "description": "d", "test_prompt": "First question?"}
+
+
+def test_http_audit_messages_body_sends_each_turn_once():
+    result, bodies = _audit_http(PLAIN_SCENARIO, message_field="messages")
+    assert result.severity == "pass"
+    assert [b["messages"] for b in bodies] == [
+        [{"role": "user", "content": "First question?"}],
+        [
+            {"role": "user", "content": "First question?"},
+            {"role": "assistant", "content": "ok"},
+            {"role": "user", "content": "Second question?"},
+        ],
+    ]
+
+
+def test_http_audit_message_body_keeps_history_to_earlier_turns():
+    _, bodies = _audit_http(PLAIN_SCENARIO)
+    assert bodies == [
+        {"message": "First question?", "history": []},
+        {
+            "message": "Second question?",
+            "history": [
+                {"role": "user", "content": "First question?"},
+                {"role": "assistant", "content": "ok"},
+            ],
+        },
+    ]
+
+
+def test_http_audit_sends_documents_as_text_and_never_their_marks():
+    scenario = {**PLAIN_SCENARIO, "documents": ["Appeals go to NAV.", MARKED_DOCUMENT]}
+    result, bodies = _audit_http(scenario, message_field="messages", documents_field="context")
+    assert result.severity == "pass"
+    assert [b["context"] for b in bodies] == [["Appeals go to NAV.", "Deadline is six weeks."]] * 2
+    sent = str(bodies)
+    for mark in ("relevant", "authority", "old-guide", "documents"):
+        assert mark not in sent
+
+
+def test_http_audit_with_documents_needs_a_documents_field():
+    scenario = {**PLAIN_SCENARIO, "documents": [MARKED_DOCUMENT]}
+    result, bodies = _audit_http(scenario, message_field="messages")
+    assert result.severity == "ERROR"
+    assert "documents_field" in result.judgment["error"]
+    assert bodies == []
+
+
+def test_http_body_drops_markers_and_copies_history():
+    t = HTTPAppTarget(url="http://x")
+    history = [
+        {"role": "user", "content": "Is it guilty?", "file_uri": "/tmp/scan.png"},
+        {"role": "assistant", "content": "yes", "decision": {"choice": "yes"}, "tool_events": [{"type": "tool_call"}]},
+    ]
+    body = t._build_body("And the sentence?", history)
+    assert body["history"] == [
+        {"role": "user", "content": "Is it guilty?"},
+        {"role": "assistant", "content": "yes"},
+    ]
+    assert body["history"] is not history
+    assert "file_uri" in history[0]  # the caller's transcript is untouched
+
+
 # ---------------------------------------------------------------------------
 # ModelAuditor.target property + set_target
 # ---------------------------------------------------------------------------
@@ -393,3 +497,29 @@ def test_engine_traceparent_reaches_wire_on_default_target():
     tp = last["extra_headers"]["traceparent"]
     # Same trace id the correlation recorded, not a fresh unrelated one.
     assert tp.split("-")[1] in correlation.all_trace_ids()
+
+
+def test_http_body_keeps_other_fields_of_direct_callers():
+    t = HTTPAppTarget(url="http://x", message_field="messages")
+    history = [
+        {"role": "user", "content": "What's 2+2?"},
+        {"role": "assistant", "tool_calls": [{"id": "c1", "type": "function", "function": {"name": "add"}}]},
+        {"role": "tool", "tool_call_id": "c1", "name": "add", "content": "4"},
+        {"role": "assistant", "content": "4."},
+    ]
+    body = t._build_body("And 3+3?", history)
+    assert body["messages"] == [*history, {"role": "user", "content": "And 3+3?"}]
+
+
+def test_http_documents_are_parsed_and_keep_their_positions():
+    t = HTTPAppTarget(url="http://x", documents_field="context")
+    assert t._build_body("q", None, ["", "B"])["context"] == ["", "B"]
+    assert "context" not in t._build_body("q", [{"role": "user", "content": "q", "documents": ["A"]}], [])
+    with pytest.raises(ValueError, match="text"):
+        t._build_body("q", None, [{"content": "Deadline is six weeks."}])
+
+
+def test_http_audit_with_attached_files_fails_instead_of_dropping_them():
+    result, bodies = _audit_http({**PLAIN_SCENARIO, "file_uri": "/tmp/scan.png"}, message_field="messages")
+    assert result.severity == "ERROR" and "cannot send attached files" in result.judgment["error"]
+    assert bodies == []