Move to Claude 5.5 models

.changeset/claude-5-5-models.md

@@ -0,0 +1,5 @@
+---
+"apollo": minor
+---
+
+Use Claude Opus 5.5, Sonnet 5.5 and Haiku 5.5 across all services

.changeset/model-refusal-error.md

@@ -0,0 +1,5 @@
+---
+"apollo": patch
+---
+
+Return a `MODEL_REFUSAL` error when the AI service declines a request, instead of an empty-output error

services/doc_agent_chat/agent.py

@@ -1,5 +1,6 @@
 import os
 from typing import List, Dict, Any, Optional
+import httpx2
 from anthropic import Anthropic
 from util import create_logger
 from doc_agent_chat.doc_search import DocSearch
@@ -24,7 +25,7 @@ def __init__(self, api_key: Optional[str] = None):
 
         self.client = Anthropic(api_key=self.api_key)
         self.model = resolve_model(config.get("model", "claude-sonnet"))
-        self.max_tokens = config.get("max_tokens", 16384)
+        self.max_tokens = config.get("max_tokens", 32768)
         self.max_tool_calls = config.get("max_tool_calls", 10)
         self.search_top_k = config.get("search_top_k", 5)
         self.search_threshold = config.get("search_threshold", 0.7)
@@ -63,7 +64,10 @@ def run(
                 max_tokens=self.max_tokens,
                 system=system_prompt,
                 messages=messages,
-                tools=TOOL_DEFINITIONS
+                tools=TOOL_DEFINITIONS,
+                # Required for non-streaming calls with max_tokens > ~21k, which
+                # the SDK otherwise rejects. Same values as the SDK default.
+                timeout=httpx2.Timeout(600.0, connect=5.0),
             )
 
             if hasattr(response, "usage"):

services/doc_agent_chat/config.yaml

@@ -1,6 +1,6 @@
 config_version: 1.0
 model: claude-sonnet
-max_tokens: 16384
+max_tokens: 32768
 temperature: 0
 max_tool_calls: 10
 search_top_k: 5

services/global_chat/config.yaml

@@ -8,7 +8,7 @@ router:
 # Planner configuration (complex orchestration)
 planner:
   model: "claude-opus"
-  max_tokens: 24576
+  max_tokens: 32768
   max_tool_calls: 20
   max_pause_continuations: 5
   web_search: # Server-side web search/fetch for planner.

services/global_chat/planner.py

@@ -18,7 +18,7 @@
 sys.path.append(str(Path(__file__).parent.parent))
 
 from langfuse import observe
-from util import create_logger, ApolloError, sum_usage, mask_secrets, format_attachments
+from util import create_logger, ApolloError, sum_usage, mask_secrets, format_attachments, refusal_error
 from streaming_util import (
     StreamManager,
     STATUS_REVIEWING_WORKFLOW,
@@ -39,7 +39,6 @@
 # max_tool_calls, so this is a backstop against a raised budget: at the budget
 # it could only fire on the wrap-up round, clearing what that round summarises.
 _CONTEXT_MANAGEMENT = {
-    "betas": ["context-management-2025-06-27"],
     "context_management": {
         "edits": [
             {
@@ -53,6 +52,12 @@
     },
 }
 
+# The model writes its notes between tool calls into thinking blocks, which
+# come back empty unless display is "updates". Those notes are the narration.
+_THINKING = {"type": "adaptive", "display": "updates"}
+
+_BETAS = ["context-management-2025-06-27", "thinking-display-updates-2026-08-18"]
+
 _FINAL_ROUND_NOTICE = (
     "Stop and reply to the user now. Say what you changed. Mention unfinished work "
     "only if there is any, and then offer to continue next turn — otherwise don't "
@@ -247,7 +252,7 @@ def __init__(
 
         planner_config = config_loader.config.get("planner", {})
         self.model = resolve_model(planner_config.get("model", "claude-opus"))
-        self.max_tokens = planner_config.get("max_tokens", 24576)
+        self.max_tokens = planner_config.get("max_tokens", 32768)
         self.max_tool_calls = planner_config.get("max_tool_calls", 20)
         self.max_pause_continuations = planner_config.get("max_pause_continuations", 5)
 
@@ -387,6 +392,10 @@ def run(
 
                     logger.info(f"Claude API call {tool_call_count + 1}: stop_reason={response.stop_reason}")
 
+                    refusal = refusal_error(response)
+                    if refusal:
+                        raise refusal
+
                     # Text from every round is part of the answer the user saw
                     # (tool rounds may narrate before calling tools).
                     round_text = self._extract_text(response)
@@ -595,10 +604,10 @@ def _call_api(self, system_prompt, messages, stream, stream_manager, tool_choice
         rounds close the open text block), so the client can weave text and
         status events with its own formatting.
 
-        Adaptive thinking is enabled for better reasoning but thinking content
-        is not streamed to the client — it exposes internal details like tool
-        names and agent architecture. User-facing progress comes from the
-        task-specific status messages sent before each tool execution.
+        Reasoning is never streamed — it exposes internal details like tool
+        names and agent architecture — and under display "updates" it comes
+        back empty. The only thinking text is the model's progress notes
+        between tool calls, which stream as narration.
         """
         choice = {"tool_choice": tool_choice} if tool_choice else {}
 
@@ -610,14 +619,18 @@ def _call_api(self, system_prompt, messages, stream, stream_manager, tool_choice
                 system=system_prompt,
                 messages=messages,
                 tools=self.tools,
-                thinking={"type": "adaptive"},
+                thinking=_THINKING,
                 output_config={"effort": "medium"},
+                betas=_BETAS,
                 **choice,
                 **_CONTEXT_MANAGEMENT,
             ) as stream_obj:
                 for event in stream_obj:
-                    if event.type == "content_block_delta" and event.delta.type == "text_delta":
-                        stream_manager.send_text(event.delta.text)
+                    if event.type == "content_block_delta":
+                        if event.delta.type == "text_delta":
+                            stream_manager.send_text(event.delta.text)
+                        elif event.delta.type == "thinking_delta" and event.delta.thinking:
+                            stream_manager.send_text(event.delta.thinking)
                     elif event.type == "content_block_start":
                         if event.content_block.type == "server_tool_use":
                             self._send_spinner(stream_manager, STATUS_SEARCHING_WEB)
@@ -636,8 +649,9 @@ def _call_api(self, system_prompt, messages, stream, stream_manager, tool_choice
                 system=system_prompt,
                 messages=messages,
                 tools=self.tools,
-                thinking={"type": "adaptive"},
+                thinking=_THINKING,
                 output_config={"effort": "medium"},
+                betas=_BETAS,
                 **choice,
                 # Per-request timeout (same values as the SDK default):
                 # required for non-streaming calls with max_tokens > ~21k,
@@ -1042,8 +1056,12 @@ def _format_display_name(name: str) -> str:
         return name.replace("-", " ").replace("_", " ").title()
 
     def _extract_text(self, response):
-        """Extract text from response content, concatenated as it was streamed."""
-        return "".join(block.text for block in response.content if block.type == "text")
+        """Extract text and progress notes, concatenated as they were streamed."""
+        return "".join(
+            block.text if block.type == "text" else block.thinking
+            for block in response.content
+            if block.type == "text" or (block.type == "thinking" and block.thinking)
+        )
 
     @staticmethod
     def _count_server_tool_uses(response) -> dict:

services/global_chat/router.py

@@ -184,6 +184,9 @@ def _make_routing_decision(
             messages=[
                 {"role": "user", "content": routing_message}
             ],
+            # Routing is meant to be fast: a request that needs reasoning is
+            # one to send to the planner, not to think about here.
+            thinking={"type": "disabled"},
             output_config={"format": {"type": "json_schema", "schema": routing_schema}}
         )
 
@@ -194,7 +197,7 @@ def _make_routing_decision(
             "cache_read_input_tokens": getattr(response.usage, "cache_read_input_tokens", 0),
         }
 
-        response_text = response.content[0].text if response.content else "{}"
+        response_text = "".join(block.text for block in response.content if block.type == "text") or "{}"
 
         try:
             decision_data = json.loads(response_text)

services/global_chat/tests/unit/test_planner.py

@@ -611,6 +611,48 @@ def test_model_api_error_is_not_reported_as_a_tool_failure() -> None:
     )
 
 
+class FakeThinking:
+    type = "thinking"
+    signature = "sig"
+
+    def __init__(self, thinking: str) -> None:
+        self.thinking = thinking
+
+
+def test_a_refusal_is_reported_as_one_not_as_an_empty_answer() -> None:
+    """A decline is a 200 with no text, which otherwise surfaces as our own
+    EMPTY_OUTPUT failure."""
+    planner = make_planner()
+    planner.model = "test-model"
+    planner.max_tool_calls = 5
+    refused = FakeResponse("refusal", [])
+    refused.stop_details = type("StopDetails", (), {"category": "cyber"})()
+
+    with (
+        patch.object(planner, "_call_api", return_value=refused),
+        patch.object(planner, "_build_system_prompt", return_value="sys"),
+        pytest.raises(ApolloError) as raised,
+    ):
+        planner.run(content="x", workflow_yaml=None, page=None, history=[], stream=False)
+
+    assert raised.value.code == 422
+    assert raised.value.type == "MODEL_REFUSAL"
+    assert raised.value.details == {"category": "cyber"}
+
+
+def test_progress_notes_count_as_narration_and_hidden_reasoning_does_not() -> None:
+    """Notes written before a tool call arrive as thinking blocks; reasoning
+    arrives as thinking blocks too, but empty."""
+    response = FakeResponse("tool_use", [
+        FakeThinking(""),
+        FakeThinking("Checking the fetch step first."),
+        FakeText(" Then the mapping."),
+        FakeToolUse("inspect_job_code", {"job_keys": ["a"]}, "tu_1"),
+    ])
+
+    assert make_planner()._extract_text(response) == "Checking the fetch step first. Then the mapping."
+
+
 def _raise_from_planner(error: Exception) -> ApolloError:
     planner = make_planner()
     planner.model = "test-model"

services/job_chat/job_chat.py

@@ -19,7 +19,7 @@
 import sentry_sdk
 from langfuse import observe, propagate_attributes, get_client as get_langfuse_client
 from langfuse_util import should_track, build_tags, build_generation_diff, mask_secrets
-from util import ApolloError, create_logger, AdaptorSpecifier, add_page_prefix, APOLLO_VERSION
+from util import ApolloError, create_logger, AdaptorSpecifier, add_page_prefix, refusal_error, APOLLO_VERSION
 from yaml_utils import INSPECT_JOB_CODE_TOOL, inspect_job_code
 from .prompt import build_prompt, build_error_correction_prompt
 from .old_prompt import build_old_prompt
@@ -192,7 +192,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "Payload":
 @dataclass
 class ChatConfig:
     model: str = _MODEL
-    max_tokens: int = 24576
+    max_tokens: int = 32768
     api_key: Optional[str] = None
 
 
@@ -378,6 +378,11 @@ def generate(
                     if hasattr(message, "usage"):
                         usage_events.append(message.usage.model_dump())
 
+                    refusal = refusal_error(message)
+                    if refusal:
+                        stream_manager.end_stream()
+                        raise refusal
+
                     for content_block in message.content:
                         if getattr(content_block, "type", None) == "text" and content_block.text:
                             text_parts.append(content_block.text)

services/job_chat/retrieve_docs.py

@@ -223,7 +223,7 @@ def call_llm(model, system_prompt, user_prompt, client, output_schema=None):
 
         kwargs = dict(
             model=model,
-            max_tokens=1024,
+            max_tokens=4096,
             system=system_prompt,
             messages=messages
         )
@@ -238,7 +238,7 @@ def call_llm(model, system_prompt, user_prompt, client, output_schema=None):
         if not message.content or len(message.content) == 0:
             raise ApolloError(500, "Empty response from AI service", type="EMPTY_LLM_RESPONSE")
 
-        response_text = message.content[0].text
+        response_text = "".join(block.text for block in message.content if block.type == "text")
         if not response_text:
             raise ApolloError(500, "Empty text in AI service response", type="EMPTY_LLM_RESPONSE")
 

services/job_chat/tests/unit/test_retrieve_docs.py

@@ -44,7 +44,7 @@ def test_generate_queries_raises_apollo_error_on_invalid_json():
 
 def test_call_llm_returns_text_and_usage_on_success():
     message = MagicMock()
-    message.content = [MagicMock(text="hello")]
+    message.content = [MagicMock(type="text", text="hello")]
     message.usage.model_dump.return_value = {"input_tokens": 5, "output_tokens": 2}
     client = MagicMock()
     client.messages.create.return_value = message

services/models.py

@@ -4,9 +4,9 @@
 """
 
 CLAUDE_MODELS: dict[str, str] = {
-    "claude-opus":   "claude-opus-4-8",
-    "claude-sonnet": "claude-sonnet-4-6",
-    "claude-haiku":  "claude-haiku-4-5-20251001",
+    "claude-opus":   "claude-opus-5-5",
+    "claude-sonnet": "claude-sonnet-5-5",
+    "claude-haiku":  "claude-haiku-5-5",
 }
 
 CLAUDE_OPUS:   str = CLAUDE_MODELS["claude-opus"]

services/testing/judge.py

@@ -423,7 +423,7 @@ def evaluate(
         ],
     )
 
-    raw_text = response.content[0].text
+    raw_text = "".join(block.text for block in response.content if block.type == "text")
     usage = {
         "input_tokens": response.usage.input_tokens,
         "output_tokens": response.usage.output_tokens,

services/util.py

@@ -397,6 +397,24 @@ def check_attachment_size(attachments: list[dict] | None) -> None:
     )
 
 
+def refusal_error(message: Any) -> ApolloError | None:  # noqa: ANN401
+    """The typed error for a safety decline, or None if the model answered.
+
+    A decline is an HTTP 200 with stop_reason "refusal", so nothing upstream
+    raises; left alone it reads as an empty or unparseable answer and gets
+    retried or reported as our failure.
+    """
+    if getattr(message, "stop_reason", None) != "refusal":
+        return None
+    category = getattr(getattr(message, "stop_details", None), "category", None)
+    return ApolloError(
+        422,
+        "The AI service declined to answer this request. Try rephrasing it.",
+        type="MODEL_REFUSAL",
+        details={"category": category},
+    )
+
+
 # Payload attachment type -> the job_chat context field that already renders it.
 # job_chat has had typed <run_logs>/<input>/<output> blocks all along; attachments
 # arriving from global_chat reuse them rather than adding a second channel.

services/vocab_mapper/vocab_mapper.py

@@ -38,7 +38,7 @@ def _call_llm(self, system_prompt: str, user_prompt: str) -> str:
         """Helper method to make LLM calls."""
         message = self.client.messages.create(
             model=CLAUDE_SONNET,
-            max_tokens=500,
+            max_tokens=4096,
             system=system_prompt,
             messages=[
                 {
@@ -52,7 +52,7 @@ def _call_llm(self, system_prompt: str, user_prompt: str) -> str:
                 }
             ]
         )
-        return message.content[0].text
+        return "".join(block.text for block in message.content if block.type == "text")
 
     def _call_llm_batch(self, system_prompt: str, user_prompts: List[str]) -> List[str]:
         """Process a batch of LLM inputs concurrently."""