From b374c56cd688e65e8951cbb2de546b636fcd9873 Mon Sep 17 00:00:00 2001 From: hanna-paasivirta Date: Thu, 8 Oct 2026 21:45:30 +0100 Subject: [PATCH 1/3] Move to Claude 5.5 models --- services/doc_agent_chat/agent.py | 8 +++- services/doc_agent_chat/config.yaml | 2 +- services/global_chat/config.yaml | 4 +- services/global_chat/planner.py | 43 +++++++++++++------ services/global_chat/router.py | 4 +- .../global_chat/tests/unit/test_planner.py | 42 ++++++++++++++++++ services/job_chat/job_chat.py | 9 +++- services/job_chat/retrieve_docs.py | 4 +- .../job_chat/tests/unit/test_retrieve_docs.py | 2 +- services/models.py | 6 +-- services/testing/judge.py | 2 +- services/util.py | 18 ++++++++ services/vocab_mapper/vocab_mapper.py | 4 +- services/workflow_chat/workflow_chat.py | 18 ++++++-- 14 files changed, 133 insertions(+), 33 deletions(-) diff --git a/services/doc_agent_chat/agent.py b/services/doc_agent_chat/agent.py index fe175ee1..fe5aa943 100644 --- a/services/doc_agent_chat/agent.py +++ b/services/doc_agent_chat/agent.py @@ -1,5 +1,6 @@ import os from typing import List, Dict, Any, Optional +import httpx2 from anthropic import Anthropic from util import create_logger from doc_agent_chat.doc_search import DocSearch @@ -24,7 +25,7 @@ def __init__(self, api_key: Optional[str] = None): self.client = Anthropic(api_key=self.api_key) self.model = resolve_model(config.get("model", "claude-sonnet")) - self.max_tokens = config.get("max_tokens", 16384) + self.max_tokens = config.get("max_tokens", 32768) self.max_tool_calls = config.get("max_tool_calls", 10) self.search_top_k = config.get("search_top_k", 5) self.search_threshold = config.get("search_threshold", 0.7) @@ -63,7 +64,10 @@ def run( max_tokens=self.max_tokens, system=system_prompt, messages=messages, - tools=TOOL_DEFINITIONS + tools=TOOL_DEFINITIONS, + # Required for non-streaming calls with max_tokens > ~21k, which + # the SDK otherwise rejects. Same values as the SDK default. + timeout=httpx2.Timeout(600.0, connect=5.0), ) if hasattr(response, "usage"): diff --git a/services/doc_agent_chat/config.yaml b/services/doc_agent_chat/config.yaml index 40f3743d..dcebcaa5 100644 --- a/services/doc_agent_chat/config.yaml +++ b/services/doc_agent_chat/config.yaml @@ -1,6 +1,6 @@ config_version: 1.0 model: claude-sonnet -max_tokens: 16384 +max_tokens: 32768 temperature: 0 max_tool_calls: 10 search_top_k: 5 diff --git a/services/global_chat/config.yaml b/services/global_chat/config.yaml index 49be678e..3b3dc12a 100644 --- a/services/global_chat/config.yaml +++ b/services/global_chat/config.yaml @@ -3,10 +3,10 @@ # Router configuration (lightweight routing) router: model: "claude-haiku" - max_tokens: 500 + max_tokens: 4096 # Planner configuration (complex orchestration) planner: model: "claude-opus" - max_tokens: 24576 + max_tokens: 32768 max_tool_calls: 20 diff --git a/services/global_chat/planner.py b/services/global_chat/planner.py index 1735c5c7..ab5744ff 100644 --- a/services/global_chat/planner.py +++ b/services/global_chat/planner.py @@ -17,7 +17,7 @@ sys.path.append(str(Path(__file__).parent.parent)) from langfuse import observe -from util import create_logger, ApolloError, sum_usage, mask_secrets, format_attachments +from util import create_logger, ApolloError, sum_usage, mask_secrets, format_attachments, refusal_error from streaming_util import ( StreamManager, STATUS_REVIEWING_WORKFLOW, @@ -37,7 +37,6 @@ # max_tool_calls, so this is a backstop against a raised budget: at the budget # it could only fire on the wrap-up round, clearing what that round summarises. _CONTEXT_MANAGEMENT = { - "betas": ["context-management-2025-06-27"], "context_management": { "edits": [ { @@ -51,6 +50,12 @@ }, } +# The model writes its notes between tool calls into thinking blocks, which +# come back empty unless display is "updates". Those notes are the narration. +_THINKING = {"type": "adaptive", "display": "updates"} + +_BETAS = ["context-management-2025-06-27", "thinking-display-updates-2026-08-18"] + _FINAL_ROUND_NOTICE = ( "Stop and reply to the user now. Say what you changed. Mention unfinished work " "only if there is any, and then offer to continue next turn — otherwise don't " @@ -233,7 +238,7 @@ def __init__(self, config_loader: ConfigLoader, api_key: Optional[str] = None): planner_config = config_loader.config.get("planner", {}) self.model = resolve_model(planner_config.get("model", "claude-opus")) - self.max_tokens = planner_config.get("max_tokens", 24576) + self.max_tokens = planner_config.get("max_tokens", 32768) self.max_tool_calls = planner_config.get("max_tool_calls", 20) self.current_yaml: Optional[str] = None @@ -335,6 +340,10 @@ def run( logger.info(f"Claude API call {tool_call_count + 1}: stop_reason={response.stop_reason}") + refusal = refusal_error(response) + if refusal: + raise refusal + # Text from every round is part of the answer the user saw # (tool rounds may narrate before calling tools). round_text = self._extract_text(response) @@ -533,10 +542,10 @@ def _call_api(self, system_prompt, messages, stream, stream_manager, tool_choice rounds close the open text block), so the client can weave text and status events with its own formatting. - Adaptive thinking is enabled for better reasoning but thinking content - is not streamed to the client — it exposes internal details like tool - names and agent architecture. User-facing progress comes from the - task-specific status messages sent before each tool execution. + Reasoning is never streamed — it exposes internal details like tool + names and agent architecture — and under display "updates" it comes + back empty. The only thinking text is the model's progress notes + between tool calls, which stream as narration. """ choice = {"tool_choice": tool_choice} if tool_choice else {} @@ -547,14 +556,19 @@ def _call_api(self, system_prompt, messages, stream, stream_manager, tool_choice system=system_prompt, messages=messages, tools=self.tools, - thinking={"type": "adaptive"}, + thinking=_THINKING, output_config={"effort": "medium"}, + betas=_BETAS, **choice, **_CONTEXT_MANAGEMENT, ) as stream_obj: for event in stream_obj: - if event.type == "content_block_delta" and event.delta.type == "text_delta": + if event.type != "content_block_delta": + continue + if event.delta.type == "text_delta": stream_manager.send_text(event.delta.text) + elif event.delta.type == "thinking_delta" and event.delta.thinking: + stream_manager.send_text(event.delta.thinking) return stream_obj.get_final_message() else: response = self.client.beta.messages.create( @@ -563,8 +577,9 @@ def _call_api(self, system_prompt, messages, stream, stream_manager, tool_choice system=system_prompt, messages=messages, tools=self.tools, - thinking={"type": "adaptive"}, + thinking=_THINKING, output_config={"effort": "medium"}, + betas=_BETAS, **choice, # Per-request timeout (same values as the SDK default): # required for non-streaming calls with max_tokens > ~21k, @@ -969,8 +984,12 @@ def _format_display_name(name: str) -> str: return name.replace("-", " ").replace("_", " ").title() def _extract_text(self, response): - """Extract text from response content, concatenated as it was streamed.""" - return "".join(block.text for block in response.content if block.type == "text") + """Extract text and progress notes, concatenated as they were streamed.""" + return "".join( + block.text if block.type == "text" else block.thinking + for block in response.content + if block.type == "text" or (block.type == "thinking" and block.thinking) + ) def _build_system_prompt(self) -> list: """Build system prompt for planner with cache control.""" diff --git a/services/global_chat/router.py b/services/global_chat/router.py index 2fd58247..918f92f8 100644 --- a/services/global_chat/router.py +++ b/services/global_chat/router.py @@ -69,7 +69,7 @@ def __init__(self, config_loader: ConfigLoader, api_key: Optional[str] = None): router_config = config_loader.config.get("router", {}) self.model = resolve_model(router_config.get("model", "claude-haiku")) - self.max_tokens = router_config.get("max_tokens", 500) + self.max_tokens = router_config.get("max_tokens", 4096) logger.info(f"RouterAgent initialized with model: {self.model}") @@ -187,7 +187,7 @@ def _make_routing_decision( "cache_read_input_tokens": getattr(response.usage, "cache_read_input_tokens", 0), } - response_text = response.content[0].text if response.content else "{}" + response_text = "".join(block.text for block in response.content if block.type == "text") or "{}" try: decision_data = json.loads(response_text) diff --git a/services/global_chat/tests/unit/test_planner.py b/services/global_chat/tests/unit/test_planner.py index f66fb9ec..0398f067 100644 --- a/services/global_chat/tests/unit/test_planner.py +++ b/services/global_chat/tests/unit/test_planner.py @@ -452,6 +452,48 @@ def test_model_api_error_is_not_reported_as_a_tool_failure() -> None: ) +class FakeThinking: + type = "thinking" + signature = "sig" + + def __init__(self, thinking: str) -> None: + self.thinking = thinking + + +def test_a_refusal_is_reported_as_one_not_as_an_empty_answer() -> None: + """A decline is a 200 with no text, which otherwise surfaces as our own + EMPTY_OUTPUT failure.""" + planner = make_planner() + planner.model = "test-model" + planner.max_tool_calls = 5 + refused = FakeResponse("refusal", []) + refused.stop_details = type("StopDetails", (), {"category": "cyber"})() + + with ( + patch.object(planner, "_call_api", return_value=refused), + patch.object(planner, "_build_system_prompt", return_value="sys"), + pytest.raises(ApolloError) as raised, + ): + planner.run(content="x", workflow_yaml=None, page=None, history=[], stream=False) + + assert raised.value.code == 422 + assert raised.value.type == "MODEL_REFUSAL" + assert raised.value.details == {"category": "cyber"} + + +def test_progress_notes_count_as_narration_and_hidden_reasoning_does_not() -> None: + """Notes written before a tool call arrive as thinking blocks; reasoning + arrives as thinking blocks too, but empty.""" + response = FakeResponse("tool_use", [ + FakeThinking(""), + FakeThinking("Checking the fetch step first."), + FakeText(" Then the mapping."), + FakeToolUse("inspect_job_code", {"job_keys": ["a"]}, "tu_1"), + ]) + + assert make_planner()._extract_text(response) == "Checking the fetch step first. Then the mapping." + + def _raise_from_planner(error: Exception) -> ApolloError: planner = make_planner() planner.model = "test-model" diff --git a/services/job_chat/job_chat.py b/services/job_chat/job_chat.py index 9cc70017..57b9965d 100644 --- a/services/job_chat/job_chat.py +++ b/services/job_chat/job_chat.py @@ -19,7 +19,7 @@ import sentry_sdk from langfuse import observe, propagate_attributes, get_client as get_langfuse_client from langfuse_util import should_track, build_tags, build_generation_diff, mask_secrets -from util import ApolloError, create_logger, AdaptorSpecifier, add_page_prefix, APOLLO_VERSION +from util import ApolloError, create_logger, AdaptorSpecifier, add_page_prefix, refusal_error, APOLLO_VERSION from yaml_utils import INSPECT_JOB_CODE_TOOL, inspect_job_code from .prompt import build_prompt, build_error_correction_prompt from .old_prompt import build_old_prompt @@ -192,7 +192,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "Payload": @dataclass class ChatConfig: model: str = _MODEL - max_tokens: int = 24576 + max_tokens: int = 32768 api_key: Optional[str] = None @@ -378,6 +378,11 @@ def generate( if hasattr(message, "usage"): usage_events.append(message.usage.model_dump()) + refusal = refusal_error(message) + if refusal: + stream_manager.end_stream() + raise refusal + for content_block in message.content: if getattr(content_block, "type", None) == "text" and content_block.text: text_parts.append(content_block.text) diff --git a/services/job_chat/retrieve_docs.py b/services/job_chat/retrieve_docs.py index 9e80cf95..cf73ebed 100644 --- a/services/job_chat/retrieve_docs.py +++ b/services/job_chat/retrieve_docs.py @@ -223,7 +223,7 @@ def call_llm(model, system_prompt, user_prompt, client, output_schema=None): kwargs = dict( model=model, - max_tokens=1024, + max_tokens=4096, system=system_prompt, messages=messages ) @@ -238,7 +238,7 @@ def call_llm(model, system_prompt, user_prompt, client, output_schema=None): if not message.content or len(message.content) == 0: raise ApolloError(500, "Empty response from AI service", type="EMPTY_LLM_RESPONSE") - response_text = message.content[0].text + response_text = "".join(block.text for block in message.content if block.type == "text") if not response_text: raise ApolloError(500, "Empty text in AI service response", type="EMPTY_LLM_RESPONSE") diff --git a/services/job_chat/tests/unit/test_retrieve_docs.py b/services/job_chat/tests/unit/test_retrieve_docs.py index 58b60b03..7ada6709 100644 --- a/services/job_chat/tests/unit/test_retrieve_docs.py +++ b/services/job_chat/tests/unit/test_retrieve_docs.py @@ -44,7 +44,7 @@ def test_generate_queries_raises_apollo_error_on_invalid_json(): def test_call_llm_returns_text_and_usage_on_success(): message = MagicMock() - message.content = [MagicMock(text="hello")] + message.content = [MagicMock(type="text", text="hello")] message.usage.model_dump.return_value = {"input_tokens": 5, "output_tokens": 2} client = MagicMock() client.messages.create.return_value = message diff --git a/services/models.py b/services/models.py index 8a70b0a1..745cb34a 100644 --- a/services/models.py +++ b/services/models.py @@ -4,9 +4,9 @@ """ CLAUDE_MODELS: dict[str, str] = { - "claude-opus": "claude-opus-4-8", - "claude-sonnet": "claude-sonnet-4-6", - "claude-haiku": "claude-haiku-4-5-20251001", + "claude-opus": "claude-opus-5-5", + "claude-sonnet": "claude-sonnet-5-5", + "claude-haiku": "claude-haiku-5-5", } CLAUDE_OPUS: str = CLAUDE_MODELS["claude-opus"] diff --git a/services/testing/judge.py b/services/testing/judge.py index b5cc9d59..b6ef8714 100644 --- a/services/testing/judge.py +++ b/services/testing/judge.py @@ -423,7 +423,7 @@ def evaluate( ], ) - raw_text = response.content[0].text + raw_text = "".join(block.text for block in response.content if block.type == "text") usage = { "input_tokens": response.usage.input_tokens, "output_tokens": response.usage.output_tokens, diff --git a/services/util.py b/services/util.py index 1be26a92..cb4ff8c5 100644 --- a/services/util.py +++ b/services/util.py @@ -397,6 +397,24 @@ def check_attachment_size(attachments: list[dict] | None) -> None: ) +def refusal_error(message: Any) -> ApolloError | None: # noqa: ANN401 + """The typed error for a safety decline, or None if the model answered. + + A decline is an HTTP 200 with stop_reason "refusal", so nothing upstream + raises; left alone it reads as an empty or unparseable answer and gets + retried or reported as our failure. + """ + if getattr(message, "stop_reason", None) != "refusal": + return None + category = getattr(getattr(message, "stop_details", None), "category", None) + return ApolloError( + 422, + "The AI service declined to answer this request. Try rephrasing it.", + type="MODEL_REFUSAL", + details={"category": category}, + ) + + # Payload attachment type -> the job_chat context field that already renders it. # job_chat has had typed // blocks all along; attachments # arriving from global_chat reuse them rather than adding a second channel. diff --git a/services/vocab_mapper/vocab_mapper.py b/services/vocab_mapper/vocab_mapper.py index 8310f1d9..8265a1f0 100644 --- a/services/vocab_mapper/vocab_mapper.py +++ b/services/vocab_mapper/vocab_mapper.py @@ -38,7 +38,7 @@ def _call_llm(self, system_prompt: str, user_prompt: str) -> str: """Helper method to make LLM calls.""" message = self.client.messages.create( model=CLAUDE_SONNET, - max_tokens=500, + max_tokens=4096, system=system_prompt, messages=[ { @@ -52,7 +52,7 @@ def _call_llm(self, system_prompt: str, user_prompt: str) -> str: } ] ) - return message.content[0].text + return "".join(block.text for block in message.content if block.type == "text") def _call_llm_batch(self, system_prompt: str, user_prompts: List[str]) -> List[str]: """Process a batch of LLM inputs concurrently.""" diff --git a/services/workflow_chat/workflow_chat.py b/services/workflow_chat/workflow_chat.py index 4b668b05..4fb3f236 100644 --- a/services/workflow_chat/workflow_chat.py +++ b/services/workflow_chat/workflow_chat.py @@ -3,6 +3,7 @@ import re import uuid from typing import List, Optional, Dict, Any +import httpx2 import yaml from dataclasses import dataclass from models import resolve_model @@ -70,7 +71,7 @@ truncate_graphemes, unicode_names_enabled, ) -from util import ApolloError, create_logger, add_page_prefix, APOLLO_VERSION +from util import ApolloError, create_logger, add_page_prefix, refusal_error, APOLLO_VERSION from yaml_utils import _remove_ids from .gen_project_prompt import build_prompt from workflow_chat.available_adaptors import get_available_adaptors @@ -150,7 +151,7 @@ def from_dict(cls, data: Dict[str, Any]) -> "Payload": @dataclass class ChatConfig: model: str = _MODEL - max_tokens: int = 16384 + max_tokens: int = 32768 api_key: Optional[str] = None @@ -304,7 +305,11 @@ def generate( message = self.client.messages.create( max_tokens=self.config.max_tokens, messages=prompt, model=self.config.model, system=system_message, output_config=output_config, - thinking={"type": "adaptive"} + thinking={"type": "adaptive"}, + # Per-request timeout (same values as the SDK default): + # required for non-streaming calls with max_tokens > ~21k, + # which the SDK otherwise rejects. + timeout=httpx2.Timeout(600.0, connect=5.0), ) # Track usage from this attempt @@ -314,6 +319,13 @@ def generate( if key in usage: accumulated_usage[key] += usage[key] + # Before parsing: a decline carries no JSON, so it would read + # as a parse failure and be retried. + refusal = refusal_error(message) + if refusal: + stream_manager.end_stream() + raise refusal + response_parts = [] for content_block in message.content: if content_block.type == "text": From 01c3c878957575b7460adb025abb3a3f7f51783b Mon Sep 17 00:00:00 2001 From: hanna-paasivirta Date: Tue, 13 Oct 2026 09:56:29 +0100 Subject: [PATCH 2/3] Add changesets for the model update --- .changeset/claude-5-5-models.md | 5 +++++ .changeset/model-refusal-error.md | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 .changeset/claude-5-5-models.md create mode 100644 .changeset/model-refusal-error.md diff --git a/.changeset/claude-5-5-models.md b/.changeset/claude-5-5-models.md new file mode 100644 index 00000000..a7253083 --- /dev/null +++ b/.changeset/claude-5-5-models.md @@ -0,0 +1,5 @@ +--- +"apollo": minor +--- + +Use Claude Opus 5.5, Sonnet 5.5 and Haiku 5.5 across all services diff --git a/.changeset/model-refusal-error.md b/.changeset/model-refusal-error.md new file mode 100644 index 00000000..8f4991cb --- /dev/null +++ b/.changeset/model-refusal-error.md @@ -0,0 +1,5 @@ +--- +"apollo": patch +--- + +Return a `MODEL_REFUSAL` error when the AI service declines a request, instead of an empty-output error From 712f7d724e77463083416971ea84ed4b688ecabd Mon Sep 17 00:00:00 2001 From: hanna-paasivirta Date: Mon, 19 Oct 2026 12:59:08 +0300 Subject: [PATCH 3/3] Turn off thinking in the router --- services/global_chat/config.yaml | 2 +- services/global_chat/router.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/services/global_chat/config.yaml b/services/global_chat/config.yaml index 81a6ad7d..f199762b 100644 --- a/services/global_chat/config.yaml +++ b/services/global_chat/config.yaml @@ -3,7 +3,7 @@ # Router configuration (lightweight routing) router: model: "claude-haiku" - max_tokens: 4096 + max_tokens: 500 # Planner configuration (complex orchestration) planner: diff --git a/services/global_chat/router.py b/services/global_chat/router.py index db8b3be6..6b08e624 100644 --- a/services/global_chat/router.py +++ b/services/global_chat/router.py @@ -69,7 +69,7 @@ def __init__(self, config_loader: ConfigLoader, api_key: Optional[str] = None): router_config = config_loader.config.get("router", {}) self.model = resolve_model(router_config.get("model", "claude-haiku")) - self.max_tokens = router_config.get("max_tokens", 4096) + self.max_tokens = router_config.get("max_tokens", 500) logger.info(f"RouterAgent initialized with model: {self.model}") @@ -184,6 +184,9 @@ def _make_routing_decision( messages=[ {"role": "user", "content": routing_message} ], + # Routing is meant to be fast: a request that needs reasoning is + # one to send to the planner, not to think about here. + thinking={"type": "disabled"}, output_config={"format": {"type": "json_schema", "schema": routing_schema}} )