From 51cfbde057c57797975bc47c4e5096c19fc1b089 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 12 Nov 2026 13:33:06 +0000 Subject: [PATCH 1/2] fix(agent_loop): tell the model when a call's name is not a tool name A gateway that parses text in its own template returned structured calls whose name was the model's whole JSON call and whose arguments were {}. The corrective echoed the blob back as "no tool with that name", and the model spent the rest of its run reasoning about key order. When the requested name cannot be a tool name (whitespace, quotes, brackets, markup, empty), the corrective now says so, names the tool a JSON call carried, says to resend with the bare name and object arguments, and lists the callable tools (or points to tool_search for a large catalog). It keeps the `unknown tool ` prefix hosts classify on and echoes at most 60 chars. Co-authored-by: Medulla --- .../src/agent_loop/unknown_tool.rs | 98 +++++++++++++++++++ .../src/agent_loop/unknown_tool_tests.rs | 66 +++++++++++++ 2 files changed, 164 insertions(+) diff --git a/crates/tinyagents-harness/src/agent_loop/unknown_tool.rs b/crates/tinyagents-harness/src/agent_loop/unknown_tool.rs index d9ecda3ce..87f674b34 100644 --- a/crates/tinyagents-harness/src/agent_loop/unknown_tool.rs +++ b/crates/tinyagents-harness/src/agent_loop/unknown_tool.rs @@ -34,6 +34,9 @@ pub(super) fn unknown_tool_message( available: &[String], tool_search_available: bool, ) -> String { + if !is_tool_identifier(requested) { + return malformed_name_message(requested, available, tool_search_available); + } let args_repr = serde_json::to_string(arguments).unwrap_or_else(|_| "".into()); let mut message = format!( "unknown tool `{requested}` (arguments: {args_repr}): no tool with that name is available to you, and calling it \ @@ -59,6 +62,101 @@ pub(super) fn unknown_tool_message( message } +/// Longest catalog the format corrective lists by name. Past this the model is +/// sent to `tool_search`, or to its tool list, as the ordinary corrective does. +const MAX_LISTED_NAMES: usize = 12; + +/// Longest slice of a malformed name echoed back. The whole value can be a +/// multi-kilobyte JSON call; the model only needs to recognise it. +const MALFORMED_ECHO_CHARS: usize = 60; + +/// Whether `name` could be a tool's name at all: non-empty, bounded, and free +/// of whitespace, quotes, brackets and markup. +/// +/// Deliberately a *negative* test. Real catalogs carry `snake_case`, +/// `SCREAMING_CASE` connector actions, and MCP names with `.`, `-`, `:` or `/`; +/// none of those carry the characters that only show up when something other +/// than a name landed in the name field: a whole `{"name":…,"arguments":…}` +/// object, a command line, or leftover `` markup. +pub(super) fn is_tool_identifier(name: &str) -> bool { + !name.is_empty() + && name.len() <= 256 + && !name.chars().any(|c| { + c.is_whitespace() + || c.is_control() + || matches!( + c, + '{' | '}' | '[' | ']' | '(' | ')' | '<' | '>' | '"' | '\'' | '`' | ',' + ) + }) +} + +/// The corrective for a call whose name is not a tool name at all. +/// +/// A gateway that parses `` text with its own template can hand +/// back the model's whole JSON call as the *name*, with `{}` arguments. Told +/// only that no such tool exists, the model has no way to see what went wrong +/// and keeps retrying the same shape. This says it plainly, names the tool the +/// value seems to carry, and lists what may be called. The +/// ``unknown tool `…` `` prefix is kept: hosts classify the result by it. +fn malformed_name_message( + requested: &str, + available: &[String], + tool_search_available: bool, +) -> String { + let echo: String = requested.chars().take(MALFORMED_ECHO_CHARS).collect(); + let echo = echo.replace('`', "'"); + let ellipsis = if requested.chars().count() > MALFORMED_ECHO_CHARS { + "…" + } else { + "" + }; + let mut message = + format!("unknown tool `{echo}{ellipsis}`: that is not a tool name, so no tool ran."); + if let Some(carried) = carried_tool_name(requested) { + message.push_str(&format!( + " It looks like a whole tool call (`{carried}` with its arguments) was put where \ + only the name belongs." + )); + } + message.push_str( + " Call the tool again with the bare tool name as the name and the arguments as a \ + JSON object.", + ); + if !available.is_empty() && available.len() <= MAX_LISTED_NAMES { + let names = available + .iter() + .map(|name| format!("`{name}`")) + .collect::>() + .join(", "); + message.push_str(&format!(" Tools you can call: {names}.")); + } else if tool_search_available { + message.push_str(&format!( + " To find the right tool, call `{TOOL_SEARCH_NAME}` with what you want to do in plain \ + words." + )); + } else { + message.push_str(" Use only the tools in your tool list."); + } + message +} + +/// The tool name inside a JSON call object, when `value` is one +/// (`{"name": "shell", "arguments": …}`, in any key order). +fn carried_tool_name(value: &str) -> Option { + let parsed: serde_json::Value = serde_json::from_str(value.trim()).ok()?; + let name = parsed + .get("name") + .or_else(|| { + parsed + .get("function") + .and_then(|function| function.get("name")) + })? + .as_str()? + .trim(); + is_tool_identifier(name).then(|| name.to_string()) +} + /// Lower-case name tokens, split on anything that is not alphanumeric. fn tokens(name: &str) -> Vec { name.split(|c: char| !c.is_ascii_alphanumeric()) diff --git a/crates/tinyagents-harness/src/agent_loop/unknown_tool_tests.rs b/crates/tinyagents-harness/src/agent_loop/unknown_tool_tests.rs index ed36cbfd5..71e0e5b45 100644 --- a/crates/tinyagents-harness/src/agent_loop/unknown_tool_tests.rs +++ b/crates/tinyagents-harness/src/agent_loop/unknown_tool_tests.rs @@ -106,3 +106,69 @@ fn message_echoes_the_attempted_arguments() { assert!(message.starts_with("unknown tool `missing`"), "{message}"); assert!(message.contains("\"path\":\"/tmp/a\""), "{message}"); } + +#[test] +fn a_json_tool_call_in_the_name_gets_a_format_corrective() { + // Seen live: an OpenAI-compatible gateway parsed the model's + // `{json}` text in its own template and handed back + // a structured call whose *name* was the whole JSON body, arguments `{}`. + // Echoing that back as "no tool with that name" sent the model reasoning + // about key order for the rest of the run. + let body = r#"{"arguments": {"command": "cd /srv/teeny/state/workbench/brain && git status --short --branch && cat brain/cron.toml"}, "name": "shell"}"#; + let message = unknown_tool_message( + body, + &serde_json::json!({}), + &names(&["shell", "file_read", "file_write"]), + false, + ); + assert!(message.starts_with("unknown tool `"), "{message}"); + assert!(message.contains("is not a tool name"), "{message}"); + assert!( + message.contains("`shell`"), + "names the tool it carried: {message}" + ); + assert!( + message.contains("`shell`, `file_read`, `file_write`"), + "lists the callable names: {message}" + ); + assert!( + !message.contains("cron.toml"), + "does not echo the whole body: {message}" + ); + assert!(message.len() < 600, "corrective must stay small: {message}"); +} + +#[test] +fn a_name_with_spaces_or_markup_is_not_treated_as_a_typo() { + for bad in ["shell ls -la", "shell", "\"shell\"", ""] { + let message = unknown_tool_message(bad, &serde_json::json!({}), &names(&["shell"]), false); + assert!(message.contains("is not a tool name"), "{bad:?}: {message}"); + } +} + +#[test] +fn ordinary_identifiers_keep_the_plain_corrective() { + for name in [ + "web_search_tool", + "GITHUB_LIST_ISSUES", + "mcp.server-tool", + "ns:tool/x", + ] { + let message = unknown_tool_message(name, &serde_json::json!({}), &names(&["shell"]), false); + assert!(!message.contains("is not a tool name"), "{name}: {message}"); + } +} + +#[test] +fn a_bad_name_with_a_large_catalog_points_to_tool_search() { + let available: Vec = (0..50).map(|i| format!("tool_{i}")).collect(); + let message = unknown_tool_message( + r#"{"name":"tool_3","arguments":{}}"#, + &serde_json::json!({}), + &available, + true, + ); + assert!(message.contains("is not a tool name"), "{message}"); + assert!(message.contains("`tool_search`"), "{message}"); + assert!(!message.contains("tool_49"), "{message}"); +} From 13a070c8a8e79ea5fc5c15ef7780b28a5f8f2088 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 12 Nov 2026 13:56:25 +0000 Subject: [PATCH 2/2] test(agent_loop): pin the carried-name explanation, not just the tool name The catalog lists shell too, so the old assertion passed even if the JSON name extraction returned nothing. Co-authored-by: Medulla --- crates/tinyagents-harness/src/agent_loop/unknown_tool_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/tinyagents-harness/src/agent_loop/unknown_tool_tests.rs b/crates/tinyagents-harness/src/agent_loop/unknown_tool_tests.rs index 71e0e5b45..4b8c72da1 100644 --- a/crates/tinyagents-harness/src/agent_loop/unknown_tool_tests.rs +++ b/crates/tinyagents-harness/src/agent_loop/unknown_tool_tests.rs @@ -124,7 +124,7 @@ fn a_json_tool_call_in_the_name_gets_a_format_corrective() { assert!(message.starts_with("unknown tool `"), "{message}"); assert!(message.contains("is not a tool name"), "{message}"); assert!( - message.contains("`shell`"), + message.contains("a whole tool call (`shell` with its arguments)"), "names the tool it carried: {message}" ); assert!(