diff --git a/crates/tinyagents-harness/src/agent_loop/unknown_tool.rs b/crates/tinyagents-harness/src/agent_loop/unknown_tool.rs index d9ecda3ce..87f674b34 100644 --- a/crates/tinyagents-harness/src/agent_loop/unknown_tool.rs +++ b/crates/tinyagents-harness/src/agent_loop/unknown_tool.rs @@ -34,6 +34,9 @@ pub(super) fn unknown_tool_message( available: &[String], tool_search_available: bool, ) -> String { + if !is_tool_identifier(requested) { + return malformed_name_message(requested, available, tool_search_available); + } let args_repr = serde_json::to_string(arguments).unwrap_or_else(|_| "".into()); let mut message = format!( "unknown tool `{requested}` (arguments: {args_repr}): no tool with that name is available to you, and calling it \ @@ -59,6 +62,101 @@ pub(super) fn unknown_tool_message( message } +/// Longest catalog the format corrective lists by name. Past this the model is +/// sent to `tool_search`, or to its tool list, as the ordinary corrective does. +const MAX_LISTED_NAMES: usize = 12; + +/// Longest slice of a malformed name echoed back. The whole value can be a +/// multi-kilobyte JSON call; the model only needs to recognise it. +const MALFORMED_ECHO_CHARS: usize = 60; + +/// Whether `name` could be a tool's name at all: non-empty, bounded, and free +/// of whitespace, quotes, brackets and markup. +/// +/// Deliberately a *negative* test. Real catalogs carry `snake_case`, +/// `SCREAMING_CASE` connector actions, and MCP names with `.`, `-`, `:` or `/`; +/// none of those carry the characters that only show up when something other +/// than a name landed in the name field: a whole `{"name":…,"arguments":…}` +/// object, a command line, or leftover `` markup. +pub(super) fn is_tool_identifier(name: &str) -> bool { + !name.is_empty() + && name.len() <= 256 + && !name.chars().any(|c| { + c.is_whitespace() + || c.is_control() + || matches!( + c, + '{' | '}' | '[' | ']' | '(' | ')' | '<' | '>' | '"' | '\'' | '`' | ',' + ) + }) +} + +/// The corrective for a call whose name is not a tool name at all. +/// +/// A gateway that parses `` text with its own template can hand +/// back the model's whole JSON call as the *name*, with `{}` arguments. Told +/// only that no such tool exists, the model has no way to see what went wrong +/// and keeps retrying the same shape. This says it plainly, names the tool the +/// value seems to carry, and lists what may be called. The +/// ``unknown tool `…` `` prefix is kept: hosts classify the result by it. +fn malformed_name_message( + requested: &str, + available: &[String], + tool_search_available: bool, +) -> String { + let echo: String = requested.chars().take(MALFORMED_ECHO_CHARS).collect(); + let echo = echo.replace('`', "'"); + let ellipsis = if requested.chars().count() > MALFORMED_ECHO_CHARS { + "…" + } else { + "" + }; + let mut message = + format!("unknown tool `{echo}{ellipsis}`: that is not a tool name, so no tool ran."); + if let Some(carried) = carried_tool_name(requested) { + message.push_str(&format!( + " It looks like a whole tool call (`{carried}` with its arguments) was put where \ + only the name belongs." + )); + } + message.push_str( + " Call the tool again with the bare tool name as the name and the arguments as a \ + JSON object.", + ); + if !available.is_empty() && available.len() <= MAX_LISTED_NAMES { + let names = available + .iter() + .map(|name| format!("`{name}`")) + .collect::>() + .join(", "); + message.push_str(&format!(" Tools you can call: {names}.")); + } else if tool_search_available { + message.push_str(&format!( + " To find the right tool, call `{TOOL_SEARCH_NAME}` with what you want to do in plain \ + words." + )); + } else { + message.push_str(" Use only the tools in your tool list."); + } + message +} + +/// The tool name inside a JSON call object, when `value` is one +/// (`{"name": "shell", "arguments": …}`, in any key order). +fn carried_tool_name(value: &str) -> Option { + let parsed: serde_json::Value = serde_json::from_str(value.trim()).ok()?; + let name = parsed + .get("name") + .or_else(|| { + parsed + .get("function") + .and_then(|function| function.get("name")) + })? + .as_str()? + .trim(); + is_tool_identifier(name).then(|| name.to_string()) +} + /// Lower-case name tokens, split on anything that is not alphanumeric. fn tokens(name: &str) -> Vec { name.split(|c: char| !c.is_ascii_alphanumeric()) diff --git a/crates/tinyagents-harness/src/agent_loop/unknown_tool_tests.rs b/crates/tinyagents-harness/src/agent_loop/unknown_tool_tests.rs index ed36cbfd5..4b8c72da1 100644 --- a/crates/tinyagents-harness/src/agent_loop/unknown_tool_tests.rs +++ b/crates/tinyagents-harness/src/agent_loop/unknown_tool_tests.rs @@ -106,3 +106,69 @@ fn message_echoes_the_attempted_arguments() { assert!(message.starts_with("unknown tool `missing`"), "{message}"); assert!(message.contains("\"path\":\"/tmp/a\""), "{message}"); } + +#[test] +fn a_json_tool_call_in_the_name_gets_a_format_corrective() { + // Seen live: an OpenAI-compatible gateway parsed the model's + // `{json}` text in its own template and handed back + // a structured call whose *name* was the whole JSON body, arguments `{}`. + // Echoing that back as "no tool with that name" sent the model reasoning + // about key order for the rest of the run. + let body = r#"{"arguments": {"command": "cd /srv/teeny/state/workbench/brain && git status --short --branch && cat brain/cron.toml"}, "name": "shell"}"#; + let message = unknown_tool_message( + body, + &serde_json::json!({}), + &names(&["shell", "file_read", "file_write"]), + false, + ); + assert!(message.starts_with("unknown tool `"), "{message}"); + assert!(message.contains("is not a tool name"), "{message}"); + assert!( + message.contains("a whole tool call (`shell` with its arguments)"), + "names the tool it carried: {message}" + ); + assert!( + message.contains("`shell`, `file_read`, `file_write`"), + "lists the callable names: {message}" + ); + assert!( + !message.contains("cron.toml"), + "does not echo the whole body: {message}" + ); + assert!(message.len() < 600, "corrective must stay small: {message}"); +} + +#[test] +fn a_name_with_spaces_or_markup_is_not_treated_as_a_typo() { + for bad in ["shell ls -la", "shell", "\"shell\"", ""] { + let message = unknown_tool_message(bad, &serde_json::json!({}), &names(&["shell"]), false); + assert!(message.contains("is not a tool name"), "{bad:?}: {message}"); + } +} + +#[test] +fn ordinary_identifiers_keep_the_plain_corrective() { + for name in [ + "web_search_tool", + "GITHUB_LIST_ISSUES", + "mcp.server-tool", + "ns:tool/x", + ] { + let message = unknown_tool_message(name, &serde_json::json!({}), &names(&["shell"]), false); + assert!(!message.contains("is not a tool name"), "{name}: {message}"); + } +} + +#[test] +fn a_bad_name_with_a_large_catalog_points_to_tool_search() { + let available: Vec = (0..50).map(|i| format!("tool_{i}")).collect(); + let message = unknown_tool_message( + r#"{"name":"tool_3","arguments":{}}"#, + &serde_json::json!({}), + &available, + true, + ); + assert!(message.contains("is not a tool name"), "{message}"); + assert!(message.contains("`tool_search`"), "{message}"); + assert!(!message.contains("tool_49"), "{message}"); +}