From ae1f09dd3664796a61a35eaeebb090e8bbc7328e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Thu, 12 Nov 2026 13:29:48 +0000 Subject: [PATCH] feat(openai): let a host turn native tool calls on for a local runtime build_local_runtime_chat_model always set native_tool_calling = false. That is the right default for Ollama or LM Studio, but local-openai also fronts hosted OpenAI-compatible gateways whose models call tools natively and whose servers parse text in the model's own template. Prompted with the JSON-in-tag protocol there, the gateway reads the JSON as the tool's name. Add build_local_runtime_chat_model_with_native_tools, which takes the choice from the host and keeps the rest of the local-runtime profile. The old builder delegates to it with false, so existing callers are unchanged. Co-authored-by: Medulla --- .../src/providers/openai/config.rs | 37 +++++++++++++++++- .../src/providers/openai/config_tests.rs | 39 +++++++++++++++++++ .../src/providers/openai/mod.rs | 5 ++- 3 files changed, 78 insertions(+), 3 deletions(-) diff --git a/crates/tinyinference-llm/src/providers/openai/config.rs b/crates/tinyinference-llm/src/providers/openai/config.rs index f2d45e09..468675ce 100644 --- a/crates/tinyinference-llm/src/providers/openai/config.rs +++ b/crates/tinyinference-llm/src/providers/openai/config.rs @@ -262,6 +262,41 @@ pub fn build_local_runtime_chat_model( temperature_unsupported_models: &[String], temperature_override: Option, num_ctx: Option, +) -> Arc> { + build_local_runtime_chat_model_with_native_tools( + provider_name, + endpoint, + api_key, + auth_style, + model, + temperature_unsupported_models, + temperature_override, + num_ctx, + false, + ) +} + +/// [`build_local_runtime_chat_model`] with native tool calling chosen by the +/// host instead of always off. +/// +/// Off is the safe default for a model on a local runtime, which often cannot +/// take `tools`. But `local-openai` also fronts hosted OpenAI-compatible +/// gateways whose models call tools natively and whose servers parse +/// `` text in the model's own template: prompted with the JSON-in-tag +/// protocol there, the gateway reads the JSON as the tool's name. A host that +/// knows its endpoint supports native calls passes `true`; nothing else about +/// the local-runtime profile changes. +#[allow(clippy::too_many_arguments)] +pub fn build_local_runtime_chat_model_with_native_tools( + provider_name: &str, + endpoint: &str, + api_key: &str, + auth_style: AuthStyle, + model: &str, + temperature_unsupported_models: &[String], + temperature_override: Option, + num_ctx: Option, + native_tool_calling: bool, ) -> Arc> { let default_provider_options = num_ctx.map(|value| { json!({ @@ -279,7 +314,7 @@ pub fn build_local_runtime_chat_model( temperature_override, merge_system_into_user: false, extra_headers: &[], - native_tool_calling: Some(false), + native_tool_calling: Some(native_tool_calling), vision: Some(false), default_provider_options, responses_api_primary: false, diff --git a/crates/tinyinference-llm/src/providers/openai/config_tests.rs b/crates/tinyinference-llm/src/providers/openai/config_tests.rs index 8eccc970..fec84552 100644 --- a/crates/tinyinference-llm/src/providers/openai/config_tests.rs +++ b/crates/tinyinference-llm/src/providers/openai/config_tests.rs @@ -49,6 +49,45 @@ fn local_runtime_builder_disables_native_tools_and_vision() { assert!(profile.hoists_system_messages); } +#[test] +fn local_runtime_builder_can_opt_into_native_tools() { + // An OpenAI-compatible endpoint behind `local-openai` may serve a model + // with working native tool calls (GLM on a hosted gateway). Prompting it + // with the text protocol instead lets the gateway's own tool parser read + // the JSON-in-tag call as a tool *name*, so the host must be able to + // turn native calls on for it. + let model = build_local_runtime_chat_model_with_native_tools( + "local-openai", + "https://gateway.example/v1", + "secret", + AuthStyle::Bearer, + "glm-5.3-flash", + &[], + None, + None, + true, + ); + let profile = model.profile().expect("openai models expose a profile"); + assert!(profile.tool_calling); + // Everything else stays the local-runtime profile. + assert!(!profile.modalities.image_in); + assert!(!profile.mid_conversation_system_messages); + assert!(profile.hoists_system_messages); + + let off = build_local_runtime_chat_model_with_native_tools( + "local-openai", + "https://gateway.example/v1", + "secret", + AuthStyle::Bearer, + "glm-5.3-flash", + &[], + None, + None, + false, + ); + assert!(!off.profile().expect("profile").tool_calling); +} + #[test] fn openrouter_endpoints_are_recognized() { assert!(endpoint_is_openrouter("https://openrouter.ai/api/v1")); diff --git a/crates/tinyinference-llm/src/providers/openai/mod.rs b/crates/tinyinference-llm/src/providers/openai/mod.rs index 740b0abf..dcbdf643 100644 --- a/crates/tinyinference-llm/src/providers/openai/mod.rs +++ b/crates/tinyinference-llm/src/providers/openai/mod.rs @@ -95,8 +95,9 @@ mod sse; mod transport; pub use config::{ - OpenAiConfig, build_local_runtime_chat_model, build_openai_chat_model, build_openai_model, - build_openai_model_with_http, endpoint_is_openrouter, + OpenAiConfig, build_local_runtime_chat_model, build_local_runtime_chat_model_with_native_tools, + build_openai_chat_model, build_openai_model, build_openai_model_with_http, + endpoint_is_openrouter, }; pub use local::{ CONTEXT_OVERFLOW_CODE, LocalProbe, LocalRuntimeKind, is_chat_template_rejection_message,