Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 36 additions & 1 deletion crates/tinyinference-llm/src/providers/openai/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,41 @@ pub fn build_local_runtime_chat_model(
temperature_unsupported_models: &[String],
temperature_override: Option<f64>,
num_ctx: Option<u32>,
) -> Arc<dyn ChatModel<()>> {
build_local_runtime_chat_model_with_native_tools(
provider_name,
endpoint,
api_key,
auth_style,
model,
temperature_unsupported_models,
temperature_override,
num_ctx,
false,
)
}

/// [`build_local_runtime_chat_model`] with native tool calling chosen by the
/// host instead of always off.
///
/// Off is the safe default for a model on a local runtime, which often cannot
/// take `tools`. But `local-openai` also fronts hosted OpenAI-compatible
/// gateways whose models call tools natively and whose servers parse
/// `<tool_call>` text in the model's own template: prompted with the JSON-in-tag
/// protocol there, the gateway reads the JSON as the tool's name. A host that
/// knows its endpoint supports native calls passes `true`; nothing else about
/// the local-runtime profile changes.
#[allow(clippy::too_many_arguments)]
pub fn build_local_runtime_chat_model_with_native_tools(
provider_name: &str,
endpoint: &str,
api_key: &str,
auth_style: AuthStyle,
model: &str,
temperature_unsupported_models: &[String],
temperature_override: Option<f64>,
num_ctx: Option<u32>,
native_tool_calling: bool,
) -> Arc<dyn ChatModel<()>> {
let default_provider_options = num_ctx.map(|value| {
json!({
Expand All @@ -279,7 +314,7 @@ pub fn build_local_runtime_chat_model(
temperature_override,
merge_system_into_user: false,
extra_headers: &[],
native_tool_calling: Some(false),
native_tool_calling: Some(native_tool_calling),
vision: Some(false),
default_provider_options,
responses_api_primary: false,
Expand Down
39 changes: 39 additions & 0 deletions crates/tinyinference-llm/src/providers/openai/config_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,45 @@ fn local_runtime_builder_disables_native_tools_and_vision() {
assert!(profile.hoists_system_messages);
}

#[test]
fn local_runtime_builder_can_opt_into_native_tools() {
// An OpenAI-compatible endpoint behind `local-openai` may serve a model
// with working native tool calls (GLM on a hosted gateway). Prompting it
// with the text protocol instead lets the gateway's own tool parser read
// the JSON-in-tag call as a tool *name*, so the host must be able to
// turn native calls on for it.
let model = build_local_runtime_chat_model_with_native_tools(
"local-openai",
"https://gateway.example/v1",
"secret",
AuthStyle::Bearer,
"glm-5.3-flash",
&[],
None,
None,
true,
);
let profile = model.profile().expect("openai models expose a profile");
assert!(profile.tool_calling);
// Everything else stays the local-runtime profile.
assert!(!profile.modalities.image_in);
assert!(!profile.mid_conversation_system_messages);
assert!(profile.hoists_system_messages);

let off = build_local_runtime_chat_model_with_native_tools(
"local-openai",
"https://gateway.example/v1",
"secret",
AuthStyle::Bearer,
"glm-5.3-flash",
&[],
None,
None,
false,
);
assert!(!off.profile().expect("profile").tool_calling);
}

#[test]
fn openrouter_endpoints_are_recognized() {
assert!(endpoint_is_openrouter("https://openrouter.ai/api/v1"));
Expand Down
5 changes: 3 additions & 2 deletions crates/tinyinference-llm/src/providers/openai/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,9 @@ mod sse;
mod transport;

pub use config::{
OpenAiConfig, build_local_runtime_chat_model, build_openai_chat_model, build_openai_model,
build_openai_model_with_http, endpoint_is_openrouter,
OpenAiConfig, build_local_runtime_chat_model, build_local_runtime_chat_model_with_native_tools,
build_openai_chat_model, build_openai_model, build_openai_model_with_http,
endpoint_is_openrouter,
};
pub use local::{
CONTEXT_OVERFLOW_CODE, LocalProbe, LocalRuntimeKind, is_chat_template_rejection_message,
Expand Down
Loading