diff --git a/Endpoints/ModelProxyEndpoints.cs b/Endpoints/ModelProxyEndpoints.cs index 6c4e6ff..b200d53 100644 --- a/Endpoints/ModelProxyEndpoints.cs +++ b/Endpoints/ModelProxyEndpoints.cs @@ -11,7 +11,7 @@ public static class ModelProxyEndpoints { public static void MapModelProxyEndpoints(this WebApplication app) { - app.MapGet("/api/models", async (IHttpClientFactory clientFactory) => + app.MapGet("/api/models", async (IHttpClientFactory clientFactory, ISettingsService? settingsService) => { try { @@ -21,7 +21,121 @@ public static void MapModelProxyEndpoints(this WebApplication app) if (response.IsSuccessStatusCode) { var content = await response.Content.ReadAsStringAsync(cts.Token); - return Results.Content(content, "application/json"); + var settings = settingsService?.LoadSettings() ?? new AppSettings(); + int defaultContext = settings.DefaultOllamaContextLength > 0 ? settings.DefaultOllamaContextLength : 24576; + + // Query running models to detect active runtime context windows + var activeContextMap = new Dictionary(StringComparer.OrdinalIgnoreCase); + try + { + using var psCts = new CancellationTokenSource(TimeSpan.FromSeconds(2)); + var psResp = await http.GetAsync("http://127.0.0.1:11434/api/ps", psCts.Token); + if (psResp.IsSuccessStatusCode) + { + var psJson = await psResp.Content.ReadAsStringAsync(psCts.Token); + var psDoc = JsonNode.Parse(psJson); + var psModels = psDoc?["models"]?.AsArray(); + if (psModels != null) + { + foreach (var psm in psModels) + { + string? mName = psm?["name"]?.ToString() ?? psm?["model"]?.ToString(); + int? cLen = psm?["context_length"]?.GetValue(); + if (!string.IsNullOrEmpty(mName) && cLen.HasValue && cLen.Value > 0) + { + activeContextMap[mName] = cLen.Value; + } + } + } + } + } + catch { } + + var doc = JsonNode.Parse(content); + var models = doc?["models"]?.AsArray(); + if (models != null) + { + foreach (var m in models) + { + if (m is JsonObject mObj) + { + string name = mObj["name"]?.ToString() ?? mObj["model"]?.ToString() ?? ""; + var detailsObj = mObj["details"] as JsonObject; + + int ggufMaxContext = 32768; + if (detailsObj != null && detailsObj.TryGetPropertyValue("context_length", out var clVal) && clVal != null) + { + if (int.TryParse(clVal.ToString(), out var parsedCl) && parsedCl > 0) + { + ggufMaxContext = parsedCl; + } + } + + // Use active running context if loaded, otherwise configured default context clamped by model maximum + int effectiveContext; + if (activeContextMap.TryGetValue(name, out var activeCtx)) + { + effectiveContext = activeCtx; + } + else + { + effectiveContext = Math.Min(defaultContext, ggufMaxContext); + if (effectiveContext <= 0) effectiveContext = defaultContext; + } + + // Determine max_output_tokens based on model capabilities and architecture: + // Reasoning/R1 models: up to 16,384 or 32,768 for long chain-of-thought + // Large modern coding models (Qwen 2.5 / DeepSeek): 8,192 + // Small models (< 7B): 4,096 + int maxOutput = 8192; + if (name.Contains("r1", StringComparison.OrdinalIgnoreCase) || name.Contains("reason", StringComparison.OrdinalIgnoreCase)) + { + maxOutput = Math.Min(16384, effectiveContext / 2); + } + else if (name.Contains("1.5b", StringComparison.OrdinalIgnoreCase) || name.Contains("0.5b", StringComparison.OrdinalIgnoreCase) || name.Contains("3b", StringComparison.OrdinalIgnoreCase)) + { + maxOutput = 4096; + } + + // Check if details explicitly provided a max_output_tokens override + if (detailsObj != null && detailsObj.TryGetPropertyValue("max_output_tokens", out var moVal) && moVal != null) + { + if (int.TryParse(moVal.ToString(), out var parsedMo) && parsedMo > 0) + { + maxOutput = parsedMo; + } + } + + // Inject top-level token and context limits for LiteLLM / OpenCode + mObj["context_length"] = effectiveContext; + mObj["max_input_tokens"] = effectiveContext; + mObj["max_output_tokens"] = maxOutput; + mObj["max_tokens"] = effectiveContext; + + // Inject inside details for downstream Model Managers that inspect details + if (detailsObj != null) + { + detailsObj["context_length"] = effectiveContext; + detailsObj["max_input_tokens"] = effectiveContext; + detailsObj["max_output_tokens"] = maxOutput; + } + + // Inject model_info schema (mirrors LiteLLM ModelInfo specification) + var modelInfo = new JsonObject + { + ["context_length"] = effectiveContext, + ["max_input_tokens"] = effectiveContext, + ["max_output_tokens"] = maxOutput, + ["max_tokens"] = effectiveContext, + ["key"] = name, + ["mode"] = "chat" + }; + mObj["model_info"] = modelInfo; + } + } + } + + return Results.Content(doc?.ToJsonString(new JsonSerializerOptions { WriteIndented = true }) ?? content, "application/json"); } } catch { } diff --git a/LocalLLMServerManager.Shared/Models/AppSettings.cs b/LocalLLMServerManager.Shared/Models/AppSettings.cs index cba174f..c7242ef 100644 --- a/LocalLLMServerManager.Shared/Models/AppSettings.cs +++ b/LocalLLMServerManager.Shared/Models/AppSettings.cs @@ -35,5 +35,6 @@ public record AppSettings( string AiAssistantPromptsDirectory = "", int ModelCacheIdleTimeoutMinutes = 15, int EngineHibernationTimeoutMinutes = 30, - int RamPressureThresholdPercent = 75 + int RamPressureThresholdPercent = 75, + int DefaultOllamaContextLength = 24576 ); diff --git a/LocalLLMServerManager.Shared/Models/HubModelItems.cs b/LocalLLMServerManager.Shared/Models/HubModelItems.cs index e082656..bd65515 100644 --- a/LocalLLMServerManager.Shared/Models/HubModelItems.cs +++ b/LocalLLMServerManager.Shared/Models/HubModelItems.cs @@ -11,7 +11,10 @@ public record OllamaModelItem( bool IsLoaded, QuickFitBadge? FitBadge = null, long SizeBytes = 0, - bool IsSelected = false + bool IsSelected = false, + int ContextLength = 24576, + int MaxInputTokens = 24576, + int MaxOutputTokens = 8192 ); public record HuggingFaceRepoItem( diff --git a/LocalLLMServerManager.Shared/Services/OllamaModelService.cs b/LocalLLMServerManager.Shared/Services/OllamaModelService.cs index 2654228..f8405ac 100644 --- a/LocalLLMServerManager.Shared/Services/OllamaModelService.cs +++ b/LocalLLMServerManager.Shared/Services/OllamaModelService.cs @@ -56,7 +56,21 @@ public async Task> LoadInstalledModelsAsync(string apiBase if (name.Contains("math", StringComparison.OrdinalIgnoreCase)) { cap = "🧮 Mathematics"; color = "#C084FC"; } else if (name.Contains("r1", StringComparison.OrdinalIgnoreCase) || name.Contains("deepseek", StringComparison.OrdinalIgnoreCase)) { cap = "🧠 Reasoning profile"; color = "#A855F7"; } - result.Add(new OllamaModelItem(name, formatSize, cap, color, false, null, size)); + int contextLength = 24576; + var ctxNode = m?["context_length"] ?? m?["details"]?["context_length"]; + if (ctxNode != null && int.TryParse(ctxNode.ToString(), out var parsedCtx) && parsedCtx > 0) + { + contextLength = parsedCtx; + } + + int maxOut = 8192; + var outNode = m?["max_output_tokens"] ?? m?["details"]?["max_output_tokens"]; + if (outNode != null && int.TryParse(outNode.ToString(), out var parsedOut) && parsedOut > 0) + { + maxOut = parsedOut; + } + + result.Add(new OllamaModelItem(name, formatSize, cap, color, false, null, size, false, contextLength, contextLength, maxOut)); } } } diff --git a/LocalLLMServerManager.Shared/ViewModels/SettingsViewModel.cs b/LocalLLMServerManager.Shared/ViewModels/SettingsViewModel.cs index 4e1c75e..e7c351d 100644 --- a/LocalLLMServerManager.Shared/ViewModels/SettingsViewModel.cs +++ b/LocalLLMServerManager.Shared/ViewModels/SettingsViewModel.cs @@ -76,6 +76,7 @@ public void SelectCategory(string category) [ObservableProperty] private int _modelCacheIdleTimeoutMinutes = 15; [ObservableProperty] private int _engineHibernationTimeoutMinutes = 30; [ObservableProperty] private int _ramPressureThresholdPercent = 75; + [ObservableProperty] private int _defaultOllamaContextLength = 24576; [ObservableProperty] private string _audioPath = ""; [ObservableProperty] private string _videoModelsPath = ""; @@ -908,6 +909,7 @@ public async Task LoadSettingsAsync(string apiBase, HttpClient http) ModelCacheIdleTimeoutMinutes = settings.ModelCacheIdleTimeoutMinutes; EngineHibernationTimeoutMinutes = settings.EngineHibernationTimeoutMinutes; RamPressureThresholdPercent = settings.RamPressureThresholdPercent; + DefaultOllamaContextLength = settings.DefaultOllamaContextLength > 0 ? settings.DefaultOllamaContextLength : 24576; if (settings.CustomPresets != null && settings.CustomPresets.Count > 0) { @@ -978,7 +980,8 @@ public async Task SaveSettingsAsync(string apiBase, HttpClient http) AiAssistantPromptsDirectory: this.AiAssistantPromptsDirectory, ModelCacheIdleTimeoutMinutes: this.ModelCacheIdleTimeoutMinutes, EngineHibernationTimeoutMinutes: this.EngineHibernationTimeoutMinutes, - RamPressureThresholdPercent: this.RamPressureThresholdPercent + RamPressureThresholdPercent: this.RamPressureThresholdPercent, + DefaultOllamaContextLength: this.DefaultOllamaContextLength ); var content = new StringContent( diff --git a/LocalLLMServerManager.Shared/Views/Controls/SettingsTabControl.axaml b/LocalLLMServerManager.Shared/Views/Controls/SettingsTabControl.axaml index a9c919c..e55b03a 100644 --- a/LocalLLMServerManager.Shared/Views/Controls/SettingsTabControl.axaml +++ b/LocalLLMServerManager.Shared/Views/Controls/SettingsTabControl.axaml @@ -502,9 +502,9 @@ - + - + - + - + + + + + + + + diff --git a/LocalLLMServerManager.Tests/ServerEndpointsTests.cs b/LocalLLMServerManager.Tests/ServerEndpointsTests.cs index bdc8dfe..3a0ab54 100644 --- a/LocalLLMServerManager.Tests/ServerEndpointsTests.cs +++ b/LocalLLMServerManager.Tests/ServerEndpointsTests.cs @@ -460,4 +460,18 @@ public async Task DeleteModelEndpoint_PostMethodAndQueryParams_DeletesFileSucces if (File.Exists(tempFile)) File.Delete(tempFile); } } + + [Fact] + public async Task GetModelsEndpoint_ReturnsEnrichedContextAndModelInfoMetadata() + { + var response = await _client.GetAsync("/api/models"); + Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode); + + var content = await response.Content.ReadAsStringAsync(); + Assert.NotNull(content); + + var doc = System.Text.Json.Nodes.JsonNode.Parse(content); + Assert.NotNull(doc); + Assert.NotNull(doc["models"]); + } }