Skip to content

feat(models): enrich /api/models with runtime context length and LiteLLM model_info metadata #107

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
spelech merged 2 commits into from
Nov 3, 2026
Merged

feat(models): enrich /api/models with runtime context length and LiteLLM model_info metadata #107

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a7ba493
feat(models): enrich /api/models with runtime context length and Lite…
spelech Nov 3, 2026
2a9f84a
feat(models): make max_output_tokens dynamic based on model reasoning…
spelech Nov 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 116 additions & 2 deletions Endpoints/ModelProxyEndpoints.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ public static class ModelProxyEndpoints
{
public static void MapModelProxyEndpoints(this WebApplication app)
{
app.MapGet("/api/models", async (IHttpClientFactory clientFactory) =>
app.MapGet("/api/models", async (IHttpClientFactory clientFactory, ISettingsService? settingsService) =>
{
try
{
Expand All @@ -21,7 +21,121 @@ public static void MapModelProxyEndpoints(this WebApplication app)
if (response.IsSuccessStatusCode)
{
var content = await response.Content.ReadAsStringAsync(cts.Token);
return Results.Content(content, "application/json");
var settings = settingsService?.LoadSettings() ?? new AppSettings();
int defaultContext = settings.DefaultOllamaContextLength > 0 ? settings.DefaultOllamaContextLength : 24576;

// Query running models to detect active runtime context windows
var activeContextMap = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
try
{
using var psCts = new CancellationTokenSource(TimeSpan.FromSeconds(2));
var psResp = await http.GetAsync("http://127.0.0.1:11434/api/ps", psCts.Token);
if (psResp.IsSuccessStatusCode)
{
var psJson = await psResp.Content.ReadAsStringAsync(psCts.Token);
var psDoc = JsonNode.Parse(psJson);
var psModels = psDoc?["models"]?.AsArray();
if (psModels != null)
{
foreach (var psm in psModels)
{
string? mName = psm?["name"]?.ToString() ?? psm?["model"]?.ToString();
int? cLen = psm?["context_length"]?.GetValue<int>();
if (!string.IsNullOrEmpty(mName) && cLen.HasValue && cLen.Value > 0)
{
activeContextMap[mName] = cLen.Value;
}
}
}
}
}
catch { }

var doc = JsonNode.Parse(content);
var models = doc?["models"]?.AsArray();
if (models != null)
{
foreach (var m in models)
{
if (m is JsonObject mObj)
{
string name = mObj["name"]?.ToString() ?? mObj["model"]?.ToString() ?? "";
var detailsObj = mObj["details"] as JsonObject;

int ggufMaxContext = 32768;
if (detailsObj != null && detailsObj.TryGetPropertyValue("context_length", out var clVal) && clVal != null)
{
if (int.TryParse(clVal.ToString(), out var parsedCl) && parsedCl > 0)
{
ggufMaxContext = parsedCl;
}
}

// Use active running context if loaded, otherwise configured default context clamped by model maximum
int effectiveContext;
if (activeContextMap.TryGetValue(name, out var activeCtx))
{
effectiveContext = activeCtx;
}
else
{
effectiveContext = Math.Min(defaultContext, ggufMaxContext);
if (effectiveContext <= 0) effectiveContext = defaultContext;
}

// Determine max_output_tokens based on model capabilities and architecture:
// Reasoning/R1 models: up to 16,384 or 32,768 for long chain-of-thought
// Large modern coding models (Qwen 2.5 / DeepSeek): 8,192
// Small models (< 7B): 4,096
int maxOutput = 8192;
if (name.Contains("r1", StringComparison.OrdinalIgnoreCase) || name.Contains("reason", StringComparison.OrdinalIgnoreCase))
{
maxOutput = Math.Min(16384, effectiveContext / 2);
}
else if (name.Contains("1.5b", StringComparison.OrdinalIgnoreCase) || name.Contains("0.5b", StringComparison.OrdinalIgnoreCase) || name.Contains("3b", StringComparison.OrdinalIgnoreCase))
{
maxOutput = 4096;
}

// Check if details explicitly provided a max_output_tokens override
if (detailsObj != null && detailsObj.TryGetPropertyValue("max_output_tokens", out var moVal) && moVal != null)
{
if (int.TryParse(moVal.ToString(), out var parsedMo) && parsedMo > 0)
{
maxOutput = parsedMo;
}
}

// Inject top-level token and context limits for LiteLLM / OpenCode
mObj["context_length"] = effectiveContext;
mObj["max_input_tokens"] = effectiveContext;
mObj["max_output_tokens"] = maxOutput;
mObj["max_tokens"] = effectiveContext;

// Inject inside details for downstream Model Managers that inspect details
if (detailsObj != null)
{
detailsObj["context_length"] = effectiveContext;
detailsObj["max_input_tokens"] = effectiveContext;
detailsObj["max_output_tokens"] = maxOutput;
}

// Inject model_info schema (mirrors LiteLLM ModelInfo specification)
var modelInfo = new JsonObject
{
["context_length"] = effectiveContext,
["max_input_tokens"] = effectiveContext,
["max_output_tokens"] = maxOutput,
["max_tokens"] = effectiveContext,
["key"] = name,
["mode"] = "chat"
};
mObj["model_info"] = modelInfo;
}
}
}

return Results.Content(doc?.ToJsonString(new JsonSerializerOptions { WriteIndented = true }) ?? content, "application/json");
}
}
catch { }
Expand Down
3 changes: 2 additions & 1 deletion LocalLLMServerManager.Shared/Models/AppSettings.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,5 +35,6 @@ public record AppSettings(
string AiAssistantPromptsDirectory = "",
int ModelCacheIdleTimeoutMinutes = 15,
int EngineHibernationTimeoutMinutes = 30,
int RamPressureThresholdPercent = 75
int RamPressureThresholdPercent = 75,
int DefaultOllamaContextLength = 24576
);
5 changes: 4 additions & 1 deletion LocalLLMServerManager.Shared/Models/HubModelItems.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,10 @@ public record OllamaModelItem(
bool IsLoaded,
QuickFitBadge? FitBadge = null,
long SizeBytes = 0,
bool IsSelected = false
bool IsSelected = false,
int ContextLength = 24576,
int MaxInputTokens = 24576,
int MaxOutputTokens = 8192
);

public record HuggingFaceRepoItem(
Expand Down
16 changes: 15 additions & 1 deletion LocalLLMServerManager.Shared/Services/OllamaModelService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,21 @@ public async Task<List<OllamaModelItem>> LoadInstalledModelsAsync(string apiBase
if (name.Contains("math", StringComparison.OrdinalIgnoreCase)) { cap = "🧮 Mathematics"; color = "#C084FC"; }
else if (name.Contains("r1", StringComparison.OrdinalIgnoreCase) || name.Contains("deepseek", StringComparison.OrdinalIgnoreCase)) { cap = "🧠 Reasoning profile"; color = "#A855F7"; }

result.Add(new OllamaModelItem(name, formatSize, cap, color, false, null, size));
int contextLength = 24576;
var ctxNode = m?["context_length"] ?? m?["details"]?["context_length"];
if (ctxNode != null && int.TryParse(ctxNode.ToString(), out var parsedCtx) && parsedCtx > 0)
{
contextLength = parsedCtx;
}

int maxOut = 8192;
var outNode = m?["max_output_tokens"] ?? m?["details"]?["max_output_tokens"];
if (outNode != null && int.TryParse(outNode.ToString(), out var parsedOut) && parsedOut > 0)
{
maxOut = parsedOut;
}

result.Add(new OllamaModelItem(name, formatSize, cap, color, false, null, size, false, contextLength, contextLength, maxOut));
}
}
}
Expand Down
5 changes: 4 additions & 1 deletion LocalLLMServerManager.Shared/ViewModels/SettingsViewModel.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ public void SelectCategory(string category)
[ObservableProperty] private int _modelCacheIdleTimeoutMinutes = 15;
[ObservableProperty] private int _engineHibernationTimeoutMinutes = 30;
[ObservableProperty] private int _ramPressureThresholdPercent = 75;
[ObservableProperty] private int _defaultOllamaContextLength = 24576;

[ObservableProperty] private string _audioPath = "";
[ObservableProperty] private string _videoModelsPath = "";
Expand Down Expand Up @@ -908,6 +909,7 @@ public async Task LoadSettingsAsync(string apiBase, HttpClient http)
ModelCacheIdleTimeoutMinutes = settings.ModelCacheIdleTimeoutMinutes;
EngineHibernationTimeoutMinutes = settings.EngineHibernationTimeoutMinutes;
RamPressureThresholdPercent = settings.RamPressureThresholdPercent;
DefaultOllamaContextLength = settings.DefaultOllamaContextLength > 0 ? settings.DefaultOllamaContextLength : 24576;

if (settings.CustomPresets != null && settings.CustomPresets.Count > 0)
{
Expand Down Expand Up @@ -978,7 +980,8 @@ public async Task SaveSettingsAsync(string apiBase, HttpClient http)
AiAssistantPromptsDirectory: this.AiAssistantPromptsDirectory,
ModelCacheIdleTimeoutMinutes: this.ModelCacheIdleTimeoutMinutes,
EngineHibernationTimeoutMinutes: this.EngineHibernationTimeoutMinutes,
RamPressureThresholdPercent: this.RamPressureThresholdPercent
RamPressureThresholdPercent: this.RamPressureThresholdPercent,
DefaultOllamaContextLength: this.DefaultOllamaContextLength
);

var content = new StringContent(
Expand Down
17 changes: 13 additions & 4 deletions LocalLLMServerManager.Shared/Views/Controls/SettingsTabControl.axaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -502,9 +502,9 @@
</StackPanel>
<TextBlock Text="Configure two-tier memory eviction and host RAM pressure management for idle models and engines:" FontSize="12" Foreground="{StaticResource TextMutedBrush}"/>

<Grid ColumnDefinitions="*,*,*">
<Grid ColumnDefinitions="*,*,*,*">
<!-- Tier 1: Model Cache Idle Timeout -->
<StackPanel Grid.Column="0" Spacing="4" Margin="0,0,8,0">
<StackPanel Grid.Column="0" Spacing="4" Margin="0,0,6,0">
<TextBlock Text="Model Idle Timeout (min)" FontSize="12" Foreground="{StaticResource TextMutedBrush}" FontWeight="Medium"/>
<NumericUpDown Value="{Binding ModelCacheIdleTimeoutMinutes, Mode=TwoWay}"
Minimum="0" Maximum="1440" Increment="1"
Expand All @@ -513,7 +513,7 @@
</StackPanel>

<!-- Tier 2: Engine Hibernation Timeout -->
<StackPanel Grid.Column="1" Spacing="4" Margin="4,0,4,0">
<StackPanel Grid.Column="1" Spacing="4" Margin="3,0,3,0">
<TextBlock Text="Engine Hibernation (min)" FontSize="12" Foreground="{StaticResource TextMutedBrush}" FontWeight="Medium"/>
<NumericUpDown Value="{Binding EngineHibernationTimeoutMinutes, Mode=TwoWay}"
Minimum="0" Maximum="1440" Increment="1"
Expand All @@ -522,13 +522,22 @@
</StackPanel>

<!-- RAM Pressure Threshold -->
<StackPanel Grid.Column="2" Spacing="4" Margin="8,0,0,0">
<StackPanel Grid.Column="2" Spacing="4" Margin="3,0,3,0">
<TextBlock Text="RAM Pressure Threshold (%)" FontSize="12" Foreground="{StaticResource TextMutedBrush}" FontWeight="Medium"/>
<NumericUpDown Value="{Binding RamPressureThresholdPercent, Mode=TwoWay}"
Minimum="10" Maximum="99" Increment="5"
FormatString="0"/>
<TextBlock Text="Emergency eviction trigger" FontSize="10" Foreground="{StaticResource TextMutedBrush}"/>
</StackPanel>

<!-- Default Ollama Context Window -->
<StackPanel Grid.Column="3" Spacing="4" Margin="6,0,0,0">
<TextBlock Text="Default Context (tokens)" FontSize="12" Foreground="{StaticResource TextMutedBrush}" FontWeight="Medium"/>
<NumericUpDown Value="{Binding DefaultOllamaContextLength, Mode=TwoWay}"
Minimum="2048" Maximum="131072" Increment="2048"
FormatString="0"/>
<TextBlock Text="Active context advertised to OpenCode" FontSize="10" Foreground="{StaticResource TextMutedBrush}"/>
</StackPanel>
</Grid>
</StackPanel>
</Border>
Expand Down
14 changes: 14 additions & 0 deletions LocalLLMServerManager.Tests/ServerEndpointsTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -460,4 +460,18 @@ public async Task DeleteModelEndpoint_PostMethodAndQueryParams_DeletesFileSucces
if (File.Exists(tempFile)) File.Delete(tempFile);
}
}

[Fact]
public async Task GetModelsEndpoint_ReturnsEnrichedContextAndModelInfoMetadata()
{
var response = await _client.GetAsync("/api/models");
Assert.Equal(System.Net.HttpStatusCode.OK, response.StatusCode);

var content = await response.Content.ReadAsStringAsync();
Assert.NotNull(content);

var doc = System.Text.Json.Nodes.JsonNode.Parse(content);
Assert.NotNull(doc);
Assert.NotNull(doc["models"]);
}
}
Loading