Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,10 @@ print(response.choices[0].message.content)
Supported capabilities depend on the selected model and can include streaming,
tool calls, parallel tool calls, structured outputs and reasoning. The current
capabilities for each model are shown in the
[model catalog](https://nexus.dappnode.com/models). They describe each model
[model catalog](https://nexus.dappnode.com/models). `GET /v1/models` lists
them per model in `capabilities`, using models.dev field names (`modalities`,
`tool_call`, `structured_output`, `reasoning`, `reasoning_options`, …); a
missing field means not stated. They describe each model
and are not checked against chat requests: the selected provider validates chat
options, so a stale catalog flag cannot reject a request the provider would
accept. Endpoint routing, privacy policy and credit checks still apply.
Expand Down
12 changes: 8 additions & 4 deletions apps/gateway/internal/adapters/http/dto/models_response.go
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package dto

import "encoding/json"

// ModelsResponse is the response envelope for GET /v1/models.
type ModelsResponse struct {
Object string `json:"object"`
Expand Down Expand Up @@ -27,8 +29,10 @@ type ModelData struct {
OutputPricePer1MTokensUSD *float64 `json:"output_price_per_1m_tokens_usd,omitempty"`
CacheReadPricePer1MTokensUSD *float64 `json:"cache_read_price_per_1m_tokens_usd,omitempty"`
CacheWritePricePer1MTokensUSD *float64 `json:"cache_write_price_per_1m_tokens_usd,omitempty"`
Features []string `json:"features"`
Endpoints []string `json:"endpoints"`
ProofMode string `json:"proof_mode"`
ProofsEnabled bool `json:"proofs_enabled"`
// Capabilities is what the operator states the model accepts, using
// models.dev field names. A missing field means not stated.
Capabilities json.RawMessage `json:"capabilities,omitempty"`
Endpoints []string `json:"endpoints"`
ProofMode string `json:"proof_mode"`
ProofsEnabled bool `json:"proofs_enabled"`
}
69 changes: 29 additions & 40 deletions apps/gateway/internal/adapters/http/handlers/handlers_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"encoding/json"
"net/http"
"net/http/httptest"
"reflect"
"strings"
"testing"

Expand Down Expand Up @@ -110,20 +111,18 @@ func TestModelsHandler_ListsModels(t *testing.T) {
catalog := &mockModelCatalog{
models: []domain.PublicModel{
{
PublicModelID: "openai/gpt-4.1-mini",
DisplayName: "GPT-4.1 Mini",
Description: &desc,
ProviderConfig: domain.ProviderConfig{ProviderName: "openai"},
Active: true,
SupportsChatCompletions: true,
SupportsChatCompletionsStream: true,
SupportsTools: true,
SupportsStructuredOutput: true,
MaxContextWindow: 128000,
MaxOutputTokens: 16000,
Currency: "EUR",
InputPricePerMillion: decimal.NewFromFloat(0.4),
OutputPricePerMillion: decimal.NewFromFloat(1.6),
PublicModelID: "openai/gpt-4.1-mini",
DisplayName: "GPT-4.1 Mini",
Description: &desc,
ProviderConfig: domain.ProviderConfig{ProviderName: "openai"},
Active: true,
SupportsChatCompletions: true,
Capabilities: json.RawMessage(`{"streaming":true,"tool_call":true,"structured_output":true}`),
MaxContextWindow: 128000,
MaxOutputTokens: 16000,
Currency: "EUR",
InputPricePerMillion: decimal.NewFromFloat(0.4),
OutputPricePerMillion: decimal.NewFromFloat(1.6),
},
{
PublicModelID: "anthropic/claude-sonnet-4",
Expand Down Expand Up @@ -154,20 +153,20 @@ func TestModelsHandler_ListsModels(t *testing.T) {
var body struct {
Object string `json:"object"`
Data []struct {
ID string `json:"id"`
Object string `json:"object"`
OwnedBy string `json:"owned_by"`
DisplayName string `json:"display_name"`
Description *string `json:"description,omitempty"`
ContextSize int `json:"context_size"`
MaxOutputTokens int `json:"max_output_tokens"`
Currency string `json:"currency"`
InputPricePer1MTokensCents int64 `json:"input_price_per_1m_tokens_cents"`
OutputPricePer1MTokensCents int64 `json:"output_price_per_1m_tokens_cents"`
InputPricePer1MTokensUSD float64 `json:"input_price_per_1m_tokens_usd"`
OutputPricePer1MTokensUSD float64 `json:"output_price_per_1m_tokens_usd"`
Features []string `json:"features"`
Endpoints []string `json:"endpoints"`
ID string `json:"id"`
Object string `json:"object"`
OwnedBy string `json:"owned_by"`
DisplayName string `json:"display_name"`
Description *string `json:"description,omitempty"`
ContextSize int `json:"context_size"`
MaxOutputTokens int `json:"max_output_tokens"`
Currency string `json:"currency"`
InputPricePer1MTokensCents int64 `json:"input_price_per_1m_tokens_cents"`
OutputPricePer1MTokensCents int64 `json:"output_price_per_1m_tokens_cents"`
InputPricePer1MTokensUSD float64 `json:"input_price_per_1m_tokens_usd"`
OutputPricePer1MTokensUSD float64 `json:"output_price_per_1m_tokens_usd"`
Capabilities map[string]bool `json:"capabilities"`
Endpoints []string `json:"endpoints"`
} `json:"data"`
}
json.NewDecoder(resp.Body).Decode(&body)
Expand Down Expand Up @@ -212,18 +211,8 @@ func TestModelsHandler_ListsModels(t *testing.T) {
if first.OutputPricePer1MTokensUSD != 1.728 {
t.Errorf("data[0].output_price_usd = %v, want 1.728", first.OutputPricePer1MTokensUSD)
}
wantFeatures := map[string]bool{
"streaming": true,
"function-calling": true,
"structured-outputs": true,
}
if len(first.Features) != len(wantFeatures) {
t.Errorf("data[0].features = %v, want %v", first.Features, wantFeatures)
}
for _, f := range first.Features {
if !wantFeatures[f] {
t.Errorf("unexpected feature %q in data[0].features", f)
}
if want := map[string]bool{"streaming": true, "tool_call": true, "structured_output": true}; !reflect.DeepEqual(first.Capabilities, want) {
t.Errorf("data[0].capabilities = %v, want %v", first.Capabilities, want)
}
if len(first.Endpoints) != 1 || first.Endpoints[0] != "chat/completions" {
t.Errorf("data[0].endpoints = %v, want [chat/completions]", first.Endpoints)
Expand Down
35 changes: 9 additions & 26 deletions apps/gateway/internal/adapters/http/handlers/models_handler.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package handlers

import (
"encoding/json"
"net/http"
"strings"

Expand Down Expand Up @@ -72,7 +73,7 @@ func toPublicModelData(m domain.PublicModel, eurToUSDRate float64) dto.ModelData
OutputPricePer1MTokensUSD: priceToUSDPtr(m.Currency, m.OutputPricePerMillion, eurToUSDRate),
CacheReadPricePer1MTokensUSD: pricePtrToUSDPtr(m.Currency, m.CacheReadPricePerMillion, eurToUSDRate),
CacheWritePricePer1MTokensUSD: pricePtrToUSDPtr(m.Currency, m.CacheWritePricePerMillion, eurToUSDRate),
Features: modelFeatures(m),
Capabilities: publishedCapabilities(m),
Endpoints: modelEndpoints(m),
ProofMode: proofMode,
ProofsEnabled: domain.ProofModeEnabled(proofMode),
Expand All @@ -88,37 +89,19 @@ func toRouterData(r domain.RouterEntry) dto.ModelData {
Created: r.CreatedAt.Unix(),
DisplayName: r.DisplayName,
Description: r.Description,
Features: []string{"routing"},
Endpoints: []string{"chat/completions"},
ProofMode: domain.ProofModeNone,
}
}

func modelFeatures(m domain.PublicModel) []string {
if m.SupportsEndpoint(domain.EndpointSystemOne) {
return []string{"structured-outputs"}
}
features := make([]string, 0, 5)
if m.SupportsChatCompletionsStream {
features = append(features, "streaming")
}
if m.SupportsTools {
features = append(features, "function-calling")
}
if m.SupportsParallelToolCalls {
features = append(features, "parallel-tool-calls")
}
if m.SupportsStructuredOutput {
features = append(features, "structured-outputs")
}
if m.SupportsReasoning {
features = append(features, "reasoning")
}
switch m.EffectiveProofMode() {
case domain.ProofModeTinfoilAttestedTransport:
features = append(features, "tinfoil-attested-transport")
// publishedCapabilities is the operator's record, unchanged, or nothing when
// the operator stated nothing.
func publishedCapabilities(m domain.PublicModel) json.RawMessage {
var object map[string]json.RawMessage
if json.Unmarshal(m.Capabilities, &object) != nil || len(object) == 0 {
return nil
}
return features
return m.Capabilities
}

func modelEndpoints(m domain.PublicModel) []string {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
package handlers

import (
"encoding/json"
"strings"
"testing"

"github.com/dappnode/dappnode-nexus-gateway/pkg/domain"
)

// The operator's record is published unchanged, including fields this Gateway
// does not know yet. There is no separate features list any more.
func TestModelsHandler_PublishesCapabilitiesUnchanged(t *testing.T) {
stored := `{"modalities":{"input":["text","image"],"output":["text"]},"tool_call":true,"reasoning":true,` +
`"reasoning_options":[{"type":"effort","values":["low","high","max"],"default":"high"}],"audio_output":false}`
data := toPublicModelData(domain.PublicModel{PublicModelID: "vision", SupportsChatCompletions: true, Capabilities: json.RawMessage(stored)}, 1)
if string(data.Capabilities) != stored {
t.Fatalf("capabilities = %s", data.Capabilities)
}
if body, _ := json.Marshal(data); strings.Contains(string(body), `"features"`) {
t.Fatalf("published the removed features list: %s", body)
}
empty := toPublicModelData(domain.PublicModel{PublicModelID: "unknown", SupportsChatCompletions: true, Capabilities: json.RawMessage(`{}`)}, 1)
body, _ := json.Marshal(empty)
if strings.Contains(string(body), `"capabilities"`) {
t.Fatalf("published capabilities nobody stated: %s", body)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ func TestStreamingUsageRetention(t *testing.T) {

model := domain.PublicModel{
PublicModelID: "local-repro", ProviderModelID: "local-repro", UpstreamModelName: "local-repro", Active: true,
SupportsChatCompletions: true, SupportsChatCompletionsStream: true, SupportsTools: true,
SupportsChatCompletions: true, Capabilities: json.RawMessage(`{"streaming":true,"tool_call":true}`),
ProviderConfig: domain.ProviderConfig{
ProviderName: "novita", BaseURL: upstream.URL,
APIKeySecretRef: "NEXUS_STREAM_REPRO_PROVIDER_KEY",
Expand Down
55 changes: 34 additions & 21 deletions apps/gateway/internal/adapters/metering/runtime_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -322,29 +322,41 @@ func jsonObject(raw json.RawMessage) bool {
return json.Unmarshal(raw, &object) == nil && object != nil
}

// capabilities is the operator's record from Metering. A Metering that does not
// send one yet still sends the flags it replaced, so they are used instead.
func (m runtimePublicModel) capabilities() json.RawMessage {
if jsonObject(m.Capabilities) {
return m.Capabilities
}
legacy, _ := json.Marshal(map[string]bool{
"streaming": *m.SupportsChatCompletionsStream,
"tool_call": *m.SupportsTools,
"parallel_tool_calls": *m.SupportsParallelToolCalls,
"structured_output": *m.SupportsStructuredOutput,
"reasoning": *m.SupportsReasoning,
})
return legacy
}

func (m runtimePublicModel) toDomain() domain.PublicModel {
model := domain.PublicModel{
ID: m.ID,
PublicModelID: m.PublicModelID,
DisplayName: m.DisplayName,
Description: m.Description,
ProviderModelID: m.ProviderModelID,
UpstreamModelName: m.UpstreamModelName,
ServiceTier: m.ServiceTier,
ProviderConfig: m.ProviderConfig.toDomain(),
SupportsChatCompletions: *m.SupportsChatCompletions,
SupportsChatCompletionsStream: *m.SupportsChatCompletionsStream,
SupportsTools: *m.SupportsTools,
SupportsParallelToolCalls: *m.SupportsParallelToolCalls,
SupportsStructuredOutput: *m.SupportsStructuredOutput,
SupportsReasoning: *m.SupportsReasoning,
ProofMode: m.ProofMode,
MaxContextWindow: m.MaxContextWindow,
MaxOutputTokens: m.MaxOutputTokens,
InputPricePerMillion: priceFromMicrocents(*m.InputPricePer1MTokensMicrocents),
OutputPricePerMillion: priceFromMicrocents(*m.OutputPricePer1MTokensMicrocents),
Currency: m.Currency,
Active: m.Active,
ID: m.ID,
PublicModelID: m.PublicModelID,
DisplayName: m.DisplayName,
Description: m.Description,
ProviderModelID: m.ProviderModelID,
UpstreamModelName: m.UpstreamModelName,
ServiceTier: m.ServiceTier,
ProviderConfig: m.ProviderConfig.toDomain(),
SupportsChatCompletions: *m.SupportsChatCompletions,
Capabilities: m.capabilities(),
ProofMode: m.ProofMode,
MaxContextWindow: m.MaxContextWindow,
MaxOutputTokens: m.MaxOutputTokens,
InputPricePerMillion: priceFromMicrocents(*m.InputPricePer1MTokensMicrocents),
OutputPricePerMillion: priceFromMicrocents(*m.OutputPricePer1MTokensMicrocents),
Currency: m.Currency,
Active: m.Active,
}
if m.Fallback != nil {
model.Fallback = &domain.ProviderTarget{
Expand Down Expand Up @@ -429,6 +441,7 @@ type runtimePublicModel struct {
SupportsParallelToolCalls *bool `json:"supports_parallel_tool_calls"`
SupportsStructuredOutput *bool `json:"supports_structured_output"`
SupportsReasoning *bool `json:"supports_reasoning"`
Capabilities json.RawMessage `json:"capabilities,omitempty"`
ProofMode string `json:"proof_mode"`
MaxContextWindow int `json:"max_context_window"`
MaxOutputTokens int `json:"max_output_tokens"`
Expand Down
21 changes: 21 additions & 0 deletions apps/gateway/internal/adapters/metering/runtime_client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -579,3 +579,24 @@ func stringPointer(value string) *string {
func int64Pointer(value int64) *int64 {
return &value
}

// The operator's record is used as sent; a Metering that does not send one yet
// still sends the flags it replaced.
func TestRuntimePublicModelCapabilities(t *testing.T) {
model := validRuntimePublicModel("openai/gpt-test")
model.Capabilities = json.RawMessage(`{"modalities":{"input":["text","image"]},"tool_call":true}`)
if got := string(model.toDomain().Capabilities); got != `{"modalities":{"input":["text","image"]},"tool_call":true}` {
t.Fatalf("capabilities = %s", got)
}
model.Capabilities = nil
var legacy map[string]bool
if err := json.Unmarshal(model.toDomain().Capabilities, &legacy); err != nil {
t.Fatal(err)
}
want := map[string]bool{"streaming": true, "tool_call": false, "parallel_tool_calls": false, "structured_output": false, "reasoning": false}
for key, value := range want {
if got, ok := legacy[key]; !ok || got != value {
t.Fatalf("capabilities from flags = %v", legacy)
}
}
}
39 changes: 20 additions & 19 deletions apps/gateway/internal/application/services/generate_service_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package services

import (
"context"
"encoding/json"
"io"
"strings"
"testing"
Expand Down Expand Up @@ -309,15 +310,15 @@ func newFallbackGenerateService(meter *stubUsageMeter, primary, fallback *stubPr
APIKey: domain.APIKey{ID: "key1", Active: true},
}},
&stubModelCatalog{model: domain.PublicModel{
PublicModelID: "model",
ProviderModelID: "primary-model",
UpstreamModelName: "primary-upstream",
ProviderConfig: domain.ProviderConfig{ProviderName: "primary"},
Fallback: &domain.ProviderTarget{ProviderModelID: "fallback-model", UpstreamModelName: "fallback-upstream", ServiceTier: &serviceTier, ProviderConfig: domain.ProviderConfig{ProviderName: "fallback"}},
SupportsChatCompletions: true,
SupportsChatCompletionsStream: true,
MaxContextWindow: 1000,
MaxOutputTokens: 100,
PublicModelID: "model",
ProviderModelID: "primary-model",
UpstreamModelName: "primary-upstream",
ProviderConfig: domain.ProviderConfig{ProviderName: "primary"},
Fallback: &domain.ProviderTarget{ProviderModelID: "fallback-model", UpstreamModelName: "fallback-upstream", ServiceTier: &serviceTier, ProviderConfig: domain.ProviderConfig{ProviderName: "fallback"}},
SupportsChatCompletions: true,
Capabilities: json.RawMessage(`{"streaming":true}`),
MaxContextWindow: 1000,
MaxOutputTokens: 100,
}},
nil,
&stubProviderRegistry{providers: map[string]ports.Provider{
Expand Down Expand Up @@ -456,16 +457,16 @@ func TestProxy_RoutesExplicitRouterToSelectedModel(t *testing.T) {
Reason: reason,
}}
selected := domain.PublicModel{
PublicModelID: "minimax/minimax-m2.7",
DisplayName: "MiniMax M2.7",
ProviderModelID: "novita-minimax-m2.7",
ProviderConfig: domain.ProviderConfig{ProviderName: "novita"},
SupportsChatCompletions: true,
SupportsChatCompletionsStream: true,
MaxContextWindow: 1000,
MaxOutputTokens: 100,
InputPricePerMillion: decimal.NewFromFloat(0.03),
OutputPricePerMillion: decimal.NewFromFloat(0.12),
PublicModelID: "minimax/minimax-m2.7",
DisplayName: "MiniMax M2.7",
ProviderModelID: "novita-minimax-m2.7",
ProviderConfig: domain.ProviderConfig{ProviderName: "novita"},
SupportsChatCompletions: true,
Capabilities: json.RawMessage(`{"streaming":true}`),
MaxContextWindow: 1000,
MaxOutputTokens: 100,
InputPricePerMillion: decimal.NewFromFloat(0.03),
OutputPricePerMillion: decimal.NewFromFloat(0.12),
}
svc := NewGenerateService(
&stubAuthService{authCtx: domain.AuthContext{
Expand Down
Loading
Loading