diff --git a/README.md b/README.md index 0f89ba7..74b54d2 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,10 @@ print(response.choices[0].message.content) Supported capabilities depend on the selected model and can include streaming, tool calls, parallel tool calls, structured outputs and reasoning. The current capabilities for each model are shown in the -[model catalog](https://nexus.dappnode.com/models). They describe each model +[model catalog](https://nexus.dappnode.com/models). `GET /v1/models` lists +them per model in `capabilities`, using models.dev field names (`modalities`, +`tool_call`, `structured_output`, `reasoning`, `reasoning_options`, …); a +missing field means not stated. They describe each model and are not checked against chat requests: the selected provider validates chat options, so a stale catalog flag cannot reject a request the provider would accept. Endpoint routing, privacy policy and credit checks still apply. diff --git a/apps/gateway/internal/adapters/http/dto/models_response.go b/apps/gateway/internal/adapters/http/dto/models_response.go index 2251fa2..af2ff09 100644 --- a/apps/gateway/internal/adapters/http/dto/models_response.go +++ b/apps/gateway/internal/adapters/http/dto/models_response.go @@ -1,5 +1,7 @@ package dto +import "encoding/json" + // ModelsResponse is the response envelope for GET /v1/models. type ModelsResponse struct { Object string `json:"object"` @@ -27,8 +29,10 @@ type ModelData struct { OutputPricePer1MTokensUSD *float64 `json:"output_price_per_1m_tokens_usd,omitempty"` CacheReadPricePer1MTokensUSD *float64 `json:"cache_read_price_per_1m_tokens_usd,omitempty"` CacheWritePricePer1MTokensUSD *float64 `json:"cache_write_price_per_1m_tokens_usd,omitempty"` - Features []string `json:"features"` - Endpoints []string `json:"endpoints"` - ProofMode string `json:"proof_mode"` - ProofsEnabled bool `json:"proofs_enabled"` + // Capabilities is what the operator states the model accepts, using + // models.dev field names. A missing field means not stated. + Capabilities json.RawMessage `json:"capabilities,omitempty"` + Endpoints []string `json:"endpoints"` + ProofMode string `json:"proof_mode"` + ProofsEnabled bool `json:"proofs_enabled"` } diff --git a/apps/gateway/internal/adapters/http/handlers/handlers_test.go b/apps/gateway/internal/adapters/http/handlers/handlers_test.go index d1a7650..642877f 100644 --- a/apps/gateway/internal/adapters/http/handlers/handlers_test.go +++ b/apps/gateway/internal/adapters/http/handlers/handlers_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "reflect" "strings" "testing" @@ -110,20 +111,18 @@ func TestModelsHandler_ListsModels(t *testing.T) { catalog := &mockModelCatalog{ models: []domain.PublicModel{ { - PublicModelID: "openai/gpt-4.1-mini", - DisplayName: "GPT-4.1 Mini", - Description: &desc, - ProviderConfig: domain.ProviderConfig{ProviderName: "openai"}, - Active: true, - SupportsChatCompletions: true, - SupportsChatCompletionsStream: true, - SupportsTools: true, - SupportsStructuredOutput: true, - MaxContextWindow: 128000, - MaxOutputTokens: 16000, - Currency: "EUR", - InputPricePerMillion: decimal.NewFromFloat(0.4), - OutputPricePerMillion: decimal.NewFromFloat(1.6), + PublicModelID: "openai/gpt-4.1-mini", + DisplayName: "GPT-4.1 Mini", + Description: &desc, + ProviderConfig: domain.ProviderConfig{ProviderName: "openai"}, + Active: true, + SupportsChatCompletions: true, + Capabilities: json.RawMessage(`{"streaming":true,"tool_call":true,"structured_output":true}`), + MaxContextWindow: 128000, + MaxOutputTokens: 16000, + Currency: "EUR", + InputPricePerMillion: decimal.NewFromFloat(0.4), + OutputPricePerMillion: decimal.NewFromFloat(1.6), }, { PublicModelID: "anthropic/claude-sonnet-4", @@ -154,20 +153,20 @@ func TestModelsHandler_ListsModels(t *testing.T) { var body struct { Object string `json:"object"` Data []struct { - ID string `json:"id"` - Object string `json:"object"` - OwnedBy string `json:"owned_by"` - DisplayName string `json:"display_name"` - Description *string `json:"description,omitempty"` - ContextSize int `json:"context_size"` - MaxOutputTokens int `json:"max_output_tokens"` - Currency string `json:"currency"` - InputPricePer1MTokensCents int64 `json:"input_price_per_1m_tokens_cents"` - OutputPricePer1MTokensCents int64 `json:"output_price_per_1m_tokens_cents"` - InputPricePer1MTokensUSD float64 `json:"input_price_per_1m_tokens_usd"` - OutputPricePer1MTokensUSD float64 `json:"output_price_per_1m_tokens_usd"` - Features []string `json:"features"` - Endpoints []string `json:"endpoints"` + ID string `json:"id"` + Object string `json:"object"` + OwnedBy string `json:"owned_by"` + DisplayName string `json:"display_name"` + Description *string `json:"description,omitempty"` + ContextSize int `json:"context_size"` + MaxOutputTokens int `json:"max_output_tokens"` + Currency string `json:"currency"` + InputPricePer1MTokensCents int64 `json:"input_price_per_1m_tokens_cents"` + OutputPricePer1MTokensCents int64 `json:"output_price_per_1m_tokens_cents"` + InputPricePer1MTokensUSD float64 `json:"input_price_per_1m_tokens_usd"` + OutputPricePer1MTokensUSD float64 `json:"output_price_per_1m_tokens_usd"` + Capabilities map[string]bool `json:"capabilities"` + Endpoints []string `json:"endpoints"` } `json:"data"` } json.NewDecoder(resp.Body).Decode(&body) @@ -212,18 +211,8 @@ func TestModelsHandler_ListsModels(t *testing.T) { if first.OutputPricePer1MTokensUSD != 1.728 { t.Errorf("data[0].output_price_usd = %v, want 1.728", first.OutputPricePer1MTokensUSD) } - wantFeatures := map[string]bool{ - "streaming": true, - "function-calling": true, - "structured-outputs": true, - } - if len(first.Features) != len(wantFeatures) { - t.Errorf("data[0].features = %v, want %v", first.Features, wantFeatures) - } - for _, f := range first.Features { - if !wantFeatures[f] { - t.Errorf("unexpected feature %q in data[0].features", f) - } + if want := map[string]bool{"streaming": true, "tool_call": true, "structured_output": true}; !reflect.DeepEqual(first.Capabilities, want) { + t.Errorf("data[0].capabilities = %v, want %v", first.Capabilities, want) } if len(first.Endpoints) != 1 || first.Endpoints[0] != "chat/completions" { t.Errorf("data[0].endpoints = %v, want [chat/completions]", first.Endpoints) diff --git a/apps/gateway/internal/adapters/http/handlers/models_handler.go b/apps/gateway/internal/adapters/http/handlers/models_handler.go index f3d44aa..5b3a533 100644 --- a/apps/gateway/internal/adapters/http/handlers/models_handler.go +++ b/apps/gateway/internal/adapters/http/handlers/models_handler.go @@ -1,6 +1,7 @@ package handlers import ( + "encoding/json" "net/http" "strings" @@ -72,7 +73,7 @@ func toPublicModelData(m domain.PublicModel, eurToUSDRate float64) dto.ModelData OutputPricePer1MTokensUSD: priceToUSDPtr(m.Currency, m.OutputPricePerMillion, eurToUSDRate), CacheReadPricePer1MTokensUSD: pricePtrToUSDPtr(m.Currency, m.CacheReadPricePerMillion, eurToUSDRate), CacheWritePricePer1MTokensUSD: pricePtrToUSDPtr(m.Currency, m.CacheWritePricePerMillion, eurToUSDRate), - Features: modelFeatures(m), + Capabilities: publishedCapabilities(m), Endpoints: modelEndpoints(m), ProofMode: proofMode, ProofsEnabled: domain.ProofModeEnabled(proofMode), @@ -88,37 +89,19 @@ func toRouterData(r domain.RouterEntry) dto.ModelData { Created: r.CreatedAt.Unix(), DisplayName: r.DisplayName, Description: r.Description, - Features: []string{"routing"}, Endpoints: []string{"chat/completions"}, ProofMode: domain.ProofModeNone, } } -func modelFeatures(m domain.PublicModel) []string { - if m.SupportsEndpoint(domain.EndpointSystemOne) { - return []string{"structured-outputs"} - } - features := make([]string, 0, 5) - if m.SupportsChatCompletionsStream { - features = append(features, "streaming") - } - if m.SupportsTools { - features = append(features, "function-calling") - } - if m.SupportsParallelToolCalls { - features = append(features, "parallel-tool-calls") - } - if m.SupportsStructuredOutput { - features = append(features, "structured-outputs") - } - if m.SupportsReasoning { - features = append(features, "reasoning") - } - switch m.EffectiveProofMode() { - case domain.ProofModeTinfoilAttestedTransport: - features = append(features, "tinfoil-attested-transport") +// publishedCapabilities is the operator's record, unchanged, or nothing when +// the operator stated nothing. +func publishedCapabilities(m domain.PublicModel) json.RawMessage { + var object map[string]json.RawMessage + if json.Unmarshal(m.Capabilities, &object) != nil || len(object) == 0 { + return nil } - return features + return m.Capabilities } func modelEndpoints(m domain.PublicModel) []string { diff --git a/apps/gateway/internal/adapters/http/handlers/models_handler_test.go b/apps/gateway/internal/adapters/http/handlers/models_handler_test.go new file mode 100644 index 0000000..b0ceae7 --- /dev/null +++ b/apps/gateway/internal/adapters/http/handlers/models_handler_test.go @@ -0,0 +1,28 @@ +package handlers + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/dappnode/dappnode-nexus-gateway/pkg/domain" +) + +// The operator's record is published unchanged, including fields this Gateway +// does not know yet. There is no separate features list any more. +func TestModelsHandler_PublishesCapabilitiesUnchanged(t *testing.T) { + stored := `{"modalities":{"input":["text","image"],"output":["text"]},"tool_call":true,"reasoning":true,` + + `"reasoning_options":[{"type":"effort","values":["low","high","max"],"default":"high"}],"audio_output":false}` + data := toPublicModelData(domain.PublicModel{PublicModelID: "vision", SupportsChatCompletions: true, Capabilities: json.RawMessage(stored)}, 1) + if string(data.Capabilities) != stored { + t.Fatalf("capabilities = %s", data.Capabilities) + } + if body, _ := json.Marshal(data); strings.Contains(string(body), `"features"`) { + t.Fatalf("published the removed features list: %s", body) + } + empty := toPublicModelData(domain.PublicModel{PublicModelID: "unknown", SupportsChatCompletions: true, Capabilities: json.RawMessage(`{}`)}, 1) + body, _ := json.Marshal(empty) + if strings.Contains(string(body), `"capabilities"`) { + t.Fatalf("published capabilities nobody stated: %s", body) + } +} diff --git a/apps/gateway/internal/adapters/http/handlers/stream_usage_repro_test.go b/apps/gateway/internal/adapters/http/handlers/stream_usage_repro_test.go index 3ce5df0..a0cfb02 100644 --- a/apps/gateway/internal/adapters/http/handlers/stream_usage_repro_test.go +++ b/apps/gateway/internal/adapters/http/handlers/stream_usage_repro_test.go @@ -107,7 +107,7 @@ func TestStreamingUsageRetention(t *testing.T) { model := domain.PublicModel{ PublicModelID: "local-repro", ProviderModelID: "local-repro", UpstreamModelName: "local-repro", Active: true, - SupportsChatCompletions: true, SupportsChatCompletionsStream: true, SupportsTools: true, + SupportsChatCompletions: true, Capabilities: json.RawMessage(`{"streaming":true,"tool_call":true}`), ProviderConfig: domain.ProviderConfig{ ProviderName: "novita", BaseURL: upstream.URL, APIKeySecretRef: "NEXUS_STREAM_REPRO_PROVIDER_KEY", diff --git a/apps/gateway/internal/adapters/metering/runtime_client.go b/apps/gateway/internal/adapters/metering/runtime_client.go index 9a3124f..3fd359b 100644 --- a/apps/gateway/internal/adapters/metering/runtime_client.go +++ b/apps/gateway/internal/adapters/metering/runtime_client.go @@ -322,29 +322,41 @@ func jsonObject(raw json.RawMessage) bool { return json.Unmarshal(raw, &object) == nil && object != nil } +// capabilities is the operator's record from Metering. A Metering that does not +// send one yet still sends the flags it replaced, so they are used instead. +func (m runtimePublicModel) capabilities() json.RawMessage { + if jsonObject(m.Capabilities) { + return m.Capabilities + } + legacy, _ := json.Marshal(map[string]bool{ + "streaming": *m.SupportsChatCompletionsStream, + "tool_call": *m.SupportsTools, + "parallel_tool_calls": *m.SupportsParallelToolCalls, + "structured_output": *m.SupportsStructuredOutput, + "reasoning": *m.SupportsReasoning, + }) + return legacy +} + func (m runtimePublicModel) toDomain() domain.PublicModel { model := domain.PublicModel{ - ID: m.ID, - PublicModelID: m.PublicModelID, - DisplayName: m.DisplayName, - Description: m.Description, - ProviderModelID: m.ProviderModelID, - UpstreamModelName: m.UpstreamModelName, - ServiceTier: m.ServiceTier, - ProviderConfig: m.ProviderConfig.toDomain(), - SupportsChatCompletions: *m.SupportsChatCompletions, - SupportsChatCompletionsStream: *m.SupportsChatCompletionsStream, - SupportsTools: *m.SupportsTools, - SupportsParallelToolCalls: *m.SupportsParallelToolCalls, - SupportsStructuredOutput: *m.SupportsStructuredOutput, - SupportsReasoning: *m.SupportsReasoning, - ProofMode: m.ProofMode, - MaxContextWindow: m.MaxContextWindow, - MaxOutputTokens: m.MaxOutputTokens, - InputPricePerMillion: priceFromMicrocents(*m.InputPricePer1MTokensMicrocents), - OutputPricePerMillion: priceFromMicrocents(*m.OutputPricePer1MTokensMicrocents), - Currency: m.Currency, - Active: m.Active, + ID: m.ID, + PublicModelID: m.PublicModelID, + DisplayName: m.DisplayName, + Description: m.Description, + ProviderModelID: m.ProviderModelID, + UpstreamModelName: m.UpstreamModelName, + ServiceTier: m.ServiceTier, + ProviderConfig: m.ProviderConfig.toDomain(), + SupportsChatCompletions: *m.SupportsChatCompletions, + Capabilities: m.capabilities(), + ProofMode: m.ProofMode, + MaxContextWindow: m.MaxContextWindow, + MaxOutputTokens: m.MaxOutputTokens, + InputPricePerMillion: priceFromMicrocents(*m.InputPricePer1MTokensMicrocents), + OutputPricePerMillion: priceFromMicrocents(*m.OutputPricePer1MTokensMicrocents), + Currency: m.Currency, + Active: m.Active, } if m.Fallback != nil { model.Fallback = &domain.ProviderTarget{ @@ -429,6 +441,7 @@ type runtimePublicModel struct { SupportsParallelToolCalls *bool `json:"supports_parallel_tool_calls"` SupportsStructuredOutput *bool `json:"supports_structured_output"` SupportsReasoning *bool `json:"supports_reasoning"` + Capabilities json.RawMessage `json:"capabilities,omitempty"` ProofMode string `json:"proof_mode"` MaxContextWindow int `json:"max_context_window"` MaxOutputTokens int `json:"max_output_tokens"` diff --git a/apps/gateway/internal/adapters/metering/runtime_client_test.go b/apps/gateway/internal/adapters/metering/runtime_client_test.go index fe01d0b..2ad1ca5 100644 --- a/apps/gateway/internal/adapters/metering/runtime_client_test.go +++ b/apps/gateway/internal/adapters/metering/runtime_client_test.go @@ -579,3 +579,24 @@ func stringPointer(value string) *string { func int64Pointer(value int64) *int64 { return &value } + +// The operator's record is used as sent; a Metering that does not send one yet +// still sends the flags it replaced. +func TestRuntimePublicModelCapabilities(t *testing.T) { + model := validRuntimePublicModel("openai/gpt-test") + model.Capabilities = json.RawMessage(`{"modalities":{"input":["text","image"]},"tool_call":true}`) + if got := string(model.toDomain().Capabilities); got != `{"modalities":{"input":["text","image"]},"tool_call":true}` { + t.Fatalf("capabilities = %s", got) + } + model.Capabilities = nil + var legacy map[string]bool + if err := json.Unmarshal(model.toDomain().Capabilities, &legacy); err != nil { + t.Fatal(err) + } + want := map[string]bool{"streaming": true, "tool_call": false, "parallel_tool_calls": false, "structured_output": false, "reasoning": false} + for key, value := range want { + if got, ok := legacy[key]; !ok || got != value { + t.Fatalf("capabilities from flags = %v", legacy) + } + } +} diff --git a/apps/gateway/internal/application/services/generate_service_test.go b/apps/gateway/internal/application/services/generate_service_test.go index f98f360..35de009 100644 --- a/apps/gateway/internal/application/services/generate_service_test.go +++ b/apps/gateway/internal/application/services/generate_service_test.go @@ -2,6 +2,7 @@ package services import ( "context" + "encoding/json" "io" "strings" "testing" @@ -309,15 +310,15 @@ func newFallbackGenerateService(meter *stubUsageMeter, primary, fallback *stubPr APIKey: domain.APIKey{ID: "key1", Active: true}, }}, &stubModelCatalog{model: domain.PublicModel{ - PublicModelID: "model", - ProviderModelID: "primary-model", - UpstreamModelName: "primary-upstream", - ProviderConfig: domain.ProviderConfig{ProviderName: "primary"}, - Fallback: &domain.ProviderTarget{ProviderModelID: "fallback-model", UpstreamModelName: "fallback-upstream", ServiceTier: &serviceTier, ProviderConfig: domain.ProviderConfig{ProviderName: "fallback"}}, - SupportsChatCompletions: true, - SupportsChatCompletionsStream: true, - MaxContextWindow: 1000, - MaxOutputTokens: 100, + PublicModelID: "model", + ProviderModelID: "primary-model", + UpstreamModelName: "primary-upstream", + ProviderConfig: domain.ProviderConfig{ProviderName: "primary"}, + Fallback: &domain.ProviderTarget{ProviderModelID: "fallback-model", UpstreamModelName: "fallback-upstream", ServiceTier: &serviceTier, ProviderConfig: domain.ProviderConfig{ProviderName: "fallback"}}, + SupportsChatCompletions: true, + Capabilities: json.RawMessage(`{"streaming":true}`), + MaxContextWindow: 1000, + MaxOutputTokens: 100, }}, nil, &stubProviderRegistry{providers: map[string]ports.Provider{ @@ -456,16 +457,16 @@ func TestProxy_RoutesExplicitRouterToSelectedModel(t *testing.T) { Reason: reason, }} selected := domain.PublicModel{ - PublicModelID: "minimax/minimax-m2.7", - DisplayName: "MiniMax M2.7", - ProviderModelID: "novita-minimax-m2.7", - ProviderConfig: domain.ProviderConfig{ProviderName: "novita"}, - SupportsChatCompletions: true, - SupportsChatCompletionsStream: true, - MaxContextWindow: 1000, - MaxOutputTokens: 100, - InputPricePerMillion: decimal.NewFromFloat(0.03), - OutputPricePerMillion: decimal.NewFromFloat(0.12), + PublicModelID: "minimax/minimax-m2.7", + DisplayName: "MiniMax M2.7", + ProviderModelID: "novita-minimax-m2.7", + ProviderConfig: domain.ProviderConfig{ProviderName: "novita"}, + SupportsChatCompletions: true, + Capabilities: json.RawMessage(`{"streaming":true}`), + MaxContextWindow: 1000, + MaxOutputTokens: 100, + InputPricePerMillion: decimal.NewFromFloat(0.03), + OutputPricePerMillion: decimal.NewFromFloat(0.12), } svc := NewGenerateService( &stubAuthService{authCtx: domain.AuthContext{ diff --git a/apps/gateway/internal/application/services/proxy_test.go b/apps/gateway/internal/application/services/proxy_test.go index 44853a0..3793732 100644 --- a/apps/gateway/internal/application/services/proxy_test.go +++ b/apps/gateway/internal/application/services/proxy_test.go @@ -2,6 +2,7 @@ package services import ( "context" + "encoding/json" "io" "strings" "testing" @@ -33,8 +34,9 @@ func proxyService(meter *resultMeter, pii string, providers map[string]ports.Pro model := domain.PublicModel{ PublicModelID: "zai-org/glm-5.3-flash", ProviderModelID: "pm", UpstreamModelName: "glm-5-3-flash", ProviderConfig: domain.ProviderConfig{ProviderName: "primary"}, - SupportsChatCompletions: true, SupportsChatCompletionsStream: true, SupportsTools: true, SupportsParallelToolCalls: true, - MaxContextWindow: 1000, MaxOutputTokens: 100, + SupportsChatCompletions: true, + Capabilities: json.RawMessage(`{"streaming":true,"tool_call":true,"parallel_tool_calls":true}`), + MaxContextWindow: 1000, MaxOutputTokens: 100, } if fallback { model.Fallback = &domain.ProviderTarget{ProviderModelID: "fm", UpstreamModelName: "glm-fallback", ProviderConfig: domain.ProviderConfig{ProviderName: "fallback"}} diff --git a/pkg/domain/model.go b/pkg/domain/model.go index 6600a77..483e1bf 100644 --- a/pkg/domain/model.go +++ b/pkg/domain/model.go @@ -1,6 +1,10 @@ package domain -import "github.com/shopspring/decimal" +import ( + "encoding/json" + + "github.com/shopspring/decimal" +) const ( ProofModeNone = "none" @@ -27,30 +31,28 @@ type ProviderTarget struct { // PublicModel is the fully-resolved model exposed by the gateway. type PublicModel struct { - ID string - PublicModelID string - DisplayName string - Description *string - ProviderModelID string - UpstreamModelName string - ServiceTier *string - ProviderConfig ProviderConfig - Fallback *ProviderTarget - SupportsChatCompletions bool - SupportsChatCompletionsStream bool - SupportsTools bool - SupportsParallelToolCalls bool - SupportsStructuredOutput bool - SupportsReasoning bool - ProofMode string - MaxContextWindow int - MaxOutputTokens int - InputPricePerMillion decimal.Decimal - OutputPricePerMillion decimal.Decimal - CacheReadPricePerMillion *decimal.Decimal - CacheWritePricePerMillion *decimal.Decimal - Currency string - Active bool + ID string + PublicModelID string + DisplayName string + Description *string + ProviderModelID string + UpstreamModelName string + ServiceTier *string + ProviderConfig ProviderConfig + Fallback *ProviderTarget + SupportsChatCompletions bool // Routing: whether chat is served. + // Capabilities is the operator's description of the model, a JSON object + // published unchanged in /v1/models. Gateway never rejects requests by it. + Capabilities json.RawMessage + ProofMode string + MaxContextWindow int + MaxOutputTokens int + InputPricePerMillion decimal.Decimal + OutputPricePerMillion decimal.Decimal + CacheReadPricePerMillion *decimal.Decimal + CacheWritePricePerMillion *decimal.Decimal + Currency string + Active bool } func (m PublicModel) EffectiveProofMode() string {