From e4ba4944a8dc9643aa12192f8fef5049ab5638e1 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sat, 12 Sep 2026 12:24:26 +0700 Subject: [PATCH 1/8] port: sync ClawRouter v0.12.278 core and harden Go validation --- .github/dependabot.yml | 14 +- .github/workflows/ci.yml | 40 +- .github/workflows/integration.yml | 29 +- README.md | 4 +- UPSTREAM_SYNC.md | 59 ++- cache/cache.go | 29 +- cache/requestkey_test.go | 78 +++ dedup/dedup.go | 31 +- dedup/requestkey_test.go | 85 ++++ docs/configuration.md | 23 + go.mod | 2 + internal/requestkey/normalize.go | 76 +++ internal/requestkey/normalize_test.go | 112 +++++ logger/logger.go | 2 + models/catalog_test.go | 149 ++++++ models/models.go | 670 ++++++++++++++++---------- proxy/prose.go | 65 +++ proxy/proxy.go | 306 ++++++++---- proxy/request.go | 116 +++++ proxy/spending.go | 177 +++++++ proxy/spending_test.go | 43 ++ proxy/upstream_sync_test.go | 505 +++++++++++++++++++ retry/retry.go | 20 +- retry/retry_test.go | 96 ++++ router/config.go | 196 +++++--- spendcontrol/spendcontrol.go | 442 ++++++++++------- spendcontrol/spendcontrol_test.go | 410 ++++++++++++++++ stats/stats.go | 155 ++++-- stats/stats_test.go | 60 +++ 29 files changed, 3253 insertions(+), 741 deletions(-) create mode 100644 cache/requestkey_test.go create mode 100644 dedup/requestkey_test.go create mode 100644 internal/requestkey/normalize.go create mode 100644 internal/requestkey/normalize_test.go create mode 100644 models/catalog_test.go create mode 100644 proxy/prose.go create mode 100644 proxy/request.go create mode 100644 proxy/spending.go create mode 100644 proxy/spending_test.go create mode 100644 proxy/upstream_sync_test.go create mode 100644 retry/retry_test.go create mode 100644 spendcontrol/spendcontrol_test.go create mode 100644 stats/stats_test.go diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 7eb16a9..5e1e461 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,14 +1,14 @@ version: 2 updates: - - package-ecosystem: npm + - package-ecosystem: gomod + directory: "/" + schedule: + interval: weekly + day: monday + open-pull-requests-limit: 5 + - package-ecosystem: github-actions directory: "/" schedule: interval: weekly day: monday open-pull-requests-limit: 5 - groups: - dev-dependencies: - dependency-type: development - typescript-eslint: - patterns: - - "@typescript-eslint/*" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3acda55..44fd318 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,9 @@ on: branches: [main] workflow_dispatch: +permissions: + contents: read + concurrency: group: ci-${{ github.ref }} cancel-in-progress: true @@ -17,38 +20,23 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4.2.2 - - - uses: actions/setup-node@v4.3.0 + - uses: actions/setup-go@v5 with: - node-version: "22" - cache: npm - - - run: npm ci - - - name: Check formatting - run: npx prettier --check . - - - name: Lint - run: npx eslint src/ - - - name: Typecheck - run: npm run typecheck + go-version-file: go.mod + cache: true + - name: Vet + run: go vet ./... build-and-test: name: Build & Test runs-on: ubuntu-latest steps: - uses: actions/checkout@v4.2.2 - - - uses: actions/setup-node@v4.3.0 + - uses: actions/setup-go@v5 with: - node-version: "22" - cache: npm - - - run: npm ci - + go-version-file: go.mod + cache: true - name: Build - run: npm run build - - - name: Integration tests (lifecycle) - run: npx vitest run --config vitest.integration.config.ts test/integration/lifecycle.test.ts + run: go build ./... + - name: Test with race detector + run: go test -race -count=1 ./... diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 53b766d..b2de60d 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -7,6 +7,9 @@ on: branches: [main] workflow_dispatch: +permissions: + contents: read + concurrency: group: security-scanner-${{ github.ref }} cancel-in-progress: true @@ -14,25 +17,13 @@ concurrency: jobs: scanner: runs-on: ubuntu-latest - steps: - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build integration image - uses: docker/build-push-action@v6 + - uses: actions/setup-go@v5 with: - context: . - file: test/integration/Dockerfile - load: true - tags: clawrouter-integration:latest - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Run security scanner - run: > - docker compose -f test/integration/docker-compose.yml run --rm - integration npx vitest run --config vitest.integration.config.ts - test/integration/security-scanner.test.ts + go-version-file: go.mod + cache: true + - name: Install Go vulnerability scanner + run: go install golang.org/x/vuln/cmd/govulncheck@v1.1.4 + - name: Scan reachable vulnerabilities + run: govulncheck ./... diff --git a/README.md b/README.md index 142aa09..5fb56e4 100644 --- a/README.md +++ b/README.md @@ -125,9 +125,9 @@ go test ./router/ -bench=. -benchmem ## Upstream Sync -This is a Go port of [BlockRunAI/ClawRouter](https://github.com/BlockRunAI/ClawRouter). Routing logic is synced periodically from upstream releases. Payment, plugin lifecycle, and CLI-specific features are excluded. See [UPSTREAM_SYNC.md](UPSTREAM_SYNC.md) for details. +This is a Go port of [BlockRunAI/ClawRouter](https://github.com/BlockRunAI/ClawRouter). Applicable routing, proxy and spending-control changes are ported from upstream source snapshots. OpenClaw/desktop lifecycle and provider-specific account services are excluded; x402 signing remains unimplemented. See [UPSTREAM_SYNC.md](UPSTREAM_SYNC.md) for details. -**Current sync**: v0.12.146 +**Current sync**: v0.12.278 source snapshot (`05de1e0`, 2026-09-12). See the tracker for adapted and deferred features. ## License diff --git a/UPSTREAM_SYNC.md b/UPSTREAM_SYNC.md index a07e455..5da0b3c 100644 --- a/UPSTREAM_SYNC.md +++ b/UPSTREAM_SYNC.md @@ -2,7 +2,7 @@ **Upstream**: [BlockRunAI/ClawRouter](https://github.com/BlockRunAI/ClawRouter) (TypeScript) **This repo**: [DOS/DOSRouter](https://github.com/DOS/DOSRouter) (Go port) -**Last synced**: v0.12.245 (2026-08-16, core ported & aligned) +**Last synced**: v0.12.278 source snapshot (`05de1e0`, 2026-09-12; Go-applicable core ported, exclusions below) ## Sync Workflow @@ -30,6 +30,63 @@ These upstream areas are excluded (TS/npm-specific): ## Sync Log +### 2026-09-12 - Sync v0.12.245 to v0.12.278 source snapshot + +Compared `v0.12.245...05de1e0` from BlockRunAI/ClawRouter. The newest +published Git tag at sync time is `v0.12.277`; upstream `package.json` and +CHANGELOG identify the main snapshot as `0.12.278`. This is a source parity +update, not a claim that every BlockRun product feature is implemented. + +**Ported or adapted:** + +| Upstream area | Go adaptation | +| --- | --- | +| v0.12.248 assistant/tool prose | Preserve assistant prose with native calls and text-recovered calls; strip tagged thinking, including split SSE tags. `DOSROUTER_TOOL_CALL_PROSE=off` restores legacy suppression. Recover syntax only when tools are supplied. | +| v0.12.252 tool-pair safety | Preserve `tool_calls`, `tool_call_id`, names and all provider extension fields during request rewriting. Avoid compressing protocol-bearing or multimodal messages. DOSRouter has no upstream-style message truncation path. | +| v0.12.254-256 cancellation/cache | Keep Go request contexts through chat/image requests, stop fallback after disconnect, reject incomplete bodies, normalize injected first-text timestamps without merging distinct user data. | +| v0.12.257-278 models/routing | Align chat catalog metadata and all four profile chains with router-core `5ee7c23c993013a8052588191569db5cf7fb793c`; retain DOS aliases and exact explicit pins. Retire dead free defaults, fix capability claims and prices. | +| v0.12.263/269/274 spend safety | Atomic in-flight reservations for direct and routed chat, including fallback attempts; pending spend counts in rolling/session caps. Persist snapshots serially with atomic file replacement. Invalid state/cost fails closed. Each Server uses one controller; embedded callers may inject a shared controller explicitly. | +| v0.12.267 ambiguous sends | Do not repeat a chat send or switch models after an ambiguous transport failure. Explicit retryable HTTP rejection statuses retain backoff/retry. DOSRouter does not yet sign x402 payments. | +| v0.12.271-275 accounting/health | Prefer settled gateway cost headers, otherwise actual token usage, then explicitly labelled estimates. Capture gateway request IDs in usage logs; report the configured gateway origin in health. Image cost reads headers/body. Reject unknown-priced images when amount limits are configured. | +| v0.12.272 credential transport | Refuse upstream redirects, avoid shared internal caching across caller-supplied bearer credentials, and mark authenticated responses `no-store`. | +| Stats day windows | Go already defaulted nonpositive windows safely; cap aggregate reporting to 30 days and test using isolated log directories. | +| Validation adaptation | Replace stale TypeScript/npm CI and missing Docker scanner targets with Go build, vet, race tests and govulncheck. Preserve job/workflow names and automatic CodeQL; prefer patched Go 1.26.6 via the toolchain directive. Dependabot follows Go modules and Actions. | + +**Catalog notes:** 114 upstream chat rows plus nine compatibility records, with +250 chat aliases. The free default is `free/nemotron-3.5-lightning`. Model +metadata reflects upstream source, not independently probed DOS providers. +Gemini 3.6/3.8 Flash's $0.75/$3.75 promotional rates end on 2027-01-01, +when upstream documents $1.50/$7.50; automated repricing is not implemented. + +**Already satisfied:** `/v1/models` lists active catalog entries; chat and image +requests derive their context from the client; full health performs no balance +RPC; nonpositive stats/log windows have safe defaults. + +**Excluded or deferred:** + +- OpenClaw plugin identity/migrations, desktop releases, npm dependencies, + Solana defaults/RPC/signing, and the added-then-removed TWZRD integration are + outside this standalone Go runtime. +- BlockRun login/account-credit/status/reconcile APIs, account service proxying, + vendor-specific paid endpoints, image/video aliases and async polling require + separate product/provider contracts. The existing image endpoint remains a + passthrough. Missing media cost must not be interpreted as proof of a free call. +- x402 counterparty policy and signing hooks remain deferred: `payment.submitPayment` + still returns `Success:false`; no signing, live charge or facilitator rollout + occurred. Earlier tracker wording about a full payment port overstated support. +- Spend limits use estimates before dispatch, then gateway/token evidence when + available. They are not a provider-enforced USD guarantee. Unconfirmed sends + conservatively consume their estimate. Reservations and session counters are + process-local and reset at restart; file storage is not a multi-process ledger. + Cross-server sharing requires an explicitly shared controller. +- Streaming textual tool-call synthesis is not implemented; native streaming + tool calls and prose are preserved. No live provider request was used to + verify catalog availability or pricing. + +Validation is recorded in the sync PR: Go unit/integration tests, race tests, +build, vet, vulnerability scan and configured automated reviews. No deployment +workflow exists in this repository, and this sync does not deploy DOS-AI. + ### 2026-08-16 - Sync to v0.12.245 (flagship models, tool-call recovery, proxy hardening) Diffed `v0.12.199...v0.12.245` (46 tags). diff --git a/cache/cache.go b/cache/cache.go index 19d1a58..d94553d 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -10,10 +10,10 @@ import ( "encoding/hex" "encoding/json" "fmt" - "regexp" - "sort" "sync" "time" + + "github.com/DOS/DOSRouter/internal/requestkey" ) const ( @@ -33,9 +33,6 @@ var skipFields = map[string]bool{ "request_id": true, } -// timestampRe matches log-style timestamps like "[Mon 2024-01-15 09:30 UTC]". -var timestampRe = regexp.MustCompile(`^\[\w{3}\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+\w+\]\s*`) - // Entry is a cached response. type Entry struct { Body []byte @@ -285,7 +282,7 @@ func CacheKey(body []byte) (string, error) { delete(raw, f) } - canonical := canonicalize(raw) + canonical := canonicalize(requestkey.Normalize(raw)) encoded, err := json.Marshal(canonical) if err != nil { return "", fmt.Errorf("cache: marshal error: %w", err) @@ -294,21 +291,16 @@ func CacheKey(body []byte) (string, error) { return hex.EncodeToString(h[:]), nil } -// canonicalize recursively sorts object keys and strips timestamp prefixes -// from string values, producing a deterministic structure for hashing. +// canonicalize preserves JSON value types while copying nested containers. +// json.Marshal sorts object keys when encoding the canonical request. func canonicalize(v interface{}) interface{} { switch val := v.(type) { case map[string]interface{}: - keys := make([]string, 0, len(val)) - for k := range val { - keys = append(keys, k) - } - sort.Strings(keys) - pairs := make([][2]interface{}, 0, len(keys)) - for _, k := range keys { - pairs = append(pairs, [2]interface{}{k, canonicalize(val[k])}) + out := make(map[string]interface{}, len(val)) + for key, item := range val { + out[key] = canonicalize(item) } - return pairs + return out case []interface{}: out := make([]interface{}, len(val)) @@ -317,9 +309,6 @@ func canonicalize(v interface{}) interface{} { } return out - case string: - return timestampRe.ReplaceAllString(val, "") - default: return val } diff --git a/cache/requestkey_test.go b/cache/requestkey_test.go new file mode 100644 index 0000000..46f5d2e --- /dev/null +++ b/cache/requestkey_test.go @@ -0,0 +1,78 @@ +package cache + +import ( + "bytes" + "strings" + "testing" +) + +const multimodalRequest = `{"model":"test/model","messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"https://example.test/a.png","detail":"high"}},{"type":"text","text":"[Sat 2026-09-12 09:00 ICT] Describe this image"},{"type":"text","text":"[Sat 2026-09-12 10:00 ICT] Event"}]},{"role":"assistant","content":null,"tool_calls":[{"type":"function","function":{"name":"record_event","arguments":"[Sat 2026-09-12 11:00 ICT] Event"}}]}]}` + +func TestCacheReusesMultimodalResponseAcrossInjectedTimestamps(t *testing.T) { + first := []byte(multimodalRequest) + original := bytes.Clone(first) + second := []byte(strings.Replace(multimodalRequest, "09:00", "09:30", 1)) + c := New() + c.Set(first, Entry{Body: []byte("cached answer"), StatusCode: 200}) + got, ok := c.Get(second, false) + if !ok || string(got.Body) != "cached answer" { + t.Fatalf("timestamp-only change missed cached response: hit=%v, body=%q", ok, got.Body) + } + if !bytes.Equal(first, original) { + t.Error("cache key normalization mutated request bytes") + } +} + +func TestCacheKeyPreservesRequestSemantics(t *testing.T) { + tests := []struct{ name, before, after string }{ + {"later text timestamp", "10:00", "10:30"}, + {"tool arguments", "11:00", "11:30"}, + {"image URL", "a.png", "b.png"}, + {"image detail", `"detail":"high"`, `"detail":"low"`}, + {"model", "test/model", "test/other-model"}, + } + base, err := CacheKey([]byte(multimodalRequest)) + if err != nil { + t.Fatal(err) + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + changed := strings.Replace(multimodalRequest, tt.before, tt.after, 1) + key, err := CacheKey([]byte(changed)) + if err != nil { + t.Fatal(err) + } + if key == base { + t.Error("different response-affecting content shared a cache key") + } + }) + } +} + +func TestCacheKeyDistinguishesObjectsFromArrays(t *testing.T) { + object, err := CacheKey([]byte(`{"model":"test/model","payload":{"a":1}}`)) + if err != nil { + t.Fatal(err) + } + array, err := CacheKey([]byte(`{"model":"test/model","payload":[["a",1]]}`)) + if err != nil { + t.Fatal(err) + } + if object == array { + t.Fatal("object and array payloads shared a cache key") + } +} + +func TestCacheKeyIgnoresObjectKeyOrder(t *testing.T) { + first, err := CacheKey([]byte(`{"model":"test/model","payload":{"a":1,"b":[2,3]}}`)) + if err != nil { + t.Fatal(err) + } + second, err := CacheKey([]byte(`{"payload":{"b":[2,3],"a":1},"model":"test/model"}`)) + if err != nil { + t.Fatal(err) + } + if first != second { + t.Fatal("equivalent JSON objects had different cache keys") + } +} diff --git a/dedup/dedup.go b/dedup/dedup.go index 50ee2eb..9583a24 100644 --- a/dedup/dedup.go +++ b/dedup/dedup.go @@ -8,10 +8,10 @@ import ( "encoding/hex" "encoding/json" "fmt" - "regexp" - "sort" "sync" "time" + + "github.com/DOS/DOSRouter/internal/requestkey" ) const ( @@ -21,9 +21,6 @@ const ( DefaultMaxBodySize = 1 << 20 ) -// timestampRe matches log-style timestamps like "[Mon 2024-01-15 09:30 UTC]". -var timestampRe = regexp.MustCompile(`^\[\w{3}\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+\w+\]\s*`) - // Response holds a cached upstream response. type Response struct { StatusCode int @@ -167,7 +164,7 @@ func HashBody(body []byte) (string, error) { if err := json.Unmarshal(body, &raw); err != nil { return "", fmt.Errorf("dedup: invalid JSON body: %w", err) } - canonical := canonicalize(raw) + canonical := canonicalize(requestkey.Normalize(raw)) encoded, err := json.Marshal(canonical) if err != nil { return "", fmt.Errorf("dedup: marshal error: %w", err) @@ -176,23 +173,16 @@ func HashBody(body []byte) (string, error) { return hex.EncodeToString(h[:]), nil } -// canonicalize recursively sorts object keys and strips timestamp prefixes -// from string values. +// canonicalize preserves JSON value types while copying nested containers. +// json.Marshal sorts object keys when encoding the canonical request. func canonicalize(v interface{}) interface{} { switch val := v.(type) { case map[string]interface{}: - keys := make([]string, 0, len(val)) - for k := range val { - keys = append(keys, k) - } - sort.Strings(keys) - // Use an ordered representation: []interface{} of [key, value] pairs. - // This ensures json.Marshal produces a deterministic byte sequence. - pairs := make([][2]interface{}, 0, len(keys)) - for _, k := range keys { - pairs = append(pairs, [2]interface{}{k, canonicalize(val[k])}) + out := make(map[string]interface{}, len(val)) + for key, item := range val { + out[key] = canonicalize(item) } - return pairs + return out case []interface{}: out := make([]interface{}, len(val)) @@ -201,9 +191,6 @@ func canonicalize(v interface{}) interface{} { } return out - case string: - return timestampRe.ReplaceAllString(val, "") - default: return val } diff --git a/dedup/requestkey_test.go b/dedup/requestkey_test.go new file mode 100644 index 0000000..15c8130 --- /dev/null +++ b/dedup/requestkey_test.go @@ -0,0 +1,85 @@ +package dedup + +import ( + "bytes" + "strings" + "testing" +) + +const multimodalRequest = `{"model":"test/model","messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"https://example.test/a.png","detail":"high"}},{"type":"text","text":"[Sat 2026-09-12 09:00 ICT] Describe this image"},{"type":"text","text":"[Sat 2026-09-12 10:00 ICT] Event"}]},{"role":"assistant","content":null,"tool_calls":[{"type":"function","function":{"name":"record_event","arguments":"[Sat 2026-09-12 11:00 ICT] Event"}}]}]}` + +func TestDedupReusesMultimodalResponseAcrossInjectedTimestamps(t *testing.T) { + first := []byte(multimodalRequest) + original := bytes.Clone(first) + second := []byte(strings.Replace(multimodalRequest, "09:00", "09:30", 1)) + d := New() + calls := 0 + call := func() (Response, error) { + calls++ + return Response{Body: []byte("upstream answer"), StatusCode: 200}, nil + } + if _, hit, err := d.Do(first, call); err != nil || hit { + t.Fatalf("first request: hit=%v err=%v", hit, err) + } + got, hit, err := d.Do(second, call) + if err != nil || !hit || calls != 1 || string(got.Body) != "upstream answer" { + t.Fatalf("timestamp-only retry: hit=%v calls=%d body=%q err=%v", hit, calls, got.Body, err) + } + if !bytes.Equal(first, original) { + t.Error("dedup normalization mutated request bytes") + } +} + +func TestHashBodyPreservesRequestSemantics(t *testing.T) { + tests := []struct{ name, before, after string }{ + {"later text timestamp", "10:00", "10:30"}, + {"tool arguments", "11:00", "11:30"}, + {"image URL", "a.png", "b.png"}, + {"image detail", `"detail":"high"`, `"detail":"low"`}, + {"model", "test/model", "test/other-model"}, + } + base, err := HashBody([]byte(multimodalRequest)) + if err != nil { + t.Fatal(err) + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + changed := strings.Replace(multimodalRequest, tt.before, tt.after, 1) + key, err := HashBody([]byte(changed)) + if err != nil { + t.Fatal(err) + } + if key == base { + t.Error("different response-affecting content shared a dedup key") + } + }) + } +} + +func TestHashBodyDistinguishesObjectsFromArrays(t *testing.T) { + object, err := HashBody([]byte(`{"model":"test/model","payload":{"a":1}}`)) + if err != nil { + t.Fatal(err) + } + array, err := HashBody([]byte(`{"model":"test/model","payload":[["a",1]]}`)) + if err != nil { + t.Fatal(err) + } + if object == array { + t.Fatal("object and array payloads shared a dedup key") + } +} + +func TestHashBodyIgnoresObjectKeyOrder(t *testing.T) { + first, err := HashBody([]byte(`{"model":"test/model","payload":{"a":1,"b":[2,3]}}`)) + if err != nil { + t.Fatal(err) + } + second, err := HashBody([]byte(`{"payload":{"b":[2,3],"a":1},"model":"test/model"}`)) + if err != nil { + t.Fatal(err) + } + if first != second { + t.Fatal("equivalent JSON objects had different dedup keys") + } +} diff --git a/docs/configuration.md b/docs/configuration.md index 8c64c98..2636abf 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -238,3 +238,26 @@ go build ./... go test ./router/ -v go test ./router/ -bench=. -benchmem ``` + + +## Tool-call prose and spending controls + +Assistant prose is preserved alongside tool calls. Set +`DOSROUTER_TOOL_CALL_PROSE=off` to restore legacy prose suppression. Tagged +thinking is stripped from content, including tags split across streaming chunks. +Request extensions and tool-call/result IDs survive model rewriting. + +The proxy loads amount limits from `~/.openclaw/DOS/spending.json`. Invalid +state refuses paid dispatch rather than silently clearing limits. Direct and +routed chat requests reserve estimated cost before dispatch, then record +settled gateway headers or token-based cost when available. Concurrent pending +reservations count against session, hourly and daily limits. Explicit upstream +rejections release a reservation; ambiguous transport failures conservatively +consume its estimate and are not retried. Unknown-priced models and image +requests are refused when limits are configured. This is a local estimate-based +control, not a provider-side USD guarantee or multi-process ledger. + +Embedded servers can supply `proxy.Config.SpendControl` to share a controller +explicitly and `UsageLogger` to direct usage records to their own sink. Defaults +remain file-backed. No BlockRun account credentials or x402 signer are enabled +by the sync. diff --git a/go.mod b/go.mod index 12eedce..48e19f8 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/DOS/DOSRouter go 1.25.0 +toolchain go1.26.6 + require ( golang.org/x/crypto v0.50.0 // indirect golang.org/x/sys v0.43.0 // indirect diff --git a/internal/requestkey/normalize.go b/internal/requestkey/normalize.go new file mode 100644 index 0000000..dce178f --- /dev/null +++ b/internal/requestkey/normalize.go @@ -0,0 +1,76 @@ +// Package requestkey normalizes only timestamps injected into message content. +package requestkey + +import ( + "maps" + "regexp" +) + +var timestamp = regexp.MustCompile(`^\[\w{3}\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+\w+\]\s*`) + +// Normalize removes an injected prefix from root request messages' string +// content or first text block. Tool results, metadata, tool arguments and later +// text blocks are preserved, and the input request is never mutated. +func Normalize(value any) any { + request, ok := value.(map[string]any) + if !ok { + return value + } + messages, ok := request["messages"].([]any) + if !ok { + return value + } + + out := maps.Clone(request) + normalized := make([]any, len(messages)) + for i, item := range messages { + normalized[i] = item + message, ok := item.(map[string]any) + if !ok { + continue + } + role, _ := message["role"].(string) + if role == "tool" || role == "function" { + continue + } + messageContent, exists := message["content"] + if !exists { + continue + } + messageCopy := maps.Clone(message) + messageCopy["content"] = content(messageContent) + normalized[i] = messageCopy + } + out["messages"] = normalized + return out +} + +func content(value any) any { + if text, ok := value.(string); ok { + return timestamp.ReplaceAllString(text, "") + } + blocks, ok := value.([]any) + if !ok { + return value + } + out := make([]any, len(blocks)) + copy(out, blocks) + for i, block := range blocks { + obj, ok := block.(map[string]any) + if !ok || obj["type"] != "text" { + continue + } + text, ok := obj["text"].(string) + if !ok { + continue + } + copy := make(map[string]any, len(obj)) + for key, value := range obj { + copy[key] = value + } + copy["text"] = timestamp.ReplaceAllString(text, "") + out[i] = copy + break + } + return out +} diff --git a/internal/requestkey/normalize_test.go b/internal/requestkey/normalize_test.go new file mode 100644 index 0000000..ccf6abe --- /dev/null +++ b/internal/requestkey/normalize_test.go @@ -0,0 +1,112 @@ +package requestkey + +import ( + "encoding/json" + "reflect" + "testing" +) + +func TestNormalizeInjectedMessageTimestamps(t *testing.T) { + tests := []struct { + name string + input string + want string + }{ + { + name: "string message", + input: `{"messages":[{"role":"user","content":"[Sat 2026-09-12 09:00 ICT] Hello"}]}`, + want: `{"messages":[{"role":"user","content":"Hello"}]}`, + }, + { + name: "first text after image", + input: `{"messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"https://example.test/image.png","detail":"high"}},{"type":"text","text":"[Sat 2026-09-12 09:00 ICT] Describe this image"},{"type":"text","text":"[Sat 2026-09-12 10:00 ICT] The event happened then"}]}]}`, + want: `{"messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"https://example.test/image.png","detail":"high"}},{"type":"text","text":"Describe this image"},{"type":"text","text":"[Sat 2026-09-12 10:00 ICT] The event happened then"}]}]}`, + }, + { + name: "later text is never stripped", + input: `{"messages":[{"content":[{"type":"text","text":"Keep the quoted timestamp"},{"type":"text","text":"[Sat 2026-09-12 10:00 ICT] Event"}]}]}`, + want: `{"messages":[{"content":[{"type":"text","text":"Keep the quoted timestamp"},{"type":"text","text":"[Sat 2026-09-12 10:00 ICT] Event"}]}]}`, + }, + { + name: "tool arguments are preserved", + input: `{"messages":[{"role":"assistant","content":null,"tool_calls":[{"type":"function","function":{"name":"record_event","arguments":"[Sat 2026-09-12 10:00 ICT] Event"}}]}]}`, + want: `{"messages":[{"role":"assistant","content":null,"tool_calls":[{"type":"function","function":{"name":"record_event","arguments":"[Sat 2026-09-12 10:00 ICT] Event"}}]}]}`, + }, + { + name: "embedded timestamp is preserved", + input: `{"messages":[{"content":"Quote [Sat 2026-09-12 10:00 ICT] Event"}]}`, + want: `{"messages":[{"content":"Quote [Sat 2026-09-12 10:00 ICT] Event"}]}`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var input, original, want any + for _, value := range []struct { + text string + target *any + }{ + {tt.input, &input}, {tt.input, &original}, {tt.want, &want}, + } { + if err := json.Unmarshal([]byte(value.text), value.target); err != nil { + t.Fatal(err) + } + } + got := Normalize(input) + if !reflect.DeepEqual(got, want) { + t.Errorf("Normalize() = %#v, want %#v", got, want) + } + if !reflect.DeepEqual(input, original) { + t.Error("Normalize mutated its input") + } + }) + } +} + +func TestNormalizePreservesNonMessageContent(t *testing.T) { + tests := []struct { + name string + input string + }{ + {"root content", `{"content":"[Sat 2026-09-12 09:00 ICT] Root data"}`}, + {"metadata content", `{"metadata":{"content":"[Sat 2026-09-12 09:00 ICT] Event"},"messages":[{"role":"user","content":"Hello"}]}`}, + {"nested message history", `{"metadata":{"messages":[{"role":"user","content":"[Sat 2026-09-12 09:00 ICT] Historical event"}]},"messages":[]}`}, + {"message metadata", `{"messages":[{"role":"user","content":"Hello","metadata":{"content":"[Sat 2026-09-12 09:00 ICT] Event"}}]}`}, + {"object tool arguments", `{"messages":[{"role":"assistant","content":null,"tool_calls":[{"type":"function","function":{"name":"record_event","arguments":{"content":"[Sat 2026-09-12 09:00 ICT] Event"}}}]}]}`}, + {"tool result string", `{"messages":[{"role":"tool","tool_call_id":"call_1","content":"[Sat 2026-09-12 09:00 ICT] Observed event"}]}`}, + {"tool result text block", `{"messages":[{"role":"tool","tool_call_id":"call_1","content":[{"type":"text","text":"[Sat 2026-09-12 09:00 ICT] Observed event"}]}]}`}, + {"legacy function result", `{"messages":[{"role":"function","name":"record_event","content":"[Sat 2026-09-12 09:00 ICT] Observed event"}]}`}, + {"non-request array", `[{"role":"user","content":"[Sat 2026-09-12 09:00 ICT] Data"}]`}, + {"empty text block array", `{"messages":[{"role":"user","content":[]}]}`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var input, original any + if err := json.Unmarshal([]byte(tt.input), &input); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal([]byte(tt.input), &original); err != nil { + t.Fatal(err) + } + if got := Normalize(input); !reflect.DeepEqual(got, original) { + t.Errorf("Normalize changed content outside injected message prefixes: got %#v, want %#v", got, original) + } + if !reflect.DeepEqual(input, original) { + t.Error("Normalize mutated its input") + } + }) + } +} + +func TestNormalizePreservesSupportedMessageRoles(t *testing.T) { + var input any + if err := json.Unmarshal([]byte(`{"messages":[{"role":"system","content":"[Sat 2026-09-12 09:00 ICT] System"},{"role":"assistant","content":"[Sat 2026-09-12 09:00 ICT] Assistant"},{"role":"user","content":"[Sat 2026-09-12 09:00 ICT] User"}]}`), &input); err != nil { + t.Fatal(err) + } + var want any + if err := json.Unmarshal([]byte(`{"messages":[{"role":"system","content":"System"},{"role":"assistant","content":"Assistant"},{"role":"user","content":"User"}]}`), &want); err != nil { + t.Fatal(err) + } + if got := Normalize(input); !reflect.DeepEqual(got, want) { + t.Errorf("injected timestamps on supported roles were not stripped: got %#v, want %#v", got, want) + } +} diff --git a/logger/logger.go b/logger/logger.go index 64c4541..19a975c 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -28,6 +28,8 @@ func ensureDir() { // UsageEntry represents a single usage log record. type UsageEntry struct { + RequestID string `json:"requestId,omitempty"` + CostSource string `json:"costSource,omitempty"` Timestamp string `json:"timestamp"` Model string `json:"model"` Tier string `json:"tier"` diff --git a/models/catalog_test.go b/models/catalog_test.go new file mode 100644 index 0000000..60c5ff4 --- /dev/null +++ b/models/catalog_test.go @@ -0,0 +1,149 @@ +package models + +import ( + "strings" + "testing" + + "github.com/DOS/DOSRouter/router" +) + +// Catalog prices and limits feed budget admission before a request is sent. +// Cover new targets and material corrections so omitted or stale paid entries +// cannot bypass cost checks or advertise invalid context/output limits. +func TestSyncedCatalogEconomicsAndLimits(t *testing.T) { + cases := []struct { + id string + input, output float64 + context, max int + }{ + {"openai/gpt-5.6-sol", 4, 20, 1050000, 128000}, + {"openai/gpt-5.6-terra", 2, 12, 1050000, 128000}, + {"openai/gpt-5.6-luna", 0.2, 1.2, 1050000, 128000}, + {"openai/chat-latest", 5, 30, 128000, 128000}, + {"anthropic/claude-fable-5", 10, 50, 1000000, 128000}, + {"anthropic/claude-sonnet-4.6", 3, 15, 1000000, 128000}, + {"anthropic/claude-sonnet-5", 3, 15, 1000000, 128000}, + {"moonshot/kimi-k3", 3, 15, 1048576, 65536}, + {"google/gemini-3.8-flash", 0.75, 3.75, 1048576, 65536}, + {"google/gemini-3.6-flash", 0.75, 3.75, 1048576, 65536}, + {"deepseek/deepseek-v4-pro", 1.32, 3.96, 1048576, 65536}, + {"deepseek/deepseek-v4-flash-vision-exp", 0.44, 1.32, 1048576, 65536}, + {"qwen/qwen3.8-flash", 0.15, 0.47, 1000000, 131072}, + {"xiaomi/mimo-v2.5", 0.14, 0.28, 1048576, 131072}, + {"zai/glm-5.3", 1.4, 4.4, 1000000, 131072}, + {"zai/glm-5.3-flash", 0.15, 0.5, 1000000, 131072}, + } + for _, tc := range cases { + t.Run(tc.id, func(t *testing.T) { + m := GetModel(tc.id) + if m == nil { + t.Fatal("model missing from budget metadata") + } + if m.InputPrice != tc.input || m.OutputPrice != tc.output { + t.Errorf("prices = %g/%g, want %g/%g", m.InputPrice, m.OutputPrice, tc.input, tc.output) + } + if m.ContextWindow != tc.context || m.MaxOutput != tc.max { + t.Errorf("context/output = %d/%d, want %d/%d", m.ContextWindow, m.MaxOutput, tc.context, tc.max) + } + }) + } +} + +func TestCatalogIdentityAndRetirementInvariants(t *testing.T) { + seen := make(map[string]bool) + for _, m := range Models { + if seen[m.ID] { + t.Errorf("duplicate model ID %q", m.ID) + } + seen[m.ID] = true + if m.ContextWindow <= 0 || m.MaxOutput <= 0 { + t.Errorf("model %q lacks positive limits", m.ID) + } + if !m.Deprecated && strings.Contains(m.ID, "/") && ResolveModelAlias(m.ID) != m.ID { + t.Errorf("active model %q is shadowed by an alias", m.ID) + } + if m.Deprecated { + fallback := GetModel(m.FallbackModel) + if fallback == nil || fallback.Deprecated { + t.Errorf("retired model %q has missing/retired fallback %q", m.ID, m.FallbackModel) + } + if strings.HasPrefix(m.ID, "free/") && !strings.HasPrefix(m.FallbackModel, "free/") { + t.Errorf("retired free model %q redirects to a paid model", m.ID) + } + } + if strings.HasPrefix(m.ID, "free/") { + if m.InputPrice != 0 || m.OutputPrice != 0 || GetActivePromoPrice(m.ID) != nil { + t.Errorf("free model %q has a charge", m.ID) + } + if m.Vision || m.ToolCalling { + t.Errorf("free model %q advertises unsupported image/tool eligibility", m.ID) + } + } + } + // ResolveModelAlias is single-pass: every target needs immediate metadata. + for alias, target := range ModelAliases { + if GetModel(target) == nil { + t.Errorf("alias %q points at uncatalogued target %q", alias, target) + } + } +} + +func TestDefaultRoutingTargetsHaveActiveBudgetMetadata(t *testing.T) { + cfg := router.DefaultRoutingConfig() + profiles := map[string]map[router.Tier]router.TierConfig{ + "auto": cfg.Tiers, "eco": cfg.EcoTiers, + "premium": cfg.PremiumTiers, "agentic": cfg.AgenticTiers, + } + for profile, tiers := range profiles { + for tier, chain := range tiers { + for _, id := range append([]string{chain.Primary}, chain.Fallback...) { + m := GetModel(id) + if m == nil || m.Deprecated { + t.Errorf("%s/%s routes to missing/retired model %q", profile, tier, id) + continue + } + if !strings.HasPrefix(id, "free/") && (m.InputPrice <= 0 || m.OutputPrice <= 0) { + t.Errorf("%s/%s paid target %q lacks budget pricing", profile, tier, id) + } + if strings.Contains(id, "gpt-oss-") { + t.Errorf("%s/%s selects withheld GPT-OSS model %q", profile, tier, id) + } + } + } + } + if free := ResolveModelAlias("free"); free != cfg.EcoTiers[router.TierSimple].Primary { + t.Errorf("free alias %q differs from eco SIMPLE primary %q", free, cfg.EcoTiers[router.TierSimple].Primary) + } +} + +func TestSyncedVisionAndExplicitAliases(t *testing.T) { + for _, id := range []string{ + "google/gemini-2.5-flash-lite", "google/gemini-3.8-flash", + "qwen/qwen3.8-flash", "deepseek/deepseek-v4-flash-vision-exp", + "xiaomi/mimo-v2.5", "zai/glm-5.3-flash", + } { + if !SupportsVision(id) { + t.Errorf("verified vision model %q is excluded from image routing", id) + } + } + for alias, want := range map[string]string{ + "anthropic/claude-opus-4.5": "anthropic/claude-opus-4.5", + "openai/o1": "openai/o1", + "opus-4.7": "anthropic/claude-opus-4.7", + "gpt-5.6-sol-pro": "openai/gpt-5.6-sol-pro", + "kimi-k2.5": "moonshot/kimi-k2.5", + "glm-5.2": "zai/glm-5.2", + "glm": "zai/glm-5.3", + "mimo-v2.5": "xiaomi/mimo-v2.5", + "mimo": "xiaomi/mimo-v2.5-pro", + "chat-latest": "openai/chat-latest", + "chatgpt-instant": "openai/chat-latest", + "gpt-120b": "free/gpt-oss-120b", + "coder-free": "free/north-mini-code", + "deepseek-v4-pro": "deepseek/deepseek-v4-pro", + } { + if got := ResolveModelAlias(alias); got != want { + t.Errorf("ResolveModelAlias(%q) = %q, want %q", alias, got, want) + } + } +} diff --git a/models/models.go b/models/models.go index 94a1351..30d75d5 100644 --- a/models/models.go +++ b/models/models.go @@ -36,254 +36,430 @@ type PromoDef struct { // ModelAliases maps shorthand names to full model IDs. var ModelAliases = map[string]string{ - // Claude - "claude": "anthropic/claude-sonnet-5", - "sonnet": "anthropic/claude-sonnet-5", - "sonnet-5": "anthropic/claude-sonnet-5", - "sonnet-4": "anthropic/claude-sonnet-4.6", - "fable": "anthropic/claude-fable-5", - "claude-fable": "anthropic/claude-fable-5", - // Opus: Opus 5 is current flagship (upstream v0.12.233). Bare and forward aliases resolve to Opus 5; - // explicit version pins stay on their version. - "opus": "anthropic/claude-opus-5", - "opus-5": "anthropic/claude-opus-5", - "opus-5.0": "anthropic/claude-opus-5", - "opus-4": "anthropic/claude-opus-4.8", - "opus-4.8": "anthropic/claude-opus-4.8", - "opus-4-8": "anthropic/claude-opus-4.8", - "opus-4.7": "anthropic/claude-opus-4.7", - "opus-4-7": "anthropic/claude-opus-4.7", - "opus-4.6": "anthropic/claude-opus-4.6", - "opus-4-6": "anthropic/claude-opus-4.6", - "haiku": "anthropic/claude-haiku-4.5", - "anthropic/sonnet": "anthropic/claude-sonnet-5", - "anthropic/opus": "anthropic/claude-opus-5", - "anthropic/claude-opus-5": "anthropic/claude-opus-5", - "anthropic/haiku": "anthropic/claude-haiku-4.5", - "anthropic/claude": "anthropic/claude-sonnet-5", - "anthropic/claude-opus-4": "anthropic/claude-opus-4.8", - "anthropic/claude-opus-4-8": "anthropic/claude-opus-4.8", - "anthropic/claude-opus-4-7": "anthropic/claude-opus-4.7", - "anthropic/claude-opus-4-6": "anthropic/claude-opus-4.6", - "anthropic/claude-opus-4.5": "anthropic/claude-opus-4.8", - - // OpenAI — GPT-5.6 Terra is the current default flagship (upstream v0.12.219). - "gpt": "openai/gpt-4o", - "gpt4": "openai/gpt-4o", - "gpt5": "openai/gpt-5.6-terra", - "gpt-5.6": "openai/gpt-5.6-terra", - "gpt-5.6-terra": "openai/gpt-5.6-terra", - "gpt-5.6-sol": "openai/gpt-5.6-sol", - "gpt-5.6-luna": "openai/gpt-5.6-luna", - "gpt-5.5": "openai/gpt-5.5", - "gpt-5.5-pro": "openai/gpt-5.5-pro", - "chat-latest": "openai/chatgpt-instant", - "chatgpt-instant": "openai/chatgpt-instant", - "mini": "openai/gpt-4o-mini", - "nano": "openai/gpt-5.4-nano", - "gpt-5.4-mini": "openai/gpt-5.4-mini", - "openai-codex/gpt-5.4-mini": "openai/gpt-5.4-mini", - "codex": "openai/gpt-5.3-codex", - "o1": "openai/o3", - "o3": "openai/o3", - - // DeepSeek - "deepseek": "deepseek/deepseek-chat", - "deepseek-chat": "deepseek/deepseek-chat", - "reasoner": "deepseek/deepseek-reasoner", - "deepseek-v4-pro": "deepseek/deepseek-v4-pro", - - // Kimi / Moonshot — K2.7 bare flagship, K3 available (upstream v0.12.229/230) - "kimi": "moonshot/kimi-k2.7", - "moonshot": "moonshot/kimi-k2.7", - "kimi-k3": "moonshot/kimi-k3", - "kimi-k2.7": "moonshot/kimi-k2.7", - "kimi-k2": "moonshot/kimi-k2.6", - "kimi-k2.6": "moonshot/kimi-k2.6", - "kimi-k2.5": "moonshot/kimi-k2.5", - "nvidia/kimi-k2.5": "moonshot/kimi-k2.5", - - // Google - "gemini": "google/gemini-2.5-pro", - "flash": "google/gemini-2.5-flash", - "gemini-3.5-flash": "google/gemini-3.5-flash", - - // xAI — Grok 4.5 flagship (upstream v0.12.225) - "grok": "xai/grok-4.5", - "grok-4.5": "xai/grok-4.5", - "grok-4.3": "xai/grok-4.3", - "grok-fast": "xai/grok-4-fast-reasoning", - "grok-4.20": "xai/grok-4.20-reasoning", - "grok-4-20": "xai/grok-4.20-reasoning", - - // MiniMax — M3 flagship (upstream v0.12.200) - "minimax": "minimax/minimax-m3", - "minimax-m3": "minimax/minimax-m3", - - // Qwen — Qwen 3.7 Max (upstream v0.12.231) - "qwen3.7-max": "qwen/qwen3.7-max", - "qwen-max": "qwen/qwen3.7-max", - - // Free models — realigned with BlockRun server up to v0.12.245 - "nvidia": "free/gpt-oss-120b", - "free": "free/gpt-oss-120b", - "qwen-coder": "free/llama-4-maverick", - "qwen-coder-free": "free/llama-4-maverick", - "qwen-thinking": "free/qwen3-next-80b-a3b-thinking", - "qwen3-next": "free/qwen3-next-80b-a3b-thinking", - "mistral-small": "free/llama-4-maverick", - "mistral-free": "free/llama-4-maverick", - // DeepSeek free redirects (V4 Flash EOL -> llama-4-maverick upstream v0.12.245) - "deepseek-free": "free/llama-4-maverick", - "deepseek-v4-flash": "free/llama-4-maverick", - "v4-flash": "free/llama-4-maverick", - "free/deepseek-v3.2": "free/llama-4-maverick", - "free/deepseek-v4-pro": "free/llama-4-maverick", - "free/deepseek-v4-flash": "free/llama-4-maverick", - "nvidia/deepseek-v3.2": "free/llama-4-maverick", - "nvidia/deepseek-v4-pro": "free/llama-4-maverick", - "nvidia/deepseek-v4-flash": "free/llama-4-maverick", - "glm-free": "free/glm-4.7", - "llama-free": "free/llama-4-maverick", - "maverick": "free/llama-4-maverick", - // Retired free IDs -> successors - "nemotron": "free/qwen3-next-80b-a3b-thinking", - "nemotron-ultra": "free/qwen3-next-80b-a3b-thinking", - "nemotron-253b": "free/qwen3-next-80b-a3b-thinking", - "nemotron-super": "free/qwen3-next-80b-a3b-thinking", - "nemotron-49b": "free/qwen3-next-80b-a3b-thinking", - "nemotron-120b": "free/qwen3-next-80b-a3b-thinking", - "devstral": "free/llama-4-maverick", - "devstral-2": "free/llama-4-maverick", - "free/nemotron-ultra-253b": "free/qwen3-next-80b-a3b-thinking", - "free/nemotron-3-super-120b": "free/qwen3-next-80b-a3b-thinking", - "free/nemotron-super-49b": "free/qwen3-next-80b-a3b-thinking", - "free/mistral-large-3-675b": "free/llama-4-maverick", - "free/mistral-small-4-119b": "free/llama-4-maverick", - "free/devstral-2-123b": "free/llama-4-maverick", - "free/qwen3-coder-480b": "free/llama-4-maverick", - "free/seed-oss-36b": "free/gpt-oss-120b", - "nvidia/nemotron-ultra-253b": "free/qwen3-next-80b-a3b-thinking", - "nvidia/nemotron-3-super-120b": "free/qwen3-next-80b-a3b-thinking", - "nvidia/nemotron-super-49b": "free/qwen3-next-80b-a3b-thinking", - "nvidia/mistral-large-3-675b": "free/llama-4-maverick", - "nvidia/devstral-2-123b": "free/llama-4-maverick", - - // Z.AI — GLM-5.2 is the flagship (upstream v0.12.211) - "glm": "zai/glm-5.2", - "glm-5.2": "zai/glm-5.2", - "glm-5.1": "zai/glm-5.1", - "glm-5": "zai/glm-5", - - // Routing profiles - "auto-router": "auto", - "router": "auto", + // Upstream chat aliases plus DOS-compatible shorthand targets. + // Bare Claude/Sonnet, o1, Gemini, Flash and paid DeepSeek Pro retain DOS behavior. + "anthropic/claude": "anthropic/claude-sonnet-5", + "anthropic/claude-fable-5.0": "anthropic/claude-fable-5", + "anthropic/claude-haiku-4": "anthropic/claude-haiku-4.5", + "anthropic/claude-haiku-4-5": "anthropic/claude-haiku-4.5", + "anthropic/claude-opus-4": "anthropic/claude-opus-4.8", + "anthropic/claude-opus-4-5": "anthropic/claude-opus-4.5", + "anthropic/claude-opus-4-6": "anthropic/claude-opus-4.6", + "anthropic/claude-opus-4-7": "anthropic/claude-opus-4.7", + "anthropic/claude-opus-4-8": "anthropic/claude-opus-4.8", + "anthropic/claude-opus-5": "anthropic/claude-opus-5", + "anthropic/claude-opus-5-0": "anthropic/claude-opus-5", + "anthropic/claude-opus-5.0": "anthropic/claude-opus-5", + "anthropic/claude-sonnet-4": "anthropic/claude-sonnet-4.6", + "anthropic/claude-sonnet-4-5": "anthropic/claude-sonnet-4.5", + "anthropic/claude-sonnet-4-6": "anthropic/claude-sonnet-4.6", + "anthropic/fable": "anthropic/claude-fable-5", + "anthropic/haiku": "anthropic/claude-haiku-4.5", + "anthropic/opus": "anthropic/claude-opus-5", + "anthropic/sonnet": "anthropic/claude-sonnet-5", + "auto-router": "auto", + "br-sonnet": "anthropic/claude-sonnet-4.6", + "chat-latest": "openai/chat-latest", + "chatgpt": "openai/chat-latest", + "chatgpt-instant": "openai/chat-latest", + "claude": "anthropic/claude-sonnet-5", + "claude-fable": "anthropic/claude-fable-5", + "coder-free": "free/north-mini-code", + "codex": "openai/gpt-5.3-codex", + "cohere/north-mini-code": "free/north-mini-code", + "deepseek": "deepseek/deepseek-chat", + "deepseek-chat": "deepseek/deepseek-chat", + "deepseek-free": "free/nemotron-3.5-lightning", + "deepseek-v4-flash": "free/nemotron-3.5-lightning", + "deepseek-v4-pro": "deepseek/deepseek-v4-pro", + "deepseek-vision": "deepseek/deepseek-v4-flash-vision-exp", + "devstral": "free/nemotron-3.5-lightning", + "devstral-2": "free/nemotron-3.5-lightning", + "fable": "anthropic/claude-fable-5", + "fable-5": "anthropic/claude-fable-5", + "fable-5.0": "anthropic/claude-fable-5", + "flash": "google/gemini-2.5-flash", + "free": "free/nemotron-3.5-lightning", + "free/deepseek-v3.2": "free/nemotron-3.5-lightning", + "free/deepseek-v4-flash": "free/nemotron-3.5-lightning", + "free/deepseek-v4-pro": "free/nemotron-3.5-lightning", + "free/devstral-2-123b": "free/nemotron-3.5-lightning", + "free/mistral-large-3-675b": "free/nemotron-3.5-lightning", + "free/mistral-small-4-119b": "free/nemotron-3.5-lightning", + "free/nemotron-3-super-120b": "free/nemotron-3.5-lightning", + "free/nemotron-super-49b": "free/nemotron-3.5-lightning", + "free/nemotron-ultra-253b": "free/nemotron-3.5-lightning", + "free/qwen3-coder-480b": "free/nemotron-3.5-lightning", + "free/seed-oss-36b": "free/nemotron-3.5-lightning", + "gemini": "google/gemini-2.5-pro", + "gemini-2.5-flash-lite": "google/gemini-2.5-flash-lite", + "gemini-3-pro": "google/gemini-3.1-pro", + "gemini-3-pro-preview": "google/gemini-3.1-pro", + "gemini-3.1-flash-lite": "google/gemini-3.1-flash-lite", + "gemini-3.1-pro": "google/gemini-3.1-pro", + "gemini-3.1-pro-preview": "google/gemini-3.1-pro", + "gemini-3.5-flash": "google/gemini-3.5-flash", + "gemini-3.5-flash-lite": "google/gemini-3.5-flash-lite", + "gemini-3.6": "google/gemini-3.6-flash", + "gemini-3.6-flash": "google/gemini-3.6-flash", + "gemini-pro": "google/gemini-3.1-pro", + "glm": "zai/glm-5.3", + "glm-5": "zai/glm-5", + "glm-5-3": "zai/glm-5.3", + "glm-5-3-flash": "zai/glm-5.3-flash", + "glm-5-turbo": "zai/glm-5-turbo", + "glm-5.1": "zai/glm-5.1", + "glm-5.2": "zai/glm-5.2", + "glm-5.3": "zai/glm-5.3", + "glm-5.3-flash": "zai/glm-5.3-flash", + "glm-flash": "zai/glm-5.3-flash", + "glm-free": "free/nemotron-3.5-lightning", + "google/gemini-3-pro-preview": "google/gemini-3.1-pro", + "google/gemini-3.1-pro-preview": "google/gemini-3.1-pro", + "gpt": "openai/gpt-4o", + "gpt-120b": "free/gpt-oss-120b", + "gpt-20b": "free/gpt-oss-20b", + "gpt-5-nano": "openai/gpt-5.4-nano", + "gpt-5.4": "openai/gpt-5.4", + "gpt-5.4-mini": "openai/gpt-5.4-mini", + "gpt-5.4-nano": "openai/gpt-5.4-nano", + "gpt-5.4-pro": "openai/gpt-5.4-pro", + "gpt-5.5": "openai/gpt-5.5", + "gpt-5.5-pro": "openai/gpt-5.5-pro", + "gpt-5.6": "openai/gpt-5.6-terra", + "gpt-5.6-luna": "openai/gpt-5.6-luna", + "gpt-5.6-luna-pro": "openai/gpt-5.6-luna-pro", + "gpt-5.6-sol": "openai/gpt-5.6-sol", + "gpt-5.6-sol-pro": "openai/gpt-5.6-sol-pro", + "gpt-5.6-terra": "openai/gpt-5.6-terra", + "gpt-5.6-terra-pro": "openai/gpt-5.6-terra-pro", + "gpt4": "openai/gpt-4o", + "gpt5": "openai/gpt-5.6-terra", + "grok": "xai/grok-4.5", + "grok-4-20": "xai/grok-4.20-reasoning", + "grok-4-5": "xai/grok-4.5", + "grok-4.20": "xai/grok-4.20-reasoning", + "grok-4.3": "xai/grok-4.3", + "grok-4.5": "xai/grok-4.5", + "grok-build": "xai/grok-build-0.1", + "grok-code": "xai/grok-build-0.1", + "grok-code-fast-1": "deepseek/deepseek-chat", + "grok-fast": "xai/grok-4-fast-reasoning", + "haiku": "anthropic/claude-haiku-4.5", + "hunyuan": "tencent/hy3", + "hy3": "tencent/hy3", + "kimi": "moonshot/kimi-k2.7", + "kimi-k2": "moonshot/kimi-k2.6", + "kimi-k2.5": "moonshot/kimi-k2.5", + "kimi-k2.6": "moonshot/kimi-k2.6", + "kimi-k2.7": "moonshot/kimi-k2.7", + "kimi-k3": "moonshot/kimi-k3", + "laguna": "free/laguna-xs-2.1", + "laguna-xs": "free/laguna-xs-2.1", + "lightning": "free/nemotron-3.5-lightning", + "llama-3.2-vision": "free/llama-3.2-11b-vision", + "llama-free": "free/llama-3.2-11b-vision", + "llama-vision": "free/llama-3.2-11b-vision", + "luna-pro": "openai/gpt-5.6-luna-pro", + "maverick": "free/llama-4-maverick", + "mimo": "xiaomi/mimo-v2.5-pro", + "mimo-v2.5": "xiaomi/mimo-v2.5", + "mimo-v2.5-pro": "xiaomi/mimo-v2.5-pro", + "mimo-vision": "xiaomi/mimo-v2.5", + "mini": "openai/gpt-4o-mini", + "minimax": "minimax/minimax-m3", + "minimax-m2.5": "minimax/minimax-m2.5", + "minimax-m2.7": "minimax/minimax-m2.7", + "minimax-m3": "minimax/minimax-m3", + "mistral-free": "free/nemotron-3.5-lightning", + "mistral-large": "free/mistral-large-3-675b", + "mistral-large-3-675b": "free/mistral-large-3-675b", + "mistral-nemotron": "free/mistral-nemotron", + "mistral-small": "free/nemotron-3.5-lightning", + "moonshot": "moonshot/kimi-k2.7", + "nano": "openai/gpt-5.4-nano", + "nano-30b": "free/nemotron-3-nano-30b", + "nano-omni": "free/nemotron-3-nano-omni-30b-a3b-reasoning", + "nano-vl": "free/nemotron-3-nano-omni-30b-a3b-reasoning", + "nemotron": "free/nemotron-3.5-lightning", + "nemotron-120b": "free/nemotron-3.5-lightning", + "nemotron-253b": "free/nemotron-3.5-lightning", + "nemotron-3.5-lightning": "free/nemotron-3.5-lightning", + "nemotron-49b": "free/nemotron-3.5-lightning", + "nemotron-lightning": "free/nemotron-3.5-lightning", + "nemotron-nano": "free/nemotron-3-nano-30b", + "nemotron-nano-30b": "free/nemotron-3-nano-30b", + "nemotron-nano-9b": "free/nemotron-3-nano-30b", + "nemotron-nano-vl": "free/nemotron-3-nano-omni-30b-a3b-reasoning", + "nemotron-omni": "free/nemotron-3-nano-omni-30b-a3b-reasoning", + "nemotron-super": "free/nemotron-3.5-lightning", + "nemotron-ultra": "free/nemotron-3.5-lightning", + "nemotron-ultra-550b": "free/nemotron-3-ultra-550b", + "north-mini": "free/north-mini-code", + "north-mini-code": "free/north-mini-code", + "nvidia": "free/nemotron-3.5-lightning", + "nvidia/deepseek-v3.2": "free/nemotron-3.5-lightning", + "nvidia/deepseek-v4-flash": "free/nemotron-3.5-lightning", + "nvidia/deepseek-v4-pro": "free/nemotron-3.5-lightning", + "nvidia/devstral-2-123b": "free/nemotron-3.5-lightning", + "nvidia/glm-4.7": "free/glm-4.7", + "nvidia/gpt-oss-120b": "free/gpt-oss-120b", + "nvidia/gpt-oss-20b": "free/gpt-oss-20b", + "nvidia/kimi-k2.5": "moonshot/kimi-k2.5", + "nvidia/llama-3.2-11b-vision": "free/llama-3.2-11b-vision", + "nvidia/llama-4-maverick": "free/llama-4-maverick", + "nvidia/mistral-large-3-675b": "free/nemotron-3.5-lightning", + "nvidia/mistral-nemotron": "free/mistral-nemotron", + "nvidia/nemotron-3-nano-30b": "free/nemotron-3-nano-30b", + "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning": "free/nemotron-3-nano-omni-30b-a3b-reasoning", + "nvidia/nemotron-3-super-120b": "free/nemotron-3.5-lightning", + "nvidia/nemotron-3-ultra-550b": "free/nemotron-3-ultra-550b", + "nvidia/nemotron-3.5-lightning": "free/nemotron-3.5-lightning", + "nvidia/nemotron-nano-12b-v2-vl": "free/nemotron-nano-12b-v2-vl", + "nvidia/nemotron-nano-9b-v2": "free/nemotron-nano-9b-v2", + "nvidia/nemotron-super-49b": "free/nemotron-3.5-lightning", + "nvidia/nemotron-ultra-253b": "free/nemotron-3.5-lightning", + "nvidia/qwen3-coder-480b": "free/qwen3-coder-480b", + "nvidia/qwen3-next-80b-a3b-instruct": "free/qwen3-next-80b-a3b-instruct", + "nvidia/qwen3-next-80b-a3b-thinking": "free/qwen3-next-80b-a3b-instruct", + "nvidia/qwen3.5-122b-a10b": "free/qwen3.5-122b-a10b", + "nvidia/seed-oss-36b": "free/seed-oss-36b", + "nvidia/step-3.7-flash": "free/step-3.7-flash", + "o1": "openai/o3", + "o1-mini": "openai/o4-mini", + "o3": "openai/o3", + "openai-codex/gpt-5.4-mini": "openai/gpt-5.4-mini", + "openai/chatgpt-instant": "openai/chat-latest", + "openai/gpt-5.6": "openai/gpt-5.6-terra", + "opus": "anthropic/claude-opus-5", + "opus-4": "anthropic/claude-opus-4.8", + "opus-4-6": "anthropic/claude-opus-4.6", + "opus-4-7": "anthropic/claude-opus-4.7", + "opus-4-8": "anthropic/claude-opus-4.8", + "opus-4.6": "anthropic/claude-opus-4.6", + "opus-4.7": "anthropic/claude-opus-4.7", + "opus-4.8": "anthropic/claude-opus-4.8", + "opus-5": "anthropic/claude-opus-5", + "opus-5-0": "anthropic/claude-opus-5", + "opus-5.0": "anthropic/claude-opus-5", + "poolside/laguna-xs-2.1": "free/laguna-xs-2.1", + "qwen-3.7-flash": "qwen/qwen3.7-flash", + "qwen-3.7-max": "qwen/qwen3.7-max", + "qwen-3.7-plus": "qwen/qwen3.7-plus", + "qwen-coder": "free/nemotron-3.5-lightning", + "qwen-coder-free": "free/nemotron-3.5-lightning", + "qwen-max": "qwen/qwen3.7-max", + "qwen-thinking": "free/nemotron-3.5-lightning", + "qwen-vision": "qwen/qwen3.8-flash", + "qwen/qwen3-coder-480b-a35b-instruct": "free/qwen3-coder-480b", + "qwen3-122b": "free/qwen3.5-122b-a10b", + "qwen3-7-max": "qwen/qwen3.7-max", + "qwen3-8-flash": "qwen/qwen3.8-flash", + "qwen3-next": "free/nemotron-3.5-lightning", + "qwen3-next-80b": "free/qwen3-next-80b-a3b-instruct", + "qwen3.5-122b": "free/qwen3.5-122b-a10b", + "qwen3.7-flash": "qwen/qwen3.7-flash", + "qwen3.7-max": "qwen/qwen3.7-max", + "qwen3.7-plus": "qwen/qwen3.7-plus", + "qwen3.8-flash": "qwen/qwen3.8-flash", + "reasoner": "deepseek/deepseek-reasoner", + "router": "auto", + "seed-oss": "free/seed-oss-36b", + "seed-oss-36b": "free/seed-oss-36b", + "sol-pro": "openai/gpt-5.6-sol-pro", + "sonnet": "anthropic/claude-sonnet-5", + "sonnet-4": "anthropic/claude-sonnet-4.6", + "sonnet-4-5": "anthropic/claude-sonnet-4.5", + "sonnet-4-6": "anthropic/claude-sonnet-4.6", + "sonnet-4.5": "anthropic/claude-sonnet-4.5", + "sonnet-4.6": "anthropic/claude-sonnet-4.6", + "sonnet-5": "anthropic/claude-sonnet-5", + "sonnet-5-0": "anthropic/claude-sonnet-5", + "sonnet-5.0": "anthropic/claude-sonnet-5", + "step-3.7-flash": "free/step-3.7-flash", + "step-flash": "free/step-3.7-flash", + "tencent": "tencent/hy3", + "terra-pro": "openai/gpt-5.6-terra-pro", + "ultra-550b": "free/nemotron-3-ultra-550b", + "v4-flash": "free/nemotron-3.5-lightning", + "v4-flash-vision": "deepseek/deepseek-v4-flash-vision-exp", + "v4-pro": "free/nemotron-3.5-lightning", + "vision-free": "free/nemotron-3-nano-omni-30b-a3b-reasoning", + "xai/grok-3-fast": "xai/grok-4-fast-reasoning", + "xai/grok-code-fast-1": "deepseek/deepseek-chat", + "xiaomi": "xiaomi/mimo-v2.5-pro", } // Models is the full catalog of supported models. var Models = []ModelDef{ - // Smart routing meta-models - {ID: "auto", Name: "Auto (Smart Router - Balanced)", ContextWindow: 1_050_000, MaxOutput: 128_000}, - {ID: "eco", Name: "Eco (Smart Router - Cost Optimized)", ContextWindow: 1_050_000, MaxOutput: 128_000}, - {ID: "premium", Name: "Premium (Smart Router - Best Quality)", ContextWindow: 2_000_000, MaxOutput: 200_000}, - {ID: "free", Name: "Free - Nemotron Ultra 253B", ContextWindow: 131_072, MaxOutput: 16_384, Reasoning: true}, - - // OpenAI — GPT-5.6 Terra is current flagship (upstream v0.12.219). - {ID: "openai/gpt-5.6-terra", Name: "GPT-5.6 Terra", Version: "5.6", InputPrice: 5.0, OutputPrice: 30.0, ContextWindow: 1_050_000, MaxOutput: 128_000, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, - {ID: "openai/gpt-5.6-sol", Name: "GPT-5.6 Sol", Version: "5.6", InputPrice: 5.0, OutputPrice: 30.0, ContextWindow: 1_050_000, MaxOutput: 128_000, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, - {ID: "openai/gpt-5.6-luna", Name: "GPT-5.6 Luna", Version: "5.6", InputPrice: 5.0, OutputPrice: 30.0, ContextWindow: 1_050_000, MaxOutput: 128_000, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, - {ID: "openai/gpt-5.5-pro", Name: "GPT-5.5 Pro", Version: "5.5", InputPrice: 21.0, OutputPrice: 168.0, ContextWindow: 1_050_000, MaxOutput: 128_000, Reasoning: true, ToolCalling: true}, - {ID: "openai/gpt-5.5", Name: "GPT-5.5", Version: "5.5", InputPrice: 5.0, OutputPrice: 30.0, ContextWindow: 1_050_000, MaxOutput: 128_000, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, - {ID: "openai/gpt-5.4", Name: "GPT-5.4", Version: "5.4", InputPrice: 2.5, OutputPrice: 10.0, ContextWindow: 1_050_000, MaxOutput: 128_000, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, - {ID: "openai/gpt-5.4-mini", Name: "GPT-5.4 Mini", Version: "5.4", InputPrice: 0.75, OutputPrice: 4.5, ContextWindow: 400_000, MaxOutput: 128_000, Vision: true, Agentic: true, ToolCalling: true}, - {ID: "openai/gpt-5.4-nano", Name: "GPT-5.4 Nano", Version: "5.4", InputPrice: 0.20, OutputPrice: 1.25, ContextWindow: 1_050_000, MaxOutput: 128_000, ToolCalling: true}, - {ID: "openai/gpt-5.4-pro", Name: "GPT-5.4 Pro", Version: "5.4", InputPrice: 21.0, OutputPrice: 168.0, ContextWindow: 1_050_000, MaxOutput: 128_000, Reasoning: true, ToolCalling: true}, - {ID: "openai/gpt-5.3-codex", Name: "GPT-5.3 Codex", Version: "5.3", InputPrice: 1.75, OutputPrice: 14.0, ContextWindow: 400_000, MaxOutput: 128_000, Reasoning: true, Agentic: true, ToolCalling: true}, - {ID: "openai/chatgpt-instant", Name: "ChatGPT Instant", InputPrice: 0.50, OutputPrice: 2.00, ContextWindow: 128_000, MaxOutput: 16_384, ToolCalling: true}, - {ID: "openai/gpt-4o", Name: "GPT-4o", InputPrice: 2.5, OutputPrice: 10.0, ContextWindow: 128_000, MaxOutput: 16_384, Vision: true, ToolCalling: true}, - {ID: "openai/gpt-4o-mini", Name: "GPT-4o Mini", InputPrice: 0.15, OutputPrice: 0.6, ContextWindow: 128_000, MaxOutput: 16_384, ToolCalling: true}, - {ID: "openai/o3", Name: "o3", InputPrice: 2.0, OutputPrice: 8.0, ContextWindow: 200_000, MaxOutput: 100_000, Reasoning: true, ToolCalling: true}, - {ID: "openai/o4-mini", Name: "o4-mini", InputPrice: 1.10, OutputPrice: 4.40, ContextWindow: 200_000, MaxOutput: 100_000, Reasoning: true, ToolCalling: true}, - - // Anthropic — Opus 5 is the current flagship (upstream v0.12.233). - {ID: "anthropic/claude-opus-5", Name: "Claude Opus 5", Version: "5.0", InputPrice: 5.0, OutputPrice: 25.0, ContextWindow: 1_000_000, MaxOutput: 128_000, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, - {ID: "anthropic/claude-opus-4.8", Name: "Claude Opus 4.8", Version: "4.8", InputPrice: 5.0, OutputPrice: 25.0, ContextWindow: 1_000_000, MaxOutput: 128_000, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, - {ID: "anthropic/claude-opus-4.7", Name: "Claude Opus 4.7", Version: "4.7", InputPrice: 5.0, OutputPrice: 25.0, ContextWindow: 1_000_000, MaxOutput: 128_000, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, - {ID: "anthropic/claude-opus-4.6", Name: "Claude Opus 4.6", Version: "4.6", InputPrice: 5.0, OutputPrice: 25.0, ContextWindow: 1_000_000, MaxOutput: 128_000, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, - {ID: "anthropic/claude-sonnet-5", Name: "Claude Sonnet 5", Version: "5.0", InputPrice: 3.0, OutputPrice: 15.0, ContextWindow: 200_000, MaxOutput: 64_000, Vision: true, Agentic: true, ToolCalling: true}, - {ID: "anthropic/claude-fable-5", Name: "Claude Fable 5", Version: "5.0", InputPrice: 3.0, OutputPrice: 15.0, ContextWindow: 200_000, MaxOutput: 64_000, Vision: true, Agentic: true, ToolCalling: true}, - {ID: "anthropic/claude-sonnet-4.6", Name: "Claude Sonnet 4.6", Version: "4.6", InputPrice: 3.0, OutputPrice: 15.0, ContextWindow: 200_000, MaxOutput: 64_000, Vision: true, Agentic: true, ToolCalling: true}, - {ID: "anthropic/claude-haiku-4.5", Name: "Claude Haiku 4.5", Version: "4.5", InputPrice: 0.80, OutputPrice: 4.0, ContextWindow: 200_000, MaxOutput: 8192, ToolCalling: true}, - - // Google - {ID: "google/gemini-3.5-flash", Name: "Gemini 3.5 Flash", Version: "3.5", InputPrice: 0.15, OutputPrice: 0.60, ContextWindow: 1_050_000, MaxOutput: 65_536, Vision: true, ToolCalling: true}, - {ID: "google/gemini-3.1-pro", Name: "Gemini 3.1 Pro", Version: "3.1", InputPrice: 1.25, OutputPrice: 10.0, ContextWindow: 2_000_000, MaxOutput: 65_536, Vision: true, ToolCalling: true}, - {ID: "google/gemini-3.1-flash-lite", Name: "Gemini 3.1 Flash Lite", Version: "3.1", InputPrice: 0.25, OutputPrice: 1.50, ContextWindow: 1_050_000, MaxOutput: 65_536, ToolCalling: true}, - {ID: "google/gemini-3-pro-preview", Name: "Gemini 3 Pro Preview", Version: "3.0", InputPrice: 1.25, OutputPrice: 10.0, ContextWindow: 1_050_000, MaxOutput: 65_536, Vision: true, ToolCalling: true, Deprecated: true, FallbackModel: "google/gemini-3.1-pro"}, - {ID: "google/gemini-3-flash-preview", Name: "Gemini 3 Flash Preview", Version: "3.0", InputPrice: 0.15, OutputPrice: 0.60, ContextWindow: 1_050_000, MaxOutput: 65_536, ToolCalling: true}, - {ID: "google/gemini-2.5-pro", Name: "Gemini 2.5 Pro", Version: "2.5", InputPrice: 1.25, OutputPrice: 10.0, ContextWindow: 1_050_000, MaxOutput: 65_536, Reasoning: true, Vision: true, ToolCalling: true}, - {ID: "google/gemini-2.5-flash", Name: "Gemini 2.5 Flash", Version: "2.5", InputPrice: 0.15, OutputPrice: 0.60, ContextWindow: 1_050_000, MaxOutput: 65_536, Vision: true, ToolCalling: true}, - {ID: "google/gemini-2.5-flash-lite", Name: "Gemini 2.5 Flash Lite", Version: "2.5", InputPrice: 0.10, OutputPrice: 0.40, ContextWindow: 1_050_000, MaxOutput: 65_536, ToolCalling: true}, - - // DeepSeek - {ID: "deepseek/deepseek-v4-pro", Name: "DeepSeek V4 Pro", Version: "v4-pro", InputPrice: 0.55, OutputPrice: 2.19, ContextWindow: 1_000_000, MaxOutput: 65_536, Reasoning: true, ToolCalling: true}, - {ID: "deepseek/deepseek-chat", Name: "DeepSeek V3", InputPrice: 0.27, OutputPrice: 1.10, ContextWindow: 128_000, MaxOutput: 16_384, ToolCalling: true}, - {ID: "deepseek/deepseek-reasoner", Name: "DeepSeek R1", InputPrice: 0.55, OutputPrice: 2.19, ContextWindow: 128_000, MaxOutput: 16_384, Reasoning: true}, - - // Kimi K3 & K2.7 - Moonshot flagship (upstream v0.12.229/230) - {ID: "moonshot/kimi-k3", Name: "Kimi K3", Version: "k3", InputPrice: 1.20, OutputPrice: 5.0, ContextWindow: 262_144, MaxOutput: 65_536, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, - {ID: "moonshot/kimi-k2.7", Name: "Kimi K2.7", Version: "k2.7", InputPrice: 0.95, OutputPrice: 4.0, ContextWindow: 262_144, MaxOutput: 65_536, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, - {ID: "moonshot/kimi-k2.6", Name: "Kimi K2.6", Version: "k2.6", InputPrice: 0.95, OutputPrice: 4.0, ContextWindow: 262_144, MaxOutput: 65_536, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, - {ID: "moonshot/kimi-k2.5", Name: "Kimi K2.5", Version: "k2.5", InputPrice: 0.60, OutputPrice: 3.0, ContextWindow: 262_144, MaxOutput: 16_384, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, - {ID: "nvidia/kimi-k2.5", Name: "Kimi K2.5 (NVIDIA, retired)", Version: "k2.5", InputPrice: 0.60, OutputPrice: 3.0, ContextWindow: 262_144, MaxOutput: 8192, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true, Deprecated: true, FallbackModel: "moonshot/kimi-k2.5"}, - - // Qwen - {ID: "qwen/qwen3.7-max", Name: "Qwen 3.7 Max", Version: "3.7", InputPrice: 1.50, OutputPrice: 6.0, ContextWindow: 1_000_000, MaxOutput: 65_536, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, - - // xAI — Grok 4.5 & 4.3 (upstream v0.12.201, v0.12.225) - {ID: "xai/grok-4.5", Name: "Grok 4.5", Version: "4.5", InputPrice: 2.0, OutputPrice: 6.0, ContextWindow: 2_000_000, MaxOutput: 16_384, Reasoning: true, ToolCalling: true}, - {ID: "xai/grok-4.3", Name: "Grok 4.3", Version: "4.3", InputPrice: 2.0, OutputPrice: 6.0, ContextWindow: 2_000_000, MaxOutput: 16_384, Reasoning: true, ToolCalling: true}, - {ID: "xai/grok-4-0709", Name: "Grok 4", InputPrice: 3.0, OutputPrice: 15.0, ContextWindow: 200_000, MaxOutput: 100_000, Vision: true, ToolCalling: true}, - {ID: "xai/grok-4-fast-reasoning", Name: "Grok 4 Fast Reasoning", InputPrice: 0.20, OutputPrice: 0.50, ContextWindow: 131_072, MaxOutput: 32_768, Reasoning: true, ToolCalling: true}, - {ID: "xai/grok-4-1-fast-reasoning", Name: "Grok 4.1 Fast Reasoning", InputPrice: 0.20, OutputPrice: 0.50, ContextWindow: 131_072, MaxOutput: 32_768, Reasoning: true, ToolCalling: true}, - {ID: "xai/grok-4-fast-non-reasoning", Name: "Grok 4 Fast", InputPrice: 0.20, OutputPrice: 0.50, ContextWindow: 131_072, MaxOutput: 32_768, ToolCalling: true}, - {ID: "xai/grok-4-1-fast-non-reasoning", Name: "Grok 4.1 Fast", InputPrice: 0.20, OutputPrice: 0.50, ContextWindow: 131_072, MaxOutput: 32_768, ToolCalling: true}, - {ID: "xai/grok-3", Name: "Grok 3", InputPrice: 3.0, OutputPrice: 15.0, ContextWindow: 131_072, MaxOutput: 32_768, ToolCalling: true}, - {ID: "xai/grok-3-mini", Name: "Grok 3 Mini", InputPrice: 0.30, OutputPrice: 0.50, ContextWindow: 131_072, MaxOutput: 32_768, Reasoning: true, ToolCalling: true}, - - // Grok 4.20 family (upstream v0.12.155): 2M context, multi-agent variant. - {ID: "xai/grok-4.20-reasoning", Name: "Grok 4.20 Reasoning", Version: "4.20", InputPrice: 2.0, OutputPrice: 6.0, ContextWindow: 2_000_000, MaxOutput: 16_384, Reasoning: true, ToolCalling: true}, - {ID: "xai/grok-4.20-non-reasoning", Name: "Grok 4.20", Version: "4.20", InputPrice: 2.0, OutputPrice: 6.0, ContextWindow: 2_000_000, MaxOutput: 16_384, ToolCalling: true}, - {ID: "xai/grok-4.20-multi-agent", Name: "Grok 4.20 Multi-Agent", Version: "4.20", InputPrice: 2.0, OutputPrice: 6.0, ContextWindow: 2_000_000, MaxOutput: 16_384, Reasoning: true, ToolCalling: true}, - - // Free (NVIDIA) - {ID: "free/gpt-oss-120b", Name: "GPT-OSS 120B (Free)", ContextWindow: 131_072, MaxOutput: 16_384}, - {ID: "free/gpt-oss-20b", Name: "GPT-OSS 20B (Free)", ContextWindow: 131_072, MaxOutput: 16_384}, - {ID: "free/qwen3-next-80b-a3b-thinking", Name: "Qwen3 Next 80B A3B Thinking (Free)", ContextWindow: 131_072, MaxOutput: 16_384, Reasoning: true}, - {ID: "free/mistral-small-4-119b", Name: "Mistral Small 4 119B (retired)", ContextWindow: 131_072, MaxOutput: 16_384, Deprecated: true, FallbackModel: "free/llama-4-maverick"}, - {ID: "free/deepseek-v4-flash", Name: "DeepSeek V4 Flash (retired)", Version: "v4-flash", ContextWindow: 1_000_000, MaxOutput: 16_384, Reasoning: true, Deprecated: true, FallbackModel: "free/llama-4-maverick"}, - {ID: "free/deepseek-v3.2", Name: "DeepSeek V3.2 (retired)", ContextWindow: 131_072, MaxOutput: 16_384, Deprecated: true, FallbackModel: "free/llama-4-maverick"}, - {ID: "free/deepseek-v4-pro", Name: "DeepSeek V4 Pro (retired)", ContextWindow: 1_000_000, MaxOutput: 16_384, Reasoning: true, Deprecated: true, FallbackModel: "free/llama-4-maverick"}, - {ID: "free/qwen3-coder-480b", Name: "Qwen3 Coder 480B (retired)", ContextWindow: 131_072, MaxOutput: 16_384, Deprecated: true, FallbackModel: "free/llama-4-maverick"}, - {ID: "free/glm-4.7", Name: "GLM 4.7 (Free)", ContextWindow: 131_072, MaxOutput: 16_384}, - {ID: "free/llama-4-maverick", Name: "Llama 4 Maverick (Free)", ContextWindow: 131_072, MaxOutput: 16_384}, - {ID: "free/seed-oss-36b", Name: "Seed OSS 36B (retired)", ContextWindow: 131_072, MaxOutput: 16_384, Deprecated: true, FallbackModel: "free/gpt-oss-120b"}, - // Retired free models (upstream v0.12.160/v0.12.245): kept for catalog back-compat, routed via FallbackModel. - {ID: "free/nemotron-ultra-253b", Name: "Nemotron Ultra 253B (retired)", ContextWindow: 131_072, MaxOutput: 16_384, Reasoning: true, Deprecated: true, FallbackModel: "free/qwen3-next-80b-a3b-thinking"}, - {ID: "free/nemotron-super-49b", Name: "Nemotron Super 49B (retired)", ContextWindow: 131_072, MaxOutput: 16_384, Deprecated: true, FallbackModel: "free/qwen3-next-80b-a3b-thinking"}, - {ID: "free/nemotron-3-super-120b", Name: "Nemotron 3 Super 120B (retired)", ContextWindow: 131_072, MaxOutput: 16_384, Deprecated: true, FallbackModel: "free/qwen3-next-80b-a3b-thinking"}, - {ID: "free/mistral-large-3-675b", Name: "Mistral Large 3 675B (retired)", ContextWindow: 131_072, MaxOutput: 16_384, Deprecated: true, FallbackModel: "free/llama-4-maverick"}, - {ID: "free/devstral-2-123b", Name: "Devstral 2 123B (retired)", ContextWindow: 131_072, MaxOutput: 16_384, Deprecated: true, FallbackModel: "free/llama-4-maverick"}, - - // Z.AI — GLM-5.2 is current flagship (upstream v0.12.211) - {ID: "zai/glm-5.2", Name: "GLM-5.2", Version: "5.2", InputPrice: 1.6, OutputPrice: 5.0, ContextWindow: 200_000, MaxOutput: 128_000, ToolCalling: true}, - {ID: "zai/glm-5.1", Name: "GLM-5.1", Version: "5.1", InputPrice: 1.4, OutputPrice: 4.4, ContextWindow: 200_000, MaxOutput: 128_000, ToolCalling: true}, - {ID: "zai/glm-5", Name: "GLM-5", InputPrice: 0.60, OutputPrice: 1.92, ContextWindow: 131_072, MaxOutput: 16_384, ToolCalling: true}, - {ID: "zai/glm-5-turbo", Name: "GLM-5 Turbo", InputPrice: 1.20, OutputPrice: 4.00, ContextWindow: 131_072, MaxOutput: 16_384, ToolCalling: true}, - - // MiniMax — M3 is current flagship with Vision (upstream v0.12.200, v0.12.236) - {ID: "minimax/minimax-m3", Name: "MiniMax M3", Version: "m3", InputPrice: 0.50, OutputPrice: 2.00, ContextWindow: 1_050_000, MaxOutput: 128_000, Vision: true, ToolCalling: true}, - {ID: "minimax/minimax-m2.7", Name: "MiniMax M2.7", InputPrice: 1.0, OutputPrice: 5.0, ContextWindow: 1_050_000, MaxOutput: 128_000, ToolCalling: true}, + // Catalog metadata: ClawRouter 05de1e0 (v0.12.278 source). + // Retired free IDs keep compatibility metadata and redirect to live successors. + // Free models deliberately do not claim working vision or structured tools. + + // Routing profiles + {ID: "auto", Name: "Auto (Smart Router - Balanced)", InputPrice: 0, OutputPrice: 0, ContextWindow: 1050000, MaxOutput: 128000}, + {ID: "free", Name: "Free - Nemotron 3.5 Lightning", InputPrice: 0, OutputPrice: 0, ContextWindow: 1000000, MaxOutput: 16384, Reasoning: true}, + {ID: "eco", Name: "Eco (Smart Router - Cost Optimized)", InputPrice: 0, OutputPrice: 0, ContextWindow: 1050000, MaxOutput: 128000}, + {ID: "premium", Name: "Premium (Smart Router - Best Quality)", InputPrice: 0, OutputPrice: 0, ContextWindow: 2000000, MaxOutput: 200000}, + + // openai + {ID: "openai/gpt-5.2", Name: "GPT-5.2", Version: "5.2", InputPrice: 1.75, OutputPrice: 14.0, ContextWindow: 400000, MaxOutput: 128000, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, + {ID: "openai/gpt-5-mini", Name: "GPT-5 Mini", Version: "5.0", InputPrice: 0.25, OutputPrice: 2.0, ContextWindow: 200000, MaxOutput: 128000, ToolCalling: true}, + {ID: "openai/gpt-5-nano", Name: "GPT-5 Nano", Version: "5.0", InputPrice: 0.05, OutputPrice: 0.4, ContextWindow: 128000, MaxOutput: 128000, ToolCalling: true, Deprecated: true, FallbackModel: "openai/gpt-5.4-nano"}, + {ID: "openai/gpt-5.2-pro", Name: "GPT-5.2 Pro", Version: "5.2", InputPrice: 21.0, OutputPrice: 168.0, ContextWindow: 400000, MaxOutput: 128000, Reasoning: true, Vision: true, ToolCalling: true}, + {ID: "openai/gpt-5.6-sol", Name: "GPT-5.6 Sol", Version: "5.6", InputPrice: 4.0, OutputPrice: 20.0, ContextWindow: 1050000, MaxOutput: 128000, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, + {ID: "openai/gpt-5.6-terra", Name: "GPT-5.6 Terra", Version: "5.6", InputPrice: 2.0, OutputPrice: 12.0, ContextWindow: 1050000, MaxOutput: 128000, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, + {ID: "openai/gpt-5.6-luna", Name: "GPT-5.6 Luna", Version: "5.6", InputPrice: 0.2, OutputPrice: 1.2, ContextWindow: 1050000, MaxOutput: 128000, Vision: true, Agentic: true, ToolCalling: true}, + {ID: "openai/gpt-5.6-sol-pro", Name: "GPT-5.6 Sol Pro", Version: "5.6", InputPrice: 4.0, OutputPrice: 20.0, ContextWindow: 1050000, MaxOutput: 128000, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, + {ID: "openai/gpt-5.6-terra-pro", Name: "GPT-5.6 Terra Pro", Version: "5.6", InputPrice: 2.0, OutputPrice: 12.0, ContextWindow: 1050000, MaxOutput: 128000, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, + {ID: "openai/gpt-5.6-luna-pro", Name: "GPT-5.6 Luna Pro", Version: "5.6", InputPrice: 0.2, OutputPrice: 1.2, ContextWindow: 1050000, MaxOutput: 128000, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, + {ID: "openai/gpt-5.5", Name: "GPT-5.5", Version: "5.5", InputPrice: 5.0, OutputPrice: 30.0, ContextWindow: 1050000, MaxOutput: 128000, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, + {ID: "openai/gpt-5.5-pro", Name: "GPT-5.5 Pro", Version: "5.5", InputPrice: 30.0, OutputPrice: 180.0, ContextWindow: 1050000, MaxOutput: 128000, Reasoning: true, Vision: true, ToolCalling: true}, + {ID: "openai/chat-latest", Name: "ChatGPT Instant (GPT-5.5)", Version: "5.5", InputPrice: 5.0, OutputPrice: 30.0, ContextWindow: 128000, MaxOutput: 128000, Vision: true, ToolCalling: true}, + {ID: "openai/gpt-5.4", Name: "GPT-5.4", Version: "5.4", InputPrice: 2.5, OutputPrice: 15.0, ContextWindow: 1050000, MaxOutput: 128000, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, + {ID: "openai/gpt-5.4-mini", Name: "GPT-5.4 Mini", Version: "5.4", InputPrice: 0.75, OutputPrice: 4.5, ContextWindow: 400000, MaxOutput: 128000, Vision: true, Agentic: true, ToolCalling: true}, + {ID: "openai/gpt-5.4-pro", Name: "GPT-5.4 Pro", Version: "5.4", InputPrice: 30.0, OutputPrice: 180.0, ContextWindow: 1050000, MaxOutput: 128000, Reasoning: true, Vision: true, ToolCalling: true}, + {ID: "openai/gpt-5.4-nano", Name: "GPT-5.4 Nano", Version: "5.4", InputPrice: 0.2, OutputPrice: 1.25, ContextWindow: 1050000, MaxOutput: 128000, ToolCalling: true}, + {ID: "openai/gpt-5.3", Name: "GPT-5.3", Version: "5.3", InputPrice: 1.75, OutputPrice: 14.0, ContextWindow: 128000, MaxOutput: 128000, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, + {ID: "openai/gpt-5.3-codex", Name: "GPT-5.3 Codex", Version: "5.3", InputPrice: 1.75, OutputPrice: 14.0, ContextWindow: 400000, MaxOutput: 128000, Agentic: true, ToolCalling: true}, + {ID: "openai/gpt-4.1", Name: "GPT-4.1", Version: "4.1", InputPrice: 2.0, OutputPrice: 8.0, ContextWindow: 128000, MaxOutput: 32768, Vision: true, ToolCalling: true}, + {ID: "openai/gpt-4.1-mini", Name: "GPT-4.1 Mini", Version: "4.1", InputPrice: 0.4, OutputPrice: 1.6, ContextWindow: 128000, MaxOutput: 32768, ToolCalling: true}, + {ID: "openai/gpt-4.1-nano", Name: "GPT-4.1 Nano", Version: "4.1", InputPrice: 0.1, OutputPrice: 0.4, ContextWindow: 128000, MaxOutput: 32768, ToolCalling: true}, + {ID: "openai/gpt-4o", Name: "GPT-4o", Version: "4o", InputPrice: 2.5, OutputPrice: 10.0, ContextWindow: 128000, MaxOutput: 16384, Vision: true, Agentic: true, ToolCalling: true}, + {ID: "openai/gpt-4o-mini", Name: "GPT-4o Mini", Version: "4o-mini", InputPrice: 0.15, OutputPrice: 0.6, ContextWindow: 128000, MaxOutput: 16384, ToolCalling: true}, + {ID: "openai/o1", Name: "o1", Version: "1", InputPrice: 15.0, OutputPrice: 60.0, ContextWindow: 200000, MaxOutput: 100000, Reasoning: true, ToolCalling: true}, + {ID: "openai/o1-mini", Name: "o1-mini", Version: "1-mini", InputPrice: 1.1, OutputPrice: 4.4, ContextWindow: 128000, MaxOutput: 65536, Reasoning: true, ToolCalling: true, Deprecated: true, FallbackModel: "openai/o4-mini"}, + {ID: "openai/o3", Name: "o3", Version: "3", InputPrice: 2.0, OutputPrice: 8.0, ContextWindow: 200000, MaxOutput: 100000, Reasoning: true, ToolCalling: true}, + {ID: "openai/o3-mini", Name: "o3-mini", Version: "3-mini", InputPrice: 1.1, OutputPrice: 4.4, ContextWindow: 128000, MaxOutput: 100000, Reasoning: true, ToolCalling: true}, + {ID: "openai/o4-mini", Name: "o4-mini", Version: "4-mini", InputPrice: 1.1, OutputPrice: 4.4, ContextWindow: 128000, MaxOutput: 100000, Reasoning: true, ToolCalling: true}, + + // anthropic + {ID: "anthropic/claude-haiku-4.5", Name: "Claude Haiku 4.5", Version: "4.5", InputPrice: 1.0, OutputPrice: 5.0, ContextWindow: 200000, MaxOutput: 64000, Vision: true, Agentic: true, ToolCalling: true}, + {ID: "anthropic/claude-sonnet-4.5", Name: "Claude Sonnet 4.5", Version: "4.5", InputPrice: 3.0, OutputPrice: 15.0, ContextWindow: 200000, MaxOutput: 64000, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, + {ID: "anthropic/claude-sonnet-4.6", Name: "Claude Sonnet 4.6", Version: "4.6", InputPrice: 3.0, OutputPrice: 15.0, ContextWindow: 1000000, MaxOutput: 128000, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, + {ID: "anthropic/claude-sonnet-5", Name: "Claude Sonnet 5", Version: "5", InputPrice: 3.0, OutputPrice: 15.0, ContextWindow: 1000000, MaxOutput: 128000, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, + {ID: "anthropic/claude-opus-4.5", Name: "Claude Opus 4.5", Version: "4.5", InputPrice: 5.0, OutputPrice: 25.0, ContextWindow: 200000, MaxOutput: 64000, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, + {ID: "anthropic/claude-opus-4.6", Name: "Claude Opus 4.6", Version: "4.6", InputPrice: 5.0, OutputPrice: 25.0, ContextWindow: 1000000, MaxOutput: 128000, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, + {ID: "anthropic/claude-fable-5", Name: "Claude Fable 5", Version: "5", InputPrice: 10.0, OutputPrice: 50.0, ContextWindow: 1000000, MaxOutput: 128000, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, + {ID: "anthropic/claude-opus-4.7", Name: "Claude Opus 4.7", Version: "4.7", InputPrice: 5.0, OutputPrice: 25.0, ContextWindow: 1000000, MaxOutput: 128000, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, + {ID: "anthropic/claude-opus-4.8", Name: "Claude Opus 4.8", Version: "4.8", InputPrice: 5.0, OutputPrice: 25.0, ContextWindow: 1000000, MaxOutput: 128000, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, + {ID: "anthropic/claude-opus-5", Name: "Claude Opus 5", Version: "5", InputPrice: 5.0, OutputPrice: 25.0, ContextWindow: 1000000, MaxOutput: 128000, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, + + // google + {ID: "google/gemini-3.1-pro", Name: "Gemini 3.1 Pro", Version: "3.1", InputPrice: 2.0, OutputPrice: 12.0, ContextWindow: 1048576, MaxOutput: 65536, Reasoning: true, Vision: true, ToolCalling: true}, + {ID: "google/gemini-3-pro-preview", Name: "Gemini 3 Pro Preview", Version: "3.0", InputPrice: 2.0, OutputPrice: 12.0, ContextWindow: 1048576, MaxOutput: 65536, Reasoning: true, Vision: true, ToolCalling: true, Deprecated: true, FallbackModel: "google/gemini-3.1-pro"}, + // Promotional $0.75/$3.75 until 2027-01-01; upstream list rate then returns to $1.50/$7.50. + {ID: "google/gemini-3.8-flash", Name: "Gemini 3.8 Flash", Version: "3.8", InputPrice: 0.75, OutputPrice: 3.75, ContextWindow: 1048576, MaxOutput: 65536, Reasoning: true, Vision: true, ToolCalling: true}, + // Promotional $0.75/$3.75 until 2027-01-01; upstream list rate then returns to $1.50/$7.50. + {ID: "google/gemini-3.6-flash", Name: "Gemini 3.6 Flash", Version: "3.6", InputPrice: 0.75, OutputPrice: 3.75, ContextWindow: 1048576, MaxOutput: 65536, Reasoning: true, Vision: true, ToolCalling: true}, + {ID: "google/gemini-3.5-flash", Name: "Gemini 3.5 Flash", Version: "3.5", InputPrice: 1.5, OutputPrice: 9.0, ContextWindow: 1048576, MaxOutput: 65536, Reasoning: true, Vision: true, ToolCalling: true}, + {ID: "google/gemini-3.5-flash-lite", Name: "Gemini 3.5 Flash Lite", Version: "3.5", InputPrice: 0.3, OutputPrice: 2.5, ContextWindow: 1048576, MaxOutput: 65536, Reasoning: true, ToolCalling: true}, + {ID: "google/gemini-3-flash-preview", Name: "Gemini 3 Flash Preview", Version: "3.0", InputPrice: 0.5, OutputPrice: 3.0, ContextWindow: 1048576, MaxOutput: 65536, Reasoning: true, Vision: true}, + {ID: "google/gemini-2.5-pro", Name: "Gemini 2.5 Pro", Version: "2.5", InputPrice: 1.25, OutputPrice: 10.0, ContextWindow: 1048576, MaxOutput: 65536, Reasoning: true, Vision: true, ToolCalling: true}, + {ID: "google/gemini-2.5-flash", Name: "Gemini 2.5 Flash", Version: "2.5", InputPrice: 0.3, OutputPrice: 2.5, ContextWindow: 1048576, MaxOutput: 65536, Vision: true, ToolCalling: true}, + {ID: "google/gemini-2.5-flash-lite", Name: "Gemini 2.5 Flash Lite", Version: "2.5", InputPrice: 0.1, OutputPrice: 0.4, ContextWindow: 1048576, MaxOutput: 65536, Vision: true, ToolCalling: true}, + {ID: "google/gemini-3.1-flash-lite", Name: "Gemini 3.1 Flash Lite", Version: "3.1", InputPrice: 0.25, OutputPrice: 1.5, ContextWindow: 1048576, MaxOutput: 65536, ToolCalling: true}, + + // deepseek + {ID: "deepseek/deepseek-chat", Name: "DeepSeek V4 Flash Chat", Version: "4-flash", InputPrice: 0.14, OutputPrice: 0.28, ContextWindow: 1048576, MaxOutput: 65536, ToolCalling: true}, + {ID: "deepseek/deepseek-reasoner", Name: "DeepSeek V4 Flash Reasoner", Version: "4-flash", InputPrice: 0.14, OutputPrice: 0.28, ContextWindow: 1048576, MaxOutput: 65536, Reasoning: true, ToolCalling: true}, + {ID: "deepseek/deepseek-v4-pro", Name: "DeepSeek V4 Pro", Version: "4-pro", InputPrice: 1.32, OutputPrice: 3.96, ContextWindow: 1048576, MaxOutput: 65536, Reasoning: true, Agentic: true, ToolCalling: true}, + + // moonshot + {ID: "moonshot/kimi-k3", Name: "Kimi K3", Version: "k3", InputPrice: 3.0, OutputPrice: 15.0, ContextWindow: 1048576, MaxOutput: 65536, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, + {ID: "moonshot/kimi-k2.7", Name: "Kimi K2.7", Version: "k2.7", InputPrice: 0.95, OutputPrice: 4.0, ContextWindow: 262144, MaxOutput: 65536, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, + + // qwen + {ID: "qwen/qwen3.7-max", Name: "Qwen3.7 Max", Version: "3.7-max", InputPrice: 1.475, OutputPrice: 4.425, ContextWindow: 1000000, MaxOutput: 65536, Reasoning: true, Agentic: true, ToolCalling: true}, + {ID: "qwen/qwen3.7-plus", Name: "Qwen3.7 Plus", Version: "3.7-plus", InputPrice: 0.32, OutputPrice: 1.28, ContextWindow: 1000000, MaxOutput: 131072, Reasoning: true, Agentic: true, ToolCalling: true}, + {ID: "qwen/qwen3.7-flash", Name: "Qwen3.7 Flash", Version: "3.7-flash", InputPrice: 0.03, OutputPrice: 0.13, ContextWindow: 1000000, MaxOutput: 65536, Reasoning: true, ToolCalling: true}, + + // moonshot + {ID: "moonshot/kimi-k2.6", Name: "Kimi K2.6", Version: "k2.6", InputPrice: 0.95, OutputPrice: 4.0, ContextWindow: 262144, MaxOutput: 65536, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, + {ID: "moonshot/kimi-k2.5", Name: "Kimi K2.5", Version: "k2.5", InputPrice: 0.6, OutputPrice: 3.0, ContextWindow: 262144, MaxOutput: 65536, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, + + // nvidia + {ID: "nvidia/kimi-k2.5", Name: "Kimi K2.5 (NVIDIA, retired)", Version: "k2.5", InputPrice: 0.6, OutputPrice: 3.0, ContextWindow: 262144, MaxOutput: 16384, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true, Deprecated: true, FallbackModel: "moonshot/kimi-k2.5"}, + + // xai + {ID: "xai/grok-3", Name: "Grok 3", Version: "3", InputPrice: 3.0, OutputPrice: 15.0, ContextWindow: 131072, MaxOutput: 16384, Reasoning: true, ToolCalling: true}, + {ID: "xai/grok-3-mini", Name: "Grok 3 Mini", Version: "3-mini", InputPrice: 0.3, OutputPrice: 0.5, ContextWindow: 131072, MaxOutput: 16384, ToolCalling: true}, + {ID: "xai/grok-4-fast-reasoning", Name: "Grok 4 Fast Reasoning", Version: "4", InputPrice: 0.2, OutputPrice: 0.5, ContextWindow: 2000000, MaxOutput: 16384, Reasoning: true, ToolCalling: true}, + {ID: "xai/grok-4-fast-non-reasoning", Name: "Grok 4 Fast", Version: "4", InputPrice: 0.2, OutputPrice: 0.5, ContextWindow: 2000000, MaxOutput: 16384, ToolCalling: true}, + {ID: "xai/grok-4-1-fast-reasoning", Name: "Grok 4.1 Fast Reasoning", Version: "4.1", InputPrice: 0.2, OutputPrice: 0.5, ContextWindow: 2000000, MaxOutput: 16384, Reasoning: true, ToolCalling: true}, + {ID: "xai/grok-4-1-fast-non-reasoning", Name: "Grok 4.1 Fast", Version: "4.1", InputPrice: 0.2, OutputPrice: 0.5, ContextWindow: 2000000, MaxOutput: 16384, ToolCalling: true}, + {ID: "xai/grok-4-0709", Name: "Grok 4 (0709)", Version: "4-0709", InputPrice: 3.0, OutputPrice: 15.0, ContextWindow: 256000, MaxOutput: 16384, Reasoning: true, ToolCalling: true}, + {ID: "xai/grok-2-vision", Name: "Grok 2 Vision", Version: "2", InputPrice: 2.0, OutputPrice: 10.0, ContextWindow: 32768, MaxOutput: 16384, Vision: true, ToolCalling: true}, + {ID: "xai/grok-4.20-reasoning", Name: "Grok 4.20 Reasoning", Version: "4.20", InputPrice: 2.0, OutputPrice: 6.0, ContextWindow: 2000000, MaxOutput: 16384, Reasoning: true, ToolCalling: true}, + {ID: "xai/grok-4.20-non-reasoning", Name: "Grok 4.20", Version: "4.20", InputPrice: 2.0, OutputPrice: 6.0, ContextWindow: 2000000, MaxOutput: 16384, ToolCalling: true}, + {ID: "xai/grok-4.20-multi-agent", Name: "Grok 4.20 Multi-Agent", Version: "4.20", InputPrice: 2.0, OutputPrice: 6.0, ContextWindow: 2000000, MaxOutput: 16384, Reasoning: true, ToolCalling: true}, + {ID: "xai/grok-4.5", Name: "Grok 4.5", Version: "4.5", InputPrice: 2.0, OutputPrice: 6.0, ContextWindow: 500000, MaxOutput: 16384, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, + {ID: "xai/grok-4.3", Name: "Grok 4.3", Version: "4.3", InputPrice: 1.25, OutputPrice: 2.5, ContextWindow: 1000000, MaxOutput: 16384, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, + {ID: "xai/grok-build-0.1", Name: "Grok Build 0.1", Version: "0.1", InputPrice: 1.0, OutputPrice: 2.0, ContextWindow: 256000, MaxOutput: 16384, Agentic: true, ToolCalling: true}, + + // tencent + {ID: "tencent/hy3", Name: "Tencent Hy3", Version: "hy3", InputPrice: 0.132, OutputPrice: 0.528, ContextWindow: 262144, MaxOutput: 128000, Reasoning: true, ToolCalling: true}, + + // xiaomi + {ID: "xiaomi/mimo-v2.5-pro", Name: "Xiaomi MiMo-V2.5 Pro", Version: "v2.5-pro", InputPrice: 0.435, OutputPrice: 0.87, ContextWindow: 1048576, MaxOutput: 131072, Reasoning: true, ToolCalling: true}, + + // minimax + {ID: "minimax/minimax-m3", Name: "MiniMax M3", Version: "m3", InputPrice: 0.3, OutputPrice: 1.2, ContextWindow: 1048576, MaxOutput: 65536, Reasoning: true, Vision: true, Agentic: true, ToolCalling: true}, + {ID: "minimax/minimax-m2.7", Name: "MiniMax M2.7", Version: "m2.7", InputPrice: 0.3, OutputPrice: 1.2, ContextWindow: 204800, MaxOutput: 16384, Reasoning: true, Agentic: true, ToolCalling: true}, + {ID: "minimax/minimax-m2.5", Name: "MiniMax M2.5", Version: "m2.5", InputPrice: 0.3, OutputPrice: 1.2, ContextWindow: 204800, MaxOutput: 16384, Reasoning: true, Agentic: true, ToolCalling: true, Deprecated: true, FallbackModel: "minimax/minimax-m2.7"}, + + // free + {ID: "free/gpt-oss-120b", Name: "[Free] GPT-OSS 120B", Version: "120b", InputPrice: 0, OutputPrice: 0, ContextWindow: 128000, MaxOutput: 16384}, + {ID: "free/gpt-oss-20b", Name: "[Free] GPT-OSS 20B", Version: "20b", InputPrice: 0, OutputPrice: 0, ContextWindow: 128000, MaxOutput: 16384}, + {ID: "free/deepseek-v4-flash", Name: "[Free] DeepSeek V4 Flash", Version: "v4-flash", InputPrice: 0, OutputPrice: 0, ContextWindow: 1000000, MaxOutput: 16384, Reasoning: true, Deprecated: true, FallbackModel: "free/nemotron-3.5-lightning"}, + {ID: "free/qwen3-coder-480b", Name: "[Free] Qwen3 Coder 480B", Version: "480b", InputPrice: 0, OutputPrice: 0, ContextWindow: 131072, MaxOutput: 16384, Deprecated: true, FallbackModel: "free/nemotron-3.5-lightning"}, + {ID: "free/glm-4.7", Name: "[Free] GLM-4.7", Version: "4.7", InputPrice: 0, OutputPrice: 0, ContextWindow: 131072, MaxOutput: 16384, Reasoning: true, Deprecated: true, FallbackModel: "free/nemotron-3.5-lightning"}, + {ID: "free/llama-4-maverick", Name: "[Free] Llama 4 Maverick", Version: "4-maverick", InputPrice: 0, OutputPrice: 0, ContextWindow: 131072, MaxOutput: 16384, Reasoning: true, Deprecated: true, FallbackModel: "free/nemotron-3.5-lightning"}, + {ID: "free/nemotron-3-nano-omni-30b-a3b-reasoning", Name: "[Free] Nemotron 3 Nano Omni", Version: "30b-a3b-omni-reasoning", InputPrice: 0, OutputPrice: 0, ContextWindow: 256000, MaxOutput: 16384, Reasoning: true}, + {ID: "free/mistral-large-3-675b", Name: "[Free] Mistral Large 3 675B", Version: "3-675b", InputPrice: 0, OutputPrice: 0, ContextWindow: 131072, MaxOutput: 16384, Reasoning: true, Deprecated: true, FallbackModel: "free/nemotron-3.5-lightning"}, + {ID: "free/qwen3.5-122b-a10b", Name: "[Free] Qwen3.5 122B", Version: "3.5-122b", InputPrice: 0, OutputPrice: 0, ContextWindow: 131072, MaxOutput: 16384, Reasoning: true, Deprecated: true, FallbackModel: "free/nemotron-3.5-lightning"}, + {ID: "free/qwen3-next-80b-a3b-instruct", Name: "[Free] Qwen3-Next 80B Instruct", Version: "next-80b-a3b", InputPrice: 0, OutputPrice: 0, ContextWindow: 262144, MaxOutput: 16384, Reasoning: true, Deprecated: true, FallbackModel: "free/nemotron-3.5-lightning"}, + {ID: "free/seed-oss-36b", Name: "[Free] Seed-OSS 36B", Version: "oss-36b", InputPrice: 0, OutputPrice: 0, ContextWindow: 131072, MaxOutput: 16384, Deprecated: true, FallbackModel: "free/nemotron-3.5-lightning"}, + {ID: "free/mistral-nemotron", Name: "[Free] Mistral Nemotron", Version: "nemotron", InputPrice: 0, OutputPrice: 0, ContextWindow: 131072, MaxOutput: 16384, Deprecated: true, FallbackModel: "free/nemotron-3.5-lightning"}, + {ID: "free/step-3.7-flash", Name: "[Free] StepFun Step 3.7 Flash", Version: "3.7-flash", InputPrice: 0, OutputPrice: 0, ContextWindow: 131072, MaxOutput: 16384, Reasoning: true, Deprecated: true, FallbackModel: "free/nemotron-3.5-lightning"}, + {ID: "free/nemotron-nano-9b-v2", Name: "[Free] Nemotron Nano 9B v2", Version: "nano-9b-v2", InputPrice: 0, OutputPrice: 0, ContextWindow: 131072, MaxOutput: 16384, Reasoning: true, Deprecated: true, FallbackModel: "free/nemotron-3-nano-30b"}, + {ID: "free/nemotron-nano-12b-v2-vl", Name: "[Free] Nemotron Nano 12B v2 VL", Version: "nano-12b-v2-vl", InputPrice: 0, OutputPrice: 0, ContextWindow: 131072, MaxOutput: 16384, Reasoning: true, Deprecated: true, FallbackModel: "free/nemotron-3-nano-omni-30b-a3b-reasoning"}, + {ID: "free/nemotron-3.5-lightning", Name: "[Free] Nemotron 3.5 Lightning", Version: "3.5-lightning", InputPrice: 0, OutputPrice: 0, ContextWindow: 1000000, MaxOutput: 16384, Reasoning: true}, + {ID: "free/nemotron-3-nano-30b", Name: "[Free] Nemotron 3 Nano 30B", Version: "3-nano-30b", InputPrice: 0, OutputPrice: 0, ContextWindow: 131072, MaxOutput: 16384, Reasoning: true}, + {ID: "free/nemotron-3-ultra-550b", Name: "[Free] Nemotron 3 Ultra 550B", Version: "3-ultra-550b", InputPrice: 0, OutputPrice: 0, ContextWindow: 1000000, MaxOutput: 16384, Reasoning: true}, + {ID: "free/llama-3.2-11b-vision", Name: "[Free] Llama 3.2 11B Vision", Version: "3.2-11b-vision", InputPrice: 0, OutputPrice: 0, ContextWindow: 128000, MaxOutput: 16384}, + {ID: "free/north-mini-code", Name: "[Free] Cohere North Mini Code", Version: "north-mini-code", InputPrice: 0, OutputPrice: 0, ContextWindow: 256000, MaxOutput: 16384, Reasoning: true}, + {ID: "free/laguna-xs-2.1", Name: "[Free] Poolside Laguna XS 2.1", Version: "xs-2.1", InputPrice: 0, OutputPrice: 0, ContextWindow: 131072, MaxOutput: 16384}, + + // qwen + {ID: "qwen/qwen3.8-flash", Name: "Qwen3.8 Flash", Version: "3.8-flash", InputPrice: 0.15, OutputPrice: 0.47, ContextWindow: 1000000, MaxOutput: 131072, Reasoning: true, Vision: true, ToolCalling: true}, + + // deepseek + {ID: "deepseek/deepseek-v4-flash-vision-exp", Name: "DeepSeek V4 Flash Vision", Version: "v4-flash-vision-exp", InputPrice: 0.44, OutputPrice: 1.32, ContextWindow: 1048576, MaxOutput: 65536, Reasoning: true, Vision: true, ToolCalling: true}, + + // xiaomi + {ID: "xiaomi/mimo-v2.5", Name: "Xiaomi MiMo V2.5", Version: "2.5", InputPrice: 0.14, OutputPrice: 0.28, ContextWindow: 1048576, MaxOutput: 131072, Reasoning: true, Vision: true, ToolCalling: true}, + + // zai + {ID: "zai/glm-5.3", Name: "GLM-5.3", Version: "5.3", InputPrice: 1.4, OutputPrice: 4.4, ContextWindow: 1000000, MaxOutput: 131072, Reasoning: true, ToolCalling: true}, + {ID: "zai/glm-5.3-flash", Name: "GLM-5.3 Flash", Version: "5.3-flash", InputPrice: 0.15, OutputPrice: 0.5, ContextWindow: 1000000, MaxOutput: 131072, Reasoning: true, Vision: true, ToolCalling: true}, + {ID: "zai/glm-5.2", Name: "GLM-5.2", Version: "5.2", InputPrice: 1.4, OutputPrice: 4.4, ContextWindow: 1000000, MaxOutput: 131072, Reasoning: true, ToolCalling: true}, + {ID: "zai/glm-5.1", Name: "GLM-5.1", Version: "5.1", InputPrice: 1.4, OutputPrice: 4.4, ContextWindow: 200000, MaxOutput: 128000, Reasoning: true, ToolCalling: true, Promo: &PromoDef{FlatPrice: 0.001, StartDate: "2026-04-01", EndDate: "2026-06-05"}}, + {ID: "zai/glm-5", Name: "GLM-5", Version: "5", InputPrice: 1.0, OutputPrice: 3.2, ContextWindow: 200000, MaxOutput: 128000, Reasoning: true, ToolCalling: true}, + {ID: "zai/glm-5-turbo", Name: "GLM-5 Turbo", Version: "5-turbo", InputPrice: 1.2, OutputPrice: 4.0, ContextWindow: 200000, MaxOutput: 128000, Reasoning: true, ToolCalling: true}, + + // openai + {ID: "openai/chatgpt-instant", Name: "ChatGPT Instant (GPT-5.5)", Version: "5.5", InputPrice: 5.0, OutputPrice: 30.0, ContextWindow: 128000, MaxOutput: 128000, Vision: true, ToolCalling: true, Deprecated: true, FallbackModel: "openai/chat-latest"}, + + // free + {ID: "free/qwen3-next-80b-a3b-thinking", Name: "qwen3-next-80b-a3b-thinking (retired)", ContextWindow: 131072, MaxOutput: 16384, InputPrice: 0, OutputPrice: 0, Deprecated: true, FallbackModel: "free/nemotron-3.5-lightning"}, + {ID: "free/mistral-small-4-119b", Name: "mistral-small-4-119b (retired)", ContextWindow: 131072, MaxOutput: 16384, InputPrice: 0, OutputPrice: 0, Deprecated: true, FallbackModel: "free/nemotron-3.5-lightning"}, + {ID: "free/deepseek-v3.2", Name: "deepseek-v3.2 (retired)", ContextWindow: 131072, MaxOutput: 16384, InputPrice: 0, OutputPrice: 0, Deprecated: true, FallbackModel: "free/nemotron-3.5-lightning"}, + {ID: "free/deepseek-v4-pro", Name: "deepseek-v4-pro (retired)", ContextWindow: 131072, MaxOutput: 16384, InputPrice: 0, OutputPrice: 0, Deprecated: true, FallbackModel: "free/nemotron-3.5-lightning"}, + {ID: "free/nemotron-ultra-253b", Name: "nemotron-ultra-253b (retired)", ContextWindow: 131072, MaxOutput: 16384, InputPrice: 0, OutputPrice: 0, Deprecated: true, FallbackModel: "free/nemotron-3.5-lightning"}, + {ID: "free/nemotron-super-49b", Name: "nemotron-super-49b (retired)", ContextWindow: 131072, MaxOutput: 16384, InputPrice: 0, OutputPrice: 0, Deprecated: true, FallbackModel: "free/nemotron-3.5-lightning"}, + {ID: "free/nemotron-3-super-120b", Name: "nemotron-3-super-120b (retired)", ContextWindow: 131072, MaxOutput: 16384, InputPrice: 0, OutputPrice: 0, Deprecated: true, FallbackModel: "free/nemotron-3.5-lightning"}, + {ID: "free/devstral-2-123b", Name: "devstral-2-123b (retired)", ContextWindow: 131072, MaxOutput: 16384, InputPrice: 0, OutputPrice: 0, Deprecated: true, FallbackModel: "free/nemotron-3.5-lightning"}, } // modelIndex is built on init for fast lookups. @@ -316,6 +492,12 @@ func ResolveModelAlias(model string) string { return withoutPrefix } + // A concrete catalog ID wins over shorthand matching after prefix removal. + // For example, the DOS o1 shorthand targets o3, but openai/o1 is an exact pin. + if _, ok := modelIndex[normalized]; ok { + return normalized + } + // Strip "openai/" prefix for virtual profiles if strings.HasPrefix(normalized, "openai/") { withoutPrefix := normalized[len("openai/"):] diff --git a/proxy/prose.go b/proxy/prose.go new file mode 100644 index 0000000..a857ebe --- /dev/null +++ b/proxy/prose.go @@ -0,0 +1,65 @@ +package proxy + +import ( + "regexp" + "strings" +) + +var thinkingTag = regexp.MustCompile(`(?i)^<\s*(/?)\s*(think(?:ing)?|thought|antthinking|antml:thinking)\b[^>]*>$`) + +// proseFilter tracks split thinking tags across SSE chunks. Tagged reasoning +// stays private while ordinary assistant prose and tool_calls remain visible. +type proseFilter struct { + pending string + hidden bool +} + +func (f *proseFilter) filter(text string, final bool) string { + f.pending += text + var out strings.Builder + for len(f.pending) > 0 { + start := strings.IndexByte(f.pending, '<') + if start < 0 { + if !f.hidden { + out.WriteString(f.pending) + } + f.pending = "" + break + } + if !f.hidden { + out.WriteString(f.pending[:start]) + } + f.pending = f.pending[start:] + end := strings.IndexByte(f.pending, '>') + if end < 0 { + if final { + if !f.hidden { + out.WriteString(f.pending) + } + f.pending = "" + } + break + } + tag := f.pending[:end+1] + f.pending = f.pending[end+1:] + if match := thinkingTag.FindStringSubmatch(tag); match != nil { + f.hidden = match[1] != "/" + } else if strings.HasPrefix(tag, "<|") || strings.HasPrefix(tag, "<｜") { + lower := strings.ToLower(tag) + if strings.Contains(lower, "begin") { + f.hidden = true + } + if strings.Contains(lower, "end") { + f.hidden = false + } + } else if !f.hidden { + out.WriteString(tag) + } + } + return out.String() +} + +func stripThinking(content string) string { + var filter proseFilter + return filter.filter(content, true) +} diff --git a/proxy/proxy.go b/proxy/proxy.go index 19ebfe8..132075c 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -61,6 +61,10 @@ type Config struct { UpstreamAPIKey string // Routing config override (nil = use default) RoutingConfig *router.RoutingConfig + // SpendControl overrides the file-backed controller (useful for embedded servers). + SpendControl *spendcontrol.SpendControl + // UsageLogger overrides usage logging. + UsageLogger func(logger.UsageEntry) } // Server is the OpenAI-compatible proxy with smart routing. @@ -76,6 +80,7 @@ type Server struct { sessions *session.Store journal *journal.SessionJournal spendControl *spendcontrol.SpendControl + spendError error // Deferred startup for OpenClaw plugin config (upstream v0.12.142) // When OpenClaw calls Register() twice, the first call has empty pluginConfig. @@ -92,21 +97,26 @@ func New(cfg Config) *Server { rc = *cfg.RoutingConfig } + sc := cfg.SpendControl + var spendErr error + if sc == nil { + sc, spendErr = spendcontrol.New(spendcontrol.NewFileStorage()) + } return &Server{ config: cfg, routingConfig: rc, modelPricing: models.BuildPricingMap(), httpClient: &http.Client{ - Timeout: 5 * time.Minute, - Transport: &http.Transport{ - Proxy: http.ProxyFromEnvironment, - }, + Timeout: 5 * time.Minute, + Transport: http.DefaultTransport.(*http.Transport).Clone(), + CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, }, dedup: dedup.New(), cache: cache.New(), sessions: session.NewStore(session.DefaultConfig()), journal: journal.New(journal.DefaultConfig()), - spendControl: mustSpendControl(), + spendControl: sc, + spendError: spendErr, } } @@ -186,18 +196,20 @@ func (s *Server) ListenAndServe() error { // chatRequest is the OpenAI-compatible request format. type chatRequest struct { - Model string `json:"model"` - Messages []chatMessage `json:"messages"` - MaxTokens int `json:"max_tokens,omitempty"` - Temperature *float64 `json:"temperature,omitempty"` - Stream bool `json:"stream,omitempty"` - Tools json.RawMessage `json:"tools,omitempty"` + Model string `json:"model"` + Messages []chatMessage `json:"messages"` + MaxTokens int `json:"max_tokens,omitempty"` + Temperature *float64 `json:"temperature,omitempty"` + Stream bool `json:"stream,omitempty"` + Tools json.RawMessage `json:"tools,omitempty"` + Extra map[string]json.RawMessage `json:"-"` } type chatMessage struct { - Role string `json:"role"` - Content json.RawMessage `json:"content"` - ReasoningContent *string `json:"reasoning_content,omitempty"` + Role string `json:"role"` + Content json.RawMessage `json:"content"` + ReasoningContent *string `json:"reasoning_content,omitempty"` + Extra map[string]json.RawMessage `json:"-"` } func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { @@ -214,6 +226,7 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { } defer r.Body.Close() + cacheBody := append([]byte(nil), body...) var req chatRequest if err := json.Unmarshal(body, &req); err != nil { http.Error(w, "Invalid JSON", http.StatusBadRequest) @@ -240,8 +253,10 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { } } + // Caller-specific credentials must never share an internal response cache. + cacheAllowed := s.config.UpstreamAPIKey != "" || r.Header.Get("Authorization") == "" // --- Response cache check (non-streaming only) --- - if !req.Stream { + if !req.Stream && cacheAllowed { if entry, ok := s.cache.Get(body, false); ok { w.Header().Set("X-DOSRouter-Cache", "hit") for k, vs := range entry.Header { @@ -249,6 +264,9 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { w.Header().Add(k, v) } } + if s.config.UpstreamAPIKey != "" || r.Header.Get("Authorization") != "" { + w.Header().Set("Cache-Control", "no-store") + } w.WriteHeader(entry.StatusCode) w.Write(entry.Body) return @@ -311,22 +329,6 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { decision = &d resolvedModel = d.Model - // --- Spend control check --- - if decision.CostEstimate > 0 { - check := s.spendControl.Check(decision.CostEstimate) - if !check.Allowed { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusTooManyRequests) - json.NewEncoder(w).Encode(map[string]interface{}{ - "error": check.Reason, - "blockedBy": check.BlockedBy, - "remaining": check.Remaining, - "resetIn": check.ResetIn, - }) - return - } - } - // Pin to session (smart-routed, not user-explicit) if sessionID != "" { s.sessions.SetSession(sessionID, resolvedModel, d.Tier, false) @@ -351,12 +353,13 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { // --- Context compression (if enabled) --- compMsgs := toNormalizedMessages(req.Messages) - if compression.ShouldCompress(compMsgs) { + if canCompressMessages(req.Messages) && compression.ShouldCompress(compMsgs) { result := compression.CompressContext(compMsgs, compression.DefaultCompressionConfig()) if result.Stats.Ratio < 0.95 && result.Stats.Ratio > 0 { // Re-marshal with compressed messages compReq := req compReq.Messages = fromNormalizedMessages(result.Messages) + req.Messages = compReq.Messages if b, err := json.Marshal(compReq); err == nil { body = b // Use compressed body for upstream } @@ -394,7 +397,7 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { w.Header().Set("X-DOSRouter-Confidence", fmt.Sprintf("%.2f", decision.Confidence)) w.Header().Set("X-DOSRouter-Savings", fmt.Sprintf("%.0f%%", decision.Savings*100)) w.Header().Set("X-DOSRouter-Profile", decision.Profile) - w.Header().Set("X-DOSRouter-Reasoning", decision.Reasoning) + w.Header().Set("X-DOSRouter-Reasoning", sanitizeHeaderValue(decision.Reasoning)) if decision.CostEstimate > 0 { w.Header().Set("X-DOSRouter-Cost", fmt.Sprintf("%.6f", decision.CostEstimate)) } @@ -434,14 +437,30 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { } var attempts []attemptResult var resp *http.Response + var spend *requestSpend + defer func() { + if spend != nil { + spend.finish(nil) + } + }() // cancelResp cancels the context of the SUCCESSFUL attempt; it is deferred // after the loop so the chosen response body stays streamable until the // handler returns, then its resources are released. var cancelResp context.CancelFunc for _, tryModel := range fallbackChain { + if r.Context().Err() != nil { + return + } + if req.MaxTokens <= 0 && req.Extra["max_completion_tokens"] == nil && s.spendControl != nil && len(s.spendControl.GetLimits()) > 0 { + req.MaxTokens = 4096 + } req.Model = tryModel tryBody, _ := json.Marshal(req) + currentSpend, ok := s.reserveChat(w, req, tryBody, tryModel) + if !ok { + return + } // Per-model timeout (upstream v0.12.182): reasoning models get 3min for // cold-start first-token (DeepSeek V4 Pro / Claude opus thinking / GPT-5 // reasoning_effort=high can take 60-120s); non-reasoning get 60s. On @@ -456,15 +475,26 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { // disconnect still cancels the in-flight attempt. attemptCtx, cancelAttempt := context.WithCancel(r.Context()) timer := time.AfterFunc(perModelTimeout(tryModel), cancelAttempt) - tryResp, tryErr := retry.Do(attemptCtx, makeReqFor(tryBody), retry.WithClient(s.httpClient)) + tryResp, tryErr := retry.Do(attemptCtx, makeReqFor(tryBody), retry.WithClient(s.httpClient), retry.WithNetworkRetries(false)) + if tryResp != nil && tryResp.StatusCode >= 300 { + tryErr = nil + } if tryErr != nil { + currentSpend.finish(nil) // A lost response may already have incurred a charge. timer.Stop() cancelAttempt() + if tryResp != nil { + tryResp.Body.Close() + } + if r.Context().Err() != nil { + return + } attempts = append(attempts, attemptResult{model: tryModel, reason: tryErr.Error()}) - continue + break } // Provider returned an error status (4xx/5xx except 429 which retry handles) - if tryResp.StatusCode >= 400 { + if tryResp.StatusCode >= 300 { + currentSpend.release() errBody, _ := io.ReadAll(tryResp.Body) tryResp.Body.Close() timer.Stop() @@ -488,6 +518,8 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { timer.Stop() cancelResp = cancelAttempt resp = tryResp + spend = currentSpend + spend.header = resp.Header resolvedModel = tryModel if tryModel != fallbackChain[0] { w.Header().Set("X-DOSRouter-Fallback", tryModel) @@ -520,7 +552,9 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { } defer resp.Body.Close() - latencyMs := time.Since(startTime).Milliseconds() + if id := gatewayRequestID(resp.Header); id != "" { + w.Header().Set("X-DOSRouter-Request-Id", id) + } // Stream response back if req.Stream { @@ -531,14 +565,19 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { flusher, ok := w.(http.Flusher) if !ok { - io.Copy(w, resp.Body) - s.logRequest(resolvedModel, decision, startTime, "success") + _, copyErr := io.Copy(w, resp.Body) + status := "success" + if copyErr != nil { + status = "interrupted" + } + s.logRequest(resolvedModel, decision, startTime, status) return } scanner := bufio.NewScanner(resp.Body) scanner.Buffer(make([]byte, 64*1024), 1024*1024) var streamInputTok, streamOutputTok int + proseFilters := make(map[int]*proseFilter) for scanner.Scan() { line := scanner.Text() // Parse and rewrite streaming chunks: inject model name + track usage @@ -547,6 +586,9 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { if json.Unmarshal([]byte(line[6:]), &chunk) == nil { // Track usage tokens if u, ok := chunk["usage"].(map[string]interface{}); ok { + _, hasInput := u["prompt_tokens"] + _, hasOutput := u["completion_tokens"] + spend.usageKnown = hasInput && hasOutput if pt, ok := u["prompt_tokens"].(float64); ok { streamInputTok = int(pt) } @@ -554,10 +596,8 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { streamOutputTok = int(ct) } } - // Strip tool-call planning prose from streamed delta content - // (upstream v0.12.165/166/169): blank delta.content when the - // chunk carries tool_calls or finish_reason=tool_calls, so - // planning prose is not forwarded to chat channels. + // Preserve tool-call prose by default (upstream v0.12.248). + // The operator can opt back into suppression. mutated := false if choices, ok := chunk["choices"].([]interface{}); ok { for _, c := range choices { @@ -565,7 +605,22 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { if !ok { continue } - if choiceEndsWithToolCalls(choice) { + if delta, ok := choice["delta"].(map[string]interface{}); ok { + index, _ := choice["index"].(float64) + f := proseFilters[int(index)] + if f == nil { + f = &proseFilter{} + proseFilters[int(index)] = f + } + content, _ := delta["content"].(string) + finish, _ := choice["finish_reason"].(string) + cleaned := f.filter(content, finish != "") + if cleaned != content { + delta["content"] = cleaned + mutated = true + } + } + if choiceEndsWithToolCalls(choice) && !forwardToolCallProse() { if delta, ok := choice["delta"].(map[string]interface{}); ok { if s, _ := delta["content"].(string); s != "" { delta["content"] = "" @@ -609,17 +664,31 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { }, } if b, err := json.Marshal(usageChunk); err == nil { - fmt.Fprintf(w, "data: %s\n\n", b) + if _, err := fmt.Fprintf(w, "data: %s\n\n", b); err != nil { + return + } flusher.Flush() } } } } - fmt.Fprintf(w, "%s\n", line) + if _, err := fmt.Fprintf(w, "%s\n", line); err != nil { + return + } flusher.Flush() } + if scanner.Err() != nil || r.Context().Err() != nil { + return + } + spend.input, spend.output = streamInputTok, streamOutputTok } else { - respBody, _ := io.ReadAll(resp.Body) + respBody, readErr := io.ReadAll(resp.Body) + if readErr != nil { + if r.Context().Err() == nil { + http.Error(w, "Incomplete upstream response", http.StatusBadGateway) + } + return + } // --- Empty turn fallback detection --- // If the response has empty content, no tool_calls, and finish_reason "stop", @@ -634,7 +703,6 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { } if nextModel != "" { log.Printf("degraded response: empty turn from %s, falling back to %s", resolvedModel, nextModel) - resolvedModel = nextModel req.Model = nextModel fbBody, _ := json.Marshal(req) @@ -643,36 +711,57 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { s.sessions.SetSession(sessionID, nextModel, decision.Tier, userExplicit) } - fbResp, fbErr := retry.Do(r.Context(), makeReqFor(fbBody), retry.WithClient(s.httpClient)) - if fbErr == nil { - defer fbResp.Body.Close() - respBody, _ = io.ReadAll(fbResp.Body) - resp = fbResp - w.Header().Set("X-DOSRouter-Fallback", nextModel) - w.Header().Set("X-DOSRouter-Model", nextModel) + spend.finish(respBody) + spend = nil + nextSpend, allowed := s.reserveChat(w, req, fbBody, nextModel) + if !allowed { + return + } + fbResp, fbErr := retry.Do(r.Context(), makeReqFor(fbBody), retry.WithClient(s.httpClient), retry.WithNetworkRetries(false)) + if fbResp != nil && fbResp.StatusCode >= 300 { + fbErr = nil + } + if fbErr != nil { + nextSpend.finish(nil) + if fbResp != nil { + fbResp.Body.Close() + } + if r.Context().Err() == nil { + http.Error(w, "Fallback request failed", http.StatusBadGateway) + } + return } + defer fbResp.Body.Close() + if fbResp.StatusCode >= 300 { + nextSpend.release() + http.Error(w, "Fallback request rejected", http.StatusBadGateway) + return + } + spend = nextSpend + spend.header = fbResp.Header + respBody, readErr = io.ReadAll(fbResp.Body) + if readErr != nil { + http.Error(w, "Incomplete fallback response", http.StatusBadGateway) + return + } + resp = fbResp + resolvedModel = nextModel + w.Header().Set("X-DOSRouter-Fallback", nextModel) + w.Header().Set("X-DOSRouter-Model", nextModel) + } } - // Cache the response - s.cache.Set(body, cache.Entry{ - Body: respBody, - StatusCode: resp.StatusCode, - Header: resp.Header, - }) - + spend.readUsage(respBody) // Inject usage.cost into non-streaming response (upstream v0.12.146) - if resp.StatusCode == http.StatusOK && decision != nil { + if resp.StatusCode == http.StatusOK { var parsed map[string]interface{} if json.Unmarshal(respBody, &parsed) == nil { // Overwrite model with actual resolved model - parsed["model"] = resolvedModel - // Strip tool-call planning prose from content (upstream - // v0.12.165/166): some providers (notably Kimi) emit planning text - // in message.content alongside tool_calls, or flag the turn via - // finish_reason=tool_calls. Tool execution only needs tool_calls; - // forwarding the prose pollutes chat channels. Blank the content - // when the turn ends in tool calls. + if decision != nil { + parsed["model"] = resolvedModel + } + // Preserve assistant prose alongside native tool calls (v0.12.248). if choices, ok := parsed["choices"].([]interface{}); ok { for _, c := range choices { choice, ok := c.(map[string]interface{}) @@ -685,7 +774,9 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { tc, _ := msg["tool_calls"].([]interface{}) if len(tc) == 0 { contentStr, _ := msg["content"].(string) - if recovered := recoverStructuredToolCalls(contentStr, ""); len(recovered) > 0 { + contentStr = stripThinking(contentStr) + if recovered, cleaned := recoverToolCallsWithProse(contentStr); len(recovered) > 0 && len(req.Tools) > 0 { + msg["content"] = cleaned recList := make([]interface{}, len(recovered)) for idx, r := range recovered { recList[idx] = r @@ -696,7 +787,12 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { } } - if choiceEndsWithToolCalls(choice) { + if msg, ok := choice["message"].(map[string]interface{}); ok { + if content, ok := msg["content"].(string); ok { + msg["content"] = stripThinking(content) + } + } + if choiceEndsWithToolCalls(choice) && !forwardToolCallProse() { if msg, ok := choice["message"].(map[string]interface{}); ok { if s, _ := msg["content"].(string); s != "" { msg["content"] = "" @@ -706,7 +802,7 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { } } // Inject cost breakdown if usage tokens available - if usage, ok := parsed["usage"].(map[string]interface{}); ok { + if usage, ok := parsed["usage"].(map[string]interface{}); ok && decision != nil { inputTok, _ := usage["prompt_tokens"].(float64) outputTok, _ := usage["completion_tokens"].(float64) cb := buildCostBreakdown(resolvedModel, string(decision.Tier), decision.Profile, s.modelPricing, int(inputTok), int(outputTok)) @@ -720,12 +816,18 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { } } + if cacheAllowed && resp.StatusCode == http.StatusOK && !isEmptyTurn(respBody) { + s.cache.Set(cacheBody, cache.Entry{Body: respBody, StatusCode: resp.StatusCode, Header: resp.Header.Clone()}) + } // Copy headers with sanitization (upstream v0.12.208) for k, v := range resp.Header { for _, vv := range v { w.Header().Add(k, sanitizeHeaderValue(vv)) } } + if authHeader != "" { + w.Header().Set("Cache-Control", "no-store") + } w.Header().Set("Content-Length", fmt.Sprintf("%d", len(respBody))) w.WriteHeader(resp.StatusCode) w.Write(respBody) @@ -748,17 +850,12 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { } } - // Record spend - if decision != nil && decision.CostEstimate > 0 { - _ = s.spendControl.Record(decision.CostEstimate, resolvedModel, "chat") - if sessionID != "" { - s.sessions.AddSessionCost(sessionID, int64(decision.CostEstimate*1_000_000)) - } + spend.finish(nil) + if sessionID != "" { + s.sessions.AddSessionCost(sessionID, int64(spend.cost*1_000_000)) } + s.logSettledRequest(resolvedModel, decision, startTime, spend) - // Log usage - _ = latencyMs - s.logRequest(resolvedModel, decision, startTime, "success") } func (s *Server) logRequest(model string, decision *router.RoutingDecision, startTime time.Time, status string) { @@ -772,7 +869,7 @@ func (s *Server) logRequest(model string, decision *router.RoutingDecision, star baselineCost = decision.BaselineCost savings = decision.Savings } - logger.LogUsage(logger.UsageEntry{ + s.writeUsage(logger.UsageEntry{ Timestamp: time.Now().UTC().Format(time.RFC3339), Model: model, Tier: tier, @@ -784,15 +881,6 @@ func (s *Server) logRequest(model string, decision *router.RoutingDecision, star }) } -func mustSpendControl() *spendcontrol.SpendControl { - sc, err := spendcontrol.New(spendcontrol.NewFileStorage()) - if err != nil { - // Non-fatal: start with empty state - sc, _ = spendcontrol.New(nil) - } - return sc -} - func flattenHeaders(h http.Header) map[string]string { flat := make(map[string]string, len(h)) for k, v := range h { @@ -859,6 +947,7 @@ func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { resp := map[string]interface{}{ "status": "ok", "version": Version, + "gateway": gatewayOrigin(s.config.UpstreamBase), } // Full health includes session/journal stats if r.URL.Query().Get("full") == "true" { @@ -867,7 +956,13 @@ func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { resp["sessions"] = sessStats.Count resp["journalSessions"] = jStats.Sessions resp["journalEntries"] = jStats.TotalEntries - resp["spendControl"] = s.spendControl.GetStatus() + if s.spendControl != nil { + resp["spendControl"] = s.spendControl.GetStatus() + } + if s.spendError != nil { + resp["status"] = "degraded" + resp["spendControlError"] = "spending state unavailable" + } } json.NewEncoder(w).Encode(resp) } @@ -937,6 +1032,14 @@ func (s *Server) handleImageGen(w http.ResponseWriter, r *http.Request) { return } + if s.spendError != nil || s.spendControl == nil { + http.Error(w, "Spending state unavailable", http.StatusServiceUnavailable) + return + } + if len(s.spendControl.GetLimits()) > 0 { + http.Error(w, "Image cost cannot be reserved under configured spend limits", http.StatusTooManyRequests) + return + } body, err := io.ReadAll(r.Body) if err != nil { http.Error(w, "Failed to read body", http.StatusBadRequest) @@ -973,7 +1076,13 @@ func (s *Server) handleImageGen(w http.ResponseWriter, r *http.Request) { } defer resp.Body.Close() - respBody, _ := io.ReadAll(resp.Body) + respBody, readErr := io.ReadAll(resp.Body) + if readErr != nil { + if r.Context().Err() == nil { + http.Error(w, "Incomplete upstream response", http.StatusBadGateway) + } + return + } for k, v := range resp.Header { for _, vv := range v { @@ -981,13 +1090,18 @@ func (s *Server) handleImageGen(w http.ResponseWriter, r *http.Request) { } } w.Header().Set("X-DOSRouter-Model", req.Model) + if upstreamReq.Header.Get("Authorization") != "" { + w.Header().Set("Cache-Control", "no-store") + } w.WriteHeader(resp.StatusCode) w.Write(respBody) - logger.LogUsage(logger.UsageEntry{ + s.writeUsage(logger.UsageEntry{ Timestamp: time.Now().UTC().Format(time.RFC3339), Model: req.Model, Tier: "IMAGE", + Cost: mediaCost(resp.Header, respBody), + RequestID: gatewayRequestID(resp.Header), Status: fmt.Sprintf("%d", resp.StatusCode), }) } @@ -995,7 +1109,7 @@ func (s *Server) handleImageGen(w http.ResponseWriter, r *http.Request) { // choiceEndsWithToolCalls reports whether a parsed choice object (streaming or // non-streaming) represents a tool-call turn: either finish_reason is // "tool_calls", or a non-empty tool_calls array is present on message or delta. -// Used to suppress planning prose in content (upstream v0.12.165/166/169). +// Used for the optional legacy prose suppression setting. func choiceEndsWithToolCalls(choice map[string]interface{}) bool { if fr, _ := choice["finish_reason"].(string); fr == "tool_calls" { return true diff --git a/proxy/request.go b/proxy/request.go new file mode 100644 index 0000000..dc37752 --- /dev/null +++ b/proxy/request.go @@ -0,0 +1,116 @@ +package proxy + +import ( + "encoding/json" + "net/url" + "os" + "strings" + + "github.com/DOS/DOSRouter/logger" +) + +// The typed view supports routing while Extra preserves the wire protocol. +func (r *chatRequest) UnmarshalJSON(data []byte) error { + type plain chatRequest + var v plain + if err := json.Unmarshal(data, &v); err != nil { + return err + } + if err := json.Unmarshal(data, &v.Extra); err != nil { + return err + } + *r = chatRequest(v) + return nil +} + +func (r chatRequest) MarshalJSON() ([]byte, error) { + type plain chatRequest + return mergeJSON(r.Extra, plain(r)) +} + +func (m *chatMessage) UnmarshalJSON(data []byte) error { + type plain chatMessage + var v plain + if err := json.Unmarshal(data, &v); err != nil { + return err + } + if err := json.Unmarshal(data, &v.Extra); err != nil { + return err + } + *m = chatMessage(v) + return nil +} + +func (m chatMessage) MarshalJSON() ([]byte, error) { + type plain chatMessage + return mergeJSON(m.Extra, plain(m)) +} + +func mergeJSON(extra map[string]json.RawMessage, typed any) ([]byte, error) { + fields := make(map[string]json.RawMessage, len(extra)) + for k, v := range extra { + fields[k] = v + } + data, err := json.Marshal(typed) + if err != nil { + return nil, err + } + if err := json.Unmarshal(data, &fields); err != nil { + return nil, err + } + return json.Marshal(fields) +} + +// Compression cannot remove or stringify protocol-bearing messages. +func canCompressMessages(messages []chatMessage) bool { + for _, m := range messages { + if m.Role == "tool" || len(m.Content) == 0 || m.Content[0] != '"' { + return false + } + for key := range m.Extra { + if key != "role" && key != "content" { + return false + } + } + if m.ReasoningContent != nil { + return false + } + } + return true +} + +func forwardToolCallProse() bool { + return !strings.EqualFold(strings.TrimSpace(os.Getenv("DOSROUTER_TOOL_CALL_PROSE")), "off") +} + +// Remove only syntax which actually recovered into a tool call. +func recoverToolCallsWithProse(content string) ([]map[string]interface{}, string) { + var calls []map[string]interface{} + clean := func(match string) string { + found := recoverStructuredToolCalls(match, "") + if len(found) == 0 { + return match + } + calls = append(calls, found...) + return "" + } + cleaned := jsonCodeBlockRegex.ReplaceAllStringFunc(content, clean) + cleaned = callToolRegex.ReplaceAllStringFunc(cleaned, clean) + return calls, strings.TrimSpace(cleaned) +} + +func gatewayOrigin(base string) string { + u, err := url.Parse(base) + if err != nil || u.Host == "" { + return "" + } + return u.Scheme + "://" + u.Host +} + +func (s *Server) writeUsage(entry logger.UsageEntry) { + if s.config.UsageLogger != nil { + s.config.UsageLogger(entry) + return + } + logger.LogUsage(entry) +} diff --git a/proxy/spending.go b/proxy/spending.go new file mode 100644 index 0000000..d1b041d --- /dev/null +++ b/proxy/spending.go @@ -0,0 +1,177 @@ +package proxy + +import ( + "encoding/json" + "math" + "net/http" + "strconv" + "strings" + "time" + + "github.com/DOS/DOSRouter/logger" + "github.com/DOS/DOSRouter/router" +) + +type requestSpend struct { + usageKnown bool + server *Server + id uint64 + model string + estimate, cost float64 + input, output int + header http.Header + finished bool + source string +} + +func (s *Server) reserveChat(w http.ResponseWriter, req chatRequest, body []byte, model string) (*requestSpend, bool) { + if s.spendError != nil || s.spendControl == nil { + http.Error(w, "Spending state unavailable", http.StatusServiceUnavailable) + return nil, false + } + price, known := s.modelPricing[model] + limited := len(s.spendControl.GetLimits()) > 0 + if !known && limited { + http.Error(w, "Unknown model cost under configured spend limits", http.StatusTooManyRequests) + return nil, false + } + if raw, ok := req.Extra["max_tokens"]; ok { + var value int + if json.Unmarshal(raw, &value) != nil || value <= 0 { + http.Error(w, "Invalid max_tokens", http.StatusBadRequest) + return nil, false + } + } + if raw, ok := req.Extra["max_completion_tokens"]; ok { + var value int + if json.Unmarshal(raw, &value) != nil || value <= 0 { + http.Error(w, "Invalid max_completion_tokens", http.StatusBadRequest) + return nil, false + } + } + output := req.MaxTokens + if raw, ok := req.Extra["max_completion_tokens"]; ok { + if json.Unmarshal(raw, &output) != nil { + output = -1 + } + } + if output <= 0 { + output = 4096 + } + n := 1 + if raw, ok := req.Extra["n"]; ok { + if json.Unmarshal(raw, &n) != nil || n < 1 { + http.Error(w, "Invalid completion count", http.StatusBadRequest) + return nil, false + } + } + // Reserve a conservative text-token estimate including history and tools. + estimate := (float64(len(body))*price.InputPrice + float64(output)*float64(n)*price.OutputPrice) / 1_000_000 + if price.FlatPrice != nil { + estimate = *price.FlatPrice * float64(n) + } + id, check := s.spendControl.Reserve(estimate) + if !check.Allowed { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusTooManyRequests) + json.NewEncoder(w).Encode(map[string]any{"error": check.Reason, "blockedBy": check.BlockedBy, "remaining": check.Remaining}) + return nil, false + } + return &requestSpend{server: s, id: id, model: model, estimate: estimate, cost: estimate, source: "estimate"}, true +} + +func (sp *requestSpend) readUsage(body []byte) { + var parsed struct { + Usage *struct { + Input *int `json:"prompt_tokens"` + Output *int `json:"completion_tokens"` + } `json:"usage"` + } + if json.Unmarshal(body, &parsed) == nil && parsed.Usage != nil && parsed.Usage.Input != nil && parsed.Usage.Output != nil { + sp.input, sp.output = *parsed.Usage.Input, *parsed.Usage.Output + sp.usageKnown = true + } +} + +func (sp *requestSpend) finish(body []byte) { + if sp == nil || sp.finished { + return + } + sp.finished = true + if len(body) > 0 { + sp.readUsage(body) + } + if cost, ok := settledCost(sp.header); ok { + sp.cost, sp.source = cost, "gateway" + } else if (sp.usageKnown || sp.input > 0 || sp.output > 0) && sp.input >= 0 && sp.output >= 0 { + if p, ok := sp.server.modelPricing[sp.model]; ok { + sp.cost = (float64(sp.input)*p.InputPrice + float64(sp.output)*p.OutputPrice) / 1_000_000 + if p.FlatPrice != nil { + sp.cost = *p.FlatPrice + } + sp.source = "tokens" + } + } + // A persistence error keeps the controller fail-closed; never retry a paid call. + _ = sp.server.spendControl.Commit(sp.id, sp.cost, sp.model, "chat") +} + +func (sp *requestSpend) release() { + if sp == nil || sp.finished { + return + } + sp.finished = true + sp.server.spendControl.Release(sp.id) +} + +func settledCost(h http.Header) (float64, bool) { + for _, name := range []string{"X-DOS-Cost-USD", "X-Blockrun-Cost-USD"} { + raw := strings.TrimSpace(h.Get(name)) + if raw == "" { + continue + } + cost, err := strconv.ParseFloat(raw, 64) + if err == nil && cost >= 0 && !math.IsNaN(cost) && !math.IsInf(cost, 0) { + return cost, true + } + } + return 0, false +} + +func gatewayRequestID(h http.Header) string { + for _, name := range []string{"X-DOS-Request-Id", "X-Blockrun-Request-Id", "X-Request-Id", "Request-Id"} { + if id := strings.TrimSpace(h.Get(name)); id != "" { + return sanitizeHeaderValue(id) + } + } + return "" +} + +func mediaCost(h http.Header, body []byte) float64 { + if cost, ok := settledCost(h); ok { + return cost + } + var data struct { + Price struct { + Amount json.RawMessage `json:"amount"` + } `json:"price"` + } + if json.Unmarshal(body, &data) == nil { + raw := strings.Trim(string(data.Price.Amount), `"`) + cost, err := strconv.ParseFloat(raw, 64) + if err == nil && cost >= 0 && !math.IsNaN(cost) && !math.IsInf(cost, 0) { + return cost + } + } + return 0 +} + +func (s *Server) logSettledRequest(model string, decision *router.RoutingDecision, start time.Time, spend *requestSpend) { + entry := logger.UsageEntry{Timestamp: time.Now().UTC().Format(time.RFC3339), Model: model, Tier: "DIRECT", Cost: spend.cost, CostSource: spend.source, RequestID: gatewayRequestID(spend.header), InputTokens: spend.input, OutputTokens: spend.output, Status: "success", LatencyMs: time.Since(start).Milliseconds()} + if decision != nil { + entry.Tier = string(decision.Tier) + entry.BaselineCost = decision.BaselineCost + entry.Savings = decision.Savings + } + s.writeUsage(entry) +} diff --git a/proxy/spending_test.go b/proxy/spending_test.go new file mode 100644 index 0000000..6f68028 --- /dev/null +++ b/proxy/spending_test.go @@ -0,0 +1,43 @@ +package proxy + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/DOS/DOSRouter/logger" +) + +func TestZeroTokenUsageIsNotAnEstimate(t *testing.T) { + var entry logger.UsageEntry + srv, sc := syncTestServer(t, func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, `{"choices":[{"message":{"content":"No charge"},"finish_reason":"stop"}],"usage":{"prompt_tokens":0,"completion_tokens":0}}`) + }, func(cfg *Config) { cfg.UsageLogger = func(e logger.UsageEntry) { entry = e } }) + got := syncTestChat(t, srv, `{"model":"openai/gpt-4o-mini","messages":[{"role":"user","content":"zero"}]}`) + if got.Code != 200 || entry.Cost != 0 || entry.CostSource != "tokens" { + t.Fatalf("status=%d log=%+v", got.Code, entry) + } + if h := sc.GetHistory(); len(h) != 1 || h[0].Amount != 0 { + t.Fatalf("history=%+v", h) + } +} + +func TestCallerCredentialsDoNotShareResponseCache(t *testing.T) { + var calls atomic.Int32 + srv, _ := syncTestServer(t, func(w http.ResponseWriter, r *http.Request) { calls.Add(1); syncTestOK(w) }, nil) + for _, credential := range []string{"Bearer test-a", "Bearer test-b"} { + r := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"openai/gpt-4o-mini","messages":[{"role":"user","content":"same"}]}`)) + r.Header.Set("Authorization", credential) + w := httptest.NewRecorder() + srv.handleChatCompletions(w, r) + if w.Code != 200 || w.Header().Get("X-DOSRouter-Cache") == "hit" || w.Header().Get("Cache-Control") != "no-store" { + t.Fatalf("status=%d headers=%v", w.Code, w.Header()) + } + } + if calls.Load() != 2 { + t.Fatalf("upstream calls=%d", calls.Load()) + } +} diff --git a/proxy/upstream_sync_test.go b/proxy/upstream_sync_test.go new file mode 100644 index 0000000..f7c3594 --- /dev/null +++ b/proxy/upstream_sync_test.go @@ -0,0 +1,505 @@ +package proxy + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "math" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/DOS/DOSRouter/logger" + "github.com/DOS/DOSRouter/router" + "github.com/DOS/DOSRouter/spendcontrol" +) + +func syncTestServer(t *testing.T, upstream http.HandlerFunc, configure func(*Config)) (*Server, *spendcontrol.SpendControl) { + t.Helper() + remote := httptest.NewServer(upstream) + t.Cleanup(remote.Close) + sc, err := spendcontrol.New(nil) + if err != nil { + t.Fatal(err) + } + cfg := Config{UpstreamBase: remote.URL, SpendControl: sc, UsageLogger: func(logger.UsageEntry) {}} + if configure != nil { + configure(&cfg) + } + srv := New(cfg) + t.Cleanup(srv.Close) + return srv, sc +} + +func syncTestJSON(t *testing.T, value any) string { + t.Helper() + b, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func syncTestChat(t *testing.T, srv *Server, payload string) *httptest.ResponseRecorder { + t.Helper() + w := httptest.NewRecorder() + srv.handleChatCompletions(w, httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(payload))) + return w +} + +func syncTestOK(w http.ResponseWriter) { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"model":"openai/gpt-4o-mini","choices":[{"index":0,"message":{"role":"assistant","content":"Done."},"finish_reason":"stop"}],"usage":{"prompt_tokens":10,"completion_tokens":5}}`) +} + +func syncTestTools() []any { + return []any{map[string]any{"type": "function", "function": map[string]any{"name": "read_file", "parameters": map[string]any{"type": "object", "properties": map[string]any{"path": map[string]any{"type": "string"}}}}}} +} + +func syncTestToolCall() []any { + return []any{map[string]any{"id": "call_test", "type": "function", "function": map[string]any{"name": "read_file", "arguments": `{"path":"README.md"}`}}} +} + +func TestUpstreamSyncPreservesExtendedToolConversation(t *testing.T) { + original := map[string]any{ + "model": "gpt5", "stream": false, "max_completion_tokens": 8192, + "reasoning_effort": "high", "response_format": map[string]any{"type": "json_schema", "json_schema": map[string]any{"name": "answer", "strict": true, "schema": map[string]any{"type": "object"}}}, + "parallel_tool_calls": false, "metadata": map[string]any{"trace": "test-only"}, "tools": syncTestTools(), + "messages": []any{ + map[string]any{"role": "user", "name": "operator", "content": strings.Repeat("The exact input must remain unchanged. ", 500)}, + map[string]any{"role": "assistant", "content": nil, "tool_calls": syncTestToolCall(), "reasoning_content": "existing reasoning", "provider_extension": map[string]any{"retain": true}}, + map[string]any{"role": "tool", "tool_call_id": "call_test", "name": "read_file", "content": strings.Repeat("Repeated but semantically meaningful tool output. ", 300)}, + }, + } + received := make(chan map[string]any, 1) + srv, _ := syncTestServer(t, func(w http.ResponseWriter, r *http.Request) { + var body map[string]any + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Error(err) + } + received <- body + syncTestOK(w) + }, nil) + result := syncTestChat(t, srv, syncTestJSON(t, original)) + if result.Code != 200 { + t.Fatalf("status = %d: %s", result.Code, result.Body.String()) + } + got := <-received + // Compare normalized JSON values, including numeric and null fields. + var want map[string]any + json.Unmarshal([]byte(syncTestJSON(t, original)), &want) + want["model"] = "openai/gpt-5.6-terra" + if !reflect.DeepEqual(got, want) { + t.Errorf("forwarded request lost or mutated tool/extension fields\ngot: %#v\nwant: %#v", got, want) + } +} + +func TestUpstreamSyncNonstreamToolProse(t *testing.T) { + for _, kind := range []string{"native", "recovered"} { + for _, optout := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/off=%t", kind, optout), func(t *testing.T) { + setting := "" + if optout { + setting = "off" + } + t.Setenv("DOSROUTER_TOOL_CALL_PROSE", setting) + content := "private deliberationI will read the file." + message := map[string]any{"role": "assistant", "content": content} + finish := "tool_calls" + if kind == "native" { + message["tool_calls"] = syncTestToolCall() + } else { + message["content"] = content + "\n```json\n{\"name\":\"read_file\",\"arguments\":{\"path\":\"README.md\"}}\n```" + finish = "stop" + } + response := syncTestJSON(t, map[string]any{"choices": []any{map[string]any{"index": 0, "message": message, "finish_reason": finish}}}) + srv, _ := syncTestServer(t, func(w http.ResponseWriter, r *http.Request) { io.WriteString(w, response) }, nil) + result := syncTestChat(t, srv, syncTestJSON(t, map[string]any{"model": "openai/gpt-4o-mini", "messages": []any{map[string]any{"role": "user", "content": "Read the file"}}, "tools": syncTestTools()})) + if result.Code != 200 { + t.Fatalf("status = %d: %s", result.Code, result.Body.String()) + } + var got struct { + Choices []struct { + Finish string `json:"finish_reason"` + Message struct { + Content string `json:"content"` + ToolCalls []any `json:"tool_calls"` + } `json:"message"` + } `json:"choices"` + } + if err := json.Unmarshal(result.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + if len(got.Choices) != 1 { + t.Fatalf("choices = %d", len(got.Choices)) + } + choice := got.Choices[0] + want := "I will read the file." + if optout { + want = "" + } + if choice.Message.Content != want || choice.Finish != "tool_calls" || len(choice.Message.ToolCalls) != 1 { + t.Errorf("tool result = %+v, want prose %q with one structured call", choice, want) + } + }) + } + } +} + +func TestUpstreamSyncDoesNotRecoverPrivateThinkingToolSyntax(t *testing.T) { + content := "call:read_file({\"path\":\"PRIVATE\"})I can explain the file." + srv, _ := syncTestServer(t, func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]any{"choices": []any{map[string]any{"index": 0, "message": map[string]any{"role": "assistant", "content": content}, "finish_reason": "stop"}}}) + }, nil) + result := syncTestChat(t, srv, syncTestJSON(t, map[string]any{"model": "openai/gpt-4o-mini", "messages": []any{map[string]any{"role": "user", "content": "Explain the file"}}, "tools": syncTestTools()})) + if result.Code != 200 { + t.Fatalf("status = %d", result.Code) + } + var body map[string]any + json.Unmarshal(result.Body.Bytes(), &body) + choice := body["choices"].([]any)[0].(map[string]any) + message := choice["message"].(map[string]any) + if tc, ok := message["tool_calls"].([]any); ok && len(tc) > 0 { + t.Errorf("private thinking became an executable tool call: %v", tc) + } + if message["content"] != "I can explain the file." || choice["finish_reason"] != "stop" { + t.Errorf("visible result = %v", choice) + } +} + +func TestUpstreamSyncStreamingToolProseAndSplitThinking(t *testing.T) { + for _, optout := range []bool{false, true} { + t.Run(fmt.Sprintf("off=%t", optout), func(t *testing.T) { + setting := "" + if optout { + setting = "off" + } + t.Setenv("DOSROUTER_TOOL_CALL_PROSE", setting) + srv, _ := syncTestServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + chunks := []map[string]any{ + {"choices": []any{map[string]any{"index": 0, "delta": map[string]any{"content": "private reasoningI will read the file.", "tool_calls": syncTestToolCall()}, "finish_reason": "tool_calls"}}}, + } + for _, chunk := range chunks { + b, _ := json.Marshal(chunk) + fmt.Fprintf(w, "data: %s\n\n", b) + w.(http.Flusher).Flush() + } + io.WriteString(w, "data: [DONE]\n\n") + }, nil) + result := syncTestChat(t, srv, syncTestJSON(t, map[string]any{"model": "openai/gpt-4o-mini", "stream": true, "messages": []any{map[string]any{"role": "user", "content": "Read the file"}}, "tools": syncTestTools()})) + if result.Code != 200 { + t.Fatalf("status = %d", result.Code) + } + var content strings.Builder + toolCount := 0 + scanner := bufio.NewScanner(strings.NewReader(result.Body.String())) + for scanner.Scan() { + line := scanner.Text() + if !strings.HasPrefix(line, "data: ") || line == "data: [DONE]" { + continue + } + var chunk struct { + Choices []struct { + Delta struct { + Content string `json:"content"` + Tools []any `json:"tool_calls"` + } `json:"delta"` + } `json:"choices"` + } + if err := json.Unmarshal([]byte(line[6:]), &chunk); err != nil { + t.Fatal(err) + } + for _, c := range chunk.Choices { + content.WriteString(c.Delta.Content) + toolCount += len(c.Delta.Tools) + } + } + want := "I will read the file." + if optout { + want = "" + } + if content.String() != want || toolCount != 1 { + t.Errorf("stream prose/tools = %q/%d, want %q/1", content.String(), toolCount, want) + } + if strings.Contains(result.Body.String(), "private reasoning") { + t.Error("thinking leaked through SSE") + } + }) + } +} + +func TestUpstreamSyncCancellationReachesChatAndImage(t *testing.T) { + for _, mode := range []string{"direct", "smart", "image"} { + t.Run(mode, func(t *testing.T) { + reached := make(chan struct{}, 1) + cancelled := make(chan struct{}, 1) + release := make(chan struct{}) + var unblock sync.Once + defer unblock.Do(func() { close(release) }) + var attempts atomic.Int32 + srv, _ := syncTestServer(t, func(w http.ResponseWriter, r *http.Request) { + attempts.Add(1) + io.Copy(io.Discard, r.Body) + select { + case reached <- struct{}{}: + default: + } + select { + case <-r.Context().Done(): + select { + case cancelled <- struct{}{}: + default: + } + case <-release: + } + }, nil) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + model := "openai/gpt-4o-mini" + if mode == "smart" { + model = "auto" + } + payload := syncTestJSON(t, map[string]any{"model": model, "messages": []any{map[string]any{"role": "user", "content": "hello"}}}) + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(payload)).WithContext(ctx) + finished := make(chan struct{}) + go func() { + defer close(finished) + w := httptest.NewRecorder() + if mode == "image" { + srv.handleImageGen(w, req) + } else { + srv.handleChatCompletions(w, req) + } + }() + select { + case <-reached: + case <-time.After(3 * time.Second): + cancel() + unblock.Do(func() { close(release) }) + t.Fatal("request did not reach test upstream") + } + cancel() + select { + case <-cancelled: + case <-time.After(3 * time.Second): + unblock.Do(func() { close(release) }) + t.Error("upstream request did not observe cancellation") + } + select { + case <-finished: + case <-time.After(3 * time.Second): + t.Error("proxy handler did not stop") + } + if attempts.Load() != 1 { + t.Errorf("cancelled request made %d upstream attempts", attempts.Load()) + } + }) + } +} + +func TestUpstreamSyncBudgetBlocksBeforeForwarding(t *testing.T) { + for _, model := range []string{"openai/gpt-4o-mini", "auto", "vendor/unknown"} { + t.Run(model, func(t *testing.T) { + var attempts atomic.Int32 + srv, sc := syncTestServer(t, func(w http.ResponseWriter, r *http.Request) { attempts.Add(1); syncTestOK(w) }, nil) + if err := sc.SetLimit(spendcontrol.WindowPerRequest, 0.000001); err != nil { + t.Fatal(err) + } + result := syncTestChat(t, srv, syncTestJSON(t, map[string]any{"model": model, "max_completion_tokens": 10000, "messages": []any{map[string]any{"role": "user", "content": "hello"}}})) + if result.Code != 429 || attempts.Load() != 0 { + t.Errorf("status/attempts = %d/%d, want 429/0", result.Code, attempts.Load()) + } + if len(sc.GetHistory()) != 0 { + t.Error("blocked request created a charge") + } + }) + } +} + +func TestUpstreamSyncConcurrentReservationAndGatewaySettlement(t *testing.T) { + for _, model := range []string{"openai/gpt-4o-mini", "auto"} { + t.Run(model, func(t *testing.T) { + reached := make(chan struct{}, 1) + release := make(chan struct{}) + var unblock sync.Once + defer unblock.Do(func() { close(release) }) + var attempts atomic.Int32 + logged := make(chan logger.UsageEntry, 3) + // Use identical bounded model chains so this test isolates admission + // across direct and smart routing instead of scorer calibration. + rc := router.DefaultRoutingConfig() + for _, tiers := range []map[router.Tier]router.TierConfig{rc.Tiers, rc.EcoTiers, rc.PremiumTiers, rc.AgenticTiers} { + for tier := range tiers { + tiers[tier] = router.TierConfig{Primary: "openai/gpt-4o-mini"} + } + } + srv, sc := syncTestServer(t, func(w http.ResponseWriter, r *http.Request) { + if attempts.Add(1) == 1 { + reached <- struct{}{} + <-release + } + w.Header().Set("X-Blockrun-Cost-USD", "0.002") + w.Header().Set("X-Blockrun-Request-Id", "request-test-only") + syncTestOK(w) + }, func(cfg *Config) { + cfg.RoutingConfig = &rc + cfg.UsageLogger = func(e logger.UsageEntry) { logged <- e } + }) + if err := sc.SetLimit(spendcontrol.WindowSession, 0.01); err != nil { + t.Fatal(err) + } + payload := func(prompt string) string { + return syncTestJSON(t, map[string]any{"model": model, "max_completion_tokens": 10000, "messages": []any{map[string]any{"role": "user", "content": prompt}}}) + } + firstDone := make(chan *httptest.ResponseRecorder, 1) + firstPayload := payload("hello first") + go func() { firstDone <- syncTestChat(t, srv, firstPayload) }() + select { + case <-reached: + case <-time.After(3 * time.Second): + unblock.Do(func() { close(release) }) + t.Fatal("first request did not reach upstream") + } + second := syncTestChat(t, srv, payload("hello second")) + if second.Code != 429 || attempts.Load() != 1 { + t.Errorf("concurrent status/attempts = %d/%d, want 429/1", second.Code, attempts.Load()) + } + unblock.Do(func() { close(release) }) + var first *httptest.ResponseRecorder + select { + case first = <-firstDone: + case <-time.After(3 * time.Second): + t.Fatal("first request did not settle") + } + if first.Code != 200 { + t.Fatalf("first status = %d: %s", first.Code, first.Body.String()) + } + if got := first.Header().Get("X-DOSRouter-Request-Id"); got != "request-test-only" { + t.Errorf("gateway request ID = %q", got) + } + history := sc.GetHistory() + if len(history) != 1 || math.Abs(history[0].Amount-0.002) > 1e-12 { + t.Fatalf("settlement history = %+v", history) + } + entry := <-logged + if entry.Cost != 0.002 || entry.CostSource != "gateway" || entry.RequestID != "request-test-only" || entry.InputTokens != 10 || entry.OutputTokens != 5 { + t.Errorf("usage log = %+v", entry) + } + third := syncTestChat(t, srv, payload("hello third")) + if third.Code != 200 || attempts.Load() != 2 { + t.Errorf("after-settlement status/attempts = %d/%d, want 200/2", third.Code, attempts.Load()) + } + }) + } +} + +func TestUpstreamSyncRejectedRequestReleasesReservation(t *testing.T) { + var attempts atomic.Int32 + srv, sc := syncTestServer(t, func(w http.ResponseWriter, r *http.Request) { + if attempts.Add(1) == 1 { + http.Error(w, "invalid request", 400) + return + } + syncTestOK(w) + }, nil) + if err := sc.SetLimit(spendcontrol.WindowSession, 0.007); err != nil { + t.Fatal(err) + } + payload := `{"model":"openai/gpt-4o-mini","max_completion_tokens":10000,"messages":[{"role":"user","content":"hello"}]}` + rejected := syncTestChat(t, srv, payload) + if rejected.Code != 502 { + t.Errorf("rejected proxy status = %d", rejected.Code) + } + if len(sc.GetHistory()) != 0 { + t.Error("explicit rejection charged the session") + } + accepted := syncTestChat(t, srv, payload) + if accepted.Code != 200 || attempts.Load() != 2 { + t.Errorf("released reservation status/attempts = %d/%d", accepted.Code, attempts.Load()) + } +} + +func TestUpstreamSyncTokenSettlementAndCacheDoNotDoubleCharge(t *testing.T) { + var attempts atomic.Int32 + var logs []logger.UsageEntry + srv, sc := syncTestServer(t, func(w http.ResponseWriter, r *http.Request) { + attempts.Add(1) + // Invalid cost headers must not replace reliable token accounting. + w.Header().Set("X-DOS-Cost-USD", "NaN") + syncTestOK(w) + }, func(cfg *Config) { cfg.UsageLogger = func(e logger.UsageEntry) { logs = append(logs, e) } }) + if err := sc.SetLimit(spendcontrol.WindowSession, 1); err != nil { + t.Fatal(err) + } + const payload = `{"model":"openai/gpt-4o-mini","messages":[{"role":"user","content":"hello cached"}]}` + first := syncTestChat(t, srv, payload) + cached := syncTestChat(t, srv, payload) + if first.Code != 200 || cached.Code != 200 || cached.Header().Get("X-DOSRouter-Cache") != "hit" || attempts.Load() != 1 { + t.Errorf("first/cache/attempts = %d/%d/%d, hit = %q", first.Code, cached.Code, attempts.Load(), cached.Header().Get("X-DOSRouter-Cache")) + } + history := sc.GetHistory() + want := (10*0.15 + 5*0.6) / 1_000_000 + if len(history) != 1 || math.Abs(history[0].Amount-want) > 1e-12 { + t.Errorf("token settlement = %+v, want one charge %g", history, want) + } + if len(logs) != 1 || logs[0].CostSource != "tokens" || math.Abs(logs[0].Cost-want) > 1e-12 { + t.Errorf("usage log = %+v", logs) + } +} + +func TestUpstreamSyncSettledCostValidation(t *testing.T) { + cases := []struct { + name, dos, blockrun string + want float64 + valid bool + }{ + {"DOS priority", "0.25", "0.9", 0.25, true}, + {"zero is authoritative", "0", "0.9", 0, true}, + {"BlockRun fallback", "", "0.4", 0.4, true}, + {"invalid primary valid fallback", "NaN", "0.4", 0.4, true}, + {"negative rejected", "-0.1", "", 0, false}, + {"infinity rejected", "+Inf", "", 0, false}, + {"non-numeric rejected", "invalid", "", 0, false}, + {"absent", "", "", 0, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + h := make(http.Header) + h.Set("X-DOS-Cost-USD", tc.dos) + h.Set("X-Blockrun-Cost-USD", tc.blockrun) + cost, valid := settledCost(h) + if cost != tc.want || valid != tc.valid { + t.Errorf("settledCost = %g/%t, want %g/%t", cost, valid, tc.want, tc.valid) + } + }) + } +} + +func TestUpstreamSyncUnknownModelWithoutLimitsRemainsForwardable(t *testing.T) { + forwarded := make(chan string, 1) + srv, _ := syncTestServer(t, func(w http.ResponseWriter, r *http.Request) { + var body struct { + Model string `json:"model"` + } + json.NewDecoder(r.Body).Decode(&body) + forwarded <- body.Model + syncTestOK(w) + }, nil) + result := syncTestChat(t, srv, `{"model":"vendor/new-model","messages":[{"role":"user","content":"hello new model"}]}`) + if result.Code != 200 { + t.Fatalf("status = %d", result.Code) + } + if got := <-forwarded; got != "vendor/new-model" { + t.Errorf("forwarded model = %q", got) + } +} diff --git a/retry/retry.go b/retry/retry.go index 7b624d8..2ba55f5 100644 --- a/retry/retry.go +++ b/retry/retry.go @@ -25,7 +25,7 @@ const ( // retryableStatusCodes are HTTP status codes that trigger a retry. var retryableStatusCodes = map[int]bool{ - http.StatusTooManyRequests: true, // 429 + http.StatusTooManyRequests: true, // 429 http.StatusBadGateway: true, // 502 http.StatusServiceUnavailable: true, // 503 http.StatusGatewayTimeout: true, // 504 @@ -38,7 +38,8 @@ type Config struct { // MaxRetries is the maximum number of retry attempts (default 2). MaxRetries int // Client is the HTTP client to use. If nil, http.DefaultClient is used. - Client *http.Client + Client *http.Client + RetryNetworkErrors bool } // Option configures retry behavior. @@ -59,10 +60,16 @@ func WithClient(cl *http.Client) Option { return func(c *Config) { c.Client = cl } } +// WithNetworkRetries controls retries when no response proves rejection. +func WithNetworkRetries(enabled bool) Option { + return func(c *Config) { c.RetryNetworkErrors = enabled } +} + func defaultConfig() Config { return Config{ - BaseDelay: DefaultBaseDelay, - MaxRetries: DefaultMaxRetries, + BaseDelay: DefaultBaseDelay, + MaxRetries: DefaultMaxRetries, + RetryNetworkErrors: true, } } @@ -87,6 +94,9 @@ func Do(ctx context.Context, buildReq func() (*http.Request, error), opts ...Opt var lastResp *http.Response for attempt := 0; attempt <= cfg.MaxRetries; attempt++ { + if err := ctx.Err(); err != nil { + return nil, err + } req, err := buildReq() if err != nil { return nil, fmt.Errorf("retry: build request: %w", err) @@ -95,7 +105,7 @@ func Do(ctx context.Context, buildReq func() (*http.Request, error), opts ...Opt resp, err := client.Do(req) if err != nil { - if !IsRetryableError(err) { + if !cfg.RetryNetworkErrors || !IsRetryableError(err) { return nil, err } lastErr = err diff --git a/retry/retry_test.go b/retry/retry_test.go new file mode 100644 index 0000000..bea9354 --- /dev/null +++ b/retry/retry_test.go @@ -0,0 +1,96 @@ +package retry + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "sync/atomic" + "testing" + "time" +) + +type retryTestTransport func(*http.Request) (*http.Response, error) + +func (f retryTestTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func TestNetworkRetryOptOutDoesNotReplayAmbiguousPost(t *testing.T) { + for _, disabled := range []bool{true, false} { + t.Run(map[bool]string{true: "disabled", false: "legacy-enabled"}[disabled], func(t *testing.T) { + var attempts atomic.Int32 + client := &http.Client{Transport: retryTestTransport(func(r *http.Request) (*http.Response, error) { + attempts.Add(1) + _, _ = io.ReadAll(r.Body) + return nil, errors.New("connection reset after upstream accepted request") + })} + _, err := Do(context.Background(), func() (*http.Request, error) { + return http.NewRequest(http.MethodPost, "http://upstream.invalid/chat", strings.NewReader(`{"model":"paid"}`)) + }, WithClient(client), WithNetworkRetries(!disabled), WithBaseDelay(time.Microsecond)) + if err == nil { + t.Fatal("expected transport failure") + } + want := int32(3) + if disabled { + want = 1 + } + if got := attempts.Load(); got != want { + t.Errorf("attempts = %d, want %d", got, want) + } + }) + } +} + +func TestNetworkRetryOptOutRetainsRejectedStatusRetries(t *testing.T) { + for _, status := range []int{429, 502, 503, 504} { + t.Run(http.StatusText(status), func(t *testing.T) { + var attempts atomic.Int32 + const payload = `{"model":"paid","messages":[]}` + client := &http.Client{Transport: retryTestTransport(func(r *http.Request) (*http.Response, error) { + got, err := io.ReadAll(r.Body) + if err != nil || string(got) != payload { + t.Errorf("retry body = %q, err = %v", got, err) + } + code := status + if attempts.Add(1) == 2 { + code = http.StatusOK + } + return &http.Response{StatusCode: code, Header: make(http.Header), Body: io.NopCloser(strings.NewReader("response"))}, nil + })} + resp, err := Do(context.Background(), func() (*http.Request, error) { + return http.NewRequest(http.MethodPost, "http://upstream.invalid/chat", strings.NewReader(payload)) + }, WithClient(client), WithNetworkRetries(false), WithBaseDelay(time.Microsecond)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK || attempts.Load() != 2 { + t.Errorf("status/attempts = %d/%d", resp.StatusCode, attempts.Load()) + } + }) + } +} + +func TestCancellationStopsStatusBackoff(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var attempts atomic.Int32 + client := &http.Client{Transport: retryTestTransport(func(r *http.Request) (*http.Response, error) { + attempts.Add(1) + cancel() + return &http.Response{StatusCode: 429, Header: http.Header{"Retry-After": []string{"90"}}, Body: io.NopCloser(strings.NewReader("rejected"))}, nil + })} + start := time.Now() + _, err := Do(ctx, func() (*http.Request, error) { + return http.NewRequest(http.MethodPost, "http://upstream.invalid/chat", nil) + }, WithClient(client), WithNetworkRetries(false)) + if !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v, want context cancellation", err) + } + if attempts.Load() != 1 { + t.Errorf("cancelled request retried %d times", attempts.Load()) + } + if time.Since(start) > time.Second { + t.Error("cancellation waited for Retry-After") + } +} diff --git a/router/config.go b/router/config.go index 63c0f27..85dcf8d 100644 --- a/router/config.go +++ b/router/config.go @@ -17,195 +17,231 @@ func DefaultRoutingConfig() RoutingConfig { Scoring: defaultScoringConfig(), - // Auto (balanced) tier configs + // Tier chains from router-core 5ee7c23c993013a8052588191569db5cf7fb793c. + // NVIDIA free IDs use the DOS catalog namespace. Tiers: map[Tier]TierConfig{ TierSimple: { - Primary: "google/gemini-3.5-flash", + Primary: "google/gemini-2.5-flash", Fallback: []string{ - "google/gemini-2.5-flash", + "google/gemini-3-flash-preview", + "google/gemini-3.5-flash-lite", "deepseek/deepseek-chat", - "openai/gpt-5.4-nano", - "moonshot/kimi-k2.5", "google/gemini-3.1-flash-lite", + "openai/gpt-5.6-luna", + "openai/gpt-5.4-nano", "google/gemini-2.5-flash-lite", - "free/gpt-oss-120b", - "free/llama-4-maverick", + "free/nemotron-3.5-lightning", }, }, TierMedium: { - Primary: "moonshot/kimi-k2.7", + Primary: "google/gemini-3.5-flash", Fallback: []string{ - "moonshot/kimi-k2.6", - "moonshot/kimi-k2.5", - "google/gemini-3.5-flash", + "google/gemini-3.6-flash", + "zai/glm-5.3-flash", + "openai/gpt-5.6-terra", + "google/gemini-3-flash-preview", "deepseek/deepseek-chat", - "openai/chatgpt-instant", "google/gemini-2.5-flash", + "minimax/minimax-m3", "google/gemini-3.1-flash-lite", - "xai/grok-4-1-fast-non-reasoning", + "openai/gpt-5.6-luna", + "google/gemini-2.5-flash-lite", }, }, TierComplex: { - Primary: "openai/gpt-5.6-terra", + Primary: "google/gemini-3.1-pro", Fallback: []string{ + "google/gemini-3.6-flash", + "google/gemini-3.5-flash", "anthropic/claude-sonnet-5", - "google/gemini-3.1-pro", "xai/grok-4.5", - "openai/gpt-5.5", - "anthropic/claude-opus-5", + "google/gemini-2.5-pro", "anthropic/claude-sonnet-4.6", + "openai/gpt-5.6-terra", + "openai/gpt-5.5", + "openai/gpt-5.4", + "zai/glm-5.3", + "moonshot/kimi-k3", + "deepseek/deepseek-v4-pro", "deepseek/deepseek-chat", - "qwen/qwen3.7-max", + "google/gemini-2.5-flash", }, }, TierReasoning: { - Primary: "xai/grok-4-1-fast-reasoning", + Primary: "deepseek/deepseek-reasoner", Fallback: []string{ - "deepseek/deepseek-reasoner", - "openai/gpt-5.6-sol", + "deepseek/deepseek-v4-pro", + "xai/grok-4.3", + "qwen/qwen3.7-plus", + "google/gemini-3.5-flash", "openai/o4-mini", "openai/o3", }, }, }, - // Eco tier configs - absolute cheapest EcoTiers: map[Tier]TierConfig{ TierSimple: { - Primary: "free/gpt-oss-120b", + Primary: "free/nemotron-3.5-lightning", Fallback: []string{ - "free/gpt-oss-20b", - "google/gemini-3.5-flash", - "google/gemini-3.1-flash-lite", + "free/nemotron-3-nano-30b", + "google/gemini-2.5-flash-lite", + "zai/glm-5.3-flash", + "openai/gpt-5.6-luna", "openai/gpt-5.4-nano", - "free/llama-4-maverick", + "google/gemini-3.1-flash-lite", }, }, TierMedium: { - Primary: "google/gemini-3.1-flash-lite", + Primary: "zai/glm-5.3-flash", Fallback: []string{ + "deepseek/deepseek-chat", + "google/gemini-3.1-flash-lite", + "openai/gpt-5.6-luna", "openai/gpt-5.4-nano", - "google/gemini-3.5-flash", "google/gemini-2.5-flash-lite", - "free/llama-4-maverick", + "google/gemini-2.5-flash", }, }, TierComplex: { - Primary: "google/gemini-3.1-flash-lite", + Primary: "zai/glm-5.3-flash", Fallback: []string{ - "google/gemini-3.5-flash", - "google/gemini-2.5-flash", "deepseek/deepseek-chat", - "free/llama-4-maverick", + "minimax/minimax-m3", + "deepseek/deepseek-v4-pro", + "google/gemini-3.1-flash-lite", + "google/gemini-2.5-flash", }, }, TierReasoning: { - Primary: "xai/grok-4-1-fast-reasoning", + Primary: "deepseek/deepseek-reasoner", Fallback: []string{ - "xai/grok-4-fast-reasoning", - "deepseek/deepseek-reasoner", + "deepseek/deepseek-v4-pro", + "qwen/qwen3.7-plus", + "minimax/minimax-m3", + "zai/glm-5.3-flash", }, }, }, - // Premium tier configs - best quality PremiumTiers: map[Tier]TierConfig{ TierSimple: { - Primary: "moonshot/kimi-k2.7", + Primary: "google/gemini-3.5-flash", Fallback: []string{ - "moonshot/kimi-k2.6", + "google/gemini-3.6-flash", "anthropic/claude-haiku-4.5", - "google/gemini-3.5-flash", + "zai/glm-5.3", + "google/gemini-2.5-flash", + "google/gemini-3.5-flash-lite", "deepseek/deepseek-chat", }, }, TierMedium: { - Primary: "anthropic/claude-sonnet-5", + Primary: "openai/gpt-5.3-codex", Fallback: []string{ - "openai/gpt-5.3-codex", + "anthropic/claude-sonnet-5", "moonshot/kimi-k3", - "moonshot/kimi-k2.7", - "zai/glm-5.2", + "zai/glm-5.3", + "google/gemini-3.6-flash", + "google/gemini-3.5-flash", + "google/gemini-2.5-pro", "xai/grok-4.5", + "anthropic/claude-sonnet-4.6", + "openai/gpt-5.6-terra", }, }, TierComplex: { - Primary: "anthropic/claude-opus-5", + Primary: "anthropic/claude-fable-5", Fallback: []string{ + "anthropic/claude-opus-5", "anthropic/claude-opus-4.8", + "anthropic/claude-opus-4.7", "anthropic/claude-sonnet-5", - "openai/gpt-5.6-terra", - "openai/gpt-5.5-pro", + "anthropic/claude-sonnet-4.6", "xai/grok-4.5", - "qwen/qwen3.7-max", "moonshot/kimi-k3", + "openai/gpt-5.6-terra", + "openai/gpt-5.5", + "openai/gpt-5.4", + "openai/gpt-5.3-codex", + "zai/glm-5.3", "deepseek/deepseek-v4-pro", + "deepseek/deepseek-chat", + "free/nemotron-3.5-lightning", }, }, TierReasoning: { - Primary: "anthropic/claude-opus-5", + Primary: "anthropic/claude-sonnet-5", Fallback: []string{ - "openai/gpt-5.6-sol", - "anthropic/claude-sonnet-5", + "anthropic/claude-sonnet-4.6", + "anthropic/claude-opus-5", "anthropic/claude-opus-4.8", + "anthropic/claude-opus-4.7", + "xai/grok-4.5", + "deepseek/deepseek-v4-pro", + "xai/grok-4.3", + "openai/o4-mini", "openai/o3", - "deepseek/deepseek-reasoner", }, }, }, - // Agentic tier configs - multi-step autonomous tasks AgenticTiers: map[Tier]TierConfig{ TierSimple: { Primary: "openai/gpt-4o-mini", Fallback: []string{ - "moonshot/kimi-k2.7", + "openai/gpt-5.6-luna", + "zai/glm-5.3-flash", "anthropic/claude-haiku-4.5", - "xai/grok-4-1-fast-non-reasoning", + "google/gemini-2.5-flash", }, }, TierMedium: { - Primary: "moonshot/kimi-k3", + Primary: "openai/gpt-5-mini", Fallback: []string{ - "moonshot/kimi-k2.7", - "anthropic/claude-sonnet-5", - "xai/grok-4-1-fast-non-reasoning", + "google/gemini-3.5-flash", + "zai/glm-5.3-flash", + "openai/gpt-5.6-terra", "openai/gpt-4o-mini", + "anthropic/claude-haiku-4.5", + "deepseek/deepseek-chat", + "moonshot/kimi-k3", }, }, TierComplex: { - Primary: "anthropic/claude-opus-5", + Primary: "anthropic/claude-sonnet-5", Fallback: []string{ - "anthropic/claude-sonnet-5", - "openai/gpt-5.6-terra", + "anthropic/claude-sonnet-4.6", + "anthropic/claude-opus-5", "anthropic/claude-opus-4.8", - "qwen/qwen3.7-max", - "moonshot/kimi-k3", + "anthropic/claude-opus-4.7", "xai/grok-4.5", + "moonshot/kimi-k3", + "openai/gpt-5.6-terra", + "openai/gpt-5.5", + "openai/gpt-5.4", + "openai/gpt-5.3-codex", + "zai/glm-5.3", + "deepseek/deepseek-v4-pro", + "deepseek/deepseek-chat", + "free/nemotron-3.5-lightning", }, }, TierReasoning: { - Primary: "anthropic/claude-opus-5", + Primary: "anthropic/claude-sonnet-5", Fallback: []string{ - "anthropic/claude-sonnet-5", - "openai/gpt-5.6-sol", + "anthropic/claude-sonnet-4.6", + "anthropic/claude-opus-5", "anthropic/claude-opus-4.8", - "openai/o3", + "anthropic/claude-opus-4.7", + "xai/grok-4.5", + "deepseek/deepseek-v4-pro", + "deepseek/deepseek-reasoner", }, }, }, - Promotions: []Promotion{ - { - Name: "GLM-5.1 Launch Promo ($0.001 flat)", - StartDate: "2026-04-01", - EndDate: "2026-04-15", - TierOverrides: map[Tier]PartialTierConfig{ - TierSimple: {Primary: "zai/glm-5.1"}, - }, - Profiles: []string{"auto"}, - }, - }, + Promotions: []Promotion{}, Overrides: OverridesConfig{ MaxTokensForceComplex: 100_000, diff --git a/spendcontrol/spendcontrol.go b/spendcontrol/spendcontrol.go index f340d6d..c007b0e 100644 --- a/spendcontrol/spendcontrol.go +++ b/spendcontrol/spendcontrol.go @@ -5,6 +5,7 @@ package spendcontrol import ( "encoding/json" "fmt" + "math" "os" "path/filepath" "sync" @@ -21,7 +22,6 @@ const ( WindowSession SpendWindow = "session" ) -// windowDuration returns the rolling duration for time-based windows. func windowDuration(w SpendWindow) time.Duration { switch w { case WindowHourly: @@ -33,6 +33,14 @@ func windowDuration(w SpendWindow) time.Duration { } } +func validWindow(w SpendWindow) bool { + return w == WindowPerRequest || w == WindowHourly || w == WindowDaily || w == WindowSession +} + +func validAmount(amount float64) bool { + return amount >= 0 && !math.IsNaN(amount) && !math.IsInf(amount, 0) +} + // SpendLimits maps each window to its maximum USD amount. type SpendLimits map[SpendWindow]float64 @@ -61,14 +69,17 @@ type SpendControl struct { sessionSpent float64 sessionCalls int storage SpendControlStorage + reservations map[uint64]float64 + nextID uint64 + storageErr error } -// New creates a SpendControl with the given storage backend. -// It loads persisted state on creation. +// New loads persisted limits and history. Session totals start at zero. func New(storage SpendControlStorage) (*SpendControl, error) { sc := &SpendControl{ - limits: make(SpendLimits), - storage: storage, + limits: make(SpendLimits), + storage: storage, + reservations: make(map[uint64]float64), } if err := sc.load(); err != nil { return nil, fmt.Errorf("spendcontrol: load: %w", err) @@ -76,188 +87,195 @@ func New(storage SpendControlStorage) (*SpendControl, error) { return sc, nil } -// SetLimit sets the maximum USD for the given window. +// SetLimit sets a finite, nonnegative maximum USD amount for a known window. func (sc *SpendControl) SetLimit(window SpendWindow, amount float64) error { + if !validWindow(window) || !validAmount(amount) { + return fmt.Errorf("spendcontrol: invalid spending limit") + } sc.mu.Lock() + defer sc.mu.Unlock() sc.limits[window] = amount - sc.mu.Unlock() - return sc.save() + return sc.saveLocked() } // ClearLimit removes the limit for the given window. func (sc *SpendControl) ClearLimit(window SpendWindow) error { + if !validWindow(window) { + return fmt.Errorf("spendcontrol: invalid spending window") + } sc.mu.Lock() + defer sc.mu.Unlock() delete(sc.limits, window) - sc.mu.Unlock() - return sc.save() + return sc.saveLocked() } // GetLimits returns a copy of the current limits. func (sc *SpendControl) GetLimits() SpendLimits { sc.mu.Lock() defer sc.mu.Unlock() - out := make(SpendLimits, len(sc.limits)) - for k, v := range sc.limits { - out[k] = v - } - return out + return cloneState(persistedState{Limits: sc.limits}).Limits } -// Check evaluates whether a request costing estimatedCost USD is allowed. +// Check evaluates a cost against committed spend and all pending reservations. +// Use Reserve before dispatching a request to make admission atomic. func (sc *SpendControl) Check(estimatedCost float64) CheckResult { sc.mu.Lock() defer sc.mu.Unlock() + return sc.checkLocked(estimatedCost, time.Now()) +} - now := time.Now() - - // Per-request check - if limit, ok := sc.limits[WindowPerRequest]; ok { - if estimatedCost > limit { - return CheckResult{ - Allowed: false, - BlockedBy: WindowPerRequest, - Remaining: limit, - Reason: fmt.Sprintf("request cost $%.4f exceeds per-request limit $%.4f", estimatedCost, limit), - } - } +// Reserve atomically checks and holds a request's estimated maximum cost. +// Pending reservations count in every cumulative window until Commit or Release, +// even if a request remains in flight longer than a rolling window. +func (sc *SpendControl) Reserve(cost float64) (uint64, CheckResult) { + sc.mu.Lock() + defer sc.mu.Unlock() + result := sc.checkLocked(cost, time.Now()) + if !result.Allowed { + return 0, result } - - // Session check - if limit, ok := sc.limits[WindowSession]; ok { - remaining := limit - sc.sessionSpent - if estimatedCost > remaining { - return CheckResult{ - Allowed: false, - BlockedBy: WindowSession, - Remaining: remaining, - Reason: fmt.Sprintf("session spend $%.4f + $%.4f would exceed limit $%.4f", sc.sessionSpent, estimatedCost, limit), - } - } + if sc.nextID == ^uint64(0) { + return 0, CheckResult{Reason: "spend reservation identifiers exhausted"} } + sc.nextID++ + sc.reservations[sc.nextID] = cost + return sc.nextID, result +} - // Rolling window checks (hourly, daily) - for _, w := range []SpendWindow{WindowHourly, WindowDaily} { +func (sc *SpendControl) checkLocked(cost float64, now time.Time) CheckResult { + if !validAmount(cost) { + return CheckResult{Reason: "request cost must be finite and nonnegative"} + } + if sc.storageErr != nil { + return CheckResult{Reason: "spending state could not be persisted"} + } + pending := sc.pendingLocked() + if !validAmount(sc.sessionSpent + pending + cost) { + return CheckResult{Reason: "spending total exceeds supported range"} + } + remaining := math.Inf(1) + for _, w := range []SpendWindow{WindowPerRequest, WindowSession, WindowHourly, WindowDaily} { limit, ok := sc.limits[w] if !ok { continue } - d := windowDuration(w) - cutoff := now.Add(-d) - var spent float64 - for _, r := range sc.history { - if r.Timestamp.After(cutoff) { - spent += r.Amount - } - } - remaining := limit - spent - if estimatedCost > remaining { - resetIn := sc.oldestInWindow(cutoff).Add(d).Sub(now) - return CheckResult{ - Allowed: false, + spent := sc.spentLocked(w, now, pending) + available := math.Max(0, limit-spent) + if !validAmount(spent) || cost > limit-spent { + result := CheckResult{ BlockedBy: w, - Remaining: remaining, - Reason: fmt.Sprintf("%s spend $%.4f + $%.4f would exceed limit $%.4f", w, spent, estimatedCost, limit), - ResetIn: formatDuration(resetIn), + Remaining: available, + Reason: fmt.Sprintf("%s spend $%.4f + $%.4f would exceed limit $%.4f", w, spent, cost, limit), } - } - } - - // Compute smallest remaining across all active limits - remaining := -1.0 - for w, limit := range sc.limits { - var r float64 - switch w { - case WindowPerRequest: - r = limit - case WindowSession: - r = limit - sc.sessionSpent - default: - d := windowDuration(w) - cutoff := now.Add(-d) - var spent float64 - for _, rec := range sc.history { - if rec.Timestamp.After(cutoff) { - spent += rec.Amount + if d := windowDuration(w); d > 0 { + if oldest, ok := sc.oldestInWindow(now.Add(-d)); ok { + result.ResetIn = formatDuration(oldest.Add(d).Sub(now)) } } - r = limit - spent - } - if remaining < 0 || r < remaining { - remaining = r + return result } + remaining = math.Min(remaining, available) } - if remaining < 0 { + if math.IsInf(remaining, 1) { remaining = 0 } - return CheckResult{Allowed: true, Remaining: remaining} } -// Record logs a completed spend and updates the session totals. +// Commit replaces a reservation with actual spend exactly once. Actual spend is +// recorded even when it exceeds the estimate; subsequent admission sees it. +// Invalid amounts retain the reservation. A persistence failure retains the +// recorded spend in memory and blocks admission until a later successful save. +func (sc *SpendControl) Commit(id uint64, amount float64, model, action string) error { + sc.mu.Lock() + defer sc.mu.Unlock() + if _, ok := sc.reservations[id]; !ok { + return fmt.Errorf("spendcontrol: unknown reservation") + } + if !validAmount(amount) || !validAmount(sc.sessionSpent+amount) { + return fmt.Errorf("spendcontrol: invalid spending amount") + } + delete(sc.reservations, id) + sc.recordLocked(amount, model, action) + return sc.saveLocked() +} + +// Release cancels a reservation when it is known that no spend occurred. +// Releasing an already settled or unknown reservation has no effect. +func (sc *SpendControl) Release(id uint64) { + sc.mu.Lock() + defer sc.mu.Unlock() + delete(sc.reservations, id) +} + +// Record logs completed spend without an admission check. New request paths +// should use Reserve and Commit to avoid check-then-record concurrency races. func (sc *SpendControl) Record(amount float64, model, action string) error { sc.mu.Lock() + defer sc.mu.Unlock() + if !validAmount(amount) || !validAmount(sc.sessionSpent+amount) { + return fmt.Errorf("spendcontrol: invalid spending amount") + } + sc.recordLocked(amount, model, action) + return sc.saveLocked() +} + +func (sc *SpendControl) recordLocked(amount float64, model, action string) { sc.history = append(sc.history, SpendRecord{ - Timestamp: time.Now(), - Amount: amount, - Model: model, - Action: action, + Timestamp: time.Now(), Amount: amount, Model: model, Action: action, }) sc.sessionSpent += amount sc.sessionCalls++ - sc.mu.Unlock() - return sc.save() } -// GetSpending returns total spent in each active window. +func (sc *SpendControl) pendingLocked() float64 { + var pending float64 + for _, amount := range sc.reservations { + pending += amount + } + return pending +} + +func (sc *SpendControl) spentLocked(window SpendWindow, now time.Time, pending float64) float64 { + switch window { + case WindowPerRequest: + return 0 + case WindowSession: + return sc.sessionSpent + pending + default: + spent := pending + cutoff := now.Add(-windowDuration(window)) + for _, record := range sc.history { + if record.Timestamp.After(cutoff) { + spent += record.Amount + } + } + return spent + } +} + +// GetSpending returns committed and reserved spend in each active window. func (sc *SpendControl) GetSpending() map[SpendWindow]float64 { sc.mu.Lock() defer sc.mu.Unlock() - now := time.Now() + now, pending := time.Now(), sc.pendingLocked() out := make(map[SpendWindow]float64) - for w := range sc.limits { - switch w { - case WindowPerRequest: - // not cumulative - case WindowSession: - out[w] = sc.sessionSpent - default: - d := windowDuration(w) - cutoff := now.Add(-d) - var spent float64 - for _, r := range sc.history { - if r.Timestamp.After(cutoff) { - spent += r.Amount - } - } - out[w] = spent + for window := range sc.limits { + if window != WindowPerRequest { + out[window] = sc.spentLocked(window, now, pending) } } return out } -// GetRemaining returns remaining budget in each active window. +// GetRemaining returns budget available after committed and reserved spend. func (sc *SpendControl) GetRemaining() map[SpendWindow]float64 { sc.mu.Lock() defer sc.mu.Unlock() - now := time.Now() + now, pending := time.Now(), sc.pendingLocked() out := make(map[SpendWindow]float64) - for w, limit := range sc.limits { - switch w { - case WindowPerRequest: - out[w] = limit - case WindowSession: - out[w] = limit - sc.sessionSpent - default: - d := windowDuration(w) - cutoff := now.Add(-d) - var spent float64 - for _, r := range sc.history { - if r.Timestamp.After(cutoff) { - spent += r.Amount - } - } - out[w] = limit - spent - } + for window, limit := range sc.limits { + out[window] = limit - sc.spentLocked(window, now, pending) } return out } @@ -269,24 +287,20 @@ type StatusEntry struct { Remaining float64 `json:"remaining"` } -// GetStatus returns a combined view of limits, spending, and remaining. +// GetStatus returns one consistent snapshot of limits, spend, and remaining. func (sc *SpendControl) GetStatus() map[SpendWindow]StatusEntry { - spending := sc.GetSpending() - remaining := sc.GetRemaining() - limits := sc.GetLimits() - - out := make(map[SpendWindow]StatusEntry, len(limits)) - for w, limit := range limits { - out[w] = StatusEntry{ - Limit: limit, - Spent: spending[w], - Remaining: remaining[w], - } + sc.mu.Lock() + defer sc.mu.Unlock() + now, pending := time.Now(), sc.pendingLocked() + out := make(map[SpendWindow]StatusEntry, len(sc.limits)) + for window, limit := range sc.limits { + spent := sc.spentLocked(window, now, pending) + out[window] = StatusEntry{Limit: limit, Spent: spent, Remaining: limit - spent} } return out } -// GetHistory returns a copy of all spending records. +// GetHistory returns a copy of all completed spending records. func (sc *SpendControl) GetHistory() []SpendRecord { sc.mu.Lock() defer sc.mu.Unlock() @@ -295,47 +309,58 @@ func (sc *SpendControl) GetHistory() []SpendRecord { return out } -// Cleanup prunes records older than 24 hours. +// Cleanup prunes completed records older than 24 hours. Pending reservations and +// the current session's totals remain intact. func (sc *SpendControl) Cleanup() error { sc.mu.Lock() + defer sc.mu.Unlock() cutoff := time.Now().Add(-24 * time.Hour) - kept := sc.history[:0] - for _, r := range sc.history { - if r.Timestamp.After(cutoff) { - kept = append(kept, r) + kept := make([]SpendRecord, 0, len(sc.history)) + for _, record := range sc.history { + if record.Timestamp.After(cutoff) { + kept = append(kept, record) } } sc.history = kept - sc.mu.Unlock() - return sc.save() + return sc.saveLocked() } -// oldestInWindow returns the oldest record timestamp within the window, -// or now if none found. Caller must hold mu. -func (sc *SpendControl) oldestInWindow(cutoff time.Time) time.Time { - oldest := time.Now() - for _, r := range sc.history { - if r.Timestamp.After(cutoff) && r.Timestamp.Before(oldest) { - oldest = r.Timestamp +func (sc *SpendControl) oldestInWindow(cutoff time.Time) (time.Time, bool) { + var oldest time.Time + for _, record := range sc.history { + if record.Timestamp.After(cutoff) && (oldest.IsZero() || record.Timestamp.Before(oldest)) { + oldest = record.Timestamp } } - return oldest + return oldest, !oldest.IsZero() } -// persistedState is the JSON shape for file storage. +// persistedState intentionally excludes process-local sessions and reservations. type persistedState struct { Limits SpendLimits `json:"limits"` History []SpendRecord `json:"history"` } -func (sc *SpendControl) save() error { +func cloneState(state persistedState) persistedState { + cp := persistedState{Limits: make(SpendLimits, len(state.Limits))} + for window, amount := range state.Limits { + cp.Limits[window] = amount + } + cp.History = append([]SpendRecord(nil), state.History...) + return cp +} + +// saveLocked serializes saves with mutations, preventing an older snapshot from +// overwriting a newer one. Storage receives a detached snapshot. +func (sc *SpendControl) saveLocked() error { if sc.storage == nil { return nil } - sc.mu.Lock() - state := persistedState{Limits: sc.limits, History: sc.history} - sc.mu.Unlock() - return sc.storage.Save(state) + sc.storageErr = sc.storage.Save(cloneState(persistedState{Limits: sc.limits, History: sc.history})) + if sc.storageErr != nil { + return fmt.Errorf("spendcontrol: save: %w", sc.storageErr) + } + return nil } func (sc *SpendControl) load() error { @@ -343,23 +368,31 @@ func (sc *SpendControl) load() error { return nil } state, err := sc.storage.Load() - if err != nil { + if err != nil || state == nil { return err } - if state != nil { - if state.Limits != nil { - sc.limits = state.Limits + for window, limit := range state.Limits { + if !validWindow(window) || !validAmount(limit) { + return fmt.Errorf("invalid persisted spending limit") + } + } + var total float64 + for _, record := range state.History { + if !validAmount(record.Amount) || record.Timestamp.IsZero() { + return fmt.Errorf("invalid persisted spending record") } - if state.History != nil { - sc.history = state.History + total += record.Amount + if !validAmount(total) { + return fmt.Errorf("persisted spending total exceeds supported range") } } + cp := cloneState(*state) + sc.limits, sc.history = cp.Limits, cp.History return nil } -// ---------- Storage interface ---------- - -// SpendControlStorage persists spending state. +// SpendControlStorage persists spending state. Implementations must not call back +// into the controller while saving; Save executes inside the controller's lock. type SpendControlStorage interface { Save(state persistedState) error Load() (*persistedState, error) @@ -381,16 +414,32 @@ func NewFileStorage() *FileSpendControlStorage { return &FileSpendControlStorage{Path: DefaultFilePath()} } +// Save writes a complete temporary file before replacing the previous state. func (fs *FileSpendControlStorage) Save(state persistedState) error { + data, err := json.MarshalIndent(state, "", " ") + if err != nil { + return err + } dir := filepath.Dir(fs.Path) - if err := os.MkdirAll(dir, 0o755); err != nil { + if err := os.MkdirAll(dir, 0o700); err != nil { return err } - data, err := json.MarshalIndent(state, "", " ") + file, err := os.CreateTemp(dir, ".spending-*.tmp") if err != nil { return err } - return os.WriteFile(fs.Path, data, 0o644) + defer os.Remove(file.Name()) + if _, err = file.Write(data); err == nil { + err = file.Sync() + } + closeErr := file.Close() + if err != nil { + return err + } + if closeErr != nil { + return closeErr + } + return os.Rename(file.Name(), fs.Path) } func (fs *FileSpendControlStorage) Load() (*persistedState, error) { @@ -401,14 +450,43 @@ func (fs *FileSpendControlStorage) Load() (*persistedState, error) { } return nil, err } - var state persistedState - if err := json.Unmarshal(data, &state); err != nil { + // Pointer amounts distinguish an explicit zero from a missing or null + // value, which could otherwise silently erase previously recorded spend. + var input struct { + Limits map[SpendWindow]*float64 `json:"limits"` + History []struct { + Timestamp time.Time `json:"timestamp"` + Amount *float64 `json:"amount"` + Model string `json:"model"` + Action string `json:"action"` + } `json:"history"` + } + if err := json.Unmarshal(data, &input); err != nil { return nil, err } + if input.Limits == nil { + return nil, fmt.Errorf("missing or null persisted spending limits") + } + state := persistedState{Limits: make(SpendLimits, len(input.Limits))} + for window, amount := range input.Limits { + if amount == nil { + return nil, fmt.Errorf("null persisted spending limit") + } + state.Limits[window] = *amount + } + for _, record := range input.History { + if record.Amount == nil { + return nil, fmt.Errorf("missing or null persisted spending amount") + } + state.History = append(state.History, SpendRecord{ + Timestamp: record.Timestamp, Amount: *record.Amount, + Model: record.Model, Action: record.Action, + }) + } return &state, nil } -// InMemorySpendControlStorage keeps state in memory only. +// InMemorySpendControlStorage keeps detached state in memory only. type InMemorySpendControlStorage struct { mu sync.Mutex state *persistedState @@ -417,7 +495,7 @@ type InMemorySpendControlStorage struct { func (m *InMemorySpendControlStorage) Save(state persistedState) error { m.mu.Lock() defer m.mu.Unlock() - cp := state + cp := cloneState(state) m.state = &cp return nil } @@ -425,12 +503,13 @@ func (m *InMemorySpendControlStorage) Save(state persistedState) error { func (m *InMemorySpendControlStorage) Load() (*persistedState, error) { m.mu.Lock() defer m.mu.Unlock() - return m.state, nil + if m.state == nil { + return nil, nil + } + cp := cloneState(*m.state) + return &cp, nil } -// ---------- Helpers ---------- - -// formatDuration produces a human-readable duration string (e.g. "1h 23m 45s"). func formatDuration(d time.Duration) string { if d <= 0 { return "0s" @@ -438,7 +517,6 @@ func formatDuration(d time.Duration) string { h := int(d.Hours()) m := int(d.Minutes()) % 60 s := int(d.Seconds()) % 60 - switch { case h > 0 && m > 0: return fmt.Sprintf("%dh %dm %ds", h, m, s) diff --git a/spendcontrol/spendcontrol_test.go b/spendcontrol/spendcontrol_test.go new file mode 100644 index 0000000..fada5d4 --- /dev/null +++ b/spendcontrol/spendcontrol_test.go @@ -0,0 +1,410 @@ +package spendcontrol + +import ( + "errors" + "math" + "os" + "path/filepath" + "sync" + "testing" + "time" +) + +func newTestControl(t *testing.T, storage SpendControlStorage) *SpendControl { + t.Helper() + sc, err := New(storage) + if err != nil { + t.Fatal(err) + } + return sc +} + +func TestConcurrentReservationsCannotOversubscribe(t *testing.T) { + for _, window := range []SpendWindow{WindowSession, WindowHourly, WindowDaily} { + t.Run(string(window), func(t *testing.T) { + sc := newTestControl(t, nil) + if err := sc.SetLimit(window, 1); err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + ids := make(chan uint64, 100) + start := make(chan struct{}) + for i := 0; i < 100; i++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + if id, result := sc.Reserve(0.25); result.Allowed { + ids <- id + } + }() + } + close(start) + wg.Wait() + close(ids) + seen := make(map[uint64]bool) + for id := range ids { + if id == 0 || seen[id] { + t.Fatalf("invalid or reused reservation: %d", id) + } + seen[id] = true + } + if len(seen) != 4 { + t.Fatalf("admitted %d requests, want 4", len(seen)) + } + if result := sc.Check(0.25); result.Allowed || result.BlockedBy != window { + t.Fatalf("pending spend did not block: %+v", result) + } + status := sc.GetStatus()[window] + if status.Spent != 1 || status.Remaining != 0 { + t.Fatalf("inconsistent pending status: %+v", status) + } + }) + } +} + +func TestReservationsSettleExactlyOnce(t *testing.T) { + sc := newTestControl(t, nil) + if err := sc.SetLimit(WindowDaily, 1); err != nil { + t.Fatal(err) + } + first, firstResult := sc.Reserve(0.75) + second, secondResult := sc.Reserve(0.25) + if !firstResult.Allowed || !secondResult.Allowed { + t.Fatal("initial reservations denied") + } + if err := sc.Commit(first, 0.5, "test-model", "chat"); err != nil { + t.Fatal(err) + } + sc.Release(second) + sc.Release(second) + sc.Release(first) + if err := sc.Commit(first, 0.5, "test-model", "chat"); err == nil { + t.Fatal("duplicate commit accepted") + } + if err := sc.Commit(second, 0.25, "test-model", "chat"); err == nil { + t.Fatal("released reservation committed") + } + if got := sc.GetSpending()[WindowDaily]; got != 0.5 { + t.Fatalf("spent = %v, want 0.5", got) + } + if got := sc.GetRemaining()[WindowDaily]; got != 0.5 { + t.Fatalf("remaining = %v, want 0.5", got) + } + history := sc.GetHistory() + if len(history) != 1 || history[0].Amount != 0.5 || history[0].Model != "test-model" || history[0].Action != "chat" { + t.Fatalf("unexpected committed history: %+v", history) + } +} + +func TestPendingReservationsNeverAgeOut(t *testing.T) { + for _, window := range []SpendWindow{WindowHourly, WindowDaily} { + t.Run(string(window), func(t *testing.T) { + sc := newTestControl(t, nil) + if err := sc.SetLimit(window, 1); err != nil { + t.Fatal(err) + } + id, result := sc.Reserve(0.75) + if !result.Allowed { + t.Fatal(result.Reason) + } + if err := sc.Record(0.25, "test", "chat"); err != nil { + t.Fatal(err) + } + sc.mu.Lock() + future := time.Now().Add(2 * windowDuration(window)) + blocked := sc.checkLocked(0.5, future) + allowed := sc.checkLocked(0.25, future) + sc.history[0].Timestamp = time.Now().Add(-48 * time.Hour) + sc.mu.Unlock() + if blocked.Allowed || !allowed.Allowed { + t.Fatalf("pending spend aged out or history did not expire: blocked=%+v allowed=%+v", blocked, allowed) + } + if err := sc.Cleanup(); err != nil { + t.Fatal(err) + } + if result := sc.Check(0.5); result.Allowed { + t.Fatal("cleanup removed pending reservation") + } + sc.Release(id) + if result := sc.Check(1); !result.Allowed { + t.Fatalf("released reservation still blocks: %+v", result) + } + }) + } +} + +func TestInvalidAmountsFailClosed(t *testing.T) { + sc := newTestControl(t, nil) + if err := sc.SetLimit(WindowSession, 1); err != nil { + t.Fatal(err) + } + id, result := sc.Reserve(0.75) + if !result.Allowed { + t.Fatal(result.Reason) + } + for _, amount := range []float64{-1, math.NaN(), math.Inf(1), math.Inf(-1)} { + if sc.Check(amount).Allowed { + t.Fatalf("invalid check accepted: %v", amount) + } + if nextID, result := sc.Reserve(amount); result.Allowed || nextID != 0 { + t.Fatalf("invalid reservation accepted: %v", amount) + } + if err := sc.Commit(id, amount, "", ""); err == nil { + t.Fatalf("invalid commit accepted: %v", amount) + } + if err := sc.Record(amount, "", ""); err == nil { + t.Fatalf("invalid record accepted: %v", amount) + } + if err := sc.SetLimit(WindowSession, amount); err == nil { + t.Fatalf("invalid limit accepted: %v", amount) + } + } + if err := sc.SetLimit(SpendWindow("typo"), 1); err == nil { + t.Fatal("unknown spending window accepted") + } + if len(sc.GetHistory()) != 0 || sc.GetLimits()[WindowSession] != 1 || sc.GetSpending()[WindowSession] != 0.75 { + t.Fatal("invalid input mutated valid state") + } + if err := sc.Commit(id, 0.5, "", ""); err != nil { + t.Fatalf("invalid commit destroyed the reservation: %v", err) + } +} + +func TestActualSpendAboveEstimateIsRetained(t *testing.T) { + sc := newTestControl(t, nil) + if err := sc.SetLimit(WindowDaily, 1); err != nil { + t.Fatal(err) + } + id, result := sc.Reserve(0.5) + if !result.Allowed { + t.Fatal(result.Reason) + } + if err := sc.Commit(id, 1.5, "", ""); err != nil { + t.Fatal(err) + } + if sc.Check(0).Allowed || sc.GetSpending()[WindowDaily] != 1.5 { + t.Fatal("actual overspend was dropped") + } +} + +type failingStorage struct { + state *persistedState + fail bool +} + +func (s *failingStorage) Save(state persistedState) error { + if s.fail { + return errors.New("injected storage failure") + } + cp := cloneState(state) + s.state = &cp + return nil +} + +func (s *failingStorage) Load() (*persistedState, error) { return s.state, nil } + +func TestPersistenceFailureRetainsSpendAndBlocksAdmission(t *testing.T) { + storage := &failingStorage{} + sc := newTestControl(t, storage) + if err := sc.SetLimit(WindowDaily, 1); err != nil { + t.Fatal(err) + } + id, _ := sc.Reserve(0.75) + storage.fail = true + if err := sc.Commit(id, 0.5, "test", "chat"); err == nil { + t.Fatal("commit did not report persistence failure") + } + if got := sc.GetSpending()[WindowDaily]; got != 0.5 { + t.Fatalf("lost completed spend on failed save: %v", got) + } + if sc.Check(0).Allowed { + t.Fatal("admission remained open after persistence failure") + } + if id, result := sc.Reserve(0); id != 0 || result.Allowed { + t.Fatal("reservation accepted after persistence failure") + } + storage.fail = false + if err := sc.Cleanup(); err != nil { + t.Fatal(err) + } + if !sc.Check(0.5).Allowed || sc.Check(0.75).Allowed { + t.Fatal("successful save did not recover the correct remaining budget") + } + if len(storage.state.History) != 1 || storage.state.History[0].Amount != 0.5 { + t.Fatal("recovered storage lost committed spend") + } +} + +func TestMemoryStorageDoesNotShareSnapshots(t *testing.T) { + storage := &InMemorySpendControlStorage{} + original := persistedState{ + Limits: SpendLimits{WindowDaily: 1}, + History: []SpendRecord{{Timestamp: time.Now(), Amount: 0.25}}, + } + if err := storage.Save(original); err != nil { + t.Fatal(err) + } + original.Limits[WindowDaily] = 100 + original.History[0].Amount = 100 + loaded, err := storage.Load() + if err != nil { + t.Fatal(err) + } + if loaded.Limits[WindowDaily] != 1 || loaded.History[0].Amount != 0.25 { + t.Fatal("saved state aliases caller memory") + } + loaded.Limits[WindowDaily] = 200 + loaded.History[0].Amount = 200 + again, err := storage.Load() + if err != nil { + t.Fatal(err) + } + if again.Limits[WindowDaily] != 1 || again.History[0].Amount != 0.25 { + t.Fatal("loaded state aliases stored memory") + } +} + +// Retaining every passed snapshot detects sharing with later controller writes. +type snapshotStorage struct { + snapshots []persistedState +} + +func (s *snapshotStorage) Save(state persistedState) error { + s.snapshots = append(s.snapshots, state) + return nil +} +func (s *snapshotStorage) Load() (*persistedState, error) { return nil, nil } + +func TestConcurrentMutationsPersistDetachedOrderedSnapshots(t *testing.T) { + storage := &snapshotStorage{} + sc := newTestControl(t, storage) + if err := sc.SetLimit(WindowDaily, 1); err != nil { + t.Fatal(err) + } + var wg sync.WaitGroup + for i := 0; i < 40; i++ { + wg.Add(1) + go func() { + defer wg.Done() + if err := sc.Record(0.25, "test", "chat"); err != nil { + t.Error(err) + } + status := sc.GetStatus()[WindowDaily] + if status.Limit-status.Spent != status.Remaining { + t.Errorf("inconsistent status: %+v", status) + } + }() + } + wg.Wait() + if err := sc.SetLimit(WindowDaily, 20); err != nil { + t.Fatal(err) + } + if len(storage.snapshots) != 42 || storage.snapshots[0].Limits[WindowDaily] != 1 { + t.Fatal("limit snapshots were lost or mutated") + } + for index := 1; index <= 40; index++ { + if len(storage.snapshots[index].History) != index { + t.Fatalf("out-of-order snapshot %d has %d records", index, len(storage.snapshots[index].History)) + } + } + sc.mu.Lock() + sc.history[0].Amount = 100 + sc.mu.Unlock() + if storage.snapshots[1].History[0].Amount != 0.25 { + t.Fatal("history snapshots alias controller memory") + } +} + +func TestRestartResetsSessionButPreservesRollingSpend(t *testing.T) { + storage := &InMemorySpendControlStorage{} + first := newTestControl(t, storage) + for _, window := range []SpendWindow{WindowSession, WindowHourly, WindowDaily} { + if err := first.SetLimit(window, 1); err != nil { + t.Fatal(err) + } + } + if err := first.Record(0.75, "test", "chat"); err != nil { + t.Fatal(err) + } + second := newTestControl(t, storage) + spending := second.GetSpending() + if spending[WindowSession] != 0 || spending[WindowHourly] != 0.75 || spending[WindowDaily] != 0.75 { + t.Fatalf("unexpected restart totals: %+v", spending) + } +} + +func TestFileStorageRejectsCorruptionAndPreservesLastGoodWrite(t *testing.T) { + path := filepath.Join(t.TempDir(), "spending.json") + storage := &FileSpendControlStorage{Path: path} + sc := newTestControl(t, storage) + if err := sc.SetLimit(WindowDaily, 1); err != nil { + t.Fatal(err) + } + if err := sc.Record(0.25, "test", "chat"); err != nil { + t.Fatal(err) + } + before, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if err := storage.Save(persistedState{Limits: SpendLimits{WindowDaily: math.NaN()}}); err == nil { + t.Fatal("invalid JSON amount saved") + } + after, err := os.ReadFile(path) + if err != nil || string(before) != string(after) { + t.Fatal("failed save changed the previous state") + } + if restarted := newTestControl(t, storage); restarted.GetSpending()[WindowDaily] != 0.25 { + t.Fatal("file round trip lost spending") + } + for _, invalid := range []string{ + `{`, + `null`, + `{}`, + `{"limits":{"daily":-1},"history":[]}`, + `{"limits":{"typo":1},"history":[]}`, + `{"limits":null,"history":[]}`, + `{"limits":{"daily":null},"history":[]}`, + `{"limits":{"daily":1},"history":[{"amount":0.5}]}`, + `{"limits":{"daily":1},"history":[{"timestamp":"2026-01-01T00:00:00Z","amount":null}]}`, + `{"limits":{"daily":1},"history":[{"timestamp":"2026-01-01T00:00:00Z","amount":-1}]}`, + } { + if err := os.WriteFile(path, []byte(invalid), 0o600); err != nil { + t.Fatal(err) + } + if _, err := New(storage); err == nil { + t.Errorf("corrupt persisted state accepted: %s", invalid) + } + } +} + +func TestPerRequestLimitDoesNotCombineSeparateReservations(t *testing.T) { + sc := newTestControl(t, nil) + if err := sc.SetLimit(WindowPerRequest, 0.5); err != nil { + t.Fatal(err) + } + for i := 0; i < 2; i++ { + if _, result := sc.Reserve(0.5); !result.Allowed { + t.Fatalf("per-request cap incorrectly accumulated reservations: %+v", result) + } + } + if _, result := sc.Reserve(0.75); result.Allowed || result.BlockedBy != WindowPerRequest { + t.Fatalf("per-request limit not enforced: %+v", result) + } +} + +func TestReservationTotalOverflowFailsClosed(t *testing.T) { + sc := newTestControl(t, nil) + id, result := sc.Reserve(math.MaxFloat64) + if !result.Allowed { + t.Fatal(result.Reason) + } + if _, result := sc.Reserve(math.MaxFloat64); result.Allowed { + t.Fatal("overflowing reservation total accepted") + } + sc.Release(id) + if _, result := sc.Reserve(0); !result.Allowed { + t.Fatal("zero-cost reservation should remain valid") + } +} diff --git a/stats/stats.go b/stats/stats.go index e18d807..b880934 100644 --- a/stats/stats.go +++ b/stats/stats.go @@ -72,26 +72,45 @@ type logEntry struct { func parseLogFile(filePath string) []logEntry { data, err := os.ReadFile(filePath) - if err != nil { return nil } + if err != nil { + return nil + } lines := strings.Split(strings.TrimSpace(string(data)), "\n") entries := make([]logEntry, 0, len(lines)) for _, line := range lines { - if line == "" { continue } + if line == "" { + continue + } var e logEntry - if err := json.Unmarshal([]byte(line), &e); err != nil { continue } - if e.Timestamp == "" { e.Timestamp = time.Now().Format(time.RFC3339) } - if e.Model == "" { e.Model = "unknown" } - if e.Tier == "" { e.Tier = "UNKNOWN" } - if e.BaselineCost == 0 { e.BaselineCost = e.Cost } + if err := json.Unmarshal([]byte(line), &e); err != nil { + continue + } + if e.Timestamp == "" { + e.Timestamp = time.Now().Format(time.RFC3339) + } + if e.Model == "" { + e.Model = "unknown" + } + if e.Tier == "" { + e.Tier = "UNKNOWN" + } + if e.BaselineCost == 0 { + e.BaselineCost = e.Cost + } entries = append(entries, e) } return entries } func getLogFiles() []string { - dir := logger.LogDir() + return getLogFilesAt(logger.LogDir()) +} + +func getLogFilesAt(dir string) []string { dirEntries, err := os.ReadDir(dir) - if err != nil { return nil } + if err != nil { + return nil + } var files []string for _, de := range dirEntries { name := de.Name() @@ -109,14 +128,22 @@ func aggregateDay(date string, entries []logEntry) DayStats { var totalLatency int64 var totalCost, totalBaselineCost float64 for _, e := range entries { - ts := byTier[e.Tier]; ts.Count++; ts.Cost += e.Cost; byTier[e.Tier] = ts - ms := byModel[e.Model]; ms.Count++; ms.Cost += e.Cost; byModel[e.Model] = ms + ts := byTier[e.Tier] + ts.Count++ + ts.Cost += e.Cost + byTier[e.Tier] = ts + ms := byModel[e.Model] + ms.Count++ + ms.Cost += e.Cost + byModel[e.Model] = ms totalLatency += e.LatencyMs totalCost += e.Cost totalBaselineCost += e.BaselineCost } avgLat := 0.0 - if len(entries) > 0 { avgLat = float64(totalLatency) / float64(len(entries)) } + if len(entries) > 0 { + avgLat = float64(totalLatency) / float64(len(entries)) + } return DayStats{ Date: date, TotalRequests: len(entries), TotalCost: totalCost, TotalBaselineCost: totalBaselineCost, TotalSavings: totalBaselineCost - totalCost, @@ -126,10 +153,20 @@ func aggregateDay(date string, entries []logEntry) DayStats { // GetStats reads log files and returns aggregated statistics for the given number of days. func GetStats(days int) AggregatedStats { - if days <= 0 { days = 7 } - logFiles := getLogFiles() - if len(logFiles) > days { logFiles = logFiles[:days] } - dir := logger.LogDir() + return getStats(days, logger.LogDir()) +} + +func getStats(days int, dir string) AggregatedStats { + if days <= 0 { + days = 7 + } + if days > 30 { + days = 30 + } + logFiles := getLogFilesAt(dir) + if len(logFiles) > days { + logFiles = logFiles[:days] + } var dailyBreakdown []DayStats allByTier := make(map[string]TierStats) allByModel := make(map[string]ModelStats) @@ -138,7 +175,9 @@ func GetStats(days int) AggregatedStats { for _, file := range logFiles { date := strings.TrimSuffix(strings.TrimPrefix(file, "usage-"), ".jsonl") entries := parseLogFile(filepath.Join(dir, file)) - if len(entries) == 0 { continue } + if len(entries) == 0 { + continue + } day := aggregateDay(date, entries) dailyBreakdown = append(dailyBreakdown, day) totalRequests += day.TotalRequests @@ -146,23 +185,35 @@ func GetStats(days int) AggregatedStats { totalBaselineCost += day.TotalBaselineCost totalLatency += day.AvgLatencyMs * float64(day.TotalRequests) for tier, ts := range day.ByTier { - a := allByTier[tier]; a.Count += ts.Count; a.Cost += ts.Cost; allByTier[tier] = a + a := allByTier[tier] + a.Count += ts.Count + a.Cost += ts.Cost + allByTier[tier] = a } for model, ms := range day.ByModel { - a := allByModel[model]; a.Count += ms.Count; a.Cost += ms.Cost; allByModel[model] = a + a := allByModel[model] + a.Count += ms.Count + a.Cost += ms.Cost + allByModel[model] = a } } for k, v := range allByTier { - if totalRequests > 0 { v.Percentage = float64(v.Count) / float64(totalRequests) * 100 } + if totalRequests > 0 { + v.Percentage = float64(v.Count) / float64(totalRequests) * 100 + } allByTier[k] = v } for k, v := range allByModel { - if totalRequests > 0 { v.Percentage = float64(v.Count) / float64(totalRequests) * 100 } + if totalRequests > 0 { + v.Percentage = float64(v.Count) / float64(totalRequests) * 100 + } allByModel[k] = v } totalSavings := totalBaselineCost - totalCost savingsPct := 0.0 - if totalBaselineCost > 0 { savingsPct = totalSavings / totalBaselineCost * 100 } + if totalBaselineCost > 0 { + savingsPct = totalSavings / totalBaselineCost * 100 + } avgLatency, avgCost := 0.0, 0.0 if totalRequests > 0 { avgLatency = totalLatency / float64(totalRequests) @@ -170,14 +221,18 @@ func GetStats(days int) AggregatedStats { } var entriesWithBaseline int for _, day := range dailyBreakdown { - if day.TotalBaselineCost != day.TotalCost { entriesWithBaseline += day.TotalRequests } + if day.TotalBaselineCost != day.TotalCost { + entriesWithBaseline += day.TotalRequests + } } // Reverse so oldest first. for i, j := 0, len(dailyBreakdown)-1; i < j; i, j = i+1, j-1 { dailyBreakdown[i], dailyBreakdown[j] = dailyBreakdown[j], dailyBreakdown[i] } period := "today" - if days != 1 { period = fmt.Sprintf("last %d days", days) } + if days != 1 { + period = fmt.Sprintf("last %d days", days) + } return AggregatedStats{ Period: period, TotalRequests: totalRequests, TotalCost: totalCost, TotalBaselineCost: totalBaselineCost, TotalSavings: totalSavings, @@ -218,7 +273,9 @@ func FormatStatsASCII(s AggregatedStats) string { for _, model := range sortedKeysByCount(s.ByModel) { ms := s.ByModel[model] name := model - if len(name) > 25 { name = name[:22] + "..." } + if len(name) > 25 { + name = name[:22] + "..." + } b.WriteString(fmt.Sprintf("| %-25s %4d $%7.4f %5.1f%%|\n", name, ms.Count, ms.Cost, ms.Percentage)) } b.WriteString(topBot + "\n") @@ -228,18 +285,28 @@ func FormatStatsASCII(s AggregatedStats) string { // FormatRecentLogs renders individual log entries as a per-request table. func FormatRecentLogs(days, limit int) string { - if days <= 0 { days = 1 } - if limit <= 0 { limit = 20 } + if days <= 0 { + days = 1 + } + if limit <= 0 { + limit = 20 + } logFiles := getLogFiles() - if len(logFiles) > days { logFiles = logFiles[:days] } + if len(logFiles) > days { + logFiles = logFiles[:days] + } dir := logger.LogDir() var all []logEntry for _, file := range logFiles { all = append(all, parseLogFile(filepath.Join(dir, file))...) } sort.Slice(all, func(i, j int) bool { return all[i].Timestamp > all[j].Timestamp }) - if len(all) > limit { all = all[:limit] } - if len(all) == 0 { return "No recent logs found.\n" } + if len(all) > limit { + all = all[:limit] + } + if len(all) == 0 { + return "No recent logs found.\n" + } var b strings.Builder b.WriteString(fmt.Sprintf("Recent Requests (last %d):\n", len(all))) b.WriteString("+-----+----------------------+----------+----------+--------+\n") @@ -247,7 +314,9 @@ func FormatRecentLogs(days, limit int) string { b.WriteString("+-----+----------------------+----------+----------+--------+\n") for i, e := range all { model := e.Model - if len(model) > 20 { model = model[:17] + "..." } + if len(model) > 20 { + model = model[:17] + "..." + } b.WriteString(fmt.Sprintf("| %3d | %-20s | %-8s | $%7.5f | %6d |\n", i+1, model, e.Tier, e.Cost, e.LatencyMs)) } b.WriteString("+-----+----------------------+----------+----------+--------+\n") @@ -259,7 +328,9 @@ func ClearStats() error { dir := logger.LogDir() entries, err := os.ReadDir(dir) if err != nil { - if os.IsNotExist(err) { return nil } + if os.IsNotExist(err) { + return nil + } return err } for _, de := range entries { @@ -272,27 +343,37 @@ func ClearStats() error { } func pad(n int) string { - if n <= 0 { return "" } + if n <= 0 { + return "" + } return strings.Repeat(" ", n) } func makeBar(pct float64, maxWidth int) string { filled := int(math.Round(pct / 100.0 * float64(maxWidth))) - if filled < 0 { filled = 0 } - if filled > maxWidth { filled = maxWidth } + if filled < 0 { + filled = 0 + } + if filled > maxWidth { + filled = maxWidth + } return strings.Repeat("#", filled) + strings.Repeat(".", maxWidth-filled) } func sortedKeys(m map[string]TierStats) []string { keys := make([]string, 0, len(m)) - for k := range m { keys = append(keys, k) } + for k := range m { + keys = append(keys, k) + } sort.Strings(keys) return keys } func sortedKeysByCount(m map[string]ModelStats) []string { keys := make([]string, 0, len(m)) - for k := range m { keys = append(keys, k) } + for k := range m { + keys = append(keys, k) + } sort.Slice(keys, func(i, j int) bool { return m[keys[i]].Count > m[keys[j]].Count }) return keys } diff --git a/stats/stats_test.go b/stats/stats_test.go new file mode 100644 index 0000000..f1a20b6 --- /dev/null +++ b/stats/stats_test.go @@ -0,0 +1,60 @@ +package stats + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestGetStatsBoundsEmptyPeriods(t *testing.T) { + dir := t.TempDir() + tests := []struct { + name string + days int + period string + }{ + {"negative defaults to week", -5, "last 7 days"}, + {"zero defaults to week", 0, "last 7 days"}, + {"one day", 1, "today"}, + {"month", 30, "last 30 days"}, + {"over a month", 31, "last 30 days"}, + {"maximum integer", int(^uint(0) >> 1), "last 30 days"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := getStats(tt.days, dir) + if got.Period != tt.period { + t.Errorf("Period = %q, want %q", got.Period, tt.period) + } + if got.TotalRequests != 0 || got.TotalCost != 0 || len(got.DailyBreakdown) != 0 { + t.Fatalf("empty log directory produced usage: %+v", got) + } + }) + } +} + +func TestGetStatsReadsAtMostThirtyNewestLogFiles(t *testing.T) { + dir := t.TempDir() + first := time.Date(2026, time.August, 1, 0, 0, 0, 0, time.UTC) + for i := 0; i < 35; i++ { + date := first.AddDate(0, 0, i).Format("2006-01-02") + entry := `{"timestamp":"` + date + `T12:00:00Z","model":"test/model","tier":"SIMPLE","cost":0.25,"baselineCost":1,"latencyMs":100}` + "\n" + if err := os.WriteFile(filepath.Join(dir, "usage-"+date+".jsonl"), []byte(entry), 0o600); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(dir, "unrelated.jsonl"), []byte("invalid log"), 0o600); err != nil { + t.Fatal(err) + } + got := getStats(365, dir) + if got.Period != "last 30 days" || got.TotalRequests != 30 || len(got.DailyBreakdown) != 30 { + t.Fatalf("GetStats exceeded the 30-file window: period=%q requests=%d days=%d", got.Period, got.TotalRequests, len(got.DailyBreakdown)) + } + if got.DailyBreakdown[0].Date != first.AddDate(0, 0, 5).Format("2006-01-02") || got.DailyBreakdown[29].Date != first.AddDate(0, 0, 34).Format("2006-01-02") { + t.Errorf("unexpected oldest/newest dates: %q / %q", got.DailyBreakdown[0].Date, got.DailyBreakdown[29].Date) + } + if got.TotalCost != 7.5 || got.TotalSavings != 22.5 || got.AvgLatencyMs != 100 { + t.Errorf("aggregates included data outside the selected window: cost=%v savings=%v latency=%v", got.TotalCost, got.TotalSavings, got.AvgLatencyMs) + } +} From 77f41cd4a3a23570982f98146d2e7d16a5f29b95 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sat, 12 Sep 2026 12:27:08 +0700 Subject: [PATCH 2/8] fix: retain unique IDs and tool declarations during recovery --- proxy/proxy.go | 4 ++-- proxy/request.go | 30 ++++++++++++++++++++++++++++-- proxy/spending_test.go | 12 ++++++++++++ 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/proxy/proxy.go b/proxy/proxy.go index 132075c..fe8fcd7 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -772,10 +772,10 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { // If model output formatted tool calls in plain text content, recover them. if msg, ok := choice["message"].(map[string]interface{}); ok { tc, _ := msg["tool_calls"].([]interface{}) - if len(tc) == 0 { + if len(tc) == 0 && strings.TrimSpace(string(req.Extra["tool_choice"])) != `"none"` { contentStr, _ := msg["content"].(string) contentStr = stripThinking(contentStr) - if recovered, cleaned := recoverToolCallsWithProse(contentStr); len(recovered) > 0 && len(req.Tools) > 0 { + if recovered, cleaned := recoverToolCallsWithProse(contentStr, req.Tools); len(recovered) > 0 && len(req.Tools) > 0 { msg["content"] = cleaned recList := make([]interface{}, len(recovered)) for idx, r := range recovered { diff --git a/proxy/request.go b/proxy/request.go index dc37752..9ffc937 100644 --- a/proxy/request.go +++ b/proxy/request.go @@ -2,6 +2,7 @@ package proxy import ( "encoding/json" + "fmt" "net/url" "os" "strings" @@ -84,14 +85,39 @@ func forwardToolCallProse() bool { } // Remove only syntax which actually recovered into a tool call. -func recoverToolCallsWithProse(content string) ([]map[string]interface{}, string) { +func recoverToolCallsWithProse(content string, tools json.RawMessage) ([]map[string]interface{}, string) { + var definitions []struct { + Type string `json:"type"` + Function struct { + Name string `json:"name"` + } `json:"function"` + } + if json.Unmarshal(tools, &definitions) != nil { + return nil, content + } + allowed := make(map[string]bool) + for _, def := range definitions { + if def.Type == "function" && def.Function.Name != "" { + allowed[def.Function.Name] = true + } + } var calls []map[string]interface{} clean := func(match string) string { found := recoverStructuredToolCalls(match, "") if len(found) == 0 { return match } - calls = append(calls, found...) + for _, call := range found { + function, _ := call["function"].(map[string]interface{}) + name, _ := function["name"].(string) + if !allowed[name] { + return match + } + } + for _, call := range found { + call["id"] = fmt.Sprintf("call_recovered_%d", len(calls)) + calls = append(calls, call) + } return "" } cleaned := jsonCodeBlockRegex.ReplaceAllStringFunc(content, clean) diff --git a/proxy/spending_test.go b/proxy/spending_test.go index 6f68028..6452ec2 100644 --- a/proxy/spending_test.go +++ b/proxy/spending_test.go @@ -41,3 +41,15 @@ func TestCallerCredentialsDoNotShareResponseCache(t *testing.T) { t.Fatalf("upstream calls=%d", calls.Load()) } } + +func TestRecoveredCallsHaveUniqueIDsAndDeclaredNames(t *testing.T) { + tools := []byte(`[{"type":"function","function":{"name":"read_file"}}]`) + input := "First. call:read_file({}) Then. call:read_file({}) call:undeclared({})" + calls, prose := recoverToolCallsWithProse(input, tools) + if len(calls) != 2 || calls[0]["id"] == calls[1]["id"] { + t.Fatalf("calls=%v", calls) + } + if !strings.Contains(prose, "call:undeclared({})") || strings.Contains(prose, "call:read_file") { + t.Fatalf("prose=%q", prose) + } +} From d6e633b28ba029520fafa44a2964493fee485a7e Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sat, 12 Sep 2026 12:29:40 +0700 Subject: [PATCH 3/8] fix: address review of optional tools and gateway origin --- internal/requestkey/normalize.go | 8 ++++---- proxy/proxy.go | 4 ++-- proxy/request.go | 7 ++++++- proxy/spending_test.go | 20 ++++++++++++++++++++ 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/internal/requestkey/normalize.go b/internal/requestkey/normalize.go index dce178f..fbc4728 100644 --- a/internal/requestkey/normalize.go +++ b/internal/requestkey/normalize.go @@ -64,12 +64,12 @@ func content(value any) any { if !ok { continue } - copy := make(map[string]any, len(obj)) + blockCopy := make(map[string]any, len(obj)) for key, value := range obj { - copy[key] = value + blockCopy[key] = value } - copy["text"] = timestamp.ReplaceAllString(text, "") - out[i] = copy + blockCopy["text"] = timestamp.ReplaceAllString(text, "") + out[i] = blockCopy break } return out diff --git a/proxy/proxy.go b/proxy/proxy.go index fe8fcd7..3682593 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -320,7 +320,7 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { Config: s.routingConfig, ModelPricing: s.modelPricing, RoutingProfile: routingProfile, - HasTools: len(req.Tools) > 0, + HasTools: requestHasTools(req.Tools), }) if err != nil { http.Error(w, "Routing error: "+err.Error(), http.StatusInternalServerError) @@ -775,7 +775,7 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { if len(tc) == 0 && strings.TrimSpace(string(req.Extra["tool_choice"])) != `"none"` { contentStr, _ := msg["content"].(string) contentStr = stripThinking(contentStr) - if recovered, cleaned := recoverToolCallsWithProse(contentStr, req.Tools); len(recovered) > 0 && len(req.Tools) > 0 { + if recovered, cleaned := recoverToolCallsWithProse(contentStr, req.Tools); len(recovered) > 0 && requestHasTools(req.Tools) { msg["content"] = cleaned recList := make([]interface{}, len(recovered)) for idx, r := range recovered { diff --git a/proxy/request.go b/proxy/request.go index 9ffc937..67bf0a2 100644 --- a/proxy/request.go +++ b/proxy/request.go @@ -127,7 +127,7 @@ func recoverToolCallsWithProse(content string, tools json.RawMessage) ([]map[str func gatewayOrigin(base string) string { u, err := url.Parse(base) - if err != nil || u.Host == "" { + if err != nil || u.Host == "" || u.Scheme == "" { return "" } return u.Scheme + "://" + u.Host @@ -140,3 +140,8 @@ func (s *Server) writeUsage(entry logger.UsageEntry) { } logger.LogUsage(entry) } + +func requestHasTools(raw json.RawMessage) bool { + var tools []json.RawMessage + return json.Unmarshal(raw, &tools) == nil && len(tools) > 0 +} diff --git a/proxy/spending_test.go b/proxy/spending_test.go index 6452ec2..a34f471 100644 --- a/proxy/spending_test.go +++ b/proxy/spending_test.go @@ -53,3 +53,23 @@ func TestRecoveredCallsHaveUniqueIDsAndDeclaredNames(t *testing.T) { t.Fatalf("prose=%q", prose) } } + +func TestToolAndGatewayEdgeCases(t *testing.T) { + for _, raw := range []string{"null", " null ", "[]", "", `{"name":"not-an-array"}`} { + if requestHasTools([]byte(raw)) { + t.Errorf("has tools for %q", raw) + } + calls, _ := recoverToolCallsWithProse("call:read_file({})", []byte(raw)) + if len(calls) != 0 { + t.Errorf("recovered undeclared calls for %q", raw) + } + } + for _, input := range []string{"//localhost:8080", "localhost", ""} { + if origin := gatewayOrigin(input); origin != "" { + t.Errorf("origin(%q)=%q", input, origin) + } + } + if got := gatewayOrigin("https://example.com/v1?not=public"); got != "https://example.com" { + t.Fatalf("origin=%q", got) + } +} From c6c9902095cd484164b512a2b4ac72134770fb11 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sat, 12 Sep 2026 12:31:18 +0700 Subject: [PATCH 4/8] fix: preserve wrapped default HTTP transports --- proxy/proxy.go | 6 +++++- proxy/spending_test.go | 13 +++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/proxy/proxy.go b/proxy/proxy.go index 3682593..1b55b63 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -97,6 +97,10 @@ func New(cfg Config) *Server { rc = *cfg.RoutingConfig } + transport := http.DefaultTransport + if standard, ok := transport.(*http.Transport); ok { + transport = standard.Clone() + } sc := cfg.SpendControl var spendErr error if sc == nil { @@ -108,7 +112,7 @@ func New(cfg Config) *Server { modelPricing: models.BuildPricingMap(), httpClient: &http.Client{ Timeout: 5 * time.Minute, - Transport: http.DefaultTransport.(*http.Transport).Clone(), + Transport: transport, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }, }, dedup: dedup.New(), diff --git a/proxy/spending_test.go b/proxy/spending_test.go index a34f471..7ec2927 100644 --- a/proxy/spending_test.go +++ b/proxy/spending_test.go @@ -73,3 +73,16 @@ func TestToolAndGatewayEdgeCases(t *testing.T) { t.Fatalf("origin=%q", got) } } + +type testWrappedTransport struct{ http.RoundTripper } + +func TestNewAcceptsWrappedDefaultTransport(t *testing.T) { + original := http.DefaultTransport + wrapper := &testWrappedTransport{original} + http.DefaultTransport = wrapper + t.Cleanup(func() { http.DefaultTransport = original }) + srv, _ := syncTestServer(t, func(w http.ResponseWriter, r *http.Request) { syncTestOK(w) }, nil) + if srv.httpClient.Transport != wrapper { + t.Fatal("wrapped transport was not preserved") + } +} From 6dd5f97cae19cf456b5031568b8e50ef3f3557e1 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sat, 12 Sep 2026 12:43:39 +0700 Subject: [PATCH 5/8] fix: address upstream sync accounting and cache review --- .github/workflows/ci.yml | 4 +- .github/workflows/integration.yml | 2 +- UPSTREAM_SYNC.md | 8 +- cache/cache.go | 10 +-- cache/requestkey_test.go | 45 +++++++++- dedup/dedup.go | 8 +- dedup/requestkey_test.go | 45 +++++++++- docs/configuration.md | 5 +- internal/requestkey/normalize.go | 76 ---------------- internal/requestkey/normalize_test.go | 112 ----------------------- models/catalog_test.go | 62 +++++++++++++ models/models.go | 24 ++--- proxy/prose.go | 13 ++- proxy/proxy.go | 37 +++++--- proxy/request.go | 9 ++ proxy/spending.go | 5 +- proxy/spending_review_test.go | 124 ++++++++++++++++++++++++++ proxy/spending_test.go | 63 +++++++++++++ spendcontrol/spendcontrol.go | 2 +- 19 files changed, 407 insertions(+), 247 deletions(-) delete mode 100644 internal/requestkey/normalize.go delete mode 100644 internal/requestkey/normalize_test.go create mode 100644 proxy/spending_review_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44fd318..9c1c927 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4.2.2 - - uses: actions/setup-go@v5 + - uses: actions/setup-go@v6 with: go-version-file: go.mod cache: true @@ -32,7 +32,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4.2.2 - - uses: actions/setup-go@v5 + - uses: actions/setup-go@v6 with: go-version-file: go.mod cache: true diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index b2de60d..5c21d92 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 + - uses: actions/setup-go@v6 with: go-version-file: go.mod cache: true diff --git a/UPSTREAM_SYNC.md b/UPSTREAM_SYNC.md index 5da0b3c..5846b59 100644 --- a/UPSTREAM_SYNC.md +++ b/UPSTREAM_SYNC.md @@ -43,12 +43,12 @@ update, not a claim that every BlockRun product feature is implemented. | --- | --- | | v0.12.248 assistant/tool prose | Preserve assistant prose with native calls and text-recovered calls; strip tagged thinking, including split SSE tags. `DOSROUTER_TOOL_CALL_PROSE=off` restores legacy suppression. Recover syntax only when tools are supplied. | | v0.12.252 tool-pair safety | Preserve `tool_calls`, `tool_call_id`, names and all provider extension fields during request rewriting. Avoid compressing protocol-bearing or multimodal messages. DOSRouter has no upstream-style message truncation path. | -| v0.12.254-256 cancellation/cache | Keep Go request contexts through chat/image requests, stop fallback after disconnect, reject incomplete bodies, normalize injected first-text timestamps without merging distinct user data. | +| v0.12.254-256 cancellation/cache | Keep Go request contexts through chat/image requests, stop fallback after disconnect, reject incomplete bodies, preserve caller-controlled timestamp content in cache keys; normalize JSON object key order without conflating arrays. | | v0.12.257-278 models/routing | Align chat catalog metadata and all four profile chains with router-core `5ee7c23c993013a8052588191569db5cf7fb793c`; retain DOS aliases and exact explicit pins. Retire dead free defaults, fix capability claims and prices. | | v0.12.263/269/274 spend safety | Atomic in-flight reservations for direct and routed chat, including fallback attempts; pending spend counts in rolling/session caps. Persist snapshots serially with atomic file replacement. Invalid state/cost fails closed. Each Server uses one controller; embedded callers may inject a shared controller explicitly. | -| v0.12.267 ambiguous sends | Do not repeat a chat send or switch models after an ambiguous transport failure. Explicit retryable HTTP rejection statuses retain backoff/retry. DOSRouter does not yet sign x402 payments. | +| v0.12.267 ambiguous sends | Do not repeat a chat send or switch models after an ambiguous transport failure. Each reservation authorizes one HTTP send. Status retries are disabled; model fallback obtains a separate reservation and ambiguous server errors retain their estimate. DOSRouter does not yet sign x402 payments. | | v0.12.271-275 accounting/health | Prefer settled gateway cost headers, otherwise actual token usage, then explicitly labelled estimates. Capture gateway request IDs in usage logs; report the configured gateway origin in health. Image cost reads headers/body. Reject unknown-priced images when amount limits are configured. | -| v0.12.272 credential transport | Refuse upstream redirects, avoid shared internal caching across caller-supplied bearer credentials, and mark authenticated responses `no-store`. | +| v0.12.272 credential transport | Refuse upstream redirects, avoid shared internal caching across caller-supplied bearer credentials (including a configured upstream key), and mark authenticated responses `no-store`. | | Stats day windows | Go already defaulted nonpositive windows safely; cap aggregate reporting to 30 days and test using isolated log directories. | | Validation adaptation | Replace stale TypeScript/npm CI and missing Docker scanner targets with Go build, vet, race tests and govulncheck. Preserve job/workflow names and automatic CodeQL; prefer patched Go 1.26.6 via the toolchain directive. Dependabot follows Go modules and Actions. | @@ -58,6 +58,8 @@ metadata reflects upstream source, not independently probed DOS providers. Gemini 3.6/3.8 Flash's $0.75/$3.75 promotional rates end on 2027-01-01, when upstream documents $1.50/$7.50; automated repricing is not implemented. +**Intentional divergence:** The upstream timestamp-stripping optimization is not enabled: a standalone server cannot distinguish injected prefixes from client-authored content. Cache/dedup keys preserve both string and first text-block timestamps until trusted injection provenance exists. + **Already satisfied:** `/v1/models` lists active catalog entries; chat and image requests derive their context from the client; full health performs no balance RPC; nonpositive stats/log windows have safe defaults. diff --git a/cache/cache.go b/cache/cache.go index d94553d..97ac39f 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -1,7 +1,7 @@ // Package cache provides a TTL + LRU response cache for LLM completions. // Cache keys are derived from canonicalized request JSON, skipping -// non-deterministic fields (stream, user, request_id) and stripping -// timestamp prefixes from message content. +// non-deterministic fields (stream, user, request_id) while preserving +// all message content, including client-supplied timestamps. package cache import ( @@ -12,8 +12,6 @@ import ( "fmt" "sync" "time" - - "github.com/DOS/DOSRouter/internal/requestkey" ) const ( @@ -270,7 +268,7 @@ func (c *Cache) removeLocked(elem *list.Element) { } // CacheKey returns a hex-encoded SHA-256 hash of the canonicalized request -// JSON, omitting non-deterministic fields and stripping timestamps. +// JSON, omitting non-deterministic fields while preserving all content. func CacheKey(body []byte) (string, error) { var raw map[string]interface{} if err := json.Unmarshal(body, &raw); err != nil { @@ -282,7 +280,7 @@ func CacheKey(body []byte) (string, error) { delete(raw, f) } - canonical := canonicalize(requestkey.Normalize(raw)) + canonical := canonicalize(raw) encoded, err := json.Marshal(canonical) if err != nil { return "", fmt.Errorf("cache: marshal error: %w", err) diff --git a/cache/requestkey_test.go b/cache/requestkey_test.go index 46f5d2e..a99ba5e 100644 --- a/cache/requestkey_test.go +++ b/cache/requestkey_test.go @@ -8,23 +8,24 @@ import ( const multimodalRequest = `{"model":"test/model","messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"https://example.test/a.png","detail":"high"}},{"type":"text","text":"[Sat 2026-09-12 09:00 ICT] Describe this image"},{"type":"text","text":"[Sat 2026-09-12 10:00 ICT] Event"}]},{"role":"assistant","content":null,"tool_calls":[{"type":"function","function":{"name":"record_event","arguments":"[Sat 2026-09-12 11:00 ICT] Event"}}]}]}` -func TestCacheReusesMultimodalResponseAcrossInjectedTimestamps(t *testing.T) { +func TestCacheSeparatesMultimodalResponsesByClientTimestamp(t *testing.T) { first := []byte(multimodalRequest) original := bytes.Clone(first) second := []byte(strings.Replace(multimodalRequest, "09:00", "09:30", 1)) c := New() c.Set(first, Entry{Body: []byte("cached answer"), StatusCode: 200}) got, ok := c.Get(second, false) - if !ok || string(got.Body) != "cached answer" { - t.Fatalf("timestamp-only change missed cached response: hit=%v, body=%q", ok, got.Body) + if ok { + t.Fatalf("different client timestamp reused cached response: hit=%v, body=%q", ok, got.Body) } if !bytes.Equal(first, original) { - t.Error("cache key normalization mutated request bytes") + t.Error("cache key generation mutated request bytes") } } func TestCacheKeyPreservesRequestSemantics(t *testing.T) { tests := []struct{ name, before, after string }{ + {"first text timestamp", "09:00", "09:30"}, {"later text timestamp", "10:00", "10:30"}, {"tool arguments", "11:00", "11:30"}, {"image URL", "a.png", "b.png"}, @@ -76,3 +77,39 @@ func TestCacheKeyIgnoresObjectKeyOrder(t *testing.T) { t.Fatal("equivalent JSON objects had different cache keys") } } + +func TestCacheKeyPreservesTimestampContent(t *testing.T) { + tests := []struct { + name string + body string + }{ + {"user string", `{"messages":[{"role":"user","content":"[Sat 2026-09-12 09:00 ICT] Event"}]}`}, + {"system string", `{"messages":[{"role":"system","content":"[Sat 2026-09-12 09:00 ICT] Event"}]}`}, + {"assistant string", `{"messages":[{"role":"assistant","content":"[Sat 2026-09-12 09:00 ICT] Event"}]}`}, + {"first text block", `{"messages":[{"role":"user","content":[{"type":"text","text":"[Sat 2026-09-12 09:00 ICT] Event"}]}]}`}, + {"tool result", `{"messages":[{"role":"tool","tool_call_id":"call_1","content":"[Sat 2026-09-12 09:00 ICT] Event"}]}`}, + {"function result", `{"messages":[{"role":"function","name":"record_event","content":"[Sat 2026-09-12 09:00 ICT] Event"}]}`}, + {"metadata", `{"metadata":{"content":"[Sat 2026-09-12 09:00 ICT] Event"},"messages":[]}`}, + {"object tool arguments", `{"messages":[{"role":"assistant","content":null,"tool_calls":[{"type":"function","function":{"name":"record_event","arguments":{"content":"[Sat 2026-09-12 09:00 ICT] Event"}}}]}]}`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + original, err := CacheKey([]byte(tt.body)) + if err != nil { + t.Fatal(err) + } + for _, changed := range []string{ + strings.ReplaceAll(tt.body, "09:00", "09:30"), + strings.ReplaceAll(tt.body, "[Sat 2026-09-12 09:00 ICT] ", ""), + } { + key, err := CacheKey([]byte(changed)) + if err != nil { + t.Fatal(err) + } + if original == key { + t.Error("different client timestamp content shared a cache key") + } + } + }) + } +} diff --git a/dedup/dedup.go b/dedup/dedup.go index 9583a24..55bc1be 100644 --- a/dedup/dedup.go +++ b/dedup/dedup.go @@ -10,8 +10,6 @@ import ( "fmt" "sync" "time" - - "github.com/DOS/DOSRouter/internal/requestkey" ) const ( @@ -157,14 +155,14 @@ func (d *Deduplicator) Len() int { } // HashBody returns a hex-encoded SHA-256 hash of the canonicalized JSON body. -// Keys are sorted recursively and timestamp prefixes are stripped from string -// values. +// Object keys are sorted recursively while preserving all content and JSON +// value types, including client-supplied timestamps. func HashBody(body []byte) (string, error) { var raw interface{} if err := json.Unmarshal(body, &raw); err != nil { return "", fmt.Errorf("dedup: invalid JSON body: %w", err) } - canonical := canonicalize(requestkey.Normalize(raw)) + canonical := canonicalize(raw) encoded, err := json.Marshal(canonical) if err != nil { return "", fmt.Errorf("dedup: marshal error: %w", err) diff --git a/dedup/requestkey_test.go b/dedup/requestkey_test.go index 15c8130..ce4d36c 100644 --- a/dedup/requestkey_test.go +++ b/dedup/requestkey_test.go @@ -8,7 +8,7 @@ import ( const multimodalRequest = `{"model":"test/model","messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"https://example.test/a.png","detail":"high"}},{"type":"text","text":"[Sat 2026-09-12 09:00 ICT] Describe this image"},{"type":"text","text":"[Sat 2026-09-12 10:00 ICT] Event"}]},{"role":"assistant","content":null,"tool_calls":[{"type":"function","function":{"name":"record_event","arguments":"[Sat 2026-09-12 11:00 ICT] Event"}}]}]}` -func TestDedupReusesMultimodalResponseAcrossInjectedTimestamps(t *testing.T) { +func TestDedupSeparatesMultimodalResponsesByClientTimestamp(t *testing.T) { first := []byte(multimodalRequest) original := bytes.Clone(first) second := []byte(strings.Replace(multimodalRequest, "09:00", "09:30", 1)) @@ -22,16 +22,17 @@ func TestDedupReusesMultimodalResponseAcrossInjectedTimestamps(t *testing.T) { t.Fatalf("first request: hit=%v err=%v", hit, err) } got, hit, err := d.Do(second, call) - if err != nil || !hit || calls != 1 || string(got.Body) != "upstream answer" { - t.Fatalf("timestamp-only retry: hit=%v calls=%d body=%q err=%v", hit, calls, got.Body, err) + if err != nil || hit || calls != 2 || string(got.Body) != "upstream answer" { + t.Fatalf("different client timestamp: hit=%v calls=%d body=%q err=%v", hit, calls, got.Body, err) } if !bytes.Equal(first, original) { - t.Error("dedup normalization mutated request bytes") + t.Error("dedup key generation mutated request bytes") } } func TestHashBodyPreservesRequestSemantics(t *testing.T) { tests := []struct{ name, before, after string }{ + {"first text timestamp", "09:00", "09:30"}, {"later text timestamp", "10:00", "10:30"}, {"tool arguments", "11:00", "11:30"}, {"image URL", "a.png", "b.png"}, @@ -83,3 +84,39 @@ func TestHashBodyIgnoresObjectKeyOrder(t *testing.T) { t.Fatal("equivalent JSON objects had different dedup keys") } } + +func TestHashBodyPreservesTimestampContent(t *testing.T) { + tests := []struct { + name string + body string + }{ + {"user string", `{"messages":[{"role":"user","content":"[Sat 2026-09-12 09:00 ICT] Event"}]}`}, + {"system string", `{"messages":[{"role":"system","content":"[Sat 2026-09-12 09:00 ICT] Event"}]}`}, + {"assistant string", `{"messages":[{"role":"assistant","content":"[Sat 2026-09-12 09:00 ICT] Event"}]}`}, + {"first text block", `{"messages":[{"role":"user","content":[{"type":"text","text":"[Sat 2026-09-12 09:00 ICT] Event"}]}]}`}, + {"tool result", `{"messages":[{"role":"tool","tool_call_id":"call_1","content":"[Sat 2026-09-12 09:00 ICT] Event"}]}`}, + {"function result", `{"messages":[{"role":"function","name":"record_event","content":"[Sat 2026-09-12 09:00 ICT] Event"}]}`}, + {"metadata", `{"metadata":{"content":"[Sat 2026-09-12 09:00 ICT] Event"},"messages":[]}`}, + {"object tool arguments", `{"messages":[{"role":"assistant","content":null,"tool_calls":[{"type":"function","function":{"name":"record_event","arguments":{"content":"[Sat 2026-09-12 09:00 ICT] Event"}}}]}]}`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + original, err := HashBody([]byte(tt.body)) + if err != nil { + t.Fatal(err) + } + for _, changed := range []string{ + strings.ReplaceAll(tt.body, "09:00", "09:30"), + strings.ReplaceAll(tt.body, "[Sat 2026-09-12 09:00 ICT] ", ""), + } { + key, err := HashBody([]byte(changed)) + if err != nil { + t.Fatal(err) + } + if original == key { + t.Error("different client timestamp content shared a dedup key") + } + } + }) + } +} diff --git a/docs/configuration.md b/docs/configuration.md index 2636abf..ff7b6e3 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -252,8 +252,9 @@ state refuses paid dispatch rather than silently clearing limits. Direct and routed chat requests reserve estimated cost before dispatch, then record settled gateway headers or token-based cost when available. Concurrent pending reservations count against session, hourly and daily limits. Explicit upstream -rejections release a reservation; ambiguous transport failures conservatively -consume its estimate and are not retried. Unknown-priced models and image +client rejections release a reservation unless a settled charge is reported; +server/transport failures conservatively consume their estimate. Each +reservation permits one HTTP send; model fallbacks reserve separately. Unknown-priced models and image requests are refused when limits are configured. This is a local estimate-based control, not a provider-side USD guarantee or multi-process ledger. diff --git a/internal/requestkey/normalize.go b/internal/requestkey/normalize.go deleted file mode 100644 index fbc4728..0000000 --- a/internal/requestkey/normalize.go +++ /dev/null @@ -1,76 +0,0 @@ -// Package requestkey normalizes only timestamps injected into message content. -package requestkey - -import ( - "maps" - "regexp" -) - -var timestamp = regexp.MustCompile(`^\[\w{3}\s+\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}\s+\w+\]\s*`) - -// Normalize removes an injected prefix from root request messages' string -// content or first text block. Tool results, metadata, tool arguments and later -// text blocks are preserved, and the input request is never mutated. -func Normalize(value any) any { - request, ok := value.(map[string]any) - if !ok { - return value - } - messages, ok := request["messages"].([]any) - if !ok { - return value - } - - out := maps.Clone(request) - normalized := make([]any, len(messages)) - for i, item := range messages { - normalized[i] = item - message, ok := item.(map[string]any) - if !ok { - continue - } - role, _ := message["role"].(string) - if role == "tool" || role == "function" { - continue - } - messageContent, exists := message["content"] - if !exists { - continue - } - messageCopy := maps.Clone(message) - messageCopy["content"] = content(messageContent) - normalized[i] = messageCopy - } - out["messages"] = normalized - return out -} - -func content(value any) any { - if text, ok := value.(string); ok { - return timestamp.ReplaceAllString(text, "") - } - blocks, ok := value.([]any) - if !ok { - return value - } - out := make([]any, len(blocks)) - copy(out, blocks) - for i, block := range blocks { - obj, ok := block.(map[string]any) - if !ok || obj["type"] != "text" { - continue - } - text, ok := obj["text"].(string) - if !ok { - continue - } - blockCopy := make(map[string]any, len(obj)) - for key, value := range obj { - blockCopy[key] = value - } - blockCopy["text"] = timestamp.ReplaceAllString(text, "") - out[i] = blockCopy - break - } - return out -} diff --git a/internal/requestkey/normalize_test.go b/internal/requestkey/normalize_test.go deleted file mode 100644 index ccf6abe..0000000 --- a/internal/requestkey/normalize_test.go +++ /dev/null @@ -1,112 +0,0 @@ -package requestkey - -import ( - "encoding/json" - "reflect" - "testing" -) - -func TestNormalizeInjectedMessageTimestamps(t *testing.T) { - tests := []struct { - name string - input string - want string - }{ - { - name: "string message", - input: `{"messages":[{"role":"user","content":"[Sat 2026-09-12 09:00 ICT] Hello"}]}`, - want: `{"messages":[{"role":"user","content":"Hello"}]}`, - }, - { - name: "first text after image", - input: `{"messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"https://example.test/image.png","detail":"high"}},{"type":"text","text":"[Sat 2026-09-12 09:00 ICT] Describe this image"},{"type":"text","text":"[Sat 2026-09-12 10:00 ICT] The event happened then"}]}]}`, - want: `{"messages":[{"role":"user","content":[{"type":"image_url","image_url":{"url":"https://example.test/image.png","detail":"high"}},{"type":"text","text":"Describe this image"},{"type":"text","text":"[Sat 2026-09-12 10:00 ICT] The event happened then"}]}]}`, - }, - { - name: "later text is never stripped", - input: `{"messages":[{"content":[{"type":"text","text":"Keep the quoted timestamp"},{"type":"text","text":"[Sat 2026-09-12 10:00 ICT] Event"}]}]}`, - want: `{"messages":[{"content":[{"type":"text","text":"Keep the quoted timestamp"},{"type":"text","text":"[Sat 2026-09-12 10:00 ICT] Event"}]}]}`, - }, - { - name: "tool arguments are preserved", - input: `{"messages":[{"role":"assistant","content":null,"tool_calls":[{"type":"function","function":{"name":"record_event","arguments":"[Sat 2026-09-12 10:00 ICT] Event"}}]}]}`, - want: `{"messages":[{"role":"assistant","content":null,"tool_calls":[{"type":"function","function":{"name":"record_event","arguments":"[Sat 2026-09-12 10:00 ICT] Event"}}]}]}`, - }, - { - name: "embedded timestamp is preserved", - input: `{"messages":[{"content":"Quote [Sat 2026-09-12 10:00 ICT] Event"}]}`, - want: `{"messages":[{"content":"Quote [Sat 2026-09-12 10:00 ICT] Event"}]}`, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - var input, original, want any - for _, value := range []struct { - text string - target *any - }{ - {tt.input, &input}, {tt.input, &original}, {tt.want, &want}, - } { - if err := json.Unmarshal([]byte(value.text), value.target); err != nil { - t.Fatal(err) - } - } - got := Normalize(input) - if !reflect.DeepEqual(got, want) { - t.Errorf("Normalize() = %#v, want %#v", got, want) - } - if !reflect.DeepEqual(input, original) { - t.Error("Normalize mutated its input") - } - }) - } -} - -func TestNormalizePreservesNonMessageContent(t *testing.T) { - tests := []struct { - name string - input string - }{ - {"root content", `{"content":"[Sat 2026-09-12 09:00 ICT] Root data"}`}, - {"metadata content", `{"metadata":{"content":"[Sat 2026-09-12 09:00 ICT] Event"},"messages":[{"role":"user","content":"Hello"}]}`}, - {"nested message history", `{"metadata":{"messages":[{"role":"user","content":"[Sat 2026-09-12 09:00 ICT] Historical event"}]},"messages":[]}`}, - {"message metadata", `{"messages":[{"role":"user","content":"Hello","metadata":{"content":"[Sat 2026-09-12 09:00 ICT] Event"}}]}`}, - {"object tool arguments", `{"messages":[{"role":"assistant","content":null,"tool_calls":[{"type":"function","function":{"name":"record_event","arguments":{"content":"[Sat 2026-09-12 09:00 ICT] Event"}}}]}]}`}, - {"tool result string", `{"messages":[{"role":"tool","tool_call_id":"call_1","content":"[Sat 2026-09-12 09:00 ICT] Observed event"}]}`}, - {"tool result text block", `{"messages":[{"role":"tool","tool_call_id":"call_1","content":[{"type":"text","text":"[Sat 2026-09-12 09:00 ICT] Observed event"}]}]}`}, - {"legacy function result", `{"messages":[{"role":"function","name":"record_event","content":"[Sat 2026-09-12 09:00 ICT] Observed event"}]}`}, - {"non-request array", `[{"role":"user","content":"[Sat 2026-09-12 09:00 ICT] Data"}]`}, - {"empty text block array", `{"messages":[{"role":"user","content":[]}]}`}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - var input, original any - if err := json.Unmarshal([]byte(tt.input), &input); err != nil { - t.Fatal(err) - } - if err := json.Unmarshal([]byte(tt.input), &original); err != nil { - t.Fatal(err) - } - if got := Normalize(input); !reflect.DeepEqual(got, original) { - t.Errorf("Normalize changed content outside injected message prefixes: got %#v, want %#v", got, original) - } - if !reflect.DeepEqual(input, original) { - t.Error("Normalize mutated its input") - } - }) - } -} - -func TestNormalizePreservesSupportedMessageRoles(t *testing.T) { - var input any - if err := json.Unmarshal([]byte(`{"messages":[{"role":"system","content":"[Sat 2026-09-12 09:00 ICT] System"},{"role":"assistant","content":"[Sat 2026-09-12 09:00 ICT] Assistant"},{"role":"user","content":"[Sat 2026-09-12 09:00 ICT] User"}]}`), &input); err != nil { - t.Fatal(err) - } - var want any - if err := json.Unmarshal([]byte(`{"messages":[{"role":"system","content":"System"},{"role":"assistant","content":"Assistant"},{"role":"user","content":"User"}]}`), &want); err != nil { - t.Fatal(err) - } - if got := Normalize(input); !reflect.DeepEqual(got, want) { - t.Errorf("injected timestamps on supported roles were not stripped: got %#v, want %#v", got, want) - } -} diff --git a/models/catalog_test.go b/models/catalog_test.go index 60c5ff4..0bdaf2e 100644 --- a/models/catalog_test.go +++ b/models/catalog_test.go @@ -147,3 +147,65 @@ func TestSyncedVisionAndExplicitAliases(t *testing.T) { } } } + +func TestRetiredFreeShorthandsResolveActiveSuccessor(t *testing.T) { + aliases := []string{ + "maverick", "mistral-large", "mistral-large-3-675b", "mistral-nemotron", + "qwen3-122b", "qwen3-next-80b", "qwen3.5-122b", + "seed-oss", "seed-oss-36b", "step-flash", "step-3.7-flash", + } + for _, alias := range aliases { + for _, prefix := range []string{"", "dosrouter/", "blockrun/", "openai/"} { + t.Run(prefix+alias, func(t *testing.T) { + resolved := ResolveModelAlias(prefix + alias) + if resolved != "free/nemotron-3.5-lightning" { + t.Fatalf("resolved = %q, want active free successor", resolved) + } + model := GetModel(resolved) + if model == nil || model.Deprecated || model.InputPrice != 0 || model.OutputPrice != 0 { + t.Errorf("free shorthand %q resolved to unavailable or paid metadata: %+v", alias, model) + } + }) + } + } + // The proxy resolves aliases once and does not interpret FallbackModel. + // A future free shorthand must therefore target an active row immediately. + for alias, target := range ModelAliases { + if strings.Contains(alias, "/") || !strings.HasPrefix(target, "free/") { + continue + } + model := GetModel(target) + if model == nil || model.Deprecated { + t.Errorf("free shorthand %q still targets missing/retired model %q", alias, target) + } + } +} + +func TestRetiredAliasFixPreservesQualifiedAndPaidVersionPins(t *testing.T) { + pins := map[string]string{ + "free/step-3.7-flash": "free/step-3.7-flash", + "free/mistral-nemotron": "free/mistral-nemotron", + "free/qwen3.5-122b-a10b": "free/qwen3.5-122b-a10b", + "nvidia/glm-4.7": "free/glm-4.7", + "nvidia/llama-4-maverick": "free/llama-4-maverick", + "nvidia/mistral-nemotron": "free/mistral-nemotron", + "nvidia/nemotron-nano-12b-v2-vl": "free/nemotron-nano-12b-v2-vl", + "nvidia/nemotron-nano-9b-v2": "free/nemotron-nano-9b-v2", + "nvidia/qwen3-coder-480b": "free/qwen3-coder-480b", + "nvidia/qwen3-next-80b-a3b-instruct": "free/qwen3-next-80b-a3b-instruct", + "nvidia/qwen3-next-80b-a3b-thinking": "free/qwen3-next-80b-a3b-instruct", + "nvidia/qwen3.5-122b-a10b": "free/qwen3.5-122b-a10b", + "nvidia/seed-oss-36b": "free/seed-oss-36b", + "nvidia/step-3.7-flash": "free/step-3.7-flash", + "qwen/qwen3-coder-480b-a35b-instruct": "free/qwen3-coder-480b", + "minimax-m2.5": "minimax/minimax-m2.5", + "minimax/minimax-m2.5": "minimax/minimax-m2.5", + } + for pin, want := range pins { + for _, prefix := range []string{"", "dosrouter/", "blockrun/"} { + if got := ResolveModelAlias(prefix + pin); got != want { + t.Errorf("ResolveModelAlias(%q) = %q, want preserved pin %q", prefix+pin, got, want) + } + } + } +} diff --git a/models/models.go b/models/models.go index 30d75d5..926b218 100644 --- a/models/models.go +++ b/models/models.go @@ -38,6 +38,8 @@ type PromoDef struct { var ModelAliases = map[string]string{ // Upstream chat aliases plus DOS-compatible shorthand targets. // Bare Claude/Sonnet, o1, Gemini, Flash and paid DeepSeek Pro retain DOS behavior. + // Retired free shorthands resolve directly to an active free successor. + // Provider-qualified compatibility pins retain their existing targets. "anthropic/claude": "anthropic/claude-sonnet-5", "anthropic/claude-fable-5.0": "anthropic/claude-fable-5", "anthropic/claude-haiku-4": "anthropic/claude-haiku-4.5", @@ -161,7 +163,7 @@ var ModelAliases = map[string]string{ "llama-free": "free/llama-3.2-11b-vision", "llama-vision": "free/llama-3.2-11b-vision", "luna-pro": "openai/gpt-5.6-luna-pro", - "maverick": "free/llama-4-maverick", + "maverick": "free/nemotron-3.5-lightning", "mimo": "xiaomi/mimo-v2.5-pro", "mimo-v2.5": "xiaomi/mimo-v2.5", "mimo-v2.5-pro": "xiaomi/mimo-v2.5-pro", @@ -172,9 +174,9 @@ var ModelAliases = map[string]string{ "minimax-m2.7": "minimax/minimax-m2.7", "minimax-m3": "minimax/minimax-m3", "mistral-free": "free/nemotron-3.5-lightning", - "mistral-large": "free/mistral-large-3-675b", - "mistral-large-3-675b": "free/mistral-large-3-675b", - "mistral-nemotron": "free/mistral-nemotron", + "mistral-large": "free/nemotron-3.5-lightning", + "mistral-large-3-675b": "free/nemotron-3.5-lightning", + "mistral-nemotron": "free/nemotron-3.5-lightning", "mistral-small": "free/nemotron-3.5-lightning", "moonshot": "moonshot/kimi-k2.7", "nano": "openai/gpt-5.4-nano", @@ -252,20 +254,20 @@ var ModelAliases = map[string]string{ "qwen-thinking": "free/nemotron-3.5-lightning", "qwen-vision": "qwen/qwen3.8-flash", "qwen/qwen3-coder-480b-a35b-instruct": "free/qwen3-coder-480b", - "qwen3-122b": "free/qwen3.5-122b-a10b", + "qwen3-122b": "free/nemotron-3.5-lightning", "qwen3-7-max": "qwen/qwen3.7-max", "qwen3-8-flash": "qwen/qwen3.8-flash", "qwen3-next": "free/nemotron-3.5-lightning", - "qwen3-next-80b": "free/qwen3-next-80b-a3b-instruct", - "qwen3.5-122b": "free/qwen3.5-122b-a10b", + "qwen3-next-80b": "free/nemotron-3.5-lightning", + "qwen3.5-122b": "free/nemotron-3.5-lightning", "qwen3.7-flash": "qwen/qwen3.7-flash", "qwen3.7-max": "qwen/qwen3.7-max", "qwen3.7-plus": "qwen/qwen3.7-plus", "qwen3.8-flash": "qwen/qwen3.8-flash", "reasoner": "deepseek/deepseek-reasoner", "router": "auto", - "seed-oss": "free/seed-oss-36b", - "seed-oss-36b": "free/seed-oss-36b", + "seed-oss": "free/nemotron-3.5-lightning", + "seed-oss-36b": "free/nemotron-3.5-lightning", "sol-pro": "openai/gpt-5.6-sol-pro", "sonnet": "anthropic/claude-sonnet-5", "sonnet-4": "anthropic/claude-sonnet-4.6", @@ -276,8 +278,8 @@ var ModelAliases = map[string]string{ "sonnet-5": "anthropic/claude-sonnet-5", "sonnet-5-0": "anthropic/claude-sonnet-5", "sonnet-5.0": "anthropic/claude-sonnet-5", - "step-3.7-flash": "free/step-3.7-flash", - "step-flash": "free/step-3.7-flash", + "step-3.7-flash": "free/nemotron-3.5-lightning", + "step-flash": "free/nemotron-3.5-lightning", "tencent": "tencent/hy3", "terra-pro": "openai/gpt-5.6-terra-pro", "ultra-550b": "free/nemotron-3-ultra-550b", diff --git a/proxy/prose.go b/proxy/prose.go index a857ebe..3f61b6b 100644 --- a/proxy/prose.go +++ b/proxy/prose.go @@ -45,12 +45,17 @@ func (f *proseFilter) filter(text string, final bool) string { if match := thinkingTag.FindStringSubmatch(tag); match != nil { f.hidden = match[1] != "/" } else if strings.HasPrefix(tag, "<|") || strings.HasPrefix(tag, "<｜") { - lower := strings.ToLower(tag) - if strings.Contains(lower, "begin") { + token := strings.ToLower(strings.Trim(tag, "<>|｜")) + token = strings.ReplaceAll(token, "▁", "_") + switch token { + case "begin_of_thinking", "thinking": f.hidden = true - } - if strings.Contains(lower, "end") { + case "end_of_thinking": f.hidden = false + default: + if !f.hidden { + out.WriteString(tag) + } } } else if !f.hidden { out.WriteString(tag) diff --git a/proxy/proxy.go b/proxy/proxy.go index 1b55b63..bbc1fae 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -258,7 +258,7 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { } // Caller-specific credentials must never share an internal response cache. - cacheAllowed := s.config.UpstreamAPIKey != "" || r.Header.Get("Authorization") == "" + cacheAllowed := r.Header.Get("Authorization") == "" // --- Response cache check (non-streaming only) --- if !req.Stream && cacheAllowed { if entry, ok := s.cache.Get(body, false); ok { @@ -465,6 +465,7 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { if !ok { return } + spend = currentSpend // Per-model timeout (upstream v0.12.182): reasoning models get 3min for // cold-start first-token (DeepSeek V4 Pro / Claude opus thinking / GPT-5 // reasoning_effort=high can take 60-120s); non-reasoning get 60s. On @@ -479,7 +480,7 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { // disconnect still cancels the in-flight attempt. attemptCtx, cancelAttempt := context.WithCancel(r.Context()) timer := time.AfterFunc(perModelTimeout(tryModel), cancelAttempt) - tryResp, tryErr := retry.Do(attemptCtx, makeReqFor(tryBody), retry.WithClient(s.httpClient), retry.WithNetworkRetries(false)) + tryResp, tryErr := retry.Do(attemptCtx, makeReqFor(tryBody), retry.WithClient(s.httpClient), retry.WithNetworkRetries(false), retry.WithMaxRetries(0)) if tryResp != nil && tryResp.StatusCode >= 300 { tryErr = nil } @@ -496,9 +497,14 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { attempts = append(attempts, attemptResult{model: tryModel, reason: tryErr.Error()}) break } - // Provider returned an error status (4xx/5xx except 429 which retry handles) + // Settle this attempt before considering a separately reserved fallback. if tryResp.StatusCode >= 300 { - currentSpend.release() + currentSpend.header = tryResp.Header + if _, settled := settledCost(tryResp.Header); settled || tryResp.StatusCode >= 500 { + currentSpend.finish(nil) + } else { + currentSpend.release() + } errBody, _ := io.ReadAll(tryResp.Body) tryResp.Body.Close() timer.Stop() @@ -590,14 +596,11 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { if json.Unmarshal([]byte(line[6:]), &chunk) == nil { // Track usage tokens if u, ok := chunk["usage"].(map[string]interface{}); ok { - _, hasInput := u["prompt_tokens"] - _, hasOutput := u["completion_tokens"] - spend.usageKnown = hasInput && hasOutput - if pt, ok := u["prompt_tokens"].(float64); ok { - streamInputTok = int(pt) - } - if ct, ok := u["completion_tokens"].(float64); ok { - streamOutputTok = int(ct) + pt, inputOK := validTokenCount(u["prompt_tokens"]) + ct, outputOK := validTokenCount(u["completion_tokens"]) + if inputOK && outputOK { + spend.usageKnown = true + streamInputTok, streamOutputTok = pt, ct } } // Preserve tool-call prose by default (upstream v0.12.248). @@ -721,7 +724,8 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { if !allowed { return } - fbResp, fbErr := retry.Do(r.Context(), makeReqFor(fbBody), retry.WithClient(s.httpClient), retry.WithNetworkRetries(false)) + spend = nextSpend + fbResp, fbErr := retry.Do(r.Context(), makeReqFor(fbBody), retry.WithClient(s.httpClient), retry.WithNetworkRetries(false), retry.WithMaxRetries(0)) if fbResp != nil && fbResp.StatusCode >= 300 { fbErr = nil } @@ -737,7 +741,12 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { } defer fbResp.Body.Close() if fbResp.StatusCode >= 300 { - nextSpend.release() + nextSpend.header = fbResp.Header + if _, settled := settledCost(fbResp.Header); settled || fbResp.StatusCode >= 500 { + nextSpend.finish(nil) + } else { + nextSpend.release() + } http.Error(w, "Fallback request rejected", http.StatusBadGateway) return } diff --git a/proxy/request.go b/proxy/request.go index 67bf0a2..88ce3c8 100644 --- a/proxy/request.go +++ b/proxy/request.go @@ -3,6 +3,7 @@ package proxy import ( "encoding/json" "fmt" + "math" "net/url" "os" "strings" @@ -145,3 +146,11 @@ func requestHasTools(raw json.RawMessage) bool { var tools []json.RawMessage return json.Unmarshal(raw, &tools) == nil && len(tools) > 0 } + +func validTokenCount(value any) (int, bool) { + number, ok := value.(float64) + if !ok || number < 0 || math.IsNaN(number) || math.IsInf(number, 0) || math.Trunc(number) != number || number >= float64(int(^uint(0)>>1)) { + return 0, false + } + return int(number), true +} diff --git a/proxy/spending.go b/proxy/spending.go index d1b041d..fbf597b 100644 --- a/proxy/spending.go +++ b/proxy/spending.go @@ -19,6 +19,7 @@ type requestSpend struct { model string estimate, cost float64 input, output int + completions int header http.Header finished bool source string @@ -77,7 +78,7 @@ func (s *Server) reserveChat(w http.ResponseWriter, req chatRequest, body []byte json.NewEncoder(w).Encode(map[string]any{"error": check.Reason, "blockedBy": check.BlockedBy, "remaining": check.Remaining}) return nil, false } - return &requestSpend{server: s, id: id, model: model, estimate: estimate, cost: estimate, source: "estimate"}, true + return &requestSpend{server: s, id: id, model: model, estimate: estimate, cost: estimate, source: "estimate", completions: n}, true } func (sp *requestSpend) readUsage(body []byte) { @@ -107,7 +108,7 @@ func (sp *requestSpend) finish(body []byte) { if p, ok := sp.server.modelPricing[sp.model]; ok { sp.cost = (float64(sp.input)*p.InputPrice + float64(sp.output)*p.OutputPrice) / 1_000_000 if p.FlatPrice != nil { - sp.cost = *p.FlatPrice + sp.cost = *p.FlatPrice * float64(sp.completions) } sp.source = "tokens" } diff --git a/proxy/spending_review_test.go b/proxy/spending_review_test.go new file mode 100644 index 0000000..f2a6473 --- /dev/null +++ b/proxy/spending_review_test.go @@ -0,0 +1,124 @@ +package proxy + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/DOS/DOSRouter/router" + "github.com/DOS/DOSRouter/spendcontrol" +) + +func TestFlatPriceSettlementPreservesCompletionCount(t *testing.T) { + const flatPrice = 0.125 + for _, tc := range []struct { + name, n, usage, gateway string + reserved, settled float64 + }{ + {"default completion", "", `{"usage":{"prompt_tokens":10,"completion_tokens":5}}`, "", flatPrice, flatPrice}, + {"multiple completions", `,"n":3`, `{"usage":{"prompt_tokens":10,"completion_tokens":15}}`, "", 3 * flatPrice, 3 * flatPrice}, + {"zero tokens", `,"n":3`, `{"usage":{"prompt_tokens":0,"completion_tokens":0}}`, "", 3 * flatPrice, 3 * flatPrice}, + {"missing usage", `,"n":3`, `{}`, "", 3 * flatPrice, 3 * flatPrice}, + {"gateway total is authoritative", `,"n":3`, `{"usage":{"prompt_tokens":10,"completion_tokens":15}}`, "0.25", 3 * flatPrice, 0.25}, + } { + t.Run(tc.name, func(t *testing.T) { + srv, sc := syncTestServer(t, func(http.ResponseWriter, *http.Request) { + t.Error("unit settlement test unexpectedly dispatched upstream") + }, nil) + flat := flatPrice + const model = "openai/gpt-4o-mini" + srv.modelPricing[model] = router.ModelPricing{FlatPrice: &flat} + if err := sc.SetLimit(spendcontrol.WindowSession, 1); err != nil { + t.Fatal(err) + } + payload := []byte(`{"model":"` + model + `","messages":[{"role":"user","content":"test"}]` + tc.n + `}`) + var req chatRequest + if err := json.Unmarshal(payload, &req); err != nil { + t.Fatal(err) + } + spend, allowed := srv.reserveChat(httptest.NewRecorder(), req, payload, model) + if !allowed || spend == nil { + t.Fatal("flat-price reservation rejected") + } + if got := sc.GetSpending()[spendcontrol.WindowSession]; got != tc.reserved { + t.Fatalf("reserved=%v, want %v", got, tc.reserved) + } + spend.header = make(http.Header) + spend.header.Set("X-DOS-Cost-USD", tc.gateway) + spend.finish([]byte(tc.usage)) + spend.finish(nil) + history := sc.GetHistory() + if len(history) != 1 || history[0].Amount != tc.settled { + t.Fatalf("settlement=%+v, want one charge of %v", history, tc.settled) + } + if got := sc.GetSpending()[spendcontrol.WindowSession]; got != tc.settled { + t.Fatalf("settlement left a pending reservation: total=%v, want %v", got, tc.settled) + } + }) + } +} + +func TestTruncatedEmptyTurnFallbackSettlesReservation(t *testing.T) { + var attempts atomic.Int32 + const firstModel = "openai/gpt-4o-mini" + const nextModel = "openai/gpt-4o" + srv, sc := syncTestServer(t, func(w http.ResponseWriter, r *http.Request) { + var req struct { + Model string `json:"model"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + } + switch attempts.Add(1) { + case 1: + if req.Model != firstModel { + t.Errorf("first model=%q, want %q", req.Model, firstModel) + } + w.Header().Set("X-DOS-Cost-USD", "0.125") + io.WriteString(w, `{"choices":[{"message":{"content":""},"finish_reason":"stop"}]}`) + case 2: + if req.Model != nextModel { + t.Errorf("fallback model=%q, want %q", req.Model, nextModel) + } + w.Header().Set("X-DOS-Cost-USD", "0.25") + w.Header().Set("Content-Length", "4096") + io.WriteString(w, `{"choices":[`) + default: + t.Error("truncated fallback was retried") + http.Error(w, "unexpected retry", http.StatusBadRequest) + } + }, func(cfg *Config) { + routing := router.DefaultRoutingConfig() + routing.Promotions = nil + for _, tiers := range []map[router.Tier]router.TierConfig{routing.Tiers, routing.AgenticTiers, routing.EcoTiers, routing.PremiumTiers} { + for _, tier := range []router.Tier{router.TierSimple, router.TierMedium, router.TierComplex, router.TierReasoning} { + if tiers != nil { + tiers[tier] = router.TierConfig{Primary: firstModel, Fallback: []string{nextModel}} + } + } + } + cfg.RoutingConfig = &routing + }) + if err := sc.SetLimit(spendcontrol.WindowSession, 1); err != nil { + t.Fatal(err) + } + result := syncTestChat(t, srv, `{"model":"auto","messages":[{"role":"user","content":"hello"}]}`) + if result.Code != http.StatusBadGateway || !strings.Contains(result.Body.String(), "Incomplete fallback response") { + t.Fatalf("status=%d, body=%q", result.Code, result.Body.String()) + } + if attempts.Load() != 2 { + t.Fatalf("upstream attempts=%d, want 2", attempts.Load()) + } + history := sc.GetHistory() + if len(history) != 2 || history[0].Model != firstModel || history[0].Amount != 0.125 || history[1].Model != nextModel || history[1].Amount != 0.25 { + t.Fatalf("incomplete fallback was not settled exactly once: %+v", history) + } + status := sc.GetStatus()[spendcontrol.WindowSession] + if status.Spent != 0.375 || status.Remaining != 0.625 { + t.Fatalf("fallback left a pending reservation: %+v", status) + } +} diff --git a/proxy/spending_test.go b/proxy/spending_test.go index 7ec2927..809ff3f 100644 --- a/proxy/spending_test.go +++ b/proxy/spending_test.go @@ -86,3 +86,66 @@ func TestNewAcceptsWrappedDefaultTransport(t *testing.T) { t.Fatal("wrapped transport was not preserved") } } + +func TestConfiguredGatewayDoesNotShareCallerCache(t *testing.T) { + var calls atomic.Int32 + srv, _ := syncTestServer(t, func(w http.ResponseWriter, r *http.Request) { calls.Add(1); syncTestOK(w) }, func(cfg *Config) { cfg.UpstreamAPIKey = "test-gateway-key" }) + for _, credential := range []string{"Bearer test-a", "Bearer test-b"} { + r := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"openai/gpt-4o-mini","messages":[{"role":"user","content":"same gateway"}]}`)) + r.Header.Set("Authorization", credential) + w := httptest.NewRecorder() + srv.handleChatCompletions(w, r) + if w.Code != 200 || w.Header().Get("X-DOSRouter-Cache") == "hit" { + t.Fatalf("status=%d headers=%v", w.Code, w.Header()) + } + } + if calls.Load() != 2 { + t.Fatalf("calls=%d", calls.Load()) + } +} + +func TestNonThinkingControlTokensKeepVisibleProse(t *testing.T) { + for _, text := range []string{"<|begin_of_text|>Visible answer", "<|end_of_turn|>Still visible"} { + if got := stripThinking(text); got != text { + t.Errorf("stripThinking(%q)=%q", text, got) + } + } + if got := stripThinking("<｜begin▁of▁thinking｜>private<｜end▁of▁thinking｜>Visible"); got != "Visible" { + t.Errorf("thinking=%q", got) + } +} + +func TestMalformedStreamingUsageRetainsEstimate(t *testing.T) { + for _, usage := range []string{`{"prompt_tokens":null,"completion_tokens":null}`, `{"prompt_tokens":"10","completion_tokens":5}`, `{"prompt_tokens":1.5,"completion_tokens":0}`, `{"prompt_tokens":-1,"completion_tokens":0}`} { + t.Run(usage, func(t *testing.T) { + var entry logger.UsageEntry + srv, _ := syncTestServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + io.WriteString(w, "data: {\"choices\":[],\"usage\":"+usage+"}\n\ndata: [DONE]\n\n") + }, func(cfg *Config) { cfg.UsageLogger = func(e logger.UsageEntry) { entry = e } }) + w := syncTestChat(t, srv, `{"model":"openai/gpt-4o-mini","stream":true,"messages":[{"role":"user","content":"stream"}]}`) + if w.Code != 200 || entry.CostSource != "estimate" || entry.Cost <= 0 { + t.Fatalf("status=%d entry=%+v", w.Code, entry) + } + }) + } +} + +func TestPaidStatusResponseDoesNotRepeatSameReservation(t *testing.T) { + for _, status := range []int{429, 502, 503, 504} { + var calls atomic.Int32 + srv, sc := syncTestServer(t, func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.Header().Set("X-Blockrun-Cost-USD", "0.002") + http.Error(w, "failed after processing", status) + }, nil) + w := syncTestChat(t, srv, `{"model":"openai/gpt-4o-mini","messages":[{"role":"user","content":"no replay"}]}`) + if w.Code != 502 || calls.Load() != 1 { + t.Fatalf("upstream=%d status=%d calls=%d", status, w.Code, calls.Load()) + } + history := sc.GetHistory() + if len(history) != 1 || history[0].Amount != 0.002 { + t.Fatalf("settled error history=%v", history) + } + } +} diff --git a/spendcontrol/spendcontrol.go b/spendcontrol/spendcontrol.go index c007b0e..68dd9d7 100644 --- a/spendcontrol/spendcontrol.go +++ b/spendcontrol/spendcontrol.go @@ -428,7 +428,7 @@ func (fs *FileSpendControlStorage) Save(state persistedState) error { if err != nil { return err } - defer os.Remove(file.Name()) + defer func() { _ = os.Remove(file.Name()) }() if _, err = file.Write(data); err == nil { err = file.Sync() } From 1210aad87835c59ca906dc463415d9317537889f Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sat, 12 Sep 2026 13:40:44 +0700 Subject: [PATCH 6/8] fix: retain observed usage when streaming is interrupted --- proxy/proxy.go | 1 + proxy/spending_review_test.go | 37 +++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/proxy/proxy.go b/proxy/proxy.go index bbc1fae..f55912c 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -599,6 +599,7 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { pt, inputOK := validTokenCount(u["prompt_tokens"]) ct, outputOK := validTokenCount(u["completion_tokens"]) if inputOK && outputOK { + spend.input, spend.output = pt, ct spend.usageKnown = true streamInputTok, streamOutputTok = pt, ct } diff --git a/proxy/spending_review_test.go b/proxy/spending_review_test.go index f2a6473..046fced 100644 --- a/proxy/spending_review_test.go +++ b/proxy/spending_review_test.go @@ -122,3 +122,40 @@ func TestTruncatedEmptyTurnFallbackSettlesReservation(t *testing.T) { t.Fatalf("fallback left a pending reservation: %+v", status) } } + +type closedStreamWriter struct{ *httptest.ResponseRecorder } + +func (w *closedStreamWriter) Write([]byte) (int, error) { return 0, io.ErrClosedPipe } + +func TestInterruptedStreamRetainsObservedUsage(t *testing.T) { + for _, failure := range []string{"client write", "upstream read"} { + t.Run(failure, func(t *testing.T) { + srv, sc := syncTestServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + if failure == "upstream read" { + w.Header().Set("Content-Length", "4096") + } + io.WriteString(w, "data: {\"choices\":[],\"usage\":{\"prompt_tokens\":1000,\"completion_tokens\":500}}\n\n") + }, nil) + if err := sc.SetLimit(spendcontrol.WindowSession, 1); err != nil { + t.Fatal(err) + } + const model = "openai/gpt-4o-mini" + srv.modelPricing[model] = router.ModelPricing{InputPrice: 1, OutputPrice: 2} + request := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"`+model+`","stream":true,"messages":[{"role":"user","content":"stream"}]}`)) + var writer http.ResponseWriter = httptest.NewRecorder() + if failure == "client write" { + writer = &closedStreamWriter{httptest.NewRecorder()} + } + srv.handleChatCompletions(writer, request) + history := sc.GetHistory() + const expectedCost = 0.002 + if len(history) != 1 || history[0].Amount != expectedCost { + t.Fatalf("observed usage lost on interruption: %+v, want one charge %v", history, expectedCost) + } + if spent := sc.GetSpending()[spendcontrol.WindowSession]; spent != expectedCost { + t.Fatalf("pending reservation remained: %v", spent) + } + }) + } +} From 24bd53fd9e48e014088d0bcbcf45004fb7b13b36 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sat, 12 Sep 2026 13:43:30 +0700 Subject: [PATCH 7/8] fix: stream ordinary comparison prose without buffering --- proxy/prose.go | 27 +++++++++++++++++++++++++++ proxy/spending_test.go | 19 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/proxy/prose.go b/proxy/prose.go index 3f61b6b..1e283e1 100644 --- a/proxy/prose.go +++ b/proxy/prose.go @@ -5,6 +5,8 @@ import ( "strings" ) +var thinkingTagPrefix = regexp.MustCompile(`(?i)^<\s*/?\s*(think(?:ing)?|thought|antthinking|antml:thinking)\b`) + var thinkingTag = regexp.MustCompile(`(?i)^<\s*(/?)\s*(think(?:ing)?|thought|antthinking|antml:thinking)\b[^>]*>$`) // proseFilter tracks split thinking tags across SSE chunks. Tagged reasoning @@ -32,6 +34,13 @@ func (f *proseFilter) filter(text string, final bool) string { f.pending = f.pending[start:] end := strings.IndexByte(f.pending, '>') if end < 0 { + if !couldBeThinkingTag(f.pending) { + if !f.hidden { + out.WriteByte('<') + } + f.pending = f.pending[1:] + continue + } if final { if !f.hidden { out.WriteString(f.pending) @@ -68,3 +77,21 @@ func stripThinking(content string) string { var filter proseFilter return filter.filter(content, true) } + +// couldBeThinkingTag buffers only prefixes that can still form a private tag. +// Ordinary comparisons must flow immediately without waiting for a closing >. +func couldBeThinkingTag(pending string) bool { + if thinkingTagPrefix.MatchString(pending) { + return true + } + candidate := strings.ToLower(strings.TrimSpace(strings.TrimPrefix(pending, "<"))) + candidate = strings.TrimSpace(strings.TrimPrefix(candidate, "/")) + candidate = strings.ReplaceAll(candidate, "｜", "|") + candidate = strings.ReplaceAll(candidate, "▁", "_") + for _, name := range []string{"think", "thinking", "thought", "antthinking", "antml:thinking", "|begin_of_thinking|", "|thinking|", "|end_of_thinking|"} { + if strings.HasPrefix(name, candidate) { + return true + } + } + return false +} diff --git a/proxy/spending_test.go b/proxy/spending_test.go index 809ff3f..461ea19 100644 --- a/proxy/spending_test.go +++ b/proxy/spending_test.go @@ -149,3 +149,22 @@ func TestPaidStatusResponseDoesNotRepeatSameReservation(t *testing.T) { } } } + +func TestStreamingComparisonDoesNotBufferOrdinaryProse(t *testing.T) { + var filter proseFilter + for _, chunk := range []string{"x < y", " and another ", "condition holds."} { + if got := filter.filter(chunk, false); got != chunk { + t.Fatalf("chunk=%q, got=%q before stream completion", chunk, got) + } + } + var split proseFilter + if got := split.filter("Visible privateFinal", false); got != "Final" { + t.Fatalf("closing tag output=%q", got) + } +} From 69ab4be661fc6c09daa5dead0d924bffedea19f4 Mon Sep 17 00:00:00 2001 From: JOY <5027251+JOY@users.noreply.github.com> Date: Sat, 12 Sep 2026 14:19:52 +0700 Subject: [PATCH 8/8] fix: finalize request accounting and preserve numeric cache inputs --- cache/cache.go | 8 ++- cache/requestkey_test.go | 38 +++++++++++++++ dedup/dedup.go | 8 ++- dedup/requestkey_test.go | 38 +++++++++++++++ docs/configuration.md | 9 ++-- proxy/proxy.go | 47 +++++++++++------- proxy/request.go | 19 +++++++- proxy/request_review_test.go | 92 +++++++++++++++++++++++++++++++++++ proxy/spending.go | 10 ++-- proxy/spending_review_test.go | 24 ++++++++- 10 files changed, 265 insertions(+), 28 deletions(-) create mode 100644 proxy/request_review_test.go diff --git a/cache/cache.go b/cache/cache.go index 97ac39f..ad6087c 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -5,6 +5,7 @@ package cache import ( + "bytes" "container/list" "crypto/sha256" "encoding/hex" @@ -270,8 +271,13 @@ func (c *Cache) removeLocked(elem *list.Element) { // CacheKey returns a hex-encoded SHA-256 hash of the canonicalized request // JSON, omitting non-deterministic fields while preserving all content. func CacheKey(body []byte) (string, error) { + if !json.Valid(body) { + return "", fmt.Errorf("cache: invalid JSON body") + } var raw map[string]interface{} - if err := json.Unmarshal(body, &raw); err != nil { + decoder := json.NewDecoder(bytes.NewReader(body)) + decoder.UseNumber() + if err := decoder.Decode(&raw); err != nil { return "", fmt.Errorf("cache: invalid JSON body: %w", err) } diff --git a/cache/requestkey_test.go b/cache/requestkey_test.go index a99ba5e..f72f188 100644 --- a/cache/requestkey_test.go +++ b/cache/requestkey_test.go @@ -113,3 +113,41 @@ func TestCacheKeyPreservesTimestampContent(t *testing.T) { }) } } + +func TestCacheKeyPreservesNumericLiterals(t *testing.T) { + for _, tt := range []struct { + name, first, second string + }{ + {"adjacent integers beyond float64 precision", "9007199254740992", "9007199254740993"}, + {"integer and decimal representation", "1", "1.0"}, + } { + t.Run(tt.name, func(t *testing.T) { + first, err := CacheKey([]byte(`{"payload":{"number":` + tt.first + `}}`)) + if err != nil { + t.Fatal(err) + } + second, err := CacheKey([]byte(`{"payload":{"number":` + tt.second + `}}`)) + if err != nil { + t.Fatal(err) + } + if first == second { + t.Fatalf("distinct numeric literals %s and %s shared a key", tt.first, tt.second) + } + }) + } +} + +func TestCacheKeyRejectsTrailingJSONValues(t *testing.T) { + for _, body := range []string{ + `{"model":"test/model"} {"model":"second"}`, + `{"model":"test/model"} 1`, + `{"model":"test/model"} garbage`, + } { + if key, err := CacheKey([]byte(body)); err == nil || key != "" { + t.Errorf("trailing JSON data accepted: key=%q err=%v", key, err) + } + } + if _, err := CacheKey([]byte("{\"model\":\"test/model\"} \n\t")); err != nil { + t.Errorf("valid trailing whitespace rejected: %v", err) + } +} diff --git a/dedup/dedup.go b/dedup/dedup.go index 55bc1be..75afce7 100644 --- a/dedup/dedup.go +++ b/dedup/dedup.go @@ -4,6 +4,7 @@ package dedup import ( + "bytes" "crypto/sha256" "encoding/hex" "encoding/json" @@ -158,8 +159,13 @@ func (d *Deduplicator) Len() int { // Object keys are sorted recursively while preserving all content and JSON // value types, including client-supplied timestamps. func HashBody(body []byte) (string, error) { + if !json.Valid(body) { + return "", fmt.Errorf("dedup: invalid JSON body") + } var raw interface{} - if err := json.Unmarshal(body, &raw); err != nil { + decoder := json.NewDecoder(bytes.NewReader(body)) + decoder.UseNumber() + if err := decoder.Decode(&raw); err != nil { return "", fmt.Errorf("dedup: invalid JSON body: %w", err) } canonical := canonicalize(raw) diff --git a/dedup/requestkey_test.go b/dedup/requestkey_test.go index ce4d36c..e906446 100644 --- a/dedup/requestkey_test.go +++ b/dedup/requestkey_test.go @@ -120,3 +120,41 @@ func TestHashBodyPreservesTimestampContent(t *testing.T) { }) } } + +func TestHashBodyPreservesNumericLiterals(t *testing.T) { + for _, tt := range []struct { + name, first, second string + }{ + {"adjacent integers beyond float64 precision", "9007199254740992", "9007199254740993"}, + {"integer and decimal representation", "1", "1.0"}, + } { + t.Run(tt.name, func(t *testing.T) { + first, err := HashBody([]byte(`{"payload":{"number":` + tt.first + `}}`)) + if err != nil { + t.Fatal(err) + } + second, err := HashBody([]byte(`{"payload":{"number":` + tt.second + `}}`)) + if err != nil { + t.Fatal(err) + } + if first == second { + t.Fatalf("distinct numeric literals %s and %s shared a key", tt.first, tt.second) + } + }) + } +} + +func TestHashBodyRejectsTrailingJSONValues(t *testing.T) { + for _, body := range []string{ + `{"model":"test/model"} {"model":"second"}`, + `{"model":"test/model"} 1`, + `{"model":"test/model"} garbage`, + } { + if key, err := HashBody([]byte(body)); err == nil || key != "" { + t.Errorf("trailing JSON data accepted: key=%q err=%v", key, err) + } + } + if _, err := HashBody([]byte("{\"model\":\"test/model\"} \n\t")); err != nil { + t.Errorf("valid trailing whitespace rejected: %v", err) + } +} diff --git a/docs/configuration.md b/docs/configuration.md index ff7b6e3..ee27118 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -252,9 +252,12 @@ state refuses paid dispatch rather than silently clearing limits. Direct and routed chat requests reserve estimated cost before dispatch, then record settled gateway headers or token-based cost when available. Concurrent pending reservations count against session, hourly and daily limits. Explicit upstream -client rejections release a reservation unless a settled charge is reported; -server/transport failures conservatively consume their estimate. Each -reservation permits one HTTP send; model fallbacks reserve separately. Unknown-priced models and image +client rejections release a reservation unless a settled charge is reported. +A valid settled gateway cost header takes priority even on server errors; +server errors without one conservatively consume their estimate. Transport +failures with no response consume the estimate and are not retried. Each +reservation permits one HTTP send; model fallbacks reserve separately. +Unknown-priced models and image requests are refused when limits are configured. This is a local estimate-based control, not a provider-side USD guarantee or multi-process ledger. diff --git a/proxy/proxy.go b/proxy/proxy.go index f55912c..564eec2 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -268,7 +268,7 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { w.Header().Add(k, v) } } - if s.config.UpstreamAPIKey != "" || r.Header.Get("Authorization") != "" { + if s.config.UpstreamAPIKey != "" { w.Header().Set("Cache-Control", "no-store") } w.WriteHeader(entry.StatusCode) @@ -442,11 +442,20 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { var attempts []attemptResult var resp *http.Response var spend *requestSpend - defer func() { - if spend != nil { - spend.finish(nil) + accounted := make(map[*requestSpend]bool) + finalStatus := "interrupted" + finalizeSpend := func(current *requestSpend, status string) { + if current == nil || accounted[current] { + return + } + current.finish(nil) + accounted[current] = true + if sessionID != "" { + s.sessions.AddSessionCost(sessionID, int64(current.cost*1_000_000)) } - }() + s.logSettledRequest(current.model, decision, startTime, current, status) + } + defer func() { finalizeSpend(spend, finalStatus) }() // cancelResp cancels the context of the SUCCESSFUL attempt; it is deferred // after the loop so the chosen response body stays streamable until the // handler returns, then its resources are released. @@ -469,7 +478,7 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { // Per-model timeout (upstream v0.12.182): reasoning models get 3min for // cold-start first-token (DeepSeek V4 Pro / Claude opus thinking / GPT-5 // reasoning_effort=high can take 60-120s); non-reasoning get 60s. On - // timeout the loop falls through to the next model rather than failing. + // timeout the request ends without another ambiguous paid send. // // Implemented as cancel-context + AfterFunc (the Go equivalent of // setTimeout/clearTimeout): the timer fires cancel() only if the attempt @@ -485,7 +494,7 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { tryErr = nil } if tryErr != nil { - currentSpend.finish(nil) // A lost response may already have incurred a charge. + finalizeSpend(currentSpend, "error") // A lost response may already have incurred a charge. timer.Stop() cancelAttempt() if tryResp != nil { @@ -501,9 +510,10 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { if tryResp.StatusCode >= 300 { currentSpend.header = tryResp.Header if _, settled := settledCost(tryResp.Header); settled || tryResp.StatusCode >= 500 { - currentSpend.finish(nil) + finalizeSpend(currentSpend, "error") } else { currentSpend.release() + accounted[currentSpend] = true } errBody, _ := io.ReadAll(tryResp.Body) tryResp.Body.Close() @@ -557,7 +567,6 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { "models": len(attempts), }, }) - s.logRequest(resolvedModel, decision, startTime, "error") return } defer resp.Body.Close() @@ -580,7 +589,7 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { if copyErr != nil { status = "interrupted" } - s.logRequest(resolvedModel, decision, startTime, status) + finalStatus = status return } @@ -719,7 +728,8 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { s.sessions.SetSession(sessionID, nextModel, decision.Tier, userExplicit) } - spend.finish(respBody) + spend.readUsage(respBody) + finalizeSpend(spend, "empty") spend = nil nextSpend, allowed := s.reserveChat(w, req, fbBody, nextModel) if !allowed { @@ -731,7 +741,7 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { fbErr = nil } if fbErr != nil { - nextSpend.finish(nil) + finalizeSpend(nextSpend, "error") if fbResp != nil { fbResp.Body.Close() } @@ -744,15 +754,20 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { if fbResp.StatusCode >= 300 { nextSpend.header = fbResp.Header if _, settled := settledCost(fbResp.Header); settled || fbResp.StatusCode >= 500 { - nextSpend.finish(nil) + finalizeSpend(nextSpend, "error") } else { nextSpend.release() + accounted[nextSpend] = true } http.Error(w, "Fallback request rejected", http.StatusBadGateway) return } spend = nextSpend spend.header = fbResp.Header + w.Header().Del("X-DOSRouter-Request-Id") + if id := gatewayRequestID(fbResp.Header); id != "" { + w.Header().Set("X-DOSRouter-Request-Id", id) + } respBody, readErr = io.ReadAll(fbResp.Body) if readErr != nil { http.Error(w, "Incomplete fallback response", http.StatusBadGateway) @@ -864,11 +879,7 @@ func (s *Server) handleChatCompletions(w http.ResponseWriter, r *http.Request) { } } - spend.finish(nil) - if sessionID != "" { - s.sessions.AddSessionCost(sessionID, int64(spend.cost*1_000_000)) - } - s.logSettledRequest(resolvedModel, decision, startTime, spend) + finalStatus = "success" } diff --git a/proxy/request.go b/proxy/request.go index 88ce3c8..dcc0abc 100644 --- a/proxy/request.go +++ b/proxy/request.go @@ -142,9 +142,26 @@ func (s *Server) writeUsage(entry logger.UsageEntry) { logger.LogUsage(entry) } +// requestHasTools recognizes the function-tool contract used by routing and +// recovery. Unknown provider extensions are forwarded without inferring tool +// capability from an arbitrary nonempty array. func requestHasTools(raw json.RawMessage) bool { var tools []json.RawMessage - return json.Unmarshal(raw, &tools) == nil && len(tools) > 0 + if json.Unmarshal(raw, &tools) != nil { + return false + } + for _, rawTool := range tools { + var tool struct { + Type string `json:"type"` + Function struct { + Name string `json:"name"` + } `json:"function"` + } + if json.Unmarshal(rawTool, &tool) == nil && tool.Type == "function" && strings.TrimSpace(tool.Function.Name) != "" { + return true + } + } + return false } func validTokenCount(value any) (int, bool) { diff --git a/proxy/request_review_test.go b/proxy/request_review_test.go new file mode 100644 index 0000000..6bd9509 --- /dev/null +++ b/proxy/request_review_test.go @@ -0,0 +1,92 @@ +package proxy + +import ( + "bytes" + "encoding/json" + "log" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/DOS/DOSRouter/spendcontrol" +) + +func TestRequestHasToolsRequiresDeclaredFunction(t *testing.T) { + for _, tc := range []struct { + name, raw string + want bool + }{ + {"missing", "", false}, + {"null", `null`, false}, + {"empty array", `[]`, false}, + {"null entry", `[null]`, false}, + {"empty object", `[{}]`, false}, + {"non-object entry", `["function"]`, false}, + {"missing type", `[{"function":{"name":"read_file"}}]`, false}, + {"missing function", `[{"type":"function"}]`, false}, + {"missing name", `[{"type":"function","function":{}}]`, false}, + {"null name", `[{"type":"function","function":{"name":null}}]`, false}, + {"wrong name type", `[{"type":"function","function":{"name":7}}]`, false}, + {"empty name", `[{"type":"function","function":{"name":""}}]`, false}, + {"whitespace name", `[{"type":"function","function":{"name":" \t "}}]`, false}, + {"unknown provider extension", `[{"type":"provider_specific","name":"extension"}]`, false}, + {"declared function", `[{"type":"function","function":{"name":"read_file"}}]`, true}, + {"valid declaration among invalid entries", `[null,{},false,{"type":"function","function":{"name":"read_file"}}]`, true}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := requestHasTools(json.RawMessage(tc.raw)); got != tc.want { + t.Fatalf("requestHasTools(%s)=%v, want %v", tc.raw, got, tc.want) + } + }) + } +} + +func TestSpendCommitFailureLogsWithoutBackendDetails(t *testing.T) { + storage := &spendcontrol.FileSpendControlStorage{Path: filepath.Join(t.TempDir(), "spending.json")} + sc, err := spendcontrol.New(storage) + if err != nil { + t.Fatal(err) + } + if err := sc.SetLimit(spendcontrol.WindowSession, 1); err != nil { + t.Fatal(err) + } + srv := New(Config{SpendControl: sc}) + t.Cleanup(srv.Close) + payload := []byte(`{"model":"openai/gpt-4o-mini","messages":[{"role":"user","content":"test"}]}`) + var request chatRequest + if err := json.Unmarshal(payload, &request); err != nil { + t.Fatal(err) + } + spend, allowed := srv.reserveChat(httptest.NewRecorder(), request, payload, request.Model) + if !allowed { + t.Fatal("reservation denied before storage failure") + } + + // Replacing a directory with the state file fails deterministically on all + // platforms, without depending on elevated users respecting file modes. + const privateDetail = "private-backend-detail-sentinel" + storage.Path = filepath.Join(t.TempDir(), privateDetail) + if err := os.Mkdir(storage.Path, 0o700); err != nil { + t.Fatal(err) + } + previous := log.Writer() + var captured bytes.Buffer + log.SetOutput(&captured) + t.Cleanup(func() { log.SetOutput(previous) }) + spend.finish([]byte(`{"usage":{"prompt_tokens":10,"completion_tokens":5}}`)) + spend.finish(nil) + if count := strings.Count(captured.String(), "spend settlement failed"); count != 1 { + t.Fatalf("expected one observable settlement failure, got %d", count) + } + if strings.Contains(captured.String(), privateDetail) || strings.Contains(captured.String(), storage.Path) { + t.Fatal("backend details leaked into logs") + } + if len(sc.GetHistory()) != 1 { + t.Fatal("settlement was lost or repeated after storage failure") + } + if sc.Check(0).Allowed { + t.Fatal("persistence failure did not keep admission closed") + } +} diff --git a/proxy/spending.go b/proxy/spending.go index fbf597b..d560080 100644 --- a/proxy/spending.go +++ b/proxy/spending.go @@ -2,6 +2,7 @@ package proxy import ( "encoding/json" + "log" "math" "net/http" "strconv" @@ -114,7 +115,10 @@ func (sp *requestSpend) finish(body []byte) { } } // A persistence error keeps the controller fail-closed; never retry a paid call. - _ = sp.server.spendControl.Commit(sp.id, sp.cost, sp.model, "chat") + if err := sp.server.spendControl.Commit(sp.id, sp.cost, sp.model, "chat"); err != nil { + // Backend errors may contain sensitive paths or data; keep the event generic. + log.Print("DOSRouter: spend settlement failed; accounting state requires attention") + } } func (sp *requestSpend) release() { @@ -167,8 +171,8 @@ func mediaCost(h http.Header, body []byte) float64 { return 0 } -func (s *Server) logSettledRequest(model string, decision *router.RoutingDecision, start time.Time, spend *requestSpend) { - entry := logger.UsageEntry{Timestamp: time.Now().UTC().Format(time.RFC3339), Model: model, Tier: "DIRECT", Cost: spend.cost, CostSource: spend.source, RequestID: gatewayRequestID(spend.header), InputTokens: spend.input, OutputTokens: spend.output, Status: "success", LatencyMs: time.Since(start).Milliseconds()} +func (s *Server) logSettledRequest(model string, decision *router.RoutingDecision, start time.Time, spend *requestSpend, status string) { + entry := logger.UsageEntry{Timestamp: time.Now().UTC().Format(time.RFC3339), Model: model, Tier: "DIRECT", Cost: spend.cost, CostSource: spend.source, RequestID: gatewayRequestID(spend.header), InputTokens: spend.input, OutputTokens: spend.output, Status: status, LatencyMs: time.Since(start).Milliseconds()} if decision != nil { entry.Tier = string(decision.Tier) entry.BaselineCost = decision.BaselineCost diff --git a/proxy/spending_review_test.go b/proxy/spending_review_test.go index 046fced..34b0a90 100644 --- a/proxy/spending_review_test.go +++ b/proxy/spending_review_test.go @@ -9,7 +9,9 @@ import ( "sync/atomic" "testing" + "github.com/DOS/DOSRouter/logger" "github.com/DOS/DOSRouter/router" + "github.com/DOS/DOSRouter/session" "github.com/DOS/DOSRouter/spendcontrol" ) @@ -64,6 +66,7 @@ func TestFlatPriceSettlementPreservesCompletionCount(t *testing.T) { func TestTruncatedEmptyTurnFallbackSettlesReservation(t *testing.T) { var attempts atomic.Int32 + var logs []logger.UsageEntry const firstModel = "openai/gpt-4o-mini" const nextModel = "openai/gpt-4o" srv, sc := syncTestServer(t, func(w http.ResponseWriter, r *http.Request) { @@ -79,12 +82,14 @@ func TestTruncatedEmptyTurnFallbackSettlesReservation(t *testing.T) { t.Errorf("first model=%q, want %q", req.Model, firstModel) } w.Header().Set("X-DOS-Cost-USD", "0.125") + w.Header().Set("X-DOS-Request-Id", "initial-attempt") io.WriteString(w, `{"choices":[{"message":{"content":""},"finish_reason":"stop"}]}`) case 2: if req.Model != nextModel { t.Errorf("fallback model=%q, want %q", req.Model, nextModel) } w.Header().Set("X-DOS-Cost-USD", "0.25") + w.Header().Set("X-DOS-Request-Id", "fallback-attempt") w.Header().Set("Content-Length", "4096") io.WriteString(w, `{"choices":[`) default: @@ -102,6 +107,7 @@ func TestTruncatedEmptyTurnFallbackSettlesReservation(t *testing.T) { } } cfg.RoutingConfig = &routing + cfg.UsageLogger = func(e logger.UsageEntry) { logs = append(logs, e) } }) if err := sc.SetLimit(spendcontrol.WindowSession, 1); err != nil { t.Fatal(err) @@ -110,6 +116,12 @@ func TestTruncatedEmptyTurnFallbackSettlesReservation(t *testing.T) { if result.Code != http.StatusBadGateway || !strings.Contains(result.Body.String(), "Incomplete fallback response") { t.Fatalf("status=%d, body=%q", result.Code, result.Body.String()) } + if result.Header().Get("X-DOSRouter-Request-Id") != "fallback-attempt" { + t.Fatalf("fallback request ID=%q", result.Header().Get("X-DOSRouter-Request-Id")) + } + if len(logs) != 2 || logs[0].Cost != 0.125 || logs[1].Cost != 0.25 || logs[0].RequestID != "initial-attempt" || logs[1].RequestID != "fallback-attempt" { + t.Fatalf("fallback usage logs=%+v", logs) + } if attempts.Load() != 2 { t.Fatalf("upstream attempts=%d, want 2", attempts.Load()) } @@ -130,19 +142,23 @@ func (w *closedStreamWriter) Write([]byte) (int, error) { return 0, io.ErrClosed func TestInterruptedStreamRetainsObservedUsage(t *testing.T) { for _, failure := range []string{"client write", "upstream read"} { t.Run(failure, func(t *testing.T) { + var logs []logger.UsageEntry srv, sc := syncTestServer(t, func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/event-stream") if failure == "upstream read" { w.Header().Set("Content-Length", "4096") } io.WriteString(w, "data: {\"choices\":[],\"usage\":{\"prompt_tokens\":1000,\"completion_tokens\":500}}\n\n") - }, nil) + }, func(cfg *Config) { cfg.UsageLogger = func(e logger.UsageEntry) { logs = append(logs, e) } }) + srv.sessions.Close() + srv.sessions = session.NewStore(session.Config{Enabled: true, TimeoutMs: 60000}) if err := sc.SetLimit(spendcontrol.WindowSession, 1); err != nil { t.Fatal(err) } const model = "openai/gpt-4o-mini" srv.modelPricing[model] = router.ModelPricing{InputPrice: 1, OutputPrice: 2} request := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"`+model+`","stream":true,"messages":[{"role":"user","content":"stream"}]}`)) + request.Header.Set("x-session-id", "interrupted-session") var writer http.ResponseWriter = httptest.NewRecorder() if failure == "client write" { writer = &closedStreamWriter{httptest.NewRecorder()} @@ -153,6 +169,12 @@ func TestInterruptedStreamRetainsObservedUsage(t *testing.T) { if len(history) != 1 || history[0].Amount != expectedCost { t.Fatalf("observed usage lost on interruption: %+v, want one charge %v", history, expectedCost) } + if len(logs) != 1 || logs[0].Cost != expectedCost || logs[0].Status != "interrupted" { + t.Fatalf("interrupted usage log=%+v", logs) + } + if cost := srv.sessions.GetSessionCostUSD("interrupted-session"); cost != expectedCost { + t.Fatalf("session cost=%v, want %v", cost, expectedCost) + } if spent := sc.GetSpending()[spendcontrol.WindowSession]; spent != expectedCost { t.Fatalf("pending reservation remained: %v", spent) }