diff --git a/.env.example b/.env.example index 3766a74..649e4ac 100644 --- a/.env.example +++ b/.env.example @@ -152,6 +152,16 @@ OPENSCALE_MODEL=deepseek/deepseek-v4.1-flash # DESCRIPTION: OpenScale Chat Completions endpoint. OPENSCALE_ENDPOINT=https://api.openscale.so/v1/chat/completions +# ------------------------------------------------------------------------------ +# UberLLM (OpenAI-compatible) +# ------------------------------------------------------------------------------ +# DESCRIPTION: UberLLM API key from https://uberllm.dev/signup. +# UBERLLM_API_KEY=your-uberllm-api-key +# DESCRIPTION: Default UberLLM text model. +UBERLLM_MODEL=deepseek/deepseek-v4.1-flash +# DESCRIPTION: UberLLM Chat Completions endpoint. +UBERLLM_ENDPOINT=https://api.uberllm.dev/v1/chat/completions + # ------------------------------------------------------------------------------ # Ollama (local models) # ------------------------------------------------------------------------------ diff --git a/README.md b/README.md index 674236f..c4da578 100644 --- a/README.md +++ b/README.md @@ -224,6 +224,7 @@ Claude Code / Cursor / Codex / Cline / Continue | **AtmoRouter** | Cloud | Claude, GPT, Gemini, GLM, Kimi, Qwen, DeepSeek, and other OpenAI-compatible models | $-$$ | | **Runcrate** | Cloud | DeepSeek, GLM, Kimi, Qwen, gpt-oss, and other open-weight models | $-$$ | | **OpenScale** | Cloud | DeepSeek V4.1 Flash, GLM 5.3 Flash | $ | +| **UberLLM** | Cloud | Claude, GPT, Gemini, DeepSeek, GLM, Kimi, Qwen, and other OpenAI-compatible models | $-$$$ | | **DeepSeek** | Cloud | DeepSeek R1, Reasoner | $ | | **Z.ai** | Cloud | GLM-4.7, GLM-4.5-Air | $ | | **Moonshot AI** | Cloud | Kimi K2.6, Kimi K3 | $ | diff --git a/bin/lynkr-init.js b/bin/lynkr-init.js index 7c5bb6b..8ba1544 100644 --- a/bin/lynkr-init.js +++ b/bin/lynkr-init.js @@ -147,6 +147,17 @@ const PROVIDERS = { ], defaultModel: 'deepseek/deepseek-v4.1-flash', }, + uberllm: { + label: 'UberLLM (OpenAI-compatible)', + local: false, + creds: [ + { key: 'UBERLLM_API_KEY', label: 'UberLLM API key', secret: true }, + ], + extras: [ + { key: 'UBERLLM_ENDPOINT', label: 'Chat Completions endpoint', default: 'https://api.uberllm.dev/v1/chat/completions' }, + ], + defaultModel: 'deepseek/deepseek-v4.1-flash', + }, openrouter: { label: 'OpenRouter (100+ models, one key)', local: false, @@ -250,7 +261,7 @@ const PROVIDERS = { const PROVIDER_ORDER = [ 'ollama', 'llamacpp', 'lmstudio', - 'azure-anthropic', 'azure-openai', 'openai', 'atlas', 'cheaperinference', 'atmorouter', 'runcrate', 'openscale', 'openrouter', 'edenai', + 'azure-anthropic', 'azure-openai', 'openai', 'atlas', 'cheaperinference', 'atmorouter', 'runcrate', 'openscale', 'uberllm', 'openrouter', 'edenai', 'databricks', 'bedrock', 'vertex', 'zai', 'moonshot', 'baidu', 'fireworks', 'orcarouter', ]; const TIERS = ['SIMPLE', 'MEDIUM', 'COMPLEX', 'REASONING']; diff --git a/documentation/providers.md b/documentation/providers.md index e186f82..2c86460 100644 --- a/documentation/providers.md +++ b/documentation/providers.md @@ -970,6 +970,42 @@ TIER_REASONING=openscale:deepseek/deepseek-v4.1-flash --- +### 8f. UberLLM (OpenAI-Compatible) + +**Best for:** One key for Claude, GPT, Gemini, DeepSeek, GLM, Kimi and Qwen models, so each Lynkr tier can use a different vendor without a second account + +#### Configuration + +```env +MODEL_PROVIDER=uberllm +UBERLLM_API_KEY=your-uberllm-api-key +UBERLLM_MODEL=deepseek/deepseek-v4.1-flash +UBERLLM_ENDPOINT=https://api.uberllm.dev/v1/chat/completions +``` + +Create a key at [uberllm.dev/signup](https://uberllm.dev/signup). The endpoint is OpenAI-compatible, so Lynkr converts Anthropic messages and tools to Chat Completions requests and transforms streamed OpenAI SSE events back to Anthropic events. + +UberLLM can also be selected per tier: + +```env +TIER_SIMPLE=uberllm:deepseek/deepseek-v4.1-flash +TIER_MEDIUM=uberllm:z-ai/glm-5.3-flash +TIER_COMPLEX=uberllm:z-ai/glm-5.3 +TIER_REASONING=uberllm:anthropic/claude-sonnet-4.6 +``` + +`UBERLLM_MODEL` is the static-routing default. A model named in `TIER_*` takes precedence for that tier. Model IDs carry the vendor prefix (`anthropic/`, `openai/`, `google/`, `deepseek/`, `z-ai/`, `moonshotai/`, `qwen/`); the full list is public at `GET https://api.uberllm.dev/v1/models` and per-model prices are at [uberllm.dev/models](https://uberllm.dev/models). + +#### Behavior + +- Supports non-streaming and streaming Chat Completions +- Supports Anthropic-format tools through Lynkr's existing OpenAI conversion path +- Relays model reasoning as `reasoning_content` +- Uses Bearer authentication and accepts a custom `UBERLLM_ENDPOINT` +- Does not automatically replay billable Chat Completions POST requests + +--- + ### 9. LM Studio (Local with GUI) **Best for:** Local models with graphical interface diff --git a/package.json b/package.json index 61b00c1..33e50c0 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "start:supervised": "while true; do node index.js 2>&1 | npx pino-pretty --sync; code=$?; echo \"[supervisor] lynkr exited ($code) — restarting in 3s\"; sleep 3; done", "lint": "eslint src index.js", "test": "npm run test:unit && npm run test:performance", - "test:unit": "LYNKR_KNN_DIR=/tmp/lynkr-test-knn DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com LOG_FILE_ENABLED=false node --test test/routing.test.js test/hybrid-routing-integration.test.js test/retry-logic.test.js test/sse-transformer.test.js test/passthrough-stream.test.js test/passthrough-mode.test.js test/openrouter-error-resilience.test.js test/format-conversion.test.js test/azure-openai-config.test.js test/azure-openai-format-conversion.test.js test/azure-openai-routing.test.js test/azure-openai-streaming.test.js test/azure-openai-error-resilience.test.js test/azure-openai-integration.test.js test/openai-integration.test.js test/atlas-integration.test.js test/cheaperinference-integration.test.js test/atmorouter-integration.test.js test/runcrate-integration.test.js test/openscale-integration.test.js test/toon-compression.test.js test/gcf-compression.test.js test/llamacpp-integration.test.js test/resilience.test.js test/telemetry-routing.test.js test/memory/store.test.js test/memory/surprise.test.js test/memory/extractor.test.js test/memory/search.test.js test/memory/retriever.test.js test/memory/distiller.test.js test/memory/distiller-freeze.test.js test/memory/wiki.test.js test/memory/skills-cache.test.js test/memory/tencentdb-launcher.test.js test/distill.test.js test/large-payload.test.js test/prompt-cache-injection.test.js test/risk-analyzer.test.js test/interaction-block.test.js test/preflight.test.js test/token-reduction.test.js test/session-affinity.test.js test/cache-state.test.js test/cache-switch-cost.test.js test/lens-recommendations.test.js test/model-registry-cost.test.js test/output-format-guard.test.js test/tier-fallback.test.js test/wrap.test.js test/init.test.js test/tool-call-response-metadata.test.js test/degradation.test.js test/routing-telemetry-columns.test.js test/sticky-routing.test.js test/knn-ambiguous-escalate.test.js test/deescalator.test.js test/client-profiles.test.js test/openai-ingress-tool-injection.test.js test/tool-result-compression-gate.test.js test/strip-internal-fields.test.js test/complexity-tool-subtraction.test.js test/bandit.test.js test/routing-propensity.test.js test/reward-pipeline.test.js test/knn-cold-start.test.js test/calibration.test.js test/feedback-loop.test.js test/session-fingerprint.test.js test/side-request-guards.test.js test/verifier.test.js test/intent-score.test.js test/difficulty-classifier.test.js test/classifier-setup.test.js test/usage-stats.test.js test/loop-guard.test.js test/moonshot-model-mapping.test.js test/baidu-model-mapping.test.js test/tenant-policy-ingress-parity.test.js test/decide.test.js test/embeddings-degradation.test.js test/health-probe.test.js test/stuck-detector.test.js test/onnx-embedder.test.js test/ope.test.js test/hierarchical-budget.test.js test/token-rate-limit.test.js test/otel-export.test.js test/mcp-broker.test.js test/compression-budget.test.js test/gpt-utils.test.js test/dedup-observe-only.test.js test/context-window-header.test.js test/token-budget-auto.test.js test/opencode-setup.test.js test/auth-mode-first-party.test.js test/harness-envelope.test.js test/task-ledger.test.js test/jev-router.test.js test/jev-routing.test.js test/force-patterns.test.js test/upstream-fidelity.test.js test/tool-schema-compression.test.js test/tool-strict-preservation.test.js test/passthrough-route.test.js test/quota-ledger.test.js test/shortfall-semantic.test.js test/routing-decisions.test.js test/routing-corpus.test.js test/switch-gate-grounding.test.js", + "test:unit": "LYNKR_KNN_DIR=/tmp/lynkr-test-knn DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com LOG_FILE_ENABLED=false node --test test/routing.test.js test/hybrid-routing-integration.test.js test/retry-logic.test.js test/sse-transformer.test.js test/passthrough-stream.test.js test/passthrough-mode.test.js test/openrouter-error-resilience.test.js test/format-conversion.test.js test/azure-openai-config.test.js test/azure-openai-format-conversion.test.js test/azure-openai-routing.test.js test/azure-openai-streaming.test.js test/azure-openai-error-resilience.test.js test/azure-openai-integration.test.js test/openai-integration.test.js test/atlas-integration.test.js test/cheaperinference-integration.test.js test/atmorouter-integration.test.js test/runcrate-integration.test.js test/openscale-integration.test.js test/uberllm-integration.test.js test/toon-compression.test.js test/gcf-compression.test.js test/llamacpp-integration.test.js test/resilience.test.js test/telemetry-routing.test.js test/memory/store.test.js test/memory/surprise.test.js test/memory/extractor.test.js test/memory/search.test.js test/memory/retriever.test.js test/memory/distiller.test.js test/memory/distiller-freeze.test.js test/memory/wiki.test.js test/memory/skills-cache.test.js test/memory/tencentdb-launcher.test.js test/distill.test.js test/large-payload.test.js test/prompt-cache-injection.test.js test/risk-analyzer.test.js test/interaction-block.test.js test/preflight.test.js test/token-reduction.test.js test/session-affinity.test.js test/cache-state.test.js test/cache-switch-cost.test.js test/lens-recommendations.test.js test/model-registry-cost.test.js test/output-format-guard.test.js test/tier-fallback.test.js test/wrap.test.js test/init.test.js test/tool-call-response-metadata.test.js test/degradation.test.js test/routing-telemetry-columns.test.js test/sticky-routing.test.js test/knn-ambiguous-escalate.test.js test/deescalator.test.js test/client-profiles.test.js test/openai-ingress-tool-injection.test.js test/tool-result-compression-gate.test.js test/strip-internal-fields.test.js test/complexity-tool-subtraction.test.js test/bandit.test.js test/routing-propensity.test.js test/reward-pipeline.test.js test/knn-cold-start.test.js test/calibration.test.js test/feedback-loop.test.js test/session-fingerprint.test.js test/side-request-guards.test.js test/verifier.test.js test/intent-score.test.js test/difficulty-classifier.test.js test/classifier-setup.test.js test/usage-stats.test.js test/loop-guard.test.js test/moonshot-model-mapping.test.js test/baidu-model-mapping.test.js test/tenant-policy-ingress-parity.test.js test/decide.test.js test/embeddings-degradation.test.js test/health-probe.test.js test/stuck-detector.test.js test/onnx-embedder.test.js test/ope.test.js test/hierarchical-budget.test.js test/token-rate-limit.test.js test/otel-export.test.js test/mcp-broker.test.js test/compression-budget.test.js test/gpt-utils.test.js test/dedup-observe-only.test.js test/context-window-header.test.js test/token-budget-auto.test.js test/opencode-setup.test.js test/auth-mode-first-party.test.js test/harness-envelope.test.js test/task-ledger.test.js test/jev-router.test.js test/jev-routing.test.js test/force-patterns.test.js test/upstream-fidelity.test.js test/tool-schema-compression.test.js test/passthrough-route.test.js test/quota-ledger.test.js test/shortfall-semantic.test.js test/routing-decisions.test.js test/routing-corpus.test.js test/switch-gate-grounding.test.js", "test:memory": "LYNKR_KNN_DIR=/tmp/lynkr-test-knn DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node --test test/memory/store.test.js test/memory/surprise.test.js test/memory/extractor.test.js test/memory/search.test.js test/memory/retriever.test.js test/memory/distiller.test.js test/memory/distiller-freeze.test.js test/memory/wiki.test.js test/memory/skills-cache.test.js test/memory/tencentdb-launcher.test.js", "test:new-features": "LYNKR_KNN_DIR=/tmp/lynkr-test-knn DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node --test test/passthrough-mode.test.js test/openrouter-error-resilience.test.js test/format-conversion.test.js", "test:performance": "LYNKR_KNN_DIR=/tmp/lynkr-test-knn DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node test/hybrid-routing-performance.test.js && DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node test/performance-tests.js", diff --git a/src/api/providers-handler.js b/src/api/providers-handler.js index 9ba1770..f70692a 100644 --- a/src/api/providers-handler.js +++ b/src/api/providers-handler.js @@ -173,6 +173,19 @@ async function getConfiguredProviders() { }); } + // Check UberLLM (OpenAI-compatible Chat Completions) + if (config.uberllm?.apiKey) { + providers.push({ + name: "uberllm", + type: "uberllm", + baseUrl: config.uberllm.endpoint || "https://api.uberllm.dev/v1/chat/completions", + enabled: true, + models: [ + { id: config.uberllm.model || "deepseek/deepseek-v4.1-flash", name: "Configured Model" }, + ] + }); + } + // Check OpenRouter if (config.openrouter?.apiKey) { providers.push({ diff --git a/src/clients/databricks.js b/src/clients/databricks.js index a92a327..72b6ce3 100644 --- a/src/clients/databricks.js +++ b/src/clients/databricks.js @@ -1998,6 +1998,72 @@ async function invokeOpenScale(body) { return response; } +async function invokeUberLLM(body) { + if (!config.uberllm?.apiKey) { + throw new Error("UberLLM API key is not configured."); + } + + const { + convertAnthropicToolsToOpenRouter, + convertAnthropicMessagesToOpenRouter + } = require("./openrouter-utils"); + + const endpoint = config.uberllm.endpoint || "https://api.uberllm.dev/v1/chat/completions"; + const headers = { + "Authorization": `Bearer ${config.uberllm.apiKey}`, + "Content-Type": "application/json", + }; + + const messages = convertAnthropicMessagesToOpenRouter(body.messages || []); + if (body.system) { + messages.unshift({ role: "system", content: body.system }); + } + + const uberllmBody = { + model: body._suggestionModeModel || body._tierModel || config.uberllm.model || "deepseek/deepseek-v4.1-flash", + messages, + temperature: body.temperature ?? 0.7, + max_tokens: body.max_tokens ?? 16384, + top_p: body.top_p ?? 1.0, + stream: body.stream ?? false, + }; + + const toolsToSend = Array.isArray(body.tools) ? body.tools : []; + + if (toolsToSend.length > 0) { + uberllmBody.tools = convertAnthropicToolsToOpenRouter(toolsToSend); + uberllmBody.parallel_tool_calls = false; + uberllmBody.tool_choice = forwardToolChoice(body); + } + + logger.debug({ + endpoint, + model: uberllmBody.model, + hasTools: !!uberllmBody.tools, + toolCount: uberllmBody.tools?.length || 0, + temperature: uberllmBody.temperature, + max_tokens: uberllmBody.max_tokens, + }, "=== UBERLLM REQUEST ==="); + + // Chat completions are billable POSTs, so UberLLM requests are never + // replayed automatically (same policy as invokeAtmoRouter). A 429 is thrown + // so invokeModel's caller can escalate to a different provider/tier. + const response = await performJsonRequest(endpoint, { + headers, + body: uberllmBody, + maxRetriesOverride: 0, + retryableStatusesOverride: [], + }, "UberLLM"); + + if (!response.ok && response.status === 429) { + const err = new Error(`UberLLM rate-limited: ${String(response.json?.error?.message || '').slice(0, 120)}`); + err.status = 429; + throw err; + } + + return response; +} + async function invokeLlamaCpp(body, _incomingHeaders = {}) { if (!config.llamacpp?.endpoint) { throw new Error("llama.cpp endpoint is not configured."); @@ -4029,6 +4095,7 @@ const PROVIDER_INVOKERS = { atmorouter: invokeAtmoRouter, runcrate: invokeRuncrate, openscale: invokeOpenScale, + uberllm: invokeUberLLM, llamacpp: invokeLlamaCpp, lmstudio: invokeLMStudio, bedrock: invokeBedrock, @@ -4968,6 +5035,7 @@ module.exports = { invokeAtmoRouter, invokeRuncrate, invokeOpenScale, + invokeUberLLM, invokeOrcaRouter, invokeCursor, PROVIDER_INVOKERS, diff --git a/src/clients/provider-capabilities.js b/src/clients/provider-capabilities.js index 5085aa2..a955ba7 100644 --- a/src/clients/provider-capabilities.js +++ b/src/clients/provider-capabilities.js @@ -9,7 +9,7 @@ const NATIVE_THINKING_BEDROCK_MODELS = [ "claude-haiku", ]; -const REASONING_CONTENT_PROVIDERS = new Set(["moonshot", "openrouter", "edenai", "openai", "azure-openai", "atlas", "cheaperinference", "atmorouter", "runcrate", "openscale", "orcarouter"]); +const REASONING_CONTENT_PROVIDERS = new Set(["moonshot", "openrouter", "edenai", "openai", "azure-openai", "atlas", "cheaperinference", "atmorouter", "runcrate", "openscale", "uberllm", "orcarouter"]); function supportsNativeThinking(providerType, model) { if (NATIVE_THINKING_PROVIDERS.has(providerType)) return true; diff --git a/src/config/index.js b/src/config/index.js index 26cba36..7cbacf2 100644 --- a/src/config/index.js +++ b/src/config/index.js @@ -62,7 +62,7 @@ function resolveConfigPath(targetPath) { return path.resolve(normalised); } -const SUPPORTED_MODEL_PROVIDERS = new Set(["databricks", "azure-anthropic", "ollama", "openrouter", "edenai", "azure-openai", "openai", "atlas", "cheaperinference", "atmorouter", "runcrate", "openscale", "llamacpp", "lmstudio", "bedrock", "zai", "vertex", "moonshot", "baidu", "fireworks", "orcarouter"]); +const SUPPORTED_MODEL_PROVIDERS = new Set(["databricks", "azure-anthropic", "ollama", "openrouter", "edenai", "azure-openai", "openai", "atlas", "cheaperinference", "atmorouter", "runcrate", "openscale", "uberllm", "llamacpp", "lmstudio", "bedrock", "zai", "vertex", "moonshot", "baidu", "fireworks", "orcarouter"]); const rawModelProvider = (process.env.MODEL_PROVIDER ?? "databricks").toLowerCase(); // Validate MODEL_PROVIDER early with a clear error message @@ -140,6 +140,11 @@ const openScaleApiKey = process.env.OPENSCALE_API_KEY?.trim() || null; const openScaleModel = process.env.OPENSCALE_MODEL?.trim() || "deepseek/deepseek-v4.1-flash"; const openScaleEndpoint = process.env.OPENSCALE_ENDPOINT?.trim() || "https://api.openscale.so/v1/chat/completions"; +// UberLLM configuration (OpenAI-compatible Chat Completions) +const uberllmApiKey = process.env.UBERLLM_API_KEY?.trim() || null; +const uberllmModel = process.env.UBERLLM_MODEL?.trim() || "deepseek/deepseek-v4.1-flash"; +const uberllmEndpoint = process.env.UBERLLM_ENDPOINT?.trim() || "https://api.uberllm.dev/v1/chat/completions"; + // llama.cpp configuration const llamacppEndpoint = process.env.LLAMACPP_ENDPOINT?.trim() || "http://localhost:8080"; const llamacppModel = process.env.LLAMACPP_MODEL?.trim() || "default"; @@ -390,6 +395,12 @@ if (modelProvider === "openscale" && !openScaleApiKey) { ); } +if (modelProvider === "uberllm" && !uberllmApiKey) { + throw new Error( + "Set UBERLLM_API_KEY before starting the proxy.", + ); +} + if (modelProvider === "orcarouter" && !orcaRouterApiKey) { throw new Error( "Set ORCAROUTER_API_KEY before starting the proxy.", @@ -444,7 +455,7 @@ const tiersConfigured = !!( if (fallbackEnabled && tiersConfigured) { const localProviders = ["ollama", "llamacpp", "lmstudio"]; if (localProviders.includes(fallbackProvider)) { - throw new Error(`FALLBACK_PROVIDER cannot be '${fallbackProvider}' (local providers should not be fallbacks). Use cloud providers: databricks, azure-anthropic, azure-openai, openrouter, edenai, openai, atlas, cheaperinference, atmorouter, runcrate, openscale, bedrock, orcarouter`); + throw new Error(`FALLBACK_PROVIDER cannot be '${fallbackProvider}' (local providers should not be fallbacks). Use cloud providers: databricks, azure-anthropic, azure-openai, openrouter, edenai, openai, atlas, cheaperinference, atmorouter, runcrate, openscale, uberllm, bedrock, orcarouter`); } let fallbackMisconfigured = false; if (fallbackProvider === "databricks" && (!rawBaseUrl || !apiKey)) { @@ -474,6 +485,9 @@ if (fallbackEnabled && tiersConfigured) { if (fallbackProvider === "openscale" && !openScaleApiKey) { fallbackMisconfigured = true; } + if (fallbackProvider === "uberllm" && !uberllmApiKey) { + fallbackMisconfigured = true; + } if (fallbackProvider === "orcarouter" && !orcaRouterApiKey) { fallbackMisconfigured = true; } @@ -748,6 +762,11 @@ var config = { model: openScaleModel, endpoint: openScaleEndpoint, }, + uberllm: { + apiKey: uberllmApiKey, + model: uberllmModel, + endpoint: uberllmEndpoint, + }, llamacpp: { endpoint: llamacppEndpoint, model: llamacppModel, @@ -1269,6 +1288,9 @@ function reloadConfig() { config.openscale.apiKey = process.env.OPENSCALE_API_KEY?.trim() || null; config.openscale.model = process.env.OPENSCALE_MODEL?.trim() || "deepseek/deepseek-v4.1-flash"; config.openscale.endpoint = process.env.OPENSCALE_ENDPOINT?.trim() || "https://api.openscale.so/v1/chat/completions"; + config.uberllm.apiKey = process.env.UBERLLM_API_KEY?.trim() || null; + config.uberllm.model = process.env.UBERLLM_MODEL?.trim() || "deepseek/deepseek-v4.1-flash"; + config.uberllm.endpoint = process.env.UBERLLM_ENDPOINT?.trim() || "https://api.uberllm.dev/v1/chat/completions"; config.bedrock.apiKey = process.env.AWS_BEDROCK_API_KEY?.trim() || null; config.zai.apiKey = process.env.ZAI_API_KEY?.trim() || null; config.zai.model = process.env.ZAI_MODEL?.trim() || "GLM-4.7"; diff --git a/src/orchestrator/index.js b/src/orchestrator/index.js index cecf832..1c347cb 100644 --- a/src/orchestrator/index.js +++ b/src/orchestrator/index.js @@ -54,6 +54,8 @@ function getDestinationUrl(providerType) { return config.runcrate?.endpoint ?? 'https://api.runcrate.ai/v1/chat/completions'; case 'openscale': return config.openscale?.endpoint ?? 'https://api.openscale.so/v1/chat/completions'; + case 'uberllm': + return config.uberllm?.endpoint ?? 'https://api.uberllm.dev/v1/chat/completions'; case 'llamacpp': return config.llamacpp?.endpoint ?? 'unknown'; case 'lmstudio': @@ -1227,7 +1229,7 @@ function sanitizePayload(payload) { } else { clean.tools = ensureAnthropicToolFormat(clean.tools); } - } else if (providerType === "azure-openai" || providerType === "openai" || providerType === "atlas" || providerType === "cheaperinference" || providerType === "atmorouter" || providerType === "runcrate" || providerType === "openscale") { + } else if (providerType === "azure-openai" || providerType === "openai" || providerType === "atlas" || providerType === "cheaperinference" || providerType === "atmorouter" || providerType === "runcrate" || providerType === "openscale" || providerType === "uberllm") { // Azure OpenAI / OpenAI-compatible providers support tools — keep Anthropic format; the // client converts to Chat Completions / Responses format. Without this // branch the unknown-provider catch-all below deletes the tools. diff --git a/src/orchestrator/sse-transformer.js b/src/orchestrator/sse-transformer.js index 6d7cf36..a0f6d59 100644 --- a/src/orchestrator/sse-transformer.js +++ b/src/orchestrator/sse-transformer.js @@ -44,6 +44,7 @@ const DEFAULT_OPENAI_SSE_PROVIDERS = [ "atmorouter", "runcrate", "openscale", + "uberllm", "azure-openai", "openrouter", "databricks", diff --git a/src/routing/index.js b/src/routing/index.js index b2286e4..09d0354 100644 --- a/src/routing/index.js +++ b/src/routing/index.js @@ -97,6 +97,7 @@ function _enabledProviders() { if (config.atmorouter?.apiKey) out.push('atmorouter'); if (config.runcrate?.apiKey) out.push('runcrate'); if (config.openscale?.apiKey) out.push('openscale'); + if (config.uberllm?.apiKey) out.push('uberllm'); if (config.azureOpenAI?.endpoint && config.azureOpenAI?.apiKey) out.push('azure-openai'); if (config.orcarouter?.apiKey) out.push('orcarouter'); if (config.ollama?.endpoint) out.push('ollama'); @@ -146,6 +147,7 @@ function getBestCloudProvider() { if (config.atmorouter?.apiKey) return 'atmorouter'; if (config.runcrate?.apiKey) return 'runcrate'; if (config.openscale?.apiKey) return 'openscale'; + if (config.uberllm?.apiKey) return 'uberllm'; if (config.azureOpenAI?.endpoint && config.azureOpenAI?.apiKey) return 'azure-openai'; if (config.orcarouter?.apiKey) return 'orcarouter'; diff --git a/src/routing/model-tiers.js b/src/routing/model-tiers.js index 56e4eea..e803141 100644 --- a/src/routing/model-tiers.js +++ b/src/routing/model-tiers.js @@ -354,6 +354,8 @@ class ModelTierSelector { return config.runcrate?.model || null; case 'openscale': return config.openscale?.model || null; + case 'uberllm': + return config.uberllm?.model || null; case 'ollama': return config.ollama?.model || null; case 'openrouter': diff --git a/test/dispatch-registry.test.js b/test/dispatch-registry.test.js index 56dc296..4347494 100644 --- a/test/dispatch-registry.test.js +++ b/test/dispatch-registry.test.js @@ -31,7 +31,7 @@ test('PROVIDER_INVOKERS covers every supported provider except databricks', () = // silent databricks dial. const SUPPORTED = [ 'azure-anthropic', 'ollama', 'openrouter', 'edenai', 'azure-openai', - 'openai', 'atlas', 'cheaperinference', 'atmorouter', 'runcrate', 'openscale', 'llamacpp', 'lmstudio', 'bedrock', 'zai', 'vertex', 'moonshot', + 'openai', 'atlas', 'cheaperinference', 'atmorouter', 'runcrate', 'openscale', 'uberllm', 'llamacpp', 'lmstudio', 'bedrock', 'zai', 'vertex', 'moonshot', 'baidu', 'orcarouter', ]; diff --git a/test/uberllm-integration.test.js b/test/uberllm-integration.test.js new file mode 100644 index 0000000..82400a7 --- /dev/null +++ b/test/uberllm-integration.test.js @@ -0,0 +1,155 @@ +"use strict"; + +const assert = require("node:assert/strict"); +const { describe, it, beforeEach, afterEach } = require("node:test"); + +function clearModules() { + for (const modulePath of [ + "../src/config", + "../src/clients/databricks", + "../src/clients/routing", + "../src/routing", + "../src/orchestrator/sse-transformer", + "../src/clients/provider-capabilities", + ]) { + delete require.cache[require.resolve(modulePath)]; + } +} + +describe("UberLLM integration", () => { + let originalEnv; + let originalFetch; + + beforeEach(() => { + originalEnv = { ...process.env }; + originalFetch = global.fetch; + + process.env.MODEL_PROVIDER = "uberllm"; + process.env.UBERLLM_API_KEY = "uberllm-test-key"; + process.env.UBERLLM_MODEL = ""; + process.env.UBERLLM_ENDPOINT = ""; + process.env.FALLBACK_ENABLED = "false"; + process.env.TIER_SIMPLE = ""; + process.env.TIER_MEDIUM = ""; + process.env.TIER_COMPLEX = ""; + process.env.TIER_REASONING = ""; + process.env.LOG_FILE_ENABLED = "false"; + clearModules(); + }); + + afterEach(() => { + process.env = originalEnv; + global.fetch = originalFetch; + clearModules(); + }); + + it("accepts uberllm and applies safe defaults", () => { + const config = require("../src/config"); + + assert.equal(config.modelProvider.type, "uberllm"); + assert.equal(config.uberllm.model, "deepseek/deepseek-v4.1-flash"); + assert.equal(config.uberllm.endpoint, "https://api.uberllm.dev/v1/chat/completions"); + }); + + it("requires UBERLLM_API_KEY for primary routing", () => { + process.env.UBERLLM_API_KEY = ""; + clearModules(); + + assert.throws( + () => require("../src/config"), + /Set UBERLLM_API_KEY before starting the proxy/, + ); + }); + + it("honors custom model and endpoint", () => { + process.env.UBERLLM_MODEL = "z-ai/glm-5.3-flash"; + process.env.UBERLLM_ENDPOINT = "https://ci.example/v1/chat/completions"; + clearModules(); + + const config = require("../src/config"); + assert.equal(config.uberllm.model, "z-ai/glm-5.3-flash"); + assert.equal(config.uberllm.endpoint, "https://ci.example/v1/chat/completions"); + }); + + it("converts Anthropic messages and tools to one UberLLM request", async () => { + const calls = []; + global.fetch = async (url, options) => { + calls.push({ url, options }); + return new Response(JSON.stringify({ + id: "chatcmpl-uberllm-test", + choices: [{ message: { role: "assistant", content: "ok" }, finish_reason: "stop" }], + usage: { prompt_tokens: 4, completion_tokens: 1, total_tokens: 5 }, + }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }; + + const { invokeUberLLM } = require("../src/clients/databricks"); + const result = await invokeUberLLM({ + system: "Be concise.", + messages: [{ role: "user", content: "Say ok" }], + tools: [{ + name: "get_weather", + description: "Get weather", + input_schema: { + type: "object", + properties: { city: { type: "string" } }, + required: ["city"], + }, + }], + max_tokens: 32, + stream: false, + }); + + assert.equal(result.ok, true); + assert.equal(calls.length, 1); + assert.equal(calls[0].url, "https://api.uberllm.dev/v1/chat/completions"); + assert.equal(calls[0].options.headers.Authorization, "Bearer uberllm-test-key"); + + const request = JSON.parse(calls[0].options.body); + assert.equal(request.model, "deepseek/deepseek-v4.1-flash"); + assert.deepEqual(request.messages[0], { role: "system", content: "Be concise." }); + assert.deepEqual(request.messages[1], { role: "user", content: "Say ok" }); + assert.equal(request.tools[0].type, "function"); + assert.equal(request.tools[0].function.name, "get_weather"); + assert.equal(request.parallel_tool_calls, false); + assert.equal(request.tool_choice, "auto"); + }); + + it("does not replay a failed billable POST", async () => { + let calls = 0; + global.fetch = async () => { + calls += 1; + return new Response(JSON.stringify({ error: { message: "temporary" } }), { + status: 503, + headers: { "content-type": "application/json" }, + }); + }; + + const { invokeUberLLM } = require("../src/clients/databricks"); + const result = await invokeUberLLM({ + messages: [{ role: "user", content: "hello" }], + tools: [{ name: "noop", description: "No-op", input_schema: { type: "object" } }], + stream: false, + }); + + assert.equal(result.status, 503); + assert.equal(calls, 1); + }); + + it("supports static routing, reasoning content, and OpenAI SSE transforms", async () => { + const routing = require("../src/clients/routing"); + const decision = await routing.determineProviderSmart({ + messages: [{ role: "user", content: "hello" }], + }); + assert.equal(decision.provider, "uberllm"); + assert.equal(decision.method, "static"); + + const { supportsReasoningContent } = require("../src/clients/provider-capabilities"); + assert.equal(supportsReasoningContent("uberllm"), true); + + const { shouldTransform } = require("../src/orchestrator/sse-transformer"); + assert.equal(shouldTransform(true, "uberllm"), true); + }); +});