Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors
TypeScript a2aai-agentsai-gatewayanthropicclaudeclaude-codeclinecodexcopilotcursordeepseekfree-aigeminikimillm-gatewaymcpopenaiopenai-proxyqwentoken-saver
feat: add ModelScope provider with circuit breaker and daily quota lock (#1430)
clousky2020d6cbdc158079
diff --git a/next.config.mjs b/next.config.mjsindex db1017ae84..375b966e96 100644--- a/next.config.mjs+++ b/next.config.mjs@@ -28,6 +28,7 @@ const nextConfig = {"./playwright-report/**/*","./app.__qa_backup/**/*","./tests/**/*",+ "./logs/**/*",],},serverExternalPackages: [diff --git a/open-sse/config/providerRegistry.ts b/open-sse/config/providerRegistry.tsindex b48b3dc9d9..ede6e0cdff 100644--- a/open-sse/config/providerRegistry.ts+++ b/open-sse/config/providerRegistry.ts@@ -1400,6 +1400,25 @@ export const REGISTRY: Record<string, RegistryEntry> = {passthroughModels: true,},+ modelscope: {+ id: "modelscope",+ alias: "ms",+ format: "openai",+ executor: "default",+ baseUrl: "https://api-inference.modelscope.cn/v1/chat/completions",+ authType: "apikey",+ authHeader: "bearer",+ // ModelScope uses per-model quotas. Setting passthroughModels: true ensures 429/404+ // only locks the specific model, not the entire connection. This allows fallback+ // to other models on the same API key.+ passthroughModels: true,+ models: [+ { id: "moonshotai/Kimi-K2.5", name: "Kimi K2.5" },+ { id: "ZhipuAI/GLM-5", name: "GLM-5" },+ { id: "stepfun-ai/Step-3.5-Flash", name: "Step-3.5-Flash" },+ ],+ },+// ── New Free Providers (2026) ─────────────────────────────────────────────longcat: {@@ -2128,8 +2147,14 @@ export function getUnsupportedParams(provider: string, modelId: string): readonlconst cached = _unsupportedParamsMap.get(modelId);if (cached) return cached;- // 3. Handle prefixed model IDs (e.g., "openai/o3" → "o3")+ // 3. Handle prefixed model IDs (e.g., "openai/o3" → "o3", "moonshotai/Kimi-K2.5" → "moonshotai/Kimi-K2.5")+ // ModelScope models have slash in ID, check both full ID and bare IDif (modelId.includes("/")) {+ // First check full model ID with provider prefix (e.g., "moonshotai/Kimi-K2.5")+ const cachedWithPrefix = _unsupportedParamsMap.get(modelId);+ if (cachedWithPrefix) return cachedWithPrefix;++ // Fall back to bare ID (e.g., "Kimi-K2.5")const bareId = modelId.split("/").pop() || "";const bare = _unsupportedParamsMap.get(bareId);if (bare) return bare;@@ -2154,6 +2179,7 @@ export function getProviderCategory(provider: string): "oauth" | "apikey" {* Derive the latest opus/sonnet/haiku model IDs from the `claude` registry entry.* Picks the first model whose ID matches each family pattern — registry order* determines precedence, so newer models should be listed first.+ * @deprecated This function will be removed in v4.0, please use REGISTRY.claude?.models directly*/export function getClaudeCodeDefaultModels(): {opus: string;diff --git a/open-sse/executors/antigravity.ts b/open-sse/executors/antigravity.tsindex b8dc191b7b..b987232735 100644--- a/open-sse/executors/antigravity.ts+++ b/open-sse/executors/antigravity.ts@@ -735,18 +735,45 @@ export class AntigravityExecutor extends BaseExecutor {// consuming the stream. The client receives the unmodified SSE data.if (response.body) {let sseBuffer = "";+ const decoder = new TextDecoder(); // Singleton for correct streaming decode+ const MAX_BUFFER_SIZE = 16 * 1024; // Limit to prevent OOM on large streams+const passThrough = new TransformStream({transform(chunk, controller) {controller.enqueue(chunk);// Accumulate text to scan for remainingCreditstry {- const text = new TextDecoder().decode(chunk, { stream: true });+ const text = decoder.decode(chunk, { stream: true });sseBuffer += text;+ // Limit buffer size to prevent unbounded growth+ // Truncate only after a complete newline to avoid splitting SSE lines mid-payload+ if (sseBuffer.length > MAX_BUFFER_SIZE) {+ const lastNewline = sseBuffer.lastIndexOf(+ "\n",+ sseBuffer.length - MAX_BUFFER_SIZE+ );+ if (lastNewline !== -1) {+ sseBuffer = sseBuffer.slice(lastNewline + 1);+ } else {+ // No newline found in discard region — buffer contains an incomplete SSE line.+ // Discard it entirely to avoid returning malformed data; the remainingCredits+ // parser won't find valid data in a truncated line anyway.+ sseBuffer = "";+ }+ }} catch {/* decoding best-effort */}},flush() {+ // Final decode for any remaining bytes+ try {+ const text = decoder.decode(); // Flush pending bytes+ sseBuffer += text;+ } catch {+ /* decoding best-effort */+ }+// Parse the accumulated SSE data for remainingCreditstry {const lines = sseBuffer.split("\n");diff --git a/open-sse/executors/codex.ts b/open-sse/executors/codex.tsindex 899c4eb00d..fcbcbab1cd 100644--- a/open-sse/executors/codex.ts+++ b/open-sse/executors/codex.ts@@ -271,6 +271,8 @@ function convertSystemToDeveloperRole(body: Record<string, unknown>): void {* server-generated prefix (rs_, fc_, resp_, msg_) — so the content is* preserved but the backend won't try to look it up* 4. Always deletes previous_response_id (endpoint doesn't persist responses)+ *+ * @deprecated This function will be removed in v4.0, Codex executor has updated processing logic*/function stripStoredItemReferences(body: Record<string, unknown>): void {// Always strip previous_response_id — the /codex/responses endpoint does notdiff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.tsindex 49aefdbc8c..8206492282 100644--- a/open-sse/handlers/chatCore.ts+++ b/open-sse/handlers/chatCore.ts@@ -14,7 +14,12 @@ import { createRequestLogger } from "../utils/requestLogger.ts";import { getModelTargetFormat, PROVIDER_ID_TO_ALIAS } from "../config/providerModels.ts";import { resolveModelAlias } from "../services/modelDeprecation.ts";import { getUnsupportedParams } from "../config/providerRegistry.ts";-import { hasPerModelQuota, lockModelIfPerModelQuota } from "../services/accountFallback.ts";+import {+ hasPerModelQuota,+ lockModelIfPerModelQuota,+ isDailyQuotaExhausted,+ getMsUntilTomorrow,+} from "../services/accountFallback.ts";import { COOLDOWN_MS } from "../config/constants.ts";import {buildErrorBody,@@ -2109,17 +2114,22 @@ export async function handleChatCore({}} else if (errorType === PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED) {// Providers with per-model quotas — lock the model only, not the connection+ // Daily quota exhausted: lock until tomorrow; otherwise use default cooldown+ const isDailyQuota = isDailyQuotaExhausted(message);+ const quotaCooldownMs = isDailyQuota+ ? getMsUntilTomorrow()+ : retryAfterMs || COOLDOWN_MS.rateLimit;if (lockModelIfPerModelQuota(provider,connectionId,model,- "quota_exhausted",- retryAfterMs || COOLDOWN_MS.rateLimit+ isDailyQuota ? "daily_quota_exhausted" : "quota_exhausted",+ quotaCooldownMs)) {console.warn(- `[provider] Node ${connectionId} model-only quota exhausted (${statusCode}) for ${model} - ${Math.ceil((retryAfterMs || COOLDOWN_MS.rateLimit) / 1000)}s (connection stays active)`+ `[provider] Node ${connectionId} ${isDailyQuota ? "daily " : ""}quota exhausted (${statusCode}) for ${model} - ${Math.ceil(quotaCooldownMs / 1000)}s (connection stays active)`);} else {await updateProviderConnection(connectionId, {diff --git a/open-sse/index.ts b/open-sse/index.tsindex fffb5ea17e..2c49410688 100644--- a/open-sse/index.ts+++ b/open-sse/index.ts@@ -50,6 +50,9 @@ export {isAccountUnavailable,getUnavailableUntil,filterAvailableAccounts,+ isProviderInCooldown,+ getProviderCooldownRemainingMs,+ getProvidersInCooldown,} from "./services/accountFallback.ts";export {diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.tsindex 95bdf510df..efadc2c25d 100644--- a/open-sse/services/accountFallback.ts+++ b/open-sse/services/accountFallback.ts@@ -24,6 +24,30 @@ type ModelFailureState = {resetAfterMs: number;};+// Provider-level failure tracking for circuit breaker behavior+type ProviderFailureEntry = {+ failureCount: number;+ lastFailureAt: number;+ resetAfterMs: number;+ cooldownUntil: number | null;+};++// Error codes that count toward provider-level failure threshold+const PROVIDER_FAILURE_ERROR_CODES = new Set([429, 408, 500, 502, 503, 504]);++// Configuration for provider-level failure tracking+const PROVIDER_FAILURE_THRESHOLD = 5;+const PROVIDER_FAILURE_WINDOW_MS = 20 * 60 * 1000; // 20 minutes+const PROVIDER_COOLDOWN_MS = 10 * 60 * 1000; // 10 minutes cooling++// Provider-level failure state map: providerId -> failure entry+const providerFailureState = new Map<string, ProviderFailureEntry>();+// Guard against synchronous re-entrant calls within the same event-loop tick.+// NOT a true mutex — Node.js is single-threaded, so different SSE streams+// can interleave across ticks. This Set prevents a single call from recursively+// re-entering recordProviderFailure within the same synchronous call stack.+const providerFailureLocks = new Set<string>();+// T06 (sub2api PR #1037): Signals that indicate permanent account deactivation.// When a 401 body contains these strings, the account is permanently dead// and should NOT be retried after token refresh.@@ -82,8 +106,12 @@ const CONTEXT_OVERFLOW_PATTERNS = [const MALFORMED_REQUEST_PATTERNS = [/\bimproperly formed request\b/i,/\binvalid.*message.*format/i,- /\bmessages must alternate/i,- /\bempty (message|content)/i,+ /\bmessages must alternate\b/i,+ /\bempty (message|content)\b/i,+ // Tool call function name errors+ /\bfunction'?s? name (?:can't|can not|is|has) (?:blank|empty|missing)/i,+ /function.*name.*(?:blank|empty|missing)/i,+ /tool_call.*name.*(?:blank|empty|missing)/i,];/**@@ -355,11 +383,21 @@ export function clearModelLock(provider, connectionId, model) {* Compatible and passthrough providers multiplex multiple upstream models behind one* connection, so transient 404/429 responses should stay model-scoped instead of* poisoning the whole connection.+ *+ * @param provider - Provider ID+ * @param _model - Model ID (reserved for future use)+ * @param connectionPassthroughModels - Optional per-connection override from providerSpecificData.+ * When provided, takes precedence over registry/provider-level logic.*/export function hasPerModelQuota(provider: string | null | undefined,- _model: string | null | undefined = null+ _model: string | null | undefined = null,+ connectionPassthroughModels?: boolean): boolean {+ // Connection-level override takes precedence (e.g., user-configured ModelScope)+ if (typeof connectionPassthroughModels === "boolean") {+ return connectionPassthroughModels;+ }if (!provider) return false;if (provider === "gemini") return true;if (getPassthroughProviders().has(provider)) return true;@@ -376,18 +414,23 @@ export function lockModelIfPerModelQuota(connectionId: string,model: string | null,reason: string,- cooldownMs: number+ cooldownMs: number,+ connectionPassthroughModels?: boolean): boolean {- if (!hasPerModelQuota(provider, model) || !model) return false;+ if (!hasPerModelQuota(provider, model, connectionPassthroughModels) || !model) return false;+ // Skip model-level lock if the entire provider is in circuit-breaker cooldown.+ // The provider cooldown already prevents all requests, so a model lock is redundant.+ if (isProviderInCooldown(provider)) return false;lockModel(provider, connectionId, model, reason, cooldownMs);return true;}export function shouldMarkAccountExhaustedFrom429(provider: string | null | undefined,- model: string | null | undefined = null+ model: string | null | undefined = null,+ connectionPassthroughModels?: boolean): boolean {- return !hasPerModelQuota(provider, model);+ return !hasPerModelQuota(provider, model, connectionPassthroughModels);}/**@@ -442,6 +485,145 @@ export function getAllModelLockouts() {return active;}+// ─── Provider-Level Failure Tracking ─────────────────────────────────────────+// Track failures at provider level: when a provider has too many transient failures+// across all its connections, cooldown the entire provider temporarily.++/**+ * Check if a provider is currently in cooldown due to too many failures+ */+export function isProviderInCooldown(provider: string | null | undefined): boolean {+ if (!provider) return false;+ const entry = providerFailureState.get(provider);+ if (!entry) return false;++ // If in cooldown, check if it has expired+ if (entry.cooldownUntil !== null && Date.now() >= entry.cooldownUntil) {+ providerFailureState.delete(provider);+ return false;+ }++ return entry.cooldownUntil !== null;+}++/**+ * Get remaining cooldown time for a provider+ */+export function getProviderCooldownRemainingMs(provider: string | null | undefined): number | null {+ if (!provider) return null;+ const entry = providerFailureState.get(provider);+ if (!entry || entry.cooldownUntil === null) return null;++ const remaining = entry.cooldownUntil - Date.now();+ return remaining > 0 ? remaining : null;+}++/**+ * Record a failure for a provider. When threshold is reached within the window,+ * the provider enters cooldown.+ */+export function recordProviderFailure(+ provider: string | null | undefined,+ log?: { warn?: (...args: unknown[]) => void }+): void {+ if (!provider) return;++ // Guard against concurrent re-entrant calls within the same tick+ if (providerFailureLocks.has(provider)) return;+ providerFailureLocks.add(provider);++ try {+ const now = Date.now();+ const entry = providerFailureState.get(provider);++ // Check if we're in cooldown period+ if (entry && entry.cooldownUntil !== null && now < entry.cooldownUntil) {+ return; // Already in cooldown, don't record+ }++ // Check if failure window has expired+ if (entry && now - entry.lastFailureAt > entry.resetAfterMs) {+ // Window expired, reset count+ providerFailureState.set(provider, {+ failureCount: 1,+ lastFailureAt: now,+ resetAfterMs: PROVIDER_FAILURE_WINDOW_MS,+ cooldownUntil: null,+ });+ return;+ }++ // Increment failure count+ const newCount = entry ? entry.failureCount + 1 : 1;++ if (newCount >= PROVIDER_FAILURE_THRESHOLD) {+ // Threshold reached, enter cooldown+ const cooldownUntil = now + PROVIDER_COOLDOWN_MS;+ providerFailureState.set(provider, {+ failureCount: newCount,+ lastFailureAt: now,+ resetAfterMs: PROVIDER_FAILURE_WINDOW_MS,+ cooldownUntil,+ });+ log?.warn?.(+ `[ProviderFailure] ${provider}: ${newCount} failures in ${PROVIDER_FAILURE_WINDOW_MS / 1000}s — entering ${PROVIDER_COOLDOWN_MS / 1000}s cooldown`+ );+ } else {+ // Just increment counter+ providerFailureState.set(provider, {+ failureCount: newCount,+ lastFailureAt: now,+ resetAfterMs: PROVIDER_FAILURE_WINDOW_MS,+ cooldownUntil: null,+ });+ }+ } finally {+ providerFailureLocks.delete(provider);+ }+}++/**+ * Clear provider failure state (e.g., after successful request)+ */+export function clearProviderFailure(provider: string | null | undefined): void {+ if (!provider) return;+ providerFailureState.delete(provider);+}++/**+ * Get all providers currently in cooldown (for debugging/dashboard)+ */+export function getProvidersInCooldown(): Array<{+ provider: string;+ failureCount: number;+ cooldownRemainingMs: number | null;+ lastFailureAt: number;+}> {+ const result = [];+ for (const [provider, entry] of providerFailureState) {+ if (entry.cooldownUntil === null) continue;+ const remaining = entry.cooldownUntil - Date.now();+ if (remaining <= 0) {+ providerFailureState.delete(provider);+ continue;+ }+ result.push({+ provider,+ failureCount: entry.failureCount,+ cooldownRemainingMs: remaining,+ lastFailureAt: entry.lastFailureAt,+ });+ }+ return result;+}++/**+ * Check if a status code should be counted toward provider failure threshold+ */+export function isProviderFailureCode(status: number): boolean {+ return PROVIDER_FAILURE_ERROR_CODES.has(status);+}+// ─── Retry-After Parsing ────────────────────────────────────────────────────/**@@ -625,6 +807,40 @@ export function classifyError(status, errorText) {return RateLimitReason.UNKNOWN;}+// ─── Daily Quota Helpers ────────────────────────────────────────────────────++/**+ * Calculate milliseconds from now until tomorrow at midnight (00:00:00).+ * Used to lock a model until the next day when daily quota is exhausted.+ * @returns {number} Milliseconds until tomorrow+ */+export function getMsUntilTomorrow(): number {+ const now = new Date();+ const tomorrow = new Date(now);+ tomorrow.setDate(tomorrow.getDate() + 1);+ tomorrow.setHours(0, 0, 0, 0);+ const ms = tomorrow.getTime() - now.getTime();+ // Guard against DST edge cases: if ms is negative (shouldn't happen) or+ // unreasonably large (>25h due to spring-forward), cap at 24 hours.+ return ms > 0 && ms <= 25 * 60 * 60 * 1000 ? ms : 24 * 60 * 60 * 1000;+}++/**+ * Check if error text indicates daily quota exhaustion (as opposed to rate limiting).+ * Daily quota errors typically mention "today's quota" or "try again tomorrow".+ * @param {string} errorText - Error message text+ * @returns {boolean} True if daily quota is exhausted+ */+export function isDailyQuotaExhausted(errorText: string): boolean {+ if (!errorText) return false;+ const lower = errorText.toLowerCase();+ return (+ lower.includes("today's quota") ||+ lower.includes("daily quota") ||+ lower.includes("try again tomorrow")+ );+}+// ─── Configurable Backoff ───────────────────────────────────────────────────/**@@ -681,6 +897,12 @@ export function checkFallbackError(const errorStr = (errorText || "").toString();const profile = profileOverride ?? (provider ? getProviderProfile(provider) : null);+ // Track provider-level failures for circuit breaker behavior+ // Only count transient errors that are likely to recover+ if (isProviderFailureCode(status)) {+ recordProviderFailure(provider);+ }+function parseResetFromHeaders(headers, errorStr = "") {if (!headers) return null;@@ -737,6 +959,19 @@ export function checkFallbackError(};}+ // Daily quota exhausted — lock model until tomorrow+ if (isDailyQuotaExhausted(errorStr)) {+ const msUntilTomorrow = getMsUntilTomorrow();+ // Cap at 24 hours to handle timezone edge cases+ const cooldownMs = Math.min(msUntilTomorrow, 24 * 60 * 60 * 1000);+ return {+ shouldFallback: true,+ cooldownMs,+ reason: RateLimitReason.QUOTA_EXHAUSTED,+ dailyQuotaExhausted: true,+ };+ }+if (lowerError.includes("no credentials")) {return {shouldFallback: true,diff --git a/open-sse/services/cloudCodeThinking.ts b/open-sse/services/cloudCodeThinking.tsindex 769c414d7b..9d3b3b944a 100644--- a/open-sse/services/cloudCodeThinking.ts+++ b/open-sse/services/cloudCodeThinking.ts@@ -23,6 +23,9 @@ function stripGeminiThinkingConfig(value: unknown): unknown {return next;}+/**+ * @deprecated This function will be removed in v4.0, reasoning configuration processing has migrated to translateRequest+ */export function shouldStripCloudCodeThinking(provider: string, model: string): boolean {if (!provider || !model) return false;const normalizedModel = normalizeCloudCodeModel(model);@@ -33,6 +36,9 @@ export function shouldStripCloudCodeThinking(provider: string, model: string): breturn CLOUD_CODE_REASONING_UNSUPPORTED_PATTERNS.some((pattern) => pattern.test(normalizedModel));}+/**+ * @deprecated This function will be removed in v4.0, reasoning configuration processing has migrated to translateRequest+ */export function stripCloudCodeThinkingConfig(body: Record<string, unknown>): Record<string, unknown> {diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.tsindex dcbd63a457..796a8a58da 100644--- a/open-sse/services/combo.ts+++ b/open-sse/services/combo.ts@@ -59,6 +59,47 @@ const COMBO_BAD_REQUEST_FALLBACK_PATTERNS = [/unsupported content part type/i,/tool(?:_call|_use)? .* not (?:available|found)/i,/third-party apps/i,+ // Context overflow — model-specific, may succeed on a model with larger context window+ /context overflow/i,+ /context length exceeded/i,+ /prompt too large/i,+ /token limit/i,+ /too many tokens/i,+ /exceeds? context/i,+ /maximum context/i,+ /input too long/i,+ /messages? exceed/i,+ // Model not supported/found — permanent model-level error, try next combo target+ /no provider supported/i,+ /model not found/i,+ /model not available/i,+ /unsupported model/i,+ /model.*has no provider/i,+ // Function calling format error — model doesn't support this capability+ /function\.?arguments.*(must be|should be|必须).*(json|JSON)/i,+ /tool.*arguments.*invalid/i,+ /function.*parameter.*(invalid|format)/i,+ // Input length range error — model-specific context limit+ /range of input length/i,+ /input length should be/i,+ // Transient 400 errors from upstream — should fallback to next combo target+ /服务遇到了一点小状况/i, // ModelScope/Qwen transient error+ /抱歉.*?敏感内容.*?请检查/i, // ModelScope/Qwen content moderation with context+ /内容.*?敏感.*?(?:无法|过滤)/i, // Content sensitivity block+ /无法响应.*?请求/i, // "unable to respond to request"+ /稍后重试/i, // "retry later" in Chinese+ /temporary.*error/i,+ /transient.*error/i,+ /service.*unavailable/i,+ /please.*try.*again/i,+ // Rate limit errors — some providers return 400 instead of 429+ /\brate.?-?limit.?(?:exceeded|reached|hit)/i,+ /too many requests/i,+ /请求过于频繁/i, // Chinese rate limit message+ // Tool call function name errors — model-specific, try next combo target+ /\bfunction'?s? name (?:can't|can not|is|has) (?:blank|empty|missing)/i,+ /function.*name.*(?:blank|empty|missing)/i,+ /tool_call.*name.*(?:blank|empty|missing)/i,];// Patterns that signal all accounts for a provider are rate-limited / exhausted.@@ -77,6 +118,7 @@ function isAllAccountsRateLimitedResponse(const MAX_COMBO_DEPTH = 3;const MAX_FALLBACK_WAIT_MS = 5000;+const MAX_GLOBAL_ATTEMPTS = 30;function comboModelNotFoundResponse(message: string) {return errorResponse(404, message);@@ -1451,6 +1493,7 @@ export async function handleComboChat({let earliestRetryAfter = null;let lastStatus = null;const startTime = Date.now();+ globalAttempts = 0;let fallbackCount = 0;let recordedAttempts = 0;@@ -1484,6 +1527,14 @@ export async function handleComboChat({// Retry loop for transient errorsfor (let retry = 0; retry <= maxRetries; retry++) {+ globalAttempts++;+ if (globalAttempts > MAX_GLOBAL_ATTEMPTS) {+ log.warn(+ "COMBO",+ `Maximum combo attempts (${MAX_GLOBAL_ATTEMPTS}) exceeded across all targets and fallbacks. Terminating loop to prevent runaway background requests.`+ );+ return errorResponse(503, "Maximum combo retry limit reached");+ }if (retry > 0) {log.info("COMBO",@@ -1799,6 +1850,7 @@ async function handleRoundRobinCombo({let lastError = null;let lastStatus = null;let earliestRetryAfter = null;+ let globalAttempts = 0;let fallbackCount = 0;let recordedAttempts = 0;@@ -1852,6 +1904,14 @@ async function handleRoundRobinCombo({// Retry loop within this modeltry {for (let retry = 0; retry <= maxRetries; retry++) {+ globalAttempts++;+ if (globalAttempts > MAX_GLOBAL_ATTEMPTS) {+ log.warn(+ "COMBO-RR",+ `Maximum combo attempts (${MAX_GLOBAL_ATTEMPTS}) exceeded. Terminating loop to prevent runaway requests.`+ );+ return errorResponse(503, "Maximum combo retry limit reached");+ }if (retry > 0) {log.info("COMBO-RR",diff --git a/open-sse/services/errorClassifier.ts b/open-sse/services/errorClassifier.tsindex 39d513ffbe..4520c2a6e4 100644--- a/open-sse/services/errorClassifier.ts+++ b/open-sse/services/errorClassifier.ts@@ -1,6 +1,7 @@import {isAccountDeactivated,isCreditsExhausted,+ isDailyQuotaExhausted,isOAuthInvalidToken,} from "./accountFallback.ts";@@ -100,7 +101,11 @@ export function classifyProviderError(statusCode: number, responseBody: unknown)return PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED;}+ // 429: Check if it's a daily quota exhaustion (lock until tomorrow) vs regular rate limitif (statusCode === 429) {+ if (isDailyQuotaExhausted(bodyStr)) {+ return PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED;+ }return PROVIDER_ERROR_TYPES.RATE_LIMITED;}diff --git a/src/app/(dashboard)/dashboard/combos/IntelligentComboPanel.tsx b/src/app/(dashboard)/dashboard/combos/IntelligentComboPanel.tsxindex 0a22d20e07..55013eab4d 100644--- a/src/app/(dashboard)/dashboard/combos/IntelligentComboPanel.tsx+++ b/src/app/(dashboard)/dashboard/combos/IntelligentComboPanel.tsx@@ -11,6 +11,7 @@ import {extractIntelligentHealthState,normalizeIntelligentRoutingConfig,} from "@/lib/combos/intelligentRouting";+import ModelStatusBadge from "@/app/(dashboard)/dashboard/providers/components/ModelStatusBadge";function getI18nOrFallback(t: any, key: string, fallback: string) {if (typeof t?.has === "function" && t.has(key)) return t(key);@@ -283,6 +284,8 @@ export default function IntelligentComboPanel({</p><p className="text-[11px] text-text-muted mt-0.5">{entry.model}</p></div>+ {/* Model status badge - shows cooldown/error state */}+ <ModelStatusBadge provider={entry.provider} model={entry.model} size="sm" /><span className="rounded-full bg-primary/10 px-2 py-1 text-[11px] font-semibold text-primary">{percentage}%</span>diff --git a/src/app/(dashboard)/dashboard/providers/[id]/page.tsx b/src/app/(dashboard)/dashboard/providers/[id]/page.tsxindex 2b9d0abfc4..9164e05718 100644--- a/src/app/(dashboard)/dashboard/providers/[id]/page.tsx+++ b/src/app/(dashboard)/dashboard/providers/[id]/page.tsx@@ -53,6 +53,7 @@ import {getCodexRequestDefaults as _getCodexRequestDefaults,} from "@/lib/providers/requestDefaults";import { resolveDashboardProviderInfo } from "../providerPageUtils";+import ModelStatusBadge from "@/app/(dashboard)/dashboard/providers/components/ModelStatusBadge";type CompatByProtocolMap = Partial<Record<@@ -326,6 +327,7 @@ function anyUpstreamHeadersBadge(interface ModelRowProps {model: { id: string; name?: string; source?: string; isHidden?: boolean };fullModel: string;+ provider: string;copied?: string;onCopy: (text: string, key: string) => void;t: (key: string, values?: Record<string, unknown>) => string;@@ -2460,6 +2462,7 @@ export default function ProviderDetailPage() {key={model.id}model={model}fullModel={`${providerDisplayAlias}/${model.id}`}+ provider={providerId}copied={copied}onCopy={copy}t={t}@@ -3329,6 +3332,7 @@ export default function ProviderDetailPage() {function ModelRow({model,fullModel,+ provider,copied,onCopy,t,@@ -3358,6 +3362,7 @@ function ModelRow({<code className="rounded bg-sidebar px-1.5 py-0.5 font-mono text-xs text-text-muted">{fullModel}</code>+ <ModelStatusBadge provider={provider} model={model.id} size="sm" /><ModelSourceBadge source={model.source} /><buttononClick={() => onCopy(fullModel, `model-${model.id}`)}@@ -3407,6 +3412,7 @@ ModelRow.propTypes = {id: PropTypes.string.isRequired,}).isRequired,fullModel: PropTypes.string.isRequired,+ provider: PropTypes.string.isRequired,copied: PropTypes.string,onCopy: PropTypes.func.isRequired,t: PropTypes.func,@@ -5402,6 +5408,7 @@ function AddApiKeyModal({accountId: "",consoleApiKey: "",ccCompatibleContext1m: false,+ passthroughModels: false,});const [validating, setValidating] = useState(false);const [validationResult, setValidationResult] = useState(null);@@ -5495,6 +5502,9 @@ function AddApiKeyModal({if (formData.excludedModels.trim()) {providerSpecificData.excludedModels = parseExcludedModelsInput(formData.excludedModels);}+ if (formData.passthroughModels) {+ providerSpecificData.passthroughModels = true;+ }if (provider === "bailian-coding-plan" && formData.consoleApiKey.trim()) {providerSpecificData.consoleApiKey = formData.consoleApiKey.trim();}@@ -5685,6 +5695,13 @@ function AddApiKeyModal({placeholder="gpt-5*, claude-opus-*, gemini-*-pro*"hint="Comma-separated wildcard patterns. This connection will never serve matching models."/>+ <Toggle+ size="sm"+ checked={formData.passthroughModels}+ onChange={(checked) => setFormData({ ...formData, passthroughModels: checked })}+ label={t("perModelQuotaLabel")}+ description={t("perModelQuotaDescription")}+ />{provider === "bailian-coding-plan" && (<Inputlabel="Console API Key (Oracle)"@@ -5824,6 +5841,7 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConneccodexOpenaiStoreEnabled: false,consoleApiKey: "",ccCompatibleContext1m: false,+ passthroughModels: connection.providerSpecificData?.passthroughModels === true,});const [testing, setTesting] = useState(false);const [testResult, setTestResult] = useState(null);@@ -5891,6 +5909,7 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConneccodexOpenaiStoreEnabled: connection.providerSpecificData?.openaiStoreEnabled === true,consoleApiKey: existingConsoleApiKey,ccCompatibleContext1m: ccRequestDefaults.context1m,+ passthroughModels: connection.providerSpecificData?.passthroughModels === true,});// Load existing extra keys from providerSpecificDataconst existing = connection.providerSpecificData?.extraApiKeys;@@ -6031,6 +6050,8 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConnectags: parseRoutingTagsInput(formData.routingTags),excludedModels: parseExcludedModelsInput(formData.excludedModels),customUserAgent: formData.customUserAgent.trim(),+ // Only write when explicitly enabled; omit to let registry default take effect+ ...(formData.passthroughModels ? { passthroughModels: true } : {}),};if (connection.provider === "bailian-coding-plan") {if (formData.consoleApiKey.trim()) {@@ -6288,6 +6309,13 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConnecplaceholder="my-app/1.0"hint="Optional override sent upstream as the User-Agent header for this connection"/>+ <Toggle+ size="sm"+ checked={formData.passthroughModels}+ onChange={(checked) => setFormData({ ...formData, passthroughModels: checked })}+ label={t("perModelQuotaLabel")}+ description={t("perModelQuotaDescription")}+ />{connection.provider === "bailian-coding-plan" && (<Inputlabel="Console API Key (Oracle)"diff --git a/src/app/(dashboard)/dashboard/providers/components/ModelStatusBadge.tsx b/src/app/(dashboard)/dashboard/providers/components/ModelStatusBadge.tsxnew file mode 100644index 0000000000..82538a6d8d--- /dev/null+++ b/src/app/(dashboard)/dashboard/providers/components/ModelStatusBadge.tsx@@ -0,0 +1,162 @@+"use client";++/**+ * ModelStatusBadge — compact single-model status indicator+ *+ * Shows a small badge with status icon (cooldown/unavailable/error)+ * with a tooltip containing additional details like remaining cooldown time+ * or last error message.+ * Only renders for non-available models to keep the UI clean.+ *+ * Uses shared polling via ModelStatusContext to avoid redundant API calls.+ */++import { useState, useEffect, useRef } from "react";+import { useTranslations } from "next-intl";+import Tooltip from "@/shared/components/Tooltip";+import { useModelStatus } from "./ModelStatusContext";++interface ModelStatusBadgeProps {+ provider: string;+ model: string;+ size?: "sm" | "md";+ className?: string;+}++function formatRemainingTime(ms: number): string {+ if (ms <= 0) return "";+ const seconds = Math.ceil(ms / 1000);+ if (seconds < 60) return `${seconds}s`;+ const minutes = Math.floor(seconds / 60);+ const remainingSeconds = seconds % 60;+ if (minutes < 60) {+ return remainingSeconds > 0 ? `${minutes}m ${remainingSeconds}s` : `${minutes}m`;+ }+ const hours = Math.floor(minutes / 60);+ const remainingMinutes = minutes % 60;+ return remainingMinutes > 0 ? `${hours}h ${remainingMinutes}m` : `${hours}h`;+}++export default function ModelStatusBadge({+ provider,+ model,+ size = "sm",+ className = "",+}: ModelStatusBadgeProps) {+ const t = useTranslations("providers");+ const status = useModelStatus(provider, model);++ // Store the latest remaining time calculated by the interval+ const [displayRemainingMs, setDisplayRemainingMs] = useState<number | null>(null);++ // Use ref to track server-provided initial cooldown duration for countdown calculation+ const initialCooldownMsRef = useRef<number | null>(null);+ // Track when we started counting down+ const countdownStartRef = useRef<number | null>(null);++ // Set up countdown timer when cooldown begins+ useEffect(() => {+ if (status?.status !== "cooldown" || !status.remainingMs) {+ // Reset refs when not in cooldown (no setState here)+ initialCooldownMsRef.current = null;+ countdownStartRef.current = null;+ return;+ }++ // Initialize timing refs (no setState here)+ initialCooldownMsRef.current = status.remainingMs;+ countdownStartRef.current = Date.now();++ // Update countdown every second via interval callback (setState allowed here)+ const interval = setInterval(() => {+ if (!initialCooldownMsRef.current || !countdownStartRef.current) {+ return;+ }++ const elapsed = Date.now() - countdownStartRef.current;+ const newRemaining = Math.max(0, initialCooldownMsRef.current - elapsed);++ setDisplayRemainingMs(newRemaining);++ // Stop updating when countdown reaches zero+ if (newRemaining === 0) {+ clearInterval(interval);+ }+ }, 1000);++ return () => clearInterval(interval);+ }, [status?.status, status?.remainingMs]);++ // Derive the displayed remaining time: use countdown value if active, else use status value+ const remainingMs =+ status?.status === "cooldown" ? (displayRemainingMs ?? status.remainingMs ?? null) : null;++ // Don't render badge for available models (keep UI clean)+ if (!status || status.status === "available" || status.status === "unknown") {+ return null;+ }++ const getStatusColor = () => {+ switch (status.status) {+ case "cooldown":+ return "#f59e0b";+ case "unavailable":+ case "error":+ return "#ef4444";+ default:+ return "#6b7280";+ }+ };++ const getStatusIcon = () => {+ switch (status.status) {+ case "cooldown":+ return "schedule";+ case "unavailable":+ case "error":+ return "error";+ default:+ return "help";+ }+ };++ const getTooltipText = () => {+ switch (status.status) {+ case "cooldown": {+ const remaining = remainingMs !== null ? formatRemainingTime(remainingMs) : "";+ const reason = status.reason ? ` (${status.reason})` : "";+ const remainingText = remaining ? ` - ${remaining}` : "";+ return `${t("cooldown")}${reason}${remainingText}`;+ }+ case "unavailable":+ return `${t("unavailable")}${status.reason ? `: ${status.reason}` : ""}`;+ case "error":+ return `${t("error")}${status.lastError ? `: ${status.lastError}` : ""}`;+ default:+ return "";+ }+ };++ const color = getStatusColor();+ const sizeClasses = size === "sm" ? "px-1.5 py-0.5" : "px-2 py-1";+ const iconSize = size === "sm" ? "text-[12px]" : "text-[14px]";++ return (+ <Tooltip content={getTooltipText()} position="top" delayMs={200}>+ <span+ className={`inline-flex items-center gap-1 rounded-full text-[10px] font-semibold ${sizeClasses} ${className}`}+ style={{+ backgroundColor: `${color}15`,+ color: color,+ }}+ >+ <span className={`material-symbols-outlined ${iconSize}`} aria-hidden="true">+ {getStatusIcon()}+ </span>+ {status.status === "cooldown" && remainingMs !== null && (+ <span>{formatRemainingTime(remainingMs)}</span>+ )}+ </span>+ </Tooltip>+ );+}diff --git a/src/app/(dashboard)/dashboard/providers/components/ModelStatusContext.tsx b/src/app/(dashboard)/dashboard/providers/components/ModelStatusContext.tsxnew file mode 100644index 0000000000..3605a9defd--- /dev/null+++ b/src/app/(dashboard)/dashboard/providers/components/ModelStatusContext.tsx@@ -0,0 +1,186 @@+"use client";++/**+ * ModelStatusContext — shared polling for model availability+ *+ * Prevents redundant API calls by having all ModelStatusBadge components+ * share a single polling interval. Only one request is made every 15 seconds+ * regardless of how many badges are on the page.+ */++import React, { createContext, useContext, useEffect, useRef, useCallback, useMemo } from "react";++export interface ModelStatus {+ status: "available" | "cooldown" | "unavailable" | "error" | "unknown";+ reason?: string;+ remainingMs?: number;+ lastError?: string;+}++interface ModelStatusContextValue {+ getStatus: (provider: string, model: string) => ModelStatus | null;+ registerModel: (key: string, provider: string, model: string) => void;+ unregisterModel: (key: string) => void;+}++const ModelStatusContext = createContext<ModelStatusContextValue | null>(null);++// Global map of model key -> status+let modelStatusMap = new Map<string, ModelStatus>();+// Global set of registered model keys+let registeredModels = new Set<string>();+// Polling interval ref (singleton)+let pollIntervalRef: NodeJS.Timeout | null = null;++function getModelKey(provider: string, model: string): string {+ return `${provider}/${model}`;+}++async function fetchModelStatus(): Promise<void> {+ try {+ const res = await fetch("/api/models/availability");+ if (!res.ok) return;++ const json = await res.json();+ const models = json?.models || [];++ // Update all registered models with fresh data+ const now = Date.now();+ registeredModels.forEach((key) => {+ const [provider, model] = key.split("/");+ // Use exact matching first to avoid gpt-4 matching gpt-4-turbo incorrectly+ const modelEntry =+ models.find((m: any) => m.provider === provider && m.model === model) ||+ models.find(+ // Fallback to prefix matching only for models that contain the registered key+ // This handles cases like "gpt-4o" matching badge for "gpt-4"+ (m: any) =>+ m.provider === provider &&+ m.model &&+ model &&+ (m.model.startsWith(model + "-") || model.startsWith(m.model + "-"))+ );++ if (modelEntry) {+ const newStatus: ModelStatus = {+ status: modelEntry.status || "unknown",+ reason: modelEntry.reason,+ remainingMs: modelEntry.remainingMs,+ lastError: modelEntry.lastError,+ };++ // For cooldown status, calculate remaining time based on server-provided value+ if (modelEntry.status === "cooldown" && modelEntry.remainingMs) {+ newStatus.remainingMs = modelEntry.remainingMs;+ }++ modelStatusMap.set(key, newStatus);+ } else {+ modelStatusMap.set(key, { status: "available" });+ }+ });++ // Trigger re-render by dispatching custom event+ window.dispatchEvent(new CustomEvent("model-status-update"));+ } catch {+ // Best-effort polling+ }+}++function ensurePolling(): void {+ if (pollIntervalRef) return;++ // Initial fetch+ fetchModelStatus();++ // Poll every 15 seconds+ pollIntervalRef = setInterval(fetchModelStatus, 15000);+}++function stopPolling(): void {+ if (pollIntervalRef) {+ clearInterval(pollIntervalRef);+ pollIntervalRef = null;+ }+}++export function ModelStatusProvider({ children }: { children: React.ReactNode }) {+ const [, forceUpdate] = React.useState(0);++ // Listen for status updates from the global poller+ useEffect(() => {+ const handleUpdate = () => forceUpdate((n) => n + 1);+ window.addEventListener("model-status-update", handleUpdate);+ return () => window.removeEventListener("model-status-update", handleUpdate);+ }, []);++ // Cleanup on unmount — stop polling only when no models remain registered+ useEffect(() => {+ return () => {+ if (registeredModels.size === 0) {+ stopPolling();+ }+ };+ }, []);++ const getStatus = useCallback((provider: string, model: string): ModelStatus | null => {+ const key = getModelKey(provider, model);+ return modelStatusMap.get(key) || null;+ }, []);++ const registerModel = useCallback((key: string, provider: string, model: string): void => {+ const wasEmpty = registeredModels.size === 0;+ registeredModels.add(key);++ // Start polling when first model registers+ if (wasEmpty) {+ ensurePolling();+ }++ // Immediately fetch if no data yet+ if (!modelStatusMap.has(key)) {+ fetchModelStatus();+ }+ }, []);++ const unregisterModel = useCallback((key: string): void => {+ registeredModels.delete(key);+ modelStatusMap.delete(key);++ // Stop polling when last model unregisters+ if (registeredModels.size === 0) {+ stopPolling();+ }+ }, []);++ const value = useMemo(+ () => ({+ getStatus,+ registerModel,+ unregisterModel,+ }),+ [getStatus, registerModel, unregisterModel]+ );++ return <ModelStatusContext.Provider value={value}>{children}</ModelStatusContext.Provider>;+}++export function useModelStatus(provider: string, model: string): ModelStatus | null {+ const context = useContext(ModelStatusContext);++ if (!context) {+ throw new Error("useModelStatus must be used within a ModelStatusProvider");+ }++ const key = useMemo(() => getModelKey(provider, model), [provider, model]);++ // Register/unregister on mount/unmount+ useEffect(() => {+ context.registerModel(key, provider, model);+ return () => context.unregisterModel(key);+ }, [context, key, provider, model]);++ return context.getStatus(provider, model);+}++export default ModelStatusContext;diff --git a/src/app/(dashboard)/layout.tsx b/src/app/(dashboard)/layout.tsxindex e640e17628..8237e8eb34 100644--- a/src/app/(dashboard)/layout.tsx+++ b/src/app/(dashboard)/layout.tsx@@ -1,5 +1,10 @@import { DashboardLayout } from "@/shared/components";+import { ModelStatusProvider } from "./dashboard/providers/components/ModelStatusContext";export default function DashboardRootLayout({ children }) {- return <DashboardLayout>{children}</DashboardLayout>;+ return (+ <ModelStatusProvider>+ <DashboardLayout>{children}</DashboardLayout>+ </ModelStatusProvider>+ );}diff --git a/src/i18n/messages/en.json b/src/i18n/messages/en.jsonindex 9e1726d107..d0f1202c61 100644--- a/src/i18n/messages/en.json+++ b/src/i18n/messages/en.json@@ -282,6 +282,10 @@"evals": "Evals","utilization": "Utilization","utilizationDescription": "Provider quota usage trends and rate limit tracking",+ "modelStatus": "Model Status",+ "modelStatusCooldown": "Cooldown",+ "modelStatusUnavailable": "Unavailable",+ "modelStatusError": "Error","comboHealth": "Combo Health","comboHealthDescription": "Combo-level quota, usage distribution, and performance metrics"},@@ -644,7 +648,8 @@"kiro": "Use when integrating Kiro and controlling model routing centrally from OmniRoute.","windsurf": "Use when you want an AI-first IDE with Codeium/Windsurf models routed through OmniRoute.","antigravity": "Use when Antigravity/Kiro traffic must be intercepted through MITM and routed to OmniRoute.",- "copilot": "Use when you want Copilot chat style UX while enforcing OmniRoute keys and routing rules."+ "copilot": "Use when you want Copilot chat style UX while enforcing OmniRoute keys and routing rules.",+ "qwen": "Use when you need Alibaba Qwen Code CLI for coding tasks."},"toolDescriptions": {"antigravity": "Google Antigravity IDE with MITM",@@ -1108,6 +1113,7 @@"builderAccount": "Account","builderPreview": "Preview","builderAddStep": "Add step",+ "builderDuplicateExact": "This exact provider/model/account step is already in the combo.","builderComboRef": "Combo Ref","builderAddComboRef": "Add combo reference","builderComboRefStep": "Add combo reference",@@ -1865,6 +1871,9 @@"compatUpstreamAddRow": "Add header","compatUpstreamRemoveRow": "Remove row","compatBadgeUpstreamHeaders": "Headers",+ "perModelQuotaLabel": "Per-Model Quota",+ "perModelQuotaDescription": "When enabled, 429/404 errors only lock the specific model, not the entire connection. Use for providers with per-model rate limits (e.g., ModelScope).",+ "perModelQuotaToggle": "Per-Model Quota Toggle","modelId": "Model ID","customModelPlaceholder": "e.g. gpt-4.5-turbo","loading": "Loading...",diff --git a/src/i18n/messages/zh-CN.json b/src/i18n/messages/zh-CN.jsonindex f2e7706f0c..5f6f7e9655 100644--- a/src/i18n/messages/zh-CN.json+++ b/src/i18n/messages/zh-CN.json@@ -282,6 +282,10 @@"evals": "评测","utilization": "利用率","utilizationDescription": "提供商配额使用趋势和速率限制跟踪",+ "modelStatus": "模型状态",+ "modelStatusCooldown": "冷却中",+ "modelStatusUnavailable": "不可用",+ "modelStatusError": "错误","comboHealth": "组合健康状况","comboHealthDescription": "组合级别配额、使用分布和性能指标"},@@ -644,7 +648,8 @@"kiro": "在集成 Kiro 并从 OmniRoute 集中控制模型路由时使用。","windsurf": "当您需要 Windsurf AI IDE 并通过 OmniRoute 路由模型时使用。","antigravity": "当必须通过 MITM 拦截 Antigravity/Kiro 流量并将其路由到 OmniRoute 时使用。",- "copilot": "当您想要 Copilot 聊天风格的 UX 同时强制执行 OmniRoute 键和路由规则时使用。"+ "copilot": "当您想要 Copilot 聊天风格的 UX 同时强制执行 OmniRoute 键和路由规则时使用。",+ "qwen": "当您需要使用阿里云 Qwen Code CLI 进行编码任务时使用。"},"toolDescriptions": {"antigravity": "带 MITM 的 Google Antigravity IDE",@@ -1022,7 +1027,8 @@"selectAccount": "选择账户","autoSelectAccount": "运行时自动选择账户","previewNextStep": "选择提供商和模型以预览下一步。",- "addStep": "添加步骤",+ "builderAddStep": "添加步骤",+ "builderDuplicateExact": "该提供商/模型/账户步骤已存在于组合中。","comboReference": "组合引用","selectComboToReference": "选择要引用的现有组合","addComboReference": "添加组合引用",@@ -1911,6 +1917,9 @@"compatUpstreamAddRow": "添加请求头","compatUpstreamRemoveRow": "删除此行","compatBadgeUpstreamHeaders": "请求头",+ "perModelQuotaLabel": "按模型配额",+ "perModelQuotaDescription": "启用后,429/404错误只会锁定特定模型,而不是整个连接。适用于具有按模型速率限制的提供商(例如ModelScope)。",+ "perModelQuotaToggle": "按模型配额开关","modelId": "模型 ID","customModelPlaceholder": "例如:gpt-4.5-turbo","loading": "正在加载...",diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.tsindex 5bc6b9629b..70f546be1b 100644--- a/src/sse/handlers/chat.ts+++ b/src/sse/handlers/chat.ts@@ -841,7 +841,11 @@ async function handleSingleModelChat(// 6. Mark account as quota-exhausted on 429 response// For providers that route quota/cooldown at model scope, a 429 on one model// does not mean the whole connection is exhausted.- if (result.status === 429 && shouldMarkAccountExhaustedFrom429(provider, model)) {+ const passthroughModels = credentials.providerSpecificData?.passthroughModels;+ if (+ result.status === 429 &&+ shouldMarkAccountExhaustedFrom429(provider, model, passthroughModels)+ ) {markAccountExhaustedFrom429(credentials.connectionId, provider);}diff --git a/src/sse/handlers/chatHelpers.ts b/src/sse/handlers/chatHelpers.tsindex f091220573..ef5dd9213e 100644--- a/src/sse/handlers/chatHelpers.ts+++ b/src/sse/handlers/chatHelpers.ts@@ -27,7 +27,10 @@ import { getCircuitBreaker, CircuitBreakerOpenError } from "../../shared/utils/cimport { getModelCooldownInfo, isModelAvailable } from "../../domain/modelAvailability";import { logProxyEvent } from "../../lib/proxyLogger";import { logTranslationEvent } from "../../lib/translatorEvents";-import { getRuntimeProviderProfile } from "@omniroute/open-sse/services/accountFallback.ts";+import {+ getRuntimeProviderProfile,+ clearProviderFailure,+} from "@omniroute/open-sse/services/accountFallback.ts";export async function resolveModelOrError(modelStr: string, body: any, endpointPath: string = "") {const modelInfo = await getModelInfo(modelStr);@@ -170,6 +173,8 @@ export async function executeChatWithBreaker({},onRequestSuccess: async () => {await clearAccountError(credentials.connectionId, credentials);+ // Clear provider-level failure state on successful request+ clearProviderFailure(provider);},}));diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.tsindex 63c611db9d..ec601a3810 100644--- a/src/sse/services/auth.ts+++ b/src/sse/services/auth.ts@@ -19,6 +19,8 @@ import {hasPerModelQuota,getRuntimeProviderProfile,recordModelLockoutFailure,+ isProviderInCooldown,+ getProviderCooldownRemainingMs,} from "@omniroute/open-sse/services/accountFallback.ts";import { isLocalProvider } from "@omniroute/open-sse/config/providerRegistry.ts";import { COOLDOWN_MS } from "@omniroute/open-sse/config/constants.ts";@@ -682,6 +684,22 @@ export async function getProviderCredentials(return true;});+ // Check if the entire provider is in cooldown (too many transient failures)+ if (isProviderInCooldown(provider)) {+ const cooldownRemaining = getProviderCooldownRemainingMs(provider);+ log.warn(+ "AUTH",+ `${provider} | provider in cooldown for ${Math.ceil((cooldownRemaining || 0) / 1000)}s (${availableConnections.length} connections bypassed)`+ );+ return {+ allRateLimited: true,+ retryAfter: new Date(Date.now() + (cooldownRemaining || 0)).toISOString(),+ retryAfterHuman: formatRetryAfter(+ new Date(Date.now() + (cooldownRemaining || 0)).toISOString()+ ),+ };+ }+log.debug("AUTH",`${provider} | available: ${availableConnections.length}/${connections.length}`@@ -1189,7 +1207,13 @@ export async function markAccountUnavailable(const effectiveProviderProfile =providerProfile || (provider ? await getRuntimeProviderProfile(provider) : null);- const isPerModelQuotaProvider = hasPerModelQuota(provider, model);+ // Read passthroughModels from connection config (user-configured per-model quota)+ const connProviderSpecificData = (conn?.providerSpecificData as Record<string, unknown>) || {};+ const connectionPassthroughModels = connProviderSpecificData.passthroughModels as+ | boolean+ | undefined;++ const isPerModelQuotaProvider = hasPerModelQuota(provider, model, connectionPassthroughModels);if (isPerModelQuotaProvider && provider && model && (status === 404 || status === 429)) {const reason = status === 404 ? "not_found" : "rate_limited";const fallbackCooldown =diff --git a/tests/integration/chat-pipeline.test.ts b/tests/integration/chat-pipeline.test.tsindex 3723cc9411..10f5d366f4 100644--- a/tests/integration/chat-pipeline.test.ts+++ b/tests/integration/chat-pipeline.test.ts@@ -26,6 +26,7 @@ const { BaseExecutor } = await import("../../open-sse/executors/base.ts");const { resetAllAvailability, setModelUnavailable } =await import("../../src/domain/modelAvailability.ts");const { resetAllCircuitBreakers } = await import("../../src/shared/utils/circuitBreaker.ts");+const { clearProviderFailure } = await import("../../open-sse/services/accountFallback.ts");const originalFetch = globalThis.fetch;const originalRetryDelayMs = BaseExecutor.RETRY_CONFIG.delayMs;@@ -1017,6 +1018,8 @@ test("chat pipeline maps upstream timeouts to 504 responses", async () => {});test("chat pipeline injects memory context before sending the upstream request", async () => {+ // Reset provider failure state to avoid circuit breaker interference+ clearProviderFailure("openai");await seedConnection("openai", { apiKey: "sk-openai-memory" });const apiKey = await seedApiKey();await settingsDb.updateSettings({@@ -1057,6 +1060,8 @@ test("chat pipeline injects memory context before sending the upstream request",});test("chat pipeline injects skills into tools and intercepts tool calls with skill output", async () => {+ // Reset provider failure state to avoid circuit breaker interference+ clearProviderFailure("openai");await seedConnection("openai", { apiKey: "sk-openai-skills" });const apiKey = await seedApiKey();await settingsDb.updateSettings({ skillsEnabled: true });@@ -1118,6 +1123,8 @@ test("chat pipeline injects skills into tools and intercepts tool calls with ski});test("chat pipeline falls back to the next account after a provider failure", async () => {+ // Reset provider failure state to avoid circuit breaker interference+ clearProviderFailure("openai");await seedConnection("openai", {name: "openai-primary",apiKey: "sk-openai-primary-fallback",@@ -1162,6 +1169,9 @@ test("chat pipeline falls back to the next account after a provider failure", as});test("chat pipeline falls back across combo models when the first provider fails", async () => {+ // Reset provider failure state to avoid circuit breaker interference+ clearProviderFailure("openai");+ clearProviderFailure("claude");await seedConnection("openai", { apiKey: "sk-openai-combo-fail" });await seedConnection("claude", { apiKey: "sk-claude-combo-fail" });await combosDb.createCombo({@@ -1206,6 +1216,8 @@ test("chat pipeline falls back across combo models when the first provider fails});test("chat pipeline deduplicates concurrent identical non-stream requests", async () => {+ // Reset provider failure state to avoid circuit breaker interference+ clearProviderFailure("openai");await seedConnection("openai", { apiKey: "sk-openai-dedup" });let fetchCount = 0;diff --git a/tests/unit/account-fallback-service.test.ts b/tests/unit/account-fallback-service.test.tsindex 4f10e7e0a8..912aa9836e 100644--- a/tests/unit/account-fallback-service.test.ts+++ b/tests/unit/account-fallback-service.test.ts@@ -22,6 +22,12 @@ const {recordModelLockoutFailure,clearModelLock,shouldMarkAccountExhaustedFrom429,+ recordProviderFailure,+ isProviderInCooldown,+ getProviderCooldownRemainingMs,+ clearProviderFailure,+ isProviderFailureCode,+ getProvidersInCooldown,} = accountFallback;const { selectAccount } = accountSelector;@@ -290,3 +296,215 @@ test("recordModelLockoutFailure uses provider profile cooldowns, backoff, and reclearModelLock("openai-compatible-custom-node", "conn-compatible", "custom-model-a");}});++// Provider-level failure circuit breaker tests+test("isProviderFailureCode correctly identifies transient error codes", () => {+ assert.equal(isProviderFailureCode(429), true);+ assert.equal(isProviderFailureCode(408), true);+ assert.equal(isProviderFailureCode(500), true);+ assert.equal(isProviderFailureCode(502), true);+ assert.equal(isProviderFailureCode(503), true);+ assert.equal(isProviderFailureCode(504), true);+ assert.equal(isProviderFailureCode(401), false);+ assert.equal(isProviderFailureCode(403), false);+ assert.equal(isProviderFailureCode(400), false);+ assert.equal(isProviderFailureCode(404), false);+ assert.equal(isProviderFailureCode(200), false);+});++test("recordProviderFailure tracks failures and triggers cooldown after threshold", () => {+ const originalNow = Date.now;+ let now = 1_700_000_000_000;+ Date.now = () => now;++ try {+ const provider = "test-provider";++ // Clear any existing state+ clearProviderFailure(provider);+ assert.equal(isProviderInCooldown(provider), false);+ assert.equal(getProviderCooldownRemainingMs(provider), null);++ // Record 4 failures - should not trigger cooldown yet+ for (let i = 0; i < 4; i++) {+ recordProviderFailure(provider);+ now += 1000; // 1 second between failures+ }+ assert.equal(isProviderInCooldown(provider), false);++ // 5th failure - should trigger cooldown+ recordProviderFailure(provider);+ assert.equal(isProviderInCooldown(provider), true);++ const remaining = getProviderCooldownRemainingMs(provider);+ assert.ok(remaining !== null);+ assert.ok(remaining > 0);+ assert.ok(remaining <= 10 * 60 * 1000); // 10 minutes max++ // Check getProvidersInCooldown returns the provider+ const inCooldown = getProvidersInCooldown();+ assert.ok(inCooldown.some((p) => p.provider === provider));+ assert.equal(inCooldown.find((p) => p.provider === provider)?.failureCount, 5);++ // Simulate cooldown expiration+ now += 11 * 60 * 1000; // 11 minutes later+ assert.equal(isProviderInCooldown(provider), false);+ assert.equal(getProviderCooldownRemainingMs(provider), null);+ assert.equal(+ getProvidersInCooldown().some((p) => p.provider === provider),+ false+ );+ } finally {+ Date.now = originalNow;+ clearProviderFailure("test-provider");+ }+});++test("recordProviderFailure resets counter after failure window expires", () => {+ const originalNow = Date.now;+ let now = 1_700_000_000_000;+ Date.now = () => now;++ try {+ const provider = "test-provider-window";+ clearProviderFailure(provider);++ // Record 3 failures+ for (let i = 0; i < 3; i++) {+ recordProviderFailure(provider);+ now += 1000;+ }+ assert.equal(isProviderInCooldown(provider), false);++ // Wait for failure window to expire (20 minutes + 1 second)+ now += 20 * 60 * 1000 + 1000;++ // Next failure should reset counter, not trigger cooldown+ recordProviderFailure(provider);+ assert.equal(isProviderInCooldown(provider), false);++ // Need 4 more failures to trigger cooldown+ for (let i = 0; i < 4; i++) {+ recordProviderFailure(provider);+ now += 1000;+ }+ assert.equal(isProviderInCooldown(provider), true);+ } finally {+ Date.now = originalNow;+ clearProviderFailure("test-provider-window");+ }+});++test("checkFallbackError records provider failure for transient errors", () => {+ const originalNow = Date.now;+ let now = 1_700_000_000_000;+ Date.now = () => now;++ try {+ const provider = "test-provider-check";+ clearProviderFailure(provider);++ // Simulate 5 transient errors through checkFallbackError+ for (let i = 0; i < 5; i++) {+ checkFallbackError(429, "rate limited", 0, null, provider);+ now += 1000;+ }++ // Provider should now be in cooldown+ assert.equal(isProviderInCooldown(provider), true);+ } finally {+ Date.now = originalNow;+ clearProviderFailure("test-provider-check");+ }+});++test("checkFallbackError does not record provider failure for non-transient errors", () => {+ const originalNow = Date.now;+ let now = 1_700_000_000_000;+ Date.now = () => now;++ try {+ const provider = "test-provider-no-record";+ clearProviderFailure(provider);++ // Simulate 5 auth errors (401) - should NOT trigger provider cooldown+ for (let i = 0; i < 5; i++) {+ checkFallbackError(401, "unauthorized", 0, null, provider);+ now += 1000;+ }++ // Provider should NOT be in cooldown+ assert.equal(isProviderInCooldown(provider), false);+ } finally {+ Date.now = originalNow;+ clearProviderFailure("test-provider-no-record");+ }+});++test("clearProviderFailure removes provider from cooldown", () => {+ const originalNow = Date.now;+ let now = 1_700_000_000_000;+ Date.now = () => now;++ try {+ const provider = "test-provider-clear";+ clearProviderFailure(provider);++ // Trigger cooldown+ for (let i = 0; i < 5; i++) {+ recordProviderFailure(provider);+ now += 1000;+ }+ assert.equal(isProviderInCooldown(provider), true);++ // Clear the failure state+ clearProviderFailure(provider);+ assert.equal(isProviderInCooldown(provider), false);+ assert.equal(getProviderCooldownRemainingMs(provider), null);+ } finally {+ Date.now = originalNow;+ clearProviderFailure("test-provider-clear");+ }+});++// Daily quota exhausted detection tests+test("isDailyQuotaExhausted detects today's quota errors", () => {+ const { isDailyQuotaExhausted } = accountFallback;+ assert.equal(isDailyQuotaExhausted("You have exceeded today's quota for model X"), true);+ assert.equal(isDailyQuotaExhausted("exceeded your daily quota"), true);+ assert.equal(isDailyQuotaExhausted("Please try again tomorrow"), true);+ assert.equal(isDailyQuotaExhausted("rate limit exceeded"), false);+ assert.equal(isDailyQuotaExhausted(""), false);+ assert.equal(isDailyQuotaExhausted(null), false);+});++test("getMsUntilTomorrow returns positive value less than 24 hours", () => {+ const { getMsUntilTomorrow } = accountFallback;+ const ms = getMsUntilTomorrow();+ assert.ok(ms > 0, "should be positive");+ assert.ok(ms <= 24 * 60 * 60 * 1000, "should be <= 24 hours");+});++test("checkFallbackError locks model until tomorrow for daily quota exhaustion", () => {+ const result = checkFallbackError(+ 429,+ "You have exceeded today's quota for model moonshotai/Kimi-K2.5, please try again tomorrow"+ );+ assert.equal(result.shouldFallback, true);+ assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED);+ assert.equal(result.dailyQuotaExhausted, true);+ assert.ok(result.cooldownMs > 0, "cooldown should be positive");+ assert.ok(result.cooldownMs <= 24 * 60 * 60 * 1000, "cooldown should be <= 24 hours");+});++test("checkFallbackError detects daily quota with 'try again tomorrow'", () => {+ const result = checkFallbackError(429, "Please try again tomorrow");+ assert.equal(result.shouldFallback, true);+ assert.equal(result.dailyQuotaExhausted, true);+});++test("checkFallbackError detects daily quota with 'daily quota'", () => {+ const result = checkFallbackError(429, "You have exceeded your daily quota");+ assert.equal(result.shouldFallback, true);+ assert.equal(result.dailyQuotaExhausted, true);+});diff --git a/tests/unit/chat-route-coverage.test.ts b/tests/unit/chat-route-coverage.test.tsindex 7473e98e28..2c7b4ca71d 100644--- a/tests/unit/chat-route-coverage.test.ts+++ b/tests/unit/chat-route-coverage.test.ts@@ -21,6 +21,7 @@ const {const { getCircuitBreaker, STATE } = await import("../../src/shared/utils/circuitBreaker.ts");const { clearModelUnavailability } = await import("../../src/domain/modelAvailability.ts");+const { clearProviderFailure } = await import("../../open-sse/services/accountFallback.ts");const { getDefaultTaskModelMap, resetTaskRoutingStats, setTaskRoutingConfig } =await import("../../open-sse/services/taskAwareRouter.ts");@@ -409,6 +410,8 @@ test("handleChat maps upstream timeouts to HTTP 504", async () => {});test("handleChat uses the emergency fallback model on budget exhaustion", async () => {+ // Reset provider failure state to avoid circuit breaker interference+ clearProviderFailure("openai");await seedConnection("openai", { apiKey: "sk-openai-billing" });await seedConnection("nvidia", { apiKey: "sk-nvidia-fallback" });const seenBodies = [];@@ -448,6 +451,8 @@ test("handleChat uses the emergency fallback model on budget exhaustion", async});test("handleChat returns the primary budget error when emergency fallback also fails", async () => {+ // Reset provider failure state to avoid circuit breaker interference+ clearProviderFailure("openai");await seedConnection("openai", { apiKey: "sk-openai-billing-fail" });await seedConnection("nvidia", { apiKey: "sk-nvidia-fallback-fail" });const seenModels = [];diff --git a/tests/unit/combo-routing-engine.test.ts b/tests/unit/combo-routing-engine.test.tsindex c3e267277a..823dfbc3bb 100644--- a/tests/unit/combo-routing-engine.test.ts+++ b/tests/unit/combo-routing-engine.test.ts@@ -740,6 +740,52 @@ test("shouldFallbackComboBadRequest only flags known provider-scoped 400 patternassert.equal(shouldFallbackComboBadRequest(429, "prohibited_content"), false);assert.equal(shouldFallbackComboBadRequest(400, null), false);assert.equal(shouldFallbackComboBadRequest(400, "generic bad request"), false);+ // Chinese transient errors (ModelScope/Qwen)+ assert.equal(+ shouldFallbackComboBadRequest(400, "[400]: 抱歉,服务遇到了一点小状况,请您稍后重试。"),+ true+ );+ assert.equal(shouldFallbackComboBadRequest(400, "服务遇到了一点小状况"), true);+ assert.equal(shouldFallbackComboBadRequest(400, "请稍后重试"), true);+ // Model not supported errors+ assert.equal(+ shouldFallbackComboBadRequest(+ 400,+ "Model id : XiaomiMiMo/MiMo-V2-Flash , has no provider supported"+ ),+ true+ );+ assert.equal(shouldFallbackComboBadRequest(400, "no provider supported"), true);+ assert.equal(shouldFallbackComboBadRequest(400, "model not found"), true);+ assert.equal(shouldFallbackComboBadRequest(400, "model not available"), true);+ // Function calling format errors+ assert.equal(+ shouldFallbackComboBadRequest(400, "function.arguments parameter must be in JSON format"),+ true+ );+ assert.equal(+ shouldFallbackComboBadRequest(+ 400,+ '[400]: <400> InternalError.Algo.InvalidParameter: The "function.arguments" parameter of the code model must be in JSON format.'+ ),+ true+ );+ assert.equal(shouldFallbackComboBadRequest(400, "tool arguments invalid format"), true);+ // Input length range errors+ assert.equal(+ shouldFallbackComboBadRequest(400, "Range of input length should be [1, 98304]"),+ true+ );+ assert.equal(shouldFallbackComboBadRequest(400, "input length should be"), true);+ // Content moderation errors (should fallback to next model)+ assert.equal(+ shouldFallbackComboBadRequest(400, "抱歉,您的内容包含敏感内容,请检查后重试"),+ true+ );+ assert.equal(shouldFallbackComboBadRequest(400, "内容存在敏感信息,无法响应"), true);+ assert.equal(shouldFallbackComboBadRequest(400, "无法响应该请求"), true);+ // Generic "please check" should NOT match (was too broad before)+ assert.equal(shouldFallbackComboBadRequest(400, "请检查您的参数"), false);});test("handleComboChat accepts binary and Responses-style 200 bodies but falls through malformed success payloads", async () => {diff --git a/tests/unit/error-classifier.test.ts b/tests/unit/error-classifier.test.tsindex 13184d076c..846f89fb87 100644--- a/tests/unit/error-classifier.test.ts+++ b/tests/unit/error-classifier.test.ts@@ -65,3 +65,21 @@ test("classifyProviderError: 403 with project string as plain string body => PROconst result = classifyProviderError(403, body);assert.equal(result, PROVIDER_ERROR_TYPES.PROJECT_ROUTE_ERROR);});++test("classifyProviderError: 429 with daily quota signal => QUOTA_EXHAUSTED", () => {+ const body = JSON.stringify({+ error: {+ message:+ "You have exceeded today's quota for model moonshotai/Kimi-K2.5, please try again tomorrow",+ },+ });+ const result = classifyProviderError(429, body);+ assert.equal(result, PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED);+});++test("classifyProviderError: 429 with 'daily quota' signal => QUOTA_EXHAUSTED", () => {+ const result = classifyProviderError(429, {+ error: { message: "You have reached your daily quota limit" },+ });+ assert.equal(result, PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED);+});