Skip to content

Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors

feat: add ModelScope provider with circuit breaker and daily quota lock (#1430)

clousky2020d6cbdc158079
diff --git a/next.config.mjs b/next.config.mjs
index db1017ae84..375b966e96 100644
--- a/next.config.mjs
+++ b/next.config.mjs
@@ -28,6 +28,7 @@ const nextConfig = {
"./playwright-report/**/*",
"./app.__qa_backup/**/*",
"./tests/**/*",
+ "./logs/**/*",
],
},
serverExternalPackages: [
diff --git a/open-sse/config/providerRegistry.ts b/open-sse/config/providerRegistry.ts
index b48b3dc9d9..ede6e0cdff 100644
--- a/open-sse/config/providerRegistry.ts
+++ b/open-sse/config/providerRegistry.ts
@@ -1400,6 +1400,25 @@ export const REGISTRY: Record<string, RegistryEntry> = {
passthroughModels: true,
},
+ modelscope: {
+ id: "modelscope",
+ alias: "ms",
+ format: "openai",
+ executor: "default",
+ baseUrl: "https://api-inference.modelscope.cn/v1/chat/completions",
+ authType: "apikey",
+ authHeader: "bearer",
+ // ModelScope uses per-model quotas. Setting passthroughModels: true ensures 429/404
+ // only locks the specific model, not the entire connection. This allows fallback
+ // to other models on the same API key.
+ passthroughModels: true,
+ models: [
+ { id: "moonshotai/Kimi-K2.5", name: "Kimi K2.5" },
+ { id: "ZhipuAI/GLM-5", name: "GLM-5" },
+ { id: "stepfun-ai/Step-3.5-Flash", name: "Step-3.5-Flash" },
+ ],
+ },
+
// ── New Free Providers (2026) ─────────────────────────────────────────────
longcat: {
@@ -2128,8 +2147,14 @@ export function getUnsupportedParams(provider: string, modelId: string): readonl
const cached = _unsupportedParamsMap.get(modelId);
if (cached) return cached;
- // 3. Handle prefixed model IDs (e.g., "openai/o3" → "o3")
+ // 3. Handle prefixed model IDs (e.g., "openai/o3" → "o3", "moonshotai/Kimi-K2.5" → "moonshotai/Kimi-K2.5")
+ // ModelScope models have slash in ID, check both full ID and bare ID
if (modelId.includes("/")) {
+ // First check full model ID with provider prefix (e.g., "moonshotai/Kimi-K2.5")
+ const cachedWithPrefix = _unsupportedParamsMap.get(modelId);
+ if (cachedWithPrefix) return cachedWithPrefix;
+
+ // Fall back to bare ID (e.g., "Kimi-K2.5")
const bareId = modelId.split("/").pop() || "";
const bare = _unsupportedParamsMap.get(bareId);
if (bare) return bare;
@@ -2154,6 +2179,7 @@ export function getProviderCategory(provider: string): "oauth" | "apikey" {
* Derive the latest opus/sonnet/haiku model IDs from the `claude` registry entry.
* Picks the first model whose ID matches each family pattern — registry order
* determines precedence, so newer models should be listed first.
+ * @deprecated This function will be removed in v4.0, please use REGISTRY.claude?.models directly
*/
export function getClaudeCodeDefaultModels(): {
opus: string;
diff --git a/open-sse/executors/antigravity.ts b/open-sse/executors/antigravity.ts
index b8dc191b7b..b987232735 100644
--- a/open-sse/executors/antigravity.ts
+++ b/open-sse/executors/antigravity.ts
@@ -735,18 +735,45 @@ export class AntigravityExecutor extends BaseExecutor {
// consuming the stream. The client receives the unmodified SSE data.
if (response.body) {
let sseBuffer = "";
+ const decoder = new TextDecoder(); // Singleton for correct streaming decode
+ const MAX_BUFFER_SIZE = 16 * 1024; // Limit to prevent OOM on large streams
+
const passThrough = new TransformStream({
transform(chunk, controller) {
controller.enqueue(chunk);
// Accumulate text to scan for remainingCredits
try {
- const text = new TextDecoder().decode(chunk, { stream: true });
+ const text = decoder.decode(chunk, { stream: true });
sseBuffer += text;
+ // Limit buffer size to prevent unbounded growth
+ // Truncate only after a complete newline to avoid splitting SSE lines mid-payload
+ if (sseBuffer.length > MAX_BUFFER_SIZE) {
+ const lastNewline = sseBuffer.lastIndexOf(
+ "\n",
+ sseBuffer.length - MAX_BUFFER_SIZE
+ );
+ if (lastNewline !== -1) {
+ sseBuffer = sseBuffer.slice(lastNewline + 1);
+ } else {
+ // No newline found in discard region — buffer contains an incomplete SSE line.
+ // Discard it entirely to avoid returning malformed data; the remainingCredits
+ // parser won't find valid data in a truncated line anyway.
+ sseBuffer = "";
+ }
+ }
} catch {
/* decoding best-effort */
}
},
flush() {
+ // Final decode for any remaining bytes
+ try {
+ const text = decoder.decode(); // Flush pending bytes
+ sseBuffer += text;
+ } catch {
+ /* decoding best-effort */
+ }
+
// Parse the accumulated SSE data for remainingCredits
try {
const lines = sseBuffer.split("\n");
diff --git a/open-sse/executors/codex.ts b/open-sse/executors/codex.ts
index 899c4eb00d..fcbcbab1cd 100644
--- a/open-sse/executors/codex.ts
+++ b/open-sse/executors/codex.ts
@@ -271,6 +271,8 @@ function convertSystemToDeveloperRole(body: Record<string, unknown>): void {
* server-generated prefix (rs_, fc_, resp_, msg_) — so the content is
* preserved but the backend won't try to look it up
* 4. Always deletes previous_response_id (endpoint doesn't persist responses)
+ *
+ * @deprecated This function will be removed in v4.0, Codex executor has updated processing logic
*/
function stripStoredItemReferences(body: Record<string, unknown>): void {
// Always strip previous_response_id — the /codex/responses endpoint does not
diff --git a/open-sse/handlers/chatCore.ts b/open-sse/handlers/chatCore.ts
index 49aefdbc8c..8206492282 100644
--- a/open-sse/handlers/chatCore.ts
+++ b/open-sse/handlers/chatCore.ts
@@ -14,7 +14,12 @@ import { createRequestLogger } from "../utils/requestLogger.ts";
import { getModelTargetFormat, PROVIDER_ID_TO_ALIAS } from "../config/providerModels.ts";
import { resolveModelAlias } from "../services/modelDeprecation.ts";
import { getUnsupportedParams } from "../config/providerRegistry.ts";
-import { hasPerModelQuota, lockModelIfPerModelQuota } from "../services/accountFallback.ts";
+import {
+ hasPerModelQuota,
+ lockModelIfPerModelQuota,
+ isDailyQuotaExhausted,
+ getMsUntilTomorrow,
+} from "../services/accountFallback.ts";
import { COOLDOWN_MS } from "../config/constants.ts";
import {
buildErrorBody,
@@ -2109,17 +2114,22 @@ export async function handleChatCore({
}
} else if (errorType === PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED) {
// Providers with per-model quotas — lock the model only, not the connection
+ // Daily quota exhausted: lock until tomorrow; otherwise use default cooldown
+ const isDailyQuota = isDailyQuotaExhausted(message);
+ const quotaCooldownMs = isDailyQuota
+ ? getMsUntilTomorrow()
+ : retryAfterMs || COOLDOWN_MS.rateLimit;
if (
lockModelIfPerModelQuota(
provider,
connectionId,
model,
- "quota_exhausted",
- retryAfterMs || COOLDOWN_MS.rateLimit
+ isDailyQuota ? "daily_quota_exhausted" : "quota_exhausted",
+ quotaCooldownMs
)
) {
console.warn(
- `[provider] Node ${connectionId} model-only quota exhausted (${statusCode}) for ${model} - ${Math.ceil((retryAfterMs || COOLDOWN_MS.rateLimit) / 1000)}s (connection stays active)`
+ `[provider] Node ${connectionId} ${isDailyQuota ? "daily " : ""}quota exhausted (${statusCode}) for ${model} - ${Math.ceil(quotaCooldownMs / 1000)}s (connection stays active)`
);
} else {
await updateProviderConnection(connectionId, {
diff --git a/open-sse/index.ts b/open-sse/index.ts
index fffb5ea17e..2c49410688 100644
--- a/open-sse/index.ts
+++ b/open-sse/index.ts
@@ -50,6 +50,9 @@ export {
isAccountUnavailable,
getUnavailableUntil,
filterAvailableAccounts,
+ isProviderInCooldown,
+ getProviderCooldownRemainingMs,
+ getProvidersInCooldown,
} from "./services/accountFallback.ts";
export {
diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts
index 95bdf510df..efadc2c25d 100644
--- a/open-sse/services/accountFallback.ts
+++ b/open-sse/services/accountFallback.ts
@@ -24,6 +24,30 @@ type ModelFailureState = {
resetAfterMs: number;
};
+// Provider-level failure tracking for circuit breaker behavior
+type ProviderFailureEntry = {
+ failureCount: number;
+ lastFailureAt: number;
+ resetAfterMs: number;
+ cooldownUntil: number | null;
+};
+
+// Error codes that count toward provider-level failure threshold
+const PROVIDER_FAILURE_ERROR_CODES = new Set([429, 408, 500, 502, 503, 504]);
+
+// Configuration for provider-level failure tracking
+const PROVIDER_FAILURE_THRESHOLD = 5;
+const PROVIDER_FAILURE_WINDOW_MS = 20 * 60 * 1000; // 20 minutes
+const PROVIDER_COOLDOWN_MS = 10 * 60 * 1000; // 10 minutes cooling
+
+// Provider-level failure state map: providerId -> failure entry
+const providerFailureState = new Map<string, ProviderFailureEntry>();
+// Guard against synchronous re-entrant calls within the same event-loop tick.
+// NOT a true mutex — Node.js is single-threaded, so different SSE streams
+// can interleave across ticks. This Set prevents a single call from recursively
+// re-entering recordProviderFailure within the same synchronous call stack.
+const providerFailureLocks = new Set<string>();
+
// T06 (sub2api PR #1037): Signals that indicate permanent account deactivation.
// When a 401 body contains these strings, the account is permanently dead
// and should NOT be retried after token refresh.
@@ -82,8 +106,12 @@ const CONTEXT_OVERFLOW_PATTERNS = [
const MALFORMED_REQUEST_PATTERNS = [
/\bimproperly formed request\b/i,
/\binvalid.*message.*format/i,
- /\bmessages must alternate/i,
- /\bempty (message|content)/i,
+ /\bmessages must alternate\b/i,
+ /\bempty (message|content)\b/i,
+ // Tool call function name errors
+ /\bfunction'?s? name (?:can't|can not|is|has) (?:blank|empty|missing)/i,
+ /function.*name.*(?:blank|empty|missing)/i,
+ /tool_call.*name.*(?:blank|empty|missing)/i,
];
/**
@@ -355,11 +383,21 @@ export function clearModelLock(provider, connectionId, model) {
* Compatible and passthrough providers multiplex multiple upstream models behind one
* connection, so transient 404/429 responses should stay model-scoped instead of
* poisoning the whole connection.
+ *
+ * @param provider - Provider ID
+ * @param _model - Model ID (reserved for future use)
+ * @param connectionPassthroughModels - Optional per-connection override from providerSpecificData.
+ * When provided, takes precedence over registry/provider-level logic.
*/
export function hasPerModelQuota(
provider: string | null | undefined,
- _model: string | null | undefined = null
+ _model: string | null | undefined = null,
+ connectionPassthroughModels?: boolean
): boolean {
+ // Connection-level override takes precedence (e.g., user-configured ModelScope)
+ if (typeof connectionPassthroughModels === "boolean") {
+ return connectionPassthroughModels;
+ }
if (!provider) return false;
if (provider === "gemini") return true;
if (getPassthroughProviders().has(provider)) return true;
@@ -376,18 +414,23 @@ export function lockModelIfPerModelQuota(
connectionId: string,
model: string | null,
reason: string,
- cooldownMs: number
+ cooldownMs: number,
+ connectionPassthroughModels?: boolean
): boolean {
- if (!hasPerModelQuota(provider, model) || !model) return false;
+ if (!hasPerModelQuota(provider, model, connectionPassthroughModels) || !model) return false;
+ // Skip model-level lock if the entire provider is in circuit-breaker cooldown.
+ // The provider cooldown already prevents all requests, so a model lock is redundant.
+ if (isProviderInCooldown(provider)) return false;
lockModel(provider, connectionId, model, reason, cooldownMs);
return true;
}
export function shouldMarkAccountExhaustedFrom429(
provider: string | null | undefined,
- model: string | null | undefined = null
+ model: string | null | undefined = null,
+ connectionPassthroughModels?: boolean
): boolean {
- return !hasPerModelQuota(provider, model);
+ return !hasPerModelQuota(provider, model, connectionPassthroughModels);
}
/**
@@ -442,6 +485,145 @@ export function getAllModelLockouts() {
return active;
}
+// ─── Provider-Level Failure Tracking ─────────────────────────────────────────
+// Track failures at provider level: when a provider has too many transient failures
+// across all its connections, cooldown the entire provider temporarily.
+
+/**
+ * Check if a provider is currently in cooldown due to too many failures
+ */
+export function isProviderInCooldown(provider: string | null | undefined): boolean {
+ if (!provider) return false;
+ const entry = providerFailureState.get(provider);
+ if (!entry) return false;
+
+ // If in cooldown, check if it has expired
+ if (entry.cooldownUntil !== null && Date.now() >= entry.cooldownUntil) {
+ providerFailureState.delete(provider);
+ return false;
+ }
+
+ return entry.cooldownUntil !== null;
+}
+
+/**
+ * Get remaining cooldown time for a provider
+ */
+export function getProviderCooldownRemainingMs(provider: string | null | undefined): number | null {
+ if (!provider) return null;
+ const entry = providerFailureState.get(provider);
+ if (!entry || entry.cooldownUntil === null) return null;
+
+ const remaining = entry.cooldownUntil - Date.now();
+ return remaining > 0 ? remaining : null;
+}
+
+/**
+ * Record a failure for a provider. When threshold is reached within the window,
+ * the provider enters cooldown.
+ */
+export function recordProviderFailure(
+ provider: string | null | undefined,
+ log?: { warn?: (...args: unknown[]) => void }
+): void {
+ if (!provider) return;
+
+ // Guard against concurrent re-entrant calls within the same tick
+ if (providerFailureLocks.has(provider)) return;
+ providerFailureLocks.add(provider);
+
+ try {
+ const now = Date.now();
+ const entry = providerFailureState.get(provider);
+
+ // Check if we're in cooldown period
+ if (entry && entry.cooldownUntil !== null && now < entry.cooldownUntil) {
+ return; // Already in cooldown, don't record
+ }
+
+ // Check if failure window has expired
+ if (entry && now - entry.lastFailureAt > entry.resetAfterMs) {
+ // Window expired, reset count
+ providerFailureState.set(provider, {
+ failureCount: 1,
+ lastFailureAt: now,
+ resetAfterMs: PROVIDER_FAILURE_WINDOW_MS,
+ cooldownUntil: null,
+ });
+ return;
+ }
+
+ // Increment failure count
+ const newCount = entry ? entry.failureCount + 1 : 1;
+
+ if (newCount >= PROVIDER_FAILURE_THRESHOLD) {
+ // Threshold reached, enter cooldown
+ const cooldownUntil = now + PROVIDER_COOLDOWN_MS;
+ providerFailureState.set(provider, {
+ failureCount: newCount,
+ lastFailureAt: now,
+ resetAfterMs: PROVIDER_FAILURE_WINDOW_MS,
+ cooldownUntil,
+ });
+ log?.warn?.(
+ `[ProviderFailure] ${provider}: ${newCount} failures in ${PROVIDER_FAILURE_WINDOW_MS / 1000}s — entering ${PROVIDER_COOLDOWN_MS / 1000}s cooldown`
+ );
+ } else {
+ // Just increment counter
+ providerFailureState.set(provider, {
+ failureCount: newCount,
+ lastFailureAt: now,
+ resetAfterMs: PROVIDER_FAILURE_WINDOW_MS,
+ cooldownUntil: null,
+ });
+ }
+ } finally {
+ providerFailureLocks.delete(provider);
+ }
+}
+
+/**
+ * Clear provider failure state (e.g., after successful request)
+ */
+export function clearProviderFailure(provider: string | null | undefined): void {
+ if (!provider) return;
+ providerFailureState.delete(provider);
+}
+
+/**
+ * Get all providers currently in cooldown (for debugging/dashboard)
+ */
+export function getProvidersInCooldown(): Array<{
+ provider: string;
+ failureCount: number;
+ cooldownRemainingMs: number | null;
+ lastFailureAt: number;
+}> {
+ const result = [];
+ for (const [provider, entry] of providerFailureState) {
+ if (entry.cooldownUntil === null) continue;
+ const remaining = entry.cooldownUntil - Date.now();
+ if (remaining <= 0) {
+ providerFailureState.delete(provider);
+ continue;
+ }
+ result.push({
+ provider,
+ failureCount: entry.failureCount,
+ cooldownRemainingMs: remaining,
+ lastFailureAt: entry.lastFailureAt,
+ });
+ }
+ return result;
+}
+
+/**
+ * Check if a status code should be counted toward provider failure threshold
+ */
+export function isProviderFailureCode(status: number): boolean {
+ return PROVIDER_FAILURE_ERROR_CODES.has(status);
+}
+
// ─── Retry-After Parsing ────────────────────────────────────────────────────
/**
@@ -625,6 +807,40 @@ export function classifyError(status, errorText) {
return RateLimitReason.UNKNOWN;
}
+// ─── Daily Quota Helpers ────────────────────────────────────────────────────
+
+/**
+ * Calculate milliseconds from now until tomorrow at midnight (00:00:00).
+ * Used to lock a model until the next day when daily quota is exhausted.
+ * @returns {number} Milliseconds until tomorrow
+ */
+export function getMsUntilTomorrow(): number {
+ const now = new Date();
+ const tomorrow = new Date(now);
+ tomorrow.setDate(tomorrow.getDate() + 1);
+ tomorrow.setHours(0, 0, 0, 0);
+ const ms = tomorrow.getTime() - now.getTime();
+ // Guard against DST edge cases: if ms is negative (shouldn't happen) or
+ // unreasonably large (>25h due to spring-forward), cap at 24 hours.
+ return ms > 0 && ms <= 25 * 60 * 60 * 1000 ? ms : 24 * 60 * 60 * 1000;
+}
+
+/**
+ * Check if error text indicates daily quota exhaustion (as opposed to rate limiting).
+ * Daily quota errors typically mention "today's quota" or "try again tomorrow".
+ * @param {string} errorText - Error message text
+ * @returns {boolean} True if daily quota is exhausted
+ */
+export function isDailyQuotaExhausted(errorText: string): boolean {
+ if (!errorText) return false;
+ const lower = errorText.toLowerCase();
+ return (
+ lower.includes("today's quota") ||
+ lower.includes("daily quota") ||
+ lower.includes("try again tomorrow")
+ );
+}
+
// ─── Configurable Backoff ───────────────────────────────────────────────────
/**
@@ -681,6 +897,12 @@ export function checkFallbackError(
const errorStr = (errorText || "").toString();
const profile = profileOverride ?? (provider ? getProviderProfile(provider) : null);
+ // Track provider-level failures for circuit breaker behavior
+ // Only count transient errors that are likely to recover
+ if (isProviderFailureCode(status)) {
+ recordProviderFailure(provider);
+ }
+
function parseResetFromHeaders(headers, errorStr = "") {
if (!headers) return null;
@@ -737,6 +959,19 @@ export function checkFallbackError(
};
}
+ // Daily quota exhausted — lock model until tomorrow
+ if (isDailyQuotaExhausted(errorStr)) {
+ const msUntilTomorrow = getMsUntilTomorrow();
+ // Cap at 24 hours to handle timezone edge cases
+ const cooldownMs = Math.min(msUntilTomorrow, 24 * 60 * 60 * 1000);
+ return {
+ shouldFallback: true,
+ cooldownMs,
+ reason: RateLimitReason.QUOTA_EXHAUSTED,
+ dailyQuotaExhausted: true,
+ };
+ }
+
if (lowerError.includes("no credentials")) {
return {
shouldFallback: true,
diff --git a/open-sse/services/cloudCodeThinking.ts b/open-sse/services/cloudCodeThinking.ts
index 769c414d7b..9d3b3b944a 100644
--- a/open-sse/services/cloudCodeThinking.ts
+++ b/open-sse/services/cloudCodeThinking.ts
@@ -23,6 +23,9 @@ function stripGeminiThinkingConfig(value: unknown): unknown {
return next;
}
+/**
+ * @deprecated This function will be removed in v4.0, reasoning configuration processing has migrated to translateRequest
+ */
export function shouldStripCloudCodeThinking(provider: string, model: string): boolean {
if (!provider || !model) return false;
const normalizedModel = normalizeCloudCodeModel(model);
@@ -33,6 +36,9 @@ export function shouldStripCloudCodeThinking(provider: string, model: string): b
return CLOUD_CODE_REASONING_UNSUPPORTED_PATTERNS.some((pattern) => pattern.test(normalizedModel));
}
+/**
+ * @deprecated This function will be removed in v4.0, reasoning configuration processing has migrated to translateRequest
+ */
export function stripCloudCodeThinkingConfig(
body: Record<string, unknown>
): Record<string, unknown> {
diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts
index dcbd63a457..796a8a58da 100644
--- a/open-sse/services/combo.ts
+++ b/open-sse/services/combo.ts
@@ -59,6 +59,47 @@ const COMBO_BAD_REQUEST_FALLBACK_PATTERNS = [
/unsupported content part type/i,
/tool(?:_call|_use)? .* not (?:available|found)/i,
/third-party apps/i,
+ // Context overflow — model-specific, may succeed on a model with larger context window
+ /context overflow/i,
+ /context length exceeded/i,
+ /prompt too large/i,
+ /token limit/i,
+ /too many tokens/i,
+ /exceeds? context/i,
+ /maximum context/i,
+ /input too long/i,
+ /messages? exceed/i,
+ // Model not supported/found — permanent model-level error, try next combo target
+ /no provider supported/i,
+ /model not found/i,
+ /model not available/i,
+ /unsupported model/i,
+ /model.*has no provider/i,
+ // Function calling format error — model doesn't support this capability
+ /function\.?arguments.*(must be|should be|必须).*(json|JSON)/i,
+ /tool.*arguments.*invalid/i,
+ /function.*parameter.*(invalid|format)/i,
+ // Input length range error — model-specific context limit
+ /range of input length/i,
+ /input length should be/i,
+ // Transient 400 errors from upstream — should fallback to next combo target
+ /服务遇到了一点小状况/i, // ModelScope/Qwen transient error
+ /抱歉.*?敏感内容.*?请检查/i, // ModelScope/Qwen content moderation with context
+ /内容.*?敏感.*?(?:无法|过滤)/i, // Content sensitivity block
+ /无法响应.*?请求/i, // "unable to respond to request"
+ /稍后重试/i, // "retry later" in Chinese
+ /temporary.*error/i,
+ /transient.*error/i,
+ /service.*unavailable/i,
+ /please.*try.*again/i,
+ // Rate limit errors — some providers return 400 instead of 429
+ /\brate.?-?limit.?(?:exceeded|reached|hit)/i,
+ /too many requests/i,
+ /请求过于频繁/i, // Chinese rate limit message
+ // Tool call function name errors — model-specific, try next combo target
+ /\bfunction'?s? name (?:can't|can not|is|has) (?:blank|empty|missing)/i,
+ /function.*name.*(?:blank|empty|missing)/i,
+ /tool_call.*name.*(?:blank|empty|missing)/i,
];
// Patterns that signal all accounts for a provider are rate-limited / exhausted.
@@ -77,6 +118,7 @@ function isAllAccountsRateLimitedResponse(
const MAX_COMBO_DEPTH = 3;
const MAX_FALLBACK_WAIT_MS = 5000;
+const MAX_GLOBAL_ATTEMPTS = 30;
function comboModelNotFoundResponse(message: string) {
return errorResponse(404, message);
@@ -1451,6 +1493,7 @@ export async function handleComboChat({
let earliestRetryAfter = null;
let lastStatus = null;
const startTime = Date.now();
+ globalAttempts = 0;
let fallbackCount = 0;
let recordedAttempts = 0;
@@ -1484,6 +1527,14 @@ export async function handleComboChat({
// Retry loop for transient errors
for (let retry = 0; retry <= maxRetries; retry++) {
+ globalAttempts++;
+ if (globalAttempts > MAX_GLOBAL_ATTEMPTS) {
+ log.warn(
+ "COMBO",
+ `Maximum combo attempts (${MAX_GLOBAL_ATTEMPTS}) exceeded across all targets and fallbacks. Terminating loop to prevent runaway background requests.`
+ );
+ return errorResponse(503, "Maximum combo retry limit reached");
+ }
if (retry > 0) {
log.info(
"COMBO",
@@ -1799,6 +1850,7 @@ async function handleRoundRobinCombo({
let lastError = null;
let lastStatus = null;
let earliestRetryAfter = null;
+ let globalAttempts = 0;
let fallbackCount = 0;
let recordedAttempts = 0;
@@ -1852,6 +1904,14 @@ async function handleRoundRobinCombo({
// Retry loop within this model
try {
for (let retry = 0; retry <= maxRetries; retry++) {
+ globalAttempts++;
+ if (globalAttempts > MAX_GLOBAL_ATTEMPTS) {
+ log.warn(
+ "COMBO-RR",
+ `Maximum combo attempts (${MAX_GLOBAL_ATTEMPTS}) exceeded. Terminating loop to prevent runaway requests.`
+ );
+ return errorResponse(503, "Maximum combo retry limit reached");
+ }
if (retry > 0) {
log.info(
"COMBO-RR",
diff --git a/open-sse/services/errorClassifier.ts b/open-sse/services/errorClassifier.ts
index 39d513ffbe..4520c2a6e4 100644
--- a/open-sse/services/errorClassifier.ts
+++ b/open-sse/services/errorClassifier.ts
@@ -1,6 +1,7 @@
import {
isAccountDeactivated,
isCreditsExhausted,
+ isDailyQuotaExhausted,
isOAuthInvalidToken,
} from "./accountFallback.ts";
@@ -100,7 +101,11 @@ export function classifyProviderError(statusCode: number, responseBody: unknown)
return PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED;
}
+ // 429: Check if it's a daily quota exhaustion (lock until tomorrow) vs regular rate limit
if (statusCode === 429) {
+ if (isDailyQuotaExhausted(bodyStr)) {
+ return PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED;
+ }
return PROVIDER_ERROR_TYPES.RATE_LIMITED;
}
diff --git a/src/app/(dashboard)/dashboard/combos/IntelligentComboPanel.tsx b/src/app/(dashboard)/dashboard/combos/IntelligentComboPanel.tsx
index 0a22d20e07..55013eab4d 100644
--- a/src/app/(dashboard)/dashboard/combos/IntelligentComboPanel.tsx
+++ b/src/app/(dashboard)/dashboard/combos/IntelligentComboPanel.tsx
@@ -11,6 +11,7 @@ import {
extractIntelligentHealthState,
normalizeIntelligentRoutingConfig,
} from "@/lib/combos/intelligentRouting";
+import ModelStatusBadge from "@/app/(dashboard)/dashboard/providers/components/ModelStatusBadge";
function getI18nOrFallback(t: any, key: string, fallback: string) {
if (typeof t?.has === "function" && t.has(key)) return t(key);
@@ -283,6 +284,8 @@ export default function IntelligentComboPanel({
</p>
<p className="text-[11px] text-text-muted mt-0.5">{entry.model}</p>
</div>
+ {/* Model status badge - shows cooldown/error state */}
+ <ModelStatusBadge provider={entry.provider} model={entry.model} size="sm" />
<span className="rounded-full bg-primary/10 px-2 py-1 text-[11px] font-semibold text-primary">
{percentage}%
</span>
diff --git a/src/app/(dashboard)/dashboard/providers/[id]/page.tsx b/src/app/(dashboard)/dashboard/providers/[id]/page.tsx
index 2b9d0abfc4..9164e05718 100644
--- a/src/app/(dashboard)/dashboard/providers/[id]/page.tsx
+++ b/src/app/(dashboard)/dashboard/providers/[id]/page.tsx
@@ -53,6 +53,7 @@ import {
getCodexRequestDefaults as _getCodexRequestDefaults,
} from "@/lib/providers/requestDefaults";
import { resolveDashboardProviderInfo } from "../providerPageUtils";
+import ModelStatusBadge from "@/app/(dashboard)/dashboard/providers/components/ModelStatusBadge";
type CompatByProtocolMap = Partial<
Record<
@@ -326,6 +327,7 @@ function anyUpstreamHeadersBadge(
interface ModelRowProps {
model: { id: string; name?: string; source?: string; isHidden?: boolean };
fullModel: string;
+ provider: string;
copied?: string;
onCopy: (text: string, key: string) => void;
t: (key: string, values?: Record<string, unknown>) => string;
@@ -2460,6 +2462,7 @@ export default function ProviderDetailPage() {
key={model.id}
model={model}
fullModel={`${providerDisplayAlias}/${model.id}`}
+ provider={providerId}
copied={copied}
onCopy={copy}
t={t}
@@ -3329,6 +3332,7 @@ export default function ProviderDetailPage() {
function ModelRow({
model,
fullModel,
+ provider,
copied,
onCopy,
t,
@@ -3358,6 +3362,7 @@ function ModelRow({
<code className="rounded bg-sidebar px-1.5 py-0.5 font-mono text-xs text-text-muted">
{fullModel}
</code>
+ <ModelStatusBadge provider={provider} model={model.id} size="sm" />
<ModelSourceBadge source={model.source} />
<button
onClick={() => onCopy(fullModel, `model-${model.id}`)}
@@ -3407,6 +3412,7 @@ ModelRow.propTypes = {
id: PropTypes.string.isRequired,
}).isRequired,
fullModel: PropTypes.string.isRequired,
+ provider: PropTypes.string.isRequired,
copied: PropTypes.string,
onCopy: PropTypes.func.isRequired,
t: PropTypes.func,
@@ -5402,6 +5408,7 @@ function AddApiKeyModal({
accountId: "",
consoleApiKey: "",
ccCompatibleContext1m: false,
+ passthroughModels: false,
});
const [validating, setValidating] = useState(false);
const [validationResult, setValidationResult] = useState(null);
@@ -5495,6 +5502,9 @@ function AddApiKeyModal({
if (formData.excludedModels.trim()) {
providerSpecificData.excludedModels = parseExcludedModelsInput(formData.excludedModels);
}
+ if (formData.passthroughModels) {
+ providerSpecificData.passthroughModels = true;
+ }
if (provider === "bailian-coding-plan" && formData.consoleApiKey.trim()) {
providerSpecificData.consoleApiKey = formData.consoleApiKey.trim();
}
@@ -5685,6 +5695,13 @@ function AddApiKeyModal({
placeholder="gpt-5*, claude-opus-*, gemini-*-pro*"
hint="Comma-separated wildcard patterns. This connection will never serve matching models."
/>
+ <Toggle
+ size="sm"
+ checked={formData.passthroughModels}
+ onChange={(checked) => setFormData({ ...formData, passthroughModels: checked })}
+ label={t("perModelQuotaLabel")}
+ description={t("perModelQuotaDescription")}
+ />
{provider === "bailian-coding-plan" && (
<Input
label="Console API Key (Oracle)"
@@ -5824,6 +5841,7 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConnec
codexOpenaiStoreEnabled: false,
consoleApiKey: "",
ccCompatibleContext1m: false,
+ passthroughModels: connection.providerSpecificData?.passthroughModels === true,
});
const [testing, setTesting] = useState(false);
const [testResult, setTestResult] = useState(null);
@@ -5891,6 +5909,7 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConnec
codexOpenaiStoreEnabled: connection.providerSpecificData?.openaiStoreEnabled === true,
consoleApiKey: existingConsoleApiKey,
ccCompatibleContext1m: ccRequestDefaults.context1m,
+ passthroughModels: connection.providerSpecificData?.passthroughModels === true,
});
// Load existing extra keys from providerSpecificData
const existing = connection.providerSpecificData?.extraApiKeys;
@@ -6031,6 +6050,8 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConnec
tags: parseRoutingTagsInput(formData.routingTags),
excludedModels: parseExcludedModelsInput(formData.excludedModels),
customUserAgent: formData.customUserAgent.trim(),
+ // Only write when explicitly enabled; omit to let registry default take effect
+ ...(formData.passthroughModels ? { passthroughModels: true } : {}),
};
if (connection.provider === "bailian-coding-plan") {
if (formData.consoleApiKey.trim()) {
@@ -6288,6 +6309,13 @@ function EditConnectionModal({ isOpen, connection, onSave, onClose }: EditConnec
placeholder="my-app/1.0"
hint="Optional override sent upstream as the User-Agent header for this connection"
/>
+ <Toggle
+ size="sm"
+ checked={formData.passthroughModels}
+ onChange={(checked) => setFormData({ ...formData, passthroughModels: checked })}
+ label={t("perModelQuotaLabel")}
+ description={t("perModelQuotaDescription")}
+ />
{connection.provider === "bailian-coding-plan" && (
<Input
label="Console API Key (Oracle)"
diff --git a/src/app/(dashboard)/dashboard/providers/components/ModelStatusBadge.tsx b/src/app/(dashboard)/dashboard/providers/components/ModelStatusBadge.tsx
new file mode 100644
index 0000000000..82538a6d8d
--- /dev/null
+++ b/src/app/(dashboard)/dashboard/providers/components/ModelStatusBadge.tsx
@@ -0,0 +1,162 @@
+"use client";
+
+/**
+ * ModelStatusBadge — compact single-model status indicator
+ *
+ * Shows a small badge with status icon (cooldown/unavailable/error)
+ * with a tooltip containing additional details like remaining cooldown time
+ * or last error message.
+ * Only renders for non-available models to keep the UI clean.
+ *
+ * Uses shared polling via ModelStatusContext to avoid redundant API calls.
+ */
+
+import { useState, useEffect, useRef } from "react";
+import { useTranslations } from "next-intl";
+import Tooltip from "@/shared/components/Tooltip";
+import { useModelStatus } from "./ModelStatusContext";
+
+interface ModelStatusBadgeProps {
+ provider: string;
+ model: string;
+ size?: "sm" | "md";
+ className?: string;
+}
+
+function formatRemainingTime(ms: number): string {
+ if (ms <= 0) return "";
+ const seconds = Math.ceil(ms / 1000);
+ if (seconds < 60) return `${seconds}s`;
+ const minutes = Math.floor(seconds / 60);
+ const remainingSeconds = seconds % 60;
+ if (minutes < 60) {
+ return remainingSeconds > 0 ? `${minutes}m ${remainingSeconds}s` : `${minutes}m`;
+ }
+ const hours = Math.floor(minutes / 60);
+ const remainingMinutes = minutes % 60;
+ return remainingMinutes > 0 ? `${hours}h ${remainingMinutes}m` : `${hours}h`;
+}
+
+export default function ModelStatusBadge({
+ provider,
+ model,
+ size = "sm",
+ className = "",
+}: ModelStatusBadgeProps) {
+ const t = useTranslations("providers");
+ const status = useModelStatus(provider, model);
+
+ // Store the latest remaining time calculated by the interval
+ const [displayRemainingMs, setDisplayRemainingMs] = useState<number | null>(null);
+
+ // Use ref to track server-provided initial cooldown duration for countdown calculation
+ const initialCooldownMsRef = useRef<number | null>(null);
+ // Track when we started counting down
+ const countdownStartRef = useRef<number | null>(null);
+
+ // Set up countdown timer when cooldown begins
+ useEffect(() => {
+ if (status?.status !== "cooldown" || !status.remainingMs) {
+ // Reset refs when not in cooldown (no setState here)
+ initialCooldownMsRef.current = null;
+ countdownStartRef.current = null;
+ return;
+ }
+
+ // Initialize timing refs (no setState here)
+ initialCooldownMsRef.current = status.remainingMs;
+ countdownStartRef.current = Date.now();
+
+ // Update countdown every second via interval callback (setState allowed here)
+ const interval = setInterval(() => {
+ if (!initialCooldownMsRef.current || !countdownStartRef.current) {
+ return;
+ }
+
+ const elapsed = Date.now() - countdownStartRef.current;
+ const newRemaining = Math.max(0, initialCooldownMsRef.current - elapsed);
+
+ setDisplayRemainingMs(newRemaining);
+
+ // Stop updating when countdown reaches zero
+ if (newRemaining === 0) {
+ clearInterval(interval);
+ }
+ }, 1000);
+
+ return () => clearInterval(interval);
+ }, [status?.status, status?.remainingMs]);
+
+ // Derive the displayed remaining time: use countdown value if active, else use status value
+ const remainingMs =
+ status?.status === "cooldown" ? (displayRemainingMs ?? status.remainingMs ?? null) : null;
+
+ // Don't render badge for available models (keep UI clean)
+ if (!status || status.status === "available" || status.status === "unknown") {
+ return null;
+ }
+
+ const getStatusColor = () => {
+ switch (status.status) {
+ case "cooldown":
+ return "#f59e0b";
+ case "unavailable":
+ case "error":
+ return "#ef4444";
+ default:
+ return "#6b7280";
+ }
+ };
+
+ const getStatusIcon = () => {
+ switch (status.status) {
+ case "cooldown":
+ return "schedule";
+ case "unavailable":
+ case "error":
+ return "error";
+ default:
+ return "help";
+ }
+ };
+
+ const getTooltipText = () => {
+ switch (status.status) {
+ case "cooldown": {
+ const remaining = remainingMs !== null ? formatRemainingTime(remainingMs) : "";
+ const reason = status.reason ? ` (${status.reason})` : "";
+ const remainingText = remaining ? ` - ${remaining}` : "";
+ return `${t("cooldown")}${reason}${remainingText}`;
+ }
+ case "unavailable":
+ return `${t("unavailable")}${status.reason ? `: ${status.reason}` : ""}`;
+ case "error":
+ return `${t("error")}${status.lastError ? `: ${status.lastError}` : ""}`;
+ default:
+ return "";
+ }
+ };
+
+ const color = getStatusColor();
+ const sizeClasses = size === "sm" ? "px-1.5 py-0.5" : "px-2 py-1";
+ const iconSize = size === "sm" ? "text-[12px]" : "text-[14px]";
+
+ return (
+ <Tooltip content={getTooltipText()} position="top" delayMs={200}>
+ <span
+ className={`inline-flex items-center gap-1 rounded-full text-[10px] font-semibold ${sizeClasses} ${className}`}
+ style={{
+ backgroundColor: `${color}15`,
+ color: color,
+ }}
+ >
+ <span className={`material-symbols-outlined ${iconSize}`} aria-hidden="true">
+ {getStatusIcon()}
+ </span>
+ {status.status === "cooldown" && remainingMs !== null && (
+ <span>{formatRemainingTime(remainingMs)}</span>
+ )}
+ </span>
+ </Tooltip>
+ );
+}
diff --git a/src/app/(dashboard)/dashboard/providers/components/ModelStatusContext.tsx b/src/app/(dashboard)/dashboard/providers/components/ModelStatusContext.tsx
new file mode 100644
index 0000000000..3605a9defd
--- /dev/null
+++ b/src/app/(dashboard)/dashboard/providers/components/ModelStatusContext.tsx
@@ -0,0 +1,186 @@
+"use client";
+
+/**
+ * ModelStatusContext — shared polling for model availability
+ *
+ * Prevents redundant API calls by having all ModelStatusBadge components
+ * share a single polling interval. Only one request is made every 15 seconds
+ * regardless of how many badges are on the page.
+ */
+
+import React, { createContext, useContext, useEffect, useRef, useCallback, useMemo } from "react";
+
+export interface ModelStatus {
+ status: "available" | "cooldown" | "unavailable" | "error" | "unknown";
+ reason?: string;
+ remainingMs?: number;
+ lastError?: string;
+}
+
+interface ModelStatusContextValue {
+ getStatus: (provider: string, model: string) => ModelStatus | null;
+ registerModel: (key: string, provider: string, model: string) => void;
+ unregisterModel: (key: string) => void;
+}
+
+const ModelStatusContext = createContext<ModelStatusContextValue | null>(null);
+
+// Global map of model key -> status
+let modelStatusMap = new Map<string, ModelStatus>();
+// Global set of registered model keys
+let registeredModels = new Set<string>();
+// Polling interval ref (singleton)
+let pollIntervalRef: NodeJS.Timeout | null = null;
+
+function getModelKey(provider: string, model: string): string {
+ return `${provider}/${model}`;
+}
+
+async function fetchModelStatus(): Promise<void> {
+ try {
+ const res = await fetch("/api/models/availability");
+ if (!res.ok) return;
+
+ const json = await res.json();
+ const models = json?.models || [];
+
+ // Update all registered models with fresh data
+ const now = Date.now();
+ registeredModels.forEach((key) => {
+ const [provider, model] = key.split("/");
+ // Use exact matching first to avoid gpt-4 matching gpt-4-turbo incorrectly
+ const modelEntry =
+ models.find((m: any) => m.provider === provider && m.model === model) ||
+ models.find(
+ // Fallback to prefix matching only for models that contain the registered key
+ // This handles cases like "gpt-4o" matching badge for "gpt-4"
+ (m: any) =>
+ m.provider === provider &&
+ m.model &&
+ model &&
+ (m.model.startsWith(model + "-") || model.startsWith(m.model + "-"))
+ );
+
+ if (modelEntry) {
+ const newStatus: ModelStatus = {
+ status: modelEntry.status || "unknown",
+ reason: modelEntry.reason,
+ remainingMs: modelEntry.remainingMs,
+ lastError: modelEntry.lastError,
+ };
+
+ // For cooldown status, calculate remaining time based on server-provided value
+ if (modelEntry.status === "cooldown" && modelEntry.remainingMs) {
+ newStatus.remainingMs = modelEntry.remainingMs;
+ }
+
+ modelStatusMap.set(key, newStatus);
+ } else {
+ modelStatusMap.set(key, { status: "available" });
+ }
+ });
+
+ // Trigger re-render by dispatching custom event
+ window.dispatchEvent(new CustomEvent("model-status-update"));
+ } catch {
+ // Best-effort polling
+ }
+}
+
+function ensurePolling(): void {
+ if (pollIntervalRef) return;
+
+ // Initial fetch
+ fetchModelStatus();
+
+ // Poll every 15 seconds
+ pollIntervalRef = setInterval(fetchModelStatus, 15000);
+}
+
+function stopPolling(): void {
+ if (pollIntervalRef) {
+ clearInterval(pollIntervalRef);
+ pollIntervalRef = null;
+ }
+}
+
+export function ModelStatusProvider({ children }: { children: React.ReactNode }) {
+ const [, forceUpdate] = React.useState(0);
+
+ // Listen for status updates from the global poller
+ useEffect(() => {
+ const handleUpdate = () => forceUpdate((n) => n + 1);
+ window.addEventListener("model-status-update", handleUpdate);
+ return () => window.removeEventListener("model-status-update", handleUpdate);
+ }, []);
+
+ // Cleanup on unmount — stop polling only when no models remain registered
+ useEffect(() => {
+ return () => {
+ if (registeredModels.size === 0) {
+ stopPolling();
+ }
+ };
+ }, []);
+
+ const getStatus = useCallback((provider: string, model: string): ModelStatus | null => {
+ const key = getModelKey(provider, model);
+ return modelStatusMap.get(key) || null;
+ }, []);
+
+ const registerModel = useCallback((key: string, provider: string, model: string): void => {
+ const wasEmpty = registeredModels.size === 0;
+ registeredModels.add(key);
+
+ // Start polling when first model registers
+ if (wasEmpty) {
+ ensurePolling();
+ }
+
+ // Immediately fetch if no data yet
+ if (!modelStatusMap.has(key)) {
+ fetchModelStatus();
+ }
+ }, []);
+
+ const unregisterModel = useCallback((key: string): void => {
+ registeredModels.delete(key);
+ modelStatusMap.delete(key);
+
+ // Stop polling when last model unregisters
+ if (registeredModels.size === 0) {
+ stopPolling();
+ }
+ }, []);
+
+ const value = useMemo(
+ () => ({
+ getStatus,
+ registerModel,
+ unregisterModel,
+ }),
+ [getStatus, registerModel, unregisterModel]
+ );
+
+ return <ModelStatusContext.Provider value={value}>{children}</ModelStatusContext.Provider>;
+}
+
+export function useModelStatus(provider: string, model: string): ModelStatus | null {
+ const context = useContext(ModelStatusContext);
+
+ if (!context) {
+ throw new Error("useModelStatus must be used within a ModelStatusProvider");
+ }
+
+ const key = useMemo(() => getModelKey(provider, model), [provider, model]);
+
+ // Register/unregister on mount/unmount
+ useEffect(() => {
+ context.registerModel(key, provider, model);
+ return () => context.unregisterModel(key);
+ }, [context, key, provider, model]);
+
+ return context.getStatus(provider, model);
+}
+
+export default ModelStatusContext;
diff --git a/src/app/(dashboard)/layout.tsx b/src/app/(dashboard)/layout.tsx
index e640e17628..8237e8eb34 100644
--- a/src/app/(dashboard)/layout.tsx
+++ b/src/app/(dashboard)/layout.tsx
@@ -1,5 +1,10 @@
import { DashboardLayout } from "@/shared/components";
+import { ModelStatusProvider } from "./dashboard/providers/components/ModelStatusContext";
export default function DashboardRootLayout({ children }) {
- return <DashboardLayout>{children}</DashboardLayout>;
+ return (
+ <ModelStatusProvider>
+ <DashboardLayout>{children}</DashboardLayout>
+ </ModelStatusProvider>
+ );
}
diff --git a/src/i18n/messages/en.json b/src/i18n/messages/en.json
index 9e1726d107..d0f1202c61 100644
--- a/src/i18n/messages/en.json
+++ b/src/i18n/messages/en.json
@@ -282,6 +282,10 @@
"evals": "Evals",
"utilization": "Utilization",
"utilizationDescription": "Provider quota usage trends and rate limit tracking",
+ "modelStatus": "Model Status",
+ "modelStatusCooldown": "Cooldown",
+ "modelStatusUnavailable": "Unavailable",
+ "modelStatusError": "Error",
"comboHealth": "Combo Health",
"comboHealthDescription": "Combo-level quota, usage distribution, and performance metrics"
},
@@ -644,7 +648,8 @@
"kiro": "Use when integrating Kiro and controlling model routing centrally from OmniRoute.",
"windsurf": "Use when you want an AI-first IDE with Codeium/Windsurf models routed through OmniRoute.",
"antigravity": "Use when Antigravity/Kiro traffic must be intercepted through MITM and routed to OmniRoute.",
- "copilot": "Use when you want Copilot chat style UX while enforcing OmniRoute keys and routing rules."
+ "copilot": "Use when you want Copilot chat style UX while enforcing OmniRoute keys and routing rules.",
+ "qwen": "Use when you need Alibaba Qwen Code CLI for coding tasks."
},
"toolDescriptions": {
"antigravity": "Google Antigravity IDE with MITM",
@@ -1108,6 +1113,7 @@
"builderAccount": "Account",
"builderPreview": "Preview",
"builderAddStep": "Add step",
+ "builderDuplicateExact": "This exact provider/model/account step is already in the combo.",
"builderComboRef": "Combo Ref",
"builderAddComboRef": "Add combo reference",
"builderComboRefStep": "Add combo reference",
@@ -1865,6 +1871,9 @@
"compatUpstreamAddRow": "Add header",
"compatUpstreamRemoveRow": "Remove row",
"compatBadgeUpstreamHeaders": "Headers",
+ "perModelQuotaLabel": "Per-Model Quota",
+ "perModelQuotaDescription": "When enabled, 429/404 errors only lock the specific model, not the entire connection. Use for providers with per-model rate limits (e.g., ModelScope).",
+ "perModelQuotaToggle": "Per-Model Quota Toggle",
"modelId": "Model ID",
"customModelPlaceholder": "e.g. gpt-4.5-turbo",
"loading": "Loading...",
diff --git a/src/i18n/messages/zh-CN.json b/src/i18n/messages/zh-CN.json
index f2e7706f0c..5f6f7e9655 100644
--- a/src/i18n/messages/zh-CN.json
+++ b/src/i18n/messages/zh-CN.json
@@ -282,6 +282,10 @@
"evals": "评测",
"utilization": "利用率",
"utilizationDescription": "提供商配额使用趋势和速率限制跟踪",
+ "modelStatus": "模型状态",
+ "modelStatusCooldown": "冷却中",
+ "modelStatusUnavailable": "不可用",
+ "modelStatusError": "错误",
"comboHealth": "组合健康状况",
"comboHealthDescription": "组合级别配额、使用分布和性能指标"
},
@@ -644,7 +648,8 @@
"kiro": "在集成 Kiro 并从 OmniRoute 集中控制模型路由时使用。",
"windsurf": "当您需要 Windsurf AI IDE 并通过 OmniRoute 路由模型时使用。",
"antigravity": "当必须通过 MITM 拦截 Antigravity/Kiro 流量并将其路由到 OmniRoute 时使用。",
- "copilot": "当您想要 Copilot 聊天风格的 UX 同时强制执行 OmniRoute 键和路由规则时使用。"
+ "copilot": "当您想要 Copilot 聊天风格的 UX 同时强制执行 OmniRoute 键和路由规则时使用。",
+ "qwen": "当您需要使用阿里云 Qwen Code CLI 进行编码任务时使用。"
},
"toolDescriptions": {
"antigravity": "带 MITM 的 Google Antigravity IDE",
@@ -1022,7 +1027,8 @@
"selectAccount": "选择账户",
"autoSelectAccount": "运行时自动选择账户",
"previewNextStep": "选择提供商和模型以预览下一步。",
- "addStep": "添加步骤",
+ "builderAddStep": "添加步骤",
+ "builderDuplicateExact": "该提供商/模型/账户步骤已存在于组合中。",
"comboReference": "组合引用",
"selectComboToReference": "选择要引用的现有组合",
"addComboReference": "添加组合引用",
@@ -1911,6 +1917,9 @@
"compatUpstreamAddRow": "添加请求头",
"compatUpstreamRemoveRow": "删除此行",
"compatBadgeUpstreamHeaders": "请求头",
+ "perModelQuotaLabel": "按模型配额",
+ "perModelQuotaDescription": "启用后,429/404错误只会锁定特定模型,而不是整个连接。适用于具有按模型速率限制的提供商(例如ModelScope)。",
+ "perModelQuotaToggle": "按模型配额开关",
"modelId": "模型 ID",
"customModelPlaceholder": "例如:gpt-4.5-turbo",
"loading": "正在加载...",
diff --git a/src/sse/handlers/chat.ts b/src/sse/handlers/chat.ts
index 5bc6b9629b..70f546be1b 100644
--- a/src/sse/handlers/chat.ts
+++ b/src/sse/handlers/chat.ts
@@ -841,7 +841,11 @@ async function handleSingleModelChat(
// 6. Mark account as quota-exhausted on 429 response
// For providers that route quota/cooldown at model scope, a 429 on one model
// does not mean the whole connection is exhausted.
- if (result.status === 429 && shouldMarkAccountExhaustedFrom429(provider, model)) {
+ const passthroughModels = credentials.providerSpecificData?.passthroughModels;
+ if (
+ result.status === 429 &&
+ shouldMarkAccountExhaustedFrom429(provider, model, passthroughModels)
+ ) {
markAccountExhaustedFrom429(credentials.connectionId, provider);
}
diff --git a/src/sse/handlers/chatHelpers.ts b/src/sse/handlers/chatHelpers.ts
index f091220573..ef5dd9213e 100644
--- a/src/sse/handlers/chatHelpers.ts
+++ b/src/sse/handlers/chatHelpers.ts
@@ -27,7 +27,10 @@ import { getCircuitBreaker, CircuitBreakerOpenError } from "../../shared/utils/c
import { getModelCooldownInfo, isModelAvailable } from "../../domain/modelAvailability";
import { logProxyEvent } from "../../lib/proxyLogger";
import { logTranslationEvent } from "../../lib/translatorEvents";
-import { getRuntimeProviderProfile } from "@omniroute/open-sse/services/accountFallback.ts";
+import {
+ getRuntimeProviderProfile,
+ clearProviderFailure,
+} from "@omniroute/open-sse/services/accountFallback.ts";
export async function resolveModelOrError(modelStr: string, body: any, endpointPath: string = "") {
const modelInfo = await getModelInfo(modelStr);
@@ -170,6 +173,8 @@ export async function executeChatWithBreaker({
},
onRequestSuccess: async () => {
await clearAccountError(credentials.connectionId, credentials);
+ // Clear provider-level failure state on successful request
+ clearProviderFailure(provider);
},
})
);
diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.ts
index 63c611db9d..ec601a3810 100644
--- a/src/sse/services/auth.ts
+++ b/src/sse/services/auth.ts
@@ -19,6 +19,8 @@ import {
hasPerModelQuota,
getRuntimeProviderProfile,
recordModelLockoutFailure,
+ isProviderInCooldown,
+ getProviderCooldownRemainingMs,
} from "@omniroute/open-sse/services/accountFallback.ts";
import { isLocalProvider } from "@omniroute/open-sse/config/providerRegistry.ts";
import { COOLDOWN_MS } from "@omniroute/open-sse/config/constants.ts";
@@ -682,6 +684,22 @@ export async function getProviderCredentials(
return true;
});
+ // Check if the entire provider is in cooldown (too many transient failures)
+ if (isProviderInCooldown(provider)) {
+ const cooldownRemaining = getProviderCooldownRemainingMs(provider);
+ log.warn(
+ "AUTH",
+ `${provider} | provider in cooldown for ${Math.ceil((cooldownRemaining || 0) / 1000)}s (${availableConnections.length} connections bypassed)`
+ );
+ return {
+ allRateLimited: true,
+ retryAfter: new Date(Date.now() + (cooldownRemaining || 0)).toISOString(),
+ retryAfterHuman: formatRetryAfter(
+ new Date(Date.now() + (cooldownRemaining || 0)).toISOString()
+ ),
+ };
+ }
+
log.debug(
"AUTH",
`${provider} | available: ${availableConnections.length}/${connections.length}`
@@ -1189,7 +1207,13 @@ export async function markAccountUnavailable(
const effectiveProviderProfile =
providerProfile || (provider ? await getRuntimeProviderProfile(provider) : null);
- const isPerModelQuotaProvider = hasPerModelQuota(provider, model);
+ // Read passthroughModels from connection config (user-configured per-model quota)
+ const connProviderSpecificData = (conn?.providerSpecificData as Record<string, unknown>) || {};
+ const connectionPassthroughModels = connProviderSpecificData.passthroughModels as
+ | boolean
+ | undefined;
+
+ const isPerModelQuotaProvider = hasPerModelQuota(provider, model, connectionPassthroughModels);
if (isPerModelQuotaProvider && provider && model && (status === 404 || status === 429)) {
const reason = status === 404 ? "not_found" : "rate_limited";
const fallbackCooldown =
diff --git a/tests/integration/chat-pipeline.test.ts b/tests/integration/chat-pipeline.test.ts
index 3723cc9411..10f5d366f4 100644
--- a/tests/integration/chat-pipeline.test.ts
+++ b/tests/integration/chat-pipeline.test.ts
@@ -26,6 +26,7 @@ const { BaseExecutor } = await import("../../open-sse/executors/base.ts");
const { resetAllAvailability, setModelUnavailable } =
await import("../../src/domain/modelAvailability.ts");
const { resetAllCircuitBreakers } = await import("../../src/shared/utils/circuitBreaker.ts");
+const { clearProviderFailure } = await import("../../open-sse/services/accountFallback.ts");
const originalFetch = globalThis.fetch;
const originalRetryDelayMs = BaseExecutor.RETRY_CONFIG.delayMs;
@@ -1017,6 +1018,8 @@ test("chat pipeline maps upstream timeouts to 504 responses", async () => {
});
test("chat pipeline injects memory context before sending the upstream request", async () => {
+ // Reset provider failure state to avoid circuit breaker interference
+ clearProviderFailure("openai");
await seedConnection("openai", { apiKey: "sk-openai-memory" });
const apiKey = await seedApiKey();
await settingsDb.updateSettings({
@@ -1057,6 +1060,8 @@ test("chat pipeline injects memory context before sending the upstream request",
});
test("chat pipeline injects skills into tools and intercepts tool calls with skill output", async () => {
+ // Reset provider failure state to avoid circuit breaker interference
+ clearProviderFailure("openai");
await seedConnection("openai", { apiKey: "sk-openai-skills" });
const apiKey = await seedApiKey();
await settingsDb.updateSettings({ skillsEnabled: true });
@@ -1118,6 +1123,8 @@ test("chat pipeline injects skills into tools and intercepts tool calls with ski
});
test("chat pipeline falls back to the next account after a provider failure", async () => {
+ // Reset provider failure state to avoid circuit breaker interference
+ clearProviderFailure("openai");
await seedConnection("openai", {
name: "openai-primary",
apiKey: "sk-openai-primary-fallback",
@@ -1162,6 +1169,9 @@ test("chat pipeline falls back to the next account after a provider failure", as
});
test("chat pipeline falls back across combo models when the first provider fails", async () => {
+ // Reset provider failure state to avoid circuit breaker interference
+ clearProviderFailure("openai");
+ clearProviderFailure("claude");
await seedConnection("openai", { apiKey: "sk-openai-combo-fail" });
await seedConnection("claude", { apiKey: "sk-claude-combo-fail" });
await combosDb.createCombo({
@@ -1206,6 +1216,8 @@ test("chat pipeline falls back across combo models when the first provider fails
});
test("chat pipeline deduplicates concurrent identical non-stream requests", async () => {
+ // Reset provider failure state to avoid circuit breaker interference
+ clearProviderFailure("openai");
await seedConnection("openai", { apiKey: "sk-openai-dedup" });
let fetchCount = 0;
diff --git a/tests/unit/account-fallback-service.test.ts b/tests/unit/account-fallback-service.test.ts
index 4f10e7e0a8..912aa9836e 100644
--- a/tests/unit/account-fallback-service.test.ts
+++ b/tests/unit/account-fallback-service.test.ts
@@ -22,6 +22,12 @@ const {
recordModelLockoutFailure,
clearModelLock,
shouldMarkAccountExhaustedFrom429,
+ recordProviderFailure,
+ isProviderInCooldown,
+ getProviderCooldownRemainingMs,
+ clearProviderFailure,
+ isProviderFailureCode,
+ getProvidersInCooldown,
} = accountFallback;
const { selectAccount } = accountSelector;
@@ -290,3 +296,215 @@ test("recordModelLockoutFailure uses provider profile cooldowns, backoff, and re
clearModelLock("openai-compatible-custom-node", "conn-compatible", "custom-model-a");
}
});
+
+// Provider-level failure circuit breaker tests
+test("isProviderFailureCode correctly identifies transient error codes", () => {
+ assert.equal(isProviderFailureCode(429), true);
+ assert.equal(isProviderFailureCode(408), true);
+ assert.equal(isProviderFailureCode(500), true);
+ assert.equal(isProviderFailureCode(502), true);
+ assert.equal(isProviderFailureCode(503), true);
+ assert.equal(isProviderFailureCode(504), true);
+ assert.equal(isProviderFailureCode(401), false);
+ assert.equal(isProviderFailureCode(403), false);
+ assert.equal(isProviderFailureCode(400), false);
+ assert.equal(isProviderFailureCode(404), false);
+ assert.equal(isProviderFailureCode(200), false);
+});
+
+test("recordProviderFailure tracks failures and triggers cooldown after threshold", () => {
+ const originalNow = Date.now;
+ let now = 1_700_000_000_000;
+ Date.now = () => now;
+
+ try {
+ const provider = "test-provider";
+
+ // Clear any existing state
+ clearProviderFailure(provider);
+ assert.equal(isProviderInCooldown(provider), false);
+ assert.equal(getProviderCooldownRemainingMs(provider), null);
+
+ // Record 4 failures - should not trigger cooldown yet
+ for (let i = 0; i < 4; i++) {
+ recordProviderFailure(provider);
+ now += 1000; // 1 second between failures
+ }
+ assert.equal(isProviderInCooldown(provider), false);
+
+ // 5th failure - should trigger cooldown
+ recordProviderFailure(provider);
+ assert.equal(isProviderInCooldown(provider), true);
+
+ const remaining = getProviderCooldownRemainingMs(provider);
+ assert.ok(remaining !== null);
+ assert.ok(remaining > 0);
+ assert.ok(remaining <= 10 * 60 * 1000); // 10 minutes max
+
+ // Check getProvidersInCooldown returns the provider
+ const inCooldown = getProvidersInCooldown();
+ assert.ok(inCooldown.some((p) => p.provider === provider));
+ assert.equal(inCooldown.find((p) => p.provider === provider)?.failureCount, 5);
+
+ // Simulate cooldown expiration
+ now += 11 * 60 * 1000; // 11 minutes later
+ assert.equal(isProviderInCooldown(provider), false);
+ assert.equal(getProviderCooldownRemainingMs(provider), null);
+ assert.equal(
+ getProvidersInCooldown().some((p) => p.provider === provider),
+ false
+ );
+ } finally {
+ Date.now = originalNow;
+ clearProviderFailure("test-provider");
+ }
+});
+
+test("recordProviderFailure resets counter after failure window expires", () => {
+ const originalNow = Date.now;
+ let now = 1_700_000_000_000;
+ Date.now = () => now;
+
+ try {
+ const provider = "test-provider-window";
+ clearProviderFailure(provider);
+
+ // Record 3 failures
+ for (let i = 0; i < 3; i++) {
+ recordProviderFailure(provider);
+ now += 1000;
+ }
+ assert.equal(isProviderInCooldown(provider), false);
+
+ // Wait for failure window to expire (20 minutes + 1 second)
+ now += 20 * 60 * 1000 + 1000;
+
+ // Next failure should reset counter, not trigger cooldown
+ recordProviderFailure(provider);
+ assert.equal(isProviderInCooldown(provider), false);
+
+ // Need 4 more failures to trigger cooldown
+ for (let i = 0; i < 4; i++) {
+ recordProviderFailure(provider);
+ now += 1000;
+ }
+ assert.equal(isProviderInCooldown(provider), true);
+ } finally {
+ Date.now = originalNow;
+ clearProviderFailure("test-provider-window");
+ }
+});
+
+test("checkFallbackError records provider failure for transient errors", () => {
+ const originalNow = Date.now;
+ let now = 1_700_000_000_000;
+ Date.now = () => now;
+
+ try {
+ const provider = "test-provider-check";
+ clearProviderFailure(provider);
+
+ // Simulate 5 transient errors through checkFallbackError
+ for (let i = 0; i < 5; i++) {
+ checkFallbackError(429, "rate limited", 0, null, provider);
+ now += 1000;
+ }
+
+ // Provider should now be in cooldown
+ assert.equal(isProviderInCooldown(provider), true);
+ } finally {
+ Date.now = originalNow;
+ clearProviderFailure("test-provider-check");
+ }
+});
+
+test("checkFallbackError does not record provider failure for non-transient errors", () => {
+ const originalNow = Date.now;
+ let now = 1_700_000_000_000;
+ Date.now = () => now;
+
+ try {
+ const provider = "test-provider-no-record";
+ clearProviderFailure(provider);
+
+ // Simulate 5 auth errors (401) - should NOT trigger provider cooldown
+ for (let i = 0; i < 5; i++) {
+ checkFallbackError(401, "unauthorized", 0, null, provider);
+ now += 1000;
+ }
+
+ // Provider should NOT be in cooldown
+ assert.equal(isProviderInCooldown(provider), false);
+ } finally {
+ Date.now = originalNow;
+ clearProviderFailure("test-provider-no-record");
+ }
+});
+
+test("clearProviderFailure removes provider from cooldown", () => {
+ const originalNow = Date.now;
+ let now = 1_700_000_000_000;
+ Date.now = () => now;
+
+ try {
+ const provider = "test-provider-clear";
+ clearProviderFailure(provider);
+
+ // Trigger cooldown
+ for (let i = 0; i < 5; i++) {
+ recordProviderFailure(provider);
+ now += 1000;
+ }
+ assert.equal(isProviderInCooldown(provider), true);
+
+ // Clear the failure state
+ clearProviderFailure(provider);
+ assert.equal(isProviderInCooldown(provider), false);
+ assert.equal(getProviderCooldownRemainingMs(provider), null);
+ } finally {
+ Date.now = originalNow;
+ clearProviderFailure("test-provider-clear");
+ }
+});
+
+// Daily quota exhausted detection tests
+test("isDailyQuotaExhausted detects today's quota errors", () => {
+ const { isDailyQuotaExhausted } = accountFallback;
+ assert.equal(isDailyQuotaExhausted("You have exceeded today's quota for model X"), true);
+ assert.equal(isDailyQuotaExhausted("exceeded your daily quota"), true);
+ assert.equal(isDailyQuotaExhausted("Please try again tomorrow"), true);
+ assert.equal(isDailyQuotaExhausted("rate limit exceeded"), false);
+ assert.equal(isDailyQuotaExhausted(""), false);
+ assert.equal(isDailyQuotaExhausted(null), false);
+});
+
+test("getMsUntilTomorrow returns positive value less than 24 hours", () => {
+ const { getMsUntilTomorrow } = accountFallback;
+ const ms = getMsUntilTomorrow();
+ assert.ok(ms > 0, "should be positive");
+ assert.ok(ms <= 24 * 60 * 60 * 1000, "should be <= 24 hours");
+});
+
+test("checkFallbackError locks model until tomorrow for daily quota exhaustion", () => {
+ const result = checkFallbackError(
+ 429,
+ "You have exceeded today's quota for model moonshotai/Kimi-K2.5, please try again tomorrow"
+ );
+ assert.equal(result.shouldFallback, true);
+ assert.equal(result.reason, RateLimitReason.QUOTA_EXHAUSTED);
+ assert.equal(result.dailyQuotaExhausted, true);
+ assert.ok(result.cooldownMs > 0, "cooldown should be positive");
+ assert.ok(result.cooldownMs <= 24 * 60 * 60 * 1000, "cooldown should be <= 24 hours");
+});
+
+test("checkFallbackError detects daily quota with 'try again tomorrow'", () => {
+ const result = checkFallbackError(429, "Please try again tomorrow");
+ assert.equal(result.shouldFallback, true);
+ assert.equal(result.dailyQuotaExhausted, true);
+});
+
+test("checkFallbackError detects daily quota with 'daily quota'", () => {
+ const result = checkFallbackError(429, "You have exceeded your daily quota");
+ assert.equal(result.shouldFallback, true);
+ assert.equal(result.dailyQuotaExhausted, true);
+});
diff --git a/tests/unit/chat-route-coverage.test.ts b/tests/unit/chat-route-coverage.test.ts
index 7473e98e28..2c7b4ca71d 100644
--- a/tests/unit/chat-route-coverage.test.ts
+++ b/tests/unit/chat-route-coverage.test.ts
@@ -21,6 +21,7 @@ const {
const { getCircuitBreaker, STATE } = await import("../../src/shared/utils/circuitBreaker.ts");
const { clearModelUnavailability } = await import("../../src/domain/modelAvailability.ts");
+const { clearProviderFailure } = await import("../../open-sse/services/accountFallback.ts");
const { getDefaultTaskModelMap, resetTaskRoutingStats, setTaskRoutingConfig } =
await import("../../open-sse/services/taskAwareRouter.ts");
@@ -409,6 +410,8 @@ test("handleChat maps upstream timeouts to HTTP 504", async () => {
});
test("handleChat uses the emergency fallback model on budget exhaustion", async () => {
+ // Reset provider failure state to avoid circuit breaker interference
+ clearProviderFailure("openai");
await seedConnection("openai", { apiKey: "sk-openai-billing" });
await seedConnection("nvidia", { apiKey: "sk-nvidia-fallback" });
const seenBodies = [];
@@ -448,6 +451,8 @@ test("handleChat uses the emergency fallback model on budget exhaustion", async
});
test("handleChat returns the primary budget error when emergency fallback also fails", async () => {
+ // Reset provider failure state to avoid circuit breaker interference
+ clearProviderFailure("openai");
await seedConnection("openai", { apiKey: "sk-openai-billing-fail" });
await seedConnection("nvidia", { apiKey: "sk-nvidia-fallback-fail" });
const seenModels = [];
diff --git a/tests/unit/combo-routing-engine.test.ts b/tests/unit/combo-routing-engine.test.ts
index c3e267277a..823dfbc3bb 100644
--- a/tests/unit/combo-routing-engine.test.ts
+++ b/tests/unit/combo-routing-engine.test.ts
@@ -740,6 +740,52 @@ test("shouldFallbackComboBadRequest only flags known provider-scoped 400 pattern
assert.equal(shouldFallbackComboBadRequest(429, "prohibited_content"), false);
assert.equal(shouldFallbackComboBadRequest(400, null), false);
assert.equal(shouldFallbackComboBadRequest(400, "generic bad request"), false);
+ // Chinese transient errors (ModelScope/Qwen)
+ assert.equal(
+ shouldFallbackComboBadRequest(400, "[400]: 抱歉,服务遇到了一点小状况,请您稍后重试。"),
+ true
+ );
+ assert.equal(shouldFallbackComboBadRequest(400, "服务遇到了一点小状况"), true);
+ assert.equal(shouldFallbackComboBadRequest(400, "请稍后重试"), true);
+ // Model not supported errors
+ assert.equal(
+ shouldFallbackComboBadRequest(
+ 400,
+ "Model id : XiaomiMiMo/MiMo-V2-Flash , has no provider supported"
+ ),
+ true
+ );
+ assert.equal(shouldFallbackComboBadRequest(400, "no provider supported"), true);
+ assert.equal(shouldFallbackComboBadRequest(400, "model not found"), true);
+ assert.equal(shouldFallbackComboBadRequest(400, "model not available"), true);
+ // Function calling format errors
+ assert.equal(
+ shouldFallbackComboBadRequest(400, "function.arguments parameter must be in JSON format"),
+ true
+ );
+ assert.equal(
+ shouldFallbackComboBadRequest(
+ 400,
+ '[400]: <400> InternalError.Algo.InvalidParameter: The "function.arguments" parameter of the code model must be in JSON format.'
+ ),
+ true
+ );
+ assert.equal(shouldFallbackComboBadRequest(400, "tool arguments invalid format"), true);
+ // Input length range errors
+ assert.equal(
+ shouldFallbackComboBadRequest(400, "Range of input length should be [1, 98304]"),
+ true
+ );
+ assert.equal(shouldFallbackComboBadRequest(400, "input length should be"), true);
+ // Content moderation errors (should fallback to next model)
+ assert.equal(
+ shouldFallbackComboBadRequest(400, "抱歉,您的内容包含敏感内容,请检查后重试"),
+ true
+ );
+ assert.equal(shouldFallbackComboBadRequest(400, "内容存在敏感信息,无法响应"), true);
+ assert.equal(shouldFallbackComboBadRequest(400, "无法响应该请求"), true);
+ // Generic "please check" should NOT match (was too broad before)
+ assert.equal(shouldFallbackComboBadRequest(400, "请检查您的参数"), false);
});
test("handleComboChat accepts binary and Responses-style 200 bodies but falls through malformed success payloads", async () => {
diff --git a/tests/unit/error-classifier.test.ts b/tests/unit/error-classifier.test.ts
index 13184d076c..846f89fb87 100644
--- a/tests/unit/error-classifier.test.ts
+++ b/tests/unit/error-classifier.test.ts
@@ -65,3 +65,21 @@ test("classifyProviderError: 403 with project string as plain string body => PRO
const result = classifyProviderError(403, body);
assert.equal(result, PROVIDER_ERROR_TYPES.PROJECT_ROUTE_ERROR);
});
+
+test("classifyProviderError: 429 with daily quota signal => QUOTA_EXHAUSTED", () => {
+ const body = JSON.stringify({
+ error: {
+ message:
+ "You have exceeded today's quota for model moonshotai/Kimi-K2.5, please try again tomorrow",
+ },
+ });
+ const result = classifyProviderError(429, body);
+ assert.equal(result, PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED);
+});
+
+test("classifyProviderError: 429 with 'daily quota' signal => QUOTA_EXHAUSTED", () => {
+ const result = classifyProviderError(429, {
+ error: { message: "You have reached your daily quota limit" },
+ });
+ assert.equal(result, PROVIDER_ERROR_TYPES.QUOTA_EXHAUSTED);
+});