Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(routing): skip the whole provider on 5xx/timeout/transport errors, not one key at a time (#788) (#817)

suantea (asuan_suan)e1d1c5aa3256
diff --git a/server/src/__tests__/lib/fallback-loop.test.ts b/server/src/__tests__/lib/fallback-loop.test.ts
index dff16745..5491b64d 100644
--- a/server/src/__tests__/lib/fallback-loop.test.ts
+++ b/server/src/__tests__/lib/fallback-loop.test.ts
@@ -31,8 +31,14 @@ import {
EMPTY_COMPLETION_STREAK_LIMIT,
type AttemptRecord,
type FallbackHooks,
+ type FallbackState,
} from '../../lib/fallback-loop.js';
-import { isKeyAuthError, isDailyQuotaExhaustedError } from '../../lib/error-classify.js';
+import {
+ isKeyAuthError,
+ isDailyQuotaExhaustedError,
+ isProviderLevelError,
+ newClientAbortError,
+} from '../../lib/error-classify.js';
import { getAllPenalties } from '../../services/router.js';
import type { RouteResult } from '../../services/router.js';
@@ -560,6 +566,115 @@ describe('runFallbackLoop: circuit-breaker guardrail (max_consecutive_upstream_f
});
});
+describe('isProviderLevelError (#788: the PROVIDER is sick, not this key)', () => {
+ it('flags what only a sick provider produces: a 5xx, a timeout, a dead socket', () => {
+ expect(isProviderLevelError(Object.assign(new Error('Bad Gateway'), { status: 502 }))).toBe(true);
+ expect(isProviderLevelError(Object.assign(new Error('Groq API error 503: Service Unavailable'), { status: 503 }))).toBe(true);
+ expect(isProviderLevelError(Object.assign(new Error('Internal Server Error'), { status: 500 }))).toBe(true);
+ // Transport-level failures carry no HTTP status at all.
+ expect(isProviderLevelError(new Error('connect ECONNREFUSED 127.0.0.1:443'))).toBe(true);
+ expect(isProviderLevelError(new Error('read ECONNRESET'))).toBe(true);
+ expect(isProviderLevelError(new Error('fetch failed'))).toBe(true);
+ expect(isProviderLevelError(new Error('The operation was aborted (groq, chat, 120s)'))).toBe(true);
+ expect(isProviderLevelError(new Error('Fake stream stalled: no data for 90000ms (timeout)'))).toBe(true);
+ // A DEGRADED deployment is provider health wearing a 400 (#522).
+ expect(isProviderLevelError(Object.assign(new Error("Function id 'x': DEGRADED function cannot be invoked"), { status: 400 }))).toBe(true);
+ });
+
+ it('does not misfire on digits or wording that merely LOOK like a 5xx', () => {
+ // The whole point of the structured-status rule: these all carry '500'/'503'
+ // /'unavailable' in their text, and condemning the platform on any of them
+ // would strand every healthy key and model behind a sick-looking sentence.
+ expect(isProviderLevelError(Object.assign(new Error('Request took 5003ms'), { status: 400 }))).toBe(false);
+ expect(isProviderLevelError(Object.assign(new Error("API error 400: This model's maximum context length is 8192 tokens. However, your messages resulted in 5000 tokens"), { status: 400 }))).toBe(false);
+ expect(isProviderLevelError(Object.assign(new Error('API error 429: quota of 5000 requests per day reached'), { status: 429 }))).toBe(false);
+ expect(isProviderLevelError(Object.assign(new Error('API error 403: qwen3-coder-480b is unavailable on your key\'s tier'), { status: 403 }))).toBe(false);
+ });
+
+ it('leaves key-scoped failures alone so a dead key still rotates to a sibling', () => {
+ expect(isProviderLevelError(Object.assign(new Error('Invalid API Key'), { status: 401 }))).toBe(false);
+ expect(isProviderLevelError(Object.assign(new Error('429 Too Many Requests'), { status: 429 }))).toBe(false);
+ expect(isProviderLevelError(Object.assign(new Error('Payment required'), { status: 402 }))).toBe(false);
+ // A vanished client says nothing about provider health.
+ expect(isProviderLevelError(newClientAbortError())).toBe(false);
+ });
+});
+
+describe('runFallbackLoop: a provider-level failure skips the whole platform (#788)', () => {
+ // A miniature routeRequest: walk a fixed candidate list in order and return
+ // the first one this request has not ruled out, applying the same three gates
+ // the real router applies (skipPlatforms, skipModels, skipKeys).
+ const CANDIDATES = [
+ { platform: 'alpha', modelId: 'alpha-big', modelDbId: 788_001, keyId: 78_801 },
+ { platform: 'alpha', modelId: 'alpha-big', modelDbId: 788_001, keyId: 78_802 },
+ { platform: 'alpha', modelId: 'alpha-small', modelDbId: 788_002, keyId: 78_801 },
+ { platform: 'beta', modelId: 'beta-one', modelDbId: 788_003, keyId: 78_803 },
+ ];
+ const miniRouter = (state: FallbackState) => () => {
+ const pick = CANDIDATES.find(c =>
+ !state.skipPlatforms.has(c.platform)
+ && !state.skipModels.has(c.modelDbId)
+ && !state.skipKeys.has(`${c.platform}:${c.modelId}:${c.keyId}`));
+ if (!pick) throw Object.assign(new Error('all candidates exhausted'), { status: 429, diagnostics: [] });
+ return fakeRoute(pick);
+ };
+ const platformsTried = (dispatch: ReturnType<typeof vi.fn>): string[] =>
+ dispatch.mock.calls.map(call => (call[0] as RouteResult).platform);
+
+ it('a 503 on alpha key1 rules out alpha entirely — every key AND every model', async () => {
+ const state = newFallbackState();
+ const dispatch = vi.fn(async () => {
+ throw Object.assign(new Error('Alpha API error 503: Service Unavailable'), { status: 503 });
+ });
+
+ await runFallbackLoop(hooksSkeleton({ maxRetries: 20, state, route: miniRouter(state), dispatch }));
+
+ // Two hops, not four: alpha's sibling key and its second model are skipped
+ // wholesale, so failover spends its budget on the NEXT provider.
+ expect(platformsTried(dispatch)).toEqual(['alpha', 'beta']);
+ expect(state.skipPlatforms.has('alpha')).toBe(true);
+ });
+
+ it('a 401 on alpha key1 still rotates to alpha key2 (key-scoped, not provider-scoped)', async () => {
+ const state = newFallbackState();
+ const dispatch = vi.fn()
+ .mockRejectedValueOnce(Object.assign(new Error('Invalid API Key'), { status: 401 }))
+ .mockResolvedValueOnce('done');
+
+ await runFallbackLoop(hooksSkeleton({ maxRetries: 20, state, route: miniRouter(state), dispatch }));
+
+ expect(dispatch).toHaveBeenCalledTimes(2);
+ const second = dispatch.mock.calls[1][0] as RouteResult;
+ expect(second.platform).toBe('alpha');
+ expect(second.keyId).toBe(78_802);
+ expect(state.skipPlatforms.size).toBe(0);
+ });
+
+ it('a plain 429 also stays key-scoped', async () => {
+ const state = newFallbackState();
+ const dispatch = vi.fn()
+ .mockRejectedValueOnce(Object.assign(new Error('429 Too Many Requests'), { status: 429 }))
+ .mockResolvedValueOnce('done');
+
+ await runFallbackLoop(hooksSkeleton({ maxRetries: 20, state, route: miniRouter(state), dispatch }));
+
+ expect((dispatch.mock.calls[1][0] as RouteResult).platform).toBe('alpha');
+ expect(state.skipPlatforms.size).toBe(0);
+ });
+
+ it('a timeout with no HTTP status rules the platform out too', async () => {
+ const state = newFallbackState();
+ const dispatch = vi.fn()
+ .mockRejectedValueOnce(new Error('The operation was aborted (alpha, chat, 120s)'))
+ .mockResolvedValueOnce('done');
+
+ await runFallbackLoop(hooksSkeleton({ maxRetries: 20, state, route: miniRouter(state), dispatch }));
+
+ expect((dispatch.mock.calls[1][0] as RouteResult).platform).toBe('beta');
+ expect(state.skipPlatforms.has('alpha')).toBe(true);
+ });
+});
+
describe('runFallbackLoop: client disconnect + attempt log', () => {
const retryable = () => Object.assign(new Error('429 Too Many Requests'), { status: 429 });
diff --git a/server/src/__tests__/routes/proxy-provider-level-skip.test.ts b/server/src/__tests__/routes/proxy-provider-level-skip.test.ts
new file mode 100644
index 00000000..553500ab
--- /dev/null
+++ b/server/src/__tests__/routes/proxy-provider-level-skip.test.ts
@@ -0,0 +1,153 @@
+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';
+import type { Express } from 'express';
+
+// #788: a provider-level failure (5xx, timeout, dead socket) is about the
+// PROVIDER, not the key that happened to catch it. Failover used to burn one
+// hop per sibling key — and then one per sibling MODEL of the same sick
+// platform — before it ever reached the next provider. The router now honors a
+// request-scoped skipPlatforms set, so the first 503 moves the chain on.
+// Key-scoped failures (401, 429) must keep rotating to the sibling key.
+//
+// End-to-end through the real router so the skip is proven where candidates are
+// actually enumerated, not just in the loop's bookkeeping.
+
+const chatCompletion = vi.fn();
+const streamChatCompletion = vi.fn();
+const fakeProvider = { name: 'fake', chatCompletion, streamChatCompletion } as any;
+
+vi.mock('../../providers/index.js', async (importOriginal) => {
+ const actual = await importOriginal() as any;
+ return {
+ ...actual,
+ getProvider: () => fakeProvider,
+ resolveProvider: () => fakeProvider,
+ };
+});
+
+const { mockCheckKeyHealth, mockMarkKeyHealthy } = vi.hoisted(() => ({
+ mockCheckKeyHealth: vi.fn(),
+ mockMarkKeyHealthy: vi.fn(),
+}));
+vi.mock('../../services/health.js', () => ({
+ checkKeyHealth: mockCheckKeyHealth,
+ markKeyHealthyFromRequest: mockMarkKeyHealthy,
+}));
+
+const { createApp } = await import('../../app.js');
+const { initDb, getDb, getUnifiedApiKey } = await import('../../db/index.js');
+const { encrypt } = await import('../../lib/crypto.js');
+const { setRoutingStrategy } = await import('../../services/router.js');
+
+async function post(app: Express, body: any, key: string) {
+ const server = app.listen(0);
+ const addr = server.address() as any;
+ const res = await fetch(`http://127.0.0.1:${addr.port}/v1/chat/completions`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` },
+ body: JSON.stringify(body),
+ });
+ const raw = await res.text();
+ server.close();
+ let json: any = null;
+ try { json = JSON.parse(raw); } catch { /* not JSON */ }
+ return { status: res.status, body: json };
+}
+
+const GOOD_RESULT = {
+ choices: [{ message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }],
+ usage: { prompt_tokens: 5, completion_tokens: 2, total_tokens: 7 },
+};
+const err503 = () => Object.assign(new Error('Groq API error 503: Service Unavailable'), { status: 503 });
+const err401 = () => Object.assign(new Error('Groq API error 401: Invalid API Key'), { status: 401 });
+
+// The decrypted key text ("<platform>-<label>") identifies every dispatched
+// attempt — chatCompletion's first argument. Which of a platform's keys the
+// round-robin cursor lands on first is not part of the contract, so the
+// platform sequence is what these tests assert on.
+const keysUsed = (): string[] => chatCompletion.mock.calls.map(call => String(call[0]));
+const platformsUsed = (): string[] => keysUsed().map(k => k.split('-')[0]);
+
+// Two groq models on two groq keys, ranked ahead of one cerebras model: the
+// pre-#788 chain would spend FOUR hops inside groq before reaching cerebras.
+function setup(): void {
+ const db = getDb();
+ setRoutingStrategy('priority');
+ db.prepare("DELETE FROM settings WHERE key = 'active_profile_id'").run();
+ db.prepare('DELETE FROM api_keys').run();
+ db.prepare('DELETE FROM rate_limit_cooldowns').run();
+ db.prepare('DELETE FROM rate_limit_usage').run();
+ db.prepare('UPDATE models SET enabled = 0').run();
+
+ const pick = (platform: string, limit: number) => db.prepare(
+ 'SELECT id FROM models WHERE platform = ? ORDER BY id LIMIT ?',
+ ).all(platform, limit) as { id: number }[];
+ const chain = [...pick('groq', 2), ...pick('cerebras', 1)];
+ const enable = db.prepare(
+ 'UPDATE models SET enabled = 1, rpm_limit = NULL, rpd_limit = NULL, tpm_limit = NULL, tpd_limit = NULL WHERE id = ?',
+ );
+ const prioritize = db.prepare('UPDATE fallback_config SET priority = ?, enabled = 1 WHERE model_db_id = ?');
+ chain.forEach((m, i) => { enable.run(m.id); prioritize.run(i + 1, m.id); });
+
+ for (const [platform, label] of [['groq', 'a'], ['groq', 'b'], ['cerebras', 'c']] as const) {
+ const { encrypted, iv, authTag } = encrypt(`${platform}-${label}`);
+ db.prepare(`
+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)
+ VALUES (?, ?, ?, ?, ?, 'healthy', 1)
+ `).run(platform, label, encrypted, iv, authTag);
+ }
+}
+
+describe('provider-level failures skip the whole platform (#788)', () => {
+ let app: Express;
+ let key: string;
+
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ app = createApp();
+ key = getUnifiedApiKey();
+ });
+
+ beforeEach(() => {
+ chatCompletion.mockReset();
+ streamChatCompletion.mockReset();
+ mockCheckKeyHealth.mockReset();
+ mockCheckKeyHealth.mockResolvedValue('invalid');
+ setup();
+ });
+
+ it('a 503 on the first groq key fails over to cerebras, not to groq key b', async () => {
+ chatCompletion
+ .mockRejectedValueOnce(err503())
+ .mockResolvedValueOnce(GOOD_RESULT);
+
+ const { status } = await post(app, { messages: [{ role: 'user', content: 'hi' }] }, key);
+
+ expect(status).toBe(200);
+ expect(chatCompletion).toHaveBeenCalledTimes(2);
+ expect(platformsUsed()).toEqual(['groq', 'cerebras']);
+ });
+
+ it('a groq-wide outage burns two hops, not four', async () => {
+ // Both groq keys x both groq models used to be four separate attempts.
+ chatCompletion.mockRejectedValue(err503());
+
+ const { status } = await post(app, { messages: [{ role: 'user', content: 'hi' }] }, key);
+
+ expect(status).toBe(502); // upstream failures, not a rate-limit exhaustion
+ expect(platformsUsed()).toEqual(['groq', 'cerebras']);
+ });
+
+ it('a 401 still rotates to the sibling groq key (key-scoped, unchanged)', async () => {
+ chatCompletion
+ .mockRejectedValueOnce(err401())
+ .mockResolvedValueOnce(GOOD_RESULT);
+
+ const { status } = await post(app, { messages: [{ role: 'user', content: 'hi' }] }, key);
+
+ expect(status).toBe(200);
+ expect(platformsUsed()).toEqual(['groq', 'groq']);
+ const [first, second] = keysUsed();
+ expect(second).not.toBe(first); // the sibling key, not a retry of the dead one
+ });
+});
diff --git a/server/src/lib/error-classify.ts b/server/src/lib/error-classify.ts
index b7bc173b..9ef37a04 100644
--- a/server/src/lib/error-classify.ts
+++ b/server/src/lib/error-classify.ts
@@ -240,6 +240,34 @@ export function isProviderDegradedError(err: any): boolean {
return msg.includes('degraded');
}
+// #788: provider-level failures — 5xx, timeouts, transport/network errors, a
+// degraded deployment — are symptoms of the PROVIDER being sick, not of this
+// particular key. Retrying the same provider with a sibling key would fail
+// identically, so the fallback loop must skip the WHOLE platform for the
+// request instead of burning one failover hop per key. Key-scoped failures
+// (auth, quota, 403 tier) stay out of here so a dead key can still rotate to
+// a healthy sibling on the same platform.
+//
+// Classified on the STRUCTURED status — every adapter attaches one to an HTTP
+// failure (providerHttpError in providers/base.ts) — plus the two families that
+// carry no status at all: a timeout and a transport-level failure. Deliberately
+// NO bare '500' / '503' / 'unavailable' / 'internal server error' substrings: a
+// token count, a duration ("… took 5003ms") or a key-scoped message naming an
+// unavailable model would each condemn a healthy platform for the whole request.
+// A message check therefore only runs when there is no status to trust.
+export function isProviderLevelError(err: any): boolean {
+ const status = typeof err?.status === 'number' ? err.status : 0;
+ if (status >= 500) return true;
+ // A DEGRADED hosted deployment (NVIDIA NIM, #522) is provider health wearing
+ // a 400 — same source of truth as everywhere else, not a second substring.
+ if (isProviderDegradedError(err)) return true;
+ if (status !== 0) return false;
+ const msg = (err?.message ?? '').toLowerCase();
+ return isTimeoutErrorText(msg)
+ || msg.includes('econnrefused') || msg.includes('econnreset')
+ || msg.includes('fetch failed'); // undici transport error (DNS/TLS/proxy down)
+}
+
// Provider-side 400s are retryable because another provider may accept the same
// request shape. If every routed provider rejects it, however, the client should
// see an invalid-request error rather than a misleading rate-limit exhaustion.
diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.ts
index 0f8fa23c..7aad94ac 100644
--- a/server/src/lib/fallback-loop.ts
+++ b/server/src/lib/fallback-loop.ts
@@ -41,6 +41,7 @@ import {
isModelAccessForbiddenError,
isProviderBadRequestError,
isProviderDegradedError,
+ isProviderLevelError,
isContextTooLargeError,
isTimeoutErrorText,
} from './error-classify.js';
@@ -146,14 +147,18 @@ export function getFallbackTimeBudgetMs(): number {
// Mutable per-request skip state threaded through the loop and mutated by
// recordRetryableFailure / recordAuthFailure. skipKeys entries are
// "platform:modelId:keyId"; skipModels holds model_db_ids ruled out for the
-// rest of this request.
+// rest of this request; skipPlatforms holds platforms ruled out wholesale
+// (#788) — every model and every key of them — for the rest of this request.
+// All three are request-scoped only: nothing here outlives the response, so a
+// provider that blipped once is a fresh candidate on the very next request.
export interface FallbackState {
skipKeys: Set<string>;
skipModels: Set<number>;
+ skipPlatforms: Set<string>;
}
export function newFallbackState(): FallbackState {
- return { skipKeys: new Set<string>(), skipModels: new Set<number>() };
+ return { skipKeys: new Set<string>(), skipModels: new Set<number>(), skipPlatforms: new Set<string>() };
}
// Milliseconds until the next UTC midnight — when most providers' daily free
@@ -294,6 +299,16 @@ export function recordRetryableFailure(route: RouteResult, err: any, state: Fall
// sibling keys counts as the single observation it is.
noteModelRetirementSignal(route, err, getRequestTrace());
state.skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);
+ // #788: provider-level failures (5xx / timeout / transport / degraded) mean
+ // the PROVIDER is sick, not this key — every key AND every model of that
+ // platform would fail identically. Rule out the whole platform for this
+ // request so the loop moves to the NEXT provider instead of burning one
+ // failover hop per key. Key-scoped failures (auth/quota) stay on the
+ // single-key path, and the per-key cooldown below is still the only thing
+ // that outlives the request.
+ if (isProviderLevelError(err)) {
+ state.skipPlatforms.add(route.platform);
+ }
if (consumeSkipBenchExemption(route, err)) return true;
const decision = cooldownDecisionForError(route, err);
setCooldown(route.platform, route.modelId, route.keyId, decision.durationMs, decision.source);
@@ -890,7 +905,8 @@ export interface FallbackHooks {
state: FallbackState;
/**
- * Pick a route for this attempt. Reads state.skipKeys / state.skipModels.
+ * Pick a route for this attempt. Reads state.skipKeys / state.skipModels /
+ * state.skipPlatforms.
* Throws the router's RouteError when the pool is exhausted before any
* upstream is tried (caught by the loop → onRoutingExhausted).
*/
diff --git a/server/src/lib/inbound-chat.ts b/server/src/lib/inbound-chat.ts
index 79b1e48b..329a3ade 100644
--- a/server/src/lib/inbound-chat.ts
+++ b/server/src/lib/inbound-chat.ts
@@ -233,6 +233,7 @@ export async function runInboundChat(
state.skipModels.size ? state.skipModels : undefined,
pin.strictChain,
input.responseFormat !== undefined,
+ state.skipPlatforms.size ? state.skipPlatforms : undefined,
),
dispatch: async (route, attempt) => {
if (!input.stream) {
diff --git a/server/src/routes/anthropic.ts b/server/src/routes/anthropic.ts
index ab5b9ab1..6a477e06 100644
--- a/server/src/routes/anthropic.ts
+++ b/server/src/routes/anthropic.ts
@@ -556,7 +556,7 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {
state,
attemptLog,
clientGone: () => clientGone,
- route: () => routeRequest(estimatedTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined),
+ route: () => routeRequest(estimatedTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, undefined, false, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined),
dispatch: async (route, attempt) => {
if (stream) {
try {
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 8ea7025a..d4e12e53 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -957,6 +957,8 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
false,
state.skipModels.size > 0 ? state.skipModels : undefined,
groupChain ?? resolvedChain?.chain,
+ false,
+ state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined,
),
dispatch: async (route, attempt) => {
traceRouteEvent('Proxy', {
@@ -1740,7 +1742,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
// model is on record). Turns where injection can't happen — every turn 1, and
// sessions that never switched — pay no headroom tax.
const routingEstimate = handoffPossible ? estimatedTotal + HANDOFF_MAX_TOKENS : estimatedTotal;
- return routeRequest(routingEstimate, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain ?? resolvedChain?.chain, samplingParams.response_format !== undefined);
+ return routeRequest(routingEstimate, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain ?? resolvedChain?.chain, samplingParams.response_format !== undefined, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined);
},
dispatch: async (route, attempt) => {
const modelKey = `${route.platform}:${route.modelId}`;
diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.ts
index 5bb73b34..76f97863 100644
--- a/server/src/routes/responses.ts
+++ b/server/src/routes/responses.ts
@@ -520,7 +520,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
state,
attemptLog,
clientGone: () => clientGone,
- route: () => routeRequest(estimatedTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, false, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain, completionOpts.response_format !== undefined),
+ route: () => routeRequest(estimatedTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, false, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain, completionOpts.response_format !== undefined, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined),
dispatch: async (route, attempt) => {
traceRouteEvent('Responses', {
event: attempt === 0 ? 'start' : 'next',
diff --git a/server/src/services/router.ts b/server/src/services/router.ts
index 906c3a8a..87ce9741 100644
--- a/server/src/services/router.ts
+++ b/server/src/services/router.ts
@@ -882,6 +882,7 @@ function orderChain(chain: ChainRow[], strategy: RoutingStrategy, sampled = true
* @param preferredModelDbId - try this model first (sticky session)
* @param requireVision - only consider models that accept image input (#118)
* @param requireTools - only consider models that emit structured tool_calls
+ * @param skipPlatforms - platforms ruled out for the rest of this request (#788)
*/
export interface ResolvedChain {
chain: ChainRow[];
@@ -1515,7 +1516,7 @@ export function resolveFusionCandidate(modelId: string): FusionCandidate | null
return null;
}
-export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preferredModelDbId?: number, requireVision = false, requireTools = false, skipModels?: Set<number>, prefetchedChain?: ChainRow[], requireStructured = false): RouteResult {
+export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preferredModelDbId?: number, requireVision = false, requireTools = false, skipModels?: Set<number>, prefetchedChain?: ChainRow[], requireStructured = false, skipPlatforms?: Set<string>): RouteResult {
const db = getDb();
const strategy = getRoutingStrategy();
@@ -1587,6 +1588,13 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre
// same dead route (PR #111, credits @barbotkonv).
if (skipModels?.has(entry.model_db_id)) { diag.push(`${label}: ruled out earlier this request`); continue; }
+ // Platforms the caller has ruled out wholesale (#788): a provider-level
+ // failure this request — a 5xx, a timeout, a dead socket — is about the
+ // PROVIDER, so its other keys and its other models would fail the same way.
+ // Skipping the platform moves failover to the next provider instead of
+ // burning one hop per key. Request-scoped; nothing is benched by this.
+ if (skipPlatforms?.has(entry.platform)) { diag.push(`${label}: provider ruled out earlier this request`); continue; }
+
// Vision requests skip text-only models — including a sticky/preferred one,
// which is correct: don't pin an image turn to a model that can't see it.
if (requireVision && !entry.supports_vision) { diag.push(`${label}: no vision support`); continue; }