Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

P1 remainder: cooldown-probe recovery, client AbortSignal propagation, honest exhaustion statuses (#601)

Tashfeen5a5b17cfaa5a
diff --git a/.gitignore b/.gitignore
index 7442529a..92b84aa6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,9 @@
node_modules/
dist/
server/data/
+# Runtime data dir when the server is launched from the repo root (same
+# contents as server/data — DB + keyfile — so same rule).
+/data/
*.db
*.db-wal
*.db-shm
diff --git a/server/src/__tests__/db/migrate/roundtrip.test.ts b/server/src/__tests__/db/migrate/roundtrip.test.ts
index cd5b2690..292f45e8 100644
--- a/server/src/__tests__/db/migrate/roundtrip.test.ts
+++ b/server/src/__tests__/db/migrate/roundtrip.test.ts
@@ -13,6 +13,7 @@ const REQUEST_CLIENT_INFO_FILENAME = '20260706_000001_request_client_info.ts';
const CUSTOM_MODEL_TOOL_SUPPORT_FILENAME = '20260706_000002_custom_model_tool_support.ts';
const PROFILE_CHAIN_BACKFILL_FILENAME = '20260714_000001_profile_chain_backfill.ts';
const KEY_HEALTH_ERROR_FILENAME = '20260720_000001_key_health_error.ts';
+const COOLDOWN_PROBE_PROVENANCE_FILENAME = '20260726_000001_cooldown_probe_provenance.ts';
interface SchemaRow {
type: string;
@@ -72,6 +73,7 @@ describe('migration round trip', () => {
CUSTOM_MODEL_TOOL_SUPPORT_FILENAME,
PROFILE_CHAIN_BACKFILL_FILENAME,
KEY_HEALTH_ERROR_FILENAME,
+ COOLDOWN_PROBE_PROVENANCE_FILENAME,
]);
} finally {
db.close();
diff --git a/server/src/__tests__/integration/full-flow.test.ts b/server/src/__tests__/integration/full-flow.test.ts
index 0265b774..0c103870 100644
--- a/server/src/__tests__/integration/full-flow.test.ts
+++ b/server/src/__tests__/integration/full-flow.test.ts
@@ -162,23 +162,23 @@ describe('Full Integration Flow', () => {
expect(s2).toBe(400);
});
- it('Step 11: Explicit unknown model returns 400 (not silent fallthrough)', async () => {
+ it('Step 11: Explicit unknown model returns an honest 404 (not silent fallthrough)', async () => {
const { status, body } = await req(app, 'POST', '/v1/chat/completions', {
model: 'definitely-not-a-real-model',
messages: [{ role: 'user', content: 'hi' }],
}, authHeaders());
- expect(status).toBe(400);
+ expect(status).toBe(404);
expect(body.error.code).toBe('model_not_found');
expect(body.error.message).toContain('not in the catalog');
});
- it('Step 12: Explicit disabled model returns 400 with disabled reason', async () => {
+ it('Step 12: Explicit disabled model returns an honest 404 with disabled reason', async () => {
// gemini-2.5-pro is disabled (V1 migration). Reuse it as a known-disabled fixture.
const { status, body } = await req(app, 'POST', '/v1/chat/completions', {
model: 'gemini-2.5-pro',
messages: [{ role: 'user', content: 'hi' }],
}, authHeaders());
- expect(status).toBe(400);
+ expect(status).toBe(404);
expect(body.error.code).toBe('model_not_found');
expect(body.error.message).toContain('is disabled');
});
diff --git a/server/src/__tests__/lib/exhaustion-statuses.test.ts b/server/src/__tests__/lib/exhaustion-statuses.test.ts
new file mode 100644
index 00000000..8f621927
--- /dev/null
+++ b/server/src/__tests__/lib/exhaustion-statuses.test.ts
@@ -0,0 +1,438 @@
+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';
+
+// Honest exhaustion statuses (P1): when the fallback ladder exhausts every
+// candidate, the terminal status must say WHY it failed and when to retry —
+// 413 when every attempt died on a context/size rejection, 429 + retryAtMs +
+// Retry-After when every candidate is unavailable until a known time, 404 when
+// the model is gone everywhere, 503 when nothing is configured at all, and 502
+// (never 500, never a lying 429) for mixed/other upstream failures.
+
+const { mockCheckKeyHealth } = vi.hoisted(() => ({ mockCheckKeyHealth: vi.fn() }));
+vi.mock('../../services/health.js', () => ({ checkKeyHealth: mockCheckKeyHealth, markKeyHealthyFromRequest: vi.fn() }));
+
+import { initDb, getDb } from '../../db/index.js';
+import {
+ runFallbackLoop,
+ newFallbackState,
+ exhaustedRetryError,
+ routingExhaustionBody,
+ exhaustionErrorPayload,
+ setExhaustionHeaders,
+ classifyAttemptError,
+ type AttemptRecord,
+ type AttemptErrorClass,
+ type FallbackHooks,
+} from '../../lib/fallback-loop.js';
+import { isContextTooLargeError } from '../../lib/error-classify.js';
+import { setCooldown } from '../../services/ratelimit.js';
+import type { RouteResult } from '../../services/router.js';
+
+let keySeq = 96_000;
+function fakeRoute(overrides: Partial<RouteResult> = {}): RouteResult {
+ const n = ++keySeq;
+ return {
+ provider: {} as any, modelId: 'fake-model', modelDbId: 962000 + n, apiKey: 'k',
+ keyId: n, platform: 'fake', displayName: 'Fake Model',
+ rpdLimit: null, tpdLimit: null,
+ ...overrides,
+ };
+}
+
+function hooksSkeleton(overrides: Partial<FallbackHooks>): FallbackHooks {
+ return {
+ state: newFallbackState(),
+ timeBudgetMs: 0,
+ route: () => fakeRoute(),
+ dispatch: async () => 'done',
+ logFailure: () => {},
+ onFatal: () => {},
+ onRoutingExhausted: () => {},
+ onExhausted: () => {},
+ ...overrides,
+ };
+}
+
+const record = (classes: AttemptErrorClass[]): AttemptRecord[] =>
+ classes.map((errorClass, i) => ({ platform: 'fake', modelId: `m${i}`, keyOrdinal: i + 1, errorClass }));
+
+// Fake key-shaped fixture built at runtime: an obvious prefix + low-entropy filler.
+const FAKE_KEY = 'sk-test-' + 'a'.repeat(20);
+
+beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+});
+
+beforeEach(() => {
+ mockCheckKeyHealth.mockReset();
+ getDb().prepare('DELETE FROM rate_limit_cooldowns').run();
+});
+
+// ── isContextTooLargeError: real provider error shapes ───────────────────────
+
+describe('isContextTooLargeError', () => {
+ it('flags OpenAI-compat context_length_exceeded shapes (message and code)', () => {
+ expect(isContextTooLargeError(Object.assign(
+ new Error("OpenAI-compat API error 400: This model's maximum context length is 8192 tokens. However, your messages resulted in 10012 tokens. Please reduce the length of the messages."),
+ { status: 400 },
+ ))).toBe(true);
+ expect(isContextTooLargeError(Object.assign(new Error('Bad request'), { status: 400, code: 'context_length_exceeded' }))).toBe(true);
+ expect(isContextTooLargeError(new Error("400 {\"error\":{\"message\":\"...\",\"code\":\"context_length_exceeded\"}}"))).toBe(true);
+ });
+
+ it('flags Anthropic-style prompt-too-long shapes', () => {
+ expect(isContextTooLargeError(Object.assign(
+ new Error('Anthropic API error 400: prompt is too long: 210021 tokens > 200000 maximum'),
+ { status: 400 },
+ ))).toBe(true);
+ expect(isContextTooLargeError(new Error('Input is too long for requested model.'))).toBe(true);
+ });
+
+ it('flags Google/Gemini token-limit shapes', () => {
+ expect(isContextTooLargeError(Object.assign(
+ new Error('Google API error 400: The input token count (1194286) exceeds the maximum number of tokens allowed (1048576).'),
+ { status: 400 },
+ ))).toBe(true);
+ expect(isContextTooLargeError(new Error('request exceeds the maximum number of tokens'))).toBe(true);
+ });
+
+ it('flags literal HTTP 413s, including Groq TPM-ceiling 413s (Requested > Limit can never fit)', () => {
+ expect(isContextTooLargeError(Object.assign(
+ new Error('Groq API error 413: Request too large for model `llama-3.3-70b-versatile` in organization `org_x` on tokens per minute (TPM): Limit 30000, Requested 33476'),
+ { status: 413 },
+ ))).toBe(true);
+ expect(isContextTooLargeError(new Error('API error 413: Payload Too Large'))).toBe(true);
+ expect(isContextTooLargeError(new Error('Request entity too large'))).toBe(true);
+ expect(isContextTooLargeError(new Error('Cloudflare API error 413: content too large'))).toBe(true);
+ });
+
+ it('does not flag rate limits, ordinary 400s, or upstream errors', () => {
+ expect(isContextTooLargeError(Object.assign(new Error('429 Too Many Requests'), { status: 429 }))).toBe(false);
+ expect(isContextTooLargeError(Object.assign(new Error('Groq API error 429: Rate limit reached on requests per day (RPD): Limit 1000'), { status: 429 }))).toBe(false);
+ expect(isContextTooLargeError(Object.assign(new Error('Google API error 400: Invalid JSON payload received. Unknown name "x"'), { status: 400 }))).toBe(false);
+ expect(isContextTooLargeError(Object.assign(new Error('Internal server error'), { status: 500 }))).toBe(false);
+ expect(isContextTooLargeError(new Error('you have used up your daily free allocation of 10,000 neurons'))).toBe(false);
+ });
+
+ it('classifyAttemptError buckets context errors ahead of provider_bad_request', () => {
+ // The OpenAI-compat shape contains "API error 400", which the generic
+ // bad-request rule would otherwise swallow.
+ expect(classifyAttemptError(Object.assign(
+ new Error("OpenAI-compat API error 400: This model's maximum context length is 8192 tokens."),
+ { status: 400 },
+ ))).toBe('context_too_large');
+ expect(classifyAttemptError(Object.assign(new Error('API error 413: Payload Too Large'), { status: 413 }))).toBe('context_too_large');
+ });
+});
+
+// ── exhaustedRetryError: failure-kind → terminal-status taxonomy ─────────────
+
+describe('exhaustedRetryError honest terminal statuses', () => {
+ it('all attempts context-too-large → 413 invalid_request_error / context_length_exceeded', () => {
+ const body = exhaustedRetryError(new Error('prompt is too long: 210000 tokens > 200000 maximum'), 20, {
+ attempts: record(['context_too_large', 'context_too_large', 'context_too_large']),
+ });
+ expect(body.status).toBe(413);
+ expect(body.kind).toBe('context_too_large');
+ expect(body.type).toBe('invalid_request_error');
+ expect(body.code).toBe('context_length_exceeded');
+ expect(body.retryAtMs).toBeUndefined();
+ expect(body.message).toContain('too large');
+ expect(body.message).toContain('Attempt trail:');
+ });
+
+ it('the all-context 413 wins even when the last error is shaped like a generic provider 400', () => {
+ // "API error 400" would match isProviderBadRequestError; the aggregate
+ // unanimity must outrank the shape of whichever error came last.
+ const lastError = Object.assign(
+ new Error("OpenAI-compat API error 400: This model's maximum context length is 8192 tokens."),
+ { status: 400 },
+ );
+ const body = exhaustedRetryError(lastError, 20, { attempts: record(['context_too_large', 'context_too_large']) });
+ expect(body.status).toBe(413);
+ expect(body.code).toBe('context_length_exceeded');
+ });
+
+ it('all attempts model-not-found → 404 model_not_found (no 410: the catalog keeps no removal tombstones)', () => {
+ const body = exhaustedRetryError(Object.assign(new Error('no endpoints found'), { status: 404 }), 20, {
+ attempts: record(['model_not_found', 'model_not_found']),
+ });
+ expect(body.status).toBe(404);
+ expect(body.kind).toBe('model_not_found');
+ expect(body.type).toBe('invalid_request_error');
+ expect(body.code).toBe('model_not_found');
+ });
+
+ it('every candidate unavailable-until-known-time → 429 with retryAtMs from the soonest cooldown', () => {
+ const before = Date.now();
+ setCooldown('fake', 'benched-model', ++keySeq, 90_000);
+ const body = exhaustedRetryError(Object.assign(new Error('429 Too Many Requests'), { status: 429 }), 20, {
+ attempts: record(['rate_limited', 'daily_quota_exhausted', 'out_of_credits', 'forbidden']),
+ });
+ expect(body.status).toBe(429);
+ expect(body.kind).toBe('rate_limit');
+ expect(body.type).toBe('rate_limit_error');
+ expect(body.code).toBe('rate_limit_exceeded');
+ expect(body.retryAtMs).toBeGreaterThanOrEqual(before + 80_000);
+ expect(body.retryAtMs).toBeLessThanOrEqual(Date.now() + 95_000);
+ });
+
+ it('the 429 omits retryAtMs when nothing is cooling down (no invented numbers)', () => {
+ const body = exhaustedRetryError(new Error('429 Too Many Requests'), 20, {
+ attempts: record(['rate_limited']),
+ });
+ expect(body.status).toBe(429);
+ expect(body.retryAtMs).toBeUndefined();
+ });
+
+ it('mixed rate-limit + upstream-error → 502 (waiting is not a promise), never 500', () => {
+ setCooldown('fake', 'benched-model', ++keySeq, 90_000); // a cooldown exists, but the mix forbids the 429
+ const body = exhaustedRetryError(Object.assign(new Error('Bad Gateway'), { status: 502 }), 20, {
+ attempts: record(['rate_limited', 'upstream_error']),
+ });
+ expect(body.status).toBe(502);
+ expect(body.kind).toBe('upstream');
+ expect(body.type).toBe('provider_error');
+ expect(body.code).toBe('upstream_failed');
+ expect(body.retryAtMs).toBeUndefined();
+ expect(body.message).toContain('provider-side failure');
+ });
+
+ it('all upstream/network failures → 502 upstream_failed (upstream failed, not our bug)', () => {
+ for (const classes of [['upstream_error', 'upstream_error'], ['timeout', 'upstream_error'], ['timeout', 'error']] as AttemptErrorClass[][]) {
+ const body = exhaustedRetryError(Object.assign(new Error('fetch failed'), {}), 20, { attempts: record(classes) });
+ expect(body.status).toBe(502);
+ expect(body.code).toBe('upstream_failed');
+ }
+ });
+
+ it('mixed context + rate-limit is neither a 413 nor a 429: falls through to 502', () => {
+ const body = exhaustedRetryError(Object.assign(new Error('429'), { status: 429 }), 20, {
+ attempts: record(['context_too_large', 'rate_limited']),
+ });
+ expect(body.status).toBe(502);
+ });
+
+ it('precedence: all-auth 502 and the aggregate 413 both outrank the circuit-breaker 503', () => {
+ const auth = exhaustedRetryError(new Error('401'), 20, { attempts: record(['auth', 'auth']), breakerFails: 2 });
+ expect(auth.status).toBe(502);
+ expect(auth.kind).toBe('auth');
+ expect(auth.code).toBe('provider_authentication_failed');
+
+ const ctx = exhaustedRetryError(new Error('prompt is too long: 9 tokens > 8 maximum'), 20, {
+ attempts: record(['context_too_large', 'context_too_large']),
+ breakerFails: 2,
+ });
+ expect(ctx.status).toBe(413);
+ });
+
+ it('precedence: a breaker stop over a MIXED pool still renders the 503 breaker body', () => {
+ const body = exhaustedRetryError(Object.assign(new Error('Bad Gateway'), { status: 502 }), 20, {
+ attempts: record(['rate_limited', 'upstream_error']),
+ breakerFails: 2,
+ });
+ expect(body.status).toBe(503);
+ expect(body.code).toBe('upstream_unhealthy');
+ expect(body.message).toContain('circuit-breaker');
+ });
+
+ it('legacy zero-attempt shape keeps the 429 contract', () => {
+ const body = exhaustedRetryError(new Error('429 Too Many Requests'), 20);
+ expect(body.status).toBe(429);
+ expect(body.type).toBe('rate_limit_error');
+ });
+});
+
+// ── routingExhaustionBody: zero attempts ran ─────────────────────────────────
+
+describe('routingExhaustionBody (router gave up before any upstream was tried)', () => {
+ it('empty chain / no diagnostics → 503 no_providers_configured', () => {
+ const body = routingExhaustionBody({ message: 'All models exhausted.', status: 429, diagnostics: [] });
+ expect(body.status).toBe(503);
+ expect(body.type).toBe('service_unavailable');
+ expect(body.code).toBe('no_providers_configured');
+ });
+
+ it('every candidate lacks a provider/usable key → 503 no_providers_configured', () => {
+ const body = routingExhaustionBody({
+ message: 'All models exhausted: 2 routes checked.',
+ status: 429,
+ diagnostics: [
+ 'openai/gpt-4o-mini: no enabled+healthy key for platform',
+ 'weird/x: no provider registered',
+ ],
+ });
+ expect(body.status).toBe(503);
+ expect(body.code).toBe('no_providers_configured');
+ });
+
+ it('every candidate rate-limited/on cooldown → 429 with retryAtMs', () => {
+ const before = Date.now();
+ setCooldown('fake', 'routing-benched', ++keySeq, 120_000);
+ const body = routingExhaustionBody({
+ message: 'All models exhausted: 2 routes checked.',
+ status: 429,
+ diagnostics: [
+ 'groq/llama-3.3-70b: 2 key(s) — cooldown:2',
+ 'google/gemini-2.0-flash: 1 key(s) — rpm/rpd-limit:1',
+ ],
+ });
+ expect(body.status).toBe(429);
+ expect(body.code).toBe('rate_limit_exceeded');
+ expect(body.retryAtMs).toBeGreaterThanOrEqual(before + 110_000);
+ });
+
+ it('rate-limited candidates mixed with unconfigured ones still → 429 (the pool recovers by itself)', () => {
+ const body = routingExhaustionBody({
+ message: 'All models exhausted.',
+ status: 429,
+ diagnostics: [
+ 'groq/llama-3.3-70b: 1 key(s) — cooldown:1',
+ 'weird/x: no provider registered',
+ ],
+ });
+ expect(body.status).toBe(429);
+ expect(body.code).toBe('rate_limit_exceeded');
+ });
+
+ it('every candidate too small for the request → 413 context_length_exceeded', () => {
+ const body = routingExhaustionBody({
+ message: 'All models exhausted: 2 routes checked (2 prompt too large for the model).',
+ status: 429,
+ diagnostics: [
+ 'groq/llama-3.1-8b: context 8192 < estimated 50000',
+ 'groq/gpt-oss-120b: tpm_limit 8000 < estimated 50000',
+ ],
+ });
+ expect(body.status).toBe(413);
+ expect(body.code).toBe('context_length_exceeded');
+ expect(body.type).toBe('invalid_request_error');
+ });
+
+ it('capability-filtered candidates keep the router status with a routing_exhausted code', () => {
+ const body = routingExhaustionBody({
+ message: 'All models exhausted.',
+ status: 429,
+ diagnostics: [
+ 'fake/text-only: no vision support',
+ 'groq/llama-3.3-70b: 1 key(s) — cooldown:1',
+ ],
+ });
+ expect(body.status).toBe(429);
+ expect(body.code).toBe('routing_exhausted');
+ });
+});
+
+// ── Wire helpers: OpenAI error payload + Retry-After header ──────────────────
+
+describe('exhaustionErrorPayload / setExhaustionHeaders', () => {
+ it('the payload carries type/code/retryAtMs and omits absent fields', () => {
+ const full = exhaustionErrorPayload({
+ status: 429, kind: 'rate_limit', type: 'rate_limit_error', code: 'rate_limit_exceeded',
+ retryAtMs: 1234, message: 'm',
+ });
+ expect(full).toEqual({ message: 'm', type: 'rate_limit_error', code: 'rate_limit_exceeded', retryAtMs: 1234 });
+
+ const bare = exhaustionErrorPayload({ status: 502, kind: 'upstream', type: 'provider_error', message: 'm' });
+ expect(bare).toEqual({ message: 'm', type: 'provider_error' });
+ expect('retryAtMs' in bare).toBe(false);
+ });
+
+ it('Retry-After is ceil((retryAtMs - now) / 1000) seconds', () => {
+ const headers: Record<string, string> = {};
+ const res = { setHeader: (n: string, v: string) => { headers[n] = v; } };
+ const now = Date.now();
+ setExhaustionHeaders(res, {
+ status: 429, kind: 'rate_limit', type: 'rate_limit_error', message: 'm',
+ retryAtMs: now + 61_500,
+ }, now);
+ expect(headers['Retry-After']).toBe('62');
+ });
+
+ it('no Retry-After without a concrete retry time; an elapsed one clamps to 0', () => {
+ const headers: Record<string, string> = {};
+ const res = { setHeader: (n: string, v: string) => { headers[n] = v; } };
+ setExhaustionHeaders(res, { status: 502, kind: 'upstream', type: 'provider_error', message: 'm' });
+ expect(headers['Retry-After']).toBeUndefined();
+ setExhaustionHeaders(res, { status: 429, kind: 'rate_limit', type: 'rate_limit_error', message: 'm', retryAtMs: Date.now() - 5_000 });
+ expect(headers['Retry-After']).toBe('0');
+ });
+});
+
+// ── End-to-end through the shared loop ───────────────────────────────────────
+
+describe('runFallbackLoop renders honest exhaustion bodies', () => {
+ it('an all-rate-limited chain exhausts into a 429 whose retryAtMs comes from the cooldowns the loop itself set', async () => {
+ const before = Date.now();
+ const onExhausted = vi.fn();
+ const dispatch = vi.fn(async () => {
+ throw Object.assign(new Error('429 Too Many Requests'), { status: 429 });
+ });
+
+ await runFallbackLoop(hooksSkeleton({
+ maxRetries: 3,
+ route: () => fakeRoute({ apiKey: FAKE_KEY }),
+ dispatch,
+ onExhausted,
+ }));
+
+ expect(onExhausted).toHaveBeenCalledTimes(1);
+ const [body] = onExhausted.mock.calls[0];
+ expect(body.status).toBe(429);
+ expect(body.code).toBe('rate_limit_exceeded');
+ // recordRetryableFailure benched each failed key, so the soonest expiry is real.
+ expect(body.retryAtMs).toBeGreaterThan(before);
+ });
+
+ it('a rate-limit + upstream-5xx mix exhausts into a 502, not a lying 429', async () => {
+ const onExhausted = vi.fn();
+ let call = 0;
+ const dispatch = vi.fn(async () => {
+ call += 1;
+ if (call === 1) throw Object.assign(new Error('429 Too Many Requests'), { status: 429 });
+ throw Object.assign(new Error('Bad Gateway'), { status: 502 });
+ });
+
+ await runFallbackLoop(hooksSkeleton({ maxRetries: 3, dispatch, onExhausted }));
+
+ const [body] = onExhausted.mock.calls[0];
+ expect(body.status).toBe(502);
+ expect(body.kind).toBe('upstream');
+ expect(body.code).toBe('upstream_failed');
+ });
+
+ it('an all-context-too-large chain exhausts into a 413', async () => {
+ const onExhausted = vi.fn();
+ const dispatch = vi.fn(async () => {
+ throw Object.assign(
+ new Error("OpenAI-compat API error 400: This model's maximum context length is 8192 tokens."),
+ { status: 400 },
+ );
+ });
+
+ await runFallbackLoop(hooksSkeleton({ maxRetries: 2, dispatch, onExhausted }));
+
+ const [body] = onExhausted.mock.calls[0];
+ expect(body.status).toBe(413);
+ expect(body.code).toBe('context_length_exceeded');
+ });
+
+ it('routing exhaustion with zero attempts hands the surface a ready 503 body when nothing is configured', async () => {
+ const onRoutingExhausted = vi.fn();
+ await runFallbackLoop(hooksSkeleton({
+ route: () => {
+ throw Object.assign(new Error('All models exhausted.'), {
+ status: 429,
+ diagnostics: ['openai/gpt-4o-mini: no enabled+healthy key for platform'],
+ });
+ },
+ onRoutingExhausted,
+ }));
+
+ expect(onRoutingExhausted).toHaveBeenCalledTimes(1);
+ const [lastError, , exhaustion] = onRoutingExhausted.mock.calls[0];
+ expect(lastError).toBeNull();
+ expect(exhaustion.status).toBe(503);
+ expect(exhaustion.code).toBe('no_providers_configured');
+ });
+});
diff --git a/server/src/__tests__/lib/fallback-loop-client-abort.test.ts b/server/src/__tests__/lib/fallback-loop-client-abort.test.ts
new file mode 100644
index 00000000..f98f3b7f
--- /dev/null
+++ b/server/src/__tests__/lib/fallback-loop-client-abort.test.ts
@@ -0,0 +1,147 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import { initDb, getDb } from '../../db/index.js';
+import { runFallbackLoop, newFallbackState, type FallbackState } from '../../lib/fallback-loop.js';
+import { isClientAbortError, newClientAbortError, isRetryableError } from '../../lib/error-classify.js';
+import { acquireLease, releaseLease, resetLeases, inFlightForKey } from '../../services/ratelimit.js';
+import type { RouteResult } from '../../services/router.js';
+
+/**
+ * Client-abort semantics in the shared fallback loop (P1 AbortSignal work):
+ * when the gateway's OWN client hangs up, the composed fetch signal rejects
+ * the in-flight attempt with the marked error from newClientAbortError. That
+ * throw must stop the ladder immediately, release the in-flight lease via the
+ * existing finally, and leave NO provider-failure trace — no cooldown, no
+ * skip-key entry, no logFailure row, no exhaustion render.
+ */
+
+let keySeq = 900;
+function leasedRoute(): { route: RouteResult; keyId: number } {
+ const keyId = ++keySeq;
+ const id = acquireLease('fake', 'fake-model', keyId, 100);
+ return {
+ keyId,
+ route: {
+ provider: {} as any,
+ modelId: 'fake-model',
+ modelDbId: 900000 + keyId,
+ apiKey: 'test-key',
+ keyId,
+ platform: 'fake',
+ displayName: 'Fake Model',
+ rpdLimit: null,
+ tpdLimit: null,
+ release: () => releaseLease(id),
+ },
+ };
+}
+
+function hooks(state: FallbackState, overrides: Record<string, unknown>) {
+ return {
+ state,
+ timeBudgetMs: 0,
+ logFailure: vi.fn(),
+ onFatal: vi.fn(),
+ onRoutingExhausted: vi.fn(),
+ onExhausted: vi.fn(),
+ ...overrides,
+ } as any;
+}
+
+describe('fallback loop on client abort', () => {
+ beforeEach(() => {
+ initDb(':memory:');
+ resetLeases();
+ vi.spyOn(console, 'log').mockImplementation(() => {});
+ });
+
+ afterEach(() => {
+ resetLeases();
+ vi.restoreAllMocks();
+ });
+
+ it('classifies the marked error as client-abort, and not as retryable', () => {
+ const err = newClientAbortError();
+ expect(isClientAbortError(err)).toBe(true);
+ // Must never enter the retryable ladder as a provider failure.
+ expect(isRetryableError(err)).toBe(false);
+ // Ordinary aborts/timeouts keep their provider-timeout classification.
+ expect(isClientAbortError(new DOMException('The operation was aborted (groq, chat, 15s)', 'AbortError'))).toBe(false);
+ });
+
+ it('stops the ladder immediately — no further attempts, nothing rendered', async () => {
+ const state = newFallbackState();
+ const dispatch = vi.fn(async () => { throw newClientAbortError(); });
+ const h = hooks(state, {
+ maxRetries: 5,
+ route: () => leasedRoute().route,
+ dispatch,
+ });
+
+ await runFallbackLoop(h);
+
+ expect(dispatch).toHaveBeenCalledTimes(1);
+ // The socket is gone: no exhaustion body, no fatal render, no error row.
+ expect(h.logFailure).not.toHaveBeenCalled();
+ expect(h.onFatal).not.toHaveBeenCalled();
+ expect(h.onExhausted).not.toHaveBeenCalled();
+ expect(h.onRoutingExhausted).not.toHaveBeenCalled();
+ });
+
+ it('releases the in-flight lease (mid-stream abort surfaces as a dispatch throw)', async () => {
+ const state = newFallbackState();
+ const { route, keyId } = leasedRoute();
+ // Mirror a stream pump: one chunk is consumed, then the client hangs up
+ // and the next upstream read rejects with the marked abort.
+ async function* abortedStream() {
+ yield { choices: [{ delta: { content: 'hel' } }] };
+ throw newClientAbortError();
+ }
+ await runFallbackLoop(hooks(state, {
+ route: () => route,
+ dispatch: async () => {
+ for await (const _chunk of abortedStream()) { /* pump until the abort */ }
+ return 'done' as const;
+ },
+ }));
+
+ expect(inFlightForKey('fake', keyId)).toBe(0);
+ });
+
+ it('records no cooldown, no skip entry, and no failure bookkeeping', async () => {
+ const state = newFallbackState();
+ const { route, keyId } = leasedRoute();
+ await runFallbackLoop(hooks(state, {
+ route: () => route,
+ dispatch: async () => { throw newClientAbortError(); },
+ }));
+
+ // No cooldown row: a vanished client is not a provider-health signal.
+ const cooldown = getDb().prepare('SELECT 1 FROM rate_limit_cooldowns WHERE platform = ? AND key_id = ?').get('fake', keyId);
+ expect(cooldown).toBeUndefined();
+ // No per-request skip state either — nothing was "tried and failed".
+ expect(state.skipKeys.size).toBe(0);
+ expect(state.skipModels.size).toBe(0);
+ });
+
+ it('clientGone still stops the loop before the next attempt after an ordinary failure', async () => {
+ // The pre-existing boundary: a NON-abort provider failure that lands after
+ // the disconnect is still recorded normally, but the loop must not start
+ // another attempt for a client that is gone.
+ const state = newFallbackState();
+ const dispatch = vi.fn(async () => {
+ throw Object.assign(new Error('fake API error 429: rate limit'), { status: 429 });
+ });
+ let gone = false;
+ await runFallbackLoop(hooks(state, {
+ maxRetries: 5,
+ clientGone: () => gone,
+ route: () => leasedRoute().route,
+ dispatch: async () => {
+ gone = true; // client vanishes during the first attempt
+ return dispatch();
+ },
+ }));
+
+ expect(dispatch).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/server/src/__tests__/providers/abort-signal.test.ts b/server/src/__tests__/providers/abort-signal.test.ts
new file mode 100644
index 00000000..a3d7970c
--- /dev/null
+++ b/server/src/__tests__/providers/abort-signal.test.ts
@@ -0,0 +1,167 @@
+import http from 'http';
+import type { AddressInfo } from 'net';
+import { describe, it, expect, afterEach } from 'vitest';
+import { OpenAICompatProvider } from '../../providers/openai-compat.js';
+import { isClientAbortError, newClientAbortError } from '../../lib/error-classify.js';
+import type { ChatMessage } from '@freellmapi/shared/types.js';
+
+/**
+ * AbortSignal-through-fetch coverage (P1, mined from the frankljlyy-ai and
+ * MLuqmanBR forks): fetchWithTimeout used to bound only time-to-headers, so a
+ * body/stream read was unbounded and a client disconnect never canceled the
+ * upstream request. These tests run a REAL loopback HTTP upstream (not a fetch
+ * mock) because the behavior under test is undici's signal wiring itself:
+ * aborting the composed signal must tear down the upstream socket mid-body and
+ * mid-stream, and reject the pending read with the marked client-abort reason.
+ */
+
+const MESSAGES: ChatMessage[] = [{ role: 'user', content: 'hi' }];
+
+interface Upstream {
+ url: string;
+ /** Resolves when the upstream saw its response connection torn down. */
+ closed: Promise<void>;
+ stop: () => Promise<void>;
+}
+
+async function startUpstream(handler: (req: http.IncomingMessage, res: http.ServerResponse) => void): Promise<Upstream> {
+ let sawClose!: () => void;
+ const closed = new Promise<void>(resolve => { sawClose = resolve; });
+ const server = http.createServer((req, res) => {
+ res.on('close', sawClose);
+ handler(req, res);
+ });
+ await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve));
+ const port = (server.address() as AddressInfo).port;
+ return {
+ url: `http://127.0.0.1:${port}/v1`,
+ closed,
+ stop: () => new Promise<void>(resolve => {
+ server.closeAllConnections();
+ server.close(() => resolve());
+ }),
+ };
+}
+
+function makeProvider(baseUrl: string): OpenAICompatProvider {
+ return new OpenAICompatProvider({ platform: 'groq', name: 'TestCompat', baseUrl });
+}
+
+describe('AbortSignal through provider fetch + body/stream reads', () => {
+ let upstream: Upstream | null = null;
+
+ afterEach(async () => {
+ await upstream?.stop();
+ upstream = null;
+ });
+
+ it('client abort mid-body-read cancels the upstream fetch', async () => {
+ // 200 + partial JSON body, then hold the connection open forever: the
+ // request is stuck inside res.json(), past the header timeout's old reach.
+ upstream = await startUpstream((_req, res) => {
+ res.writeHead(200, { 'content-type': 'application/json' });
+ res.write('{"id":"chatcmpl-hang","choices":[');
+ });
+ const provider = makeProvider(upstream.url);
+
+ const clientAbort = new AbortController();
+ const pending = provider.chatCompletion('test-key', MESSAGES, 'test-model', { signal: clientAbort.signal });
+ setTimeout(() => clientAbort.abort(newClientAbortError()), 100);
+
+ const err = await pending.then(() => null, e => e);
+ expect(err).not.toBeNull();
+ // The rejection must carry the marked client-abort reason (not a generic
+ // parse/abort error), so the fallback loop can skip all failure bookkeeping.
+ expect(isClientAbortError(err)).toBe(true);
+ // And the upstream connection must actually be torn down — tokens stop
+ // burning — not left to drain in the background.
+ await upstream.closed;
+ });
+
+ it('client abort mid-stream stops iteration and rejects with the marked reason', async () => {
+ // SSE headers + one real chunk, then hold: the consumer is parked inside
+ // reader.read() when the client hangs up.
+ upstream = await startUpstream((_req, res) => {
+ res.writeHead(200, { 'content-type': 'text/event-stream' });
+ res.write('data: {"id":"c1","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"hello"},"finish_reason":null}]}\n\n');
+ });
+ const provider = makeProvider(upstream.url);
+
+ const clientAbort = new AbortController();
+ const received: string[] = [];
+ const err = await (async () => {
+ for await (const chunk of provider.streamChatCompletion('test-key', MESSAGES, 'test-model', { signal: clientAbort.signal })) {
+ received.push((chunk as any).choices?.[0]?.delta?.content ?? '');
+ // Simulate the route pump's client vanishing after the first token.
+ clientAbort.abort(newClientAbortError());
+ }
+ return null;
+ })().catch(e => e);
+
+ expect(received).toEqual(['hello']);
+ expect(err).not.toBeNull();
+ expect(isClientAbortError(err)).toBe(true);
+ await upstream.closed;
+ });
+
+ it('per-attempt timeout still fires on slow headers', async () => {
+ // Upstream never writes headers — the historical fetchWithTimeout contract.
+ upstream = await startUpstream(() => { /* never respond */ });
+ const provider = makeProvider(upstream.url);
+
+ const startedAt = Date.now();
+ const err = await provider
+ .chatCompletion('test-key', MESSAGES, 'test-model', { timeoutMs: 300 })
+ .then(() => null, e => e);
+
+ expect(err).not.toBeNull();
+ expect(/aborted/i.test(err.message)).toBe(true);
+ // A timeout is a provider-health signal — it must NOT read as client-caused.
+ expect(isClientAbortError(err)).toBe(false);
+ expect(Date.now() - startedAt).toBeLessThan(3000);
+ });
+
+ it('per-attempt timeout now also fires on a hung body read', async () => {
+ // 200 arrives instantly (so the old header-scoped timer is disarmed), but
+ // the JSON body never completes. The 'request'-bounds deadline must abort
+ // res.json() instead of hanging the attempt forever.
+ upstream = await startUpstream((_req, res) => {
+ res.writeHead(200, { 'content-type': 'application/json' });
+ res.write('{"id":"chatcmpl-hang","choices":[');
+ });
+ const provider = makeProvider(upstream.url);
+
+ const startedAt = Date.now();
+ const err = await provider
+ .chatCompletion('test-key', MESSAGES, 'test-model', { timeoutMs: 300 })
+ .then(() => null, e => e);
+
+ expect(err).not.toBeNull();
+ // Must surface as an abort/timeout, not as the "non-OpenAI-compatible
+ // endpoint" body-parse diagnosis (isAbortLikeError rethrow in the adapter).
+ expect(/aborted/i.test(err.message)).toBe(true);
+ expect(isClientAbortError(err)).toBe(false);
+ expect(Date.now() - startedAt).toBeLessThan(3000);
+ await upstream.closed;
+ });
+
+ it('a completed request is unaffected by a later client abort', async () => {
+ // Guard against over-eager wiring: abort AFTER the response is fully
+ // consumed must not retro-fail anything (the timer in 'request' bounds is
+ // never cleared, so this exercises the fire-after-completion no-op too).
+ upstream = await startUpstream((_req, res) => {
+ res.writeHead(200, { 'content-type': 'application/json' });
+ res.end(JSON.stringify({
+ id: 'chatcmpl-ok', object: 'chat.completion', created: 1, model: 'test-model',
+ choices: [{ index: 0, message: { role: 'assistant', content: 'done' }, finish_reason: 'stop' }],
+ usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
+ }));
+ });
+ const provider = makeProvider(upstream.url);
+
+ const clientAbort = new AbortController();
+ const result = await provider.chatCompletion('test-key', MESSAGES, 'test-model', { signal: clientAbort.signal });
+ clientAbort.abort(newClientAbortError());
+ expect(result.choices[0]?.message?.content).toBe('done');
+ });
+});
diff --git a/server/src/__tests__/routes/proxy-auto-model.test.ts b/server/src/__tests__/routes/proxy-auto-model.test.ts
index 4f304962..097cd70f 100644
--- a/server/src/__tests__/routes/proxy-auto-model.test.ts
+++ b/server/src/__tests__/routes/proxy-auto-model.test.ts
@@ -209,7 +209,7 @@ describe('Virtual "auto" model', () => {
messages: [{ role: 'user', content: 'hello' }],
}, authHeaders());
- expect(status).toBe(400);
+ expect(status).toBe(404);
expect(body.error.code).toBe('model_not_found');
});
});
diff --git a/server/src/__tests__/routes/proxy-max-tokens-routing.test.ts b/server/src/__tests__/routes/proxy-max-tokens-routing.test.ts
index db599868..e3e2e4df 100644
--- a/server/src/__tests__/routes/proxy-max-tokens-routing.test.ts
+++ b/server/src/__tests__/routes/proxy-max-tokens-routing.test.ts
@@ -101,18 +101,22 @@ describe('max_tokens no longer starves routing (#470)', () => {
expect(chatCompletion).toHaveBeenCalledTimes(1);
});
- it('still excludes the model when the INPUT itself exceeds the TPM budget', async () => {
+ it('still excludes the model when the INPUT itself exceeds the TPM budget — now an honest 413', async () => {
chatCompletion.mockResolvedValueOnce(GOOD_RESULT);
// ~8k input tokens (32k chars / 4) > the 6k TPM budget → the only enabled
- // model is filtered out before any upstream call.
+ // model is filtered out before any upstream call. Every candidate rejected
+ // the request as too big for its window, so the exhaustion ladder renders
+ // a 413 context_length_exceeded (waiting would not help), not the old
+ // misleading 429.
const bigPrompt = 'x'.repeat(32_000);
- const { status } = await post(app, {
+ const { status, body } = await post(app, {
messages: [{ role: 'user', content: bigPrompt }],
max_tokens: 50,
}, key);
- expect(status).toBe(429);
+ expect(status).toBe(413);
+ expect(body.error.code).toBe('context_length_exceeded');
expect(chatCompletion).not.toHaveBeenCalled();
});
});
diff --git a/server/src/__tests__/services/cooldown-probe.test.ts b/server/src/__tests__/services/cooldown-probe.test.ts
new file mode 100644
index 00000000..e4f80947
--- /dev/null
+++ b/server/src/__tests__/services/cooldown-probe.test.ts
@@ -0,0 +1,347 @@
+import { describe, it, expect, beforeAll, beforeEach, afterEach, vi } from 'vitest';
+
+// Unit tests for cooldown-probe early recovery: which cooldowns are eligible
+// (heuristic only — authoritative/credit/tier are provider-stated facts and are
+// never probed), the half-bench ripeness gate, the restart stagger, the failed-
+// probe backoff progression, the per-pass probe budget, and early clearing on a
+// successful probe. Plus the provenance classification the whole feature keys
+// off (cooldownDecisionForError / setCooldown source tagging).
+
+import { initDb, getDb } from '../../db/index.js';
+import {
+ setCooldown,
+ isOnCooldown,
+ getProbeableCooldowns,
+} from '../../services/ratelimit.js';
+import {
+ runCooldownProbePass,
+ resetCooldownProbeState,
+ startCooldownProbe,
+ stopCooldownProbe,
+} from '../../services/cooldown-probe.js';
+import { cooldownDecisionForError } from '../../lib/fallback-loop.js';
+import type { RouteResult } from '../../services/router.js';
+import type { KeyProbeOutcome } from '../../services/health.js';
+
+const MINUTE = 60_000;
+
+// Distinct keyId per bench: the cooldown store is module-global (memory map +
+// DB), so shared ids would leak state across tests.
+let keySeq = 910_000;
+
+function bench(
+ source: 'heuristic' | 'authoritative' | 'credit' | 'tier',
+ opts: { durationMs?: number; modelId?: string; keyId?: number } = {},
+): number {
+ const keyId = opts.keyId ?? ++keySeq;
+ setCooldown('probefake', opts.modelId ?? 'model-a', keyId, opts.durationMs ?? 10 * MINUTE, source);
+ return keyId;
+}
+
+// A probe that always answers the same thing, wrapped in a spy so tests can
+// assert exactly which keys were probed and how often.
+const probeReturning = (outcome: KeyProbeOutcome) => vi.fn(async (_keyId: number) => outcome);
+
+// jitter() => 0 makes a first-sighted key probeable on the very next pass,
+// which keeps the "seed pass then probe pass" choreography deterministic.
+const noJitter = () => 0;
+
+beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+});
+
+beforeEach(() => {
+ resetCooldownProbeState();
+ getDb().prepare('DELETE FROM rate_limit_cooldowns').run();
+});
+
+describe('probe eligibility', () => {
+ it('only heuristic cooldowns are visible to the prober', () => {
+ const heuristic = bench('heuristic');
+ bench('authoritative');
+ bench('credit');
+ bench('tier');
+
+ const rows = getProbeableCooldowns();
+ expect(rows).toHaveLength(1);
+ expect(rows[0].keyId).toBe(heuristic);
+ });
+
+ it('never probes authoritative, credit, or tier cooldowns', async () => {
+ const heuristic = bench('heuristic');
+ const authoritative = bench('authoritative');
+ const credit = bench('credit');
+ const tier = bench('tier');
+ const probe = probeReturning('valid');
+ const now = Date.now() + 6 * MINUTE; // past the half-bench ripeness gate
+
+ await runCooldownProbePass({ now, probe, jitter: noJitter }); // seed pass
+ const result = await runCooldownProbePass({ now, probe, jitter: noJitter });
+
+ expect(result.probedKeyIds).toEqual([heuristic]);
+ expect(probe).toHaveBeenCalledTimes(1);
+ expect(probe).toHaveBeenCalledWith(heuristic);
+ // The provider-stated benches stay exactly where they were.
+ expect(isOnCooldown('probefake', 'model-a', authoritative)).toBe(true);
+ expect(isOnCooldown('probefake', 'model-a', credit)).toBe(true);
+ expect(isOnCooldown('probefake', 'model-a', tier)).toBe(true);
+ });
+
+ it('does not probe before half the bench has elapsed', async () => {
+ bench('heuristic', { durationMs: 10 * MINUTE });
+ const probe = probeReturning('valid');
+ const early = Date.now() + 2 * MINUTE; // 20% served — not ripe
+
+ await runCooldownProbePass({ now: early, probe, jitter: noJitter });
+ await runCooldownProbePass({ now: early, probe, jitter: noJitter });
+ expect(probe).not.toHaveBeenCalled();
+
+ const ripe = Date.now() + 6 * MINUTE; // 60% served — ripe
+ await runCooldownProbePass({ now: ripe, probe, jitter: noJitter }); // seed
+ const result = await runCooldownProbePass({ now: ripe, probe, jitter: noJitter });
+ expect(result.probedKeyIds).toHaveLength(1);
+ });
+
+ it('skips cooldowns that are about to expire on their own', async () => {
+ bench('heuristic', { durationMs: 10 * MINUTE });
+ const probe = probeReturning('valid');
+ // 9.5 minutes in: past half-bench, but the remaining 30s is less than a
+ // scan interval — the probe would cost more than the idle time it saves.
+ const now = Date.now() + 9 * MINUTE + 30_000;
+
+ await runCooldownProbePass({ now, probe, jitter: noJitter });
+ await runCooldownProbePass({ now, probe, jitter: noJitter });
+ expect(probe).not.toHaveBeenCalled();
+ });
+});
+
+describe('successful probe', () => {
+ it('clears every heuristic cooldown on the key early, with one probe', async () => {
+ const keyId = ++keySeq;
+ bench('heuristic', { keyId, modelId: 'model-a' });
+ bench('heuristic', { keyId, modelId: 'model-b' });
+ expect(isOnCooldown('probefake', 'model-a', keyId)).toBe(true);
+ expect(isOnCooldown('probefake', 'model-b', keyId)).toBe(true);
+
+ const probe = probeReturning('valid');
+ const now = Date.now() + 6 * MINUTE;
+ await runCooldownProbePass({ now, probe, jitter: noJitter }); // seed
+ const result = await runCooldownProbePass({ now, probe, jitter: noJitter });
+
+ // One validate call is the same evidence for both benched models.
+ expect(probe).toHaveBeenCalledTimes(1);
+ expect(result.clearedCooldowns).toBe(2);
+ expect(isOnCooldown('probefake', 'model-a', keyId)).toBe(false);
+ expect(isOnCooldown('probefake', 'model-b', keyId)).toBe(false);
+ expect(getProbeableCooldowns(now)).toHaveLength(0);
+ });
+});
+
+describe('failed probe', () => {
+ function expiryOf(keyId: number): number {
+ const row = getDb().prepare(
+ "SELECT expires_at_ms FROM rate_limit_cooldowns WHERE key_id = ?",
+ ).get(keyId) as { expires_at_ms: number };
+ return row.expires_at_ms;
+ }
+
+ it('leaves the cooldown exactly as it was — never extends it', async () => {
+ const keyId = bench('heuristic', { durationMs: 30 * MINUTE });
+ const expiryBefore = expiryOf(keyId);
+ const probe = probeReturning('invalid');
+ const now = Date.now() + 16 * MINUTE;
+
+ await runCooldownProbePass({ now, probe, jitter: noJitter }); // seed
+ const result = await runCooldownProbePass({ now, probe, jitter: noJitter });
+
+ expect(result.probedKeyIds).toEqual([keyId]);
+ expect(result.clearedCooldowns).toBe(0);
+ expect(isOnCooldown('probefake', 'model-a', keyId)).toBe(true);
+ expect(expiryOf(keyId)).toBe(expiryBefore);
+ });
+
+ it('backs off with increasing intervals between probes', async () => {
+ const keyId = bench('heuristic', { durationMs: 60 * MINUTE });
+ const probe = probeReturning('invalid');
+ const t0 = Date.now() + 31 * MINUTE;
+
+ await runCooldownProbePass({ now: t0, probe, jitter: noJitter }); // seed
+ await runCooldownProbePass({ now: t0, probe, jitter: noJitter }); // probe #1 fails
+ expect(probe).toHaveBeenCalledTimes(1);
+
+ // Within the first 2-minute backoff: no probe.
+ await runCooldownProbePass({ now: t0 + 1 * MINUTE, probe, jitter: noJitter });
+ expect(probe).toHaveBeenCalledTimes(1);
+
+ // Backoff served: probe #2 fails, backoff doubles to 4 minutes.
+ await runCooldownProbePass({ now: t0 + 2 * MINUTE, probe, jitter: noJitter });
+ expect(probe).toHaveBeenCalledTimes(2);
+
+ // 3 minutes after probe #2 — still inside the 4-minute backoff.
+ await runCooldownProbePass({ now: t0 + 5 * MINUTE, probe, jitter: noJitter });
+ expect(probe).toHaveBeenCalledTimes(2);
+
+ // 4 minutes after probe #2: due again.
+ await runCooldownProbePass({ now: t0 + 6 * MINUTE, probe, jitter: noJitter });
+ expect(probe).toHaveBeenCalledTimes(3);
+ expect(isOnCooldown('probefake', 'model-a', keyId)).toBe(true);
+ });
+
+ it('treats an inconclusive transport error like a failure', async () => {
+ const keyId = bench('heuristic');
+ const probe = probeReturning('error');
+ const now = Date.now() + 6 * MINUTE;
+
+ await runCooldownProbePass({ now, probe, jitter: noJitter }); // seed
+ const result = await runCooldownProbePass({ now, probe, jitter: noJitter });
+
+ expect(result.clearedCooldowns).toBe(0);
+ expect(isOnCooldown('probefake', 'model-a', keyId)).toBe(true);
+ });
+});
+
+describe('stagger and probe budget', () => {
+ it('a first-sighted key gets a jittered schedule instead of an instant probe', async () => {
+ bench('heuristic', { durationMs: 60 * MINUTE });
+ const probe = probeReturning('valid');
+ const maxJitter = () => 0.999; // ~45s stagger
+ const t0 = Date.now() + 31 * MINUTE;
+
+ await runCooldownProbePass({ now: t0, probe, jitter: maxJitter }); // seed only
+ expect(probe).not.toHaveBeenCalled();
+
+ // Still inside the stagger window.
+ await runCooldownProbePass({ now: t0 + 30_000, probe, jitter: maxJitter });
+ expect(probe).not.toHaveBeenCalled();
+
+ // Stagger served.
+ await runCooldownProbePass({ now: t0 + 46_000, probe, jitter: maxJitter });
+ expect(probe).toHaveBeenCalledTimes(1);
+ });
+
+ it('caps how many keys one pass may probe', async () => {
+ for (let i = 0; i < 5; i++) bench('heuristic');
+ const probe = probeReturning('valid');
+ const now = Date.now() + 6 * MINUTE;
+
+ await runCooldownProbePass({ now, probe, jitter: noJitter, maxProbes: 2 }); // seed
+ const first = await runCooldownProbePass({ now, probe, jitter: noJitter, maxProbes: 2 });
+ expect(first.probedKeyIds).toHaveLength(2);
+
+ // The rest are picked up by later passes, not all at once.
+ const second = await runCooldownProbePass({ now, probe, jitter: noJitter, maxProbes: 2 });
+ expect(second.probedKeyIds).toHaveLength(2);
+ expect(probe).toHaveBeenCalledTimes(4);
+ });
+});
+
+describe('cooldown provenance (cooldownDecisionForError)', () => {
+ function fakeRoute(overrides: Partial<RouteResult> = {}): RouteResult {
+ const n = ++keySeq;
+ return {
+ provider: {} as any, modelId: 'fake-model', modelDbId: 910_000 + n, apiKey: 'k',
+ keyId: n, platform: 'probefake', displayName: 'Fake Model',
+ rpdLimit: null, tpdLimit: null,
+ ...overrides,
+ };
+ }
+
+ it('tags 402 out-of-credits as credit (never probed)', () => {
+ const decision = cooldownDecisionForError(fakeRoute(), new Error('402 Payment Required'));
+ expect(decision.source).toBe('credit');
+ });
+
+ it('tags 403 model-not-on-tier as tier (never probed)', () => {
+ const err = Object.assign(new Error('forbidden'), { status: 403 });
+ expect(cooldownDecisionForError(fakeRoute(), err).source).toBe('tier');
+ });
+
+ it('tags a daily-quota bench as authoritative — the expiry is a reset time, not a guess', () => {
+ const err = new Error('You have used up your daily free allocation');
+ const decision = cooldownDecisionForError(fakeRoute(), err);
+ expect(decision.source).toBe('authoritative');
+ expect(decision.durationMs).toBeGreaterThan(0);
+ });
+
+ it('tags a plain transient 429 as heuristic (probe-eligible)', () => {
+ const decision = cooldownDecisionForError(fakeRoute(), new Error('429 Too Many Requests'));
+ expect(decision.source).toBe('heuristic');
+ expect(decision.durationMs).toBe(90_000);
+ });
+
+ it('a Retry-After that determined the expiry is authoritative; a shorter one is not', () => {
+ // Retry-After beyond our 90s transient bench: the provider's word set the
+ // expiry, so probing early could never be justified.
+ const longRetry = Object.assign(new Error('429 Too Many Requests'), { retryAfterMs: 10 * MINUTE });
+ const long = cooldownDecisionForError(fakeRoute(), longRetry);
+ expect(long).toEqual({ durationMs: 10 * MINUTE, source: 'authoritative' });
+
+ // Retry-After SHORTER than our bench: everything past the provider's own
+ // retry time is our pessimism, so early recovery stays on the table.
+ const shortRetry = Object.assign(new Error('429 Too Many Requests'), { retryAfterMs: 1_000 });
+ const short = cooldownDecisionForError(fakeRoute(), shortRetry);
+ expect(short).toEqual({ durationMs: 90_000, source: 'heuristic' });
+ });
+});
+
+describe('startCooldownProbe / stopCooldownProbe', () => {
+ function makeScheduler() {
+ const every: { ms: number; fn: () => void | Promise<void> }[] = [];
+ const cancels: ReturnType<typeof vi.fn>[] = [];
+ return {
+ scheduler: {
+ every(ms: number, fn: () => void | Promise<void>) {
+ const cancel = vi.fn();
+ every.push({ ms, fn });
+ cancels.push(cancel);
+ return cancel;
+ },
+ after(_ms: number, _fn: () => void | Promise<void>) {
+ return vi.fn();
+ },
+ },
+ every,
+ cancels,
+ };
+ }
+
+ afterEach(() => {
+ stopCooldownProbe();
+ delete process.env.COOLDOWN_PROBE_DISABLED;
+ });
+
+ it('registers one every-minute scan job', () => {
+ const { scheduler, every } = makeScheduler();
+ startCooldownProbe(scheduler);
+ expect(every).toHaveLength(1);
+ expect(every[0].ms).toBe(60_000);
+ });
+
+ it('is idempotent — double-start registers only one job', () => {
+ const { scheduler, every } = makeScheduler();
+ startCooldownProbe(scheduler);
+ startCooldownProbe(scheduler);
+ expect(every).toHaveLength(1);
+ });
+
+ it('the kill switch registers nothing', () => {
+ process.env.COOLDOWN_PROBE_DISABLED = '1';
+ const { scheduler, every } = makeScheduler();
+ startCooldownProbe(scheduler);
+ expect(every).toHaveLength(0);
+ });
+
+ it('stop invokes the cancel handle', () => {
+ const { scheduler, cancels } = makeScheduler();
+ startCooldownProbe(scheduler);
+ stopCooldownProbe();
+ expect(cancels[0]).toHaveBeenCalledOnce();
+ });
+
+ it('the registered job runs a pass without throwing', async () => {
+ const { scheduler, every } = makeScheduler();
+ startCooldownProbe(scheduler);
+ await expect(every[0].fn()).resolves.toBeUndefined();
+ });
+});
diff --git a/server/src/db/migrate/defaults.ts b/server/src/db/migrate/defaults.ts
index daa9c526..99ac28a4 100644
--- a/server/src/db/migrate/defaults.ts
+++ b/server/src/db/migrate/defaults.ts
@@ -8,6 +8,7 @@ import * as requestClientInfo from '../migrations/20260706_000001_request_client
import * as customModelToolSupport from '../migrations/20260706_000002_custom_model_tool_support.js';
import * as profileChainBackfill from '../migrations/20260714_000001_profile_chain_backfill.js';
import * as keyHealthError from '../migrations/20260720_000001_key_health_error.js';
+import * as cooldownProbeProvenance from '../migrations/20260726_000001_cooldown_probe_provenance.js';
export interface MigrationModule {
up(db: Db): void;
@@ -28,6 +29,7 @@ export const REQUEST_CLIENT_INFO_FILENAME = '20260706_000001_request_client_info
export const CUSTOM_MODEL_TOOL_SUPPORT_FILENAME = '20260706_000002_custom_model_tool_support.ts';
export const PROFILE_CHAIN_BACKFILL_FILENAME = '20260714_000001_profile_chain_backfill.ts';
export const KEY_HEALTH_ERROR_FILENAME = '20260720_000001_key_health_error.ts';
+export const COOLDOWN_PROBE_PROVENANCE_FILENAME = '20260726_000001_cooldown_probe_provenance.ts';
export const DEFAULT_MIGRATIONS: readonly DefaultMigration[] = [
{ filename: LEGACY_BASELINE_FILENAME, module: legacyBaseline },
@@ -39,4 +41,5 @@ export const DEFAULT_MIGRATIONS: readonly DefaultMigration[] = [
{ filename: CUSTOM_MODEL_TOOL_SUPPORT_FILENAME, module: customModelToolSupport },
{ filename: PROFILE_CHAIN_BACKFILL_FILENAME, module: profileChainBackfill },
{ filename: KEY_HEALTH_ERROR_FILENAME, module: keyHealthError },
+ { filename: COOLDOWN_PROBE_PROVENANCE_FILENAME, module: cooldownProbeProvenance },
];
diff --git a/server/src/db/migrations/20260726_000001_cooldown_probe_provenance.ts b/server/src/db/migrations/20260726_000001_cooldown_probe_provenance.ts
new file mode 100644
index 00000000..049be43f
--- /dev/null
+++ b/server/src/db/migrations/20260726_000001_cooldown_probe_provenance.ts
@@ -0,0 +1,42 @@
+// Migration: cooldown provenance for probe-based early recovery
+// Created: 2026-07-26
+//
+// DOWN: reversible
+//
+// The cooldown-probe recovery job (services/cooldown-probe.ts) may only probe
+// cooldowns that are OUR OWN guesses. A cooldown backed by an authoritative
+// retry time (explicit Retry-After, daily-quota reset) is a fact the provider
+// told us; probing those wastes validate calls and can never legitimately clear
+// them early. `source` records that provenance at setCooldown time; `set_at_ms`
+// records when the bench started, so the prober can wait out a fraction of the
+// bench before the first probe instead of probing a seconds-old cooldown.
+
+import type { Db } from '../types.js';
+
+function hasColumn(db: Db, table: string, column: string): boolean {
+ const columns = db.prepare(`PRAGMA table_info(${table})`).all() as { name: string }[];
+ return columns.some((candidate) => candidate.name === column);
+}
+
+export function up(db: Db): void {
+ if (!hasColumn(db, 'rate_limit_cooldowns', 'source')) {
+ // Pre-existing rows default to 'heuristic' (probe-eligible): before this
+ // migration every cooldown kind was mixed together, and a spurious probe on
+ // an old authoritative row costs one cheap validate call at worst.
+ db.prepare("ALTER TABLE rate_limit_cooldowns ADD COLUMN source TEXT NOT NULL DEFAULT 'heuristic'").run();
+ }
+ if (!hasColumn(db, 'rate_limit_cooldowns', 'set_at_ms')) {
+ // Nullable: rows written before this migration have an unknown start time,
+ // which the prober treats as "old enough to probe".
+ db.prepare('ALTER TABLE rate_limit_cooldowns ADD COLUMN set_at_ms INTEGER').run();
+ }
+}
+
+export function down(db: Db): void {
+ if (hasColumn(db, 'rate_limit_cooldowns', 'source')) {
+ db.prepare('ALTER TABLE rate_limit_cooldowns DROP COLUMN source').run();
+ }
+ if (hasColumn(db, 'rate_limit_cooldowns', 'set_at_ms')) {
+ db.prepare('ALTER TABLE rate_limit_cooldowns DROP COLUMN set_at_ms').run();
+ }
+}
diff --git a/server/src/index.ts b/server/src/index.ts
index b202c14a..4bfc6fec 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -5,6 +5,7 @@ import { startHealthChecker, checkAllKeys } from './services/health.js';
import { applyProxyUrl, applyProxyEnabled, applyProxyBypass, flushProxyCache } from './lib/proxy.js';
import { startWakeDetect } from './lib/wake-detect.js';
import { startCatalogSync } from './services/catalog-sync.js';
+import { startCooldownProbe } from './services/cooldown-probe.js';
import { installProcessSafetyNet } from './lib/process-safety-net.js';
import { NodeScheduler } from './lib/scheduler.js';
import { loadConfig } from './lib/config.js';
@@ -60,6 +61,7 @@ async function main() {
console.log(`Proxy endpoint: http://${display}:${PORT}/v1/chat/completions`);
startHealthChecker(scheduler);
startCatalogSync(scheduler);
+ startCooldownProbe(scheduler);
startDbBackupPump(getDb(), scheduler, config.dbPath ?? undefined);
// Post-sleep recovery: while the host was suspended (laptop lid, VM
diff --git a/server/src/lib/error-classify.ts b/server/src/lib/error-classify.ts
index c37e5010..6208ebfb 100644
--- a/server/src/lib/error-classify.ts
+++ b/server/src/lib/error-classify.ts
@@ -38,6 +38,12 @@ export function isRetryableError(err: any): boolean {
// provider in the fallback chain may have a larger limit. Same reasoning as 503.
|| msg.includes('413') || msg.includes('payload too large') || msg.includes('request body too large')
|| msg.includes('request entity too large') || msg.includes('content too large')
+ // Context/prompt-too-large in all its provider wordings (Anthropic "prompt
+ // is too long", Google "input token count ... exceeds", OpenAI "maximum
+ // context length", …): another candidate may have a larger window, so fail
+ // over; if EVERY candidate rejects it the exhaustion ladder renders an
+ // honest 413 instead of a generic failure.
+ || isContextTooLargeError(err)
// 404: model deprecated/removed upstream (e.g. OpenRouter's "no endpoints found"
// for a model that's been pulled). Rotate to the next model in the chain —
// setCooldown + the health checker will avoid this model on subsequent requests.
@@ -90,6 +96,43 @@ export function isRetryableError(err: any): boolean {
|| msg.includes('unparseable inline tool-call dialect');
}
+// ── Client-caused aborts ─────────────────────────────────────────────────────
+// When the gateway's OWN client hangs up, the proxy surfaces abort the composed
+// fetch signal with this marked error, and undici rejects the in-flight fetch /
+// body read / stream read with the same object (fetch propagates the signal's
+// abort reason). It is deliberately NOT an AbortError and its message contains
+// neither "aborted" nor "timeout": enrichAbort (lib/proxy.ts) must pass it
+// through untouched, and it must never classify as a retryable provider
+// timeout — a vanished client says nothing about provider health, so the
+// fallback loop stops without a cooldown, health penalty, or failure stats.
+export function newClientAbortError(): Error {
+ const err = new Error('client disconnected — upstream request canceled');
+ (err as Error & { clientAbort?: boolean }).clientAbort = true;
+ return err;
+}
+
+/** True when an error is (or wraps) the client-disconnect abort above. The
+ * structured marker is the primary signal; `cause` is checked because some
+ * transports re-wrap the abort reason, and the message substring is the last
+ * resort for errors that were stringified across a boundary. */
+export function isClientAbortError(err: any): boolean {
+ if (err?.clientAbort === true) return true;
+ if (err?.cause && (err.cause as { clientAbort?: boolean }).clientAbort === true) return true;
+ const msg = (err?.message ?? '').toLowerCase();
+ return msg.includes('client disconnected');
+}
+
+/** True for any fetch-abort rejection surfacing out of a body read — the
+ * per-attempt timeout ('request'-bounds deadline in fetchWithTimeout), an
+ * AbortSignal.timeout, or the client disconnect above. Adapters that wrap
+ * res.json() in a diagnostic catch (openai-compat's non-JSON-body split) must
+ * rethrow these instead of classifying them as a malformed provider body. */
+export function isAbortLikeError(err: any): boolean {
+ if (isClientAbortError(err)) return true;
+ const name = (err as { name?: string })?.name;
+ return name === 'AbortError' || name === 'TimeoutError' || /\baborted\b/i.test(err?.message ?? '');
+}
+
// A 401 / invalid-API-key error from a provider. KEY-fatal, not request-fatal:
// the same request is fine on the provider's sibling key or on another provider,
// so the fallback loop rotates past the bad key (and triggers an immediate
@@ -163,6 +206,47 @@ export function isProviderBadRequestError(err: any): boolean {
return msg.includes('api error 400') || msg.includes('api error 422') || msg.includes('unprocessable entity');
}
+// A "this request/prompt is too large" rejection. Providers word it very
+// differently and even disagree on the HTTP status:
+// - OpenAI-compat: 400 with code `context_length_exceeded` — "This model's
+// maximum context length is 8192 tokens. However, your messages resulted
+// in 10001 tokens. Please reduce the length of the messages."
+// - Anthropic: 400 invalid_request_error — "prompt is too long: 210000
+// tokens > 200000 maximum" (Bedrock words it "Input is too long for
+// requested model.")
+// - Google/Gemini: 400 INVALID_ARGUMENT — "The input token count (1200000)
+// exceeds the maximum number of tokens allowed (1048576)."
+// - Groq: a literal HTTP 413 — "Request too large for model `x` ... on
+// tokens per minute (TPM): Limit 30000, Requested 33476". Requested >
+// Limit can never fit that candidate's window, so it belongs here (too
+// large for the candidate), not with transient per-minute 429s.
+// - Reverse proxies / gateways: 413 "Payload Too Large" / "request entity
+// too large" / "content too large".
+// Structured status first (providerHttpError attaches err.status on every
+// adapter), then the code field, then message markers — mirroring how the
+// other classifiers in this file work. Retryable (a sibling candidate may have
+// a larger window), but when EVERY attempt dies this way the exhaustion ladder
+// renders an honest 413 instead of a generic failure.
+export function isContextTooLargeError(err: any): boolean {
+ if (err?.status === 413) return true;
+ if (typeof err?.code === 'string' && err.code.toLowerCase() === 'context_length_exceeded') return true;
+ const msg = (err?.message ?? '').toLowerCase();
+ return msg.includes('context_length_exceeded')
+ || msg.includes('maximum context length')
+ || msg.includes('context length exceeded')
+ || /exceeds[^.]*context (length|window|size)/.test(msg)
+ || msg.includes('prompt is too long')
+ || msg.includes('input is too long')
+ || /input token count[^.]*exceeds/.test(msg)
+ || msg.includes('exceeds the maximum number of tokens')
+ || msg.includes('request too large')
+ || msg.includes('payload too large')
+ || msg.includes('request entity too large')
+ || msg.includes('request body too large')
+ || msg.includes('content too large')
+ || msg.includes('api error 413');
+}
+
// A 402 Payment Required / out-of-credits error. Distinct from a transient 429:
// it won't recover on the next window, so the caller benches the model+key with
// PAYMENT_REQUIRED_COOLDOWN_MS (a full day) rather than the 90s transient cooldown.
diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.ts
index 39d3cc33..f9a91033 100644
--- a/server/src/lib/fallback-loop.ts
+++ b/server/src/lib/fallback-loop.ts
@@ -21,21 +21,24 @@ import {
recordRequest,
recordTokens,
setCooldown,
- getCooldownDurationForLimit,
+ getCooldownDecisionForLimit,
getSoonestCooldownExpiry,
PAYMENT_REQUIRED_COOLDOWN_MS,
MODEL_FORBIDDEN_COOLDOWN_MS,
learnLimitFromError,
+ type CooldownDecision,
} from '../services/ratelimit.js';
import {
isRetryableError,
isKeyAuthError,
+ isClientAbortError,
isDailyQuotaExhaustedError,
isPaymentRequiredError,
isModelNotFoundError,
isModelAccessForbiddenError,
isProviderBadRequestError,
isProviderDegradedError,
+ isContextTooLargeError,
} from './error-classify.js';
import { sanitizeProviderErrorMessage } from './error-redaction.js';
import { checkKeyHealth, markKeyHealthyFromRequest } from '../services/health.js';
@@ -112,10 +115,26 @@ export function msUntilNextUtcMidnight(now = Date.now()): number {
* provider's Retry-After as a floor (getCooldownDurationForLimit).
*/
export function cooldownForError(route: RouteResult, err: any): number {
- if (isPaymentRequiredError(err)) return PAYMENT_REQUIRED_COOLDOWN_MS;
- if (isModelAccessForbiddenError(err)) return MODEL_FORBIDDEN_COOLDOWN_MS;
- if (isDailyQuotaExhaustedError(err)) return err?.retryAfterMs ?? msUntilNextUtcMidnight();
- return getCooldownDurationForLimit(
+ return cooldownDecisionForError(route, err).durationMs;
+}
+
+/**
+ * cooldownForError plus the provenance tag the cooldown-probe recovery job
+ * keys off (see CooldownSource in services/ratelimit.ts): 402 → 'credit' and
+ * 403 → 'tier' (a key-validation probe passing proves nothing about credits or
+ * tier, so those are never probed); a daily-quota bench → 'authoritative' (the
+ * expiry is the provider's own reset time, whether from Retry-After or the
+ * UTC-midnight convention — a fact, not a guess); everything else defers to
+ * getCooldownDecisionForLimit, which tags 'authoritative' only when an explicit
+ * Retry-After actually determined the expiry.
+ */
+export function cooldownDecisionForError(route: RouteResult, err: any): CooldownDecision {
+ if (isPaymentRequiredError(err)) return { durationMs: PAYMENT_REQUIRED_COOLDOWN_MS, source: 'credit' };
+ if (isModelAccessForbiddenError(err)) return { durationMs: MODEL_FORBIDDEN_COOLDOWN_MS, source: 'tier' };
+ if (isDailyQuotaExhaustedError(err)) {
+ return { durationMs: err?.retryAfterMs ?? msUntilNextUtcMidnight(), source: 'authoritative' };
+ }
+ return getCooldownDecisionForLimit(
route.platform,
route.modelId,
route.keyId,
@@ -152,12 +171,16 @@ export function recordRetryableFailure(route: RouteResult, err: any, state: Fall
// state (ignored response_format, JSON truncated at max_tokens): a sibling
// key would reproduce it exactly, so rule out the whole model for this
// request instead of burning one failover hop per key.
- if (isModelNotFoundError(err) || isModelAccessForbiddenError(err) || err?.skipModelForRequest === true) {
+ // Context-too-large is MODEL-level too: a sibling key serves the same model
+ // with the same context window (and, for Groq-style per-key TPM 413s, the
+ // same tier ceiling), so it would reject the same request identically.
+ if (isModelNotFoundError(err) || isModelAccessForbiddenError(err) || isContextTooLargeError(err) || err?.skipModelForRequest === true) {
state.skipModels.add(route.modelDbId);
}
state.skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);
if (err?.skipBench === true) return;
- setCooldown(route.platform, route.modelId, route.keyId, cooldownForError(route, err));
+ const decision = cooldownDecisionForError(route, err);
+ setCooldown(route.platform, route.modelId, route.keyId, decision.durationMs, decision.source);
// Model-level penalty only when no sibling key can still serve (#454).
if (!hasOtherUsableKey(route.modelDbId, route.keyId, state.skipKeys)) {
recordRateLimitHit(route.modelDbId);
@@ -231,6 +254,7 @@ export type AttemptErrorClass =
| 'daily_quota_exhausted'
| 'model_not_found'
| 'forbidden'
+ | 'context_too_large'
| 'provider_bad_request'
| 'empty_completion'
| 'format_ignored'
@@ -252,6 +276,10 @@ export function classifyAttemptError(err: any): AttemptErrorClass {
if (isDailyQuotaExhaustedError(err)) return 'daily_quota_exhausted';
if (isModelNotFoundError(err)) return 'model_not_found';
if (isModelAccessForbiddenError(err)) return 'forbidden';
+ // Before the bad-request check: OpenAI-compat context errors arrive as
+ // "API error 400: This model's maximum context length is …", which the
+ // generic provider_bad_request rule would otherwise swallow.
+ if (isContextTooLargeError(err)) return 'context_too_large';
// A DEGRADED-function 400 (NVIDIA NIM, #522) is provider health, not request
// shape — keep it out of provider_bad_request so the trail reads honestly.
if (isProviderDegradedError(err)) return 'upstream_error';
@@ -305,9 +333,47 @@ export interface ExhaustionBody {
type: string;
message: string;
// Coarse class of the exhaustion, for surfaces that need to remap `type` to
- // their own wire vocabulary (the Anthropic route maps 'auth' → 'api_error'
- // and 'unavailable' → 'overloaded_error').
- kind: 'auth' | 'bad_request' | 'rate_limit' | 'unavailable';
+ // their own wire vocabulary (the Anthropic route maps 'auth' → 'api_error',
+ // 'unavailable' → 'overloaded_error', 'context_too_large' →
+ // 'request_too_large', 'model_not_found' → 'not_found_error', 'upstream' →
+ // 'api_error').
+ kind: 'auth' | 'bad_request' | 'rate_limit' | 'unavailable' | 'context_too_large' | 'model_not_found' | 'upstream';
+ // Machine-readable code for the OpenAI-compatible error object.
+ code?: string;
+ // 429 exhaustions only: epoch ms of the earliest moment any benched candidate
+ // becomes available again (soonest cooldown expiry). Rendered in the error
+ // body and as a Retry-After header (seconds, ceil) by setExhaustionHeaders.
+ retryAtMs?: number;
+}
+
+/**
+ * The OpenAI-compatible `error` object for an exhaustion body — shared by every
+ * OpenAI-shaped surface so the wire shape (message/type/code/retryAtMs) cannot
+ * drift between them.
+ */
+export function exhaustionErrorPayload(body: ExhaustionBody): { message: string; type: string; code?: string; retryAtMs?: number } {
+ const payload: { message: string; type: string; code?: string; retryAtMs?: number } = {
+ message: body.message,
+ type: body.type,
+ };
+ if (body.code) payload.code = body.code;
+ if (body.retryAtMs != null) payload.retryAtMs = body.retryAtMs;
+ return payload;
+}
+
+/**
+ * Stamp the standard retry headers for an exhaustion body: a Retry-After of
+ * ceil((retryAtMs - now) / 1000) seconds when the body carries a concrete
+ * retry time. Callers must only invoke this before headers are flushed (i.e.
+ * never on a committed SSE stream).
+ */
+export function setExhaustionHeaders(
+ res: { setHeader(name: string, value: string): void },
+ body: ExhaustionBody,
+ now = Date.now(),
+): void {
+ if (body.retryAtMs == null) return;
+ res.setHeader('Retry-After', String(Math.max(0, Math.ceil((body.retryAtMs - now) / 1000))));
}
export interface ExhaustionContext {
@@ -320,18 +386,46 @@ export interface ExhaustionContext {
breakerFails?: number;
}
+// Attempt classes that mean "this candidate is unavailable until a KNOWN time"
+// — a rate-limit window, a benched daily allocation, an out-of-credits key
+// (day bench), or a tier-forbidden model (day bench). When EVERY attempt is in
+// this family the pool recovers by itself, so the honest exhaustion is a 429
+// with a concrete retry time. Anything else in the mix (a 5xx, a timeout, a
+// vanished model) means waiting is not a promise, so the mixed case renders a
+// 502 instead.
+const UNAVAILABLE_UNTIL_KNOWN_TIME: ReadonlySet<AttemptErrorClass> = new Set([
+ 'rate_limited',
+ 'daily_quota_exhausted',
+ 'out_of_credits',
+ 'forbidden',
+]);
+
/**
- * The shared exhaustion response body.
+ * The shared exhaustion response body — the single failure-kind → terminal-
+ * status ladder every surface renders. Aggregated over the per-attempt failure
+ * classes (most-specific first):
* - Every attempt failed auth (401/invalid key) → 502 provider_error saying the
* PROVIDER keys are bad — distinct from a rate-limit exhaustion, and never
* 'authentication_error' (which would wrongly blame the CLIENT's key).
+ * - Every attempt died on a context/prompt-too-large rejection → 413: no
+ * candidate can fit this request; retrying cannot help, shrinking it can.
+ * - Every attempt got a model-not-found/gone from its provider → 404: the
+ * model has been removed upstream everywhere we route it. (No 410 — the
+ * catalog keeps no removal tombstones, so "verifiably existed before" is
+ * not determinable here.)
+ * - A chain that died on a DEGRADED-function 400 (NVIDIA NIM, #522) → 503:
+ * provider capacity, not a bad request.
* - A request every routed provider rejected as invalid → 400
* invalid_request_error, not a misleading rate-limit exhaustion.
- * - Otherwise → 429 rate_limit_error.
- * All bodies carry the attempt trail (what was tried, per attempt) and the
- * soonest-cooldown-reset hint that previously only reached clients when routing
- * failed before any attempt (summarizeExhaustion) — after one attempt they got a
- * terse "Last error" line with none of that context.
+ * - Circuit-breaker stop → 503 (the pool looks unhealthy; retry later).
+ * - Every attempt unavailable-until-known-time (rate limits, benched
+ * quotas/credits/tiers) → 429 rate_limit_error with `retryAtMs` (soonest
+ * cooldown expiry) for a matching Retry-After header.
+ * - Mixed/other upstream failures (5xx, timeouts, transport errors) → 502
+ * provider_error: the UPSTREAMS failed. Never 500 — that status is
+ * reserved for our own bugs.
+ * All bodies carry the attempt trail (what was tried, per attempt) and, where
+ * meaningful, the soonest-cooldown-reset hint.
*/
export function exhaustedRetryError(lastError: any, maxRetries?: number, ctx?: ExhaustionContext): ExhaustionBody {
const safeLastError = sanitizeProviderErrorMessage(lastError?.message);
@@ -340,18 +434,48 @@ export function exhaustedRetryError(lastError: any, maxRetries?: number, ctx?: E
const budgetNote = ctx?.timedOut
? ` (stopped early: retry time budget ${Math.round((ctx.budgetMs ?? 0) / 1000)}s exceeded)`
: '';
+ const everyAttempt = (cls: AttemptErrorClass | ReadonlySet<AttemptErrorClass>): boolean =>
+ attempts.length > 0 && attempts.every(a => (cls instanceof Set ? cls.has(a.errorClass) : a.errorClass === cls));
- if (attempts.length > 0 && attempts.every(a => a.errorClass === 'auth')) {
+ if (everyAttempt('auth')) {
return {
kind: 'auth',
status: 502,
type: 'provider_error',
+ code: 'provider_authentication_failed',
message: `All ${attempts.length} attempted provider key(s) failed authentication${budgetNote}. ` +
'The configured upstream API key(s) look invalid or expired; they are being revalidated now and will be marked invalid automatically. ' +
`Check the provider keys in the dashboard.${trail} Last error: ${safeLastError}`,
};
}
+ // Aggregate diagnoses before the lastError-shape ones below: when EVERY
+ // attempt failed the same way, that unanimity is stronger evidence than the
+ // shape of whichever error happened to come last.
+ if (everyAttempt('context_too_large')) {
+ return {
+ kind: 'context_too_large',
+ status: 413,
+ type: 'invalid_request_error',
+ code: 'context_length_exceeded',
+ message: `The request is too large for every routed candidate: all ${attempts.length} attempt(s) were rejected as ` +
+ `over the model's context/size limit${budgetNote}. Retrying will not help — reduce the prompt/history size or ` +
+ `enable a larger-context model.${trail} Last error: ${safeLastError}`,
+ };
+ }
+
+ if (everyAttempt('model_not_found')) {
+ return {
+ kind: 'model_not_found',
+ status: 404,
+ type: 'invalid_request_error',
+ code: 'model_not_found',
+ message: `Every routed provider reports the model as not found or removed upstream (${attempts.length} attempt(s))` +
+ `${budgetNote}. The catalog entry looks stale; pick another model or call /v1/models for the available list.` +
+ `${trail} Last error: ${safeLastError}`,
+ };
+ }
+
// A chain that died on a DEGRADED-function 400 (NVIDIA NIM, #522) is a
// provider-capacity outage, not a bad request: render 503 so clients retry
// later instead of "fixing" a request that was never broken.
@@ -360,6 +484,7 @@ export function exhaustedRetryError(lastError: any, maxRetries?: number, ctx?: E
kind: 'unavailable',
status: 503,
type: 'service_unavailable',
+ code: 'provider_degraded',
message: `The routed provider reported the model's hosted deployment as temporarily degraded${budgetNote}. ` +
`This is a provider-side condition; retry later or route to another provider.${trail} Last error: ${safeLastError}`,
};
@@ -370,11 +495,12 @@ export function exhaustedRetryError(lastError: any, maxRetries?: number, ctx?: E
kind: 'bad_request',
status: 400,
type: 'invalid_request_error',
+ code: 'provider_rejected_request',
message: `All routed providers rejected the request as invalid${budgetNote}.${trail} Last error: ${safeLastError}`,
};
}
- // Circuit-breaker guardrail stop. Checked after the all-auth and bad-request
+ // Circuit-breaker guardrail stop. Checked after the aggregate and bad-request
// diagnoses, which are more specific about WHY the pool is failing.
if (ctx?.breakerFails) {
const breakerEta = formatResetEta(getSoonestCooldownExpiry());
@@ -383,6 +509,7 @@ export function exhaustedRetryError(lastError: any, maxRetries?: number, ctx?: E
kind: 'unavailable',
status: 503,
type: 'service_unavailable',
+ code: 'upstream_unhealthy',
message: `Failover stopped early by the circuit-breaker guardrail: ${ctx.breakerFails} consecutive upstream ` +
`failure${ctx.breakerFails === 1 ? '' : 's'} (max_consecutive_upstream_fails). The enabled pool looks ` +
`unhealthy right now, so the remaining candidates were skipped instead of burning quota on them.` +
@@ -390,17 +517,116 @@ export function exhaustedRetryError(lastError: any, maxRetries?: number, ctx?: E
};
}
- const attemptCount = attempts.length > 0 ? attempts.length : maxRetries;
- const scope = attemptCount == null
- ? 'All models rate-limited'
- : `All models rate-limited after ${attemptCount} attempt${attemptCount === 1 ? '' : 's'}`;
- const eta = formatResetEta(getSoonestCooldownExpiry());
- const etaNote = eta ? ` Soonest cooldown reset ${eta}.` : '';
+ // 429 only when EVERY candidate is unavailable until a known time (or the
+ // caller gave us no per-attempt classes to aggregate — the legacy shape).
+ if (attempts.length === 0 || everyAttempt(UNAVAILABLE_UNTIL_KNOWN_TIME)) {
+ const attemptCount = attempts.length > 0 ? attempts.length : maxRetries;
+ const scope = attemptCount == null
+ ? 'All models rate-limited'
+ : `All models rate-limited after ${attemptCount} attempt${attemptCount === 1 ? '' : 's'}`;
+ const retryAtMs = getSoonestCooldownExpiry() ?? undefined;
+ const eta = formatResetEta(retryAtMs);
+ const etaNote = eta ? ` Soonest cooldown reset ${eta}.` : '';
+ return {
+ kind: 'rate_limit',
+ status: 429,
+ type: 'rate_limit_error',
+ code: 'rate_limit_exceeded',
+ ...(retryAtMs != null ? { retryAtMs } : {}),
+ message: `${scope}${budgetNote}.${etaNote}${trail} Last error: ${safeLastError}`,
+ };
+ }
+
+ // Mixed/other upstream failures (5xx, timeouts, transport errors, or a mix of
+ // those with rate limits): the upstream pool failed us, so say 502 — not 429
+ // (which would promise recovery-by-waiting the attempts don't support) and
+ // not 500 (which would blame our own code).
+ return {
+ kind: 'upstream',
+ status: 502,
+ type: 'provider_error',
+ code: 'upstream_failed',
+ message: `All ${attempts.length} routed attempt(s) failed with upstream provider errors${budgetNote}. ` +
+ `This is a provider-side failure, not a problem with your request; retry, or check provider status.` +
+ `${trail} Last error: ${safeLastError}`,
+ };
+}
+
+// ── Routing exhaustion (zero attempts ran) ───────────────────────────────────
+// When routeRequest gives up before ANY upstream was tried, the only evidence
+// is its per-candidate diagnostics. Map them onto the same honest-terminal-
+// status taxonomy the attempt ladder uses:
+// - nothing configured at all (empty chain, or every candidate lacks a
+// provider/usable key) → 503: no amount of waiting or request-shrinking
+// helps; the operator must add keys.
+// - every candidate rejected the request as too big for its context/TPM
+// window → 413.
+// - at least one candidate is merely rate-limited/on cooldown (and the rest
+// are at worst unconfigured/too-small) → 429 with the soonest cooldown
+// expiry as retryAtMs.
+// - anything else (capability filters like vision/tools, mixed reasons) →
+// the router's status (429) with a generic routing_exhausted code.
+type RoutingDiagClass = 'config' | 'too_large' | 'time_bound' | 'other';
+
+function classifyRoutingDiagLine(line: string): RoutingDiagClass {
+ const l = line.toLowerCase();
+ // "< estimated" first: the tpm_limit-too-small line also contains 'tpm',
+ // which would otherwise misread as a transient window.
+ if (l.includes('< estimated')) return 'too_large';
+ if (/no provider registered|no enabled\+healthy key|no usable key|decrypt-error|no-resolved-provider|custom-key-mismatch/.test(l)) return 'config';
+ if (/cooldown|rpm|rpd|tpm|tpd|provider-daily-cap|provider-minute-cap|provider-daily-token-cap|key-concurrency/.test(l)) return 'time_bound';
+ return 'other';
+}
+
+export function routingExhaustionBody(routeErr: any): ExhaustionBody {
+ const diag: string[] = Array.isArray(routeErr?.diagnostics) ? routeErr.diagnostics : [];
+ const message: string = routeErr?.message ?? 'No model available to route this request';
+ const classes = diag.map(classifyRoutingDiagLine);
+
+ if (diag.length === 0 || classes.every(c => c === 'config')) {
+ return {
+ kind: 'unavailable',
+ status: 503,
+ type: 'service_unavailable',
+ code: 'no_providers_configured',
+ message: diag.length === 0
+ ? `No models are enabled/configured to serve this request. Add provider API keys and enable models in the dashboard. ${message}`
+ : `No candidate model has a configured, usable provider key. Add provider API keys in the dashboard. ${message}`,
+ };
+ }
+
+ if (classes.every(c => c === 'too_large' || c === 'config') && classes.includes('too_large')) {
+ return {
+ kind: 'context_too_large',
+ status: 413,
+ type: 'invalid_request_error',
+ code: 'context_length_exceeded',
+ message: `The request is too large for every available candidate's context/token window. ` +
+ `Reduce the prompt/history size or enable a larger-context model. ${message}`,
+ };
+ }
+
+ if (classes.some(c => c === 'time_bound') && classes.every(c => c !== 'other')) {
+ const retryAtMs = getSoonestCooldownExpiry() ?? undefined;
+ return {
+ kind: 'rate_limit',
+ status: 429,
+ type: 'rate_limit_error',
+ code: 'rate_limit_exceeded',
+ ...(retryAtMs != null ? { retryAtMs } : {}),
+ message,
+ };
+ }
+
+ // Capability filters or mixed reasons: keep the router's verdict (429) but
+ // stamp a machine-readable code so clients can tell it from a plain
+ // rate-limit exhaustion.
return {
kind: 'rate_limit',
- status: 429,
+ status: typeof routeErr?.status === 'number' ? routeErr.status : 429,
type: 'rate_limit_error',
- message: `${scope}${budgetNote}.${etaNote}${trail} Last error: ${safeLastError}`,
+ code: 'routing_exhausted',
+ message,
};
}
@@ -435,9 +661,11 @@ export interface FallbackHooks {
attemptLog?: AttemptRecord[];
// Returns true once the client has hung up. Checked before STARTING each
// retry: a chain nobody is waiting for must not keep burning provider
- // quota. The in-flight attempt still completes (same boundary as the
- // wall-clock budget); stream pumps additionally stop reading upstream on
- // disconnect, which cancels the upstream request via reader.cancel().
+ // quota. Surfaces additionally thread a client-disconnect AbortSignal into
+ // the provider options (CompletionOptions.signal), so the in-flight
+ // attempt's fetch/body/stream is canceled the moment the client goes; the
+ // resulting client-abort throw stops the loop without any failure
+ // bookkeeping (see the isClientAbortError branch below).
clientGone?: () => boolean;
// Skip state; recordRetryableFailure / recordAuthFailure (called by the loop)
// mutate it, and the surface's route() reads it to exclude failed keys/models.
@@ -471,11 +699,13 @@ export interface FallbackHooks {
onFatal(route: RouteResult, err: any, attempt: number): void;
/**
- * Render exhaustion when route() threw. `exhaustion` is the shared body when
- * at least one attempt ran (render it); null when routing gave up before any
- * upstream was tried (render routeErr as a routing error instead).
+ * Render exhaustion when route() threw. `exhaustion` is always the shared
+ * honest-status body: built from the attempt trail when at least one attempt
+ * ran, or from routeErr's routing diagnostics (routingExhaustionBody) when
+ * routing gave up before any upstream was tried. `lastError` is null in the
+ * zero-attempt case; `routeErr` is passed for logging/diagnostics.
*/
- onRoutingExhausted(lastError: any, routeErr: any, exhaustion: ExhaustionBody | null, info: ExhaustionInfo): void;
+ onRoutingExhausted(lastError: any, routeErr: any, exhaustion: ExhaustionBody, info: ExhaustionInfo): void;
/** Render exhaustion after the attempt cap or the time budget was hit. */
onExhausted(exhaustion: ExhaustionBody, info: ExhaustionInfo): void;
@@ -555,7 +785,7 @@ export async function runFallbackLoop(hooks: FallbackHooks): Promise<void> {
} catch (routeErr) {
const exhaustion = lastError
? exhaustedRetryError(lastError, undefined, { attempts })
- : null;
+ : routingExhaustionBody(routeErr);
hooks.onRoutingExhausted(lastError, routeErr, exhaustion, { attempts, timedOut: false });
return;
}
@@ -571,6 +801,16 @@ export async function runFallbackLoop(hooks: FallbackHooks): Promise<void> {
try {
outcome = await hooks.dispatch(route, attempt);
} catch (err: any) {
+ // Client-caused abort: the composed fetch signal fired because OUR
+ // client hung up mid-attempt (see newClientAbortError). Not a
+ // provider-health signal — no cooldown, no penalty, no failure stats,
+ // no logFailure 'error' row — and no further attempts either: nobody is
+ // waiting, and there is no socket to render anything to. The finally
+ // below still frees the in-flight lease.
+ if (isClientAbortError(err)) {
+ console.log(`[FallbackLoop] client disconnected mid-attempt on ${route.platform}/${route.modelId} — upstream canceled, stopping without benching`);
+ return;
+ }
hooks.logFailure(route, err, attempt);
if (isKeyAuthError(err)) {
// KEY-fatal, not request-fatal: rotate past the bad key and revalidate
diff --git a/server/src/lib/proxy.ts b/server/src/lib/proxy.ts
index d6f360e5..ce2d208c 100644
--- a/server/src/lib/proxy.ts
+++ b/server/src/lib/proxy.ts
@@ -237,6 +237,17 @@ function socksFetch(
const signal = init?.signal;
const startedAt = Date.now();
+ // What to reject with when the signal fires. A client-caused abort carries
+ // its own marked reason (newClientAbortError in lib/error-classify.ts) —
+ // preserve it so the failure isn't misclassified downstream as a provider
+ // timeout; a plain timer abort keeps the tagged AbortError.
+ const abortRejection = (): Error => {
+ const reason = signal?.reason;
+ return reason instanceof Error && reason.name !== 'AbortError' && reason.name !== 'TimeoutError'
+ ? reason
+ : abortError(platform, type, timeoutMs, Date.now() - startedAt);
+ };
+
return new Promise((resolve, reject) => {
const req = transport.request({
hostname: url.hostname,
@@ -251,7 +262,7 @@ function socksFetch(
}, (res) => {
if (signal?.aborted) {
res.destroy();
- reject(abortError(platform, type, timeoutMs, Date.now() - startedAt));
+ reject(abortRejection());
return;
}
@@ -286,12 +297,12 @@ function socksFetch(
if (signal) {
if (signal.aborted) {
req.destroy();
- reject(abortError(platform, type, timeoutMs, Date.now() - startedAt));
+ reject(abortRejection());
return;
}
signal.addEventListener('abort', () => {
req.destroy();
- reject(abortError(platform, type, timeoutMs, Date.now() - startedAt));
+ reject(abortRejection());
}, { once: true });
}
diff --git a/server/src/providers/aihorde.ts b/server/src/providers/aihorde.ts
index 2f9c6796..196ee34a 100644
--- a/server/src/providers/aihorde.ts
+++ b/server/src/providers/aihorde.ts
@@ -139,7 +139,9 @@ export class AIHordeProvider extends BaseProvider {
'Content-Type': 'application/json',
},
body: JSON.stringify(this.buildBody(messages, modelId, options)),
- }, options?.timeoutMs ?? HORDE_TIMEOUT_MS);
+ // 'request' bounds: the queued generation is one blocking body read, so
+ // the deadline must cover it too (a hung body used to stall forever).
+ }, options?.timeoutMs ?? HORDE_TIMEOUT_MS, { signal: options?.signal, timeoutBounds: 'request' });
recordQuotaObservationsFromResponse(res, {
platform: this.platform,
@@ -201,7 +203,7 @@ export class AIHordeProvider extends BaseProvider {
const res = await this.fetchWithTimeout(`${this.baseUrl}/models`, {
method: 'GET',
headers: { 'Authorization': `Bearer ${this.resolveBearer(apiKey)}` },
- }, 30000);
+ }, 30000, { timeoutBounds: 'request' });
recordQuotaObservationsFromResponse(res, {
platform: this.platform,
keyId: quotaContext?.keyId,
diff --git a/server/src/providers/base.ts b/server/src/providers/base.ts
index 99e1b4de..4fc0adbe 100644
--- a/server/src/providers/base.ts
+++ b/server/src/providers/base.ts
@@ -66,6 +66,31 @@ export interface CompletionOptions extends ExtendedSamplingOptions {
* stripped before the request body is built); used by the probe script so
* NVIDIA's 15-60s serverless cold starts don't read as failures. */
timeoutMs?: number;
+ /** Abort signal for the gateway's OWN client (request socket closed). The
+ * proxy surfaces thread it here so a disconnect cancels the upstream fetch
+ * AND any in-progress body/stream read — tokens stop burning and the
+ * in-flight lease frees immediately instead of when the read happens to
+ * finish. Composed with the per-attempt timeout in fetchWithTimeout; never
+ * serialized into the request body. */
+ signal?: AbortSignal;
+}
+
+/** Per-call abort/timeout wiring for fetchWithTimeout. */
+export interface ProviderFetchOptions {
+ /** Client-disconnect signal, composed with the per-attempt timeout via
+ * AbortSignal.any. Unlike the timeout it is never disarmed at headers, so it
+ * also aborts body reads and stream iteration through undici. */
+ signal?: AbortSignal;
+ /** What the per-attempt timeout bounds:
+ * - 'headers' (default, historical): the abort timer dies the moment
+ * response HEADERS arrive — right for streams, whose body legitimately
+ * outlives any fixed deadline (readSseStream's stall watchdog owns
+ * mid-stream hangs, the client signal owns "nobody is listening").
+ * - 'request': the deadline stays armed across the body read too, so a 200
+ * whose body never finishes aborts at timeoutMs instead of hanging
+ * res.json() forever. Use for non-streaming calls, which read the whole
+ * body as one unit. */
+ timeoutBounds?: 'headers' | 'request';
}
export interface KeyValidationFailure {
@@ -140,20 +165,48 @@ export abstract class BaseProvider {
// Adapters that don't pass a timeout inherit the platform's env override
// (PROVIDER_TIMEOUT_<PLATFORM>, issue #547) over the historical 15s.
timeoutMs = providerTimeoutMs(this.platform, 15000),
+ fetchOpts?: ProviderFetchOptions,
): Promise<Response> {
+ const signals: AbortSignal[] = [];
+ if (fetchOpts?.signal) signals.push(fetchOpts.signal);
+
// timeoutMs <= 0 means "no timeout" (PROVIDER_TIMEOUT_<PLATFORM>=0).
// setTimeout(abort, 0) would abort every request on the next macrotask.
- if (timeoutMs <= 0) {
- return proxyFetch(url, init, this.platform, 'chat', timeoutMs);
+ let headerTimer: ReturnType<typeof setTimeout> | undefined;
+ if (timeoutMs > 0) {
+ const controller = new AbortController();
+ const timer = setTimeout(() => controller.abort(), timeoutMs);
+ signals.push(controller.signal);
+ if (fetchOpts?.timeoutBounds === 'request') {
+ // Whole-request deadline: stays armed across the body read, so a 200
+ // whose body never finishes aborts res.json() at timeoutMs instead of
+ // hanging forever. Never cleared — firing after the response is fully
+ // consumed is a no-op — and unref'd so a spent request's leftover
+ // deadline can't hold the process open.
+ timer.unref?.();
+ } else {
+ // Historical header-only deadline: disarmed the moment response
+ // HEADERS arrive (see finally). Streams need this — an SSE body
+ // legitimately outlives any fixed deadline, and mid-stream hangs are
+ // owned by the stall watchdog (readSseStream) + the client signal.
+ headerTimer = timer;
+ }
}
- const controller = new AbortController();
- const timeout = setTimeout(() => controller.abort(), timeoutMs);
+
+ // Compose the per-attempt timeout with the client-disconnect signal.
+ // Unlike the timeout, the client signal is never disarmed: undici ties it
+ // to the whole request, so a disconnect also aborts body reads and stream
+ // iteration, rejecting them with the marked reason from newClientAbortError.
+ const signal = signals.length === 0
+ ? init.signal ?? undefined
+ : signals.length === 1 ? signals[0] : AbortSignal.any(signals);
+
try {
// requestType='chat' + timeoutMs makes the AbortError message read
// `<platform>, chat, 15s` for triage from the requests.error column.
- return await proxyFetch(url, { ...init, signal: controller.signal }, this.platform, 'chat', timeoutMs);
+ return await proxyFetch(url, signal ? { ...init, signal } : init, this.platform, 'chat', timeoutMs);
} finally {
- clearTimeout(timeout);
+ if (headerTimer !== undefined) clearTimeout(headerTimer);
}
}
@@ -161,6 +214,32 @@ export abstract class BaseProvider {
return `chatcmpl-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
}
+ /**
+ * One reader.read() bounded by the mid-stream inactivity watchdog (#553).
+ * Shared by readSseStream and adapters that parse their own wire format
+ * (google.ts) so every stream gets the same stall semantics:
+ * PROVIDER_STREAM_STALL_TIMEOUT_MS, default 90s, 0 disables. Client-abort
+ * rejections pass through untouched — undici errors the body with the
+ * request signal's reason, so a disconnect surfaces here as the marked
+ * error from newClientAbortError, not as a stall.
+ */
+ protected async readWithStallTimeout<T>(
+ read: () => Promise<T>,
+ inactivityTimeoutMs: number,
+ ): Promise<T> {
+ if (inactivityTimeoutMs <= 0) return read();
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ return Promise.race([
+ read(),
+ new Promise<never>((_, reject) => {
+ timer = setTimeout(
+ () => reject(new Error(`${this.name} stream stalled: no data for ${inactivityTimeoutMs}ms (timeout)`)),
+ inactivityTimeoutMs,
+ );
+ }),
+ ]).finally(() => clearTimeout(timer));
+ }
+
/**
* Shared SSE reader for OpenAI-wire streaming endpoints (#231 audit).
*
@@ -195,20 +274,7 @@ export abstract class BaseProvider {
try {
while (true) {
- let timer: ReturnType<typeof setTimeout> | undefined;
- const result = inactivityTimeoutMs <= 0
- ? await reader.read()
- : await Promise.race([
- reader.read(),
- new Promise<never>((_, reject) => {
- timer = setTimeout(
- () => reject(new Error(`${this.name} stream stalled: no data for ${inactivityTimeoutMs}ms (timeout)`)),
- inactivityTimeoutMs,
- );
- }),
- ]).finally(() => clearTimeout(timer));
-
- const { done, value } = result;
+ const { done, value } = await this.readWithStallTimeout(() => reader.read(), inactivityTimeoutMs);
if (done) break;
buffer += decoder.decode(value, { stream: true });
diff --git a/server/src/providers/cloudflare.ts b/server/src/providers/cloudflare.ts
index 65eed370..d48daf8a 100644
--- a/server/src/providers/cloudflare.ts
+++ b/server/src/providers/cloudflare.ts
@@ -76,7 +76,9 @@ export class CloudflareProvider extends BaseProvider {
parallel_tool_calls: options?.parallel_tool_calls,
...extendedBodyParams(this.platform, options),
}),
- }, this.timeoutFor(modelId, options?.timeoutMs));
+ // 'request' bounds: the deadline covers the body read too, so a 200
+ // whose body hangs aborts instead of stalling res.json() forever.
+ }, this.timeoutFor(modelId, options?.timeoutMs), { signal: options?.signal, timeoutBounds: 'request' });
recordQuotaObservationsFromResponse(res, {
platform: this.platform,
keyId: quotaContext?.keyId,
@@ -125,7 +127,9 @@ export class CloudflareProvider extends BaseProvider {
...extendedBodyParams(this.platform, options),
stream: true,
}),
- }, this.timeoutFor(modelId, options?.timeoutMs));
+ // Default 'headers' bounds: the deadline dies at response headers, and
+ // the client signal + stall watchdog own the stream from there.
+ }, this.timeoutFor(modelId, options?.timeoutMs), { signal: options?.signal });
recordQuotaObservationsFromResponse(res, {
platform: this.platform,
keyId: quotaContext?.keyId,
@@ -177,6 +181,7 @@ export class CloudflareProvider extends BaseProvider {
url,
{ method: 'GET', headers: { 'Authorization': `Bearer ${token}` } },
10000,
+ { timeoutBounds: 'request' },
);
recordQuotaObservationsFromResponse(res, {
platform: this.platform,
diff --git a/server/src/providers/cohere.ts b/server/src/providers/cohere.ts
index 074d54aa..9e1ef29c 100644
--- a/server/src/providers/cohere.ts
+++ b/server/src/providers/cohere.ts
@@ -59,7 +59,9 @@ export class CohereProvider extends BaseProvider {
'Content-Type': 'application/json',
},
body: JSON.stringify(body),
- });
+ // 'request' bounds: the deadline covers the body read too, so a 200
+ // whose body hangs aborts instead of stalling res.json() forever.
+ }, undefined, { signal: options?.signal, timeoutBounds: 'request' });
recordQuotaObservationsFromResponse(res, {
platform: this.platform,
keyId: quotaContext?.keyId,
@@ -106,7 +108,9 @@ export class CohereProvider extends BaseProvider {
'Content-Type': 'application/json',
},
body: JSON.stringify(body),
- });
+ // Default 'headers' bounds: the deadline dies at response headers, and
+ // the client signal + stall watchdog own the stream from there.
+ }, undefined, { signal: options?.signal });
recordQuotaObservationsFromResponse(res, {
platform: this.platform,
keyId: quotaContext?.keyId,
@@ -130,7 +134,7 @@ export class CohereProvider extends BaseProvider {
const res = await this.fetchWithTimeout(`${API_BASE}/models`, {
method: 'GET',
headers: { 'Authorization': `Bearer ${apiKey}` },
- }, 10000);
+ }, 10000, { timeoutBounds: 'request' });
recordQuotaObservationsFromResponse(res, {
platform: this.platform,
keyId: quotaContext?.keyId,
diff --git a/server/src/providers/google.ts b/server/src/providers/google.ts
index 09d992bc..46664332 100644
--- a/server/src/providers/google.ts
+++ b/server/src/providers/google.ts
@@ -11,7 +11,7 @@ import { BaseProvider, providerHttpError, type CompletionOptions, type KeyValida
import { contentToString } from '../lib/content.js';
import { proxyFetch } from '../lib/proxy.js';
import { recordQuotaObservationsFromResponse, type QuotaObservationContext } from '../services/provider-quota.js';
-import { providerTimeoutMs } from '../lib/provider-timeout.js';
+import { providerTimeoutMs, streamStallTimeoutMs } from '../lib/provider-timeout.js';
const API_BASE = 'https://generativelanguage.googleapis.com/v1beta';
@@ -573,7 +573,9 @@ export class GoogleProvider extends BaseProvider {
method: 'POST',
headers: { 'Content-Type': 'application/json', 'x-goog-api-key': apiKey },
body: JSON.stringify(body),
- }, options?.timeoutMs ?? this.timeoutMs);
+ // 'request' bounds: the deadline covers the body read too, so a 200
+ // whose body hangs aborts instead of stalling res.json() forever.
+ }, options?.timeoutMs ?? this.timeoutMs, { signal: options?.signal, timeoutBounds: 'request' });
recordQuotaObservationsFromResponse(res, {
platform: this.platform,
@@ -653,7 +655,9 @@ export class GoogleProvider extends BaseProvider {
method: 'POST',
headers: { 'Content-Type': 'application/json', 'x-goog-api-key': apiKey },
body: JSON.stringify(body),
- }, options?.timeoutMs ?? this.timeoutMs);
+ // Default 'headers' bounds: the deadline dies at response headers, and
+ // the client signal + stall watchdog own the stream from there.
+ }, options?.timeoutMs ?? this.timeoutMs, { signal: options?.signal });
recordQuotaObservationsFromResponse(res, {
platform: this.platform,
@@ -680,9 +684,14 @@ export class GoogleProvider extends BaseProvider {
const seenToolCallKeys = new Set<string>();
+ // Same mid-stream inactivity watchdog as readSseStream (#553): this adapter
+ // parses Gemini's own frame format, so it reads the body itself and used to
+ // have no bound at all on a stalled read.
+ const inactivityTimeoutMs = streamStallTimeoutMs();
+
try {
while (true) {
- const { done, value } = await reader.read();
+ const { done, value } = await this.readWithStallTimeout(() => reader.read(), inactivityTimeoutMs);
if (done) break;
buffer += decoder.decode(value, { stream: true });
@@ -800,6 +809,7 @@ export class GoogleProvider extends BaseProvider {
`${API_BASE}/models`,
{ method: 'GET', headers: { 'x-goog-api-key': apiKey } },
10000,
+ { timeoutBounds: 'request' },
);
recordQuotaObservationsFromResponse(res, {
platform: this.platform,
diff --git a/server/src/providers/openai-compat.ts b/server/src/providers/openai-compat.ts
index 56e5ca53..d0af2bb9 100644
--- a/server/src/providers/openai-compat.ts
+++ b/server/src/providers/openai-compat.ts
@@ -11,6 +11,7 @@ import { rescueInlineToolCalls } from '../lib/tool-call-rescue.js';
import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';
import { recordQuotaObservationsFromResponse, type QuotaObservationContext } from '../services/provider-quota.js';
import { providerTimeoutMs } from '../lib/provider-timeout.js';
+import { isAbortLikeError } from '../lib/error-classify.js';
/**
* Generic provider for platforms that use an OpenAI-compatible API.
@@ -193,7 +194,9 @@ export class OpenAICompatProvider extends BaseProvider {
parallel_tool_calls: this.resolveParallelToolCalls(options),
...extendedBodyParams(this.platform, options),
}),
- }, options?.timeoutMs ?? this.timeoutMs);
+ // 'request' bounds: the deadline covers the body read too, so a 200
+ // whose body hangs aborts instead of stalling res.json() forever.
+ }, options?.timeoutMs ?? this.timeoutMs, { signal: options?.signal, timeoutBounds: 'request' });
recordQuotaObservationsFromResponse(res, {
platform: this.platform,
@@ -228,6 +231,10 @@ export class OpenAICompatProvider extends BaseProvider {
try {
data = await res.json() as ChatCompletionResponse;
} catch (err) {
+ // An aborted body read (per-attempt deadline, client disconnect) is not
+ // a malformed body — rethrow so it keeps its abort classification
+ // instead of reading as a "non-OpenAI-compatible endpoint" below.
+ if (isAbortLikeError(err)) throw err;
parseErr = err;
data = undefined as unknown as ChatCompletionResponse;
}
@@ -307,7 +314,9 @@ export class OpenAICompatProvider extends BaseProvider {
...extendedBodyParams(this.platform, options),
stream: true,
}),
- }, options?.timeoutMs ?? this.timeoutMs);
+ // Default 'headers' bounds: the deadline dies at response headers, and
+ // the client signal + stall watchdog own the stream from there.
+ }, options?.timeoutMs ?? this.timeoutMs, { signal: options?.signal });
recordQuotaObservationsFromResponse(res, {
platform: this.platform,
@@ -351,7 +360,9 @@ export class OpenAICompatProvider extends BaseProvider {
...this.authHeader(apiKey),
...this.extraHeaders,
},
- }, 30000);
+ // 'request' bounds: a catalog body that hangs mid-transfer must not
+ // stall the health cycle past the deadline.
+ }, 30000, { timeoutBounds: 'request' });
recordQuotaObservationsFromResponse(res, {
platform: this.platform,
keyId: quotaContext?.keyId,
diff --git a/server/src/routes/anthropic.ts b/server/src/routes/anthropic.ts
index 3abd4ec4..2b4b5f9c 100644
--- a/server/src/routes/anthropic.ts
+++ b/server/src/routes/anthropic.ts
@@ -15,9 +15,10 @@ import { contentToString } from '../lib/content.js';
import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';
import { rescueInlineToolCalls, startsWithDialectMarker, couldBecomeDialectMarker, containsDialectMarker } from '../lib/tool-call-rescue.js';
import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';
+import { isClientAbortError, newClientAbortError } from '../lib/error-classify.js';
import { logRequest } from '../lib/request-log.js';
import { extractApiToken, timingSafeStringEqual, getStickyModel, setStickyModel } from './proxy.js';
-import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, type ExhaustionBody, setFallbackHeaders, type AttemptRecord } from '../lib/fallback-loop.js';
+import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, type ExhaustionBody, setFallbackHeaders, setExhaustionHeaders, type AttemptRecord } from '../lib/fallback-loop.js';
import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';
import { resolveAnthropicModel } from '../services/anthropic-map.js';
import { buildModelListing } from '../services/model-listing.js';
@@ -113,17 +114,36 @@ function sendError(res: Response, status: number, errorType: string, message: st
}
// The shared exhaustion body's `type` strings are chosen to be valid on the
-// OpenAI-shaped surfaces; the all-keys-failed-auth exhaustion carries
-// 'provider_error' and the circuit-breaker stop carries 'service_unavailable',
-// neither of which is an Anthropic error type. Remap them onto Anthropic's
-// vocabulary ('api_error' / 'overloaded_error') so this surface stays
-// wire-correct.
+// OpenAI-shaped surfaces; several of them are not Anthropic error types. Remap
+// them onto Anthropic's vocabulary via the body's coarse `kind` so this
+// surface stays wire-correct: auth/upstream failures → 'api_error', pool
+// unavailability → 'overloaded_error', context exhaustion → Anthropic's 413
+// 'request_too_large', model-gone → 'not_found_error'.
function anthropicErrorType(body: ExhaustionBody): string {
- if (body.kind === 'auth') return 'api_error';
+ if (body.kind === 'auth' || body.kind === 'upstream') return 'api_error';
if (body.kind === 'unavailable') return 'overloaded_error';
+ if (body.kind === 'context_too_large') return 'request_too_large';
+ if (body.kind === 'model_not_found') return 'not_found_error';
return body.type;
}
+// Render an exhaustion body on the Anthropic wire shape, carrying the
+// machine-readable extras (`code`, `retryAtMs`) alongside Anthropic's standard
+// type/message and stamping the matching Retry-After header. Extra keys on the
+// error object are ignored by Anthropic SDKs, so this stays wire-compatible.
+function sendExhaustion(res: Response, body: ExhaustionBody): void {
+ setExhaustionHeaders(res, body);
+ res.status(body.status).json({
+ type: 'error',
+ error: {
+ type: anthropicErrorType(body),
+ message: body.message,
+ ...(body.code ? { code: body.code } : {}),
+ ...(body.retryAtMs != null ? { retryAtMs: body.retryAtMs } : {}),
+ },
+ });
+}
+
function newMessageId(): string {
return `msg_${crypto.randomBytes(12).toString('hex')}`;
}
@@ -422,8 +442,21 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {
// inline tool-call dialect rescue that the OpenAI/Responses surfaces carry.
const state = newFallbackState();
const attemptLog: AttemptRecord[] = [];
+ // Client-disconnect fan-out: the flag stops the loop before the NEXT
+ // attempt; the AbortController (threaded to the provider as
+ // CompletionOptions.signal) additionally cancels the IN-FLIGHT upstream
+ // fetch and any body/stream read, so tokens stop burning and the in-flight
+ // lease frees immediately. 'close' also fires on normal completion —
+ // writableEnded distinguishes a real disconnect.
let clientGone = false;
- res.on('close', () => { if (!res.writableEnded) clientGone = true; });
+ const clientAbort = new AbortController();
+ res.on('close', () => {
+ if (!res.writableEnded) {
+ clientGone = true;
+ clientAbort.abort(newClientAbortError());
+ }
+ });
+ const dispatchOptions = { ...completionOptions, signal: clientAbort.signal };
await runFallbackLoop({
maxRetries: MAX_RETRIES,
@@ -434,7 +467,7 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {
dispatch: async (route, attempt) => {
if (stream) {
try {
- await streamCompletion(res, route, messages, completionOptions, {
+ await streamCompletion(res, route, messages, dispatchOptions, {
start, attempt, attemptLog, clientGone: () => clientGone, requestedModel, estimatedInputTokens, tools, pinnedModelId,
sessionId, pinned: resolved.pinned,
});
@@ -447,7 +480,7 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {
}
}
- const result = await route.provider.chatCompletion(route.apiKey, messages, route.modelId, completionOptions);
+ const result = await route.provider.chatCompletion(route.apiKey, messages, route.modelId, dispatchOptions);
const respMsg = result.choices?.[0]?.message;
const respText = contentToString(respMsg?.content ?? '');
let respToolCalls = respMsg?.tool_calls ?? [];
@@ -526,16 +559,12 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {
sendError(res, 502, 'api_error', `Provider error (${route.displayName}): ${sanitizeProviderErrorMessage(err.message)}`);
},
onRoutingExhausted: (lastError, routeErr, exhaustion, info) => {
- if (exhaustion) {
- setFallbackHeaders(res, info.attempts.length, info.attempts);
- sendError(res, exhaustion.status, anthropicErrorType(exhaustion), exhaustion.message);
- } else {
- sendError(res, routeErr?.status ?? 503, 'api_error', routeErr?.message ?? 'No model available to route this request');
- }
+ setFallbackHeaders(res, info.attempts.length, info.attempts);
+ sendExhaustion(res, exhaustion);
},
onExhausted: (exhaustion, info) => {
setFallbackHeaders(res, info.attempts.length, info.attempts);
- sendError(res, exhaustion.status, anthropicErrorType(exhaustion), exhaustion.message);
+ sendExhaustion(res, exhaustion);
},
});
});
@@ -771,6 +800,12 @@ async function streamCompletion(
logRequest(route.platform, route.modelId, route.keyId, 'success', ctx.estimatedInputTokens, outputTokens, Date.now() - ctx.start, null, null, ctx.pinnedModelId);
} catch (err: any) {
if (err instanceof StreamAlreadyStarted) throw err;
+ // Client abort mid-stream: the pump's own `if (ctx.clientGone()) break`
+ // can lose the race against the fetch-signal rejection, so the abort may
+ // surface here instead. Rethrow — the shared loop's client-abort branch
+ // stops the ladder without benching or an error log row (the socket is
+ // gone; nothing to render).
+ if (isClientAbortError(err)) throw err;
if (messageStarted) {
// Real payload already reached the client — finish the SSE response
// honestly instead of leaving Claude Code hanging, and stop the retry loop.
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index f68ed10e..9f839b6a 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -14,10 +14,10 @@ import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';
import { rescueInlineToolCalls, startsWithDialectMarker, couldBecomeDialectMarker, containsDialectMarker } from '../lib/tool-call-rescue.js';
import { getContextHandoffMode, recordIncomingMessages, maybeInjectContextHandoff, recordSuccessfulModel, hasPriorModel, HANDOFF_MAX_TOKENS } from '../services/context-handoff.js';
import { isFusionModel, runFusion, fusionConfigSchema, FusionError, FUSION_MODEL_ID } from '../services/fusion.js';
-import { isRetryableError, isPaymentRequiredError, isModelNotFoundError, isModelAccessForbiddenError } from '../lib/error-classify.js';
+import { isRetryableError, isPaymentRequiredError, isModelNotFoundError, isModelAccessForbiddenError, isClientAbortError, newClientAbortError } from '../lib/error-classify.js';
import { logRequest } from '../lib/request-log.js';
import { parseCacheDirective, cacheActive, isCacheableTemperature, computeCacheKey, getCachedResponse, storeCachedResponse } from '../services/cache.js';
-import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, exhaustedRetryError, setFallbackHeaders, type AttemptRecord } from '../lib/fallback-loop.js';
+import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, exhaustedRetryError, setFallbackHeaders, exhaustionErrorPayload, setExhaustionHeaders, type AttemptRecord } from '../lib/fallback-loop.js';
import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';
import { samplingParamSchemaFields, pickSamplingParams, supportedParametersForPlatforms } from '../lib/sampling-params.js';
import { enforceJsonContent } from '../lib/structured-output.js';
@@ -693,14 +693,26 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
if (groupChain.length === 0) {
const placeholders = members.map(() => '?').join(',');
const anyEnabled = db.prepare(`SELECT 1 FROM models WHERE id IN (${placeholders}) AND enabled = 1 LIMIT 1`).get(...members);
- const reason = anyEnabled ? 'has no providers with an enabled key' : 'is disabled';
- res.status(400).json({
- error: {
- message: `Model '${requestedModel}' ${reason}. Use 'auto' (or omit the 'model' field) to auto-route, or call /v1/models for the available list.`,
- type: 'invalid_request_error',
- code: 'model_not_found',
- },
- });
+ // Honest statuses: a model whose providers exist but have no usable key
+ // is a server-side configuration gap (503), not a client mistake; a
+ // disabled/unknown model is a 404 model_not_found (OpenAI semantics).
+ if (anyEnabled) {
+ res.status(503).json({
+ error: {
+ message: `Model '${requestedModel}' has no providers with an enabled key. Add a provider API key for it, use 'auto' (or omit the 'model' field) to auto-route, or call /v1/models for the available list.`,
+ type: 'service_unavailable',
+ code: 'no_providers_configured',
+ },
+ });
+ } else {
+ res.status(404).json({
+ error: {
+ message: `Model '${requestedModel}' is disabled. Use 'auto' (or omit the 'model' field) to auto-route, or call /v1/models for the available list.`,
+ type: 'invalid_request_error',
+ code: 'model_not_found',
+ },
+ });
+ }
return;
}
} else {
@@ -710,7 +722,7 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
} else {
const disabled = db.prepare('SELECT id FROM models WHERE model_id = ?').get(requestedModel) as { id: number } | undefined;
const reason = disabled ? 'is disabled' : 'is not in the catalog';
- res.status(400).json({
+ res.status(404).json({
error: {
message: `Model '${requestedModel}' ${reason}. Use 'auto' (or omit the 'model' field) to auto-route, or call /v1/models for the available list.`,
type: 'invalid_request_error',
@@ -725,8 +737,20 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
const pinnedModelId = requestedModel && !isAutoModel(requestedModel) ? requestedModel : null;
const state = newFallbackState();
const attemptLog: AttemptRecord[] = [];
+ // Client-disconnect fan-out: the flag stops the loop before the NEXT
+ // attempt; the AbortController (threaded to the provider as
+ // CompletionOptions.signal) additionally cancels the IN-FLIGHT upstream
+ // fetch and any body/stream read, so tokens stop burning and the in-flight
+ // lease frees immediately. 'close' also fires on normal completion —
+ // writableEnded distinguishes a real disconnect.
let clientGone = false;
- res.on('close', () => { if (!res.writableEnded) clientGone = true; });
+ const clientAbort = new AbortController();
+ res.on('close', () => {
+ if (!res.writableEnded) {
+ clientGone = true;
+ clientAbort.abort(newClientAbortError());
+ }
+ });
// Legacy /completions is a thin adapter over the shared fallback loop
// (lib/fallback-loop.ts): the cooldown/skip/penalty/exhaustion machinery is
@@ -781,7 +805,7 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
route.apiKey,
messages,
route.modelId,
- { temperature, max_tokens, top_p, stop },
+ { temperature, max_tokens, top_p, stop, signal: clientAbort.signal },
quotaContextForRoute(route, 'chat/completions'),
);
@@ -841,6 +865,12 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens, totalOutputTokens, Date.now() - start, null, ttfbMs, pinnedModelId);
return 'done';
} catch (streamErr: any) {
+ // Client abort mid-stream: the pump's own `if (clientGone) break`
+ // can lose the race against the fetch-signal rejection, so the
+ // abort may surface here instead. Rethrow — the shared loop's
+ // client-abort branch stops the ladder without benching or an
+ // error log row (the socket is gone; nothing to render).
+ if (isClientAbortError(streamErr)) throw streamErr;
if (headerSent) {
console.error(`[Proxy] Mid-stream legacy completion error from ${route.displayName}:`, streamErr.message);
const payload = { error: { message: `Provider error (${route.displayName}): stream interrupted`, type: 'stream_error' } };
@@ -866,7 +896,7 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
route.apiKey,
messages,
route.modelId,
- { temperature, max_tokens, top_p, stop },
+ { temperature, max_tokens, top_p, stop, signal: clientAbort.signal },
quotaContextForRoute(route, 'chat/completions'),
);
@@ -941,22 +971,24 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
});
},
onRoutingExhausted: (lastError, routeErr, exhaustion, info) => {
- if (exhaustion) {
- setFallbackHeaders(res, info.attempts.length, info.attempts);
- res.status(exhaustion.status).json({ error: { message: exhaustion.message, type: exhaustion.type } });
- } else {
+ if (!lastError) {
+ // Synchronous exhaustion: the router rejected every candidate before
+ // any upstream was tried — log the per-candidate disposition.
const disposition: string[] = Array.isArray(routeErr.diagnostics) ? routeErr.diagnostics : [];
console.warn(
`[Proxy] legacy completions routing exhausted (no upstream tried) req=${shortRequestId(requestGroupId)} ` +
`requested=${requestedModelLabel} candidates=${disposition.length}` +
(disposition.length ? `:\n ${disposition.join('\n ')}` : ''),
);
- res.status(routeErr.status ?? 503).json({ error: { message: routeErr.message, type: 'routing_error' } });
}
+ setFallbackHeaders(res, info.attempts.length, info.attempts);
+ setExhaustionHeaders(res, exhaustion);
+ res.status(exhaustion.status).json({ error: exhaustionErrorPayload(exhaustion) });
},
onExhausted: (exhaustion, info) => {
setFallbackHeaders(res, info.attempts.length, info.attempts);
- res.status(exhaustion.status).json({ error: { message: exhaustion.message, type: exhaustion.type } });
+ setExhaustionHeaders(res, exhaustion);
+ res.status(exhaustion.status).json({ error: exhaustionErrorPayload(exhaustion) });
},
});
});
@@ -1386,19 +1418,29 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
if (members && members.length > 0) {
groupChain = resolveModelGroupCandidates(members);
if (groupChain.length === 0) {
- // Distinguish a catalog-disabled model from one whose providers are
- // present but unusable (chain-disabled / no key), so the 400 stays
- // actionable and matches the legacy single-row "is disabled" wording.
+ // Distinguish a catalog-disabled model (404 model_not_found, OpenAI
+ // semantics) from one whose providers are present but unusable
+ // (chain-disabled / no key) — the latter is a server-side
+ // configuration gap, so it renders an honest 503.
const placeholders = members.map(() => '?').join(',');
const anyEnabled = db.prepare(`SELECT 1 FROM models WHERE id IN (${placeholders}) AND enabled = 1 LIMIT 1`).get(...members);
- const reason = anyEnabled ? 'has no providers with an enabled key' : 'is disabled';
- res.status(400).json({
- error: {
- message: `Model '${requestedModel}' ${reason}. Use 'auto' (or omit the 'model' field) to auto-route, or call /v1/models for the available list.`,
- type: 'invalid_request_error',
- code: 'model_not_found',
- },
- });
+ if (anyEnabled) {
+ res.status(503).json({
+ error: {
+ message: `Model '${requestedModel}' has no providers with an enabled key. Add a provider API key for it, use 'auto' (or omit the 'model' field) to auto-route, or call /v1/models for the available list.`,
+ type: 'service_unavailable',
+ code: 'no_providers_configured',
+ },
+ });
+ } else {
+ res.status(404).json({
+ error: {
+ message: `Model '${requestedModel}' is disabled. Use 'auto' (or omit the 'model' field) to auto-route, or call /v1/models for the available list.`,
+ type: 'invalid_request_error',
+ code: 'model_not_found',
+ },
+ });
+ }
return;
}
stickyStrategyKey = requestedModel;
@@ -1415,7 +1457,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
} else {
const disabled = db.prepare('SELECT id FROM models WHERE model_id = ?').get(requestedModel) as { id: number } | undefined;
const reason = disabled ? 'is disabled' : 'is not in the catalog';
- res.status(400).json({
+ res.status(404).json({
error: {
message: `Model '${requestedModel}' ${reason}. Use 'auto' (or omit the 'model' field) to auto-route, or call /v1/models for the available list.`,
type: 'invalid_request_error',
@@ -1442,8 +1484,20 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
// stream turn-integrity framing.
const state = newFallbackState();
const attemptLog: AttemptRecord[] = [];
+ // Client-disconnect fan-out: the flag stops the loop before the NEXT
+ // attempt; the AbortController (threaded to the provider as
+ // CompletionOptions.signal) additionally cancels the IN-FLIGHT upstream
+ // fetch and any body/stream read, so tokens stop burning and the in-flight
+ // lease frees immediately. 'close' also fires on normal completion —
+ // writableEnded distinguishes a real disconnect.
let clientGone = false;
- res.on('close', () => { if (!res.writableEnded) clientGone = true; });
+ const clientAbort = new AbortController();
+ res.on('close', () => {
+ if (!res.writableEnded) {
+ clientGone = true;
+ clientAbort.abort(newClientAbortError());
+ }
+ });
await runFallbackLoop({
maxRetries: MAX_RETRIES,
@@ -1539,7 +1593,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
try {
const gen = route.provider.streamChatCompletion(
route.apiKey, outboundMessages, route.modelId,
- { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, ...samplingParams },
+ { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, ...samplingParams, signal: clientAbort.signal },
quotaContextForRoute(route, 'chat/completions'),
);
@@ -1730,6 +1784,12 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens + injectedHandoffTokens, totalOutputTokens, Date.now() - start, null, ttfbMs, pinnedModelId);
return 'done';
} catch (streamErr: any) {
+ // Client abort mid-stream: the pump's own `if (clientGone) break`
+ // can lose the race against the fetch-signal rejection, so the
+ // abort may surface here instead. Rethrow — the shared loop's
+ // client-abort branch stops the ladder without benching or an
+ // error log row (the socket is gone; nothing to render).
+ if (isClientAbortError(streamErr)) throw streamErr;
if (headerSent) {
// Mid-stream error after real payload reached the client — finish
// the SSE response honestly instead of leaving the client hanging.
@@ -1758,7 +1818,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
} else {
const result = await route.provider.chatCompletion(
route.apiKey, outboundMessages, route.modelId,
- { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, ...samplingParams },
+ { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, ...samplingParams, signal: clientAbort.signal },
quotaContextForRoute(route, 'chat/completions'),
);
@@ -1930,26 +1990,26 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
},
onRoutingExhausted: (lastError, routeErr, exhaustion, info) => {
// No more models available.
- if (exhaustion) {
- setFallbackHeaders(res, info.attempts.length, info.attempts);
- res.status(exhaustion.status).json({ error: { message: exhaustion.message, type: exhaustion.type } });
- } else {
+ if (!lastError) {
// Synchronous exhaustion: the router rejected every candidate before any
// upstream was tried, so this is the ONLY place the per-model disposition
- // is recorded. Without it a routing_error 429 is opaque — you can't tell a
- // genuinely dry pool from cooldowns/quota/context narrowing (issue _1).
+ // is recorded. Without it the exhaustion status is opaque — you can't tell
+ // a genuinely dry pool from cooldowns/quota/context narrowing (issue _1).
const disposition: string[] = Array.isArray(routeErr.diagnostics) ? routeErr.diagnostics : [];
console.warn(
`[Proxy] routing exhausted (no upstream tried) req=${shortRequestId(requestGroupId)} ` +
`requested=${requestedModelLabel} candidates=${disposition.length}` +
(disposition.length ? `:\n ${disposition.join('\n ')}` : ''),
);
- res.status(routeErr.status ?? 503).json({ error: { message: routeErr.message, type: 'routing_error' } });
}
+ setFallbackHeaders(res, info.attempts.length, info.attempts);
+ setExhaustionHeaders(res, exhaustion);
+ res.status(exhaustion.status).json({ error: exhaustionErrorPayload(exhaustion) });
},
onExhausted: (exhaustion, info) => {
setFallbackHeaders(res, info.attempts.length, info.attempts);
- res.status(exhaustion.status).json({ error: { message: exhaustion.message, type: exhaustion.type } });
+ setExhaustionHeaders(res, exhaustion);
+ res.status(exhaustion.status).json({ error: exhaustionErrorPayload(exhaustion) });
},
});
});
diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.ts
index 97656c26..016ea616 100644
--- a/server/src/routes/responses.ts
+++ b/server/src/routes/responses.ts
@@ -23,11 +23,12 @@ import {
traceRouteEvent,
logRequest,
} from './proxy.js';
-import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, setFallbackHeaders, type AttemptRecord } from '../lib/fallback-loop.js';
+import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, setFallbackHeaders, exhaustionErrorPayload, setExhaustionHeaders, type AttemptRecord } from '../lib/fallback-loop.js';
import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';
import { samplingParamSchemaFields, pickSamplingParams, type ResponseFormat } from '../lib/sampling-params.js';
import { enforceJsonContent } from '../lib/structured-output.js';
import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';
+import { isClientAbortError, newClientAbortError } from '../lib/error-classify.js';
import { inferQuotaPoolKey, type QuotaObservationContext } from '../services/provider-quota.js';
export const responsesRouter = Router();
@@ -407,8 +408,21 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
const responseId = newId('resp');
const state = newFallbackState();
const attemptLog: AttemptRecord[] = [];
+ // Client-disconnect fan-out: the flag stops the loop before the NEXT
+ // attempt; the AbortController (threaded to the provider as
+ // CompletionOptions.signal) additionally cancels the IN-FLIGHT upstream
+ // fetch and any body/stream read, so tokens stop burning and the in-flight
+ // lease frees immediately. 'close' also fires on normal completion —
+ // writableEnded distinguishes a real disconnect.
let clientGone = false;
- res.on('close', () => { if (!res.writableEnded) clientGone = true; });
+ const clientAbort = new AbortController();
+ res.on('close', () => {
+ if (!res.writableEnded) {
+ clientGone = true;
+ clientAbort.abort(newClientAbortError());
+ }
+ });
+ const dispatchOpts = { ...completionOpts, signal: clientAbort.signal };
// Stream bookkeeping (used only when stream === true). `streamStarted` is the
// commit flag: true once the response.created/in_progress skeleton has left,
@@ -498,7 +512,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
route.apiKey,
messages,
route.modelId,
- completionOpts,
+ dispatchOpts,
quotaContextForRoute(route, 'responses'),
);
@@ -679,6 +693,12 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens, totalOutputTokens, Date.now() - start, null);
return 'done';
} catch (streamErr: any) {
+ // Client abort mid-stream: the pump's own `if (clientGone) break`
+ // can lose the race against the fetch-signal rejection, so the
+ // abort may surface here instead. Rethrow — the shared loop's
+ // client-abort branch stops the ladder without benching or an
+ // error log row (the socket is gone; nothing to render).
+ if (isClientAbortError(streamErr)) throw streamErr;
// A committed stream can't fail over (bytes already sent) — surface a
// response.failed event honestly and stop. A pre-commit failure throws
// through to the shared loop for cooldown + failover.
@@ -706,7 +726,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
route.apiKey,
messages,
route.modelId,
- completionOpts,
+ dispatchOpts,
quotaContextForRoute(route, 'responses'),
);
@@ -809,26 +829,27 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
res.status(502).json({ error: { message: `Provider error (${route.displayName}): ${sanitizeProviderErrorMessage(err.message)}`, type: 'provider_error' } });
},
onRoutingExhausted: (lastError, routeErr, exhaustion, info) => {
- const status = exhaustion?.status ?? routeErr.status ?? 503;
- const message = exhaustion?.message ?? routeErr.message;
- const type = exhaustion?.type ?? 'routing_error';
if (streamStarted) {
- sse('response.failed', { response: { id: responseId, object: 'response', status: 'failed', error: { message, type } } });
+ // Headers are already on the wire — the honest status/Retry-After can
+ // only travel in the failed-event payload.
+ sse('response.failed', { response: { id: responseId, object: 'response', status: 'failed', error: exhaustionErrorPayload(exhaustion) } });
res.end();
} else {
setFallbackHeaders(res, info.attempts.length, info.attempts);
- res.status(status).json({ error: { message, type } });
+ setExhaustionHeaders(res, exhaustion);
+ res.status(exhaustion.status).json({ error: exhaustionErrorPayload(exhaustion) });
}
},
onExhausted: (exhaustion, info) => {
// The streaming skeleton may already be on the wire — close the SSE stream
// with a failed event instead of writing JSON onto a committed response.
if (streamStarted) {
- sse('response.failed', { response: { id: responseId, object: 'response', status: 'failed', error: { message: exhaustion.message, type: exhaustion.type } } });
+ sse('response.failed', { response: { id: responseId, object: 'response', status: 'failed', error: exhaustionErrorPayload(exhaustion) } });
res.end();
} else {
setFallbackHeaders(res, info.attempts.length, info.attempts);
- res.status(exhaustion.status).json({ error: { message: exhaustion.message, type: exhaustion.type } });
+ setExhaustionHeaders(res, exhaustion);
+ res.status(exhaustion.status).json({ error: exhaustionErrorPayload(exhaustion) });
}
},
});
diff --git a/server/src/services/cooldown-probe.ts b/server/src/services/cooldown-probe.ts
new file mode 100644
index 00000000..868c908e
--- /dev/null
+++ b/server/src/services/cooldown-probe.ts
@@ -0,0 +1,222 @@
+// cooldown-probe — probe-based early recovery for benched keys.
+//
+// Cooldowns are deliberately pessimistic: a transient 429 benches a model+key
+// for 90s, the escalation ladder for up to a day, and an upstream 401 for the
+// whole 5-minute health cycle. When the provider recovers sooner (a per-minute
+// window rolled over, a provider incident ended, a key was un-revoked), that
+// capacity sits idle until the timer runs out. This job periodically issues the
+// same cheap validateKey call the health checker uses against keys sitting in a
+// HEURISTIC cooldown, and clears those cooldowns early when the probe passes.
+//
+// What it will never do:
+// - probe a cooldown backed by an authoritative retry time (explicit
+// Retry-After, daily-quota reset) or a credit/tier bench — those are
+// provider-stated facts, filtered out at the query (getProbeableCooldowns);
+// - extend a cooldown: a failed probe only schedules the next probe further
+// out (exponential backoff), the bench itself is untouched;
+// - feed the health checker's consecutive-failure counter (probeKeyValidity
+// is side-effect-free by design);
+// - thundering-herd providers after a restart: a key's first sighting only
+// schedules a jittered probe, and each pass probes at most a handful of
+// keys.
+//
+// The probe unit is the KEY, not the model: validateKey exercises the
+// credential, so one pass/fail is the same evidence for every heuristic
+// cooldown that key holds, and probing per key is what keeps probe traffic
+// bounded no matter how many models are benched on it.
+//
+// Kill switch: COOLDOWN_PROBE_DISABLED=1 (same convention as
+// CATALOG_SYNC_DISABLED). Per-pass probe budget: COOLDOWN_PROBE_MAX_PER_PASS.
+
+import type { Scheduler } from '../lib/scheduler.js';
+import {
+ getProbeableCooldowns,
+ clearCooldownEarly,
+ type ProbeableCooldown,
+} from './ratelimit.js';
+import { probeKeyValidity, type KeyProbeOutcome } from './health.js';
+
+// How often the scanner wakes up to look at the cooldown table. Scanning is one
+// cheap indexed SELECT; actual probes are gated far harder below.
+const SCAN_INTERVAL_MS = 60 * 1000;
+
+// A cooldown only becomes probe-ripe after this fraction of its bench has been
+// served. Probing a seconds-old cooldown would mostly re-confirm the failure
+// that caused it; by half-time a transient condition has had a real chance to
+// clear, and long escalated benches still recover hours early.
+const MIN_ELAPSED_FRACTION = 0.5;
+
+// Not worth probing a cooldown that is about to expire on its own — the probe
+// would cost a validate call to save less than a scan interval of idle time.
+const MIN_REMAINING_MS = 60 * 1000;
+
+// Failed-probe backoff per key: 2m, 4m, 8m, capped at 15m. Keys that stay bad
+// converge to one validate call per 15 minutes — a third of what the 5-minute
+// health pass already sends them.
+const PROBE_BACKOFF_BASE_MS = 2 * 60 * 1000;
+const PROBE_BACKOFF_MAX_MS = 15 * 60 * 1000;
+
+// Stagger window for a key's FIRST probe after it is sighted (covers the
+// restart case, where every persisted cooldown is sighted at once).
+const FIRST_PROBE_STAGGER_MS = 45 * 1000;
+
+const DEFAULT_MAX_PROBES_PER_PASS = 3;
+
+/** Per-pass probe budget. Tunable for large key fleets; 0 or a bad value falls
+ * back to the default (mirrors HEALTH_CHECK_CONCURRENCY). */
+function getMaxProbesPerPass(): number {
+ const raw = process.env.COOLDOWN_PROBE_MAX_PER_PASS;
+ if (raw !== undefined && raw.trim() !== '') {
+ const n = Number(raw);
+ if (Number.isInteger(n) && n > 0) return n;
+ }
+ return DEFAULT_MAX_PROBES_PER_PASS;
+}
+
+interface KeyProbeState {
+ failures: number;
+ nextProbeAtMs: number;
+}
+
+// keyId -> probe pacing. In-memory only, like the escalation ladder: a restart
+// forgetting the backoff is fine because first-sighting staggering re-spaces
+// the probes anyway.
+const keyState = new Map<number, KeyProbeState>();
+
+export interface ProbePassOptions {
+ now?: number;
+ // Test seams: injectable probe + RNG so unit tests need no network or timers.
+ probe?: (keyId: number) => Promise<KeyProbeOutcome>;
+ jitter?: () => number; // [0, 1), defaults to Math.random
+ maxProbes?: number;
+}
+
+export interface ProbePassResult {
+ probedKeyIds: number[];
+ clearedCooldowns: number;
+}
+
+/** True when this cooldown has served enough of its bench to be worth probing. */
+function isRipe(cooldown: ProbeableCooldown, now: number): boolean {
+ if (cooldown.expiresAtMs - now < MIN_REMAINING_MS) return false;
+ // Rows persisted before the provenance migration have no start time; treat
+ // them as old enough rather than never probing them.
+ if (cooldown.setAtMs == null) return true;
+ const duration = cooldown.expiresAtMs - cooldown.setAtMs;
+ return now - cooldown.setAtMs >= duration * MIN_ELAPSED_FRACTION;
+}
+
+/**
+ * One probe pass: find ripe heuristic cooldowns, probe up to maxProbes of their
+ * keys (respecting per-key backoff), and clear every heuristic cooldown on a
+ * key whose probe passed. Exported for tests; production runs it on the
+ * scheduler via startCooldownProbe.
+ */
+export async function runCooldownProbePass(opts: ProbePassOptions = {}): Promise<ProbePassResult> {
+ const now = opts.now ?? Date.now();
+ const probe = opts.probe ?? probeKeyValidity;
+ const jitter = opts.jitter ?? Math.random;
+ const maxProbes = opts.maxProbes ?? getMaxProbesPerPass();
+
+ const all = getProbeableCooldowns(now);
+
+ // Drop pacing state for keys with no probeable cooldowns left (expired,
+ // cleared, or re-benched as authoritative) so old backoff cannot delay a
+ // future, unrelated bench.
+ const keysWithCooldowns = new Set(all.map(c => c.keyId));
+ for (const keyId of [...keyState.keys()]) {
+ if (!keysWithCooldowns.has(keyId)) keyState.delete(keyId);
+ }
+
+ const ripeByKey = new Map<number, ProbeableCooldown[]>();
+ for (const cooldown of all) {
+ if (!isRipe(cooldown, now)) continue;
+ const list = ripeByKey.get(cooldown.keyId) ?? [];
+ list.push(cooldown);
+ ripeByKey.set(cooldown.keyId, list);
+ }
+
+ const probedKeyIds: number[] = [];
+ let clearedCooldowns = 0;
+
+ for (const [keyId, cooldownsForKey] of ripeByKey) {
+ if (probedKeyIds.length >= maxProbes) break;
+
+ let state = keyState.get(keyId);
+ if (!state) {
+ // First sighting: schedule, don't probe. This is the restart stagger —
+ // a boot with dozens of persisted cooldowns spreads its first probes
+ // across the jitter window instead of validating everything at once.
+ state = { failures: 0, nextProbeAtMs: now + Math.floor(jitter() * FIRST_PROBE_STAGGER_MS) };
+ keyState.set(keyId, state);
+ continue;
+ }
+ if (now < state.nextProbeAtMs) continue;
+
+ probedKeyIds.push(keyId);
+ const outcome = await probe(keyId);
+
+ if (outcome === 'valid') {
+ for (const cooldown of cooldownsForKey) {
+ clearCooldownEarly(cooldown.platform, cooldown.modelId, cooldown.keyId);
+ clearedCooldowns++;
+ }
+ keyState.delete(keyId);
+ console.log(
+ `[CooldownProbe] key ${keyId} validated — cleared ${cooldownsForKey.length} cooldown(s) early ` +
+ `(${cooldownsForKey.map(c => `${c.platform}/${c.modelId}`).join(', ')})`,
+ );
+ } else {
+ // Failed or inconclusive: the bench stays EXACTLY as it was — a probe
+ // must never extend a cooldown — and this key backs off before the next
+ // attempt so probes cannot burn quota against a provider that is down.
+ state.failures++;
+ const backoff = Math.min(PROBE_BACKOFF_BASE_MS * 2 ** (state.failures - 1), PROBE_BACKOFF_MAX_MS);
+ state.nextProbeAtMs = now + backoff;
+ }
+ }
+
+ return { probedKeyIds, clearedCooldowns };
+}
+
+// Overlap guard, same shape as checkAllKeys: a slow provider validate must not
+// let the next scheduled pass stack a second set of probes on top.
+let passInFlight: Promise<ProbePassResult> | null = null;
+
+function runGuardedPass(): Promise<ProbePassResult> {
+ if (passInFlight) return passInFlight;
+ passInFlight = runCooldownProbePass().finally(() => {
+ passInFlight = null;
+ });
+ return passInFlight;
+}
+
+let cancelProbeJob: (() => void) | null = null;
+
+export function startCooldownProbe(scheduler: Scheduler): void {
+ if (cancelProbeJob) return;
+ if (process.env.COOLDOWN_PROBE_DISABLED === '1') {
+ console.log('[CooldownProbe] disabled via COOLDOWN_PROBE_DISABLED=1');
+ return;
+ }
+ console.log(`[CooldownProbe] starting cooldown-probe recovery (scan every ${SCAN_INTERVAL_MS / 1000}s)`);
+ cancelProbeJob = scheduler.every(
+ SCAN_INTERVAL_MS,
+ async () => {
+ await runGuardedPass().catch(err => console.error('[CooldownProbe] pass failed:', err));
+ },
+ { name: 'cooldown-probe' },
+ );
+}
+
+export function stopCooldownProbe(): void {
+ if (cancelProbeJob) {
+ cancelProbeJob();
+ cancelProbeJob = null;
+ }
+}
+
+/** Test seam: drop all per-key probe pacing state. */
+export function resetCooldownProbeState(): void {
+ keyState.clear();
+}
diff --git a/server/src/services/fusion.ts b/server/src/services/fusion.ts
index d1f0f1a2..8cb2627c 100644
--- a/server/src/services/fusion.ts
+++ b/server/src/services/fusion.ts
@@ -6,6 +6,7 @@ import {
} from './router.js';
import {
recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit,
+ getCooldownDecisionForLimit,
PAYMENT_REQUIRED_COOLDOWN_MS, MODEL_FORBIDDEN_COOLDOWN_MS,
} from './ratelimit.js';
import { logRequest } from '../lib/request-log.js';
@@ -257,14 +258,15 @@ async function runModelCall(
if (isRetryableError(err)) {
if (isModelNotFoundError(err) || isModelAccessForbiddenError(err)) skipModels.add(route.modelDbId);
skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);
- setCooldown(
- route.platform, route.modelId, route.keyId,
- isPaymentRequiredError(err)
- ? PAYMENT_REQUIRED_COOLDOWN_MS
- : isModelAccessForbiddenError(err)
- ? MODEL_FORBIDDEN_COOLDOWN_MS
- : getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }, err.retryAfterMs),
- );
+ // Provenance mirrors cooldownDecisionForError (lib/fallback-loop.ts):
+ // credit/tier benches are never probe-recovered, Retry-After-backed
+ // ones only when our heuristic outlasted the provider's own retry time.
+ const decision = isPaymentRequiredError(err)
+ ? { durationMs: PAYMENT_REQUIRED_COOLDOWN_MS, source: 'credit' as const }
+ : isModelAccessForbiddenError(err)
+ ? { durationMs: MODEL_FORBIDDEN_COOLDOWN_MS, source: 'tier' as const }
+ : getCooldownDecisionForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }, err.retryAfterMs);
+ setCooldown(route.platform, route.modelId, route.keyId, decision.durationMs, decision.source);
recordRateLimitHit(route.modelDbId);
continue;
}
@@ -348,12 +350,13 @@ async function runJudgeStreaming(
if (isRetryableError(err)) {
if (isModelNotFoundError(err) || isModelAccessForbiddenError(err)) skipModels.add(route.modelDbId);
skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);
- setCooldown(
- route.platform, route.modelId, route.keyId,
- isPaymentRequiredError(err) ? PAYMENT_REQUIRED_COOLDOWN_MS
- : isModelAccessForbiddenError(err) ? MODEL_FORBIDDEN_COOLDOWN_MS
- : getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }, err.retryAfterMs),
- );
+ // Same provenance mapping as the non-stream path above.
+ const decision = isPaymentRequiredError(err)
+ ? { durationMs: PAYMENT_REQUIRED_COOLDOWN_MS, source: 'credit' as const }
+ : isModelAccessForbiddenError(err)
+ ? { durationMs: MODEL_FORBIDDEN_COOLDOWN_MS, source: 'tier' as const }
+ : getCooldownDecisionForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }, err.retryAfterMs);
+ setCooldown(route.platform, route.modelId, route.keyId, decision.durationMs, decision.source);
recordRateLimitHit(route.modelDbId);
continue;
}
diff --git a/server/src/services/health.ts b/server/src/services/health.ts
index 79833b26..cb8cd8c0 100644
--- a/server/src/services/health.ts
+++ b/server/src/services/health.ts
@@ -103,6 +103,43 @@ export async function checkKeyHealth(keyId: number): Promise<KeyStatus> {
}
}
+// ── Cooldown-probe validation (side-effect-free) ─────────────────────────────
+// The cooldown-probe recovery job (services/cooldown-probe.ts) needs the same
+// cheap validateKey call checkKeyHealth makes, WITHOUT any of its bookkeeping:
+// no status writes, no last_checked_at, and above all no failureCount — probes
+// run far more often than the 5-minute health pass, so letting them feed the
+// consecutive-failure counter would auto-disable a genuinely-bad key in a
+// fraction of the "3 consecutive checks" the threshold promises. A probe is a
+// question, never a verdict.
+
+export type KeyProbeOutcome = 'valid' | 'invalid' | 'error';
+
+export async function probeKeyValidity(keyId: number): Promise<KeyProbeOutcome> {
+ try {
+ const db = getDb();
+ const row = db.prepare('SELECT * FROM api_keys WHERE id = ? AND enabled = 1').get(keyId) as any;
+ if (!row) return 'error';
+
+ const provider = resolveProvider(row.platform as Platform, row.base_url);
+ if (!provider) return 'error';
+
+ const apiKey = decrypt(row.encrypted_key, row.iv, row.auth_tag);
+ const validation = await provider.validateKey(apiKey, {
+ platform: row.platform as Platform,
+ keyId,
+ quotaPoolKey: inferQuotaPoolKey(row.platform as Platform, null),
+ endpoint: 'models',
+ origin: 'probe',
+ });
+ const isValid = typeof validation === 'boolean' ? validation : validation.valid;
+ return isValid ? 'valid' : 'invalid';
+ } catch {
+ // Transport error (DNS/timeout/TLS): inconclusive, same as checkKeyHealth's
+ // reasoning — evidence about the network, not the key.
+ return 'error';
+ }
+}
+
/**
* Promote a key out of 'error' after it successfully served a live request.
*
diff --git a/server/src/services/ratelimit.ts b/server/src/services/ratelimit.ts
index b7690f26..b8fc48bc 100644
--- a/server/src/services/ratelimit.ts
+++ b/server/src/services/ratelimit.ts
@@ -600,6 +600,21 @@ export function recordTokens(
// Cooldown: when a provider returns 429, block that model+key for a period
const cooldowns = new Map<string, number>(); // key -> expiry timestamp
+// ── Cooldown provenance ───────────────────────────────────────────────────────
+// Recorded at setCooldown time so the cooldown-probe recovery job (services/
+// cooldown-probe.ts) can tell OUR guesses apart from provider-stated facts:
+// 'heuristic' — our own pessimistic bench (transient 90s, escalation
+// ladder, auth-failure bench, empty-completion). Often
+// outlives the actual outage, so probe-eligible.
+// 'authoritative' — backed by an explicit provider retry time (Retry-After
+// that determined the expiry, daily-quota reset). A fact,
+// not a guess: never probed.
+// 'credit' — 402 out-of-credits. A key-validation probe succeeding is
+// no evidence the account was topped up, so never probed.
+// 'tier' — 403 model-not-on-tier. Same reasoning: the key validates
+// fine while the model stays gated, so never probed.
+export type CooldownSource = 'heuristic' | 'authoritative' | 'credit' | 'tier';
+
// Escalating cooldown: track hits per key over a rolling 24h window so a
// daily-quota exhaustion (OpenRouter free: 50/day, Cohere free: 33/day, etc.)
// quarantines the key for the rest of the day instead of looping through
@@ -699,6 +714,11 @@ export function recentHitCount(
// prompts 429s on TPM while the daily quota is barely touched. Daily counters
// are persisted (countPersistedRequests / sumPersistedTokens), so this verdict
// is stable across restarts.
+export interface CooldownDecision {
+ durationMs: number;
+ source: CooldownSource;
+}
+
export function getCooldownDurationForLimit(
platform: string,
modelId: string,
@@ -706,6 +726,24 @@ export function getCooldownDurationForLimit(
limits: { rpd: number | null; tpd: number | null },
retryAfterMs?: number | null,
): number {
+ return getCooldownDecisionForLimit(platform, modelId, keyId, limits, retryAfterMs).durationMs;
+}
+
+/**
+ * Same verdict as getCooldownDurationForLimit, plus the provenance the
+ * cooldown-probe recovery job needs: 'authoritative' when an explicit provider
+ * Retry-After actually determined the expiry (a fact — never probed early),
+ * 'heuristic' when the duration is our own transient/escalation guess (probe-
+ * eligible). A Retry-After SHORTER than our bench does not make the cooldown
+ * authoritative: everything past the provider's own retry time is our guess.
+ */
+export function getCooldownDecisionForLimit(
+ platform: string,
+ modelId: string,
+ keyId: number,
+ limits: { rpd: number | null; tpd: number | null },
+ retryAfterMs?: number | null,
+): CooldownDecision {
const now = Date.now();
const rpdExhausted =
limits.rpd !== null && requestCount(platform, modelId, keyId, DAY, now) >= limits.rpd;
@@ -730,8 +768,10 @@ export function getCooldownDurationForLimit(
// Honor an upstream Retry-After as a floor: never bench shorter than our own
// heuristic, but extend (capped at a day) when the provider explicitly asks
// to wait longer than we otherwise would.
- if (retryAfterMs != null && retryAfterMs > base) return Math.min(retryAfterMs, DAY);
- return base;
+ if (retryAfterMs != null && retryAfterMs > base) {
+ return { durationMs: Math.min(retryAfterMs, DAY), source: 'authoritative' };
+ }
+ return { durationMs: base, source: 'heuristic' };
}
function persistedCooldownExpiry(
@@ -751,14 +791,23 @@ function persistedCooldownExpiry(
});
}
-function persistCooldown(platform: string, modelId: string, keyId: number, expiresAtMs: number) {
+function persistCooldown(
+ platform: string,
+ modelId: string,
+ keyId: number,
+ expiresAtMs: number,
+ source: CooldownSource,
+ setAtMs: number,
+) {
withDb(db => {
db.prepare(`
- INSERT INTO rate_limit_cooldowns (platform, model_id, key_id, expires_at_ms)
- VALUES (?, ?, ?, ?)
+ INSERT INTO rate_limit_cooldowns (platform, model_id, key_id, expires_at_ms, source, set_at_ms)
+ VALUES (?, ?, ?, ?, ?, ?)
ON CONFLICT(platform, model_id, key_id)
- DO UPDATE SET expires_at_ms = excluded.expires_at_ms
- `).run(platform, modelId, keyId, expiresAtMs);
+ DO UPDATE SET expires_at_ms = excluded.expires_at_ms,
+ source = excluded.source,
+ set_at_ms = excluded.set_at_ms
+ `).run(platform, modelId, keyId, expiresAtMs, source, setAtMs);
});
}
@@ -773,11 +822,18 @@ function clearPersistedCooldown(platform: string, modelId: string, keyId: number
});
}
-export function setCooldown(platform: string, modelId: string, keyId: number, durationMs = 60_000) {
+export function setCooldown(
+ platform: string,
+ modelId: string,
+ keyId: number,
+ durationMs = 60_000,
+ source: CooldownSource = 'heuristic',
+) {
const key = `${platform}:${modelId}:${keyId}:cooldown`;
- const expiresAtMs = Date.now() + durationMs;
+ const now = Date.now();
+ const expiresAtMs = now + durationMs;
cooldowns.set(key, expiresAtMs);
- persistCooldown(platform, modelId, keyId, expiresAtMs);
+ persistCooldown(platform, modelId, keyId, expiresAtMs, source, now);
}
export function isOnCooldown(platform: string, modelId: string, keyId: number): boolean {
@@ -880,6 +936,55 @@ export function clearCooldownsForKey(keyId: number): number {
return Math.max(cleared, memoryCleared);
}
+// ── Cooldown-probe surface (services/cooldown-probe.ts) ───────────────────────
+
+export interface ProbeableCooldown {
+ platform: string;
+ modelId: string;
+ keyId: number;
+ expiresAtMs: number;
+ // Null for rows persisted before the provenance migration; the prober treats
+ // an unknown start time as "old enough to probe".
+ setAtMs: number | null;
+}
+
+/**
+ * Active cooldowns whose expiry is OUR OWN guess ('heuristic'), i.e. the only
+ * ones probe-based early recovery may touch. Authoritative/credit/tier rows are
+ * excluded at the query so the prober cannot even see them. Deliberately
+ * DB-only, no in-memory fallback: the probe job is a best-effort optimisation,
+ * and when the DB is unavailable the right amount of probing is none.
+ */
+export function getProbeableCooldowns(now = Date.now()): ProbeableCooldown[] {
+ const rows = withDb(db => db.prepare(`
+ SELECT platform, model_id, key_id, expires_at_ms, set_at_ms
+ FROM rate_limit_cooldowns
+ WHERE expires_at_ms > ?
+ AND source = 'heuristic'
+ ORDER BY expires_at_ms ASC
+ `).all(now) as { platform: string; model_id: string; key_id: number; expires_at_ms: number; set_at_ms: number | null }[]) ?? [];
+
+ return rows.map(row => ({
+ platform: row.platform,
+ modelId: row.model_id,
+ keyId: row.key_id,
+ expiresAtMs: row.expires_at_ms,
+ setAtMs: row.set_at_ms,
+ }));
+}
+
+/**
+ * Clear ONE model+key cooldown before its timer expires, after a probe showed
+ * the key is serving again. Narrower than clearCooldownsForKey (the operator
+ * override): the escalation history in cooldownHits is deliberately kept, so a
+ * key that 429s again right after an early recovery re-enters the ladder where
+ * it left off instead of starting back at 2 minutes.
+ */
+export function clearCooldownEarly(platform: string, modelId: string, keyId: number): void {
+ cooldowns.delete(`${platform}:${modelId}:${keyId}:cooldown`);
+ clearPersistedCooldown(platform, modelId, keyId);
+}
+
/**
* Soonest moment any active cooldown expires, in ms since epoch, or null when
* nothing is cooling down. Used to tell an exhausted caller roughly when to