7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
P1 remainder: cooldown-probe recovery, client AbortSignal propagation, honest exhaustion statuses (#601)
Tashfeen5a5b17cfaa5a
diff --git a/.gitignore b/.gitignoreindex 7442529a..92b84aa6 100644--- a/.gitignore+++ b/.gitignore@@ -1,6 +1,9 @@node_modules/dist/server/data/+# Runtime data dir when the server is launched from the repo root (same+# contents as server/data — DB + keyfile — so same rule).+/data/*.db*.db-wal*.db-shmdiff --git a/server/src/__tests__/db/migrate/roundtrip.test.ts b/server/src/__tests__/db/migrate/roundtrip.test.tsindex cd5b2690..292f45e8 100644--- a/server/src/__tests__/db/migrate/roundtrip.test.ts+++ b/server/src/__tests__/db/migrate/roundtrip.test.ts@@ -13,6 +13,7 @@ const REQUEST_CLIENT_INFO_FILENAME = '20260706_000001_request_client_info.ts';const CUSTOM_MODEL_TOOL_SUPPORT_FILENAME = '20260706_000002_custom_model_tool_support.ts';const PROFILE_CHAIN_BACKFILL_FILENAME = '20260714_000001_profile_chain_backfill.ts';const KEY_HEALTH_ERROR_FILENAME = '20260720_000001_key_health_error.ts';+const COOLDOWN_PROBE_PROVENANCE_FILENAME = '20260726_000001_cooldown_probe_provenance.ts';interface SchemaRow {type: string;@@ -72,6 +73,7 @@ describe('migration round trip', () => {CUSTOM_MODEL_TOOL_SUPPORT_FILENAME,PROFILE_CHAIN_BACKFILL_FILENAME,KEY_HEALTH_ERROR_FILENAME,+ COOLDOWN_PROBE_PROVENANCE_FILENAME,]);} finally {db.close();diff --git a/server/src/__tests__/integration/full-flow.test.ts b/server/src/__tests__/integration/full-flow.test.tsindex 0265b774..0c103870 100644--- a/server/src/__tests__/integration/full-flow.test.ts+++ b/server/src/__tests__/integration/full-flow.test.ts@@ -162,23 +162,23 @@ describe('Full Integration Flow', () => {expect(s2).toBe(400);});- it('Step 11: Explicit unknown model returns 400 (not silent fallthrough)', async () => {+ it('Step 11: Explicit unknown model returns an honest 404 (not silent fallthrough)', async () => {const { status, body } = await req(app, 'POST', '/v1/chat/completions', {model: 'definitely-not-a-real-model',messages: [{ role: 'user', content: 'hi' }],}, authHeaders());- expect(status).toBe(400);+ expect(status).toBe(404);expect(body.error.code).toBe('model_not_found');expect(body.error.message).toContain('not in the catalog');});- it('Step 12: Explicit disabled model returns 400 with disabled reason', async () => {+ it('Step 12: Explicit disabled model returns an honest 404 with disabled reason', async () => {// gemini-2.5-pro is disabled (V1 migration). Reuse it as a known-disabled fixture.const { status, body } = await req(app, 'POST', '/v1/chat/completions', {model: 'gemini-2.5-pro',messages: [{ role: 'user', content: 'hi' }],}, authHeaders());- expect(status).toBe(400);+ expect(status).toBe(404);expect(body.error.code).toBe('model_not_found');expect(body.error.message).toContain('is disabled');});diff --git a/server/src/__tests__/lib/exhaustion-statuses.test.ts b/server/src/__tests__/lib/exhaustion-statuses.test.tsnew file mode 100644index 00000000..8f621927--- /dev/null+++ b/server/src/__tests__/lib/exhaustion-statuses.test.ts@@ -0,0 +1,438 @@+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';++// Honest exhaustion statuses (P1): when the fallback ladder exhausts every+// candidate, the terminal status must say WHY it failed and when to retry —+// 413 when every attempt died on a context/size rejection, 429 + retryAtMs ++// Retry-After when every candidate is unavailable until a known time, 404 when+// the model is gone everywhere, 503 when nothing is configured at all, and 502+// (never 500, never a lying 429) for mixed/other upstream failures.++const { mockCheckKeyHealth } = vi.hoisted(() => ({ mockCheckKeyHealth: vi.fn() }));+vi.mock('../../services/health.js', () => ({ checkKeyHealth: mockCheckKeyHealth, markKeyHealthyFromRequest: vi.fn() }));++import { initDb, getDb } from '../../db/index.js';+import {+ runFallbackLoop,+ newFallbackState,+ exhaustedRetryError,+ routingExhaustionBody,+ exhaustionErrorPayload,+ setExhaustionHeaders,+ classifyAttemptError,+ type AttemptRecord,+ type AttemptErrorClass,+ type FallbackHooks,+} from '../../lib/fallback-loop.js';+import { isContextTooLargeError } from '../../lib/error-classify.js';+import { setCooldown } from '../../services/ratelimit.js';+import type { RouteResult } from '../../services/router.js';++let keySeq = 96_000;+function fakeRoute(overrides: Partial<RouteResult> = {}): RouteResult {+ const n = ++keySeq;+ return {+ provider: {} as any, modelId: 'fake-model', modelDbId: 962000 + n, apiKey: 'k',+ keyId: n, platform: 'fake', displayName: 'Fake Model',+ rpdLimit: null, tpdLimit: null,+ ...overrides,+ };+}++function hooksSkeleton(overrides: Partial<FallbackHooks>): FallbackHooks {+ return {+ state: newFallbackState(),+ timeBudgetMs: 0,+ route: () => fakeRoute(),+ dispatch: async () => 'done',+ logFailure: () => {},+ onFatal: () => {},+ onRoutingExhausted: () => {},+ onExhausted: () => {},+ ...overrides,+ };+}++const record = (classes: AttemptErrorClass[]): AttemptRecord[] =>+ classes.map((errorClass, i) => ({ platform: 'fake', modelId: `m${i}`, keyOrdinal: i + 1, errorClass }));++// Fake key-shaped fixture built at runtime: an obvious prefix + low-entropy filler.+const FAKE_KEY = 'sk-test-' + 'a'.repeat(20);++beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+});++beforeEach(() => {+ mockCheckKeyHealth.mockReset();+ getDb().prepare('DELETE FROM rate_limit_cooldowns').run();+});++// ── isContextTooLargeError: real provider error shapes ───────────────────────++describe('isContextTooLargeError', () => {+ it('flags OpenAI-compat context_length_exceeded shapes (message and code)', () => {+ expect(isContextTooLargeError(Object.assign(+ new Error("OpenAI-compat API error 400: This model's maximum context length is 8192 tokens. However, your messages resulted in 10012 tokens. Please reduce the length of the messages."),+ { status: 400 },+ ))).toBe(true);+ expect(isContextTooLargeError(Object.assign(new Error('Bad request'), { status: 400, code: 'context_length_exceeded' }))).toBe(true);+ expect(isContextTooLargeError(new Error("400 {\"error\":{\"message\":\"...\",\"code\":\"context_length_exceeded\"}}"))).toBe(true);+ });++ it('flags Anthropic-style prompt-too-long shapes', () => {+ expect(isContextTooLargeError(Object.assign(+ new Error('Anthropic API error 400: prompt is too long: 210021 tokens > 200000 maximum'),+ { status: 400 },+ ))).toBe(true);+ expect(isContextTooLargeError(new Error('Input is too long for requested model.'))).toBe(true);+ });++ it('flags Google/Gemini token-limit shapes', () => {+ expect(isContextTooLargeError(Object.assign(+ new Error('Google API error 400: The input token count (1194286) exceeds the maximum number of tokens allowed (1048576).'),+ { status: 400 },+ ))).toBe(true);+ expect(isContextTooLargeError(new Error('request exceeds the maximum number of tokens'))).toBe(true);+ });++ it('flags literal HTTP 413s, including Groq TPM-ceiling 413s (Requested > Limit can never fit)', () => {+ expect(isContextTooLargeError(Object.assign(+ new Error('Groq API error 413: Request too large for model `llama-3.3-70b-versatile` in organization `org_x` on tokens per minute (TPM): Limit 30000, Requested 33476'),+ { status: 413 },+ ))).toBe(true);+ expect(isContextTooLargeError(new Error('API error 413: Payload Too Large'))).toBe(true);+ expect(isContextTooLargeError(new Error('Request entity too large'))).toBe(true);+ expect(isContextTooLargeError(new Error('Cloudflare API error 413: content too large'))).toBe(true);+ });++ it('does not flag rate limits, ordinary 400s, or upstream errors', () => {+ expect(isContextTooLargeError(Object.assign(new Error('429 Too Many Requests'), { status: 429 }))).toBe(false);+ expect(isContextTooLargeError(Object.assign(new Error('Groq API error 429: Rate limit reached on requests per day (RPD): Limit 1000'), { status: 429 }))).toBe(false);+ expect(isContextTooLargeError(Object.assign(new Error('Google API error 400: Invalid JSON payload received. Unknown name "x"'), { status: 400 }))).toBe(false);+ expect(isContextTooLargeError(Object.assign(new Error('Internal server error'), { status: 500 }))).toBe(false);+ expect(isContextTooLargeError(new Error('you have used up your daily free allocation of 10,000 neurons'))).toBe(false);+ });++ it('classifyAttemptError buckets context errors ahead of provider_bad_request', () => {+ // The OpenAI-compat shape contains "API error 400", which the generic+ // bad-request rule would otherwise swallow.+ expect(classifyAttemptError(Object.assign(+ new Error("OpenAI-compat API error 400: This model's maximum context length is 8192 tokens."),+ { status: 400 },+ ))).toBe('context_too_large');+ expect(classifyAttemptError(Object.assign(new Error('API error 413: Payload Too Large'), { status: 413 }))).toBe('context_too_large');+ });+});++// ── exhaustedRetryError: failure-kind → terminal-status taxonomy ─────────────++describe('exhaustedRetryError honest terminal statuses', () => {+ it('all attempts context-too-large → 413 invalid_request_error / context_length_exceeded', () => {+ const body = exhaustedRetryError(new Error('prompt is too long: 210000 tokens > 200000 maximum'), 20, {+ attempts: record(['context_too_large', 'context_too_large', 'context_too_large']),+ });+ expect(body.status).toBe(413);+ expect(body.kind).toBe('context_too_large');+ expect(body.type).toBe('invalid_request_error');+ expect(body.code).toBe('context_length_exceeded');+ expect(body.retryAtMs).toBeUndefined();+ expect(body.message).toContain('too large');+ expect(body.message).toContain('Attempt trail:');+ });++ it('the all-context 413 wins even when the last error is shaped like a generic provider 400', () => {+ // "API error 400" would match isProviderBadRequestError; the aggregate+ // unanimity must outrank the shape of whichever error came last.+ const lastError = Object.assign(+ new Error("OpenAI-compat API error 400: This model's maximum context length is 8192 tokens."),+ { status: 400 },+ );+ const body = exhaustedRetryError(lastError, 20, { attempts: record(['context_too_large', 'context_too_large']) });+ expect(body.status).toBe(413);+ expect(body.code).toBe('context_length_exceeded');+ });++ it('all attempts model-not-found → 404 model_not_found (no 410: the catalog keeps no removal tombstones)', () => {+ const body = exhaustedRetryError(Object.assign(new Error('no endpoints found'), { status: 404 }), 20, {+ attempts: record(['model_not_found', 'model_not_found']),+ });+ expect(body.status).toBe(404);+ expect(body.kind).toBe('model_not_found');+ expect(body.type).toBe('invalid_request_error');+ expect(body.code).toBe('model_not_found');+ });++ it('every candidate unavailable-until-known-time → 429 with retryAtMs from the soonest cooldown', () => {+ const before = Date.now();+ setCooldown('fake', 'benched-model', ++keySeq, 90_000);+ const body = exhaustedRetryError(Object.assign(new Error('429 Too Many Requests'), { status: 429 }), 20, {+ attempts: record(['rate_limited', 'daily_quota_exhausted', 'out_of_credits', 'forbidden']),+ });+ expect(body.status).toBe(429);+ expect(body.kind).toBe('rate_limit');+ expect(body.type).toBe('rate_limit_error');+ expect(body.code).toBe('rate_limit_exceeded');+ expect(body.retryAtMs).toBeGreaterThanOrEqual(before + 80_000);+ expect(body.retryAtMs).toBeLessThanOrEqual(Date.now() + 95_000);+ });++ it('the 429 omits retryAtMs when nothing is cooling down (no invented numbers)', () => {+ const body = exhaustedRetryError(new Error('429 Too Many Requests'), 20, {+ attempts: record(['rate_limited']),+ });+ expect(body.status).toBe(429);+ expect(body.retryAtMs).toBeUndefined();+ });++ it('mixed rate-limit + upstream-error → 502 (waiting is not a promise), never 500', () => {+ setCooldown('fake', 'benched-model', ++keySeq, 90_000); // a cooldown exists, but the mix forbids the 429+ const body = exhaustedRetryError(Object.assign(new Error('Bad Gateway'), { status: 502 }), 20, {+ attempts: record(['rate_limited', 'upstream_error']),+ });+ expect(body.status).toBe(502);+ expect(body.kind).toBe('upstream');+ expect(body.type).toBe('provider_error');+ expect(body.code).toBe('upstream_failed');+ expect(body.retryAtMs).toBeUndefined();+ expect(body.message).toContain('provider-side failure');+ });++ it('all upstream/network failures → 502 upstream_failed (upstream failed, not our bug)', () => {+ for (const classes of [['upstream_error', 'upstream_error'], ['timeout', 'upstream_error'], ['timeout', 'error']] as AttemptErrorClass[][]) {+ const body = exhaustedRetryError(Object.assign(new Error('fetch failed'), {}), 20, { attempts: record(classes) });+ expect(body.status).toBe(502);+ expect(body.code).toBe('upstream_failed');+ }+ });++ it('mixed context + rate-limit is neither a 413 nor a 429: falls through to 502', () => {+ const body = exhaustedRetryError(Object.assign(new Error('429'), { status: 429 }), 20, {+ attempts: record(['context_too_large', 'rate_limited']),+ });+ expect(body.status).toBe(502);+ });++ it('precedence: all-auth 502 and the aggregate 413 both outrank the circuit-breaker 503', () => {+ const auth = exhaustedRetryError(new Error('401'), 20, { attempts: record(['auth', 'auth']), breakerFails: 2 });+ expect(auth.status).toBe(502);+ expect(auth.kind).toBe('auth');+ expect(auth.code).toBe('provider_authentication_failed');++ const ctx = exhaustedRetryError(new Error('prompt is too long: 9 tokens > 8 maximum'), 20, {+ attempts: record(['context_too_large', 'context_too_large']),+ breakerFails: 2,+ });+ expect(ctx.status).toBe(413);+ });++ it('precedence: a breaker stop over a MIXED pool still renders the 503 breaker body', () => {+ const body = exhaustedRetryError(Object.assign(new Error('Bad Gateway'), { status: 502 }), 20, {+ attempts: record(['rate_limited', 'upstream_error']),+ breakerFails: 2,+ });+ expect(body.status).toBe(503);+ expect(body.code).toBe('upstream_unhealthy');+ expect(body.message).toContain('circuit-breaker');+ });++ it('legacy zero-attempt shape keeps the 429 contract', () => {+ const body = exhaustedRetryError(new Error('429 Too Many Requests'), 20);+ expect(body.status).toBe(429);+ expect(body.type).toBe('rate_limit_error');+ });+});++// ── routingExhaustionBody: zero attempts ran ─────────────────────────────────++describe('routingExhaustionBody (router gave up before any upstream was tried)', () => {+ it('empty chain / no diagnostics → 503 no_providers_configured', () => {+ const body = routingExhaustionBody({ message: 'All models exhausted.', status: 429, diagnostics: [] });+ expect(body.status).toBe(503);+ expect(body.type).toBe('service_unavailable');+ expect(body.code).toBe('no_providers_configured');+ });++ it('every candidate lacks a provider/usable key → 503 no_providers_configured', () => {+ const body = routingExhaustionBody({+ message: 'All models exhausted: 2 routes checked.',+ status: 429,+ diagnostics: [+ 'openai/gpt-4o-mini: no enabled+healthy key for platform',+ 'weird/x: no provider registered',+ ],+ });+ expect(body.status).toBe(503);+ expect(body.code).toBe('no_providers_configured');+ });++ it('every candidate rate-limited/on cooldown → 429 with retryAtMs', () => {+ const before = Date.now();+ setCooldown('fake', 'routing-benched', ++keySeq, 120_000);+ const body = routingExhaustionBody({+ message: 'All models exhausted: 2 routes checked.',+ status: 429,+ diagnostics: [+ 'groq/llama-3.3-70b: 2 key(s) — cooldown:2',+ 'google/gemini-2.0-flash: 1 key(s) — rpm/rpd-limit:1',+ ],+ });+ expect(body.status).toBe(429);+ expect(body.code).toBe('rate_limit_exceeded');+ expect(body.retryAtMs).toBeGreaterThanOrEqual(before + 110_000);+ });++ it('rate-limited candidates mixed with unconfigured ones still → 429 (the pool recovers by itself)', () => {+ const body = routingExhaustionBody({+ message: 'All models exhausted.',+ status: 429,+ diagnostics: [+ 'groq/llama-3.3-70b: 1 key(s) — cooldown:1',+ 'weird/x: no provider registered',+ ],+ });+ expect(body.status).toBe(429);+ expect(body.code).toBe('rate_limit_exceeded');+ });++ it('every candidate too small for the request → 413 context_length_exceeded', () => {+ const body = routingExhaustionBody({+ message: 'All models exhausted: 2 routes checked (2 prompt too large for the model).',+ status: 429,+ diagnostics: [+ 'groq/llama-3.1-8b: context 8192 < estimated 50000',+ 'groq/gpt-oss-120b: tpm_limit 8000 < estimated 50000',+ ],+ });+ expect(body.status).toBe(413);+ expect(body.code).toBe('context_length_exceeded');+ expect(body.type).toBe('invalid_request_error');+ });++ it('capability-filtered candidates keep the router status with a routing_exhausted code', () => {+ const body = routingExhaustionBody({+ message: 'All models exhausted.',+ status: 429,+ diagnostics: [+ 'fake/text-only: no vision support',+ 'groq/llama-3.3-70b: 1 key(s) — cooldown:1',+ ],+ });+ expect(body.status).toBe(429);+ expect(body.code).toBe('routing_exhausted');+ });+});++// ── Wire helpers: OpenAI error payload + Retry-After header ──────────────────++describe('exhaustionErrorPayload / setExhaustionHeaders', () => {+ it('the payload carries type/code/retryAtMs and omits absent fields', () => {+ const full = exhaustionErrorPayload({+ status: 429, kind: 'rate_limit', type: 'rate_limit_error', code: 'rate_limit_exceeded',+ retryAtMs: 1234, message: 'm',+ });+ expect(full).toEqual({ message: 'm', type: 'rate_limit_error', code: 'rate_limit_exceeded', retryAtMs: 1234 });++ const bare = exhaustionErrorPayload({ status: 502, kind: 'upstream', type: 'provider_error', message: 'm' });+ expect(bare).toEqual({ message: 'm', type: 'provider_error' });+ expect('retryAtMs' in bare).toBe(false);+ });++ it('Retry-After is ceil((retryAtMs - now) / 1000) seconds', () => {+ const headers: Record<string, string> = {};+ const res = { setHeader: (n: string, v: string) => { headers[n] = v; } };+ const now = Date.now();+ setExhaustionHeaders(res, {+ status: 429, kind: 'rate_limit', type: 'rate_limit_error', message: 'm',+ retryAtMs: now + 61_500,+ }, now);+ expect(headers['Retry-After']).toBe('62');+ });++ it('no Retry-After without a concrete retry time; an elapsed one clamps to 0', () => {+ const headers: Record<string, string> = {};+ const res = { setHeader: (n: string, v: string) => { headers[n] = v; } };+ setExhaustionHeaders(res, { status: 502, kind: 'upstream', type: 'provider_error', message: 'm' });+ expect(headers['Retry-After']).toBeUndefined();+ setExhaustionHeaders(res, { status: 429, kind: 'rate_limit', type: 'rate_limit_error', message: 'm', retryAtMs: Date.now() - 5_000 });+ expect(headers['Retry-After']).toBe('0');+ });+});++// ── End-to-end through the shared loop ───────────────────────────────────────++describe('runFallbackLoop renders honest exhaustion bodies', () => {+ it('an all-rate-limited chain exhausts into a 429 whose retryAtMs comes from the cooldowns the loop itself set', async () => {+ const before = Date.now();+ const onExhausted = vi.fn();+ const dispatch = vi.fn(async () => {+ throw Object.assign(new Error('429 Too Many Requests'), { status: 429 });+ });++ await runFallbackLoop(hooksSkeleton({+ maxRetries: 3,+ route: () => fakeRoute({ apiKey: FAKE_KEY }),+ dispatch,+ onExhausted,+ }));++ expect(onExhausted).toHaveBeenCalledTimes(1);+ const [body] = onExhausted.mock.calls[0];+ expect(body.status).toBe(429);+ expect(body.code).toBe('rate_limit_exceeded');+ // recordRetryableFailure benched each failed key, so the soonest expiry is real.+ expect(body.retryAtMs).toBeGreaterThan(before);+ });++ it('a rate-limit + upstream-5xx mix exhausts into a 502, not a lying 429', async () => {+ const onExhausted = vi.fn();+ let call = 0;+ const dispatch = vi.fn(async () => {+ call += 1;+ if (call === 1) throw Object.assign(new Error('429 Too Many Requests'), { status: 429 });+ throw Object.assign(new Error('Bad Gateway'), { status: 502 });+ });++ await runFallbackLoop(hooksSkeleton({ maxRetries: 3, dispatch, onExhausted }));++ const [body] = onExhausted.mock.calls[0];+ expect(body.status).toBe(502);+ expect(body.kind).toBe('upstream');+ expect(body.code).toBe('upstream_failed');+ });++ it('an all-context-too-large chain exhausts into a 413', async () => {+ const onExhausted = vi.fn();+ const dispatch = vi.fn(async () => {+ throw Object.assign(+ new Error("OpenAI-compat API error 400: This model's maximum context length is 8192 tokens."),+ { status: 400 },+ );+ });++ await runFallbackLoop(hooksSkeleton({ maxRetries: 2, dispatch, onExhausted }));++ const [body] = onExhausted.mock.calls[0];+ expect(body.status).toBe(413);+ expect(body.code).toBe('context_length_exceeded');+ });++ it('routing exhaustion with zero attempts hands the surface a ready 503 body when nothing is configured', async () => {+ const onRoutingExhausted = vi.fn();+ await runFallbackLoop(hooksSkeleton({+ route: () => {+ throw Object.assign(new Error('All models exhausted.'), {+ status: 429,+ diagnostics: ['openai/gpt-4o-mini: no enabled+healthy key for platform'],+ });+ },+ onRoutingExhausted,+ }));++ expect(onRoutingExhausted).toHaveBeenCalledTimes(1);+ const [lastError, , exhaustion] = onRoutingExhausted.mock.calls[0];+ expect(lastError).toBeNull();+ expect(exhaustion.status).toBe(503);+ expect(exhaustion.code).toBe('no_providers_configured');+ });+});diff --git a/server/src/__tests__/lib/fallback-loop-client-abort.test.ts b/server/src/__tests__/lib/fallback-loop-client-abort.test.tsnew file mode 100644index 00000000..f98f3b7f--- /dev/null+++ b/server/src/__tests__/lib/fallback-loop-client-abort.test.ts@@ -0,0 +1,147 @@+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';+import { initDb, getDb } from '../../db/index.js';+import { runFallbackLoop, newFallbackState, type FallbackState } from '../../lib/fallback-loop.js';+import { isClientAbortError, newClientAbortError, isRetryableError } from '../../lib/error-classify.js';+import { acquireLease, releaseLease, resetLeases, inFlightForKey } from '../../services/ratelimit.js';+import type { RouteResult } from '../../services/router.js';++/**+ * Client-abort semantics in the shared fallback loop (P1 AbortSignal work):+ * when the gateway's OWN client hangs up, the composed fetch signal rejects+ * the in-flight attempt with the marked error from newClientAbortError. That+ * throw must stop the ladder immediately, release the in-flight lease via the+ * existing finally, and leave NO provider-failure trace — no cooldown, no+ * skip-key entry, no logFailure row, no exhaustion render.+ */++let keySeq = 900;+function leasedRoute(): { route: RouteResult; keyId: number } {+ const keyId = ++keySeq;+ const id = acquireLease('fake', 'fake-model', keyId, 100);+ return {+ keyId,+ route: {+ provider: {} as any,+ modelId: 'fake-model',+ modelDbId: 900000 + keyId,+ apiKey: 'test-key',+ keyId,+ platform: 'fake',+ displayName: 'Fake Model',+ rpdLimit: null,+ tpdLimit: null,+ release: () => releaseLease(id),+ },+ };+}++function hooks(state: FallbackState, overrides: Record<string, unknown>) {+ return {+ state,+ timeBudgetMs: 0,+ logFailure: vi.fn(),+ onFatal: vi.fn(),+ onRoutingExhausted: vi.fn(),+ onExhausted: vi.fn(),+ ...overrides,+ } as any;+}++describe('fallback loop on client abort', () => {+ beforeEach(() => {+ initDb(':memory:');+ resetLeases();+ vi.spyOn(console, 'log').mockImplementation(() => {});+ });++ afterEach(() => {+ resetLeases();+ vi.restoreAllMocks();+ });++ it('classifies the marked error as client-abort, and not as retryable', () => {+ const err = newClientAbortError();+ expect(isClientAbortError(err)).toBe(true);+ // Must never enter the retryable ladder as a provider failure.+ expect(isRetryableError(err)).toBe(false);+ // Ordinary aborts/timeouts keep their provider-timeout classification.+ expect(isClientAbortError(new DOMException('The operation was aborted (groq, chat, 15s)', 'AbortError'))).toBe(false);+ });++ it('stops the ladder immediately — no further attempts, nothing rendered', async () => {+ const state = newFallbackState();+ const dispatch = vi.fn(async () => { throw newClientAbortError(); });+ const h = hooks(state, {+ maxRetries: 5,+ route: () => leasedRoute().route,+ dispatch,+ });++ await runFallbackLoop(h);++ expect(dispatch).toHaveBeenCalledTimes(1);+ // The socket is gone: no exhaustion body, no fatal render, no error row.+ expect(h.logFailure).not.toHaveBeenCalled();+ expect(h.onFatal).not.toHaveBeenCalled();+ expect(h.onExhausted).not.toHaveBeenCalled();+ expect(h.onRoutingExhausted).not.toHaveBeenCalled();+ });++ it('releases the in-flight lease (mid-stream abort surfaces as a dispatch throw)', async () => {+ const state = newFallbackState();+ const { route, keyId } = leasedRoute();+ // Mirror a stream pump: one chunk is consumed, then the client hangs up+ // and the next upstream read rejects with the marked abort.+ async function* abortedStream() {+ yield { choices: [{ delta: { content: 'hel' } }] };+ throw newClientAbortError();+ }+ await runFallbackLoop(hooks(state, {+ route: () => route,+ dispatch: async () => {+ for await (const _chunk of abortedStream()) { /* pump until the abort */ }+ return 'done' as const;+ },+ }));++ expect(inFlightForKey('fake', keyId)).toBe(0);+ });++ it('records no cooldown, no skip entry, and no failure bookkeeping', async () => {+ const state = newFallbackState();+ const { route, keyId } = leasedRoute();+ await runFallbackLoop(hooks(state, {+ route: () => route,+ dispatch: async () => { throw newClientAbortError(); },+ }));++ // No cooldown row: a vanished client is not a provider-health signal.+ const cooldown = getDb().prepare('SELECT 1 FROM rate_limit_cooldowns WHERE platform = ? AND key_id = ?').get('fake', keyId);+ expect(cooldown).toBeUndefined();+ // No per-request skip state either — nothing was "tried and failed".+ expect(state.skipKeys.size).toBe(0);+ expect(state.skipModels.size).toBe(0);+ });++ it('clientGone still stops the loop before the next attempt after an ordinary failure', async () => {+ // The pre-existing boundary: a NON-abort provider failure that lands after+ // the disconnect is still recorded normally, but the loop must not start+ // another attempt for a client that is gone.+ const state = newFallbackState();+ const dispatch = vi.fn(async () => {+ throw Object.assign(new Error('fake API error 429: rate limit'), { status: 429 });+ });+ let gone = false;+ await runFallbackLoop(hooks(state, {+ maxRetries: 5,+ clientGone: () => gone,+ route: () => leasedRoute().route,+ dispatch: async () => {+ gone = true; // client vanishes during the first attempt+ return dispatch();+ },+ }));++ expect(dispatch).toHaveBeenCalledTimes(1);+ });+});diff --git a/server/src/__tests__/providers/abort-signal.test.ts b/server/src/__tests__/providers/abort-signal.test.tsnew file mode 100644index 00000000..a3d7970c--- /dev/null+++ b/server/src/__tests__/providers/abort-signal.test.ts@@ -0,0 +1,167 @@+import http from 'http';+import type { AddressInfo } from 'net';+import { describe, it, expect, afterEach } from 'vitest';+import { OpenAICompatProvider } from '../../providers/openai-compat.js';+import { isClientAbortError, newClientAbortError } from '../../lib/error-classify.js';+import type { ChatMessage } from '@freellmapi/shared/types.js';++/**+ * AbortSignal-through-fetch coverage (P1, mined from the frankljlyy-ai and+ * MLuqmanBR forks): fetchWithTimeout used to bound only time-to-headers, so a+ * body/stream read was unbounded and a client disconnect never canceled the+ * upstream request. These tests run a REAL loopback HTTP upstream (not a fetch+ * mock) because the behavior under test is undici's signal wiring itself:+ * aborting the composed signal must tear down the upstream socket mid-body and+ * mid-stream, and reject the pending read with the marked client-abort reason.+ */++const MESSAGES: ChatMessage[] = [{ role: 'user', content: 'hi' }];++interface Upstream {+ url: string;+ /** Resolves when the upstream saw its response connection torn down. */+ closed: Promise<void>;+ stop: () => Promise<void>;+}++async function startUpstream(handler: (req: http.IncomingMessage, res: http.ServerResponse) => void): Promise<Upstream> {+ let sawClose!: () => void;+ const closed = new Promise<void>(resolve => { sawClose = resolve; });+ const server = http.createServer((req, res) => {+ res.on('close', sawClose);+ handler(req, res);+ });+ await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve));+ const port = (server.address() as AddressInfo).port;+ return {+ url: `http://127.0.0.1:${port}/v1`,+ closed,+ stop: () => new Promise<void>(resolve => {+ server.closeAllConnections();+ server.close(() => resolve());+ }),+ };+}++function makeProvider(baseUrl: string): OpenAICompatProvider {+ return new OpenAICompatProvider({ platform: 'groq', name: 'TestCompat', baseUrl });+}++describe('AbortSignal through provider fetch + body/stream reads', () => {+ let upstream: Upstream | null = null;++ afterEach(async () => {+ await upstream?.stop();+ upstream = null;+ });++ it('client abort mid-body-read cancels the upstream fetch', async () => {+ // 200 + partial JSON body, then hold the connection open forever: the+ // request is stuck inside res.json(), past the header timeout's old reach.+ upstream = await startUpstream((_req, res) => {+ res.writeHead(200, { 'content-type': 'application/json' });+ res.write('{"id":"chatcmpl-hang","choices":[');+ });+ const provider = makeProvider(upstream.url);++ const clientAbort = new AbortController();+ const pending = provider.chatCompletion('test-key', MESSAGES, 'test-model', { signal: clientAbort.signal });+ setTimeout(() => clientAbort.abort(newClientAbortError()), 100);++ const err = await pending.then(() => null, e => e);+ expect(err).not.toBeNull();+ // The rejection must carry the marked client-abort reason (not a generic+ // parse/abort error), so the fallback loop can skip all failure bookkeeping.+ expect(isClientAbortError(err)).toBe(true);+ // And the upstream connection must actually be torn down — tokens stop+ // burning — not left to drain in the background.+ await upstream.closed;+ });++ it('client abort mid-stream stops iteration and rejects with the marked reason', async () => {+ // SSE headers + one real chunk, then hold: the consumer is parked inside+ // reader.read() when the client hangs up.+ upstream = await startUpstream((_req, res) => {+ res.writeHead(200, { 'content-type': 'text/event-stream' });+ res.write('data: {"id":"c1","object":"chat.completion.chunk","choices":[{"index":0,"delta":{"content":"hello"},"finish_reason":null}]}\n\n');+ });+ const provider = makeProvider(upstream.url);++ const clientAbort = new AbortController();+ const received: string[] = [];+ const err = await (async () => {+ for await (const chunk of provider.streamChatCompletion('test-key', MESSAGES, 'test-model', { signal: clientAbort.signal })) {+ received.push((chunk as any).choices?.[0]?.delta?.content ?? '');+ // Simulate the route pump's client vanishing after the first token.+ clientAbort.abort(newClientAbortError());+ }+ return null;+ })().catch(e => e);++ expect(received).toEqual(['hello']);+ expect(err).not.toBeNull();+ expect(isClientAbortError(err)).toBe(true);+ await upstream.closed;+ });++ it('per-attempt timeout still fires on slow headers', async () => {+ // Upstream never writes headers — the historical fetchWithTimeout contract.+ upstream = await startUpstream(() => { /* never respond */ });+ const provider = makeProvider(upstream.url);++ const startedAt = Date.now();+ const err = await provider+ .chatCompletion('test-key', MESSAGES, 'test-model', { timeoutMs: 300 })+ .then(() => null, e => e);++ expect(err).not.toBeNull();+ expect(/aborted/i.test(err.message)).toBe(true);+ // A timeout is a provider-health signal — it must NOT read as client-caused.+ expect(isClientAbortError(err)).toBe(false);+ expect(Date.now() - startedAt).toBeLessThan(3000);+ });++ it('per-attempt timeout now also fires on a hung body read', async () => {+ // 200 arrives instantly (so the old header-scoped timer is disarmed), but+ // the JSON body never completes. The 'request'-bounds deadline must abort+ // res.json() instead of hanging the attempt forever.+ upstream = await startUpstream((_req, res) => {+ res.writeHead(200, { 'content-type': 'application/json' });+ res.write('{"id":"chatcmpl-hang","choices":[');+ });+ const provider = makeProvider(upstream.url);++ const startedAt = Date.now();+ const err = await provider+ .chatCompletion('test-key', MESSAGES, 'test-model', { timeoutMs: 300 })+ .then(() => null, e => e);++ expect(err).not.toBeNull();+ // Must surface as an abort/timeout, not as the "non-OpenAI-compatible+ // endpoint" body-parse diagnosis (isAbortLikeError rethrow in the adapter).+ expect(/aborted/i.test(err.message)).toBe(true);+ expect(isClientAbortError(err)).toBe(false);+ expect(Date.now() - startedAt).toBeLessThan(3000);+ await upstream.closed;+ });++ it('a completed request is unaffected by a later client abort', async () => {+ // Guard against over-eager wiring: abort AFTER the response is fully+ // consumed must not retro-fail anything (the timer in 'request' bounds is+ // never cleared, so this exercises the fire-after-completion no-op too).+ upstream = await startUpstream((_req, res) => {+ res.writeHead(200, { 'content-type': 'application/json' });+ res.end(JSON.stringify({+ id: 'chatcmpl-ok', object: 'chat.completion', created: 1, model: 'test-model',+ choices: [{ index: 0, message: { role: 'assistant', content: 'done' }, finish_reason: 'stop' }],+ usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },+ }));+ });+ const provider = makeProvider(upstream.url);++ const clientAbort = new AbortController();+ const result = await provider.chatCompletion('test-key', MESSAGES, 'test-model', { signal: clientAbort.signal });+ clientAbort.abort(newClientAbortError());+ expect(result.choices[0]?.message?.content).toBe('done');+ });+});diff --git a/server/src/__tests__/routes/proxy-auto-model.test.ts b/server/src/__tests__/routes/proxy-auto-model.test.tsindex 4f304962..097cd70f 100644--- a/server/src/__tests__/routes/proxy-auto-model.test.ts+++ b/server/src/__tests__/routes/proxy-auto-model.test.ts@@ -209,7 +209,7 @@ describe('Virtual "auto" model', () => {messages: [{ role: 'user', content: 'hello' }],}, authHeaders());- expect(status).toBe(400);+ expect(status).toBe(404);expect(body.error.code).toBe('model_not_found');});});diff --git a/server/src/__tests__/routes/proxy-max-tokens-routing.test.ts b/server/src/__tests__/routes/proxy-max-tokens-routing.test.tsindex db599868..e3e2e4df 100644--- a/server/src/__tests__/routes/proxy-max-tokens-routing.test.ts+++ b/server/src/__tests__/routes/proxy-max-tokens-routing.test.ts@@ -101,18 +101,22 @@ describe('max_tokens no longer starves routing (#470)', () => {expect(chatCompletion).toHaveBeenCalledTimes(1);});- it('still excludes the model when the INPUT itself exceeds the TPM budget', async () => {+ it('still excludes the model when the INPUT itself exceeds the TPM budget — now an honest 413', async () => {chatCompletion.mockResolvedValueOnce(GOOD_RESULT);// ~8k input tokens (32k chars / 4) > the 6k TPM budget → the only enabled- // model is filtered out before any upstream call.+ // model is filtered out before any upstream call. Every candidate rejected+ // the request as too big for its window, so the exhaustion ladder renders+ // a 413 context_length_exceeded (waiting would not help), not the old+ // misleading 429.const bigPrompt = 'x'.repeat(32_000);- const { status } = await post(app, {+ const { status, body } = await post(app, {messages: [{ role: 'user', content: bigPrompt }],max_tokens: 50,}, key);- expect(status).toBe(429);+ expect(status).toBe(413);+ expect(body.error.code).toBe('context_length_exceeded');expect(chatCompletion).not.toHaveBeenCalled();});});diff --git a/server/src/__tests__/services/cooldown-probe.test.ts b/server/src/__tests__/services/cooldown-probe.test.tsnew file mode 100644index 00000000..e4f80947--- /dev/null+++ b/server/src/__tests__/services/cooldown-probe.test.ts@@ -0,0 +1,347 @@+import { describe, it, expect, beforeAll, beforeEach, afterEach, vi } from 'vitest';++// Unit tests for cooldown-probe early recovery: which cooldowns are eligible+// (heuristic only — authoritative/credit/tier are provider-stated facts and are+// never probed), the half-bench ripeness gate, the restart stagger, the failed-+// probe backoff progression, the per-pass probe budget, and early clearing on a+// successful probe. Plus the provenance classification the whole feature keys+// off (cooldownDecisionForError / setCooldown source tagging).++import { initDb, getDb } from '../../db/index.js';+import {+ setCooldown,+ isOnCooldown,+ getProbeableCooldowns,+} from '../../services/ratelimit.js';+import {+ runCooldownProbePass,+ resetCooldownProbeState,+ startCooldownProbe,+ stopCooldownProbe,+} from '../../services/cooldown-probe.js';+import { cooldownDecisionForError } from '../../lib/fallback-loop.js';+import type { RouteResult } from '../../services/router.js';+import type { KeyProbeOutcome } from '../../services/health.js';++const MINUTE = 60_000;++// Distinct keyId per bench: the cooldown store is module-global (memory map ++// DB), so shared ids would leak state across tests.+let keySeq = 910_000;++function bench(+ source: 'heuristic' | 'authoritative' | 'credit' | 'tier',+ opts: { durationMs?: number; modelId?: string; keyId?: number } = {},+): number {+ const keyId = opts.keyId ?? ++keySeq;+ setCooldown('probefake', opts.modelId ?? 'model-a', keyId, opts.durationMs ?? 10 * MINUTE, source);+ return keyId;+}++// A probe that always answers the same thing, wrapped in a spy so tests can+// assert exactly which keys were probed and how often.+const probeReturning = (outcome: KeyProbeOutcome) => vi.fn(async (_keyId: number) => outcome);++// jitter() => 0 makes a first-sighted key probeable on the very next pass,+// which keeps the "seed pass then probe pass" choreography deterministic.+const noJitter = () => 0;++beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+});++beforeEach(() => {+ resetCooldownProbeState();+ getDb().prepare('DELETE FROM rate_limit_cooldowns').run();+});++describe('probe eligibility', () => {+ it('only heuristic cooldowns are visible to the prober', () => {+ const heuristic = bench('heuristic');+ bench('authoritative');+ bench('credit');+ bench('tier');++ const rows = getProbeableCooldowns();+ expect(rows).toHaveLength(1);+ expect(rows[0].keyId).toBe(heuristic);+ });++ it('never probes authoritative, credit, or tier cooldowns', async () => {+ const heuristic = bench('heuristic');+ const authoritative = bench('authoritative');+ const credit = bench('credit');+ const tier = bench('tier');+ const probe = probeReturning('valid');+ const now = Date.now() + 6 * MINUTE; // past the half-bench ripeness gate++ await runCooldownProbePass({ now, probe, jitter: noJitter }); // seed pass+ const result = await runCooldownProbePass({ now, probe, jitter: noJitter });++ expect(result.probedKeyIds).toEqual([heuristic]);+ expect(probe).toHaveBeenCalledTimes(1);+ expect(probe).toHaveBeenCalledWith(heuristic);+ // The provider-stated benches stay exactly where they were.+ expect(isOnCooldown('probefake', 'model-a', authoritative)).toBe(true);+ expect(isOnCooldown('probefake', 'model-a', credit)).toBe(true);+ expect(isOnCooldown('probefake', 'model-a', tier)).toBe(true);+ });++ it('does not probe before half the bench has elapsed', async () => {+ bench('heuristic', { durationMs: 10 * MINUTE });+ const probe = probeReturning('valid');+ const early = Date.now() + 2 * MINUTE; // 20% served — not ripe++ await runCooldownProbePass({ now: early, probe, jitter: noJitter });+ await runCooldownProbePass({ now: early, probe, jitter: noJitter });+ expect(probe).not.toHaveBeenCalled();++ const ripe = Date.now() + 6 * MINUTE; // 60% served — ripe+ await runCooldownProbePass({ now: ripe, probe, jitter: noJitter }); // seed+ const result = await runCooldownProbePass({ now: ripe, probe, jitter: noJitter });+ expect(result.probedKeyIds).toHaveLength(1);+ });++ it('skips cooldowns that are about to expire on their own', async () => {+ bench('heuristic', { durationMs: 10 * MINUTE });+ const probe = probeReturning('valid');+ // 9.5 minutes in: past half-bench, but the remaining 30s is less than a+ // scan interval — the probe would cost more than the idle time it saves.+ const now = Date.now() + 9 * MINUTE + 30_000;++ await runCooldownProbePass({ now, probe, jitter: noJitter });+ await runCooldownProbePass({ now, probe, jitter: noJitter });+ expect(probe).not.toHaveBeenCalled();+ });+});++describe('successful probe', () => {+ it('clears every heuristic cooldown on the key early, with one probe', async () => {+ const keyId = ++keySeq;+ bench('heuristic', { keyId, modelId: 'model-a' });+ bench('heuristic', { keyId, modelId: 'model-b' });+ expect(isOnCooldown('probefake', 'model-a', keyId)).toBe(true);+ expect(isOnCooldown('probefake', 'model-b', keyId)).toBe(true);++ const probe = probeReturning('valid');+ const now = Date.now() + 6 * MINUTE;+ await runCooldownProbePass({ now, probe, jitter: noJitter }); // seed+ const result = await runCooldownProbePass({ now, probe, jitter: noJitter });++ // One validate call is the same evidence for both benched models.+ expect(probe).toHaveBeenCalledTimes(1);+ expect(result.clearedCooldowns).toBe(2);+ expect(isOnCooldown('probefake', 'model-a', keyId)).toBe(false);+ expect(isOnCooldown('probefake', 'model-b', keyId)).toBe(false);+ expect(getProbeableCooldowns(now)).toHaveLength(0);+ });+});++describe('failed probe', () => {+ function expiryOf(keyId: number): number {+ const row = getDb().prepare(+ "SELECT expires_at_ms FROM rate_limit_cooldowns WHERE key_id = ?",+ ).get(keyId) as { expires_at_ms: number };+ return row.expires_at_ms;+ }++ it('leaves the cooldown exactly as it was — never extends it', async () => {+ const keyId = bench('heuristic', { durationMs: 30 * MINUTE });+ const expiryBefore = expiryOf(keyId);+ const probe = probeReturning('invalid');+ const now = Date.now() + 16 * MINUTE;++ await runCooldownProbePass({ now, probe, jitter: noJitter }); // seed+ const result = await runCooldownProbePass({ now, probe, jitter: noJitter });++ expect(result.probedKeyIds).toEqual([keyId]);+ expect(result.clearedCooldowns).toBe(0);+ expect(isOnCooldown('probefake', 'model-a', keyId)).toBe(true);+ expect(expiryOf(keyId)).toBe(expiryBefore);+ });++ it('backs off with increasing intervals between probes', async () => {+ const keyId = bench('heuristic', { durationMs: 60 * MINUTE });+ const probe = probeReturning('invalid');+ const t0 = Date.now() + 31 * MINUTE;++ await runCooldownProbePass({ now: t0, probe, jitter: noJitter }); // seed+ await runCooldownProbePass({ now: t0, probe, jitter: noJitter }); // probe #1 fails+ expect(probe).toHaveBeenCalledTimes(1);++ // Within the first 2-minute backoff: no probe.+ await runCooldownProbePass({ now: t0 + 1 * MINUTE, probe, jitter: noJitter });+ expect(probe).toHaveBeenCalledTimes(1);++ // Backoff served: probe #2 fails, backoff doubles to 4 minutes.+ await runCooldownProbePass({ now: t0 + 2 * MINUTE, probe, jitter: noJitter });+ expect(probe).toHaveBeenCalledTimes(2);++ // 3 minutes after probe #2 — still inside the 4-minute backoff.+ await runCooldownProbePass({ now: t0 + 5 * MINUTE, probe, jitter: noJitter });+ expect(probe).toHaveBeenCalledTimes(2);++ // 4 minutes after probe #2: due again.+ await runCooldownProbePass({ now: t0 + 6 * MINUTE, probe, jitter: noJitter });+ expect(probe).toHaveBeenCalledTimes(3);+ expect(isOnCooldown('probefake', 'model-a', keyId)).toBe(true);+ });++ it('treats an inconclusive transport error like a failure', async () => {+ const keyId = bench('heuristic');+ const probe = probeReturning('error');+ const now = Date.now() + 6 * MINUTE;++ await runCooldownProbePass({ now, probe, jitter: noJitter }); // seed+ const result = await runCooldownProbePass({ now, probe, jitter: noJitter });++ expect(result.clearedCooldowns).toBe(0);+ expect(isOnCooldown('probefake', 'model-a', keyId)).toBe(true);+ });+});++describe('stagger and probe budget', () => {+ it('a first-sighted key gets a jittered schedule instead of an instant probe', async () => {+ bench('heuristic', { durationMs: 60 * MINUTE });+ const probe = probeReturning('valid');+ const maxJitter = () => 0.999; // ~45s stagger+ const t0 = Date.now() + 31 * MINUTE;++ await runCooldownProbePass({ now: t0, probe, jitter: maxJitter }); // seed only+ expect(probe).not.toHaveBeenCalled();++ // Still inside the stagger window.+ await runCooldownProbePass({ now: t0 + 30_000, probe, jitter: maxJitter });+ expect(probe).not.toHaveBeenCalled();++ // Stagger served.+ await runCooldownProbePass({ now: t0 + 46_000, probe, jitter: maxJitter });+ expect(probe).toHaveBeenCalledTimes(1);+ });++ it('caps how many keys one pass may probe', async () => {+ for (let i = 0; i < 5; i++) bench('heuristic');+ const probe = probeReturning('valid');+ const now = Date.now() + 6 * MINUTE;++ await runCooldownProbePass({ now, probe, jitter: noJitter, maxProbes: 2 }); // seed+ const first = await runCooldownProbePass({ now, probe, jitter: noJitter, maxProbes: 2 });+ expect(first.probedKeyIds).toHaveLength(2);++ // The rest are picked up by later passes, not all at once.+ const second = await runCooldownProbePass({ now, probe, jitter: noJitter, maxProbes: 2 });+ expect(second.probedKeyIds).toHaveLength(2);+ expect(probe).toHaveBeenCalledTimes(4);+ });+});++describe('cooldown provenance (cooldownDecisionForError)', () => {+ function fakeRoute(overrides: Partial<RouteResult> = {}): RouteResult {+ const n = ++keySeq;+ return {+ provider: {} as any, modelId: 'fake-model', modelDbId: 910_000 + n, apiKey: 'k',+ keyId: n, platform: 'probefake', displayName: 'Fake Model',+ rpdLimit: null, tpdLimit: null,+ ...overrides,+ };+ }++ it('tags 402 out-of-credits as credit (never probed)', () => {+ const decision = cooldownDecisionForError(fakeRoute(), new Error('402 Payment Required'));+ expect(decision.source).toBe('credit');+ });++ it('tags 403 model-not-on-tier as tier (never probed)', () => {+ const err = Object.assign(new Error('forbidden'), { status: 403 });+ expect(cooldownDecisionForError(fakeRoute(), err).source).toBe('tier');+ });++ it('tags a daily-quota bench as authoritative — the expiry is a reset time, not a guess', () => {+ const err = new Error('You have used up your daily free allocation');+ const decision = cooldownDecisionForError(fakeRoute(), err);+ expect(decision.source).toBe('authoritative');+ expect(decision.durationMs).toBeGreaterThan(0);+ });++ it('tags a plain transient 429 as heuristic (probe-eligible)', () => {+ const decision = cooldownDecisionForError(fakeRoute(), new Error('429 Too Many Requests'));+ expect(decision.source).toBe('heuristic');+ expect(decision.durationMs).toBe(90_000);+ });++ it('a Retry-After that determined the expiry is authoritative; a shorter one is not', () => {+ // Retry-After beyond our 90s transient bench: the provider's word set the+ // expiry, so probing early could never be justified.+ const longRetry = Object.assign(new Error('429 Too Many Requests'), { retryAfterMs: 10 * MINUTE });+ const long = cooldownDecisionForError(fakeRoute(), longRetry);+ expect(long).toEqual({ durationMs: 10 * MINUTE, source: 'authoritative' });++ // Retry-After SHORTER than our bench: everything past the provider's own+ // retry time is our pessimism, so early recovery stays on the table.+ const shortRetry = Object.assign(new Error('429 Too Many Requests'), { retryAfterMs: 1_000 });+ const short = cooldownDecisionForError(fakeRoute(), shortRetry);+ expect(short).toEqual({ durationMs: 90_000, source: 'heuristic' });+ });+});++describe('startCooldownProbe / stopCooldownProbe', () => {+ function makeScheduler() {+ const every: { ms: number; fn: () => void | Promise<void> }[] = [];+ const cancels: ReturnType<typeof vi.fn>[] = [];+ return {+ scheduler: {+ every(ms: number, fn: () => void | Promise<void>) {+ const cancel = vi.fn();+ every.push({ ms, fn });+ cancels.push(cancel);+ return cancel;+ },+ after(_ms: number, _fn: () => void | Promise<void>) {+ return vi.fn();+ },+ },+ every,+ cancels,+ };+ }++ afterEach(() => {+ stopCooldownProbe();+ delete process.env.COOLDOWN_PROBE_DISABLED;+ });++ it('registers one every-minute scan job', () => {+ const { scheduler, every } = makeScheduler();+ startCooldownProbe(scheduler);+ expect(every).toHaveLength(1);+ expect(every[0].ms).toBe(60_000);+ });++ it('is idempotent — double-start registers only one job', () => {+ const { scheduler, every } = makeScheduler();+ startCooldownProbe(scheduler);+ startCooldownProbe(scheduler);+ expect(every).toHaveLength(1);+ });++ it('the kill switch registers nothing', () => {+ process.env.COOLDOWN_PROBE_DISABLED = '1';+ const { scheduler, every } = makeScheduler();+ startCooldownProbe(scheduler);+ expect(every).toHaveLength(0);+ });++ it('stop invokes the cancel handle', () => {+ const { scheduler, cancels } = makeScheduler();+ startCooldownProbe(scheduler);+ stopCooldownProbe();+ expect(cancels[0]).toHaveBeenCalledOnce();+ });++ it('the registered job runs a pass without throwing', async () => {+ const { scheduler, every } = makeScheduler();+ startCooldownProbe(scheduler);+ await expect(every[0].fn()).resolves.toBeUndefined();+ });+});diff --git a/server/src/db/migrate/defaults.ts b/server/src/db/migrate/defaults.tsindex daa9c526..99ac28a4 100644--- a/server/src/db/migrate/defaults.ts+++ b/server/src/db/migrate/defaults.ts@@ -8,6 +8,7 @@ import * as requestClientInfo from '../migrations/20260706_000001_request_clientimport * as customModelToolSupport from '../migrations/20260706_000002_custom_model_tool_support.js';import * as profileChainBackfill from '../migrations/20260714_000001_profile_chain_backfill.js';import * as keyHealthError from '../migrations/20260720_000001_key_health_error.js';+import * as cooldownProbeProvenance from '../migrations/20260726_000001_cooldown_probe_provenance.js';export interface MigrationModule {up(db: Db): void;@@ -28,6 +29,7 @@ export const REQUEST_CLIENT_INFO_FILENAME = '20260706_000001_request_client_infoexport const CUSTOM_MODEL_TOOL_SUPPORT_FILENAME = '20260706_000002_custom_model_tool_support.ts';export const PROFILE_CHAIN_BACKFILL_FILENAME = '20260714_000001_profile_chain_backfill.ts';export const KEY_HEALTH_ERROR_FILENAME = '20260720_000001_key_health_error.ts';+export const COOLDOWN_PROBE_PROVENANCE_FILENAME = '20260726_000001_cooldown_probe_provenance.ts';export const DEFAULT_MIGRATIONS: readonly DefaultMigration[] = [{ filename: LEGACY_BASELINE_FILENAME, module: legacyBaseline },@@ -39,4 +41,5 @@ export const DEFAULT_MIGRATIONS: readonly DefaultMigration[] = [{ filename: CUSTOM_MODEL_TOOL_SUPPORT_FILENAME, module: customModelToolSupport },{ filename: PROFILE_CHAIN_BACKFILL_FILENAME, module: profileChainBackfill },{ filename: KEY_HEALTH_ERROR_FILENAME, module: keyHealthError },+ { filename: COOLDOWN_PROBE_PROVENANCE_FILENAME, module: cooldownProbeProvenance },];diff --git a/server/src/db/migrations/20260726_000001_cooldown_probe_provenance.ts b/server/src/db/migrations/20260726_000001_cooldown_probe_provenance.tsnew file mode 100644index 00000000..049be43f--- /dev/null+++ b/server/src/db/migrations/20260726_000001_cooldown_probe_provenance.ts@@ -0,0 +1,42 @@+// Migration: cooldown provenance for probe-based early recovery+// Created: 2026-07-26+//+// DOWN: reversible+//+// The cooldown-probe recovery job (services/cooldown-probe.ts) may only probe+// cooldowns that are OUR OWN guesses. A cooldown backed by an authoritative+// retry time (explicit Retry-After, daily-quota reset) is a fact the provider+// told us; probing those wastes validate calls and can never legitimately clear+// them early. `source` records that provenance at setCooldown time; `set_at_ms`+// records when the bench started, so the prober can wait out a fraction of the+// bench before the first probe instead of probing a seconds-old cooldown.++import type { Db } from '../types.js';++function hasColumn(db: Db, table: string, column: string): boolean {+ const columns = db.prepare(`PRAGMA table_info(${table})`).all() as { name: string }[];+ return columns.some((candidate) => candidate.name === column);+}++export function up(db: Db): void {+ if (!hasColumn(db, 'rate_limit_cooldowns', 'source')) {+ // Pre-existing rows default to 'heuristic' (probe-eligible): before this+ // migration every cooldown kind was mixed together, and a spurious probe on+ // an old authoritative row costs one cheap validate call at worst.+ db.prepare("ALTER TABLE rate_limit_cooldowns ADD COLUMN source TEXT NOT NULL DEFAULT 'heuristic'").run();+ }+ if (!hasColumn(db, 'rate_limit_cooldowns', 'set_at_ms')) {+ // Nullable: rows written before this migration have an unknown start time,+ // which the prober treats as "old enough to probe".+ db.prepare('ALTER TABLE rate_limit_cooldowns ADD COLUMN set_at_ms INTEGER').run();+ }+}++export function down(db: Db): void {+ if (hasColumn(db, 'rate_limit_cooldowns', 'source')) {+ db.prepare('ALTER TABLE rate_limit_cooldowns DROP COLUMN source').run();+ }+ if (hasColumn(db, 'rate_limit_cooldowns', 'set_at_ms')) {+ db.prepare('ALTER TABLE rate_limit_cooldowns DROP COLUMN set_at_ms').run();+ }+}diff --git a/server/src/index.ts b/server/src/index.tsindex b202c14a..4bfc6fec 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -5,6 +5,7 @@ import { startHealthChecker, checkAllKeys } from './services/health.js';import { applyProxyUrl, applyProxyEnabled, applyProxyBypass, flushProxyCache } from './lib/proxy.js';import { startWakeDetect } from './lib/wake-detect.js';import { startCatalogSync } from './services/catalog-sync.js';+import { startCooldownProbe } from './services/cooldown-probe.js';import { installProcessSafetyNet } from './lib/process-safety-net.js';import { NodeScheduler } from './lib/scheduler.js';import { loadConfig } from './lib/config.js';@@ -60,6 +61,7 @@ async function main() {console.log(`Proxy endpoint: http://${display}:${PORT}/v1/chat/completions`);startHealthChecker(scheduler);startCatalogSync(scheduler);+ startCooldownProbe(scheduler);startDbBackupPump(getDb(), scheduler, config.dbPath ?? undefined);// Post-sleep recovery: while the host was suspended (laptop lid, VMdiff --git a/server/src/lib/error-classify.ts b/server/src/lib/error-classify.tsindex c37e5010..6208ebfb 100644--- a/server/src/lib/error-classify.ts+++ b/server/src/lib/error-classify.ts@@ -38,6 +38,12 @@ export function isRetryableError(err: any): boolean {// provider in the fallback chain may have a larger limit. Same reasoning as 503.|| msg.includes('413') || msg.includes('payload too large') || msg.includes('request body too large')|| msg.includes('request entity too large') || msg.includes('content too large')+ // Context/prompt-too-large in all its provider wordings (Anthropic "prompt+ // is too long", Google "input token count ... exceeds", OpenAI "maximum+ // context length", …): another candidate may have a larger window, so fail+ // over; if EVERY candidate rejects it the exhaustion ladder renders an+ // honest 413 instead of a generic failure.+ || isContextTooLargeError(err)// 404: model deprecated/removed upstream (e.g. OpenRouter's "no endpoints found"// for a model that's been pulled). Rotate to the next model in the chain —// setCooldown + the health checker will avoid this model on subsequent requests.@@ -90,6 +96,43 @@ export function isRetryableError(err: any): boolean {|| msg.includes('unparseable inline tool-call dialect');}+// ── Client-caused aborts ─────────────────────────────────────────────────────+// When the gateway's OWN client hangs up, the proxy surfaces abort the composed+// fetch signal with this marked error, and undici rejects the in-flight fetch /+// body read / stream read with the same object (fetch propagates the signal's+// abort reason). It is deliberately NOT an AbortError and its message contains+// neither "aborted" nor "timeout": enrichAbort (lib/proxy.ts) must pass it+// through untouched, and it must never classify as a retryable provider+// timeout — a vanished client says nothing about provider health, so the+// fallback loop stops without a cooldown, health penalty, or failure stats.+export function newClientAbortError(): Error {+ const err = new Error('client disconnected — upstream request canceled');+ (err as Error & { clientAbort?: boolean }).clientAbort = true;+ return err;+}++/** True when an error is (or wraps) the client-disconnect abort above. The+ * structured marker is the primary signal; `cause` is checked because some+ * transports re-wrap the abort reason, and the message substring is the last+ * resort for errors that were stringified across a boundary. */+export function isClientAbortError(err: any): boolean {+ if (err?.clientAbort === true) return true;+ if (err?.cause && (err.cause as { clientAbort?: boolean }).clientAbort === true) return true;+ const msg = (err?.message ?? '').toLowerCase();+ return msg.includes('client disconnected');+}++/** True for any fetch-abort rejection surfacing out of a body read — the+ * per-attempt timeout ('request'-bounds deadline in fetchWithTimeout), an+ * AbortSignal.timeout, or the client disconnect above. Adapters that wrap+ * res.json() in a diagnostic catch (openai-compat's non-JSON-body split) must+ * rethrow these instead of classifying them as a malformed provider body. */+export function isAbortLikeError(err: any): boolean {+ if (isClientAbortError(err)) return true;+ const name = (err as { name?: string })?.name;+ return name === 'AbortError' || name === 'TimeoutError' || /\baborted\b/i.test(err?.message ?? '');+}+// A 401 / invalid-API-key error from a provider. KEY-fatal, not request-fatal:// the same request is fine on the provider's sibling key or on another provider,// so the fallback loop rotates past the bad key (and triggers an immediate@@ -163,6 +206,47 @@ export function isProviderBadRequestError(err: any): boolean {return msg.includes('api error 400') || msg.includes('api error 422') || msg.includes('unprocessable entity');}+// A "this request/prompt is too large" rejection. Providers word it very+// differently and even disagree on the HTTP status:+// - OpenAI-compat: 400 with code `context_length_exceeded` — "This model's+// maximum context length is 8192 tokens. However, your messages resulted+// in 10001 tokens. Please reduce the length of the messages."+// - Anthropic: 400 invalid_request_error — "prompt is too long: 210000+// tokens > 200000 maximum" (Bedrock words it "Input is too long for+// requested model.")+// - Google/Gemini: 400 INVALID_ARGUMENT — "The input token count (1200000)+// exceeds the maximum number of tokens allowed (1048576)."+// - Groq: a literal HTTP 413 — "Request too large for model `x` ... on+// tokens per minute (TPM): Limit 30000, Requested 33476". Requested >+// Limit can never fit that candidate's window, so it belongs here (too+// large for the candidate), not with transient per-minute 429s.+// - Reverse proxies / gateways: 413 "Payload Too Large" / "request entity+// too large" / "content too large".+// Structured status first (providerHttpError attaches err.status on every+// adapter), then the code field, then message markers — mirroring how the+// other classifiers in this file work. Retryable (a sibling candidate may have+// a larger window), but when EVERY attempt dies this way the exhaustion ladder+// renders an honest 413 instead of a generic failure.+export function isContextTooLargeError(err: any): boolean {+ if (err?.status === 413) return true;+ if (typeof err?.code === 'string' && err.code.toLowerCase() === 'context_length_exceeded') return true;+ const msg = (err?.message ?? '').toLowerCase();+ return msg.includes('context_length_exceeded')+ || msg.includes('maximum context length')+ || msg.includes('context length exceeded')+ || /exceeds[^.]*context (length|window|size)/.test(msg)+ || msg.includes('prompt is too long')+ || msg.includes('input is too long')+ || /input token count[^.]*exceeds/.test(msg)+ || msg.includes('exceeds the maximum number of tokens')+ || msg.includes('request too large')+ || msg.includes('payload too large')+ || msg.includes('request entity too large')+ || msg.includes('request body too large')+ || msg.includes('content too large')+ || msg.includes('api error 413');+}+// A 402 Payment Required / out-of-credits error. Distinct from a transient 429:// it won't recover on the next window, so the caller benches the model+key with// PAYMENT_REQUIRED_COOLDOWN_MS (a full day) rather than the 90s transient cooldown.diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.tsindex 39d3cc33..f9a91033 100644--- a/server/src/lib/fallback-loop.ts+++ b/server/src/lib/fallback-loop.ts@@ -21,21 +21,24 @@ import {recordRequest,recordTokens,setCooldown,- getCooldownDurationForLimit,+ getCooldownDecisionForLimit,getSoonestCooldownExpiry,PAYMENT_REQUIRED_COOLDOWN_MS,MODEL_FORBIDDEN_COOLDOWN_MS,learnLimitFromError,+ type CooldownDecision,} from '../services/ratelimit.js';import {isRetryableError,isKeyAuthError,+ isClientAbortError,isDailyQuotaExhaustedError,isPaymentRequiredError,isModelNotFoundError,isModelAccessForbiddenError,isProviderBadRequestError,isProviderDegradedError,+ isContextTooLargeError,} from './error-classify.js';import { sanitizeProviderErrorMessage } from './error-redaction.js';import { checkKeyHealth, markKeyHealthyFromRequest } from '../services/health.js';@@ -112,10 +115,26 @@ export function msUntilNextUtcMidnight(now = Date.now()): number {* provider's Retry-After as a floor (getCooldownDurationForLimit).*/export function cooldownForError(route: RouteResult, err: any): number {- if (isPaymentRequiredError(err)) return PAYMENT_REQUIRED_COOLDOWN_MS;- if (isModelAccessForbiddenError(err)) return MODEL_FORBIDDEN_COOLDOWN_MS;- if (isDailyQuotaExhaustedError(err)) return err?.retryAfterMs ?? msUntilNextUtcMidnight();- return getCooldownDurationForLimit(+ return cooldownDecisionForError(route, err).durationMs;+}++/**+ * cooldownForError plus the provenance tag the cooldown-probe recovery job+ * keys off (see CooldownSource in services/ratelimit.ts): 402 → 'credit' and+ * 403 → 'tier' (a key-validation probe passing proves nothing about credits or+ * tier, so those are never probed); a daily-quota bench → 'authoritative' (the+ * expiry is the provider's own reset time, whether from Retry-After or the+ * UTC-midnight convention — a fact, not a guess); everything else defers to+ * getCooldownDecisionForLimit, which tags 'authoritative' only when an explicit+ * Retry-After actually determined the expiry.+ */+export function cooldownDecisionForError(route: RouteResult, err: any): CooldownDecision {+ if (isPaymentRequiredError(err)) return { durationMs: PAYMENT_REQUIRED_COOLDOWN_MS, source: 'credit' };+ if (isModelAccessForbiddenError(err)) return { durationMs: MODEL_FORBIDDEN_COOLDOWN_MS, source: 'tier' };+ if (isDailyQuotaExhaustedError(err)) {+ return { durationMs: err?.retryAfterMs ?? msUntilNextUtcMidnight(), source: 'authoritative' };+ }+ return getCooldownDecisionForLimit(route.platform,route.modelId,route.keyId,@@ -152,12 +171,16 @@ export function recordRetryableFailure(route: RouteResult, err: any, state: Fall// state (ignored response_format, JSON truncated at max_tokens): a sibling// key would reproduce it exactly, so rule out the whole model for this// request instead of burning one failover hop per key.- if (isModelNotFoundError(err) || isModelAccessForbiddenError(err) || err?.skipModelForRequest === true) {+ // Context-too-large is MODEL-level too: a sibling key serves the same model+ // with the same context window (and, for Groq-style per-key TPM 413s, the+ // same tier ceiling), so it would reject the same request identically.+ if (isModelNotFoundError(err) || isModelAccessForbiddenError(err) || isContextTooLargeError(err) || err?.skipModelForRequest === true) {state.skipModels.add(route.modelDbId);}state.skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);if (err?.skipBench === true) return;- setCooldown(route.platform, route.modelId, route.keyId, cooldownForError(route, err));+ const decision = cooldownDecisionForError(route, err);+ setCooldown(route.platform, route.modelId, route.keyId, decision.durationMs, decision.source);// Model-level penalty only when no sibling key can still serve (#454).if (!hasOtherUsableKey(route.modelDbId, route.keyId, state.skipKeys)) {recordRateLimitHit(route.modelDbId);@@ -231,6 +254,7 @@ export type AttemptErrorClass =| 'daily_quota_exhausted'| 'model_not_found'| 'forbidden'+ | 'context_too_large'| 'provider_bad_request'| 'empty_completion'| 'format_ignored'@@ -252,6 +276,10 @@ export function classifyAttemptError(err: any): AttemptErrorClass {if (isDailyQuotaExhaustedError(err)) return 'daily_quota_exhausted';if (isModelNotFoundError(err)) return 'model_not_found';if (isModelAccessForbiddenError(err)) return 'forbidden';+ // Before the bad-request check: OpenAI-compat context errors arrive as+ // "API error 400: This model's maximum context length is …", which the+ // generic provider_bad_request rule would otherwise swallow.+ if (isContextTooLargeError(err)) return 'context_too_large';// A DEGRADED-function 400 (NVIDIA NIM, #522) is provider health, not request// shape — keep it out of provider_bad_request so the trail reads honestly.if (isProviderDegradedError(err)) return 'upstream_error';@@ -305,9 +333,47 @@ export interface ExhaustionBody {type: string;message: string;// Coarse class of the exhaustion, for surfaces that need to remap `type` to- // their own wire vocabulary (the Anthropic route maps 'auth' → 'api_error'- // and 'unavailable' → 'overloaded_error').- kind: 'auth' | 'bad_request' | 'rate_limit' | 'unavailable';+ // their own wire vocabulary (the Anthropic route maps 'auth' → 'api_error',+ // 'unavailable' → 'overloaded_error', 'context_too_large' →+ // 'request_too_large', 'model_not_found' → 'not_found_error', 'upstream' →+ // 'api_error').+ kind: 'auth' | 'bad_request' | 'rate_limit' | 'unavailable' | 'context_too_large' | 'model_not_found' | 'upstream';+ // Machine-readable code for the OpenAI-compatible error object.+ code?: string;+ // 429 exhaustions only: epoch ms of the earliest moment any benched candidate+ // becomes available again (soonest cooldown expiry). Rendered in the error+ // body and as a Retry-After header (seconds, ceil) by setExhaustionHeaders.+ retryAtMs?: number;+}++/**+ * The OpenAI-compatible `error` object for an exhaustion body — shared by every+ * OpenAI-shaped surface so the wire shape (message/type/code/retryAtMs) cannot+ * drift between them.+ */+export function exhaustionErrorPayload(body: ExhaustionBody): { message: string; type: string; code?: string; retryAtMs?: number } {+ const payload: { message: string; type: string; code?: string; retryAtMs?: number } = {+ message: body.message,+ type: body.type,+ };+ if (body.code) payload.code = body.code;+ if (body.retryAtMs != null) payload.retryAtMs = body.retryAtMs;+ return payload;+}++/**+ * Stamp the standard retry headers for an exhaustion body: a Retry-After of+ * ceil((retryAtMs - now) / 1000) seconds when the body carries a concrete+ * retry time. Callers must only invoke this before headers are flushed (i.e.+ * never on a committed SSE stream).+ */+export function setExhaustionHeaders(+ res: { setHeader(name: string, value: string): void },+ body: ExhaustionBody,+ now = Date.now(),+): void {+ if (body.retryAtMs == null) return;+ res.setHeader('Retry-After', String(Math.max(0, Math.ceil((body.retryAtMs - now) / 1000))));}export interface ExhaustionContext {@@ -320,18 +386,46 @@ export interface ExhaustionContext {breakerFails?: number;}+// Attempt classes that mean "this candidate is unavailable until a KNOWN time"+// — a rate-limit window, a benched daily allocation, an out-of-credits key+// (day bench), or a tier-forbidden model (day bench). When EVERY attempt is in+// this family the pool recovers by itself, so the honest exhaustion is a 429+// with a concrete retry time. Anything else in the mix (a 5xx, a timeout, a+// vanished model) means waiting is not a promise, so the mixed case renders a+// 502 instead.+const UNAVAILABLE_UNTIL_KNOWN_TIME: ReadonlySet<AttemptErrorClass> = new Set([+ 'rate_limited',+ 'daily_quota_exhausted',+ 'out_of_credits',+ 'forbidden',+]);+/**- * The shared exhaustion response body.+ * The shared exhaustion response body — the single failure-kind → terminal-+ * status ladder every surface renders. Aggregated over the per-attempt failure+ * classes (most-specific first):* - Every attempt failed auth (401/invalid key) → 502 provider_error saying the* PROVIDER keys are bad — distinct from a rate-limit exhaustion, and never* 'authentication_error' (which would wrongly blame the CLIENT's key).+ * - Every attempt died on a context/prompt-too-large rejection → 413: no+ * candidate can fit this request; retrying cannot help, shrinking it can.+ * - Every attempt got a model-not-found/gone from its provider → 404: the+ * model has been removed upstream everywhere we route it. (No 410 — the+ * catalog keeps no removal tombstones, so "verifiably existed before" is+ * not determinable here.)+ * - A chain that died on a DEGRADED-function 400 (NVIDIA NIM, #522) → 503:+ * provider capacity, not a bad request.* - A request every routed provider rejected as invalid → 400* invalid_request_error, not a misleading rate-limit exhaustion.- * - Otherwise → 429 rate_limit_error.- * All bodies carry the attempt trail (what was tried, per attempt) and the- * soonest-cooldown-reset hint that previously only reached clients when routing- * failed before any attempt (summarizeExhaustion) — after one attempt they got a- * terse "Last error" line with none of that context.+ * - Circuit-breaker stop → 503 (the pool looks unhealthy; retry later).+ * - Every attempt unavailable-until-known-time (rate limits, benched+ * quotas/credits/tiers) → 429 rate_limit_error with `retryAtMs` (soonest+ * cooldown expiry) for a matching Retry-After header.+ * - Mixed/other upstream failures (5xx, timeouts, transport errors) → 502+ * provider_error: the UPSTREAMS failed. Never 500 — that status is+ * reserved for our own bugs.+ * All bodies carry the attempt trail (what was tried, per attempt) and, where+ * meaningful, the soonest-cooldown-reset hint.*/export function exhaustedRetryError(lastError: any, maxRetries?: number, ctx?: ExhaustionContext): ExhaustionBody {const safeLastError = sanitizeProviderErrorMessage(lastError?.message);@@ -340,18 +434,48 @@ export function exhaustedRetryError(lastError: any, maxRetries?: number, ctx?: Econst budgetNote = ctx?.timedOut? ` (stopped early: retry time budget ${Math.round((ctx.budgetMs ?? 0) / 1000)}s exceeded)`: '';+ const everyAttempt = (cls: AttemptErrorClass | ReadonlySet<AttemptErrorClass>): boolean =>+ attempts.length > 0 && attempts.every(a => (cls instanceof Set ? cls.has(a.errorClass) : a.errorClass === cls));- if (attempts.length > 0 && attempts.every(a => a.errorClass === 'auth')) {+ if (everyAttempt('auth')) {return {kind: 'auth',status: 502,type: 'provider_error',+ code: 'provider_authentication_failed',message: `All ${attempts.length} attempted provider key(s) failed authentication${budgetNote}. ` +'The configured upstream API key(s) look invalid or expired; they are being revalidated now and will be marked invalid automatically. ' +`Check the provider keys in the dashboard.${trail} Last error: ${safeLastError}`,};}+ // Aggregate diagnoses before the lastError-shape ones below: when EVERY+ // attempt failed the same way, that unanimity is stronger evidence than the+ // shape of whichever error happened to come last.+ if (everyAttempt('context_too_large')) {+ return {+ kind: 'context_too_large',+ status: 413,+ type: 'invalid_request_error',+ code: 'context_length_exceeded',+ message: `The request is too large for every routed candidate: all ${attempts.length} attempt(s) were rejected as ` ++ `over the model's context/size limit${budgetNote}. Retrying will not help — reduce the prompt/history size or ` ++ `enable a larger-context model.${trail} Last error: ${safeLastError}`,+ };+ }++ if (everyAttempt('model_not_found')) {+ return {+ kind: 'model_not_found',+ status: 404,+ type: 'invalid_request_error',+ code: 'model_not_found',+ message: `Every routed provider reports the model as not found or removed upstream (${attempts.length} attempt(s))` ++ `${budgetNote}. The catalog entry looks stale; pick another model or call /v1/models for the available list.` ++ `${trail} Last error: ${safeLastError}`,+ };+ }+// A chain that died on a DEGRADED-function 400 (NVIDIA NIM, #522) is a// provider-capacity outage, not a bad request: render 503 so clients retry// later instead of "fixing" a request that was never broken.@@ -360,6 +484,7 @@ export function exhaustedRetryError(lastError: any, maxRetries?: number, ctx?: Ekind: 'unavailable',status: 503,type: 'service_unavailable',+ code: 'provider_degraded',message: `The routed provider reported the model's hosted deployment as temporarily degraded${budgetNote}. ` +`This is a provider-side condition; retry later or route to another provider.${trail} Last error: ${safeLastError}`,};@@ -370,11 +495,12 @@ export function exhaustedRetryError(lastError: any, maxRetries?: number, ctx?: Ekind: 'bad_request',status: 400,type: 'invalid_request_error',+ code: 'provider_rejected_request',message: `All routed providers rejected the request as invalid${budgetNote}.${trail} Last error: ${safeLastError}`,};}- // Circuit-breaker guardrail stop. Checked after the all-auth and bad-request+ // Circuit-breaker guardrail stop. Checked after the aggregate and bad-request// diagnoses, which are more specific about WHY the pool is failing.if (ctx?.breakerFails) {const breakerEta = formatResetEta(getSoonestCooldownExpiry());@@ -383,6 +509,7 @@ export function exhaustedRetryError(lastError: any, maxRetries?: number, ctx?: Ekind: 'unavailable',status: 503,type: 'service_unavailable',+ code: 'upstream_unhealthy',message: `Failover stopped early by the circuit-breaker guardrail: ${ctx.breakerFails} consecutive upstream ` +`failure${ctx.breakerFails === 1 ? '' : 's'} (max_consecutive_upstream_fails). The enabled pool looks ` +`unhealthy right now, so the remaining candidates were skipped instead of burning quota on them.` +@@ -390,17 +517,116 @@ export function exhaustedRetryError(lastError: any, maxRetries?: number, ctx?: E};}- const attemptCount = attempts.length > 0 ? attempts.length : maxRetries;- const scope = attemptCount == null- ? 'All models rate-limited'- : `All models rate-limited after ${attemptCount} attempt${attemptCount === 1 ? '' : 's'}`;- const eta = formatResetEta(getSoonestCooldownExpiry());- const etaNote = eta ? ` Soonest cooldown reset ${eta}.` : '';+ // 429 only when EVERY candidate is unavailable until a known time (or the+ // caller gave us no per-attempt classes to aggregate — the legacy shape).+ if (attempts.length === 0 || everyAttempt(UNAVAILABLE_UNTIL_KNOWN_TIME)) {+ const attemptCount = attempts.length > 0 ? attempts.length : maxRetries;+ const scope = attemptCount == null+ ? 'All models rate-limited'+ : `All models rate-limited after ${attemptCount} attempt${attemptCount === 1 ? '' : 's'}`;+ const retryAtMs = getSoonestCooldownExpiry() ?? undefined;+ const eta = formatResetEta(retryAtMs);+ const etaNote = eta ? ` Soonest cooldown reset ${eta}.` : '';+ return {+ kind: 'rate_limit',+ status: 429,+ type: 'rate_limit_error',+ code: 'rate_limit_exceeded',+ ...(retryAtMs != null ? { retryAtMs } : {}),+ message: `${scope}${budgetNote}.${etaNote}${trail} Last error: ${safeLastError}`,+ };+ }++ // Mixed/other upstream failures (5xx, timeouts, transport errors, or a mix of+ // those with rate limits): the upstream pool failed us, so say 502 — not 429+ // (which would promise recovery-by-waiting the attempts don't support) and+ // not 500 (which would blame our own code).+ return {+ kind: 'upstream',+ status: 502,+ type: 'provider_error',+ code: 'upstream_failed',+ message: `All ${attempts.length} routed attempt(s) failed with upstream provider errors${budgetNote}. ` ++ `This is a provider-side failure, not a problem with your request; retry, or check provider status.` ++ `${trail} Last error: ${safeLastError}`,+ };+}++// ── Routing exhaustion (zero attempts ran) ───────────────────────────────────+// When routeRequest gives up before ANY upstream was tried, the only evidence+// is its per-candidate diagnostics. Map them onto the same honest-terminal-+// status taxonomy the attempt ladder uses:+// - nothing configured at all (empty chain, or every candidate lacks a+// provider/usable key) → 503: no amount of waiting or request-shrinking+// helps; the operator must add keys.+// - every candidate rejected the request as too big for its context/TPM+// window → 413.+// - at least one candidate is merely rate-limited/on cooldown (and the rest+// are at worst unconfigured/too-small) → 429 with the soonest cooldown+// expiry as retryAtMs.+// - anything else (capability filters like vision/tools, mixed reasons) →+// the router's status (429) with a generic routing_exhausted code.+type RoutingDiagClass = 'config' | 'too_large' | 'time_bound' | 'other';++function classifyRoutingDiagLine(line: string): RoutingDiagClass {+ const l = line.toLowerCase();+ // "< estimated" first: the tpm_limit-too-small line also contains 'tpm',+ // which would otherwise misread as a transient window.+ if (l.includes('< estimated')) return 'too_large';+ if (/no provider registered|no enabled\+healthy key|no usable key|decrypt-error|no-resolved-provider|custom-key-mismatch/.test(l)) return 'config';+ if (/cooldown|rpm|rpd|tpm|tpd|provider-daily-cap|provider-minute-cap|provider-daily-token-cap|key-concurrency/.test(l)) return 'time_bound';+ return 'other';+}++export function routingExhaustionBody(routeErr: any): ExhaustionBody {+ const diag: string[] = Array.isArray(routeErr?.diagnostics) ? routeErr.diagnostics : [];+ const message: string = routeErr?.message ?? 'No model available to route this request';+ const classes = diag.map(classifyRoutingDiagLine);++ if (diag.length === 0 || classes.every(c => c === 'config')) {+ return {+ kind: 'unavailable',+ status: 503,+ type: 'service_unavailable',+ code: 'no_providers_configured',+ message: diag.length === 0+ ? `No models are enabled/configured to serve this request. Add provider API keys and enable models in the dashboard. ${message}`+ : `No candidate model has a configured, usable provider key. Add provider API keys in the dashboard. ${message}`,+ };+ }++ if (classes.every(c => c === 'too_large' || c === 'config') && classes.includes('too_large')) {+ return {+ kind: 'context_too_large',+ status: 413,+ type: 'invalid_request_error',+ code: 'context_length_exceeded',+ message: `The request is too large for every available candidate's context/token window. ` ++ `Reduce the prompt/history size or enable a larger-context model. ${message}`,+ };+ }++ if (classes.some(c => c === 'time_bound') && classes.every(c => c !== 'other')) {+ const retryAtMs = getSoonestCooldownExpiry() ?? undefined;+ return {+ kind: 'rate_limit',+ status: 429,+ type: 'rate_limit_error',+ code: 'rate_limit_exceeded',+ ...(retryAtMs != null ? { retryAtMs } : {}),+ message,+ };+ }++ // Capability filters or mixed reasons: keep the router's verdict (429) but+ // stamp a machine-readable code so clients can tell it from a plain+ // rate-limit exhaustion.return {kind: 'rate_limit',- status: 429,+ status: typeof routeErr?.status === 'number' ? routeErr.status : 429,type: 'rate_limit_error',- message: `${scope}${budgetNote}.${etaNote}${trail} Last error: ${safeLastError}`,+ code: 'routing_exhausted',+ message,};}@@ -435,9 +661,11 @@ export interface FallbackHooks {attemptLog?: AttemptRecord[];// Returns true once the client has hung up. Checked before STARTING each// retry: a chain nobody is waiting for must not keep burning provider- // quota. The in-flight attempt still completes (same boundary as the- // wall-clock budget); stream pumps additionally stop reading upstream on- // disconnect, which cancels the upstream request via reader.cancel().+ // quota. Surfaces additionally thread a client-disconnect AbortSignal into+ // the provider options (CompletionOptions.signal), so the in-flight+ // attempt's fetch/body/stream is canceled the moment the client goes; the+ // resulting client-abort throw stops the loop without any failure+ // bookkeeping (see the isClientAbortError branch below).clientGone?: () => boolean;// Skip state; recordRetryableFailure / recordAuthFailure (called by the loop)// mutate it, and the surface's route() reads it to exclude failed keys/models.@@ -471,11 +699,13 @@ export interface FallbackHooks {onFatal(route: RouteResult, err: any, attempt: number): void;/**- * Render exhaustion when route() threw. `exhaustion` is the shared body when- * at least one attempt ran (render it); null when routing gave up before any- * upstream was tried (render routeErr as a routing error instead).+ * Render exhaustion when route() threw. `exhaustion` is always the shared+ * honest-status body: built from the attempt trail when at least one attempt+ * ran, or from routeErr's routing diagnostics (routingExhaustionBody) when+ * routing gave up before any upstream was tried. `lastError` is null in the+ * zero-attempt case; `routeErr` is passed for logging/diagnostics.*/- onRoutingExhausted(lastError: any, routeErr: any, exhaustion: ExhaustionBody | null, info: ExhaustionInfo): void;+ onRoutingExhausted(lastError: any, routeErr: any, exhaustion: ExhaustionBody, info: ExhaustionInfo): void;/** Render exhaustion after the attempt cap or the time budget was hit. */onExhausted(exhaustion: ExhaustionBody, info: ExhaustionInfo): void;@@ -555,7 +785,7 @@ export async function runFallbackLoop(hooks: FallbackHooks): Promise<void> {} catch (routeErr) {const exhaustion = lastError? exhaustedRetryError(lastError, undefined, { attempts })- : null;+ : routingExhaustionBody(routeErr);hooks.onRoutingExhausted(lastError, routeErr, exhaustion, { attempts, timedOut: false });return;}@@ -571,6 +801,16 @@ export async function runFallbackLoop(hooks: FallbackHooks): Promise<void> {try {outcome = await hooks.dispatch(route, attempt);} catch (err: any) {+ // Client-caused abort: the composed fetch signal fired because OUR+ // client hung up mid-attempt (see newClientAbortError). Not a+ // provider-health signal — no cooldown, no penalty, no failure stats,+ // no logFailure 'error' row — and no further attempts either: nobody is+ // waiting, and there is no socket to render anything to. The finally+ // below still frees the in-flight lease.+ if (isClientAbortError(err)) {+ console.log(`[FallbackLoop] client disconnected mid-attempt on ${route.platform}/${route.modelId} — upstream canceled, stopping without benching`);+ return;+ }hooks.logFailure(route, err, attempt);if (isKeyAuthError(err)) {// KEY-fatal, not request-fatal: rotate past the bad key and revalidatediff --git a/server/src/lib/proxy.ts b/server/src/lib/proxy.tsindex d6f360e5..ce2d208c 100644--- a/server/src/lib/proxy.ts+++ b/server/src/lib/proxy.ts@@ -237,6 +237,17 @@ function socksFetch(const signal = init?.signal;const startedAt = Date.now();+ // What to reject with when the signal fires. A client-caused abort carries+ // its own marked reason (newClientAbortError in lib/error-classify.ts) —+ // preserve it so the failure isn't misclassified downstream as a provider+ // timeout; a plain timer abort keeps the tagged AbortError.+ const abortRejection = (): Error => {+ const reason = signal?.reason;+ return reason instanceof Error && reason.name !== 'AbortError' && reason.name !== 'TimeoutError'+ ? reason+ : abortError(platform, type, timeoutMs, Date.now() - startedAt);+ };+return new Promise((resolve, reject) => {const req = transport.request({hostname: url.hostname,@@ -251,7 +262,7 @@ function socksFetch(}, (res) => {if (signal?.aborted) {res.destroy();- reject(abortError(platform, type, timeoutMs, Date.now() - startedAt));+ reject(abortRejection());return;}@@ -286,12 +297,12 @@ function socksFetch(if (signal) {if (signal.aborted) {req.destroy();- reject(abortError(platform, type, timeoutMs, Date.now() - startedAt));+ reject(abortRejection());return;}signal.addEventListener('abort', () => {req.destroy();- reject(abortError(platform, type, timeoutMs, Date.now() - startedAt));+ reject(abortRejection());}, { once: true });}diff --git a/server/src/providers/aihorde.ts b/server/src/providers/aihorde.tsindex 2f9c6796..196ee34a 100644--- a/server/src/providers/aihorde.ts+++ b/server/src/providers/aihorde.ts@@ -139,7 +139,9 @@ export class AIHordeProvider extends BaseProvider {'Content-Type': 'application/json',},body: JSON.stringify(this.buildBody(messages, modelId, options)),- }, options?.timeoutMs ?? HORDE_TIMEOUT_MS);+ // 'request' bounds: the queued generation is one blocking body read, so+ // the deadline must cover it too (a hung body used to stall forever).+ }, options?.timeoutMs ?? HORDE_TIMEOUT_MS, { signal: options?.signal, timeoutBounds: 'request' });recordQuotaObservationsFromResponse(res, {platform: this.platform,@@ -201,7 +203,7 @@ export class AIHordeProvider extends BaseProvider {const res = await this.fetchWithTimeout(`${this.baseUrl}/models`, {method: 'GET',headers: { 'Authorization': `Bearer ${this.resolveBearer(apiKey)}` },- }, 30000);+ }, 30000, { timeoutBounds: 'request' });recordQuotaObservationsFromResponse(res, {platform: this.platform,keyId: quotaContext?.keyId,diff --git a/server/src/providers/base.ts b/server/src/providers/base.tsindex 99e1b4de..4fc0adbe 100644--- a/server/src/providers/base.ts+++ b/server/src/providers/base.ts@@ -66,6 +66,31 @@ export interface CompletionOptions extends ExtendedSamplingOptions {* stripped before the request body is built); used by the probe script so* NVIDIA's 15-60s serverless cold starts don't read as failures. */timeoutMs?: number;+ /** Abort signal for the gateway's OWN client (request socket closed). The+ * proxy surfaces thread it here so a disconnect cancels the upstream fetch+ * AND any in-progress body/stream read — tokens stop burning and the+ * in-flight lease frees immediately instead of when the read happens to+ * finish. Composed with the per-attempt timeout in fetchWithTimeout; never+ * serialized into the request body. */+ signal?: AbortSignal;+}++/** Per-call abort/timeout wiring for fetchWithTimeout. */+export interface ProviderFetchOptions {+ /** Client-disconnect signal, composed with the per-attempt timeout via+ * AbortSignal.any. Unlike the timeout it is never disarmed at headers, so it+ * also aborts body reads and stream iteration through undici. */+ signal?: AbortSignal;+ /** What the per-attempt timeout bounds:+ * - 'headers' (default, historical): the abort timer dies the moment+ * response HEADERS arrive — right for streams, whose body legitimately+ * outlives any fixed deadline (readSseStream's stall watchdog owns+ * mid-stream hangs, the client signal owns "nobody is listening").+ * - 'request': the deadline stays armed across the body read too, so a 200+ * whose body never finishes aborts at timeoutMs instead of hanging+ * res.json() forever. Use for non-streaming calls, which read the whole+ * body as one unit. */+ timeoutBounds?: 'headers' | 'request';}export interface KeyValidationFailure {@@ -140,20 +165,48 @@ export abstract class BaseProvider {// Adapters that don't pass a timeout inherit the platform's env override// (PROVIDER_TIMEOUT_<PLATFORM>, issue #547) over the historical 15s.timeoutMs = providerTimeoutMs(this.platform, 15000),+ fetchOpts?: ProviderFetchOptions,): Promise<Response> {+ const signals: AbortSignal[] = [];+ if (fetchOpts?.signal) signals.push(fetchOpts.signal);+// timeoutMs <= 0 means "no timeout" (PROVIDER_TIMEOUT_<PLATFORM>=0).// setTimeout(abort, 0) would abort every request on the next macrotask.- if (timeoutMs <= 0) {- return proxyFetch(url, init, this.platform, 'chat', timeoutMs);+ let headerTimer: ReturnType<typeof setTimeout> | undefined;+ if (timeoutMs > 0) {+ const controller = new AbortController();+ const timer = setTimeout(() => controller.abort(), timeoutMs);+ signals.push(controller.signal);+ if (fetchOpts?.timeoutBounds === 'request') {+ // Whole-request deadline: stays armed across the body read, so a 200+ // whose body never finishes aborts res.json() at timeoutMs instead of+ // hanging forever. Never cleared — firing after the response is fully+ // consumed is a no-op — and unref'd so a spent request's leftover+ // deadline can't hold the process open.+ timer.unref?.();+ } else {+ // Historical header-only deadline: disarmed the moment response+ // HEADERS arrive (see finally). Streams need this — an SSE body+ // legitimately outlives any fixed deadline, and mid-stream hangs are+ // owned by the stall watchdog (readSseStream) + the client signal.+ headerTimer = timer;+ }}- const controller = new AbortController();- const timeout = setTimeout(() => controller.abort(), timeoutMs);++ // Compose the per-attempt timeout with the client-disconnect signal.+ // Unlike the timeout, the client signal is never disarmed: undici ties it+ // to the whole request, so a disconnect also aborts body reads and stream+ // iteration, rejecting them with the marked reason from newClientAbortError.+ const signal = signals.length === 0+ ? init.signal ?? undefined+ : signals.length === 1 ? signals[0] : AbortSignal.any(signals);+try {// requestType='chat' + timeoutMs makes the AbortError message read// `<platform>, chat, 15s` for triage from the requests.error column.- return await proxyFetch(url, { ...init, signal: controller.signal }, this.platform, 'chat', timeoutMs);+ return await proxyFetch(url, signal ? { ...init, signal } : init, this.platform, 'chat', timeoutMs);} finally {- clearTimeout(timeout);+ if (headerTimer !== undefined) clearTimeout(headerTimer);}}@@ -161,6 +214,32 @@ export abstract class BaseProvider {return `chatcmpl-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;}+ /**+ * One reader.read() bounded by the mid-stream inactivity watchdog (#553).+ * Shared by readSseStream and adapters that parse their own wire format+ * (google.ts) so every stream gets the same stall semantics:+ * PROVIDER_STREAM_STALL_TIMEOUT_MS, default 90s, 0 disables. Client-abort+ * rejections pass through untouched — undici errors the body with the+ * request signal's reason, so a disconnect surfaces here as the marked+ * error from newClientAbortError, not as a stall.+ */+ protected async readWithStallTimeout<T>(+ read: () => Promise<T>,+ inactivityTimeoutMs: number,+ ): Promise<T> {+ if (inactivityTimeoutMs <= 0) return read();+ let timer: ReturnType<typeof setTimeout> | undefined;+ return Promise.race([+ read(),+ new Promise<never>((_, reject) => {+ timer = setTimeout(+ () => reject(new Error(`${this.name} stream stalled: no data for ${inactivityTimeoutMs}ms (timeout)`)),+ inactivityTimeoutMs,+ );+ }),+ ]).finally(() => clearTimeout(timer));+ }+/*** Shared SSE reader for OpenAI-wire streaming endpoints (#231 audit).*@@ -195,20 +274,7 @@ export abstract class BaseProvider {try {while (true) {- let timer: ReturnType<typeof setTimeout> | undefined;- const result = inactivityTimeoutMs <= 0- ? await reader.read()- : await Promise.race([- reader.read(),- new Promise<never>((_, reject) => {- timer = setTimeout(- () => reject(new Error(`${this.name} stream stalled: no data for ${inactivityTimeoutMs}ms (timeout)`)),- inactivityTimeoutMs,- );- }),- ]).finally(() => clearTimeout(timer));-- const { done, value } = result;+ const { done, value } = await this.readWithStallTimeout(() => reader.read(), inactivityTimeoutMs);if (done) break;buffer += decoder.decode(value, { stream: true });diff --git a/server/src/providers/cloudflare.ts b/server/src/providers/cloudflare.tsindex 65eed370..d48daf8a 100644--- a/server/src/providers/cloudflare.ts+++ b/server/src/providers/cloudflare.ts@@ -76,7 +76,9 @@ export class CloudflareProvider extends BaseProvider {parallel_tool_calls: options?.parallel_tool_calls,...extendedBodyParams(this.platform, options),}),- }, this.timeoutFor(modelId, options?.timeoutMs));+ // 'request' bounds: the deadline covers the body read too, so a 200+ // whose body hangs aborts instead of stalling res.json() forever.+ }, this.timeoutFor(modelId, options?.timeoutMs), { signal: options?.signal, timeoutBounds: 'request' });recordQuotaObservationsFromResponse(res, {platform: this.platform,keyId: quotaContext?.keyId,@@ -125,7 +127,9 @@ export class CloudflareProvider extends BaseProvider {...extendedBodyParams(this.platform, options),stream: true,}),- }, this.timeoutFor(modelId, options?.timeoutMs));+ // Default 'headers' bounds: the deadline dies at response headers, and+ // the client signal + stall watchdog own the stream from there.+ }, this.timeoutFor(modelId, options?.timeoutMs), { signal: options?.signal });recordQuotaObservationsFromResponse(res, {platform: this.platform,keyId: quotaContext?.keyId,@@ -177,6 +181,7 @@ export class CloudflareProvider extends BaseProvider {url,{ method: 'GET', headers: { 'Authorization': `Bearer ${token}` } },10000,+ { timeoutBounds: 'request' },);recordQuotaObservationsFromResponse(res, {platform: this.platform,diff --git a/server/src/providers/cohere.ts b/server/src/providers/cohere.tsindex 074d54aa..9e1ef29c 100644--- a/server/src/providers/cohere.ts+++ b/server/src/providers/cohere.ts@@ -59,7 +59,9 @@ export class CohereProvider extends BaseProvider {'Content-Type': 'application/json',},body: JSON.stringify(body),- });+ // 'request' bounds: the deadline covers the body read too, so a 200+ // whose body hangs aborts instead of stalling res.json() forever.+ }, undefined, { signal: options?.signal, timeoutBounds: 'request' });recordQuotaObservationsFromResponse(res, {platform: this.platform,keyId: quotaContext?.keyId,@@ -106,7 +108,9 @@ export class CohereProvider extends BaseProvider {'Content-Type': 'application/json',},body: JSON.stringify(body),- });+ // Default 'headers' bounds: the deadline dies at response headers, and+ // the client signal + stall watchdog own the stream from there.+ }, undefined, { signal: options?.signal });recordQuotaObservationsFromResponse(res, {platform: this.platform,keyId: quotaContext?.keyId,@@ -130,7 +134,7 @@ export class CohereProvider extends BaseProvider {const res = await this.fetchWithTimeout(`${API_BASE}/models`, {method: 'GET',headers: { 'Authorization': `Bearer ${apiKey}` },- }, 10000);+ }, 10000, { timeoutBounds: 'request' });recordQuotaObservationsFromResponse(res, {platform: this.platform,keyId: quotaContext?.keyId,diff --git a/server/src/providers/google.ts b/server/src/providers/google.tsindex 09d992bc..46664332 100644--- a/server/src/providers/google.ts+++ b/server/src/providers/google.ts@@ -11,7 +11,7 @@ import { BaseProvider, providerHttpError, type CompletionOptions, type KeyValidaimport { contentToString } from '../lib/content.js';import { proxyFetch } from '../lib/proxy.js';import { recordQuotaObservationsFromResponse, type QuotaObservationContext } from '../services/provider-quota.js';-import { providerTimeoutMs } from '../lib/provider-timeout.js';+import { providerTimeoutMs, streamStallTimeoutMs } from '../lib/provider-timeout.js';const API_BASE = 'https://generativelanguage.googleapis.com/v1beta';@@ -573,7 +573,9 @@ export class GoogleProvider extends BaseProvider {method: 'POST',headers: { 'Content-Type': 'application/json', 'x-goog-api-key': apiKey },body: JSON.stringify(body),- }, options?.timeoutMs ?? this.timeoutMs);+ // 'request' bounds: the deadline covers the body read too, so a 200+ // whose body hangs aborts instead of stalling res.json() forever.+ }, options?.timeoutMs ?? this.timeoutMs, { signal: options?.signal, timeoutBounds: 'request' });recordQuotaObservationsFromResponse(res, {platform: this.platform,@@ -653,7 +655,9 @@ export class GoogleProvider extends BaseProvider {method: 'POST',headers: { 'Content-Type': 'application/json', 'x-goog-api-key': apiKey },body: JSON.stringify(body),- }, options?.timeoutMs ?? this.timeoutMs);+ // Default 'headers' bounds: the deadline dies at response headers, and+ // the client signal + stall watchdog own the stream from there.+ }, options?.timeoutMs ?? this.timeoutMs, { signal: options?.signal });recordQuotaObservationsFromResponse(res, {platform: this.platform,@@ -680,9 +684,14 @@ export class GoogleProvider extends BaseProvider {const seenToolCallKeys = new Set<string>();+ // Same mid-stream inactivity watchdog as readSseStream (#553): this adapter+ // parses Gemini's own frame format, so it reads the body itself and used to+ // have no bound at all on a stalled read.+ const inactivityTimeoutMs = streamStallTimeoutMs();+try {while (true) {- const { done, value } = await reader.read();+ const { done, value } = await this.readWithStallTimeout(() => reader.read(), inactivityTimeoutMs);if (done) break;buffer += decoder.decode(value, { stream: true });@@ -800,6 +809,7 @@ export class GoogleProvider extends BaseProvider {`${API_BASE}/models`,{ method: 'GET', headers: { 'x-goog-api-key': apiKey } },10000,+ { timeoutBounds: 'request' },);recordQuotaObservationsFromResponse(res, {platform: this.platform,diff --git a/server/src/providers/openai-compat.ts b/server/src/providers/openai-compat.tsindex 56e5ca53..d0af2bb9 100644--- a/server/src/providers/openai-compat.ts+++ b/server/src/providers/openai-compat.ts@@ -11,6 +11,7 @@ import { rescueInlineToolCalls } from '../lib/tool-call-rescue.js';import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';import { recordQuotaObservationsFromResponse, type QuotaObservationContext } from '../services/provider-quota.js';import { providerTimeoutMs } from '../lib/provider-timeout.js';+import { isAbortLikeError } from '../lib/error-classify.js';/*** Generic provider for platforms that use an OpenAI-compatible API.@@ -193,7 +194,9 @@ export class OpenAICompatProvider extends BaseProvider {parallel_tool_calls: this.resolveParallelToolCalls(options),...extendedBodyParams(this.platform, options),}),- }, options?.timeoutMs ?? this.timeoutMs);+ // 'request' bounds: the deadline covers the body read too, so a 200+ // whose body hangs aborts instead of stalling res.json() forever.+ }, options?.timeoutMs ?? this.timeoutMs, { signal: options?.signal, timeoutBounds: 'request' });recordQuotaObservationsFromResponse(res, {platform: this.platform,@@ -228,6 +231,10 @@ export class OpenAICompatProvider extends BaseProvider {try {data = await res.json() as ChatCompletionResponse;} catch (err) {+ // An aborted body read (per-attempt deadline, client disconnect) is not+ // a malformed body — rethrow so it keeps its abort classification+ // instead of reading as a "non-OpenAI-compatible endpoint" below.+ if (isAbortLikeError(err)) throw err;parseErr = err;data = undefined as unknown as ChatCompletionResponse;}@@ -307,7 +314,9 @@ export class OpenAICompatProvider extends BaseProvider {...extendedBodyParams(this.platform, options),stream: true,}),- }, options?.timeoutMs ?? this.timeoutMs);+ // Default 'headers' bounds: the deadline dies at response headers, and+ // the client signal + stall watchdog own the stream from there.+ }, options?.timeoutMs ?? this.timeoutMs, { signal: options?.signal });recordQuotaObservationsFromResponse(res, {platform: this.platform,@@ -351,7 +360,9 @@ export class OpenAICompatProvider extends BaseProvider {...this.authHeader(apiKey),...this.extraHeaders,},- }, 30000);+ // 'request' bounds: a catalog body that hangs mid-transfer must not+ // stall the health cycle past the deadline.+ }, 30000, { timeoutBounds: 'request' });recordQuotaObservationsFromResponse(res, {platform: this.platform,keyId: quotaContext?.keyId,diff --git a/server/src/routes/anthropic.ts b/server/src/routes/anthropic.tsindex 3abd4ec4..2b4b5f9c 100644--- a/server/src/routes/anthropic.ts+++ b/server/src/routes/anthropic.ts@@ -15,9 +15,10 @@ import { contentToString } from '../lib/content.js';import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';import { rescueInlineToolCalls, startsWithDialectMarker, couldBecomeDialectMarker, containsDialectMarker } from '../lib/tool-call-rescue.js';import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';+import { isClientAbortError, newClientAbortError } from '../lib/error-classify.js';import { logRequest } from '../lib/request-log.js';import { extractApiToken, timingSafeStringEqual, getStickyModel, setStickyModel } from './proxy.js';-import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, type ExhaustionBody, setFallbackHeaders, type AttemptRecord } from '../lib/fallback-loop.js';+import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, type ExhaustionBody, setFallbackHeaders, setExhaustionHeaders, type AttemptRecord } from '../lib/fallback-loop.js';import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';import { resolveAnthropicModel } from '../services/anthropic-map.js';import { buildModelListing } from '../services/model-listing.js';@@ -113,17 +114,36 @@ function sendError(res: Response, status: number, errorType: string, message: st}// The shared exhaustion body's `type` strings are chosen to be valid on the-// OpenAI-shaped surfaces; the all-keys-failed-auth exhaustion carries-// 'provider_error' and the circuit-breaker stop carries 'service_unavailable',-// neither of which is an Anthropic error type. Remap them onto Anthropic's-// vocabulary ('api_error' / 'overloaded_error') so this surface stays-// wire-correct.+// OpenAI-shaped surfaces; several of them are not Anthropic error types. Remap+// them onto Anthropic's vocabulary via the body's coarse `kind` so this+// surface stays wire-correct: auth/upstream failures → 'api_error', pool+// unavailability → 'overloaded_error', context exhaustion → Anthropic's 413+// 'request_too_large', model-gone → 'not_found_error'.function anthropicErrorType(body: ExhaustionBody): string {- if (body.kind === 'auth') return 'api_error';+ if (body.kind === 'auth' || body.kind === 'upstream') return 'api_error';if (body.kind === 'unavailable') return 'overloaded_error';+ if (body.kind === 'context_too_large') return 'request_too_large';+ if (body.kind === 'model_not_found') return 'not_found_error';return body.type;}+// Render an exhaustion body on the Anthropic wire shape, carrying the+// machine-readable extras (`code`, `retryAtMs`) alongside Anthropic's standard+// type/message and stamping the matching Retry-After header. Extra keys on the+// error object are ignored by Anthropic SDKs, so this stays wire-compatible.+function sendExhaustion(res: Response, body: ExhaustionBody): void {+ setExhaustionHeaders(res, body);+ res.status(body.status).json({+ type: 'error',+ error: {+ type: anthropicErrorType(body),+ message: body.message,+ ...(body.code ? { code: body.code } : {}),+ ...(body.retryAtMs != null ? { retryAtMs: body.retryAtMs } : {}),+ },+ });+}+function newMessageId(): string {return `msg_${crypto.randomBytes(12).toString('hex')}`;}@@ -422,8 +442,21 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {// inline tool-call dialect rescue that the OpenAI/Responses surfaces carry.const state = newFallbackState();const attemptLog: AttemptRecord[] = [];+ // Client-disconnect fan-out: the flag stops the loop before the NEXT+ // attempt; the AbortController (threaded to the provider as+ // CompletionOptions.signal) additionally cancels the IN-FLIGHT upstream+ // fetch and any body/stream read, so tokens stop burning and the in-flight+ // lease frees immediately. 'close' also fires on normal completion —+ // writableEnded distinguishes a real disconnect.let clientGone = false;- res.on('close', () => { if (!res.writableEnded) clientGone = true; });+ const clientAbort = new AbortController();+ res.on('close', () => {+ if (!res.writableEnded) {+ clientGone = true;+ clientAbort.abort(newClientAbortError());+ }+ });+ const dispatchOptions = { ...completionOptions, signal: clientAbort.signal };await runFallbackLoop({maxRetries: MAX_RETRIES,@@ -434,7 +467,7 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {dispatch: async (route, attempt) => {if (stream) {try {- await streamCompletion(res, route, messages, completionOptions, {+ await streamCompletion(res, route, messages, dispatchOptions, {start, attempt, attemptLog, clientGone: () => clientGone, requestedModel, estimatedInputTokens, tools, pinnedModelId,sessionId, pinned: resolved.pinned,});@@ -447,7 +480,7 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {}}- const result = await route.provider.chatCompletion(route.apiKey, messages, route.modelId, completionOptions);+ const result = await route.provider.chatCompletion(route.apiKey, messages, route.modelId, dispatchOptions);const respMsg = result.choices?.[0]?.message;const respText = contentToString(respMsg?.content ?? '');let respToolCalls = respMsg?.tool_calls ?? [];@@ -526,16 +559,12 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {sendError(res, 502, 'api_error', `Provider error (${route.displayName}): ${sanitizeProviderErrorMessage(err.message)}`);},onRoutingExhausted: (lastError, routeErr, exhaustion, info) => {- if (exhaustion) {- setFallbackHeaders(res, info.attempts.length, info.attempts);- sendError(res, exhaustion.status, anthropicErrorType(exhaustion), exhaustion.message);- } else {- sendError(res, routeErr?.status ?? 503, 'api_error', routeErr?.message ?? 'No model available to route this request');- }+ setFallbackHeaders(res, info.attempts.length, info.attempts);+ sendExhaustion(res, exhaustion);},onExhausted: (exhaustion, info) => {setFallbackHeaders(res, info.attempts.length, info.attempts);- sendError(res, exhaustion.status, anthropicErrorType(exhaustion), exhaustion.message);+ sendExhaustion(res, exhaustion);},});});@@ -771,6 +800,12 @@ async function streamCompletion(logRequest(route.platform, route.modelId, route.keyId, 'success', ctx.estimatedInputTokens, outputTokens, Date.now() - ctx.start, null, null, ctx.pinnedModelId);} catch (err: any) {if (err instanceof StreamAlreadyStarted) throw err;+ // Client abort mid-stream: the pump's own `if (ctx.clientGone()) break`+ // can lose the race against the fetch-signal rejection, so the abort may+ // surface here instead. Rethrow — the shared loop's client-abort branch+ // stops the ladder without benching or an error log row (the socket is+ // gone; nothing to render).+ if (isClientAbortError(err)) throw err;if (messageStarted) {// Real payload already reached the client — finish the SSE response// honestly instead of leaving Claude Code hanging, and stop the retry loop.diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex f68ed10e..9f839b6a 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -14,10 +14,10 @@ import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';import { rescueInlineToolCalls, startsWithDialectMarker, couldBecomeDialectMarker, containsDialectMarker } from '../lib/tool-call-rescue.js';import { getContextHandoffMode, recordIncomingMessages, maybeInjectContextHandoff, recordSuccessfulModel, hasPriorModel, HANDOFF_MAX_TOKENS } from '../services/context-handoff.js';import { isFusionModel, runFusion, fusionConfigSchema, FusionError, FUSION_MODEL_ID } from '../services/fusion.js';-import { isRetryableError, isPaymentRequiredError, isModelNotFoundError, isModelAccessForbiddenError } from '../lib/error-classify.js';+import { isRetryableError, isPaymentRequiredError, isModelNotFoundError, isModelAccessForbiddenError, isClientAbortError, newClientAbortError } from '../lib/error-classify.js';import { logRequest } from '../lib/request-log.js';import { parseCacheDirective, cacheActive, isCacheableTemperature, computeCacheKey, getCachedResponse, storeCachedResponse } from '../services/cache.js';-import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, exhaustedRetryError, setFallbackHeaders, type AttemptRecord } from '../lib/fallback-loop.js';+import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, exhaustedRetryError, setFallbackHeaders, exhaustionErrorPayload, setExhaustionHeaders, type AttemptRecord } from '../lib/fallback-loop.js';import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';import { samplingParamSchemaFields, pickSamplingParams, supportedParametersForPlatforms } from '../lib/sampling-params.js';import { enforceJsonContent } from '../lib/structured-output.js';@@ -693,14 +693,26 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {if (groupChain.length === 0) {const placeholders = members.map(() => '?').join(',');const anyEnabled = db.prepare(`SELECT 1 FROM models WHERE id IN (${placeholders}) AND enabled = 1 LIMIT 1`).get(...members);- const reason = anyEnabled ? 'has no providers with an enabled key' : 'is disabled';- res.status(400).json({- error: {- message: `Model '${requestedModel}' ${reason}. Use 'auto' (or omit the 'model' field) to auto-route, or call /v1/models for the available list.`,- type: 'invalid_request_error',- code: 'model_not_found',- },- });+ // Honest statuses: a model whose providers exist but have no usable key+ // is a server-side configuration gap (503), not a client mistake; a+ // disabled/unknown model is a 404 model_not_found (OpenAI semantics).+ if (anyEnabled) {+ res.status(503).json({+ error: {+ message: `Model '${requestedModel}' has no providers with an enabled key. Add a provider API key for it, use 'auto' (or omit the 'model' field) to auto-route, or call /v1/models for the available list.`,+ type: 'service_unavailable',+ code: 'no_providers_configured',+ },+ });+ } else {+ res.status(404).json({+ error: {+ message: `Model '${requestedModel}' is disabled. Use 'auto' (or omit the 'model' field) to auto-route, or call /v1/models for the available list.`,+ type: 'invalid_request_error',+ code: 'model_not_found',+ },+ });+ }return;}} else {@@ -710,7 +722,7 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {} else {const disabled = db.prepare('SELECT id FROM models WHERE model_id = ?').get(requestedModel) as { id: number } | undefined;const reason = disabled ? 'is disabled' : 'is not in the catalog';- res.status(400).json({+ res.status(404).json({error: {message: `Model '${requestedModel}' ${reason}. Use 'auto' (or omit the 'model' field) to auto-route, or call /v1/models for the available list.`,type: 'invalid_request_error',@@ -725,8 +737,20 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {const pinnedModelId = requestedModel && !isAutoModel(requestedModel) ? requestedModel : null;const state = newFallbackState();const attemptLog: AttemptRecord[] = [];+ // Client-disconnect fan-out: the flag stops the loop before the NEXT+ // attempt; the AbortController (threaded to the provider as+ // CompletionOptions.signal) additionally cancels the IN-FLIGHT upstream+ // fetch and any body/stream read, so tokens stop burning and the in-flight+ // lease frees immediately. 'close' also fires on normal completion —+ // writableEnded distinguishes a real disconnect.let clientGone = false;- res.on('close', () => { if (!res.writableEnded) clientGone = true; });+ const clientAbort = new AbortController();+ res.on('close', () => {+ if (!res.writableEnded) {+ clientGone = true;+ clientAbort.abort(newClientAbortError());+ }+ });// Legacy /completions is a thin adapter over the shared fallback loop// (lib/fallback-loop.ts): the cooldown/skip/penalty/exhaustion machinery is@@ -781,7 +805,7 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {route.apiKey,messages,route.modelId,- { temperature, max_tokens, top_p, stop },+ { temperature, max_tokens, top_p, stop, signal: clientAbort.signal },quotaContextForRoute(route, 'chat/completions'),);@@ -841,6 +865,12 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens, totalOutputTokens, Date.now() - start, null, ttfbMs, pinnedModelId);return 'done';} catch (streamErr: any) {+ // Client abort mid-stream: the pump's own `if (clientGone) break`+ // can lose the race against the fetch-signal rejection, so the+ // abort may surface here instead. Rethrow — the shared loop's+ // client-abort branch stops the ladder without benching or an+ // error log row (the socket is gone; nothing to render).+ if (isClientAbortError(streamErr)) throw streamErr;if (headerSent) {console.error(`[Proxy] Mid-stream legacy completion error from ${route.displayName}:`, streamErr.message);const payload = { error: { message: `Provider error (${route.displayName}): stream interrupted`, type: 'stream_error' } };@@ -866,7 +896,7 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {route.apiKey,messages,route.modelId,- { temperature, max_tokens, top_p, stop },+ { temperature, max_tokens, top_p, stop, signal: clientAbort.signal },quotaContextForRoute(route, 'chat/completions'),);@@ -941,22 +971,24 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {});},onRoutingExhausted: (lastError, routeErr, exhaustion, info) => {- if (exhaustion) {- setFallbackHeaders(res, info.attempts.length, info.attempts);- res.status(exhaustion.status).json({ error: { message: exhaustion.message, type: exhaustion.type } });- } else {+ if (!lastError) {+ // Synchronous exhaustion: the router rejected every candidate before+ // any upstream was tried — log the per-candidate disposition.const disposition: string[] = Array.isArray(routeErr.diagnostics) ? routeErr.diagnostics : [];console.warn(`[Proxy] legacy completions routing exhausted (no upstream tried) req=${shortRequestId(requestGroupId)} ` +`requested=${requestedModelLabel} candidates=${disposition.length}` +(disposition.length ? `:\n ${disposition.join('\n ')}` : ''),);- res.status(routeErr.status ?? 503).json({ error: { message: routeErr.message, type: 'routing_error' } });}+ setFallbackHeaders(res, info.attempts.length, info.attempts);+ setExhaustionHeaders(res, exhaustion);+ res.status(exhaustion.status).json({ error: exhaustionErrorPayload(exhaustion) });},onExhausted: (exhaustion, info) => {setFallbackHeaders(res, info.attempts.length, info.attempts);- res.status(exhaustion.status).json({ error: { message: exhaustion.message, type: exhaustion.type } });+ setExhaustionHeaders(res, exhaustion);+ res.status(exhaustion.status).json({ error: exhaustionErrorPayload(exhaustion) });},});});@@ -1386,19 +1418,29 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {if (members && members.length > 0) {groupChain = resolveModelGroupCandidates(members);if (groupChain.length === 0) {- // Distinguish a catalog-disabled model from one whose providers are- // present but unusable (chain-disabled / no key), so the 400 stays- // actionable and matches the legacy single-row "is disabled" wording.+ // Distinguish a catalog-disabled model (404 model_not_found, OpenAI+ // semantics) from one whose providers are present but unusable+ // (chain-disabled / no key) — the latter is a server-side+ // configuration gap, so it renders an honest 503.const placeholders = members.map(() => '?').join(',');const anyEnabled = db.prepare(`SELECT 1 FROM models WHERE id IN (${placeholders}) AND enabled = 1 LIMIT 1`).get(...members);- const reason = anyEnabled ? 'has no providers with an enabled key' : 'is disabled';- res.status(400).json({- error: {- message: `Model '${requestedModel}' ${reason}. Use 'auto' (or omit the 'model' field) to auto-route, or call /v1/models for the available list.`,- type: 'invalid_request_error',- code: 'model_not_found',- },- });+ if (anyEnabled) {+ res.status(503).json({+ error: {+ message: `Model '${requestedModel}' has no providers with an enabled key. Add a provider API key for it, use 'auto' (or omit the 'model' field) to auto-route, or call /v1/models for the available list.`,+ type: 'service_unavailable',+ code: 'no_providers_configured',+ },+ });+ } else {+ res.status(404).json({+ error: {+ message: `Model '${requestedModel}' is disabled. Use 'auto' (or omit the 'model' field) to auto-route, or call /v1/models for the available list.`,+ type: 'invalid_request_error',+ code: 'model_not_found',+ },+ });+ }return;}stickyStrategyKey = requestedModel;@@ -1415,7 +1457,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {} else {const disabled = db.prepare('SELECT id FROM models WHERE model_id = ?').get(requestedModel) as { id: number } | undefined;const reason = disabled ? 'is disabled' : 'is not in the catalog';- res.status(400).json({+ res.status(404).json({error: {message: `Model '${requestedModel}' ${reason}. Use 'auto' (or omit the 'model' field) to auto-route, or call /v1/models for the available list.`,type: 'invalid_request_error',@@ -1442,8 +1484,20 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {// stream turn-integrity framing.const state = newFallbackState();const attemptLog: AttemptRecord[] = [];+ // Client-disconnect fan-out: the flag stops the loop before the NEXT+ // attempt; the AbortController (threaded to the provider as+ // CompletionOptions.signal) additionally cancels the IN-FLIGHT upstream+ // fetch and any body/stream read, so tokens stop burning and the in-flight+ // lease frees immediately. 'close' also fires on normal completion —+ // writableEnded distinguishes a real disconnect.let clientGone = false;- res.on('close', () => { if (!res.writableEnded) clientGone = true; });+ const clientAbort = new AbortController();+ res.on('close', () => {+ if (!res.writableEnded) {+ clientGone = true;+ clientAbort.abort(newClientAbortError());+ }+ });await runFallbackLoop({maxRetries: MAX_RETRIES,@@ -1539,7 +1593,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {try {const gen = route.provider.streamChatCompletion(route.apiKey, outboundMessages, route.modelId,- { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, ...samplingParams },+ { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, ...samplingParams, signal: clientAbort.signal },quotaContextForRoute(route, 'chat/completions'),);@@ -1730,6 +1784,12 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens + injectedHandoffTokens, totalOutputTokens, Date.now() - start, null, ttfbMs, pinnedModelId);return 'done';} catch (streamErr: any) {+ // Client abort mid-stream: the pump's own `if (clientGone) break`+ // can lose the race against the fetch-signal rejection, so the+ // abort may surface here instead. Rethrow — the shared loop's+ // client-abort branch stops the ladder without benching or an+ // error log row (the socket is gone; nothing to render).+ if (isClientAbortError(streamErr)) throw streamErr;if (headerSent) {// Mid-stream error after real payload reached the client — finish// the SSE response honestly instead of leaving the client hanging.@@ -1758,7 +1818,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {} else {const result = await route.provider.chatCompletion(route.apiKey, outboundMessages, route.modelId,- { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, ...samplingParams },+ { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, ...samplingParams, signal: clientAbort.signal },quotaContextForRoute(route, 'chat/completions'),);@@ -1930,26 +1990,26 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {},onRoutingExhausted: (lastError, routeErr, exhaustion, info) => {// No more models available.- if (exhaustion) {- setFallbackHeaders(res, info.attempts.length, info.attempts);- res.status(exhaustion.status).json({ error: { message: exhaustion.message, type: exhaustion.type } });- } else {+ if (!lastError) {// Synchronous exhaustion: the router rejected every candidate before any// upstream was tried, so this is the ONLY place the per-model disposition- // is recorded. Without it a routing_error 429 is opaque — you can't tell a- // genuinely dry pool from cooldowns/quota/context narrowing (issue _1).+ // is recorded. Without it the exhaustion status is opaque — you can't tell+ // a genuinely dry pool from cooldowns/quota/context narrowing (issue _1).const disposition: string[] = Array.isArray(routeErr.diagnostics) ? routeErr.diagnostics : [];console.warn(`[Proxy] routing exhausted (no upstream tried) req=${shortRequestId(requestGroupId)} ` +`requested=${requestedModelLabel} candidates=${disposition.length}` +(disposition.length ? `:\n ${disposition.join('\n ')}` : ''),);- res.status(routeErr.status ?? 503).json({ error: { message: routeErr.message, type: 'routing_error' } });}+ setFallbackHeaders(res, info.attempts.length, info.attempts);+ setExhaustionHeaders(res, exhaustion);+ res.status(exhaustion.status).json({ error: exhaustionErrorPayload(exhaustion) });},onExhausted: (exhaustion, info) => {setFallbackHeaders(res, info.attempts.length, info.attempts);- res.status(exhaustion.status).json({ error: { message: exhaustion.message, type: exhaustion.type } });+ setExhaustionHeaders(res, exhaustion);+ res.status(exhaustion.status).json({ error: exhaustionErrorPayload(exhaustion) });},});});diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.tsindex 97656c26..016ea616 100644--- a/server/src/routes/responses.ts+++ b/server/src/routes/responses.ts@@ -23,11 +23,12 @@ import {traceRouteEvent,logRequest,} from './proxy.js';-import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, setFallbackHeaders, type AttemptRecord } from '../lib/fallback-loop.js';+import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, setFallbackHeaders, exhaustionErrorPayload, setExhaustionHeaders, type AttemptRecord } from '../lib/fallback-loop.js';import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';import { samplingParamSchemaFields, pickSamplingParams, type ResponseFormat } from '../lib/sampling-params.js';import { enforceJsonContent } from '../lib/structured-output.js';import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';+import { isClientAbortError, newClientAbortError } from '../lib/error-classify.js';import { inferQuotaPoolKey, type QuotaObservationContext } from '../services/provider-quota.js';export const responsesRouter = Router();@@ -407,8 +408,21 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {const responseId = newId('resp');const state = newFallbackState();const attemptLog: AttemptRecord[] = [];+ // Client-disconnect fan-out: the flag stops the loop before the NEXT+ // attempt; the AbortController (threaded to the provider as+ // CompletionOptions.signal) additionally cancels the IN-FLIGHT upstream+ // fetch and any body/stream read, so tokens stop burning and the in-flight+ // lease frees immediately. 'close' also fires on normal completion —+ // writableEnded distinguishes a real disconnect.let clientGone = false;- res.on('close', () => { if (!res.writableEnded) clientGone = true; });+ const clientAbort = new AbortController();+ res.on('close', () => {+ if (!res.writableEnded) {+ clientGone = true;+ clientAbort.abort(newClientAbortError());+ }+ });+ const dispatchOpts = { ...completionOpts, signal: clientAbort.signal };// Stream bookkeeping (used only when stream === true). `streamStarted` is the// commit flag: true once the response.created/in_progress skeleton has left,@@ -498,7 +512,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {route.apiKey,messages,route.modelId,- completionOpts,+ dispatchOpts,quotaContextForRoute(route, 'responses'),);@@ -679,6 +693,12 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens, totalOutputTokens, Date.now() - start, null);return 'done';} catch (streamErr: any) {+ // Client abort mid-stream: the pump's own `if (clientGone) break`+ // can lose the race against the fetch-signal rejection, so the+ // abort may surface here instead. Rethrow — the shared loop's+ // client-abort branch stops the ladder without benching or an+ // error log row (the socket is gone; nothing to render).+ if (isClientAbortError(streamErr)) throw streamErr;// A committed stream can't fail over (bytes already sent) — surface a// response.failed event honestly and stop. A pre-commit failure throws// through to the shared loop for cooldown + failover.@@ -706,7 +726,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {route.apiKey,messages,route.modelId,- completionOpts,+ dispatchOpts,quotaContextForRoute(route, 'responses'),);@@ -809,26 +829,27 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {res.status(502).json({ error: { message: `Provider error (${route.displayName}): ${sanitizeProviderErrorMessage(err.message)}`, type: 'provider_error' } });},onRoutingExhausted: (lastError, routeErr, exhaustion, info) => {- const status = exhaustion?.status ?? routeErr.status ?? 503;- const message = exhaustion?.message ?? routeErr.message;- const type = exhaustion?.type ?? 'routing_error';if (streamStarted) {- sse('response.failed', { response: { id: responseId, object: 'response', status: 'failed', error: { message, type } } });+ // Headers are already on the wire — the honest status/Retry-After can+ // only travel in the failed-event payload.+ sse('response.failed', { response: { id: responseId, object: 'response', status: 'failed', error: exhaustionErrorPayload(exhaustion) } });res.end();} else {setFallbackHeaders(res, info.attempts.length, info.attempts);- res.status(status).json({ error: { message, type } });+ setExhaustionHeaders(res, exhaustion);+ res.status(exhaustion.status).json({ error: exhaustionErrorPayload(exhaustion) });}},onExhausted: (exhaustion, info) => {// The streaming skeleton may already be on the wire — close the SSE stream// with a failed event instead of writing JSON onto a committed response.if (streamStarted) {- sse('response.failed', { response: { id: responseId, object: 'response', status: 'failed', error: { message: exhaustion.message, type: exhaustion.type } } });+ sse('response.failed', { response: { id: responseId, object: 'response', status: 'failed', error: exhaustionErrorPayload(exhaustion) } });res.end();} else {setFallbackHeaders(res, info.attempts.length, info.attempts);- res.status(exhaustion.status).json({ error: { message: exhaustion.message, type: exhaustion.type } });+ setExhaustionHeaders(res, exhaustion);+ res.status(exhaustion.status).json({ error: exhaustionErrorPayload(exhaustion) });}},});diff --git a/server/src/services/cooldown-probe.ts b/server/src/services/cooldown-probe.tsnew file mode 100644index 00000000..868c908e--- /dev/null+++ b/server/src/services/cooldown-probe.ts@@ -0,0 +1,222 @@+// cooldown-probe — probe-based early recovery for benched keys.+//+// Cooldowns are deliberately pessimistic: a transient 429 benches a model+key+// for 90s, the escalation ladder for up to a day, and an upstream 401 for the+// whole 5-minute health cycle. When the provider recovers sooner (a per-minute+// window rolled over, a provider incident ended, a key was un-revoked), that+// capacity sits idle until the timer runs out. This job periodically issues the+// same cheap validateKey call the health checker uses against keys sitting in a+// HEURISTIC cooldown, and clears those cooldowns early when the probe passes.+//+// What it will never do:+// - probe a cooldown backed by an authoritative retry time (explicit+// Retry-After, daily-quota reset) or a credit/tier bench — those are+// provider-stated facts, filtered out at the query (getProbeableCooldowns);+// - extend a cooldown: a failed probe only schedules the next probe further+// out (exponential backoff), the bench itself is untouched;+// - feed the health checker's consecutive-failure counter (probeKeyValidity+// is side-effect-free by design);+// - thundering-herd providers after a restart: a key's first sighting only+// schedules a jittered probe, and each pass probes at most a handful of+// keys.+//+// The probe unit is the KEY, not the model: validateKey exercises the+// credential, so one pass/fail is the same evidence for every heuristic+// cooldown that key holds, and probing per key is what keeps probe traffic+// bounded no matter how many models are benched on it.+//+// Kill switch: COOLDOWN_PROBE_DISABLED=1 (same convention as+// CATALOG_SYNC_DISABLED). Per-pass probe budget: COOLDOWN_PROBE_MAX_PER_PASS.++import type { Scheduler } from '../lib/scheduler.js';+import {+ getProbeableCooldowns,+ clearCooldownEarly,+ type ProbeableCooldown,+} from './ratelimit.js';+import { probeKeyValidity, type KeyProbeOutcome } from './health.js';++// How often the scanner wakes up to look at the cooldown table. Scanning is one+// cheap indexed SELECT; actual probes are gated far harder below.+const SCAN_INTERVAL_MS = 60 * 1000;++// A cooldown only becomes probe-ripe after this fraction of its bench has been+// served. Probing a seconds-old cooldown would mostly re-confirm the failure+// that caused it; by half-time a transient condition has had a real chance to+// clear, and long escalated benches still recover hours early.+const MIN_ELAPSED_FRACTION = 0.5;++// Not worth probing a cooldown that is about to expire on its own — the probe+// would cost a validate call to save less than a scan interval of idle time.+const MIN_REMAINING_MS = 60 * 1000;++// Failed-probe backoff per key: 2m, 4m, 8m, capped at 15m. Keys that stay bad+// converge to one validate call per 15 minutes — a third of what the 5-minute+// health pass already sends them.+const PROBE_BACKOFF_BASE_MS = 2 * 60 * 1000;+const PROBE_BACKOFF_MAX_MS = 15 * 60 * 1000;++// Stagger window for a key's FIRST probe after it is sighted (covers the+// restart case, where every persisted cooldown is sighted at once).+const FIRST_PROBE_STAGGER_MS = 45 * 1000;++const DEFAULT_MAX_PROBES_PER_PASS = 3;++/** Per-pass probe budget. Tunable for large key fleets; 0 or a bad value falls+ * back to the default (mirrors HEALTH_CHECK_CONCURRENCY). */+function getMaxProbesPerPass(): number {+ const raw = process.env.COOLDOWN_PROBE_MAX_PER_PASS;+ if (raw !== undefined && raw.trim() !== '') {+ const n = Number(raw);+ if (Number.isInteger(n) && n > 0) return n;+ }+ return DEFAULT_MAX_PROBES_PER_PASS;+}++interface KeyProbeState {+ failures: number;+ nextProbeAtMs: number;+}++// keyId -> probe pacing. In-memory only, like the escalation ladder: a restart+// forgetting the backoff is fine because first-sighting staggering re-spaces+// the probes anyway.+const keyState = new Map<number, KeyProbeState>();++export interface ProbePassOptions {+ now?: number;+ // Test seams: injectable probe + RNG so unit tests need no network or timers.+ probe?: (keyId: number) => Promise<KeyProbeOutcome>;+ jitter?: () => number; // [0, 1), defaults to Math.random+ maxProbes?: number;+}++export interface ProbePassResult {+ probedKeyIds: number[];+ clearedCooldowns: number;+}++/** True when this cooldown has served enough of its bench to be worth probing. */+function isRipe(cooldown: ProbeableCooldown, now: number): boolean {+ if (cooldown.expiresAtMs - now < MIN_REMAINING_MS) return false;+ // Rows persisted before the provenance migration have no start time; treat+ // them as old enough rather than never probing them.+ if (cooldown.setAtMs == null) return true;+ const duration = cooldown.expiresAtMs - cooldown.setAtMs;+ return now - cooldown.setAtMs >= duration * MIN_ELAPSED_FRACTION;+}++/**+ * One probe pass: find ripe heuristic cooldowns, probe up to maxProbes of their+ * keys (respecting per-key backoff), and clear every heuristic cooldown on a+ * key whose probe passed. Exported for tests; production runs it on the+ * scheduler via startCooldownProbe.+ */+export async function runCooldownProbePass(opts: ProbePassOptions = {}): Promise<ProbePassResult> {+ const now = opts.now ?? Date.now();+ const probe = opts.probe ?? probeKeyValidity;+ const jitter = opts.jitter ?? Math.random;+ const maxProbes = opts.maxProbes ?? getMaxProbesPerPass();++ const all = getProbeableCooldowns(now);++ // Drop pacing state for keys with no probeable cooldowns left (expired,+ // cleared, or re-benched as authoritative) so old backoff cannot delay a+ // future, unrelated bench.+ const keysWithCooldowns = new Set(all.map(c => c.keyId));+ for (const keyId of [...keyState.keys()]) {+ if (!keysWithCooldowns.has(keyId)) keyState.delete(keyId);+ }++ const ripeByKey = new Map<number, ProbeableCooldown[]>();+ for (const cooldown of all) {+ if (!isRipe(cooldown, now)) continue;+ const list = ripeByKey.get(cooldown.keyId) ?? [];+ list.push(cooldown);+ ripeByKey.set(cooldown.keyId, list);+ }++ const probedKeyIds: number[] = [];+ let clearedCooldowns = 0;++ for (const [keyId, cooldownsForKey] of ripeByKey) {+ if (probedKeyIds.length >= maxProbes) break;++ let state = keyState.get(keyId);+ if (!state) {+ // First sighting: schedule, don't probe. This is the restart stagger —+ // a boot with dozens of persisted cooldowns spreads its first probes+ // across the jitter window instead of validating everything at once.+ state = { failures: 0, nextProbeAtMs: now + Math.floor(jitter() * FIRST_PROBE_STAGGER_MS) };+ keyState.set(keyId, state);+ continue;+ }+ if (now < state.nextProbeAtMs) continue;++ probedKeyIds.push(keyId);+ const outcome = await probe(keyId);++ if (outcome === 'valid') {+ for (const cooldown of cooldownsForKey) {+ clearCooldownEarly(cooldown.platform, cooldown.modelId, cooldown.keyId);+ clearedCooldowns++;+ }+ keyState.delete(keyId);+ console.log(+ `[CooldownProbe] key ${keyId} validated — cleared ${cooldownsForKey.length} cooldown(s) early ` ++ `(${cooldownsForKey.map(c => `${c.platform}/${c.modelId}`).join(', ')})`,+ );+ } else {+ // Failed or inconclusive: the bench stays EXACTLY as it was — a probe+ // must never extend a cooldown — and this key backs off before the next+ // attempt so probes cannot burn quota against a provider that is down.+ state.failures++;+ const backoff = Math.min(PROBE_BACKOFF_BASE_MS * 2 ** (state.failures - 1), PROBE_BACKOFF_MAX_MS);+ state.nextProbeAtMs = now + backoff;+ }+ }++ return { probedKeyIds, clearedCooldowns };+}++// Overlap guard, same shape as checkAllKeys: a slow provider validate must not+// let the next scheduled pass stack a second set of probes on top.+let passInFlight: Promise<ProbePassResult> | null = null;++function runGuardedPass(): Promise<ProbePassResult> {+ if (passInFlight) return passInFlight;+ passInFlight = runCooldownProbePass().finally(() => {+ passInFlight = null;+ });+ return passInFlight;+}++let cancelProbeJob: (() => void) | null = null;++export function startCooldownProbe(scheduler: Scheduler): void {+ if (cancelProbeJob) return;+ if (process.env.COOLDOWN_PROBE_DISABLED === '1') {+ console.log('[CooldownProbe] disabled via COOLDOWN_PROBE_DISABLED=1');+ return;+ }+ console.log(`[CooldownProbe] starting cooldown-probe recovery (scan every ${SCAN_INTERVAL_MS / 1000}s)`);+ cancelProbeJob = scheduler.every(+ SCAN_INTERVAL_MS,+ async () => {+ await runGuardedPass().catch(err => console.error('[CooldownProbe] pass failed:', err));+ },+ { name: 'cooldown-probe' },+ );+}++export function stopCooldownProbe(): void {+ if (cancelProbeJob) {+ cancelProbeJob();+ cancelProbeJob = null;+ }+}++/** Test seam: drop all per-key probe pacing state. */+export function resetCooldownProbeState(): void {+ keyState.clear();+}diff --git a/server/src/services/fusion.ts b/server/src/services/fusion.tsindex d1f0f1a2..8cb2627c 100644--- a/server/src/services/fusion.ts+++ b/server/src/services/fusion.ts@@ -6,6 +6,7 @@ import {} from './router.js';import {recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit,+ getCooldownDecisionForLimit,PAYMENT_REQUIRED_COOLDOWN_MS, MODEL_FORBIDDEN_COOLDOWN_MS,} from './ratelimit.js';import { logRequest } from '../lib/request-log.js';@@ -257,14 +258,15 @@ async function runModelCall(if (isRetryableError(err)) {if (isModelNotFoundError(err) || isModelAccessForbiddenError(err)) skipModels.add(route.modelDbId);skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);- setCooldown(- route.platform, route.modelId, route.keyId,- isPaymentRequiredError(err)- ? PAYMENT_REQUIRED_COOLDOWN_MS- : isModelAccessForbiddenError(err)- ? MODEL_FORBIDDEN_COOLDOWN_MS- : getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }, err.retryAfterMs),- );+ // Provenance mirrors cooldownDecisionForError (lib/fallback-loop.ts):+ // credit/tier benches are never probe-recovered, Retry-After-backed+ // ones only when our heuristic outlasted the provider's own retry time.+ const decision = isPaymentRequiredError(err)+ ? { durationMs: PAYMENT_REQUIRED_COOLDOWN_MS, source: 'credit' as const }+ : isModelAccessForbiddenError(err)+ ? { durationMs: MODEL_FORBIDDEN_COOLDOWN_MS, source: 'tier' as const }+ : getCooldownDecisionForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }, err.retryAfterMs);+ setCooldown(route.platform, route.modelId, route.keyId, decision.durationMs, decision.source);recordRateLimitHit(route.modelDbId);continue;}@@ -348,12 +350,13 @@ async function runJudgeStreaming(if (isRetryableError(err)) {if (isModelNotFoundError(err) || isModelAccessForbiddenError(err)) skipModels.add(route.modelDbId);skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);- setCooldown(- route.platform, route.modelId, route.keyId,- isPaymentRequiredError(err) ? PAYMENT_REQUIRED_COOLDOWN_MS- : isModelAccessForbiddenError(err) ? MODEL_FORBIDDEN_COOLDOWN_MS- : getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }, err.retryAfterMs),- );+ // Same provenance mapping as the non-stream path above.+ const decision = isPaymentRequiredError(err)+ ? { durationMs: PAYMENT_REQUIRED_COOLDOWN_MS, source: 'credit' as const }+ : isModelAccessForbiddenError(err)+ ? { durationMs: MODEL_FORBIDDEN_COOLDOWN_MS, source: 'tier' as const }+ : getCooldownDecisionForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }, err.retryAfterMs);+ setCooldown(route.platform, route.modelId, route.keyId, decision.durationMs, decision.source);recordRateLimitHit(route.modelDbId);continue;}diff --git a/server/src/services/health.ts b/server/src/services/health.tsindex 79833b26..cb8cd8c0 100644--- a/server/src/services/health.ts+++ b/server/src/services/health.ts@@ -103,6 +103,43 @@ export async function checkKeyHealth(keyId: number): Promise<KeyStatus> {}}+// ── Cooldown-probe validation (side-effect-free) ─────────────────────────────+// The cooldown-probe recovery job (services/cooldown-probe.ts) needs the same+// cheap validateKey call checkKeyHealth makes, WITHOUT any of its bookkeeping:+// no status writes, no last_checked_at, and above all no failureCount — probes+// run far more often than the 5-minute health pass, so letting them feed the+// consecutive-failure counter would auto-disable a genuinely-bad key in a+// fraction of the "3 consecutive checks" the threshold promises. A probe is a+// question, never a verdict.++export type KeyProbeOutcome = 'valid' | 'invalid' | 'error';++export async function probeKeyValidity(keyId: number): Promise<KeyProbeOutcome> {+ try {+ const db = getDb();+ const row = db.prepare('SELECT * FROM api_keys WHERE id = ? AND enabled = 1').get(keyId) as any;+ if (!row) return 'error';++ const provider = resolveProvider(row.platform as Platform, row.base_url);+ if (!provider) return 'error';++ const apiKey = decrypt(row.encrypted_key, row.iv, row.auth_tag);+ const validation = await provider.validateKey(apiKey, {+ platform: row.platform as Platform,+ keyId,+ quotaPoolKey: inferQuotaPoolKey(row.platform as Platform, null),+ endpoint: 'models',+ origin: 'probe',+ });+ const isValid = typeof validation === 'boolean' ? validation : validation.valid;+ return isValid ? 'valid' : 'invalid';+ } catch {+ // Transport error (DNS/timeout/TLS): inconclusive, same as checkKeyHealth's+ // reasoning — evidence about the network, not the key.+ return 'error';+ }+}+/*** Promote a key out of 'error' after it successfully served a live request.*diff --git a/server/src/services/ratelimit.ts b/server/src/services/ratelimit.tsindex b7690f26..b8fc48bc 100644--- a/server/src/services/ratelimit.ts+++ b/server/src/services/ratelimit.ts@@ -600,6 +600,21 @@ export function recordTokens(// Cooldown: when a provider returns 429, block that model+key for a periodconst cooldowns = new Map<string, number>(); // key -> expiry timestamp+// ── Cooldown provenance ───────────────────────────────────────────────────────+// Recorded at setCooldown time so the cooldown-probe recovery job (services/+// cooldown-probe.ts) can tell OUR guesses apart from provider-stated facts:+// 'heuristic' — our own pessimistic bench (transient 90s, escalation+// ladder, auth-failure bench, empty-completion). Often+// outlives the actual outage, so probe-eligible.+// 'authoritative' — backed by an explicit provider retry time (Retry-After+// that determined the expiry, daily-quota reset). A fact,+// not a guess: never probed.+// 'credit' — 402 out-of-credits. A key-validation probe succeeding is+// no evidence the account was topped up, so never probed.+// 'tier' — 403 model-not-on-tier. Same reasoning: the key validates+// fine while the model stays gated, so never probed.+export type CooldownSource = 'heuristic' | 'authoritative' | 'credit' | 'tier';+// Escalating cooldown: track hits per key over a rolling 24h window so a// daily-quota exhaustion (OpenRouter free: 50/day, Cohere free: 33/day, etc.)// quarantines the key for the rest of the day instead of looping through@@ -699,6 +714,11 @@ export function recentHitCount(// prompts 429s on TPM while the daily quota is barely touched. Daily counters// are persisted (countPersistedRequests / sumPersistedTokens), so this verdict// is stable across restarts.+export interface CooldownDecision {+ durationMs: number;+ source: CooldownSource;+}+export function getCooldownDurationForLimit(platform: string,modelId: string,@@ -706,6 +726,24 @@ export function getCooldownDurationForLimit(limits: { rpd: number | null; tpd: number | null },retryAfterMs?: number | null,): number {+ return getCooldownDecisionForLimit(platform, modelId, keyId, limits, retryAfterMs).durationMs;+}++/**+ * Same verdict as getCooldownDurationForLimit, plus the provenance the+ * cooldown-probe recovery job needs: 'authoritative' when an explicit provider+ * Retry-After actually determined the expiry (a fact — never probed early),+ * 'heuristic' when the duration is our own transient/escalation guess (probe-+ * eligible). A Retry-After SHORTER than our bench does not make the cooldown+ * authoritative: everything past the provider's own retry time is our guess.+ */+export function getCooldownDecisionForLimit(+ platform: string,+ modelId: string,+ keyId: number,+ limits: { rpd: number | null; tpd: number | null },+ retryAfterMs?: number | null,+): CooldownDecision {const now = Date.now();const rpdExhausted =limits.rpd !== null && requestCount(platform, modelId, keyId, DAY, now) >= limits.rpd;@@ -730,8 +768,10 @@ export function getCooldownDurationForLimit(// Honor an upstream Retry-After as a floor: never bench shorter than our own// heuristic, but extend (capped at a day) when the provider explicitly asks// to wait longer than we otherwise would.- if (retryAfterMs != null && retryAfterMs > base) return Math.min(retryAfterMs, DAY);- return base;+ if (retryAfterMs != null && retryAfterMs > base) {+ return { durationMs: Math.min(retryAfterMs, DAY), source: 'authoritative' };+ }+ return { durationMs: base, source: 'heuristic' };}function persistedCooldownExpiry(@@ -751,14 +791,23 @@ function persistedCooldownExpiry(});}-function persistCooldown(platform: string, modelId: string, keyId: number, expiresAtMs: number) {+function persistCooldown(+ platform: string,+ modelId: string,+ keyId: number,+ expiresAtMs: number,+ source: CooldownSource,+ setAtMs: number,+) {withDb(db => {db.prepare(`- INSERT INTO rate_limit_cooldowns (platform, model_id, key_id, expires_at_ms)- VALUES (?, ?, ?, ?)+ INSERT INTO rate_limit_cooldowns (platform, model_id, key_id, expires_at_ms, source, set_at_ms)+ VALUES (?, ?, ?, ?, ?, ?)ON CONFLICT(platform, model_id, key_id)- DO UPDATE SET expires_at_ms = excluded.expires_at_ms- `).run(platform, modelId, keyId, expiresAtMs);+ DO UPDATE SET expires_at_ms = excluded.expires_at_ms,+ source = excluded.source,+ set_at_ms = excluded.set_at_ms+ `).run(platform, modelId, keyId, expiresAtMs, source, setAtMs);});}@@ -773,11 +822,18 @@ function clearPersistedCooldown(platform: string, modelId: string, keyId: number});}-export function setCooldown(platform: string, modelId: string, keyId: number, durationMs = 60_000) {+export function setCooldown(+ platform: string,+ modelId: string,+ keyId: number,+ durationMs = 60_000,+ source: CooldownSource = 'heuristic',+) {const key = `${platform}:${modelId}:${keyId}:cooldown`;- const expiresAtMs = Date.now() + durationMs;+ const now = Date.now();+ const expiresAtMs = now + durationMs;cooldowns.set(key, expiresAtMs);- persistCooldown(platform, modelId, keyId, expiresAtMs);+ persistCooldown(platform, modelId, keyId, expiresAtMs, source, now);}export function isOnCooldown(platform: string, modelId: string, keyId: number): boolean {@@ -880,6 +936,55 @@ export function clearCooldownsForKey(keyId: number): number {return Math.max(cleared, memoryCleared);}+// ── Cooldown-probe surface (services/cooldown-probe.ts) ───────────────────────++export interface ProbeableCooldown {+ platform: string;+ modelId: string;+ keyId: number;+ expiresAtMs: number;+ // Null for rows persisted before the provenance migration; the prober treats+ // an unknown start time as "old enough to probe".+ setAtMs: number | null;+}++/**+ * Active cooldowns whose expiry is OUR OWN guess ('heuristic'), i.e. the only+ * ones probe-based early recovery may touch. Authoritative/credit/tier rows are+ * excluded at the query so the prober cannot even see them. Deliberately+ * DB-only, no in-memory fallback: the probe job is a best-effort optimisation,+ * and when the DB is unavailable the right amount of probing is none.+ */+export function getProbeableCooldowns(now = Date.now()): ProbeableCooldown[] {+ const rows = withDb(db => db.prepare(`+ SELECT platform, model_id, key_id, expires_at_ms, set_at_ms+ FROM rate_limit_cooldowns+ WHERE expires_at_ms > ?+ AND source = 'heuristic'+ ORDER BY expires_at_ms ASC+ `).all(now) as { platform: string; model_id: string; key_id: number; expires_at_ms: number; set_at_ms: number | null }[]) ?? [];++ return rows.map(row => ({+ platform: row.platform,+ modelId: row.model_id,+ keyId: row.key_id,+ expiresAtMs: row.expires_at_ms,+ setAtMs: row.set_at_ms,+ }));+}++/**+ * Clear ONE model+key cooldown before its timer expires, after a probe showed+ * the key is serving again. Narrower than clearCooldownsForKey (the operator+ * override): the escalation history in cooldownHits is deliberately kept, so a+ * key that 429s again right after an early recovery re-enters the ladder where+ * it left off instead of starting back at 2 minutes.+ */+export function clearCooldownEarly(platform: string, modelId: string, keyId: number): void {+ cooldowns.delete(`${platform}:${modelId}:${keyId}:cooldown`);+ clearPersistedCooldown(platform, modelId, keyId);+}+/*** Soonest moment any active cooldown expires, in ms since epoch, or null when* nothing is cooling down. Used to tell an exhausted caller roughly when to