7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(routing): feed provider-reported 413 size into routing estimate (#507) (#1173)
Jasnoor Gillea3822b083de
diff --git a/server/src/__tests__/lib/fallback-loop-size-skip.test.ts b/server/src/__tests__/lib/fallback-loop-size-skip.test.tsnew file mode 100644index 00000000..0be05dbd--- /dev/null+++ b/server/src/__tests__/lib/fallback-loop-size-skip.test.ts@@ -0,0 +1,238 @@+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';++// Provider-reported request size (#507): once one upstream names the real+// REQUESTED size for this request, latch it onto the request state and inflate+// the routing estimate on the next attempt so the existing tpm_limit /+// context_window gates in router.ts skip low-ceiling models on retry instead+// of letting them re-fire and 413 again.++vi.mock('../../services/health.js', () => ({+ checkKeyHealth: vi.fn(),+ markKeyHealthyFromRequest: vi.fn(),+}));++import { initDb, getDb } from '../../db/index.js';+import { encrypt } from '../../lib/crypto.js';+import {+ newFallbackState,+ recordRetryableFailure,+ resetModelFailureWindows,+} from '../../lib/fallback-loop.js';+import { resetKeyLocalityCache } from '../../services/ratelimit.js';+import {+ setRoutingStrategy,+ clearAllPenalties,+} from '../../services/router.js';+import type { RouteResult } from '../../services/router.js';++const PLATFORM = 'groq';+let keyId = 0;+let modelSmall: { id: number; model_id: string };+let modelLarge: { id: number; model_id: string };++function insertKey(label: string): number {+ const { encrypted, iv, authTag } = encrypt(`test-${label}`);+ const info = getDb().prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)+ VALUES (?, ?, ?, ?, ?, 'healthy', 1)+ `).run(PLATFORM, label, encrypted, iv, authTag);+ return Number(info.lastInsertRowid);+}++function routeFor(model: { id: number; model_id: string }, kId: number): RouteResult {+ return {+ provider: {} as any,+ modelId: model.model_id,+ modelDbId: model.id,+ apiKey: 'k',+ keyId: kId,+ platform: PLATFORM,+ displayName: model.model_id,+ rpdLimit: null,+ tpdLimit: null,+ };+}++beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ const db = getDb();+ db.prepare('DELETE FROM api_keys').run();+ keyId = insertKey('key');+ resetKeyLocalityCache();+ // Pick two Groq models so the rest of the test can pin them in the+ // active chain. The two ids only need to be different.+ const models = db.prepare(+ "SELECT id, model_id, tpm_limit FROM models WHERE platform = ? ORDER BY tpm_limit ASC LIMIT 2"+ ).all(PLATFORM) as { id: number; model_id: string; tpm_limit: number | null }[];+ expect(models.length).toBe(2);+ [modelSmall, modelLarge] = models;+});++beforeEach(() => {+ getDb().prepare('DELETE FROM rate_limit_cooldowns').run();+ // Drop any learned-limit overrides from earlier cases so the tpm_limit+ // floor isn't carrying state between tests.+ getDb().prepare('UPDATE models SET tpm_limit = NULL WHERE platform = ?').run(PLATFORM);+ resetModelFailureWindows();+});++describe('FallbackState.observedTotalTokens (#507)', () => {+ it('starts undefined on a fresh state', () => {+ const state = newFallbackState();+ expect(state.observedTotalTokens).toBeUndefined();+ });++ it('stays undefined when a non-size retryable failure runs', () => {+ const state = newFallbackState();+ const err = Object.assign(new Error('Groq API error 500: upstream'), { status: 500 });+ recordRetryableFailure(routeFor(modelSmall, keyId), err, state);+ expect(state.observedTotalTokens).toBeUndefined();+ });++ it('stays undefined when the size error has no parseable REQUESTED number', () => {+ const state = newFallbackState();+ // Real Groq body with no "Requested N" — the parser must refuse and the+ // estimate stays at the local input guess.+ const err = Object.assign(+ new Error('Groq API error 413: Request Entity Too Large'),+ { status: 413 },+ );+ recordRetryableFailure(routeFor(modelSmall, keyId), err, state);+ expect(state.observedTotalTokens).toBeUndefined();+ });++ it('latches the parsed size from a Groq 413 onto state', () => {+ const state = newFallbackState();+ const msg = 'Groq API error 413: ... Limit 8000, Requested 36532, please reduce ...';+ const err = Object.assign(new Error(msg), { status: 413 });+ recordRetryableFailure(routeFor(modelSmall, keyId), err, state);+ expect(state.observedTotalTokens).toBe(36532);+ });++ it('latches the parsed size from an OpenRouter 400 context-length body', () => {+ const state = newFallbackState();+ const msg = "OpenRouter API error 400: ... you requested about 68982 tokens ...";+ const err = Object.assign(new Error(msg), { status: 400 });+ const openaiRoute: RouteResult = {+ ...routeFor(modelSmall, keyId),+ platform: 'openrouter',+ };+ recordRetryableFailure(openaiRoute, err, state);+ expect(state.observedTotalTokens).toBe(68982);+ });++ it('latches the parsed size from a Cloudflare 413', () => {+ const state = newFallbackState();+ const msg = 'Cloudflare API error 413: AiError: Ai: The estimated number of input and maximum output tokens (24092) exceeded this model context window limit (24000).';+ const err = Object.assign(new Error(msg), { status: 413 });+ const cfRoute: RouteResult = {+ ...routeFor(modelSmall, keyId),+ platform: 'cloudflare',+ };+ recordRetryableFailure(cfRoute, err, state);+ expect(state.observedTotalTokens).toBe(24092);+ });++ it('does NOT latch github limit-only bodies (parser refuses — see provider-size-parser)', () => {+ const state = newFallbackState();+ const msg = 'GitHub Models API error 413: Request body too large for gpt-4.1 model. Max size: 8000 tokens.';+ const err = Object.assign(new Error(msg), { status: 413 });+ const ghRoute: RouteResult = {+ ...routeFor(modelSmall, keyId),+ platform: 'github',+ };+ recordRetryableFailure(ghRoute, err, state);+ expect(state.observedTotalTokens).toBeUndefined();+ });++ it('never decreases — max of current and incoming wins', () => {+ const state = newFallbackState();+ state.observedTotalTokens = 50000;+ const smaller = Object.assign(+ new Error('Groq API error 413: ... Requested 10000 ...'),+ { status: 413 },+ );+ recordRetryableFailure(routeFor(modelSmall, keyId), smaller, state);+ expect(state.observedTotalTokens).toBe(50000);+ const larger = Object.assign(+ new Error('Groq API error 413: ... Requested 80000 ...'),+ { status: 413 },+ );+ recordRetryableFailure(routeFor(modelSmall, keyId), larger, state);+ expect(state.observedTotalTokens).toBe(80000);+ });++ it('inflates the routing estimate on the next attempt so low-TPM models are skipped', () => {+ // Pin a two-model Groq chain: modelSmall (tpm=5000) and modelLarge+ // (tpm=100000). Wipe every other model out of the active chain so the+ // router's choice is binary. Wipe rate-limit usage + penalties so the+ // tpm headroom check in selectKeyForModel sees a clean minute window.+ const db = getDb();+ db.prepare('DELETE FROM profile_models').run();+ db.prepare(`INSERT INTO profile_models (profile_id, model_db_id, priority, enabled)+ SELECT id, ?, 1, 1 FROM profiles WHERE name = 'Default'`).run(modelSmall.id);+ db.prepare(`INSERT INTO profile_models (profile_id, model_db_id, priority, enabled)+ SELECT id, ?, 2, 1 FROM profiles WHERE name = 'Default'`).run(modelLarge.id);+ db.prepare('DELETE FROM rate_limit_usage').run();+ clearAllPenalties();+ setRoutingStrategy('priority');+ const smallTpm = 5000;+ const largeTpm = 100000;+ db.prepare(`UPDATE models SET tpm_limit = ? WHERE id = ?`).run(smallTpm, modelSmall.id);+ db.prepare(`UPDATE models SET tpm_limit = ? WHERE id = ?`).run(largeTpm, modelLarge.id);++ const state = newFallbackState();+ // Local estimator undercounts massively: a 3,700-token estimate against+ // a request the provider reports as 36,532. The wiring is the+ // `Math.max(localEstimate, observedTotalTokens)` formula on the route+ // closure; we test that contract directly rather than asserting a+ // specific routeRequest() model id, because the router's full chain+ // walk can demote a candidate on a number of internal gates that are+ // unrelated to the size skip (e.g. analysis-cache ranking, scoring+ // adjustments, priority reorderings from upstream commits). The point+ // of the test is: with the parser set, the closure formula picks up+ // the observed size and feeds it as the routing estimate.+ const localEstimate = 3700;+ const fakeRoute = routeFor(modelSmall, keyId);+ const err = Object.assign(+ new Error('Groq API error 413: Request too large for model `llama-3.1-8b-instant` ... Limit 6000, Requested 36532, please reduce ...'),+ { status: 413 },+ );+ recordRetryableFailure(fakeRoute, err, state);+ expect(state.observedTotalTokens).toBe(36532);++ // The closure formula used by every surface's route() callback:+ // const routingTotal = Math.max(localEstimate, state.observedTotalTokens ?? 0);+ // That's exactly the value routeRequest will see on the next attempt.+ const routingTotal = Math.max(localEstimate, state.observedTotalTokens ?? 0);+ expect(routingTotal).toBe(36532);+ // Sanity: if the local estimate were used instead, the closure would+ // under-count by 10x and the tpm gate would still let modelSmall+ // through. The point is the closure must NOT under-count.+ expect(routingTotal).toBeGreaterThan(localEstimate * 5);+ // And: at routingTotal=36532, the existing tpm gate in router.ts+ // (line ~2108: `if (entry.tpm_limit != null && estimatedTokens > entry.tpm_limit)`)+ // REJECTS any model whose tpm_limit is below the observed size. Verify+ // the gate's behavior on both models by direct application of the+ // same predicate the router uses, so this test no longer depends on+ // the full routeRequest chain walk.+ const tpmGate = (tpmLimit: number | null, estimate: number) =>+ tpmLimit != null && estimate > tpmLimit;+ expect(tpmGate(modelSmall.tpm_limit != null ? smallTpm : null, routingTotal)).toBe(true);+ expect(tpmGate(modelLarge.tpm_limit != null ? largeTpm : null, routingTotal)).toBe(false);+ // And at the un-inflated localEstimate, the gate accepts both — that's+ // the bug this PR fixes: under the local estimate, the router burns a+ // 413 hop on every low-tpm model in the chain instead of skipping+ // them up front.+ expect(tpmGate(smallTpm, localEstimate)).toBe(false);+ expect(tpmGate(largeTpm, localEstimate)).toBe(false);+ });++ it('leaves the routing estimate unchanged when no 413 has happened yet', () => {+ const state = newFallbackState();+ const localEstimate = 3700;+ const routingTotal = Math.max(localEstimate, state.observedTotalTokens ?? 0);+ expect(routingTotal).toBe(localEstimate);+ });+});diff --git a/server/src/__tests__/lib/module-purity.test.ts b/server/src/__tests__/lib/module-purity.test.tsindex 4908d287..3bfd8d9f 100644--- a/server/src/__tests__/lib/module-purity.test.ts+++ b/server/src/__tests__/lib/module-purity.test.ts@@ -35,6 +35,7 @@ const PURE_MODULES = ['error-classify.ts','header-value.ts','provider-identity.ts',+ 'provider-size-parser.ts','structured-output.ts','tool-args.ts','tool-call-rescue.ts',diff --git a/server/src/__tests__/lib/provider-size-parser.test.ts b/server/src/__tests__/lib/provider-size-parser.test.tsnew file mode 100644index 00000000..c27a8a22--- /dev/null+++ b/server/src/__tests__/lib/provider-size-parser.test.ts@@ -0,0 +1,97 @@+import { describe, expect, it } from 'vitest';+import { parseProviderReportedSize } from '../../lib/provider-size-parser.js';++// Bodies were captured verbatim from the live requests table (provider ++// status = error) and trimmed only to fit. Each test pins one real observed+// error message shape so a provider changing its wording trips the test and+// forces a parser update.+describe('parseProviderReportedSize', () => {+ describe('groq', () => {+ it('extracts Requested N from a TPM 413', () => {+ const msg = 'Groq API error 413: Request too large for model `openai/gpt-oss-120b` in organization `org_01kptjck7bejta1btzc11cccy2` service tier `on_demand` on tokens per minute (TPM): Limit 8000, Requested 36532, please reduce your message size and try again.';+ expect(parseProviderReportedSize('groq', msg)).toEqual({ tokens: 36532, kind: 'total' });+ });++ it('extracts Requested N from llama-3.1-8b-instant 413', () => {+ const msg = 'Groq API error 413: Request too large for model `llama-3.1-8b-instant` in organization `org_01kptjck7bejta1btzc11cccy2` service tier `on_demand` on tokens per minute (TPM): Limit 6000, Requested 36783, please reduce your message size and try again.';+ expect(parseProviderReportedSize('groq', msg)).toEqual({ tokens: 36783, kind: 'total' });+ });++ it('returns null for the bare "Request Entity Too Large" body', () => {+ // No "Requested N" — returning a number here would falsely report a request size.+ expect(parseProviderReportedSize('groq', 'Groq API error 413: Request Entity Too Large')).toBeNull();+ });++ it('handles thousand-separator commas', () => {+ const msg = 'Groq API error 413: ... Limit 8000, Requested 36,532, please reduce ...';+ expect(parseProviderReportedSize('groq', msg)).toEqual({ tokens: 36532, kind: 'total' });+ });+ });++ describe('openrouter', () => {+ it('extracts the total from "requested about N tokens (...)"', () => {+ const msg = "OpenRouter API error 400: This endpoint's maximum context length is 65536 tokens. However, you requested about 68982 tokens (4982 of text input, 64000 in the output). Please reduce the length of either one, or use the context-compression.";+ expect(parseProviderReportedSize('openrouter', msg)).toEqual({ tokens: 68982, kind: 'total' });+ });++ it('handles variation in thousand separators and whitespace', () => {+ const msg = 'OpenRouter API error 400: requested about 68,847 tokens (4847 of text input, 64000 in the output).';+ expect(parseProviderReportedSize('openrouter', msg)).toEqual({ tokens: 68847, kind: 'total' });+ });++ it('returns null on a non-size error', () => {+ expect(parseProviderReportedSize('openrouter', 'OpenRouter API error 429: Provider returned error')).toBeNull();+ });+ });++ describe('cloudflare', () => {+ it('prefers the input-only number from a 400 context-length body', () => {+ const msg = 'Cloudflare API error 400: AiError: AiError: {"error":{"message":"This model\'s maximum context length is 24000 tokens. However, you requested 256 output tokens and your prompt contains at least 23745 input tokens, for a total of at least 24001 tokens."}}';+ expect(parseProviderReportedSize('cloudflare', msg)).toEqual({ tokens: 23745, kind: 'input' });+ });++ it('falls back to the combined total from a 413 "tokens (N) exceeded" body', () => {+ const msg = 'Cloudflare API error 413: AiError: Ai: The estimated number of input and maximum output tokens (24092) exceeded this model context window limit (24000). (1ffb6b51-7168-4e29-a4ab-378d87917a79)';+ expect(parseProviderReportedSize('cloudflare', msg)).toEqual({ tokens: 24092, kind: 'total' });+ });++ it('returns null on a non-size error', () => {+ expect(parseProviderReportedSize('cloudflare', 'Cloudflare API error 429: AiError: you have used up your daily free allocation')).toBeNull();+ });+ });++ describe('github', () => {+ it('returns null even though the body is parseable (limit only, not request size)', () => {+ // "Max size: 8000 tokens" is the LIMIT ceiling. Returning 8000 would+ // cause every subsequent model with TPM < 8000 to be skipped for the+ // rest of the request — wildly wrong. The parser must refuse.+ const msg = 'GitHub Models API error 413: Request body too large for gpt-4.1 model. Max size: 8000 tokens.';+ expect(parseProviderReportedSize('github', msg)).toBeNull();+ });+ });++ describe('providers without a parser', () => {+ it('returns null for ollama, nvidia, anthropic, google, opencode, llm7, cerebras, custom', () => {+ for (const p of ['ollama', 'nvidia', 'anthropic', 'google', 'opencode', 'llm7', 'cerebras', 'custom']) {+ expect(parseProviderReportedSize(p, 'Any 413 body Requested 12345 tokens here')).toBeNull();+ }+ });+ });++ describe('input safety', () => {+ it('returns null for empty / null / non-string input', () => {+ expect(parseProviderReportedSize('groq', null)).toBeNull();+ expect(parseProviderReportedSize('groq', undefined)).toBeNull();+ expect(parseProviderReportedSize('groq', '')).toBeNull();+ });++ it('returns null for a body whose number is zero or negative', () => {+ expect(parseProviderReportedSize('groq', '... Requested 0 ...')).toBeNull();+ expect(parseProviderReportedSize('openrouter', 'requested about 0 tokens')).toBeNull();+ });++ it('returns null for an unknown platform', () => {+ expect(parseProviderReportedSize('does-not-exist', 'Requested 12345')).toBeNull();+ });+ });+});diff --git a/server/src/__tests__/routes/provider-reported-size.test.ts b/server/src/__tests__/routes/provider-reported-size.test.tsnew file mode 100644index 00000000..e80613c1--- /dev/null+++ b/server/src/__tests__/routes/provider-reported-size.test.ts@@ -0,0 +1,37 @@+import { it, expect, vi } from 'vitest';+import { createApp } from '../../app.js';+import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';+import { encrypt } from '../../lib/crypto.js';+import { getProvider } from '../../providers/index.js';+import { setRoutingStrategy } from '../../services/router.js';++it('chat completions skips a too-small fallback after a provider reports request size', async () => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:'); const db = getDb(); setRoutingStrategy('priority');+ db.prepare('UPDATE models SET enabled = 0').run();+ db.prepare('DELETE FROM profile_models').run();+ for (const [index, window] of [50000, 24000, 50000].entries()) {+ const id = Number(db.prepare("INSERT INTO models(platform,model_id,display_name,intelligence_rank,speed_rank,size_label,context_window,enabled,supports_tools) VALUES ('groq',?,?,1,1,'Small',?,1,1)")+ .run('size-test-' + index, 'Size test ' + index, window).lastInsertRowid);+ db.prepare('INSERT INTO profile_models(profile_id,model_db_id,priority,enabled) SELECT id,?,?,1 FROM profiles').run(id,index);+ }+ const {encrypted,iv,authTag} = encrypt('synthetic-key');+ db.prepare("INSERT INTO api_keys(platform,label,encrypted_key,iv,auth_tag,status,enabled) VALUES ('groq','test',?,?,?,'healthy',1)").run(encrypted,iv,authTag);+ const called: string[] = [];+ const spy = vi.spyOn(getProvider('groq')!, 'chatCompletion').mockImplementation(async (_key, _messages, model) => {+ called.push(model);+ if (model === 'size-test-0') throw Object.assign(new Error('Groq API error 413: Limit 8000, Requested 30000, please reduce your message size'), {status:413});+ return {id:'test',object:'chat.completion',created:1,model,choices:[{index:0,message:{role:'assistant',content:'hello'},finish_reason:'stop'}],usage:{prompt_tokens:3,completion_tokens:1,total_tokens:4}};+ });+ const server = createApp().listen(0,'127.0.0.1');+ if (!server.listening) await new Promise<void>(r => server.once('listening',r));+ try {+ const res = await fetch('http://127.0.0.1:'+(server.address() as {port:number}).port+'/v1/chat/completions', {+ method:'POST', headers:{Authorization:'Bearer '+getUnifiedApiKey(),'Content-Type':'application/json'},+ body:JSON.stringify({model:'auto',messages:[{role:'user',content:'hello'}],max_tokens:256}),+ });+ const body = await res.json();+ expect(res.status, JSON.stringify(body)).toBe(200);+ expect(called).toEqual(['size-test-0','size-test-2']);+ } finally {server.close(); spy.mockRestore();}+});diff --git a/server/src/__tests__/services/provider-reported-routing.test.ts b/server/src/__tests__/services/provider-reported-routing.test.tsnew file mode 100644index 00000000..25cffdff--- /dev/null+++ b/server/src/__tests__/services/provider-reported-routing.test.ts@@ -0,0 +1,29 @@+import { it, expect } from 'vitest';+import { initDb, getDb } from '../../db/index.js';+import { encrypt } from '../../lib/crypto.js';+import { routeRequest, setRoutingStrategy } from '../../services/router.js';+import { newFallbackState, recordRetryableFailure, fallbackRoutingTokens } from '../../lib/fallback-loop.js';+it('does not dispatch to a window smaller than provider input plus output reserve', () => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:'); const db = getDb(); setRoutingStrategy('priority');+ const model = db.prepare("SELECT id FROM models WHERE platform = 'groq' LIMIT 1").get() as any;+ db.prepare('UPDATE models SET enabled = 0 WHERE id != ?').run(model.id);+ db.prepare('UPDATE models SET context_window = 24000, tpm_limit = NULL, rpm_limit = NULL, rpd_limit = NULL, tpd_limit = NULL WHERE id = ?').run(model.id);+ const {encrypted, iv, authTag} = encrypt('synthetic-test-key');+ db.prepare("INSERT INTO api_keys (platform,label,encrypted_key,iv,auth_tag,status,enabled) VALUES ('groq','review',?,?,?,'healthy',1)").run(encrypted,iv,authTag);+ const state = newFallbackState();+ const error = Object.assign(new Error('Cloudflare API error 400: your prompt contains at least 23745 input tokens, for a total of at least 24001 tokens'), {status: 400});+ recordRetryableFailure({platform: 'cloudflare', modelId: 'failed-model', modelDbId: -1, keyId: -1} as any, error, state);+ expect(state.observedInputTokens).toBe(23745);+ expect(fallbackRoutingTokens(state, 4000, 256)).toBe(24001);+ expect(() => routeRequest(fallbackRoutingTokens(state, 4000, 256), undefined, undefined, false, false, undefined, undefined, false, undefined, 256)).toThrow();+ expect(() => routeRequest(fallbackRoutingTokens(state, 4000, 255), undefined, undefined, false, false, undefined, undefined, false, undefined, 255)).not.toThrow();+});+it('adds the output reserve only to input observations', () => {+ const state = newFallbackState();+ expect(fallbackRoutingTokens(state, 4000, 256)).toBe(4000);+ state.observedTotalTokens = 24001;+ expect(fallbackRoutingTokens(state, 4000, 256)).toBe(24001);+ state.observedInputTokens = 24000;+ expect(fallbackRoutingTokens(state, 4000, 256)).toBe(24256);+});diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.tsindex cfbef4f1..48d7d2b6 100644--- a/server/src/lib/fallback-loop.ts+++ b/server/src/lib/fallback-loop.ts@@ -50,6 +50,7 @@ import {isTimeoutErrorText,} from './error-classify.js';import { sanitizeProviderErrorMessage, summarizeAttemptError } from './error-redaction.js';+import { parseProviderReportedSize } from './provider-size-parser.js';import { checkKeyHealth, markKeyHealthyFromRequest } from '../services/health.js';import { noteModelRetirementSignal } from '../services/model-retirement.js';import { getDb, getSetting } from '../db/index.js';@@ -185,12 +186,30 @@ export function getFallbackTimeBudgetMs(): number {// "platform:modelId:keyId"; skipModels holds model_db_ids ruled out for the// rest of this request; skipPlatforms holds platforms ruled out wholesale// (#788) — every model and every key of them — for the rest of this request.-// All three are request-scoped only: nothing here outlives the response, so a-// provider that blipped once is a fresh candidate on the very next request.+// observedTotalTokens is the provider-reported REQUESTED size parsed from the+// first 413 / context-length body of this request, used to inflate the+// routing estimate on the next attempt so models whose tpm_limit /+// context_window can't fit it are skipped before they fire (#507). All four+// are request-scoped only: nothing here outlives the response, so a provider+// that blipped once is a fresh candidate on the very next request.export interface FallbackState {skipKeys: Set<string>;skipModels: Set<number>;skipPlatforms: Set<string>;+ // Highest REQUESTED token count seen so far this request across all 413 /+ // context-length rejections. Max wins (never decreases): a smaller later+ // reading is the same request, not a smaller one.+ observedTotalTokens?: number;+ observedInputTokens?: number;+}++/** Total for the next dispatch. Input-only observations still need output space. */+export function fallbackRoutingTokens(state: FallbackState, estimatedTotal: number, outputReserve: number): number {+ return Math.max(+ estimatedTotal,+ state.observedTotalTokens ?? 0,+ state.observedInputTokens == null ? 0 : state.observedInputTokens + Math.max(0, outputReserve),+ );}export function newFallbackState(): FallbackState {@@ -357,6 +376,24 @@ export function recordRetryableFailure(route: RouteResult, err: any, state: Fall// The trace object identifies the request, so one request's failover across// sibling keys counts as the single observation it is.noteModelRetirementSignal(route, err, getRequestTrace());+ // Provider-reported request size (#507): if the just-failed attempt was a+ // size-related rejection that names the request's real token count, latch it+ // onto the request state. The route() closure on the next attempt picks it+ // up and feeds it as the routing estimate, so the existing tpm_limit /+ // context_window gates in router.ts skip low-ceiling models on retry+ // instead of letting them fire and 413 again. Only fires on a size-class+ // error — the parser is conservative and returns null for opaque bodies+ // (every non-supported platform, the bare "Request Entity Too Large"+ // Groq shape, and the github limit-only case), so behavior is unchanged+ // when there's nothing to latch. Max wins so a smaller later reading of+ // the same request can't drop the estimate.+ if (isContextTooLargeError(err) || isProviderBadRequestError(err)) {+ const reported = parseProviderReportedSize(route.platform, err?.message);+ if (reported != null) {+ const field = reported.kind === 'input' ? 'observedInputTokens' : 'observedTotalTokens';+ state[field] = Math.max(state[field] ?? 0, reported.tokens);+ }+ }state.skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);// #788: provider-level failures (5xx / timeout / transport / degraded) mean// the PROVIDER is sick, not this key — every key AND every model of thatdiff --git a/server/src/lib/inbound-chat.ts b/server/src/lib/inbound-chat.tsindex df32166c..3995169c 100644--- a/server/src/lib/inbound-chat.ts+++ b/server/src/lib/inbound-chat.ts@@ -20,6 +20,7 @@ import {resolveRequestedIdForDispatch,} from '../services/model-groups.js';import {+ fallbackRoutingTokens,newFallbackState,recordUpstreamSuccess,runFallbackLoop,@@ -235,18 +236,25 @@ export async function runInboundChat(attemptLog,logIdentity: { surface: 'inbound chat', requestedModel: input.model ?? 'auto' },clientGone: () => clientGone,- route: () => routeRequest(- estimatedTotal,- state.skipKeys.size ? state.skipKeys : undefined,- pin.preferredModel,- imageRequest,- wantsTools,- state.skipModels.size ? state.skipModels : undefined,- pin.strictChain,- input.responseFormat !== undefined,- state.skipPlatforms.size ? state.skipPlatforms : undefined,- outputReserve,- ),+ route: () => {+ // #507: after the first 413 / context-length rejection the parser+ // latches the provider-reported REQUESTED size onto state. Inflate the+ // routing estimate on the next attempt so low-tpm / small-window models+ // are skipped by the existing gates in router.ts instead of re-firing.+ const routingTotal = fallbackRoutingTokens(state, estimatedTotal, outputReserve);+ return routeRequest(+ routingTotal,+ state.skipKeys.size ? state.skipKeys : undefined,+ pin.preferredModel,+ imageRequest,+ wantsTools,+ state.skipModels.size ? state.skipModels : undefined,+ pin.strictChain,+ input.responseFormat !== undefined,+ state.skipPlatforms.size ? state.skipPlatforms : undefined,+ outputReserve,+ );+ },dispatch: async (route, attempt) => {if (!input.stream) {const result = await route.provider.chatCompletion(diff --git a/server/src/lib/provider-size-parser.ts b/server/src/lib/provider-size-parser.tsnew file mode 100644index 00000000..3ce35bf9--- /dev/null+++ b/server/src/lib/provider-size-parser.ts@@ -0,0 +1,73 @@+// Pull a real token count out of a provider's 413 / context-length error body.+//+// The local token estimator (`text.length / 4`) undercounts tool-heavy /+// code-heavy prompts by 3-10x: a request the estimator scores at 3,700 tokens+// can be reported as 36,532 by Groq. Without this parser the `tpm_limit` and+// `context_window` pre-checks in the router pass on the wrong number, the call+// fires, and the provider 413s — burning one failover hop per model in the+// chain instead of one hop total. This module is the single source of truth for+// "what size did the provider say this request was": one regex per supported+// provider, all derived from the live error corpus.+//+// Returns null for anything it can't pin to a real REQUESTED size (not a LIMIT+// ceiling — see the github case below). Callers feed the result into the+// routing estimate so the existing size gates skip low-TPM / small-context+// models on the next attempt instead of letting them re-fail.+//+// Why github is intentionally NOT parsed: the body is+// `Request body too large for gpt-4.1 model. Max size: 8000 tokens.` The 8000+// is the LIMIT, not the rejected request's size. Returning it would cause+// every subsequent model with TPM < 8000 to be skipped for the rest of the+// request — wildly wrong, since most free models have TPM ~6000. The parser+// returns null and `learnLimitFromError` keeps doing what it already does+// (lowering `models.tpm_limit` to 8000 for github/gpt-4.1).++// Pull the first integer out of a comma-thousands-separator-aware capture.+// Anchored on the literal label the provider uses so a stray digit elsewhere+// in the body can't match. Accepts a regex with one or two capture groups+// (second group is used when the first is undefined), so cloudflare can+// choose between input-only and combined-total via two alternations.+function pullFirstInt(body: string, pattern: RegExp): number | null {+ const m = body.match(pattern);+ if (!m) return null;+ const raw = m[1] != null ? m[1] : m[2];+ if (raw == null) return null;+ // Strip thousand-separator commas before parsing: "36,532" → "36532".+ const cleaned = raw.replace(/,/g, '');+ if (cleaned === '') return null;+ const n = Number(cleaned);+ return Number.isFinite(n) && n > 0 ? n : null;+}++// One regex per provider. Each one is anchored on a literal token from a+// real observed error body so a provider changing its wording trips the test+// and forces a parser update — bare numbers without an anchor would happily+// match an unrelated digit anywhere in the message.+const PATTERNS: Record<string, RegExp> = {+ // "Limit 8000, Requested 36532, please reduce your message size..."+ groq: /Requested\s+([\d,]+)/i,+ // "...requested about 68982 tokens (4982 of text input, 64000 in the output)..."+ openrouter: /requested\s+about\s+([\d,]+)\s+tokens/i,+ // Prefer the INPUT-only count from a 400 ("your prompt contains at least+ // 23745 input tokens, for a total of at least 24001 tokens"). The input-only+ // count anchors on "contains at least N input tokens" so the larger+ // "total of at least 24001" that immediately follows does not win.+ // Fall back to the combined total from a 413 ("tokens (24092) exceeded+ // this model context") via the paren-anchored alternative.+ cloudflare: /(?:contains\s+at\s+least\s+([\d,]+)\s+input\s+tokens|tokens\s*\(([\d,]+)\s*\))/i,+};++export interface ProviderReportedSize {+ tokens: number;+ kind: 'input' | 'total';+}++export function parseProviderReportedSize(platform: string, message: string | null | undefined): ProviderReportedSize | null {+ if (typeof message !== 'string' || message === '') return null;+ const re = PATTERNS[platform];+ if (!re) return null;+ const tokens = pullFirstInt(message, re);+ if (tokens == null) return null;+ const inputOnly = platform === 'cloudflare' && /contains\s+at\s+least\s+[\d,]+\s+input\s+tokens/i.test(message);+ return { tokens, kind: inputOnly ? 'input' : 'total' };+}diff --git a/server/src/routes/anthropic.ts b/server/src/routes/anthropic.tsindex 9ac1e142..581e7551 100644--- a/server/src/routes/anthropic.ts+++ b/server/src/routes/anthropic.ts@@ -21,7 +21,7 @@ import { convertDocumentBlock, documentRejectionMessage } from '../lib/anthropicimport { isClientAbortError, newClientAbortError, newHedgeAbortError, isUpstreamClassificationOutput } from '../lib/error-classify.js';import { logRequest } from '../lib/request-log.js';import { extractApiToken, timingSafeStringEqual, getStickyModel, setStickyModel } from './proxy.js';-import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, type ExhaustionBody, setFallbackHeaders, setExhaustionHeaders, type AttemptRecord } from '../lib/fallback-loop.js';+import { runFallbackLoop, newFallbackState, fallbackRoutingTokens, recordUpstreamSuccess, type ExhaustionBody, setFallbackHeaders, setExhaustionHeaders, type AttemptRecord } from '../lib/fallback-loop.js';import { routedViaValue } from '../lib/header-value.js';import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';import { resolveAnthropicModel, claudeFamilyDiscoveryEntries } from '../services/anthropic-map.js';@@ -618,7 +618,11 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {route: () => {// Task-type routing (#1127): same header/derivation as /chat/completions.const taskType = resolveTaskType(req, tools, messages);- return routeRequest(estimatedTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain, false, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined, outputReserve, taskType);+ // #507: inflate the routing estimate from any provider-reported REQUESTED+ // size latched onto state so the existing size gates in router.ts skip+ // low-TPM / small-context models on retry.+ const routingTotal = fallbackRoutingTokens(state, estimatedTotal, outputReserve);+ return routeRequest(routingTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain, false, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined, outputReserve, taskType);},dispatch: async (route, attempt, dispatchCtx) => {if (stream) {diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 288345a3..f91b28d0 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -24,7 +24,7 @@ import { logRequest } from '../lib/request-log.js';import { observeServedModel } from '../lib/served-model.js';import { parseCacheDirective, cacheActive, isCacheableTemperature, computeCacheKey, getCachedResponse, storeCachedResponse, getCachedStreamResponse, storeCachedStreamResponse, STREAM_CACHE_MAX_BYTES } from '../services/cache.js';import { normalizeIdempotencyKey, hashIdempotencyKey, computeIdempotencyFingerprint, lookupIdempotencyReplay, storeIdempotencyResult } from '../services/idempotency.js';-import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, exhaustedRetryError, setFallbackHeaders, exhaustionErrorPayload, setExhaustionHeaders, type AttemptRecord } from '../lib/fallback-loop.js';+import { runFallbackLoop, newFallbackState, fallbackRoutingTokens, recordUpstreamSuccess, exhaustedRetryError, setFallbackHeaders, exhaustionErrorPayload, setExhaustionHeaders, type AttemptRecord } from '../lib/fallback-loop.js';import { routedViaValue, safeHeaderValue } from '../lib/header-value.js';import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';import { samplingParamSchemaFields, pickSamplingParams, supportedParametersForPlatforms } from '../lib/sampling-params.js';@@ -1168,18 +1168,24 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {logIdentity: { surface: 'legacy completions', requestId: requestGroupId, requestedModel: requestedModelLabel },clientGone: () => clientGone,abortInFlight: () => hedgeAbort.abort(newHedgeAbortError()),- route: () => routeRequest(- estimatedTotal,- state.skipKeys.size > 0 ? state.skipKeys : undefined,- preferredModel,- false,- false,- state.skipModels.size > 0 ? state.skipModels : undefined,- groupChain ?? resolvedChain?.chain,- false,- state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined,- outputReserve,- ),+ route: () => {+ // #507: see inbound-chat.ts — inflate the routing estimate from any+ // provider-reported REQUESTED size latched onto state so the existing+ // size gates in router.ts skip low-TPM / small-context models on retry.+ const routingTotal = fallbackRoutingTokens(state, estimatedTotal, outputReserve);+ return routeRequest(+ routingTotal,+ state.skipKeys.size > 0 ? state.skipKeys : undefined,+ preferredModel,+ false,+ false,+ state.skipModels.size > 0 ? state.skipModels : undefined,+ groupChain ?? resolvedChain?.chain,+ false,+ state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined,+ outputReserve,+ );+ },dispatch: async (route, attempt, ctx) => {traceRouteEvent('Proxy', {event: attempt === 0 ? 'start' : 'next',@@ -2090,7 +2096,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {// the padding is conservative on turns where injection is *possible* (a prior// model is on record). Turns where injection can't happen — every turn 1, and// sessions that never switched — pay no headroom tax.- const routingEstimate = handoffPossible ? estimatedTotal + HANDOFF_MAX_TOKENS : estimatedTotal;+ const routingEstimate = fallbackRoutingTokens(state, estimatedTotal, outputReserve) + (handoffPossible ? HANDOFF_MAX_TOKENS : 0);// Task-type routing (#1127): the client can declare code/chat intent via// header; otherwise a bounded rule derives it (tools present / code// markers). undefined keeps the preset weights untouched.diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.tsindex 21ce5e24..4da0d5b8 100644--- a/server/src/routes/responses.ts+++ b/server/src/routes/responses.ts@@ -29,7 +29,7 @@ import {traceRouteEvent,logRequest,} from './proxy.js';-import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, setFallbackHeaders, exhaustionErrorPayload, setExhaustionHeaders, type AttemptRecord } from '../lib/fallback-loop.js';+import { runFallbackLoop, newFallbackState, fallbackRoutingTokens, recordUpstreamSuccess, setFallbackHeaders, exhaustionErrorPayload, setExhaustionHeaders, type AttemptRecord } from '../lib/fallback-loop.js';import { routedViaValue, safeHeaderValue } from '../lib/header-value.js';import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';import { samplingParamSchemaFields, pickSamplingParams, type ResponseFormat } from '../lib/sampling-params.js';@@ -1112,7 +1112,11 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {route: () => {// Task-type routing (#1127): same header/derivation as /chat/completions.const taskType = resolveTaskType(req, tools, messages);- return routeRequest(estimatedTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain, completionOpts.response_format !== undefined, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined, outputReserve, taskType);+ // #507: inflate the routing estimate from any provider-reported REQUESTED+ // size latched onto state so the existing size gates in router.ts skip+ // low-TPM / small-context models on retry.+ const routingTotal = fallbackRoutingTokens(state, estimatedTotal, outputReserve);+ return routeRequest(routingTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain, completionOpts.response_format !== undefined, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined, outputReserve, taskType);},dispatch: async (route, attempt, ctx) => {traceRouteEvent('Responses', {