7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(fusion): return a non-retryable 413 when the prompt is too large for the panel (#1479) (#1485)
Tashfeend000f3ff5682
diff --git a/server/src/__tests__/lib/fallback-loop.test.ts b/server/src/__tests__/lib/fallback-loop.test.tsindex 45421823..65711e1f 100644--- a/server/src/__tests__/lib/fallback-loop.test.ts+++ b/server/src/__tests__/lib/fallback-loop.test.ts@@ -200,6 +200,29 @@ describe('isDailyQuotaExhaustedError + midnight benching (drift: 90s cooldown on});});+describe('recordRetryableFailure context-length rejection (#1479)', () => {+ it('learns the reported window and rules the model out without a cooldown or penalty', () => {+ const modelDbId = getDb().prepare(+ `INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, context_window)+ VALUES ('fake', ?, 'Fake', 50, 50, 1000000)`,+ ).run(`ctx-${keySeq}`).lastInsertRowid as number;+ const route = fakeRoute({ modelDbId });+ const state = newFallbackState();+ const err = Object.assign(+ new Error("OpenRouter API error 400: This endpoint's maximum context length is 262144 tokens. However, you requested about 404225 tokens (396033 of text input, 8192 in the output)."),+ { status: 400 },+ );+ recordRetryableFailure(route, err, state);++ expect(state.skipModels.has(modelDbId)).toBe(true);+ const window = (getDb().prepare('SELECT context_window FROM models WHERE id = ?').get(modelDbId) as { context_window: number }).context_window;+ expect(window).toBe(262144);+ const cooldown = getDb().prepare('SELECT 1 FROM rate_limit_cooldowns WHERE platform = ? AND key_id = ?').get('fake', route.keyId);+ expect(cooldown).toBeUndefined();+ expect(getAllPenalties().some(p => p.modelDbId === modelDbId)).toBe(false);+ });+});+describe('recordRetryableFailure skipBench exemption (reasoning truncation)', () => {it('skips cooldown + penalty but still rules the key out for this request', () => {const route = fakeRoute();diff --git a/server/src/__tests__/routes/fusion.test.ts b/server/src/__tests__/routes/fusion.test.tsindex 1dada99c..a8652e0b 100644--- a/server/src/__tests__/routes/fusion.test.ts+++ b/server/src/__tests__/routes/fusion.test.ts@@ -6,7 +6,7 @@ import { mintDashboardToken, isGatedApiPath } from '../helpers/auth.js';import { isFusionModel, fusionConfigSchema, familyKey, diversifyChain } from '../../services/fusion.js';import { getOrderedFusionChain, setRoutingStrategy, getRoutingStrategy, type FusionCandidate } from '../../services/router.js';import * as router from '../../services/router.js';-import { setCooldown } from '../../services/ratelimit.js';+import { setCooldown, isOnCooldown } from '../../services/ratelimit.js';let dashToken = '';@@ -62,7 +62,11 @@ function mockUpstreams(answers: Record<string, MockAnswer>, rateLimited: Set<strcalls.push({ url: u, body });for (const host of rateLimited) {if (u.includes(host)) {- return { ok: false, status: 429, headers: new Headers(), text: () => Promise.resolve('rate limited') } as any;+ return {+ ok: false, status: 429, headers: new Headers(),+ text: () => Promise.resolve('rate limited'),+ json: () => Promise.resolve({ error: { message: 'rate limited' } }),+ } as any;}}// Streaming (judge) hosts return an SSE body consumed by streamChatCompletion.@@ -100,6 +104,49 @@ function mockUpstreams(answers: Record<string, MockAnswer>, rateLimited: Set<strreturn { calls };}+// Stage upstream HTTP errors by host ({ status, message } in the OpenAI error+// shape), with optional successful answers for other hosts. Records every call.+function mockUpstreamErrors(errors: Record<string, { status: number; message: string }>, answers: Record<string, string> = {}) {+ const origFetch = global.fetch;+ const calls: string[] = [];+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {+ const u = typeof url === 'string' ? url : url.toString();+ calls.push(u);+ for (const [host, e] of Object.entries(errors)) {+ if (u.includes(host)) {+ const body = { error: { message: e.message } };+ return {+ ok: false, status: e.status, statusText: 'error', headers: new Headers(),+ text: () => Promise.resolve(JSON.stringify(body)),+ json: () => Promise.resolve(body),+ } as any;+ }+ }+ for (const [host, content] of Object.entries(answers)) {+ if (u.includes(host)) {+ return {+ ok: true, status: 200, headers: new Headers(),+ json: () => Promise.resolve({+ id: 'chatcmpl-x', object: 'chat.completion', created: 1, model: 'm',+ choices: [{ index: 0, message: { role: 'assistant', content }, finish_reason: 'stop' }],+ usage: { prompt_tokens: 5, completion_tokens: 7, total_tokens: 12 },+ }),+ } as any;+ }+ }+ return origFetch(url as any, init as any);+ });+ return { calls };+}++// Real context-length rejection bodies. The OpenRouter one is the #1479 body+// with the window scaled down so the test prompt can stay small.+const OPENROUTER_TOO_LONG = "This endpoint's maximum context length is 2048 tokens. However, you requested about 4300 tokens (4044 of text input, 256 in the output). Please reduce the length of either one, or use the \"middle-out\" transform to compress your prompt automatically.";+const GROQ_TOO_LONG = "This model's maximum context length is 3000 tokens. However, your messages resulted in 4044 tokens. Please reduce the length of the messages.";+// ~4000 estimated tokens: larger than either learned window, far below the+// catalog windows the tests stage.+const BIG_PROMPT = 'x'.repeat(16_000);+describe('isFusionModel', () => {it('matches the virtual fusion id and its suffix form, nothing else', () => {expect(isFusionModel('fusion')).toBe(true);@@ -237,6 +284,147 @@ describe('fusion route (/v1/chat/completions, model: "fusion")', () => {expect(body.error.type).toBe('rate_limit_error');});+ describe('prompt too large for the panel (#1479)', () => {+ // Stage roomy catalog windows and no TPM/TPD caps so the prompt passes the+ // pre-dispatch size gates and only the provider's real (smaller) window+ // rejects it, as in the issue. Restored after each test: the DB is shared.+ let saved: Array<{ id: number; context_window: number | null; tpm_limit: number | null; tpd_limit: number | null }> = [];+ beforeEach(() => {+ const db = getDb();+ saved = db.prepare(+ "SELECT id, context_window, tpm_limit, tpd_limit FROM models WHERE platform IN ('groq', 'cerebras', 'openrouter')",+ ).all() as typeof saved;+ db.prepare(+ "UPDATE models SET context_window = 1000000, tpm_limit = NULL, tpd_limit = NULL WHERE platform IN ('groq', 'cerebras', 'openrouter')",+ ).run();+ });+ afterEach(() => {+ const restore = getDb().prepare('UPDATE models SET context_window = ?, tpm_limit = ?, tpd_limit = ? WHERE id = ?');+ for (const r of saved) restore.run(r.context_window, r.tpm_limit, r.tpd_limit, r.id);+ });++ const keyId = (platform: string) =>+ (getDb().prepare('SELECT id FROM api_keys WHERE platform = ?').get(platform) as { id: number }).id;+ const contextWindow = (platform: string, modelId: string) =>+ (getDb().prepare('SELECT context_window FROM models WHERE platform = ? AND model_id = ?').get(platform, modelId) as { context_window: number | null }).context_window;++ it('returns a non-retryable 413 when every panel model rejects the prompt as too long', async () => {+ const { calls } = mockUpstreamErrors({+ 'openrouter.ai': { status: 400, message: OPENROUTER_TOO_LONG },+ 'api.groq.com': { status: 400, message: GROQ_TOO_LONG },+ });+ const body = {+ model: 'fusion',+ max_tokens: 256,+ messages: [{ role: 'user', content: BIG_PROMPT }],+ fusion: { models: [openrouterModel, groqModel] },+ };+ const first = await request(app, 'POST', '/v1/chat/completions', body, authHeaders());++ expect(first.status).toBe(413);+ expect(first.body.error.type).toBe('invalid_request_error');+ expect(first.body.error.code).toBe('context_length_exceeded');+ expect(first.body.error.message).toContain('Retrying will not help');+ expect(first.body.error.message).toContain('maximum context length is 2048');+ // One dispatch per model: a sibling key would hit the same window.+ expect(calls.filter(u => u.includes('openrouter.ai'))).toHaveLength(1);+ expect(calls.filter(u => u.includes('api.groq.com'))).toHaveLength(1);+ // Too big for the request is not a model-health problem: no cooldown.+ expect(isOnCooldown('openrouter', openrouterModel, keyId('openrouter'))).toBe(false);+ expect(isOnCooldown('groq', groqModel, keyId('groq'))).toBe(false);+ // The real windows were learned over the catalog's 1M.+ expect(contextWindow('openrouter', openrouterModel)).toBe(2048);+ expect(contextWindow('groq', groqModel)).toBe(3000);++ // The retry is sized out before dispatch and still answers 413, not 429.+ calls.length = 0;+ const second = await request(app, 'POST', '/v1/chat/completions', body, authHeaders());+ expect(second.status).toBe(413);+ expect(second.body.error.code).toBe('context_length_exceeded');+ expect(calls.filter(u => u.includes('openrouter.ai') || u.includes('api.groq.com'))).toHaveLength(0);+ });++ it('keeps 429 when a panel model too small for the prompt sits beside a rate-limited one', async () => {+ mockUpstreamErrors({+ 'openrouter.ai': { status: 400, message: OPENROUTER_TOO_LONG },+ 'api.groq.com': { status: 429, message: 'Rate limit reached for requests per minute' },+ });+ const { status, body } = await request(app, 'POST', '/v1/chat/completions', {+ model: 'fusion',+ max_tokens: 256,+ messages: [{ role: 'user', content: BIG_PROMPT }],+ fusion: { models: [openrouterModel, groqModel] },+ }, authHeaders());+ // The rate-limited model may still answer once it recovers.+ expect(status).toBe(429);+ expect(body.error.type).toBe('rate_limit_error');+ });++ it('returns 502 when the panel fails upstream without a rate limit', async () => {+ mockUpstreamErrors({+ 'openrouter.ai': { status: 400, message: OPENROUTER_TOO_LONG },+ 'api.groq.com': { status: 500, message: 'internal server error' },+ });+ const { status, body } = await request(app, 'POST', '/v1/chat/completions', {+ model: 'fusion',+ max_tokens: 256,+ messages: [{ role: 'user', content: BIG_PROMPT }],+ fusion: { models: [openrouterModel, groqModel] },+ }, authHeaders());+ expect(status).toBe(502);+ expect(body.error.type).toBe('server_error');+ expect(body.error.message).toContain(groqModel);+ });++ it('still answers when one panel model is too small and another fits', async () => {+ mockUpstreamErrors(+ { 'openrouter.ai': { status: 400, message: OPENROUTER_TOO_LONG } },+ { 'api.groq.com': 'groq fits' },+ );+ const { status, body } = await request(app, 'POST', '/v1/chat/completions', {+ model: 'fusion',+ max_tokens: 256,+ messages: [{ role: 'user', content: BIG_PROMPT }],+ fusion: { models: [openrouterModel, groqModel] },+ }, authHeaders());+ expect(status).toBe(200);+ expect(body.choices[0].message.content).toBe('groq fits');+ });++ it('returns 413 without dispatching when the auto-panel has no model large enough', async () => {+ getDb().prepare("UPDATE models SET context_window = 1000 WHERE platform IN ('groq', 'cerebras', 'openrouter')").run();+ const { calls } = mockUpstreamErrors({});+ const { status, body } = await request(app, 'POST', '/v1/chat/completions', {+ model: 'fusion',+ max_tokens: 256,+ messages: [{ role: 'user', content: BIG_PROMPT }],+ }, authHeaders());+ expect(status).toBe(413);+ expect(body.error.code).toBe('context_length_exceeded');+ expect(calls.filter(u => !u.startsWith('http://127.0.0.1'))).toHaveLength(0);+ });++ it('carries the 413 code on the streaming error frame', async () => {+ mockUpstreamErrors({+ 'openrouter.ai': { status: 400, message: OPENROUTER_TOO_LONG },+ 'api.groq.com': { status: 400, message: GROQ_TOO_LONG },+ });+ const { text } = await request(app, 'POST', '/v1/chat/completions', {+ model: 'fusion',+ stream: true,+ max_tokens: 256,+ messages: [{ role: 'user', content: BIG_PROMPT }],+ fusion: { models: [openrouterModel, groqModel] },+ }, authHeaders());+ const errorFrame = text.split('\n')+ .filter(l => l.startsWith('data: ') && l !== 'data: [DONE]')+ .map(l => JSON.parse(l.slice(6)))+ .find(f => f.error);+ expect(errorFrame.error.code).toBe('context_length_exceeded');+ expect(errorFrame.error.type).toBe('invalid_request_error');+ });+ });+it('drops unknown models from an explicit panel and reports them', async () => {mockUpstreams({ 'api.groq.com': 'groq answer' });const { status, body } = await request(app, 'POST', '/v1/chat/completions', {diff --git a/server/src/__tests__/services/learn-context-window.test.ts b/server/src/__tests__/services/learn-context-window.test.tsnew file mode 100644index 00000000..1dfc29f9--- /dev/null+++ b/server/src/__tests__/services/learn-context-window.test.ts@@ -0,0 +1,88 @@+import { describe, it, expect, beforeAll } from 'vitest';+import { initDb, getDb } from '../../db/index.js';+import { parseProviderContextWindow, learnContextWindowFromError } from '../../services/ratelimit.js';++describe('parseProviderContextWindow', () => {+ it('parses the OpenRouter endpoint body from #1479', () => {+ const msg = "OpenRouter API error 400: This endpoint's maximum context length is 262144 tokens. However, you requested about 404225 tokens (396033 of text input, 8192 in the output). Please reduce the length of either one, or use the \"middle-out\" transform to compress your prompt automatically.";+ expect(parseProviderContextWindow(msg)).toBe(262144);+ });++ it('parses the OpenAI-compatible wording', () => {+ expect(parseProviderContextWindow("Groq API error 400: This model's maximum context length is 131072 tokens. However, your messages resulted in 140000 tokens.")).toBe(131072);+ });++ it('parses the Mistral wording', () => {+ expect(parseProviderContextWindow('Prompt contains 140000 tokens and 0 draft tokens, too large for model with 131072 maximum context length')).toBe(131072);+ });++ it('parses the Anthropic wording', () => {+ expect(parseProviderContextWindow('prompt is too long: 210000 tokens > 200000 maximum')).toBe(200000);+ });++ it('parses the Google wording', () => {+ expect(parseProviderContextWindow('The input token count (1200000) exceeds the maximum number of tokens allowed (1048576).')).toBe(1048576);+ });++ it('strips thousands separators', () => {+ expect(parseProviderContextWindow('maximum context length is 262,144 tokens')).toBe(262144);+ });++ it('never reads the requested size or a TPM limit as the window', () => {+ expect(parseProviderContextWindow('you requested about 404225 tokens (396033 of text input, 8192 in the output)')).toBeNull();+ expect(parseProviderContextWindow('Request too large for model on tokens per minute (TPM): Limit 30000, Requested 33476')).toBeNull();+ expect(parseProviderContextWindow('Request body too large for gpt-4.1 model. Max size: 8000 tokens.')).toBeNull();+ });++ it('rejects an implausibly small window as a misread', () => {+ expect(parseProviderContextWindow('maximum context length is 512 tokens')).toBeNull();+ });++ it('returns null for empty and unrelated errors', () => {+ expect(parseProviderContextWindow('')).toBeNull();+ expect(parseProviderContextWindow(undefined)).toBeNull();+ expect(parseProviderContextWindow('API error 500: internal server error')).toBeNull();+ });+});++describe('learnContextWindowFromError (persistence)', () => {+ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ });++ function insertModel(contextWindow: number | null): number {+ const info = getDb().prepare(+ `INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, context_window)+ VALUES ('openrouter', ?, 'M', 50, 50, ?)`,+ ).run(`m-${Math.random().toString(36).slice(2)}`, contextWindow);+ return info.lastInsertRowid as number;+ }+ const windowOf = (id: number) =>+ (getDb().prepare('SELECT context_window FROM models WHERE id = ?').get(id) as { context_window: number | null }).context_window;+ const tooLong = { message: "This endpoint's maximum context length is 262144 tokens. However, you requested about 404225 tokens." };++ it('lowers a catalog window the endpoint does not honor', () => {+ const id = insertModel(1_000_000);+ expect(learnContextWindowFromError(id, tooLong)).toBe(262144);+ expect(windowOf(id)).toBe(262144);+ });++ it('fills an unknown window', () => {+ const id = insertModel(null);+ expect(learnContextWindowFromError(id, tooLong)).toBe(262144);+ expect(windowOf(id)).toBe(262144);+ });++ it('never raises a window that is already lower', () => {+ const id = insertModel(131072);+ expect(learnContextWindowFromError(id, tooLong)).toBeNull();+ expect(windowOf(id)).toBe(131072);+ });++ it('is a no-op for an error that names no window', () => {+ const id = insertModel(200000);+ expect(learnContextWindowFromError(id, { message: 'Request Entity Too Large' })).toBeNull();+ expect(windowOf(id)).toBe(200000);+ });+});diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.tsindex 49f0dc4d..ac725f5c 100644--- a/server/src/lib/fallback-loop.ts+++ b/server/src/lib/fallback-loop.ts@@ -29,6 +29,7 @@ import {getPaymentRequiredCooldownMs,getModelForbiddenCooldownMs,learnLimitFromError,+ learnContextWindowFromError,type CooldownDecision,type CooldownSource,} from '../services/ratelimit.js';@@ -508,6 +509,7 @@ export function recordRetryableFailure(route: RouteResult, err: any, state: Fall// model it touched, so later ordinary requests found the pool rate limited.if (isContextTooLargeError(err)) {learnLimitFromError(route.modelDbId, err);+ learnContextWindowFromError(route.modelDbId, err);return false;}// A max_tokens above this model's output ceiling (Claude Code asks fordiff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 49761731..0449e750 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -1829,7 +1829,8 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {const type = err instanceof FusionError? (err.status === 429 ? 'rate_limit_error' : err.status >= 500 ? 'server_error' : 'invalid_request_error'): 'server_error';- writeFrame({ error: { message, type } });+ const code = err instanceof FusionError ? err.code : undefined;+ writeFrame({ error: { message, type, ...(code ? { code } : {}) } });}try { res.write('data: [DONE]\n\n'); res.end(); } catch { /* socket gone */ }return;@@ -1869,6 +1870,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {error: {message: err.message,type: err.status === 429 ? 'rate_limit_error' : err.status >= 500 ? 'server_error' : 'invalid_request_error',+ ...(err.code ? { code: err.code } : {}),},execution_id: requestGroupId,});diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.tsindex f39b1986..faa30f53 100644--- a/server/src/routes/responses.ts+++ b/server/src/routes/responses.ts@@ -914,6 +914,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {error: {message: err.message,type: err.status === 429 ? 'rate_limit_error' : err.status >= 500 ? 'server_error' : 'invalid_request_error',+ ...(err.code ? { code: err.code } : {}),},});} else {@@ -1096,11 +1097,12 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {const type = err instanceof FusionError? (err.status === 429 ? 'rate_limit_error' : err.status >= 500 ? 'server_error' : 'invalid_request_error'): 'server_error';+ const code = err instanceof FusionError ? err.code : undefined;fusionSse('response.failed', {response: {...skeleton,status: 'failed',- error: { message, type },+ error: { message, type, ...(code ? { code } : {}) },},});res.end();diff --git a/server/src/services/fusion.ts b/server/src/services/fusion.tsindex 9ac52ccb..b3108a79 100644--- a/server/src/services/fusion.ts+++ b/server/src/services/fusion.ts@@ -2,17 +2,20 @@ import { z } from 'zod';import type { ChatMessage, ChatCompletionChoice, ChatCompletionResponse, ChatToolCall, TokenUsage } from '@freellmapi/shared/types.js';import {routePinnedModel, routeRequest, getOrderedFusionChain, resolveFusionCandidate,+ pinnedModelFitsRequestSize,recordRateLimitHit, recordSuccess, type RouteResult, type FusionCandidate,} from './router.js';import {recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit,getCooldownDecisionForLimit,getPaymentRequiredCooldownMs, getModelForbiddenCooldownMs,+ learnLimitFromError, learnContextWindowFromError,} from './ratelimit.js';import { logRequest } from '../lib/request-log.js';import {isRetryableError, isRateLimitSignal, isPaymentRequiredError,isModelNotFoundError, isModelAccessForbiddenError,+ isContextTooLargeError, isDailyQuotaExhaustedError,} from '../lib/error-classify.js';import { contentToString, stripImagesFromMessages } from '../lib/content.js';import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';@@ -222,9 +225,15 @@ interface PanelAnswer {toolCalls?: ChatToolCall[];rawChoice?: ChatCompletionChoice;error?: string;+ failure?: CallFailure;usage?: TokenUsage;}+// Why a model call produced no answer, so an all-failed panel can report the+// right status: a prompt too big for every model will never fit (413), a+// rate-limited panel may recover (429), anything else is an upstream failure.+type CallFailure = 'context_too_large' | 'rate_limited' | 'error';+interface CallOutcome {ok: boolean;route?: RouteResult;@@ -233,6 +242,30 @@ interface CallOutcome {rawChoice?: ChatCompletionChoice;usage?: TokenUsage;error?: string;+ failure?: CallFailure;+}++function classifyCallFailure(err: any): CallFailure {+ if (isContextTooLargeError(err)) return 'context_too_large';+ if (isRateLimitSignal(err) || isPaymentRequiredError(err) || isDailyQuotaExhaustedError(err)) return 'rate_limited';+ return 'error';+}++// A model too small for this request stays too small for it on every key, so+// once one attempt says so the call's failure is context_too_large for good.+function mergeCallFailure(current: CallFailure | undefined, next: CallFailure): CallFailure {+ return current === 'context_too_large' ? current : next;+}++// Too big for this model is a fact about the REQUEST, not the model's health,+// the same rule as recordRetryableFailure in lib/fallback-loop.ts: skip the+// model for this request and learn the ceiling the provider reported so the+// router sizes it out before the next oversized dispatch, but never cool it+// down or penalize it (#1479).+function noteContextTooLarge(route: RouteResult, err: any, skipModels: Set<number>): void {+ learnLimitFromError(route.modelDbId, err);+ learnContextWindowFromError(route.modelDbId, err);+ skipModels.add(route.modelDbId);}const ZERO_USAGE: TokenUsage = { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 };@@ -259,10 +292,14 @@ async function runModelCall(options: CompletionOptions,estimatedTokens: number,maxAttempts: number,+ // Why the call failed when no route was ever available. Defaults to+ // rate_limited (every key cooling down or over quota).+ noRouteFailure: () => CallFailure = () => 'rate_limited',): Promise<CallOutcome> {const skipKeys = new Set<string>();const skipModels = new Set<number>();let lastError: string | undefined;+ let failure: CallFailure | undefined;for (let attempt = 0; attempt < maxAttempts; attempt++) {let route: RouteResult | null;@@ -274,6 +311,12 @@ async function runModelCall(break;}if (!route) break;+ // A pinned panel slot keeps handing back its one model; once that model is+ // ruled out for this request there is nothing left to try.+ if (skipModels.has(route.modelDbId)) {+ route.release?.();+ break;+ }const startedAt = Date.now();const routeOpts = { ...options, contextBudget: routeOutputBudget(route, estimatedTokens) };@@ -291,6 +334,7 @@ async function runModelCall(setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));recordRateLimitHit(route.modelDbId);lastError = `empty completion from ${route.displayName}`;+ failure = mergeCallFailure(failure, 'error');continue;}@@ -311,7 +355,12 @@ async function runModelCall(const safe = sanitizeProviderErrorMessage(err?.message);logRequest(route.platform, route.modelId, route.keyId, 'error', 0, 0, Date.now() - startedAt, safe, null, FUSION_TAG, null, 'http');lastError = safe;+ failure = mergeCallFailure(failure, classifyCallFailure(err));+ if (isContextTooLargeError(err)) {+ noteContextTooLarge(route, err, skipModels);+ continue;+ }if (isRetryableError(err)) {if (isModelNotFoundError(err) || isModelAccessForbiddenError(err)) skipModels.add(route.modelDbId);skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);@@ -337,7 +386,13 @@ async function runModelCall(}}- return { ok: false, error: lastError ?? 'no available key for model' };+ if (failure == null) {+ failure = noRouteFailure();+ if (failure === 'context_too_large' && lastError == null) {+ lastError = `request (~${estimatedTokens} tokens) does not fit this model's context window`;+ }+ }+ return { ok: false, error: lastError ?? 'no available key for model', failure };}/**@@ -363,6 +418,11 @@ async function runJudgeStreaming(let route: RouteResult | null;try { route = getRoute(skipKeys, skipModels); } catch (err: any) { lastError = sanitizeProviderErrorMessage(err?.message); break; }if (!route) break;+ // A pinned judge keeps handing back its one model (see runModelCall).+ if (skipModels.has(route.modelDbId)) {+ route.release?.();+ break;+ }const startedAt = Date.now();const routeOpts = { ...options, contextBudget: routeOutputBudget(route, estimatedTokens) };@@ -405,6 +465,10 @@ async function runJudgeStreaming(}break;}+ if (isContextTooLargeError(err)) {+ noteContextTooLarge(route, err, skipModels);+ continue;+ }if (isRetryableError(err)) {if (isModelNotFoundError(err) || isModelAccessForbiddenError(err)) skipModels.add(route.modelDbId);skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);@@ -572,12 +636,56 @@ export interface FusionResult {*/export class FusionError extends Error {status: number;- constructor(message: string, status: number) {+ code?: string;+ constructor(message: string, status: number, code?: string) {super(message);this.status = status;+ this.code = code;}}+function contextTooLargeError(estimatedTokens: number, trail = ''): FusionError {+ return new FusionError(+ `fusion: the request (~${estimatedTokens} tokens) is too large for every panel model's context window or token limit. ` ++ 'Retrying will not help: reduce the prompt/history size, or send it to a larger-context model instead of fusion.' ++ trail,+ 413,+ 'context_length_exceeded',+ );+}++/**+ * The error for a panel with no survivors, chosen from why each slot failed+ * (the same ladder as exhaustedRetryError in lib/fallback-loop.ts):+ * - every slot too small for the prompt → 413: it can never fit, and a 429+ * here sent agent clients into a retry loop on a request that cannot+ * succeed (#1479);+ * - any slot rate-limited → 429: that model may answer once it recovers;+ * - otherwise (timeouts, 5xx, bad responses) → 502: the upstreams failed.+ */+function emptyPanelError(answers: PanelAnswer[], estimatedTokens: number): FusionError {+ const failures = answers.map(a => a.failure ?? 'error');+ const trail = ` Panel: ${answers.map(a => `${a.modelId} (${truncateReason(a.error)})`).join('; ')}.`;+ if (failures.length > 0 && failures.every(f => f === 'context_too_large')) {+ return contextTooLargeError(estimatedTokens, trail);+ }+ if (failures.includes('rate_limited')) {+ return new FusionError(+ 'fusion: every panel model failed or was rate-limited. Try again shortly or pick different `fusion.models`.' + trail,+ 429,+ );+ }+ return new FusionError(+ 'fusion: every panel model failed upstream. Try again or pick different `fusion.models`.' + trail,+ 502,+ );+}++function truncateReason(reason: string | undefined): string {+ const text = reason ?? 'failed';+ return text.length > 200 ? `${text.slice(0, 197)}...` : text;+}+// Progress hooks for a streaming fusion request. Fired as each panel slot// settles and when the judge succeeds, so a client (the Playground) can show the// panel answers arriving and the judge kicking in before the final answer.@@ -616,6 +724,13 @@ export async function runFusion(params: {const acceptsToolCall = options.tool_choice !== 'none';const { panel, overflow, dropped } = selectPanel(config, { requireTools, requireVision: vision, estimatedTokens });if (panel.length === 0) {+ // The auto-panel drops models whose context window or TPM ceiling cannot+ // hold the prompt. When that is the only reason the panel is empty, the+ // request can never be served as sent: say so with a 413 instead of+ // pointing the user at their model setup.+ if (selectPanel(config, { requireTools, requireVision: vision, estimatedTokens: 1 }).panel.length > 0) {+ throw contextTooLargeError(estimatedTokens);+ }const hint = vision? 'No vision-capable model is servable for the panel. Enable a vision model in the Fallback Chain or pass `fusion.models` with vision-capable model ids.': 'Provide `fusion.models` with enabled model ids, or enable models in the Fallback Chain.';@@ -633,6 +748,7 @@ export async function runFusion(params: {runModelCall((skipKeys) => routePinnedModel(cand.modelDbId, estimatedTokens, skipKeys),messages, slotOptions, estimatedTokens, MAX_SLOT_ATTEMPTS,+ () => pinnedModelFitsRequestSize(cand.modelDbId, estimatedTokens) ? 'rate_limited' : 'context_too_large',).then((outcome): PanelAnswer => {const returnedToolCalls = outcome.toolCalls;const forbiddenToolCall = options.tool_choice === 'none' && !!returnedToolCalls?.length;@@ -662,6 +778,7 @@ export async function runFusion(params: {: forbiddenToolCall? 'provider returned a tool call despite tool_choice=none': outcome.error,+ failure: outcome.ok ? 'error' : outcome.failure,};hooks?.onPanel?.({ platform: answer.platform, model: answer.modelId, status: answer.status, content: answer.content, tool_calls: answer.toolCalls, error: answer.error });return answer;@@ -713,7 +830,7 @@ export async function runFusion(params: {settled.forEach((s, i) => {const a: PanelAnswer = s.status === 'fulfilled'? s.value- : { modelDbId: wave[i].modelDbId, platform: wave[i].platform, modelId: wave[i].modelId, displayName: wave[i].displayName, status: 'failed', error: sanitizeProviderErrorMessage((s as PromiseRejectedResult).reason?.message) };+ : { modelDbId: wave[i].modelDbId, platform: wave[i].platform, modelId: wave[i].modelId, displayName: wave[i].displayName, status: 'failed', error: sanitizeProviderErrorMessage((s as PromiseRejectedResult).reason?.message), failure: 'error' };answers.push(a);if (a.status === 'ok' && (a.content || (a.toolCalls?.length ?? 0) > 0)) okCount++;});@@ -724,12 +841,7 @@ export async function runFusion(params: {let totalUsage: TokenUsage = { ...ZERO_USAGE };for (const a of survivors) totalUsage = addUsage(totalUsage, a.usage);- if (survivors.length === 0) {- throw new FusionError(- 'fusion: every panel model failed or was rate-limited. Try again shortly or pick different `fusion.models`.',- 429,- );- }+ if (survivors.length === 0) throw emptyPanelError(answers, estimatedTokens);// Tool calls are actions, not prose. They cannot be safely synthesized across// models, so the first panel survivor that returned structured tool_callsdiff --git a/server/src/services/ratelimit.ts b/server/src/services/ratelimit.tsindex fbc9c2be..06dc5156 100644--- a/server/src/services/ratelimit.ts+++ b/server/src/services/ratelimit.ts@@ -1515,3 +1515,59 @@ export function learnLimitFromError(modelDbId: number, err: { message?: string });return result && result.changes > 0 ? parsed : null;}++// ── Learning real context windows from context-length rejections (#1479) ──+// The catalog's context_window is the model's advertised window, but a relay+// can route to an endpoint with a smaller one. Observed live: OpenRouter's+// nvidia/nemotron-3-super-120b-a12b:free passed the pre-dispatch size gate and+// then answered "This endpoint's maximum context length is 262144 tokens".+// Without learning that ceiling, every later oversized request dispatches to+// the same endpoint and fails the same way.+//+// Each pattern is anchored on a context-window phrase from a real provider+// body, so a token count elsewhere in the message (the requested size, an+// output budget) is never read as the window.+const CONTEXT_WINDOW_PATTERNS: RegExp[] = [+ // OpenAI / OpenRouter / vLLM / DeepSeek: "maximum context length is 262144 tokens"+ /maximum context length (?:is|of)\s+([\d,]+)/i,+ // Mistral: "too large for model with 131072 maximum context length"+ /([\d,]+)\s+maximum context length/i,+ // Anthropic: "prompt is too long: 210000 tokens > 200000 maximum"+ /prompt is too long:\s*[\d,]+\s*tokens\s*>\s*([\d,]+)\s*maximum/i,+ // Google: "exceeds the maximum number of tokens allowed (1048576)"+ /exceeds the maximum number of tokens allowed\s*\(([\d,]+)\)/i,+];+// A window below this is almost certainly a misread, and a wrong low value+// would size the model out of nearly every request until the next catalog sync.+export const MIN_LEARNED_CONTEXT_WINDOW = 1024;++/** The context window a provider names when it rejects a prompt as too long,+ * or null when the message names none. */+export function parseProviderContextWindow(message: string | undefined | null): number | null {+ if (!message) return null;+ for (const re of CONTEXT_WINDOW_PATTERNS) {+ const m = message.match(re);+ if (!m) continue;+ const window = Number(m[1]!.replace(/,/g, ''));+ if (Number.isFinite(window) && window >= MIN_LEARNED_CONTEXT_WINDOW) return window;+ }+ return null;+}++/**+ * Persist a provider-reported context window onto the model row, with the same+ * only-more-conservative rule as learnLimitFromError: fill an unknown window+ * or lower one that was too high, never raise it. The router's context gates+ * then size the model out of prompts it cannot hold before they dispatch.+ * Returns the learned window when the row changed, else null.+ */+export function learnContextWindowFromError(modelDbId: number, err: { message?: string }): number | null {+ const window = parseProviderContextWindow(err?.message);+ if (window == null) return null;+ const result = withDb(db =>+ db.prepare(+ 'UPDATE models SET context_window = ? WHERE id = ? AND (context_window IS NULL OR context_window > ?)',+ ).run(window, modelDbId, window),+ );+ return result && result.changes > 0 ? window : null;+}diff --git a/server/src/services/router.ts b/server/src/services/router.tsindex c47ce4aa..c5ab8338 100644--- a/server/src/services/router.ts+++ b/server/src/services/router.ts@@ -1844,11 +1844,26 @@ export function routePinnedModel(modelDbId: number, estimatedTokens = 1000, skip// pre-margin behavior was one dispatch attempt (a mid-chain context_too_large// 400 is classified and retried downstream). Nothing is multiplied here —// estimatedTokens already carries the exact capped output reserve (#470).- if (!fitsContextWindowStrict(entry.context_window, estimatedTokens)) return null;- if (entry.tpm_limit != null && estimatedTokens > entry.tpm_limit) return null;+ if (!rowFitsRequestSize(entry, estimatedTokens)) return null;return selectKeyForModel(entry, estimatedTokens, skipKeys);}+function rowFitsRequestSize(entry: ChainRow, estimatedTokens: number): boolean {+ if (!fitsContextWindowStrict(entry.context_window, estimatedTokens)) return false;+ return entry.tpm_limit == null || estimatedTokens <= entry.tpm_limit;+}++/**+ * Whether a pinned model's known context window and TPM ceiling can hold a+ * request this size. Lets a caller tell "routePinnedModel returned null because+ * the request is too big" (permanent for this request) from "no key is free+ * right now" (transient). An unknown model is not a size problem.+ */+export function pinnedModelFitsRequestSize(modelDbId: number, estimatedTokens: number): boolean {+ const entry = getModelChainRow(getDb(), modelDbId);+ return !entry || rowFitsRequestSize(entry, estimatedTokens);+}+/*** Resolve a logical model group's member db ids to an ordered ChainRow[] for* strict group-pin routing (the "unify" feature). Each catalog-enabled member