7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(router): context-window safety margin against chars/4 under-counting (#956)
Icesenatorf08e17c44be8
diff --git a/server/src/__tests__/services/router.test.ts b/server/src/__tests__/services/router.test.tsindex dab6b599..b9cd8949 100644--- a/server/src/__tests__/services/router.test.ts+++ b/server/src/__tests__/services/router.test.ts@@ -172,6 +172,58 @@ describe('Router', () => {expect(routeRequest(9000).modelDbId).toBe(groq.id);});+ it('prefers margin-fitting models but still serves raw-window fits (#956 review)', () => {+ const db = getDb();+ const groqKey = encrypt('test-groq-key');+ db.prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)+ VALUES (?, ?, ?, ?, ?, ?, ?)+ `).run('groq', 'groq', groqKey.encrypted, groqKey.iv, groqKey.authTag, 'healthy', 1);++ const groq = db.prepare(`+ SELECT id FROM models+ WHERE platform = 'groq' AND context_window = 131072+ LIMIT 1+ `).get() as { id: number };++ db.prepare('UPDATE fallback_config SET priority = 1000, enabled = 1').run();+ db.prepare('UPDATE fallback_config SET priority = 1 WHERE model_db_id = ?').run(groq.id);+ db.prepare('UPDATE models SET tpm_limit = NULL, tpd_limit = NULL WHERE id = ?').run(groq.id);++ // CONTEXT_WINDOW_SAFETY_FACTOR: the chars/4 estimate under-counts dense+ // payloads (JSON, code), so the preferred ceiling is window / 1.25.+ // 131072 / 1.25 = 104857.6. A margin-fitting runner-up on another platform+ // must NOT outrank the priority-1 model while the estimate is inside the+ // margin.+ const googleKey = encrypt('test-google-key');+ db.prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)+ VALUES (?, ?, ?, ?, ?, ?, ?)+ `).run('google', 'google', googleKey.encrypted, googleKey.iv, googleKey.authTag, 'healthy', 1);+ const google = db.prepare(`+ SELECT id FROM models+ WHERE platform = 'google' AND context_window >= 132500 AND enabled = 1+ ORDER BY intelligence_rank ASC+ LIMIT 1+ `).get() as { id: number };+ expect(google).toBeDefined();+ db.prepare('UPDATE fallback_config SET priority = 2 WHERE model_db_id = ?').run(google.id);+ db.prepare('UPDATE models SET tpm_limit = NULL, tpd_limit = NULL WHERE id = ?').run(google.id);++ expect(routeRequest(104000).modelDbId).toBe(groq.id);++ // Above the margin ceiling the priority-1 model is DEMOTED, not excluded:+ // /v1/models advertises the raw window, so a client that packs past the+ // margin gets the margin-fitting runner-up instead of "all models exhausted".+ expect(routeRequest(106000).modelDbId).not.toBe(groq.id);+ expect(routeRequest(106000).modelDbId).toBe(google.id);++ // When nothing fits WITH the margin anywhere, the raw advertised window is+ // still honored — one attempt rather than an empty pool.+ db.prepare("DELETE FROM api_keys WHERE platform = 'google'").run();+ expect(routeRequest(106000).modelDbId).toBe(groq.id);+ });+it('still routes a model with an unknown (null) context window (#167)', () => {const db = getDb();const groqKey = encrypt('test-groq-key');diff --git a/server/src/lib/inbound-chat.ts b/server/src/lib/inbound-chat.tsindex 3aa80979..8eed30ae 100644--- a/server/src/lib/inbound-chat.ts+++ b/server/src/lib/inbound-chat.ts@@ -201,7 +201,11 @@ export async function runInboundChat(const attemptLog: AttemptRecord[] = [];const wantsTools = (input.tools?.length ?? 0) > 0;const imageRequest = hasImages(input.messages);- const estimatedTotal = estimatedInputTokens + routingReserveTokens(maxTokens);+ // Capped reserve (#470); threaded to the router separately because it is an+ // exact count and must not be inflated by the context-window safety margin+ // (#956 review).+ const outputReserve = routingReserveTokens(maxTokens);+ const estimatedTotal = estimatedInputTokens + outputReserve;const schemas = toolSchemaMap(input.tools);let clientGone = false;const clientAbort = new AbortController();@@ -240,6 +244,7 @@ export async function runInboundChat(pin.strictChain,input.responseFormat !== undefined,state.skipPlatforms.size ? state.skipPlatforms : undefined,+ outputReserve,),dispatch: async (route, attempt) => {if (!input.stream) {diff --git a/server/src/routes/anthropic.ts b/server/src/routes/anthropic.tsindex 1593dba1..38bd7ba1 100644--- a/server/src/routes/anthropic.ts+++ b/server/src/routes/anthropic.ts@@ -517,8 +517,11 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {reasoning_effort: effortFromAnthropicThinking(body.thinking),};// Capped output reserve so a large max_tokens can't falsely exclude the model- // pool (#470); input + images count in full.- const estimatedTotal = estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE + routingReserveTokens(max_tokens);+ // pool (#470); input + images count in full. Threaded to the router+ // separately: it is exact and must not be inflated by the context-window+ // safety margin (#956 review).+ const outputReserve = routingReserveTokens(max_tokens);+ const estimatedTotal = estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE + outputReserve;// Resolve the model through the operator's Claude-family map (opus/sonnet/// haiku/default → auto | a pinned catalog model). A concrete catalog id pins@@ -569,7 +572,7 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {attemptLog,clientGone: () => clientGone,abortInFlight: () => hedgeAbort.abort(newHedgeAbortError()),- route: () => routeRequest(estimatedTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, undefined, false, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined),+ route: () => routeRequest(estimatedTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, undefined, false, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined, outputReserve),dispatch: async (route, attempt, dispatchCtx) => {if (stream) {try {diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 22ea63a3..45b058a9 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -878,8 +878,11 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {const messages = prependSystemPrompt(completionPromptToMessages(prompt, suffix), auth.systemPrompt);const estimatedInputTokens = messages.reduce((sum, m) => sum + Math.ceil(contentToString(m.content).length / 4), 0);// Cap the reserved output so a huge client-set max_tokens doesn't falsely- // exclude the whole model pool (#470); input is still counted in full.- const estimatedTotal = estimatedInputTokens + routingReserveTokens(max_tokens);+ // exclude the whole model pool (#470); input is still counted in full. The+ // reserve is passed to the router separately: it is an exact count and must+ // not be inflated by the context-window safety margin (#956 review).+ const outputReserve = routingReserveTokens(max_tokens);+ const estimatedTotal = estimatedInputTokens + outputReserve;// Guardrail: per-request token budget (request_max_tokens_budget, default// off). max_tokens always has a value on this surface (default 128), so a@@ -991,6 +994,7 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {groupChain ?? resolvedChain?.chain,false,state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined,+ outputReserve,),dispatch: async (route, attempt, ctx) => {traceRouteEvent('Proxy', {@@ -1448,8 +1452,11 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {const imageCount = messages.reduce((n, m) =>n + (Array.isArray(m.content) ? m.content.filter(b => (b as { type?: string })?.type === 'image_url' || (b as { type?: string })?.type === 'image').length : 0), 0);// The reserved output is capped (routingReserveTokens, #470) so an oversized- // client max_tokens can't starve routing; input + images count in full.- const estimatedTotal = estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE + routingReserveTokens(max_tokens);+ // client max_tokens can't starve routing; input + images count in full. The+ // reserve is threaded to the router separately: it is exact and must not be+ // inflated by the context-window safety margin (#956 review).+ const outputReserve = routingReserveTokens(max_tokens);+ const estimatedTotal = estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE + outputReserve;// Tool-bearing requests must route to a model that emits STRUCTURED// tool_calls. A model without real function-calling support serializes the@@ -1814,7 +1821,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {// model is on record). Turns where injection can't happen — every turn 1, and// sessions that never switched — pay no headroom tax.const routingEstimate = handoffPossible ? estimatedTotal + HANDOFF_MAX_TOKENS : estimatedTotal;- return routeRequest(routingEstimate, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain ?? resolvedChain?.chain, samplingParams.response_format !== undefined, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined);+ return routeRequest(routingEstimate, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain ?? resolvedChain?.chain, samplingParams.response_format !== undefined, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined, outputReserve);},dispatch: async (route, attempt, ctx) => {const modelKey = `${route.platform}:${route.modelId}`;diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.tsindex ec718a99..18c3f025 100644--- a/server/src/routes/responses.ts+++ b/server/src/routes/responses.ts@@ -640,8 +640,11 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {const imageCount = messages.reduce((n, m) =>n + (Array.isArray(m.content) ? m.content.filter(b => (b as { type?: string })?.type === 'image_url' || (b as { type?: string })?.type === 'image').length : 0), 0);// Capped output reserve so a large max_output_tokens can't falsely exclude the- // model pool (#470); input counts in full.- const estimatedTotal = estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE + routingReserveTokens(reqData.max_output_tokens);+ // model pool (#470); input counts in full. Threaded to the router separately:+ // it is exact and must not be inflated by the context-window safety margin+ // (#956 review).+ const outputReserve = routingReserveTokens(reqData.max_output_tokens);+ const estimatedTotal = estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE + outputReserve;// Guardrail: per-request token budget (request_max_tokens_budget, default// off). A request with no max_output_tokens gets its output capped to the@@ -768,7 +771,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {attemptLog,clientGone: () => clientGone,abortInFlight: () => hedgeAbort.abort(newHedgeAbortError()),- route: () => routeRequest(estimatedTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain, completionOpts.response_format !== undefined, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined),+ route: () => routeRequest(estimatedTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain, completionOpts.response_format !== undefined, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined, outputReserve),dispatch: async (route, attempt, ctx) => {traceRouteEvent('Responses', {event: attempt === 0 ? 'start' : 'next',diff --git a/server/src/services/router.ts b/server/src/services/router.tsindex 879863ce..c3288891 100644--- a/server/src/services/router.ts+++ b/server/src/services/router.ts@@ -1347,6 +1347,54 @@ function getModelChainRow(db: Db, modelDbId: number): ChainRow | undefined {`).get(modelDbId) as ChainRow | undefined;}+/**+ * Safety margin applied when ranking a model against an estimated request size.+ * The estimate is a chars/4 heuristic that under-counts dense payloads (JSON,+ * code, CJK) by up to ~2x; without any accounting for that gap such requests+ * were routed to models whose real tokenizer count exceeded the window and the+ * provider rejected them with a 400 mid-chain (kilo: "maximum context length is+ * 262144 tokens" on requests estimated <=256000).+ *+ * The margin is a SOFT preference, not a hard filter (#956 review): /v1/models+ * advertises the RAW window, so clients legitimately pack requests right up to+ * it. Excluding margin-violating models outright would turn an upstream 400+ * that the retry loop already classifies and handles (`context_too_large`)+ * into a regression: "all models exhausted" with zero attempts. Callers+ * therefore try margin-fitting candidates first and only fall back to raw+ * advertised-window fits (see fitsContextWindowStrict) when nothing else can+ * serve the request.+ */+export const CONTEXT_WINDOW_SAFETY_FACTOR = 1.25;++// Platforms whose pre-dispatch trim guard already caps the dispatched input+// below the live context ceiling (lib/content.ts truncateMessagesForGithub):+// the guard — not the routing estimate — is what guarantees the fit there, so+// applying the factor too would only make that guard unreachable. The margin+// checks below treat these platforms as strict comparisons.+const TRIM_GUARDED_PLATFORMS = new Set(['github']);++/** True when `estimatedTokens` fits the RAW advertised window (null window =+ * unknown, never filtered — same convention as the auto-router). This is the+ * comparison /v1/models publishes and the soft-preference fallback tier. */+export function fitsContextWindowStrict(contextWindow: number | null | undefined, estimatedTokens: number): boolean {+ return contextWindow == null || estimatedTokens <= contextWindow;+}++/** True when `estimatedTokens` plausibly fits `contextWindow` WITH the safety+ * margin. The chars/4 heuristic portion is scaled by the factor; an explicit+ * output reserve derived from the client's max_tokens (`routingReserveTokens`)+ * is already an exact count and is added UNSCALED (#956 review). Trim-guarded+ * platforms compare strictly — their guard guarantees the fit. */+export function fitsContextWindow(platform: string, contextWindow: number | null | undefined, estimatedTokens: number, exactOutputReserve = 0): boolean {+ if (contextWindow == null) return true;+ // Raw advertised comparison first — the margin can only shrink eligibility.+ if (estimatedTokens > contextWindow) return false;+ if (TRIM_GUARDED_PLATFORMS.has(platform)) return true;+ const reserve = Math.max(0, exactOutputReserve);+ const heuristic = Math.max(0, estimatedTokens - reserve);+ return heuristic * CONTEXT_WINDOW_SAFETY_FACTOR + reserve <= contextWindow;+}+/*** Route to ONE specific model, hard-pinned. Rotates across that model's keys* (cooldowns, quotas, decryption all honored) but NEVER substitutes a different@@ -1359,7 +1407,12 @@ export function routePinnedModel(modelDbId: number, estimatedTokens = 1000, skipconst db = getDb();const entry = getModelChainRow(db, modelDbId);if (!entry) return null;- if (entry.context_window != null && estimatedTokens > entry.context_window) return null;+ // Strict comparison only (#956 review): a pinned slot has no substitute, so+ // refusing on a margin violation would drop the slot outright where the+ // pre-margin behavior was one dispatch attempt (a mid-chain context_too_large+ // 400 is classified and retried downstream). Nothing is multiplied here —+ // estimatedTokens already carries the exact capped output reserve (#470).+ if (!fitsContextWindowStrict(entry.context_window, estimatedTokens)) return null;if (entry.tpm_limit != null && estimatedTokens > entry.tpm_limit) return null;return selectKeyForModel(entry, estimatedTokens, skipKeys);}@@ -1446,7 +1499,7 @@ export interface FusionCandidate {* so the panel's auto-pick draws from the highest-scored models first and the* fusion layer just needs to apply provider-diversity on top.*/-export function getOrderedFusionChain(estimatedTokens: number): FusionCandidate[] {+export function getOrderedFusionChain(estimatedTokens: number, exactOutputReserve = 0): FusionCandidate[] {const db = getDb();const strategy = getRoutingStrategy();if (strategy !== 'priority') refreshStatsCache(db);@@ -1479,10 +1532,20 @@ export function getOrderedFusionChain(estimatedTokens: number): FusionCandidate[const arr = keysByPlatform.get(k.platform);if (arr) arr.push(entry); else keysByPlatform.set(k.platform, [entry]);}- const servable = chain.filter(e => {+ // Soft preference (#956 review): prefer models whose window holds the estimate+ // WITH the safety margin; /v1/models still advertises the raw window, so if+ // NOTHING survives that pass, re-run allowing raw advertised-window fits+ // rather than handing back an empty chain — a request packed to the+ // advertised window keeps its one attempt (the mid-chain context_too_large+ // 400 is classified and retried downstream).+ const passesContextGate = (e: ChainRow, allowMarginViolators: boolean) => {// A null context_window means "unknown", not "zero": same convention the// auto-router uses, so an unspecified window is never itself a reason to skip.- if (e.context_window != null && estimatedTokens > e.context_window) return false;+ if (fitsContextWindow(e.platform, e.context_window, estimatedTokens, exactOutputReserve)) return true;+ return allowMarginViolators && fitsContextWindowStrict(e.context_window, estimatedTokens);+ };+ const servableFilter = (allowMarginViolators: boolean) => chain.filter(e => {+ if (!passesContextGate(e, allowMarginViolators)) return false;const keyIds = keysByPlatform.get(e.platform);if (!keyIds) return false;// Same endpoint-pool rule the router applies (#619).@@ -1500,6 +1563,8 @@ export function getOrderedFusionChain(estimatedTokens: number): FusionCandidate[canUseProviderTokens(e.platform, kid, e.model_id, estimatedTokens),);});+ let servable = servableFilter(false);+ if (servable.length === 0) servable = servableFilter(true);// Deterministic (expected-score) ordering so the panel faithfully follows the// user's picked routing strategy instead of re-sampling a fresh draw each call.@@ -1570,7 +1635,7 @@ export function resolveFusionCandidate(modelId: string): FusionCandidate | nullreturn null;}-export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preferredModelDbId?: number, requireVision = false, requireTools = false, skipModels?: Set<number>, prefetchedChain?: ChainRow[], requireStructured = false, skipPlatforms?: Set<string>): RouteResult {+export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preferredModelDbId?: number, requireVision = false, requireTools = false, skipModels?: Set<number>, prefetchedChain?: ChainRow[], requireStructured = false, skipPlatforms?: Set<string>, exactOutputReserve = 0): RouteResult {const db = getDb();const strategy = getRoutingStrategy();@@ -1612,7 +1677,7 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preif (requireVision && !e.supports_vision) return false;if (requireTools && !e.supports_tools) return false;if (requireStructured && platformDropsResponseFormat(e.platform)) return false;- if (e.context_window != null && estimatedTokens > e.context_window) return false;+ if (!fitsContextWindow(e.platform, e.context_window, estimatedTokens, exactOutputReserve)) return false;if (e.tpm_limit != null && estimatedTokens > e.tpm_limit) return false;return true;});@@ -1659,7 +1724,21 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre// synchronous "all exhausted" path (nothing downstream logs it). See issue _1.const diag: string[] = [];- for (const entry of sortedChain) {+ // Margin as a SOFT preference (#956 review): /v1/models advertises the raw+ // window, so clients legitimately pack requests right up to it — excluding+ // those models outright turned an already-handled upstream 400 into "all+ // models exhausted" with zero attempts. Keep the operator's order intact but+ // sweep margin-fitting models first; ones that only fit the advertised window+ // stay eligible behind them. Worst case is one classified context_too_large+ // hop instead of no route at all.+ const servingChain: ChainRow[] = [];+ const marginDeferred: ChainRow[] = [];+ for (const e of sortedChain) {+ (fitsContextWindow(e.platform, e.context_window, estimatedTokens, exactOutputReserve) ? servingChain : marginDeferred).push(e);+ }+ servingChain.push(...marginDeferred);++ for (const entry of servingChain) {const label = `${entry.platform}/${entry.model_id}`;// Models the caller has ruled out for this request — e.g. a 404// "model removed upstream" already seen this request: trying the same@@ -1693,18 +1772,28 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre// it are caught by the non-stream JSON enforcement downstream.if (requireStructured && platformDropsResponseFormat(entry.platform)) { diag.push(`${label}: platform drops response_format`); continue; }- // Context-aware routing: skip a model whose context window can't hold the- // request, so a large prompt never selects a small-context model and burns- // a failover hop on a 413 "request too large" (#167). Only enforced when we- // know the model's window; estimatedTokens is the INPUT estimate plus a- // CAPPED output reserve (routingReserveTokens, #470), so a huge client-set- // max_tokens no longer excludes the model — the input must fit, not- // input+full max_tokens. A 413 that slips through is still retryable- // downstream, and the failed model is put on cooldown — so this is a- // fast-path, not the only guard. If every model is too small, the loop falls- // through and the caller gets the normal "all models exhausted" error rather- // than a wasted sweep.- if (entry.context_window != null && estimatedTokens > entry.context_window) { diag.push(`${label}: context ${entry.context_window} < estimated ${estimatedTokens}`); continue; }+ // Context-aware routing fast path (#167): skip a model whose RAW advertised+ // window cannot hold the request — a dispatch there is a guaranteed 413.+ // Margin-violating-but-raw-fitting models are NOT skipped (soft preference,+ // #956 review): they were merely deferred to the back of the sweep above.+ // estimatedTokens is the INPUT estimate plus a CAPPED output reserve+ // (routingReserveTokens, #470), so a huge client-set max_tokens no longer+ // excludes the model — the input must fit, not input+full max_tokens. A 413+ // that slips through is still retryable downstream, and the failed model is+ // put on cooldown — so this is a fast-path, not the only guard. If every+ // model is too small, the loop falls through and the caller gets the normal+ // "all models exhausted" error rather than a wasted sweep.+ if (!fitsContextWindowStrict(entry.context_window, estimatedTokens)) {+ // Keep the `< estimated` substring — summarizeExhaustion buckets prompt+ // overflow off it. Emit the EFFECTIVE number so the line doesn't read as+ // false (#956 review): e.g. `context 131072 < estimated 106000 x1.25 = 132500`.+ const reserve = Math.max(0, exactOutputReserve);+ const requiredWithMargin = Math.ceil(Math.max(0, estimatedTokens - reserve) * CONTEXT_WINDOW_SAFETY_FACTOR + reserve);+ diag.push(TRIM_GUARDED_PLATFORMS.has(entry.platform)+ ? `${label}: context ${entry.context_window} < estimated ${estimatedTokens}`+ : `${label}: context ${entry.context_window} < estimated ${estimatedTokens} x${CONTEXT_WINDOW_SAFETY_FACTOR} = ${requiredWithMargin}`);+ continue;+ }// Same guard for a model with a small per-minute token budget: a request// whose input alone exceeds tpm_limit can never fit one minute of quota and