7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(routing): size agent requests with their tool schemas, stop reading size errors as rate limits (#1362)
Tashfeen8fd72ca7a349
diff --git a/server/src/__tests__/lib/content.test.ts b/server/src/__tests__/lib/content.test.tsindex ede11d52..cfb10c0f 100644--- a/server/src/__tests__/lib/content.test.ts+++ b/server/src/__tests__/lib/content.test.ts@@ -1,5 +1,5 @@import { describe, it, expect } from 'vitest';-import { GITHUB_MAX_INPUT_TOKENS, contentToString, flattenMessageContent, messageHasImage, normalizeOutboundContent, sanitizeResponse, stripImagesFromMessages, truncateMessagesForGithub } from '../../lib/content.js';+import { GITHUB_MAX_INPUT_TOKENS, contentToString, estimateInputTokens, flattenMessageContent, messageHasImage, normalizeOutboundContent, sanitizeResponse, stripImagesFromMessages, truncateMessagesForGithub } from '../../lib/content.js';import type { ChatMessage } from '@freellmapi/shared/types.js';describe('contentToString', () => {@@ -270,3 +270,29 @@ describe('truncateMessagesForGithub', () => {expect(out).toEqual([image]);});});++describe('estimateInputTokens', () => {+ const messages: ChatMessage[] = [{ role: 'user', content: 'x'.repeat(400) }];++ it('counts message text at ~4 chars per token', () => {+ expect(estimateInputTokens(messages)).toBe(100);+ });++ it('counts the tool schemas the request carries', () => {+ // Claude Code ships ~65 KB of tool schemas next to a few KB of messages;+ // an estimate without them routed its turns to models they could not fit.+ const tools = [{ type: 'function', function: { name: 'Read', description: 'd'.repeat(4000), parameters: { type: 'object' } } }];+ const withTools = estimateInputTokens(messages, tools);+ expect(withTools).toBeGreaterThan(100 + 1000);+ expect(estimateInputTokens(messages, [])).toBe(100);+ });++ it('counts assistant tool-call arguments', () => {+ const withCall: ChatMessage[] = [...messages, {+ role: 'assistant',+ content: null,+ tool_calls: [{ id: 'c1', type: 'function', function: { name: 'Write', arguments: 'a'.repeat(795) } }],+ } as ChatMessage];+ expect(estimateInputTokens(withCall)).toBe(100 + 200);+ });+});diff --git a/server/src/__tests__/lib/fallback-loop-size-skip.test.ts b/server/src/__tests__/lib/fallback-loop-size-skip.test.tsindex 0be05dbd..d794d219 100644--- a/server/src/__tests__/lib/fallback-loop-size-skip.test.ts+++ b/server/src/__tests__/lib/fallback-loop-size-skip.test.ts@@ -18,7 +18,7 @@ import {recordRetryableFailure,resetModelFailureWindows,} from '../../lib/fallback-loop.js';-import { resetKeyLocalityCache } from '../../services/ratelimit.js';+import { isOnCooldown, resetKeyLocalityCache } from '../../services/ratelimit.js';import {setRoutingStrategy,clearAllPenalties,@@ -236,3 +236,19 @@ describe('FallbackState.observedTotalTokens (#507)', () => {expect(routingTotal).toBe(localEstimate);});});++describe('request-too-large failures', () => {+ it('skip the model for this request without benching it for smaller ones', () => {+ // An oversized agent turn says nothing about the model's health; a+ // cooldown here made the next ordinary request find the pool rate limited.+ // A key of its own: earlier cases leave in-memory cooldowns on `keyId`.+ const freshKey = insertKey('size-no-bench');+ resetKeyLocalityCache();+ const state = newFallbackState();+ const err = Object.assign(new Error('Groq API error 413: Request too large for model `x` on tokens per minute (TPM): Limit 8000, Requested 17679, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing'), { status: 413 });+ recordRetryableFailure(routeFor(modelSmall, freshKey), err, state);+ expect(state.skipModels.has(modelSmall.id)).toBe(true);+ expect(state.observedTotalTokens).toBe(17679);+ expect(isOnCooldown(PLATFORM, modelSmall.model_id, freshKey)).toBe(false);+ });+});diff --git a/server/src/__tests__/lib/fallback-loop.test.ts b/server/src/__tests__/lib/fallback-loop.test.tsindex 0686087a..45421823 100644--- a/server/src/__tests__/lib/fallback-loop.test.ts+++ b/server/src/__tests__/lib/fallback-loop.test.ts@@ -156,6 +156,23 @@ describe('isDailyQuotaExhaustedError + midnight benching (drift: 90s cooldown onexpect(isDailyQuotaExhaustedError(new Error('503 Service Unavailable'))).toBe(false);});+ // Live Groq bodies (2026-09-29): every limit error ends with the same+ // "Upgrade to Dev Tier today" upsell, which used to read as a daily marker.+ const GROQ_UPSELL = 'Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing';++ it('does not read Groq\'s "Upgrade to Dev Tier today" upsell as a daily quota', () => {+ const tooLarge = Object.assign(new Error(`Groq API error 413: Request too large for model \`openai/gpt-oss-20b\` in organization \`org_x\` service tier \`on_demand\` on tokens per minute (TPM): Limit 8000, Requested 12122, please reduce your message size and try again. ${GROQ_UPSELL}`), { status: 413 });+ const perMinute = Object.assign(new Error(`Groq API error 429: Rate limit reached for model \`openai/gpt-oss-120b\` in organization \`org_x\` service tier \`on_demand\` on tokens per minute (TPM): Limit 8000, Used 6000, Requested 4000. Please try again in 15s. ${GROQ_UPSELL}`), { status: 429 });+ expect(isDailyQuotaExhaustedError(tooLarge)).toBe(false);+ expect(classifyAttemptError(tooLarge)).toBe('context_too_large');+ expect(isDailyQuotaExhaustedError(perMinute)).toBe(false);+ });++ it('still flags a Groq requests-per-day 429 that carries the upsell', () => {+ const perDay = Object.assign(new Error(`Groq API error 429: Rate limit reached for model \`llama-3.3-70b-versatile\` in organization \`org_x\` service tier \`on_demand\` on requests per day (RPD): Limit 1000, Used 1000, Requested 1. Please try again in 1m26s. Need more requests? Upgrade to Dev Tier today at https://console.groq.com/settings/billing`), { status: 429 });+ expect(isDailyQuotaExhaustedError(perDay)).toBe(true);+ });+it('cooldownForError benches a daily-allocation 429 until the next UTC midnight', () => {const route = fakeRoute();const err = Object.assign(new Error('you have used up your daily free allocation of 10,000 neurons'), { status: 429 });diff --git a/server/src/__tests__/routes/anthropic.test.ts b/server/src/__tests__/routes/anthropic.test.tsindex e69c1ed9..c1917338 100644--- a/server/src/__tests__/routes/anthropic.test.ts+++ b/server/src/__tests__/routes/anthropic.test.ts@@ -390,6 +390,17 @@ describe('Anthropic-compatible /v1/messages', () => {expect(body.input_tokens).toBeGreaterThan(0);});+ it('counts tool schemas in /v1/messages/count_tokens', async () => {+ const messages = [{ role: 'user', content: 'count these tokens please' }];+ const bare = await request(app, '/v1/messages/count_tokens', { model: 'claude-sonnet-4-5', messages }, anthropicHeaders());+ const withTools = await request(app, '/v1/messages/count_tokens', {+ model: 'claude-sonnet-4-5',+ messages,+ tools: [{ name: 'Read', description: 'r'.repeat(4000), input_schema: { type: 'object', properties: {} } }],+ }, anthropicHeaders());+ expect(withTools.body.input_tokens).toBeGreaterThan(bare.body.input_tokens + 1000);+ });+it('returns an Anthropic invalid_request_error on a malformed body', async () => {const { status, body } = await request(app, '/v1/messages', {model: 'claude-sonnet-4-5', messages: [],diff --git a/server/src/lib/content.ts b/server/src/lib/content.tsindex a2c7eb69..e2c1eed7 100644--- a/server/src/lib/content.ts+++ b/server/src/lib/content.ts@@ -33,6 +33,23 @@ export function contentToString(content: unknown): string {return '';}+// Rough input-token count (~4 chars per token) used for routing and budget+// checks: message text, assistant tool-call arguments, and the tool schemas+// the request carries. Tool schemas go upstream on every turn and count+// against the provider's context window and TPM ceiling. An agent client like+// Claude Code sends ~16k tokens of them next to ~4k of messages, so leaving+// them out routed its requests to models they could never fit (a Groq 8k-TPM+// model answered 413 to an "estimated" 4k request that was really 17.7k).+export function estimateInputTokens(messages: ChatMessage[], tools?: readonly unknown[]): number {+ const messageTokens = messages.reduce((sum, m) => {+ const calls = (m.tool_calls ?? [])+ .reduce((n, call) => n + call.function.name.length + call.function.arguments.length, 0);+ return sum + Math.ceil((contentToString(m.content).length + calls) / 4);+ }, 0);+ const toolTokens = tools && tools.length > 0 ? Math.ceil(JSON.stringify(tools).length / 4) : 0;+ return messageTokens + toolTokens;+}+export function flattenMessageContent(messages: ChatMessage[]): ChatMessage[] {return messages.map((m) => ({...m,diff --git a/server/src/lib/error-classify.ts b/server/src/lib/error-classify.tsindex e1949c3e..a17a1a9a 100644--- a/server/src/lib/error-classify.ts+++ b/server/src/lib/error-classify.ts@@ -352,9 +352,17 @@ export function isKeyAuthError(err: any): boolean {// makes the router re-pick a dead-for-the-day provider all day, so the caller// benches until the next UTC midnight instead. Requires BOTH a daily marker and// a quota/allocation marker so an ordinary per-minute 429 never matches.+//+// Groq ends every limit error, per-minute and request-too-large alike, with+// "Need more tokens? Upgrade to Dev Tier today at …". That "today" is sales+// copy, not a window, so it is cut before looking for a daily marker; and a+// request too big for the model is never a spent allowance.+const UPGRADE_UPSELL = /\bupgrade to [^.?!]*?\btoday\b/g;+export function isDailyQuotaExhaustedError(err: any): boolean {if (err?.dailyQuotaExhausted === true) return true;- const msg = (err?.message ?? '').toLowerCase();+ if (isContextTooLargeError(err)) return false;+ const msg = (err?.message ?? '').toLowerCase().replace(UPGRADE_UPSELL, '');if (!/daily|per[ -_]?day|\btoday\b/.test(msg)) return false;return /allocation|quota|limit|exhaust|used up/.test(msg);}diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.tsindex 22dfd585..c437e6ee 100644--- a/server/src/lib/fallback-loop.ts+++ b/server/src/lib/fallback-loop.ts@@ -499,6 +499,16 @@ export function recordRetryableFailure(route: RouteResult, err: any, state: Fallif (isProviderLevelError(err)) {state.skipPlatforms.add(route.platform);}+ // Too big for this model is a fact about the REQUEST, not the model's+ // health: it still serves every smaller request. Skip it for this request+ // (above) and learn the reported ceiling so the router sizes it out next+ // time, but no cooldown and no penalty. Benching it let one oversized agent+ // turn (Claude Code ships ~16k tokens of tool schemas) sink every small-TPM+ // model it touched, so later ordinary requests found the pool rate limited.+ if (isContextTooLargeError(err)) {+ learnLimitFromError(route.modelDbId, err);+ return false;+ }if (consumeSkipBenchExemption(route, err)) return true;const decision = cooldownDecisionForError(route, err);setCooldown(route.platform, route.modelId, route.keyId, decision.durationMs, decision.source);diff --git a/server/src/lib/inbound-chat.ts b/server/src/lib/inbound-chat.tsindex d7692fb3..bc2647fe 100644--- a/server/src/lib/inbound-chat.ts+++ b/server/src/lib/inbound-chat.ts@@ -31,7 +31,7 @@ import {} from './fallback-loop.js';import { routedViaValue } from './header-value.js';import { applyTokenBudget, tokenBudgetMessage } from './guardrails.js';-import { contentToString } from './content.js';+import { contentToString, estimateInputTokens } from './content.js';import { normalizeMessageImages } from './image-normalize.js';import { repairToolArguments, toolSchemaMap } from './tool-args.js';import { invalidToolArgumentsError, invalidToolCallReasons, isToolArgumentValidationEnabled } from './tool-validate.js';@@ -86,15 +86,6 @@ export interface InboundChatWire {sendStreamError?(res: Response, message: string): void;}-function estimateTokens(messages: ChatMessage[]): number {- return messages.reduce((sum, message) => {- const text = contentToString(message.content);- const calls = (message.tool_calls ?? [])- .reduce((n, call) => n + call.function.name.length + call.function.arguments.length, 0);- return sum + Math.ceil((text.length + calls) / 4);- }, 0);-}-function hasImages(messages: ChatMessage[]): boolean {return messages.some(message =>Array.isArray(message.content)@@ -181,7 +172,7 @@ export async function runInboundChat(// token budgets, payload limits, and upstream transfers all see the shrunk// bytes (see lib/image-normalize.ts). Mutates the image blocks in place.await normalizeMessageImages(input.messages);- const estimatedInputTokens = estimateTokens(input.messages);+ const estimatedInputTokens = estimateInputTokens(input.messages, input.tools);const budget = applyTokenBudget(estimatedInputTokens, input.maxTokens);if (budget.rejection) {wire.sendError(res, 413, tokenBudgetMessage(budget.rejection), 'request_token_budget');diff --git a/server/src/routes/anthropic.ts b/server/src/routes/anthropic.tsindex e4ea5898..b12826e9 100644--- a/server/src/routes/anthropic.ts+++ b/server/src/routes/anthropic.ts@@ -11,7 +11,7 @@ import type {} from '@freellmapi/shared/types.js';import { routeRequest, resolveModelGroupCandidates, resolveRoutingChain, resolveStickyPreference, routingReserveTokens, type RouteResult, type ResolvedChain, type ChainRow } from '../services/router.js';import { getSetting, getUnifiedApiKey } from '../db/index.js';-import { contentToString } from '../lib/content.js';+import { contentToString, estimateInputTokens } from '../lib/content.js';import { resolveTaskType } from '../lib/task-type.js';import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';import { invalidToolArgumentsError, invalidToolCallReasons, isToolArgumentValidationEnabled } from '../lib/tool-validate.js';@@ -374,10 +374,6 @@ function convertRequest(input: AnthropicRequest): ConvertedRequest {};}-function estimateTokens(messages: ChatMessage[]): number {- return messages.reduce((sum, m) => sum + Math.ceil(contentToString(m.content).length / 4), 0);-}-// Model resolution (Claude family → auto | pinned catalog model) lives in// services/anthropic-map.ts so the dashboard mapping editor and this route// share one source of truth. Claude Code keeps its built-in `claude-*` names;@@ -505,7 +501,7 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {messages = compressionResult.messages;res.setHeader('X-FreeLLM-Compress', formatCompressionHeader(compressionResult));- const estimatedInputTokens = estimateTokens(messages);+ const estimatedInputTokens = estimateInputTokens(messages, tools);const imageCount = messages.reduce((n, m) =>n + (Array.isArray(m.content) ? m.content.filter(b => (b as any)?.type === 'image_url').length : 0), 0);@@ -1109,7 +1105,7 @@ anthropicRouter.post('/messages/count_tokens', (req: Request, res: Response) =>recordStats: false,});res.setHeader('X-FreeLLM-Compress', formatCompressionHeader(compressionResult));- res.json({ input_tokens: estimateTokens(compressionResult.messages) });+ res.json({ input_tokens: estimateInputTokens(compressionResult.messages, tools) });});// Anthropic-compatible GET /v1/models. Content-negotiated: only answers whendiff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 0383cfcf..864454b5 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -11,7 +11,7 @@ import { runImageGeneration, runVideoGeneration, runSpeech, runTranscription, Meimport multer from 'multer';import { getDb } from '../db/index.js';import { resolveAuth, prependSystemPrompt, type ResolvedAuth } from '../lib/system-prompt.js';-import { contentToString, messageHasImage, normalizeOutboundContent, sanitizeResponse, truncateMessagesForGithub } from '../lib/content.js';+import { contentToString, estimateInputTokens, messageHasImage, normalizeOutboundContent, sanitizeResponse, truncateMessagesForGithub } from '../lib/content.js';import { resolveTaskType } from '../lib/task-type.js';import { normalizeMessageImages } from '../lib/image-normalize.js';import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';@@ -1629,10 +1629,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {// Non-streaming requests reconcile against the provider's real `usage` block;// streaming does the same when stream_options.include_usage produces a final// usage frame, and otherwise falls back to this estimate.- const estimatedInputTokens = messages.reduce((sum, m) => {- const text = contentToString(m.content);- return sum + Math.ceil(text.length / 4);- }, 0);+ const estimatedInputTokens = estimateInputTokens(messages, tools);// Image requests must route to a vision-capable model. Reject up front with a// clear message when none is enabled, rather than silently dropping the imagediff --git a/server/src/routes/responses.ts b/server/src/routes/responses.tsindex 69e7ec27..8b7445cc 100644--- a/server/src/routes/responses.ts+++ b/server/src/routes/responses.ts@@ -13,7 +13,7 @@ import { routeRequest, hasEnabledVisionModel, hasEnabledToolsModel, resolveStickimport { getDb } from '../db/index.js';import { resolveAuth, prependSystemPrompt } from '../lib/system-prompt.js';import { isUnifyEnabled, getModelGroups, resolveRequestedIdForDispatch } from '../services/model-groups.js';-import { contentToString, messageHasImage } from '../lib/content.js';+import { contentToString, estimateInputTokens, messageHasImage } from '../lib/content.js';import { resolveTaskType } from '../lib/task-type.js';import { normalizeMessageImages } from '../lib/image-normalize.js';import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';@@ -725,10 +725,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {// bytes (see lib/image-normalize.ts). Mutates the image blocks in place.await normalizeMessageImages(messages);- const estimatedInputTokens = messages.reduce(- (sum, m) => sum + Math.ceil(contentToString(m.content).length / 4),- 0,- );+ const estimatedInputTokens = estimateInputTokens(messages, tools);// Image requests must route to a vision-capable model (mirrors// /chat/completions, proxy.ts). Reject up front with a clear message when