Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(routing): size agent requests with their tool schemas, stop reading size errors as rate limits (#1362)

Tashfeen8fd72ca7a349
diff --git a/server/src/__tests__/lib/content.test.ts b/server/src/__tests__/lib/content.test.ts
index ede11d52..cfb10c0f 100644
--- a/server/src/__tests__/lib/content.test.ts
+++ b/server/src/__tests__/lib/content.test.ts
@@ -1,5 +1,5 @@
import { describe, it, expect } from 'vitest';
-import { GITHUB_MAX_INPUT_TOKENS, contentToString, flattenMessageContent, messageHasImage, normalizeOutboundContent, sanitizeResponse, stripImagesFromMessages, truncateMessagesForGithub } from '../../lib/content.js';
+import { GITHUB_MAX_INPUT_TOKENS, contentToString, estimateInputTokens, flattenMessageContent, messageHasImage, normalizeOutboundContent, sanitizeResponse, stripImagesFromMessages, truncateMessagesForGithub } from '../../lib/content.js';
import type { ChatMessage } from '@freellmapi/shared/types.js';
describe('contentToString', () => {
@@ -270,3 +270,29 @@ describe('truncateMessagesForGithub', () => {
expect(out).toEqual([image]);
});
});
+
+describe('estimateInputTokens', () => {
+ const messages: ChatMessage[] = [{ role: 'user', content: 'x'.repeat(400) }];
+
+ it('counts message text at ~4 chars per token', () => {
+ expect(estimateInputTokens(messages)).toBe(100);
+ });
+
+ it('counts the tool schemas the request carries', () => {
+ // Claude Code ships ~65 KB of tool schemas next to a few KB of messages;
+ // an estimate without them routed its turns to models they could not fit.
+ const tools = [{ type: 'function', function: { name: 'Read', description: 'd'.repeat(4000), parameters: { type: 'object' } } }];
+ const withTools = estimateInputTokens(messages, tools);
+ expect(withTools).toBeGreaterThan(100 + 1000);
+ expect(estimateInputTokens(messages, [])).toBe(100);
+ });
+
+ it('counts assistant tool-call arguments', () => {
+ const withCall: ChatMessage[] = [...messages, {
+ role: 'assistant',
+ content: null,
+ tool_calls: [{ id: 'c1', type: 'function', function: { name: 'Write', arguments: 'a'.repeat(795) } }],
+ } as ChatMessage];
+ expect(estimateInputTokens(withCall)).toBe(100 + 200);
+ });
+});
diff --git a/server/src/__tests__/lib/fallback-loop-size-skip.test.ts b/server/src/__tests__/lib/fallback-loop-size-skip.test.ts
index 0be05dbd..d794d219 100644
--- a/server/src/__tests__/lib/fallback-loop-size-skip.test.ts
+++ b/server/src/__tests__/lib/fallback-loop-size-skip.test.ts
@@ -18,7 +18,7 @@ import {
recordRetryableFailure,
resetModelFailureWindows,
} from '../../lib/fallback-loop.js';
-import { resetKeyLocalityCache } from '../../services/ratelimit.js';
+import { isOnCooldown, resetKeyLocalityCache } from '../../services/ratelimit.js';
import {
setRoutingStrategy,
clearAllPenalties,
@@ -236,3 +236,19 @@ describe('FallbackState.observedTotalTokens (#507)', () => {
expect(routingTotal).toBe(localEstimate);
});
});
+
+describe('request-too-large failures', () => {
+ it('skip the model for this request without benching it for smaller ones', () => {
+ // An oversized agent turn says nothing about the model's health; a
+ // cooldown here made the next ordinary request find the pool rate limited.
+ // A key of its own: earlier cases leave in-memory cooldowns on `keyId`.
+ const freshKey = insertKey('size-no-bench');
+ resetKeyLocalityCache();
+ const state = newFallbackState();
+ const err = Object.assign(new Error('Groq API error 413: Request too large for model `x` on tokens per minute (TPM): Limit 8000, Requested 17679, please reduce your message size and try again. Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing'), { status: 413 });
+ recordRetryableFailure(routeFor(modelSmall, freshKey), err, state);
+ expect(state.skipModels.has(modelSmall.id)).toBe(true);
+ expect(state.observedTotalTokens).toBe(17679);
+ expect(isOnCooldown(PLATFORM, modelSmall.model_id, freshKey)).toBe(false);
+ });
+});
diff --git a/server/src/__tests__/lib/fallback-loop.test.ts b/server/src/__tests__/lib/fallback-loop.test.ts
index 0686087a..45421823 100644
--- a/server/src/__tests__/lib/fallback-loop.test.ts
+++ b/server/src/__tests__/lib/fallback-loop.test.ts
@@ -156,6 +156,23 @@ describe('isDailyQuotaExhaustedError + midnight benching (drift: 90s cooldown on
expect(isDailyQuotaExhaustedError(new Error('503 Service Unavailable'))).toBe(false);
});
+ // Live Groq bodies (2026-09-29): every limit error ends with the same
+ // "Upgrade to Dev Tier today" upsell, which used to read as a daily marker.
+ const GROQ_UPSELL = 'Need more tokens? Upgrade to Dev Tier today at https://console.groq.com/settings/billing';
+
+ it('does not read Groq\'s "Upgrade to Dev Tier today" upsell as a daily quota', () => {
+ const tooLarge = Object.assign(new Error(`Groq API error 413: Request too large for model \`openai/gpt-oss-20b\` in organization \`org_x\` service tier \`on_demand\` on tokens per minute (TPM): Limit 8000, Requested 12122, please reduce your message size and try again. ${GROQ_UPSELL}`), { status: 413 });
+ const perMinute = Object.assign(new Error(`Groq API error 429: Rate limit reached for model \`openai/gpt-oss-120b\` in organization \`org_x\` service tier \`on_demand\` on tokens per minute (TPM): Limit 8000, Used 6000, Requested 4000. Please try again in 15s. ${GROQ_UPSELL}`), { status: 429 });
+ expect(isDailyQuotaExhaustedError(tooLarge)).toBe(false);
+ expect(classifyAttemptError(tooLarge)).toBe('context_too_large');
+ expect(isDailyQuotaExhaustedError(perMinute)).toBe(false);
+ });
+
+ it('still flags a Groq requests-per-day 429 that carries the upsell', () => {
+ const perDay = Object.assign(new Error(`Groq API error 429: Rate limit reached for model \`llama-3.3-70b-versatile\` in organization \`org_x\` service tier \`on_demand\` on requests per day (RPD): Limit 1000, Used 1000, Requested 1. Please try again in 1m26s. Need more requests? Upgrade to Dev Tier today at https://console.groq.com/settings/billing`), { status: 429 });
+ expect(isDailyQuotaExhaustedError(perDay)).toBe(true);
+ });
+
it('cooldownForError benches a daily-allocation 429 until the next UTC midnight', () => {
const route = fakeRoute();
const err = Object.assign(new Error('you have used up your daily free allocation of 10,000 neurons'), { status: 429 });
diff --git a/server/src/__tests__/routes/anthropic.test.ts b/server/src/__tests__/routes/anthropic.test.ts
index e69c1ed9..c1917338 100644
--- a/server/src/__tests__/routes/anthropic.test.ts
+++ b/server/src/__tests__/routes/anthropic.test.ts
@@ -390,6 +390,17 @@ describe('Anthropic-compatible /v1/messages', () => {
expect(body.input_tokens).toBeGreaterThan(0);
});
+ it('counts tool schemas in /v1/messages/count_tokens', async () => {
+ const messages = [{ role: 'user', content: 'count these tokens please' }];
+ const bare = await request(app, '/v1/messages/count_tokens', { model: 'claude-sonnet-4-5', messages }, anthropicHeaders());
+ const withTools = await request(app, '/v1/messages/count_tokens', {
+ model: 'claude-sonnet-4-5',
+ messages,
+ tools: [{ name: 'Read', description: 'r'.repeat(4000), input_schema: { type: 'object', properties: {} } }],
+ }, anthropicHeaders());
+ expect(withTools.body.input_tokens).toBeGreaterThan(bare.body.input_tokens + 1000);
+ });
+
it('returns an Anthropic invalid_request_error on a malformed body', async () => {
const { status, body } = await request(app, '/v1/messages', {
model: 'claude-sonnet-4-5', messages: [],
diff --git a/server/src/lib/content.ts b/server/src/lib/content.ts
index a2c7eb69..e2c1eed7 100644
--- a/server/src/lib/content.ts
+++ b/server/src/lib/content.ts
@@ -33,6 +33,23 @@ export function contentToString(content: unknown): string {
return '';
}
+// Rough input-token count (~4 chars per token) used for routing and budget
+// checks: message text, assistant tool-call arguments, and the tool schemas
+// the request carries. Tool schemas go upstream on every turn and count
+// against the provider's context window and TPM ceiling. An agent client like
+// Claude Code sends ~16k tokens of them next to ~4k of messages, so leaving
+// them out routed its requests to models they could never fit (a Groq 8k-TPM
+// model answered 413 to an "estimated" 4k request that was really 17.7k).
+export function estimateInputTokens(messages: ChatMessage[], tools?: readonly unknown[]): number {
+ const messageTokens = messages.reduce((sum, m) => {
+ const calls = (m.tool_calls ?? [])
+ .reduce((n, call) => n + call.function.name.length + call.function.arguments.length, 0);
+ return sum + Math.ceil((contentToString(m.content).length + calls) / 4);
+ }, 0);
+ const toolTokens = tools && tools.length > 0 ? Math.ceil(JSON.stringify(tools).length / 4) : 0;
+ return messageTokens + toolTokens;
+}
+
export function flattenMessageContent(messages: ChatMessage[]): ChatMessage[] {
return messages.map((m) => ({
...m,
diff --git a/server/src/lib/error-classify.ts b/server/src/lib/error-classify.ts
index e1949c3e..a17a1a9a 100644
--- a/server/src/lib/error-classify.ts
+++ b/server/src/lib/error-classify.ts
@@ -352,9 +352,17 @@ export function isKeyAuthError(err: any): boolean {
// makes the router re-pick a dead-for-the-day provider all day, so the caller
// benches until the next UTC midnight instead. Requires BOTH a daily marker and
// a quota/allocation marker so an ordinary per-minute 429 never matches.
+//
+// Groq ends every limit error, per-minute and request-too-large alike, with
+// "Need more tokens? Upgrade to Dev Tier today at …". That "today" is sales
+// copy, not a window, so it is cut before looking for a daily marker; and a
+// request too big for the model is never a spent allowance.
+const UPGRADE_UPSELL = /\bupgrade to [^.?!]*?\btoday\b/g;
+
export function isDailyQuotaExhaustedError(err: any): boolean {
if (err?.dailyQuotaExhausted === true) return true;
- const msg = (err?.message ?? '').toLowerCase();
+ if (isContextTooLargeError(err)) return false;
+ const msg = (err?.message ?? '').toLowerCase().replace(UPGRADE_UPSELL, '');
if (!/daily|per[ -_]?day|\btoday\b/.test(msg)) return false;
return /allocation|quota|limit|exhaust|used up/.test(msg);
}
diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.ts
index 22dfd585..c437e6ee 100644
--- a/server/src/lib/fallback-loop.ts
+++ b/server/src/lib/fallback-loop.ts
@@ -499,6 +499,16 @@ export function recordRetryableFailure(route: RouteResult, err: any, state: Fall
if (isProviderLevelError(err)) {
state.skipPlatforms.add(route.platform);
}
+ // Too big for this model is a fact about the REQUEST, not the model's
+ // health: it still serves every smaller request. Skip it for this request
+ // (above) and learn the reported ceiling so the router sizes it out next
+ // time, but no cooldown and no penalty. Benching it let one oversized agent
+ // turn (Claude Code ships ~16k tokens of tool schemas) sink every small-TPM
+ // model it touched, so later ordinary requests found the pool rate limited.
+ if (isContextTooLargeError(err)) {
+ learnLimitFromError(route.modelDbId, err);
+ return false;
+ }
if (consumeSkipBenchExemption(route, err)) return true;
const decision = cooldownDecisionForError(route, err);
setCooldown(route.platform, route.modelId, route.keyId, decision.durationMs, decision.source);
diff --git a/server/src/lib/inbound-chat.ts b/server/src/lib/inbound-chat.ts
index d7692fb3..bc2647fe 100644
--- a/server/src/lib/inbound-chat.ts
+++ b/server/src/lib/inbound-chat.ts
@@ -31,7 +31,7 @@ import {
} from './fallback-loop.js';
import { routedViaValue } from './header-value.js';
import { applyTokenBudget, tokenBudgetMessage } from './guardrails.js';
-import { contentToString } from './content.js';
+import { contentToString, estimateInputTokens } from './content.js';
import { normalizeMessageImages } from './image-normalize.js';
import { repairToolArguments, toolSchemaMap } from './tool-args.js';
import { invalidToolArgumentsError, invalidToolCallReasons, isToolArgumentValidationEnabled } from './tool-validate.js';
@@ -86,15 +86,6 @@ export interface InboundChatWire {
sendStreamError?(res: Response, message: string): void;
}
-function estimateTokens(messages: ChatMessage[]): number {
- return messages.reduce((sum, message) => {
- const text = contentToString(message.content);
- const calls = (message.tool_calls ?? [])
- .reduce((n, call) => n + call.function.name.length + call.function.arguments.length, 0);
- return sum + Math.ceil((text.length + calls) / 4);
- }, 0);
-}
-
function hasImages(messages: ChatMessage[]): boolean {
return messages.some(message =>
Array.isArray(message.content)
@@ -181,7 +172,7 @@ export async function runInboundChat(
// token budgets, payload limits, and upstream transfers all see the shrunk
// bytes (see lib/image-normalize.ts). Mutates the image blocks in place.
await normalizeMessageImages(input.messages);
- const estimatedInputTokens = estimateTokens(input.messages);
+ const estimatedInputTokens = estimateInputTokens(input.messages, input.tools);
const budget = applyTokenBudget(estimatedInputTokens, input.maxTokens);
if (budget.rejection) {
wire.sendError(res, 413, tokenBudgetMessage(budget.rejection), 'request_token_budget');
diff --git a/server/src/routes/anthropic.ts b/server/src/routes/anthropic.ts
index e4ea5898..b12826e9 100644
--- a/server/src/routes/anthropic.ts
+++ b/server/src/routes/anthropic.ts
@@ -11,7 +11,7 @@ import type {
} from '@freellmapi/shared/types.js';
import { routeRequest, resolveModelGroupCandidates, resolveRoutingChain, resolveStickyPreference, routingReserveTokens, type RouteResult, type ResolvedChain, type ChainRow } from '../services/router.js';
import { getSetting, getUnifiedApiKey } from '../db/index.js';
-import { contentToString } from '../lib/content.js';
+import { contentToString, estimateInputTokens } from '../lib/content.js';
import { resolveTaskType } from '../lib/task-type.js';
import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';
import { invalidToolArgumentsError, invalidToolCallReasons, isToolArgumentValidationEnabled } from '../lib/tool-validate.js';
@@ -374,10 +374,6 @@ function convertRequest(input: AnthropicRequest): ConvertedRequest {
};
}
-function estimateTokens(messages: ChatMessage[]): number {
- return messages.reduce((sum, m) => sum + Math.ceil(contentToString(m.content).length / 4), 0);
-}
-
// Model resolution (Claude family → auto | pinned catalog model) lives in
// services/anthropic-map.ts so the dashboard mapping editor and this route
// share one source of truth. Claude Code keeps its built-in `claude-*` names;
@@ -505,7 +501,7 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {
messages = compressionResult.messages;
res.setHeader('X-FreeLLM-Compress', formatCompressionHeader(compressionResult));
- const estimatedInputTokens = estimateTokens(messages);
+ const estimatedInputTokens = estimateInputTokens(messages, tools);
const imageCount = messages.reduce((n, m) =>
n + (Array.isArray(m.content) ? m.content.filter(b => (b as any)?.type === 'image_url').length : 0), 0);
@@ -1109,7 +1105,7 @@ anthropicRouter.post('/messages/count_tokens', (req: Request, res: Response) =>
recordStats: false,
});
res.setHeader('X-FreeLLM-Compress', formatCompressionHeader(compressionResult));
- res.json({ input_tokens: estimateTokens(compressionResult.messages) });
+ res.json({ input_tokens: estimateInputTokens(compressionResult.messages, tools) });
});
// Anthropic-compatible GET /v1/models. Content-negotiated: only answers when
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 0383cfcf..864454b5 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -11,7 +11,7 @@ import { runImageGeneration, runVideoGeneration, runSpeech, runTranscription, Me
import multer from 'multer';
import { getDb } from '../db/index.js';
import { resolveAuth, prependSystemPrompt, type ResolvedAuth } from '../lib/system-prompt.js';
-import { contentToString, messageHasImage, normalizeOutboundContent, sanitizeResponse, truncateMessagesForGithub } from '../lib/content.js';
+import { contentToString, estimateInputTokens, messageHasImage, normalizeOutboundContent, sanitizeResponse, truncateMessagesForGithub } from '../lib/content.js';
import { resolveTaskType } from '../lib/task-type.js';
import { normalizeMessageImages } from '../lib/image-normalize.js';
import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';
@@ -1629,10 +1629,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
// Non-streaming requests reconcile against the provider's real `usage` block;
// streaming does the same when stream_options.include_usage produces a final
// usage frame, and otherwise falls back to this estimate.
- const estimatedInputTokens = messages.reduce((sum, m) => {
- const text = contentToString(m.content);
- return sum + Math.ceil(text.length / 4);
- }, 0);
+ const estimatedInputTokens = estimateInputTokens(messages, tools);
// Image requests must route to a vision-capable model. Reject up front with a
// clear message when none is enabled, rather than silently dropping the image
diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.ts
index 69e7ec27..8b7445cc 100644
--- a/server/src/routes/responses.ts
+++ b/server/src/routes/responses.ts
@@ -13,7 +13,7 @@ import { routeRequest, hasEnabledVisionModel, hasEnabledToolsModel, resolveStick
import { getDb } from '../db/index.js';
import { resolveAuth, prependSystemPrompt } from '../lib/system-prompt.js';
import { isUnifyEnabled, getModelGroups, resolveRequestedIdForDispatch } from '../services/model-groups.js';
-import { contentToString, messageHasImage } from '../lib/content.js';
+import { contentToString, estimateInputTokens, messageHasImage } from '../lib/content.js';
import { resolveTaskType } from '../lib/task-type.js';
import { normalizeMessageImages } from '../lib/image-normalize.js';
import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';
@@ -725,10 +725,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
// bytes (see lib/image-normalize.ts). Mutates the image blocks in place.
await normalizeMessageImages(messages);
- const estimatedInputTokens = messages.reduce(
- (sum, m) => sum + Math.ceil(contentToString(m.content).length / 4),
- 0,
- );
+ const estimatedInputTokens = estimateInputTokens(messages, tools);
// Image requests must route to a vision-capable model (mirrors
// /chat/completions, proxy.ts). Reject up front with a clear message when