7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(router): clamp max_tokens to the model's remaining context budget (#1217)
Chong Jia Zhen5bf71fdaf0ac
diff --git a/server/src/__tests__/lib/unified-max-tokens.test.ts b/server/src/__tests__/lib/unified-max-tokens.test.tsindex 25289303..0c7de2aa 100644--- a/server/src/__tests__/lib/unified-max-tokens.test.ts+++ b/server/src/__tests__/lib/unified-max-tokens.test.ts@@ -128,3 +128,31 @@ describe('per-platform max_tokens ceiling', () => {expect(resolveMaxTokens('github', 65536)).toBe(128);});});++describe('contextBudget clamping', () => {+ it('clamps max_tokens to the remaining context budget', () => {+ expect(resolveMaxTokens('groq', 64000, 48000)).toBe(48000);+ });++ it('leaves max_tokens alone when it fits', () => {+ expect(resolveMaxTokens('groq', 4096, 60000)).toBe(4096);+ });++ it('ignores contextBudget when undefined', () => {+ expect(resolveMaxTokens('groq', 64000, undefined)).toBe(64000);+ });++ it('ignores contextBudget when zero or negative', () => {+ expect(resolveMaxTokens('groq', 64000, 0)).toBe(64000);+ expect(resolveMaxTokens('groq', 64000, -100)).toBe(64000);+ });++ it('floors the result at 1', () => {+ expect(resolveMaxTokens('groq', 64000, 1)).toBe(1);+ });++ it('takes the tightest of context budget, platform cap, and operator cap', () => {+ settingStore.set(UNIFIED_MAX_TOKENS_SETTING, '16000');+ expect(resolveMaxTokens('github', 64000, 48000)).toBe(Math.min(GITHUB_MAX_OUTPUT_TOKENS, 16000, 48000));+ });+});diff --git a/server/src/__tests__/providers/aclide.test.ts b/server/src/__tests__/providers/aclide.test.tsindex ab77ce0c..7de9fb60 100644--- a/server/src/__tests__/providers/aclide.test.ts+++ b/server/src/__tests__/providers/aclide.test.ts@@ -38,6 +38,12 @@ describe('ACLIDE Responses adapter', () => {expect(result._routed_via).toEqual({ platform: 'aclide', model });});+ it('clamps max_output_tokens to the route\'s remaining context budget', async () => {+ const fetch = vi.spyOn(global, 'fetch').mockResolvedValue(json(completed()));+ await new AclideProvider().chatCompletion('k', messages, model, { max_tokens: 4096, contextBudget: 300 });+ expect(JSON.parse(String(fetch.mock.calls[0][1]?.body)).max_output_tokens).toBe(300);+ });+it('preserves multimodal input, tool history and JSON schema', async () => {const fetch = vi.spyOn(global, 'fetch').mockResolvedValue(json({ ...completed(), output: [{ type: 'function_call', call_id: 'call_2', name: 'weather', arguments: '{}' },diff --git a/server/src/__tests__/providers/moondream.test.ts b/server/src/__tests__/providers/moondream.test.tsindex 1c711e7f..53b0ffc6 100644--- a/server/src/__tests__/providers/moondream.test.ts+++ b/server/src/__tests__/providers/moondream.test.ts@@ -49,6 +49,12 @@ describe('Moondream hosted adapter', () => {expect(result._routed_via).toEqual({ platform: 'moondream', model });});+ it('clamps max_completion_tokens to the route\'s remaining context budget', async () => {+ const fetch = vi.spyOn(global, 'fetch').mockResolvedValue(json(completion()));+ await new MoondreamProvider().chatCompletion('k', messages, model, { max_tokens: 2048, contextBudget: 300 });+ expect(JSON.parse(String(fetch.mock.calls[0][1]?.body)).max_completion_tokens).toBe(300);+ });+it('preserves vision input, strips replay-only fields and obeys upstream output ceiling', async () => {const fetch = vi.spyOn(global, 'fetch').mockResolvedValue(json(completion()));await new MoondreamProvider().chatCompletion('k', [diff --git a/server/src/lib/sampling-params.ts b/server/src/lib/sampling-params.tsindex 390ae338..53e994ef 100644--- a/server/src/lib/sampling-params.ts+++ b/server/src/lib/sampling-params.ts@@ -386,13 +386,13 @@ export function maxTokensCapFor(platform: string): number | undefined {* openai-compat (and its subclasses), cloudflare, cohere, google and aihorde* all do.*/-export function resolveMaxTokens(platform: string, requested: number | undefined): number | undefined {+export function resolveMaxTokens(platform: string, requested: number | undefined, contextBudget?: number): number | undefined {const resolved = requested ?? defaultMaxTokensFor(platform);if (resolved == null) return resolved;// The tighter ceiling wins: a platform's hard reject applies even with the// operator cap off, and an operator cap below it applies everywhere.- const caps = [unifiedMaxTokensCap(), maxTokensCapFor(platform)].filter((c): c is number => c != null);- return caps.length === 0 ? resolved : Math.min(resolved, ...caps);+ const caps = [unifiedMaxTokensCap(), maxTokensCapFor(platform), contextBudget].filter((c): c is number => c != null && c > 0);+ return caps.length === 0 ? resolved : Math.max(1, Math.min(resolved, ...caps));}// ── Unified output-token cap ─────────────────────────────────────────────────diff --git a/server/src/providers/aclide.ts b/server/src/providers/aclide.tsindex 581ab511..74d3a17b 100644--- a/server/src/providers/aclide.ts+++ b/server/src/providers/aclide.ts@@ -88,7 +88,7 @@ export class AclideProvider extends BaseProvider {}private body(messages: ChatMessage[], modelId: string, options?: CompletionOptions): Record<string, unknown> {- const maxOutput = resolveMaxTokens(this.platform, options?.max_tokens);+ const maxOutput = resolveMaxTokens(this.platform, options?.max_tokens, options?.contextBudget);const format = options?.response_format;const body: Record<string, unknown> = { model: modelId, input: this.input(messages), store: false, stream: false };if (maxOutput !== undefined) body.max_output_tokens = maxOutput;diff --git a/server/src/providers/aihorde.ts b/server/src/providers/aihorde.tsindex 3dfb5b43..c7c64768 100644--- a/server/src/providers/aihorde.ts+++ b/server/src/providers/aihorde.ts@@ -92,7 +92,7 @@ export class AIHordeProvider extends BaseProvider {// applies to it as well — it can only ever lower what we send.max_tokens: Math.max(MIN_MAX_TOKENS,- resolveMaxTokens(this.platform, options?.max_tokens ?? DEFAULT_MAX_TOKENS) ?? DEFAULT_MAX_TOKENS,+ resolveMaxTokens(this.platform, options?.max_tokens ?? DEFAULT_MAX_TOKENS, options?.contextBudget) ?? DEFAULT_MAX_TOKENS,),};if (options?.temperature != null) body.temperature = options.temperature;diff --git a/server/src/providers/base.ts b/server/src/providers/base.tsindex 63a6a043..8126d761 100644--- a/server/src/providers/base.ts+++ b/server/src/providers/base.ts@@ -163,6 +163,9 @@ export interface CompletionOptions extends ExtendedSamplingOptions {stream_options?: {include_usage?: boolean;};+ /** Remaining context budget (context_window − estimated_input_tokens) for+ * this route. resolveMaxTokens clamps max_tokens to fit. */+ contextBudget?: number;/** Per-call HTTP timeout override. Not part of the OpenAI wire format (it is* stripped before the request body is built); used by the probe script so* NVIDIA's 15-60s serverless cold starts don't read as failures. */diff --git a/server/src/providers/cloudflare.ts b/server/src/providers/cloudflare.tsindex bc681b9f..604948e1 100644--- a/server/src/providers/cloudflare.ts+++ b/server/src/providers/cloudflare.ts@@ -69,7 +69,7 @@ export class CloudflareProvider extends BaseProvider {model: modelId,messages: this.normalizeMessages(messages),temperature: options?.temperature,- max_tokens: resolveMaxTokens(this.platform, options?.max_tokens),+ max_tokens: resolveMaxTokens(this.platform, options?.max_tokens, options?.contextBudget),top_p: options?.top_p,stop: options?.stop,tools: options?.tools,@@ -125,7 +125,7 @@ export class CloudflareProvider extends BaseProvider {model: modelId,messages: this.normalizeMessages(messages),temperature: options?.temperature,- max_tokens: resolveMaxTokens(this.platform, options?.max_tokens),+ max_tokens: resolveMaxTokens(this.platform, options?.max_tokens, options?.contextBudget),top_p: options?.top_p,stop: options?.stop,tools: options?.tools,diff --git a/server/src/providers/cohere.ts b/server/src/providers/cohere.tsindex f0f8a282..c9734917 100644--- a/server/src/providers/cohere.ts+++ b/server/src/providers/cohere.ts@@ -44,7 +44,7 @@ export class CohereProvider extends BaseProvider {model: modelId,messages: flattenMessageContent(messages),temperature: options?.temperature,- max_tokens: resolveMaxTokens(this.platform, options?.max_tokens),+ max_tokens: resolveMaxTokens(this.platform, options?.max_tokens, options?.contextBudget),top_p: options?.top_p,stop: options?.stop,tools: sanitizeCohereTools(options?.tools),@@ -92,7 +92,7 @@ export class CohereProvider extends BaseProvider {model: modelId,messages: flattenMessageContent(messages),temperature: options?.temperature,- max_tokens: resolveMaxTokens(this.platform, options?.max_tokens),+ max_tokens: resolveMaxTokens(this.platform, options?.max_tokens, options?.contextBudget),top_p: options?.top_p,stop: options?.stop,tools: sanitizeCohereTools(options?.tools),diff --git a/server/src/providers/google.ts b/server/src/providers/google.tsindex 0ef71b8b..45faa88a 100644--- a/server/src/providers/google.ts+++ b/server/src/providers/google.ts@@ -560,7 +560,7 @@ export class GoogleProvider extends BaseProvider {contents: request.contents,generationConfig: {temperature: options?.temperature,- maxOutputTokens: resolveMaxTokens(this.platform, options?.max_tokens),+ maxOutputTokens: resolveMaxTokens(this.platform, options?.max_tokens, options?.contextBudget),topP: options?.top_p,stopSequences: toGeminiStopSequences(options?.stop),...toGeminiExtendedConfig(options),@@ -643,7 +643,7 @@ export class GoogleProvider extends BaseProvider {contents: request.contents,generationConfig: {temperature: options?.temperature,- maxOutputTokens: resolveMaxTokens(this.platform, options?.max_tokens),+ maxOutputTokens: resolveMaxTokens(this.platform, options?.max_tokens, options?.contextBudget),topP: options?.top_p,stopSequences: toGeminiStopSequences(options?.stop),...toGeminiExtendedConfig(options),diff --git a/server/src/providers/moondream.ts b/server/src/providers/moondream.tsindex 5eaac226..f23076cf 100644--- a/server/src/providers/moondream.ts+++ b/server/src/providers/moondream.ts@@ -52,7 +52,7 @@ export class MoondreamProvider extends BaseProvider {return {model: modelId, messages: input, stream,temperature: options?.temperature, top_p: options?.top_p,- max_completion_tokens: resolveMaxTokens(this.platform, options?.max_tokens),+ max_completion_tokens: resolveMaxTokens(this.platform, options?.max_tokens, options?.contextBudget),...(options?.reasoning_effort !== undefined ? { reasoning: options.reasoning_effort !== 'none' } : {}),...(stream && options?.stream_options ? { stream_options: options.stream_options } : {}),};diff --git a/server/src/providers/openai-compat.ts b/server/src/providers/openai-compat.tsindex c5aff35d..d1a9977b 100644--- a/server/src/providers/openai-compat.ts+++ b/server/src/providers/openai-compat.ts@@ -284,7 +284,7 @@ export class OpenAICompatProvider extends BaseProvider {model: modelId,messages: this.messagesForPlatform(messages, modelId),temperature: sampling.temperature,- max_tokens: resolveMaxTokens(this.platform, options?.max_tokens),+ max_tokens: resolveMaxTokens(this.platform, options?.max_tokens, options?.contextBudget),top_p: sampling.topP,stop: options?.stop,tools: options?.tools,@@ -413,7 +413,7 @@ export class OpenAICompatProvider extends BaseProvider {model: modelId,messages: this.messagesForPlatform(messages, modelId),temperature: sampling.temperature,- max_tokens: resolveMaxTokens(this.platform, options?.max_tokens),+ max_tokens: resolveMaxTokens(this.platform, options?.max_tokens, options?.contextBudget),top_p: sampling.topP,stop: options?.stop,tools: options?.tools,diff --git a/server/src/providers/sail.ts b/server/src/providers/sail.tsindex f6107335..b73220ff 100644--- a/server/src/providers/sail.ts+++ b/server/src/providers/sail.ts@@ -175,7 +175,7 @@ export class SailProvider extends BaseProvider {}private buildBody(messages: ChatMessage[], modelId: string, options?: CompletionOptions): Record<string, unknown> {- const maxOutputTokens = resolveMaxTokens(this.platform, options?.max_tokens);+ const maxOutputTokens = resolveMaxTokens(this.platform, options?.max_tokens, options?.contextBudget);const reasoningEffort = this.reasoningEffort(modelId, options);const textFormat = this.textFormat(options);const tools = options?.tools?.map(tool => ({diff --git a/server/src/routes/anthropic.ts b/server/src/routes/anthropic.tsindex 63cd5029..e4ea5898 100644--- a/server/src/routes/anthropic.ts+++ b/server/src/routes/anthropic.ts@@ -660,9 +660,11 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {return routeRequest(routingTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain ?? resolvedChain?.chain, false, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined, outputReserve, taskType);},dispatch: async (route, attempt, dispatchCtx) => {+ const contextBudget = route.contextWindow != null ? route.contextWindow - estimatedInputTokens : undefined;+ const routeOptions = { ...dispatchOptions, contextBudget };if (stream) {try {- await streamCompletion(res, route, messages, dispatchOptions, {+ await streamCompletion(res, route, messages, routeOptions, {start, attempt, attemptLog, clientGone: () => clientGone, requestedModel, estimatedInputTokens, tools, pinnedModelId,sessionId, pinned: resolved.pinned, stickyScope, strategyKey, disarmHedge: dispatchCtx.disarmHedge, state,});@@ -675,7 +677,7 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {}}- const result = await route.provider.chatCompletion(route.apiKey, messages, route.modelId, dispatchOptions);+ const result = await route.provider.chatCompletion(route.apiKey, messages, route.modelId, routeOptions);const respMsg = result.choices?.[0]?.message;const respText = contentToString(respMsg?.content ?? '');let respToolCalls = respMsg?.tool_calls ?? [];diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 6d7204c3..a361343b 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -1192,6 +1192,7 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {);},dispatch: async (route, attempt, ctx) => {+ const contextBudget = route.contextWindow != null ? route.contextWindow - estimatedInputTokens : undefined;traceRouteEvent('Proxy', {event: attempt === 0 ? 'start' : 'next',requestId: requestGroupId,@@ -1239,7 +1240,7 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {route.apiKey,dispatchMessages,route.modelId,- { temperature, max_tokens, top_p, stop, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) },+ { temperature, max_tokens, top_p, stop, contextBudget, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) },quotaContextForRoute(route, 'chat/completions'),);@@ -1339,7 +1340,7 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {route.apiKey,dispatchMessages,route.modelId,- { temperature, max_tokens, top_p, stop, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) },+ { temperature, max_tokens, top_p, stop, contextBudget, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) },quotaContextForRoute(route, 'chat/completions'),);@@ -2111,6 +2112,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {return routeRequest(routingEstimate, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain ?? resolvedChain?.chain, samplingParams.response_format !== undefined, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined, outputReserve, taskType);},dispatch: async (route, attempt, ctx) => {+ const contextBudget = route.contextWindow != null ? route.contextWindow - estimatedInputTokens : undefined;const modelKey = `${route.platform}:${route.modelId}`;traceRouteEvent('Proxy', {event: attempt === 0 ? 'start' : 'next',@@ -2253,7 +2255,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {try {const gen = route.provider.streamChatCompletion(route.apiKey, outboundMessages, route.modelId,- { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, stream_options: parsed.data.stream_options, ...samplingParams, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) },+ { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, stream_options: parsed.data.stream_options, ...samplingParams, contextBudget, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) },quotaContextForRoute(route, 'chat/completions'),);@@ -2635,7 +2637,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {} else {const result = await route.provider.chatCompletion(route.apiKey, outboundMessages, route.modelId,- { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, ...samplingParams, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) },+ { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, ...samplingParams, contextBudget, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) },quotaContextForRoute(route, 'chat/completions'),);diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.tsindex 74b64f12..69e7ec27 100644--- a/server/src/routes/responses.ts+++ b/server/src/routes/responses.ts@@ -1141,6 +1141,8 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {return routeRequest(routingTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain ?? resolvedChain?.chain, completionOpts.response_format !== undefined, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined, outputReserve, taskType);},dispatch: async (route, attempt, ctx) => {+ const contextBudget = route.contextWindow != null ? route.contextWindow - estimatedInputTokens : undefined;+ const routeOpts = contextBudget != null ? { ...dispatchOpts, contextBudget } : dispatchOpts;traceRouteEvent('Responses', {event: attempt === 0 ? 'start' : 'next',requestId: requestGroupId,@@ -1235,7 +1237,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {route.apiKey,messages,route.modelId,- dispatchOpts,+ routeOpts,quotaContextForRoute(route, 'responses'),);@@ -1473,7 +1475,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {route.apiKey,messages,route.modelId,- dispatchOpts,+ routeOpts,quotaContextForRoute(route, 'responses'),);diff --git a/server/src/services/fusion.ts b/server/src/services/fusion.tsindex c9afdbd1..83bab2f5 100644--- a/server/src/services/fusion.ts+++ b/server/src/services/fusion.ts@@ -275,8 +275,11 @@ async function runModelCall(if (!route) break;const startedAt = Date.now();+ const routeOpts = route.contextWindow != null+ ? { ...options, contextBudget: route.contextWindow - estimatedTokens }+ : options;try {- const result = await route.provider.chatCompletion(route.apiKey, messages, route.modelId, options);+ const result = await route.provider.chatCompletion(route.apiKey, messages, route.modelId, routeOpts);const choice = result.choices?.[0];const text = contentToString(choice?.message?.content ?? '');const toolCalls = choice?.message?.tool_calls;@@ -363,10 +366,13 @@ async function runJudgeStreaming(if (!route) break;const startedAt = Date.now();+ const routeOpts = route.contextWindow != null+ ? { ...options, contextBudget: route.contextWindow - estimatedTokens }+ : options;let text = '';let started = false;try {- for await (const chunk of route.provider.streamChatCompletion(route.apiKey, messages, route.modelId, options)) {+ for await (const chunk of route.provider.streamChatCompletion(route.apiKey, messages, route.modelId, routeOpts)) {const delta = (chunk as any)?.choices?.[0]?.delta?.content;if (typeof delta === 'string' && delta.length > 0) {if (!started) { started = true; cb.onStart?.({ platform: route.platform, model: route.modelId }); }diff --git a/server/src/services/router.ts b/server/src/services/router.tsindex 2cc15d71..9f17506b 100644--- a/server/src/services/router.ts+++ b/server/src/services/router.ts@@ -198,6 +198,7 @@ export interface RouteResult {keyLabel: string | null;platform: string;displayName: string;+ contextWindow: number | null;/*** The custom endpoint this route belongs to, '' for catalog platforms (#651).* Carried on the route so the failure path can attribute a retirement signal@@ -1540,6 +1541,7 @@ function selectKeyForModel(entry: ChainRow, estimatedTokens: number, skipKeys?:proxyUrl,platform: entry.platform,displayName: entry.display_name,+ contextWindow: entry.context_window,endpointScope: entry.endpoint_scope ?? '',rpdLimit: limits.rpd,tpdLimit: limits.tpd,