Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(router): clamp max_tokens to the model's remaining context budget (#1217)

Chong Jia Zhen5bf71fdaf0ac
diff --git a/server/src/__tests__/lib/unified-max-tokens.test.ts b/server/src/__tests__/lib/unified-max-tokens.test.ts
index 25289303..0c7de2aa 100644
--- a/server/src/__tests__/lib/unified-max-tokens.test.ts
+++ b/server/src/__tests__/lib/unified-max-tokens.test.ts
@@ -128,3 +128,31 @@ describe('per-platform max_tokens ceiling', () => {
expect(resolveMaxTokens('github', 65536)).toBe(128);
});
});
+
+describe('contextBudget clamping', () => {
+ it('clamps max_tokens to the remaining context budget', () => {
+ expect(resolveMaxTokens('groq', 64000, 48000)).toBe(48000);
+ });
+
+ it('leaves max_tokens alone when it fits', () => {
+ expect(resolveMaxTokens('groq', 4096, 60000)).toBe(4096);
+ });
+
+ it('ignores contextBudget when undefined', () => {
+ expect(resolveMaxTokens('groq', 64000, undefined)).toBe(64000);
+ });
+
+ it('ignores contextBudget when zero or negative', () => {
+ expect(resolveMaxTokens('groq', 64000, 0)).toBe(64000);
+ expect(resolveMaxTokens('groq', 64000, -100)).toBe(64000);
+ });
+
+ it('floors the result at 1', () => {
+ expect(resolveMaxTokens('groq', 64000, 1)).toBe(1);
+ });
+
+ it('takes the tightest of context budget, platform cap, and operator cap', () => {
+ settingStore.set(UNIFIED_MAX_TOKENS_SETTING, '16000');
+ expect(resolveMaxTokens('github', 64000, 48000)).toBe(Math.min(GITHUB_MAX_OUTPUT_TOKENS, 16000, 48000));
+ });
+});
diff --git a/server/src/__tests__/providers/aclide.test.ts b/server/src/__tests__/providers/aclide.test.ts
index ab77ce0c..7de9fb60 100644
--- a/server/src/__tests__/providers/aclide.test.ts
+++ b/server/src/__tests__/providers/aclide.test.ts
@@ -38,6 +38,12 @@ describe('ACLIDE Responses adapter', () => {
expect(result._routed_via).toEqual({ platform: 'aclide', model });
});
+ it('clamps max_output_tokens to the route\'s remaining context budget', async () => {
+ const fetch = vi.spyOn(global, 'fetch').mockResolvedValue(json(completed()));
+ await new AclideProvider().chatCompletion('k', messages, model, { max_tokens: 4096, contextBudget: 300 });
+ expect(JSON.parse(String(fetch.mock.calls[0][1]?.body)).max_output_tokens).toBe(300);
+ });
+
it('preserves multimodal input, tool history and JSON schema', async () => {
const fetch = vi.spyOn(global, 'fetch').mockResolvedValue(json({ ...completed(), output: [
{ type: 'function_call', call_id: 'call_2', name: 'weather', arguments: '{}' },
diff --git a/server/src/__tests__/providers/moondream.test.ts b/server/src/__tests__/providers/moondream.test.ts
index 1c711e7f..53b0ffc6 100644
--- a/server/src/__tests__/providers/moondream.test.ts
+++ b/server/src/__tests__/providers/moondream.test.ts
@@ -49,6 +49,12 @@ describe('Moondream hosted adapter', () => {
expect(result._routed_via).toEqual({ platform: 'moondream', model });
});
+ it('clamps max_completion_tokens to the route\'s remaining context budget', async () => {
+ const fetch = vi.spyOn(global, 'fetch').mockResolvedValue(json(completion()));
+ await new MoondreamProvider().chatCompletion('k', messages, model, { max_tokens: 2048, contextBudget: 300 });
+ expect(JSON.parse(String(fetch.mock.calls[0][1]?.body)).max_completion_tokens).toBe(300);
+ });
+
it('preserves vision input, strips replay-only fields and obeys upstream output ceiling', async () => {
const fetch = vi.spyOn(global, 'fetch').mockResolvedValue(json(completion()));
await new MoondreamProvider().chatCompletion('k', [
diff --git a/server/src/lib/sampling-params.ts b/server/src/lib/sampling-params.ts
index 390ae338..53e994ef 100644
--- a/server/src/lib/sampling-params.ts
+++ b/server/src/lib/sampling-params.ts
@@ -386,13 +386,13 @@ export function maxTokensCapFor(platform: string): number | undefined {
* openai-compat (and its subclasses), cloudflare, cohere, google and aihorde
* all do.
*/
-export function resolveMaxTokens(platform: string, requested: number | undefined): number | undefined {
+export function resolveMaxTokens(platform: string, requested: number | undefined, contextBudget?: number): number | undefined {
const resolved = requested ?? defaultMaxTokensFor(platform);
if (resolved == null) return resolved;
// The tighter ceiling wins: a platform's hard reject applies even with the
// operator cap off, and an operator cap below it applies everywhere.
- const caps = [unifiedMaxTokensCap(), maxTokensCapFor(platform)].filter((c): c is number => c != null);
- return caps.length === 0 ? resolved : Math.min(resolved, ...caps);
+ const caps = [unifiedMaxTokensCap(), maxTokensCapFor(platform), contextBudget].filter((c): c is number => c != null && c > 0);
+ return caps.length === 0 ? resolved : Math.max(1, Math.min(resolved, ...caps));
}
// ── Unified output-token cap ─────────────────────────────────────────────────
diff --git a/server/src/providers/aclide.ts b/server/src/providers/aclide.ts
index 581ab511..74d3a17b 100644
--- a/server/src/providers/aclide.ts
+++ b/server/src/providers/aclide.ts
@@ -88,7 +88,7 @@ export class AclideProvider extends BaseProvider {
}
private body(messages: ChatMessage[], modelId: string, options?: CompletionOptions): Record<string, unknown> {
- const maxOutput = resolveMaxTokens(this.platform, options?.max_tokens);
+ const maxOutput = resolveMaxTokens(this.platform, options?.max_tokens, options?.contextBudget);
const format = options?.response_format;
const body: Record<string, unknown> = { model: modelId, input: this.input(messages), store: false, stream: false };
if (maxOutput !== undefined) body.max_output_tokens = maxOutput;
diff --git a/server/src/providers/aihorde.ts b/server/src/providers/aihorde.ts
index 3dfb5b43..c7c64768 100644
--- a/server/src/providers/aihorde.ts
+++ b/server/src/providers/aihorde.ts
@@ -92,7 +92,7 @@ export class AIHordeProvider extends BaseProvider {
// applies to it as well — it can only ever lower what we send.
max_tokens: Math.max(
MIN_MAX_TOKENS,
- resolveMaxTokens(this.platform, options?.max_tokens ?? DEFAULT_MAX_TOKENS) ?? DEFAULT_MAX_TOKENS,
+ resolveMaxTokens(this.platform, options?.max_tokens ?? DEFAULT_MAX_TOKENS, options?.contextBudget) ?? DEFAULT_MAX_TOKENS,
),
};
if (options?.temperature != null) body.temperature = options.temperature;
diff --git a/server/src/providers/base.ts b/server/src/providers/base.ts
index 63a6a043..8126d761 100644
--- a/server/src/providers/base.ts
+++ b/server/src/providers/base.ts
@@ -163,6 +163,9 @@ export interface CompletionOptions extends ExtendedSamplingOptions {
stream_options?: {
include_usage?: boolean;
};
+ /** Remaining context budget (context_window − estimated_input_tokens) for
+ * this route. resolveMaxTokens clamps max_tokens to fit. */
+ contextBudget?: number;
/** Per-call HTTP timeout override. Not part of the OpenAI wire format (it is
* stripped before the request body is built); used by the probe script so
* NVIDIA's 15-60s serverless cold starts don't read as failures. */
diff --git a/server/src/providers/cloudflare.ts b/server/src/providers/cloudflare.ts
index bc681b9f..604948e1 100644
--- a/server/src/providers/cloudflare.ts
+++ b/server/src/providers/cloudflare.ts
@@ -69,7 +69,7 @@ export class CloudflareProvider extends BaseProvider {
model: modelId,
messages: this.normalizeMessages(messages),
temperature: options?.temperature,
- max_tokens: resolveMaxTokens(this.platform, options?.max_tokens),
+ max_tokens: resolveMaxTokens(this.platform, options?.max_tokens, options?.contextBudget),
top_p: options?.top_p,
stop: options?.stop,
tools: options?.tools,
@@ -125,7 +125,7 @@ export class CloudflareProvider extends BaseProvider {
model: modelId,
messages: this.normalizeMessages(messages),
temperature: options?.temperature,
- max_tokens: resolveMaxTokens(this.platform, options?.max_tokens),
+ max_tokens: resolveMaxTokens(this.platform, options?.max_tokens, options?.contextBudget),
top_p: options?.top_p,
stop: options?.stop,
tools: options?.tools,
diff --git a/server/src/providers/cohere.ts b/server/src/providers/cohere.ts
index f0f8a282..c9734917 100644
--- a/server/src/providers/cohere.ts
+++ b/server/src/providers/cohere.ts
@@ -44,7 +44,7 @@ export class CohereProvider extends BaseProvider {
model: modelId,
messages: flattenMessageContent(messages),
temperature: options?.temperature,
- max_tokens: resolveMaxTokens(this.platform, options?.max_tokens),
+ max_tokens: resolveMaxTokens(this.platform, options?.max_tokens, options?.contextBudget),
top_p: options?.top_p,
stop: options?.stop,
tools: sanitizeCohereTools(options?.tools),
@@ -92,7 +92,7 @@ export class CohereProvider extends BaseProvider {
model: modelId,
messages: flattenMessageContent(messages),
temperature: options?.temperature,
- max_tokens: resolveMaxTokens(this.platform, options?.max_tokens),
+ max_tokens: resolveMaxTokens(this.platform, options?.max_tokens, options?.contextBudget),
top_p: options?.top_p,
stop: options?.stop,
tools: sanitizeCohereTools(options?.tools),
diff --git a/server/src/providers/google.ts b/server/src/providers/google.ts
index 0ef71b8b..45faa88a 100644
--- a/server/src/providers/google.ts
+++ b/server/src/providers/google.ts
@@ -560,7 +560,7 @@ export class GoogleProvider extends BaseProvider {
contents: request.contents,
generationConfig: {
temperature: options?.temperature,
- maxOutputTokens: resolveMaxTokens(this.platform, options?.max_tokens),
+ maxOutputTokens: resolveMaxTokens(this.platform, options?.max_tokens, options?.contextBudget),
topP: options?.top_p,
stopSequences: toGeminiStopSequences(options?.stop),
...toGeminiExtendedConfig(options),
@@ -643,7 +643,7 @@ export class GoogleProvider extends BaseProvider {
contents: request.contents,
generationConfig: {
temperature: options?.temperature,
- maxOutputTokens: resolveMaxTokens(this.platform, options?.max_tokens),
+ maxOutputTokens: resolveMaxTokens(this.platform, options?.max_tokens, options?.contextBudget),
topP: options?.top_p,
stopSequences: toGeminiStopSequences(options?.stop),
...toGeminiExtendedConfig(options),
diff --git a/server/src/providers/moondream.ts b/server/src/providers/moondream.ts
index 5eaac226..f23076cf 100644
--- a/server/src/providers/moondream.ts
+++ b/server/src/providers/moondream.ts
@@ -52,7 +52,7 @@ export class MoondreamProvider extends BaseProvider {
return {
model: modelId, messages: input, stream,
temperature: options?.temperature, top_p: options?.top_p,
- max_completion_tokens: resolveMaxTokens(this.platform, options?.max_tokens),
+ max_completion_tokens: resolveMaxTokens(this.platform, options?.max_tokens, options?.contextBudget),
...(options?.reasoning_effort !== undefined ? { reasoning: options.reasoning_effort !== 'none' } : {}),
...(stream && options?.stream_options ? { stream_options: options.stream_options } : {}),
};
diff --git a/server/src/providers/openai-compat.ts b/server/src/providers/openai-compat.ts
index c5aff35d..d1a9977b 100644
--- a/server/src/providers/openai-compat.ts
+++ b/server/src/providers/openai-compat.ts
@@ -284,7 +284,7 @@ export class OpenAICompatProvider extends BaseProvider {
model: modelId,
messages: this.messagesForPlatform(messages, modelId),
temperature: sampling.temperature,
- max_tokens: resolveMaxTokens(this.platform, options?.max_tokens),
+ max_tokens: resolveMaxTokens(this.platform, options?.max_tokens, options?.contextBudget),
top_p: sampling.topP,
stop: options?.stop,
tools: options?.tools,
@@ -413,7 +413,7 @@ export class OpenAICompatProvider extends BaseProvider {
model: modelId,
messages: this.messagesForPlatform(messages, modelId),
temperature: sampling.temperature,
- max_tokens: resolveMaxTokens(this.platform, options?.max_tokens),
+ max_tokens: resolveMaxTokens(this.platform, options?.max_tokens, options?.contextBudget),
top_p: sampling.topP,
stop: options?.stop,
tools: options?.tools,
diff --git a/server/src/providers/sail.ts b/server/src/providers/sail.ts
index f6107335..b73220ff 100644
--- a/server/src/providers/sail.ts
+++ b/server/src/providers/sail.ts
@@ -175,7 +175,7 @@ export class SailProvider extends BaseProvider {
}
private buildBody(messages: ChatMessage[], modelId: string, options?: CompletionOptions): Record<string, unknown> {
- const maxOutputTokens = resolveMaxTokens(this.platform, options?.max_tokens);
+ const maxOutputTokens = resolveMaxTokens(this.platform, options?.max_tokens, options?.contextBudget);
const reasoningEffort = this.reasoningEffort(modelId, options);
const textFormat = this.textFormat(options);
const tools = options?.tools?.map(tool => ({
diff --git a/server/src/routes/anthropic.ts b/server/src/routes/anthropic.ts
index 63cd5029..e4ea5898 100644
--- a/server/src/routes/anthropic.ts
+++ b/server/src/routes/anthropic.ts
@@ -660,9 +660,11 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {
return routeRequest(routingTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain ?? resolvedChain?.chain, false, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined, outputReserve, taskType);
},
dispatch: async (route, attempt, dispatchCtx) => {
+ const contextBudget = route.contextWindow != null ? route.contextWindow - estimatedInputTokens : undefined;
+ const routeOptions = { ...dispatchOptions, contextBudget };
if (stream) {
try {
- await streamCompletion(res, route, messages, dispatchOptions, {
+ await streamCompletion(res, route, messages, routeOptions, {
start, attempt, attemptLog, clientGone: () => clientGone, requestedModel, estimatedInputTokens, tools, pinnedModelId,
sessionId, pinned: resolved.pinned, stickyScope, strategyKey, disarmHedge: dispatchCtx.disarmHedge, state,
});
@@ -675,7 +677,7 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {
}
}
- const result = await route.provider.chatCompletion(route.apiKey, messages, route.modelId, dispatchOptions);
+ const result = await route.provider.chatCompletion(route.apiKey, messages, route.modelId, routeOptions);
const respMsg = result.choices?.[0]?.message;
const respText = contentToString(respMsg?.content ?? '');
let respToolCalls = respMsg?.tool_calls ?? [];
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 6d7204c3..a361343b 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -1192,6 +1192,7 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
);
},
dispatch: async (route, attempt, ctx) => {
+ const contextBudget = route.contextWindow != null ? route.contextWindow - estimatedInputTokens : undefined;
traceRouteEvent('Proxy', {
event: attempt === 0 ? 'start' : 'next',
requestId: requestGroupId,
@@ -1239,7 +1240,7 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
route.apiKey,
dispatchMessages,
route.modelId,
- { temperature, max_tokens, top_p, stop, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) },
+ { temperature, max_tokens, top_p, stop, contextBudget, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) },
quotaContextForRoute(route, 'chat/completions'),
);
@@ -1339,7 +1340,7 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
route.apiKey,
dispatchMessages,
route.modelId,
- { temperature, max_tokens, top_p, stop, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) },
+ { temperature, max_tokens, top_p, stop, contextBudget, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) },
quotaContextForRoute(route, 'chat/completions'),
);
@@ -2111,6 +2112,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
return routeRequest(routingEstimate, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain ?? resolvedChain?.chain, samplingParams.response_format !== undefined, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined, outputReserve, taskType);
},
dispatch: async (route, attempt, ctx) => {
+ const contextBudget = route.contextWindow != null ? route.contextWindow - estimatedInputTokens : undefined;
const modelKey = `${route.platform}:${route.modelId}`;
traceRouteEvent('Proxy', {
event: attempt === 0 ? 'start' : 'next',
@@ -2253,7 +2255,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
try {
const gen = route.provider.streamChatCompletion(
route.apiKey, outboundMessages, route.modelId,
- { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, stream_options: parsed.data.stream_options, ...samplingParams, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) },
+ { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, stream_options: parsed.data.stream_options, ...samplingParams, contextBudget, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) },
quotaContextForRoute(route, 'chat/completions'),
);
@@ -2635,7 +2637,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
} else {
const result = await route.provider.chatCompletion(
route.apiKey, outboundMessages, route.modelId,
- { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, ...samplingParams, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) },
+ { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, ...samplingParams, contextBudget, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) },
quotaContextForRoute(route, 'chat/completions'),
);
diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.ts
index 74b64f12..69e7ec27 100644
--- a/server/src/routes/responses.ts
+++ b/server/src/routes/responses.ts
@@ -1141,6 +1141,8 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
return routeRequest(routingTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain ?? resolvedChain?.chain, completionOpts.response_format !== undefined, state.skipPlatforms.size > 0 ? state.skipPlatforms : undefined, outputReserve, taskType);
},
dispatch: async (route, attempt, ctx) => {
+ const contextBudget = route.contextWindow != null ? route.contextWindow - estimatedInputTokens : undefined;
+ const routeOpts = contextBudget != null ? { ...dispatchOpts, contextBudget } : dispatchOpts;
traceRouteEvent('Responses', {
event: attempt === 0 ? 'start' : 'next',
requestId: requestGroupId,
@@ -1235,7 +1237,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
route.apiKey,
messages,
route.modelId,
- dispatchOpts,
+ routeOpts,
quotaContextForRoute(route, 'responses'),
);
@@ -1473,7 +1475,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
route.apiKey,
messages,
route.modelId,
- dispatchOpts,
+ routeOpts,
quotaContextForRoute(route, 'responses'),
);
diff --git a/server/src/services/fusion.ts b/server/src/services/fusion.ts
index c9afdbd1..83bab2f5 100644
--- a/server/src/services/fusion.ts
+++ b/server/src/services/fusion.ts
@@ -275,8 +275,11 @@ async function runModelCall(
if (!route) break;
const startedAt = Date.now();
+ const routeOpts = route.contextWindow != null
+ ? { ...options, contextBudget: route.contextWindow - estimatedTokens }
+ : options;
try {
- const result = await route.provider.chatCompletion(route.apiKey, messages, route.modelId, options);
+ const result = await route.provider.chatCompletion(route.apiKey, messages, route.modelId, routeOpts);
const choice = result.choices?.[0];
const text = contentToString(choice?.message?.content ?? '');
const toolCalls = choice?.message?.tool_calls;
@@ -363,10 +366,13 @@ async function runJudgeStreaming(
if (!route) break;
const startedAt = Date.now();
+ const routeOpts = route.contextWindow != null
+ ? { ...options, contextBudget: route.contextWindow - estimatedTokens }
+ : options;
let text = '';
let started = false;
try {
- for await (const chunk of route.provider.streamChatCompletion(route.apiKey, messages, route.modelId, options)) {
+ for await (const chunk of route.provider.streamChatCompletion(route.apiKey, messages, route.modelId, routeOpts)) {
const delta = (chunk as any)?.choices?.[0]?.delta?.content;
if (typeof delta === 'string' && delta.length > 0) {
if (!started) { started = true; cb.onStart?.({ platform: route.platform, model: route.modelId }); }
diff --git a/server/src/services/router.ts b/server/src/services/router.ts
index 2cc15d71..9f17506b 100644
--- a/server/src/services/router.ts
+++ b/server/src/services/router.ts
@@ -198,6 +198,7 @@ export interface RouteResult {
keyLabel: string | null;
platform: string;
displayName: string;
+ contextWindow: number | null;
/**
* The custom endpoint this route belongs to, '' for catalog platforms (#651).
* Carried on the route so the failure path can attribute a retirement signal
@@ -1540,6 +1541,7 @@ function selectKeyForModel(entry: ChainRow, estimatedTokens: number, skipKeys?:
proxyUrl,
platform: entry.platform,
displayName: entry.display_name,
+ contextWindow: entry.context_window,
endpointScope: entry.endpoint_scope ?? '',
rpdLimit: limits.rpd,
tpdLimit: limits.tpd,