Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

refactor(proxy): tighten context-handoff follow-ups

Tashfeen4a1d3b77abf1
diff --git a/README.md b/README.md
index fbd5c801..2dd39f26 100644
--- a/README.md
+++ b/README.md
@@ -446,7 +446,7 @@ When FreeLLMAPI falls over to a different model mid-conversation (quota, rate li
```
FreeLLMAPI context handoff:
-You are taking over an ongoing coding-agent conversation from another model (groq:llama-3 → google:gemini-flash).
+You are taking over an ongoing conversation from another model (groq:llama-3 → google:gemini-flash).
Continue the user's task using the conversation context already provided in this request.
Do not restart the task, re-ask already answered setup questions, or discard prior tool results.
Respect the user's latest message as the highest-priority instruction.
@@ -542,6 +542,7 @@ PRs should include a test, keep the existing test suite green, and match the `.e
<a href="https://github.com/hmm183"><img src="https://images.weserv.nl/?url=github.com/hmm183.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@hmm183" /></a>
<a href="https://github.com/duemilionidieuro-bot"><img src="https://images.weserv.nl/?url=github.com/duemilionidieuro-bot.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@duemilionidieuro-bot" /></a>
<a href="https://github.com/hjhhoni"><img src="https://images.weserv.nl/?url=github.com/hjhhoni.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@hjhhoni" /></a>
+<a href="https://github.com/immanuelsavio"><img src="https://images.weserv.nl/?url=github.com/immanuelsavio.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@immanuelsavio" /></a>
## Terms of Service review
diff --git a/package-lock.json b/package-lock.json
index c6d2e4f4..7f390b1c 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -1238,6 +1238,7 @@
"os": [
"aix"
],
+ "peer": true,
"engines": {
"node": ">=18"
}
@@ -1254,6 +1255,7 @@
"os": [
"android"
],
+ "peer": true,
"engines": {
"node": ">=18"
}
@@ -1270,6 +1272,7 @@
"os": [
"android"
],
+ "peer": true,
"engines": {
"node": ">=18"
}
@@ -1286,6 +1289,7 @@
"os": [
"android"
],
+ "peer": true,
"engines": {
"node": ">=18"
}
@@ -1302,6 +1306,7 @@
"os": [
"darwin"
],
+ "peer": true,
"engines": {
"node": ">=18"
}
@@ -1318,6 +1323,7 @@
"os": [
"darwin"
],
+ "peer": true,
"engines": {
"node": ">=18"
}
@@ -1334,6 +1340,7 @@
"os": [
"freebsd"
],
+ "peer": true,
"engines": {
"node": ">=18"
}
@@ -1350,6 +1357,7 @@
"os": [
"freebsd"
],
+ "peer": true,
"engines": {
"node": ">=18"
}
@@ -1366,6 +1374,7 @@
"os": [
"linux"
],
+ "peer": true,
"engines": {
"node": ">=18"
}
@@ -1382,6 +1391,7 @@
"os": [
"linux"
],
+ "peer": true,
"engines": {
"node": ">=18"
}
@@ -1398,6 +1408,7 @@
"os": [
"linux"
],
+ "peer": true,
"engines": {
"node": ">=18"
}
@@ -1414,6 +1425,7 @@
"os": [
"linux"
],
+ "peer": true,
"engines": {
"node": ">=18"
}
@@ -1430,6 +1442,7 @@
"os": [
"linux"
],
+ "peer": true,
"engines": {
"node": ">=18"
}
@@ -1446,6 +1459,7 @@
"os": [
"linux"
],
+ "peer": true,
"engines": {
"node": ">=18"
}
@@ -1462,6 +1476,7 @@
"os": [
"linux"
],
+ "peer": true,
"engines": {
"node": ">=18"
}
@@ -1478,6 +1493,7 @@
"os": [
"linux"
],
+ "peer": true,
"engines": {
"node": ">=18"
}
@@ -1494,6 +1510,7 @@
"os": [
"linux"
],
+ "peer": true,
"engines": {
"node": ">=18"
}
@@ -1510,6 +1527,7 @@
"os": [
"netbsd"
],
+ "peer": true,
"engines": {
"node": ">=18"
}
@@ -1526,6 +1544,7 @@
"os": [
"netbsd"
],
+ "peer": true,
"engines": {
"node": ">=18"
}
@@ -1542,6 +1561,7 @@
"os": [
"openbsd"
],
+ "peer": true,
"engines": {
"node": ">=18"
}
@@ -1558,6 +1578,7 @@
"os": [
"openbsd"
],
+ "peer": true,
"engines": {
"node": ">=18"
}
@@ -1574,6 +1595,7 @@
"os": [
"openharmony"
],
+ "peer": true,
"engines": {
"node": ">=18"
}
@@ -1590,6 +1612,7 @@
"os": [
"sunos"
],
+ "peer": true,
"engines": {
"node": ">=18"
}
@@ -1606,6 +1629,7 @@
"os": [
"win32"
],
+ "peer": true,
"engines": {
"node": ">=18"
}
@@ -1622,6 +1646,7 @@
"os": [
"win32"
],
+ "peer": true,
"engines": {
"node": ">=18"
}
@@ -1638,6 +1663,7 @@
"os": [
"win32"
],
+ "peer": true,
"engines": {
"node": ">=18"
}
diff --git a/server/src/__tests__/services/context-handoff.test.ts b/server/src/__tests__/services/context-handoff.test.ts
index 209c41ae..9af10f68 100644
--- a/server/src/__tests__/services/context-handoff.test.ts
+++ b/server/src/__tests__/services/context-handoff.test.ts
@@ -4,6 +4,7 @@ import {
recordIncomingMessages,
maybeInjectContextHandoff,
recordSuccessfulModel,
+ hasPriorModel,
HANDOFF_MAX_TOKENS,
_clearStoreForTesting,
} from '../../services/context-handoff.js';
@@ -310,3 +311,60 @@ describe('HANDOFF_MAX_TOKENS', () => {
expect(typeof HANDOFF_MAX_TOKENS).toBe('number');
});
});
+
+describe('hasPriorModel', () => {
+ it('is false for an unknown session', () => {
+ expect(hasPriorModel('nope')).toBe(false);
+ });
+
+ it('is false for an empty session key', () => {
+ expect(hasPriorModel('')).toBe(false);
+ });
+
+ it('is false after recordIncomingMessages alone (no model yet)', () => {
+ recordIncomingMessages('sess1', messages);
+ expect(hasPriorModel('sess1')).toBe(false);
+ });
+
+ it('is true once a model has succeeded for the session', () => {
+ recordIncomingMessages('sess1', messages);
+ recordSuccessfulModel({ sessionKey: 'sess1', modelKey: 'groq:llama-3' });
+ expect(hasPriorModel('sess1')).toBe(true);
+ });
+
+ it('flips back to false when a fresh conversation clears the prior model', () => {
+ recordIncomingMessages('sess1', messages);
+ recordSuccessfulModel({ sessionKey: 'sess1', modelKey: 'groq:llama-3' });
+ // Reused session ID, brand-new conversation (no assistant turns) → cleared.
+ recordIncomingMessages('sess1', [msg('user', 'brand new')]);
+ expect(hasPriorModel('sess1')).toBe(false);
+ });
+});
+
+describe('injectedTokens', () => {
+ it('is 0 when no handoff is injected', () => {
+ const result = maybeInjectContextHandoff({
+ mode: 'on_model_switch',
+ sessionKey: 'sess1',
+ messages,
+ selectedModelKey: 'groq:llama-3',
+ });
+ expect(result.injected).toBe(false);
+ expect(result.injectedTokens).toBe(0);
+ });
+
+ it('is a positive estimate when a handoff is injected', () => {
+ recordIncomingMessages('sess1', messages);
+ recordSuccessfulModel({ sessionKey: 'sess1', modelKey: 'groq:llama-3' });
+ const result = maybeInjectContextHandoff({
+ mode: 'on_model_switch',
+ sessionKey: 'sess1',
+ messages,
+ selectedModelKey: 'google:gemini-flash',
+ });
+ expect(result.injected).toBe(true);
+ expect(result.injectedTokens).toBeGreaterThan(0);
+ // Never exceeds the conservative routing-pad upper bound.
+ expect(result.injectedTokens).toBeLessThanOrEqual(HANDOFF_MAX_TOKENS);
+ });
+});
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 907acf61..06b300fe 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -12,7 +12,7 @@ import { contentToString, messageHasImage, normalizeOutboundContent } from '../l
import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';
import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';
import { rescueInlineToolCalls, startsWithDialectMarker, couldBecomeDialectMarker, containsDialectMarker } from '../lib/tool-call-rescue.js';
-import { getContextHandoffMode, recordIncomingMessages, maybeInjectContextHandoff, recordSuccessfulModel, HANDOFF_MAX_TOKENS } from '../services/context-handoff.js';
+import { getContextHandoffMode, recordIncomingMessages, maybeInjectContextHandoff, recordSuccessfulModel, hasPriorModel, HANDOFF_MAX_TOKENS } from '../services/context-handoff.js';
export const proxyRouter = Router();
@@ -633,6 +633,11 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
if (handoffMode !== 'off' && sessionKey) {
recordIncomingMessages(sessionKey, messages);
}
+ // A handoff can only fire when a prior model is on record for this session.
+ // Check after recordIncomingMessages, which clears the prior model on a
+ // fresh conversation. Stable across the retry loop (the prior model only
+ // changes on a success, which returns), so compute it once here.
+ const handoffPossible = handoffMode !== 'off' && !!sessionKey && hasPriorModel(sessionKey);
// Explicit `model` field pins routing. If the catalog has no enabled row
// matching the requested id, return 400 — silently auto-routing to a
@@ -676,13 +681,13 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
let route: RouteResult;
try {
- // When handoff injection is active, pad the token estimate so the router's
+ // When a handoff could fire this turn, pad the token estimate so the router's
// context-window and TPM checks account for the extra system message overhead.
- // This padding is conservative and fires on every auto-routed request (not just
- // on turns where injection will actually occur), because we don't know the
- // selected model key until after routeRequest() returns. The trade-off is a
- // ~1600-token headroom tax that prevents under-counting on turns that do inject.
- const routingEstimate = (handoffMode !== 'off' && sessionKey) ? estimatedTotal + HANDOFF_MAX_TOKENS : estimatedTotal;
+ // We don't know the selected model key until after routeRequest() returns, so
+ // the padding is conservative on turns where injection is *possible* (a prior
+ // model is on record). Turns where injection can't happen — every turn 1, and
+ // sessions that never switched — pay no headroom tax.
+ const routingEstimate = handoffPossible ? estimatedTotal + HANDOFF_MAX_TOKENS : estimatedTotal;
route = routeRequest(routingEstimate, skipKeys.size > 0 ? skipKeys : undefined, preferredModel, hasImage, wantsTools, skipModels.size > 0 ? skipModels : undefined);
} catch (err: any) {
// No more models available
@@ -704,10 +709,16 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
const modelKey = `${route.platform}:${route.modelId}`;
let outboundMessages = messages;
+ // Extra input tokens the injected handoff adds on this turn (0 when not
+ // injected). Folded into the streaming success accounting, where token
+ // counts are estimated; the non-stream path uses the provider's usage,
+ // which already counts the injected message.
+ let injectedHandoffTokens = 0;
if (handoffMode !== 'off' && sessionKey) {
const handoff = maybeInjectContextHandoff({ mode: handoffMode, sessionKey, messages, selectedModelKey: modelKey });
if (handoff.injected) console.log(`[Proxy] Context handoff injected (session ${sessionKey.slice(0, 8)}…, model switch detected)`);
outboundMessages = handoff.messages;
+ injectedHandoffTokens = handoff.injectedTokens;
}
try {
@@ -909,11 +920,11 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
res.end();
recordRequest(route.platform, route.modelId, route.keyId);
- recordTokens(route.platform, route.modelId, route.keyId, estimatedInputTokens + totalOutputTokens);
+ recordTokens(route.platform, route.modelId, route.keyId, estimatedInputTokens + injectedHandoffTokens + totalOutputTokens);
recordSuccess(route.modelDbId);
setStickyModel(messages, route.modelDbId, sessionIdHeader);
if (handoffMode !== 'off' && sessionKey) recordSuccessfulModel({ sessionKey, modelKey });
- logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens, totalOutputTokens, Date.now() - start, null, ttfbMs, pinnedModelId);
+ logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens + injectedHandoffTokens, totalOutputTokens, Date.now() - start, null, ttfbMs, pinnedModelId);
return;
} catch (streamErr: any) {
if (headerSent) {
diff --git a/server/src/services/context-handoff.ts b/server/src/services/context-handoff.ts
index 7ede5449..d9de756f 100644
--- a/server/src/services/context-handoff.ts
+++ b/server/src/services/context-handoff.ts
@@ -29,9 +29,20 @@ export function getContextHandoffMode(): ContextHandoffMode {
return raw === 'on_model_switch' ? 'on_model_switch' : 'off';
}
+// Slice without cutting through a surrogate pair. A bare String.slice can
+// leave a lone high surrogate when it lands mid astral-plane char (emoji etc.),
+// and some providers 400 on a lone surrogate. Trim that trailing half-char off.
+// (Lib-agnostic — String.prototype.toWellFormed needs the es2024 lib target.)
+function safeSlice(text: string, max: number): string {
+ if (text.length <= max) return text;
+ const lastCode = text.charCodeAt(max - 1);
+ const end = lastCode >= 0xd800 && lastCode <= 0xdbff ? max - 1 : max;
+ return text.slice(0, end);
+}
+
function trimContent(content: ChatMessage['content']): string {
const text = contentToString(content);
- return text.length > MAX_CONTENT_PER_MSG ? text.slice(0, MAX_CONTENT_PER_MSG) + '…' : text;
+ return text.length > MAX_CONTENT_PER_MSG ? safeSlice(text, MAX_CONTENT_PER_MSG) + '…' : text;
}
function pruneExpired(): void {
@@ -84,22 +95,31 @@ function buildSummary(messages: TrimmedMessage[]): string {
const lines = messages.map(m => `${m.role === 'user' ? 'User' : 'Assistant'}: ${m.content}`);
const joined = lines.join('\n');
return joined.length > MAX_HANDOFF_CHARS
- ? joined.slice(0, MAX_HANDOFF_CHARS) + '\n…[truncated]'
+ ? safeSlice(joined, MAX_HANDOFF_CHARS) + '\n…[truncated]'
: joined;
}
+// True when a prior successful model is on record for this session, i.e. the
+// next request *could* trigger a handoff. proxy.ts uses this to skip padding
+// the routing token estimate on turns where injection is impossible (every
+// turn 1, and sessions that never switched), avoiding a needless headroom tax.
+export function hasPriorModel(sessionKey: string): boolean {
+ if (!sessionKey) return false;
+ return !!store.get(sessionKey)?.lastModelKey;
+}
+
export function maybeInjectContextHandoff(params: {
mode: ContextHandoffMode;
sessionKey: string;
messages: ChatMessage[];
selectedModelKey: string;
-}): { messages: ChatMessage[]; injected: boolean } {
+}): { messages: ChatMessage[]; injected: boolean; injectedTokens: number } {
const { mode, sessionKey, messages, selectedModelKey } = params;
- if (mode === 'off' || !sessionKey) return { messages, injected: false };
+ if (mode === 'off' || !sessionKey) return { messages, injected: false, injectedTokens: 0 };
const ctx = store.get(sessionKey);
if (!ctx?.lastModelKey || ctx.lastModelKey === selectedModelKey) {
- return { messages, injected: false };
+ return { messages, injected: false, injectedTokens: 0 };
}
// Skip if a handoff message is already present — handles both plain strings
@@ -109,12 +129,12 @@ export function maybeInjectContextHandoff(params: {
const text = typeof m.content === 'string' ? m.content : contentToString(m.content);
return text.startsWith('FreeLLMAPI context handoff:');
});
- if (alreadyPresent) return { messages, injected: false };
+ if (alreadyPresent) return { messages, injected: false, injectedTokens: 0 };
const summary = buildSummary(ctx.recentMessages);
const handoffContent = [
'FreeLLMAPI context handoff:',
- `You are taking over an ongoing coding-agent conversation from another model (${ctx.lastModelKey} → ${selectedModelKey}).`,
+ `You are taking over an ongoing conversation from another model (${ctx.lastModelKey} → ${selectedModelKey}).`,
'Continue the user\'s task using the conversation context already provided in this request.',
'Do not restart the task, re-ask already answered setup questions, or discard prior tool results.',
'Respect the user\'s latest message as the highest-priority instruction.',
@@ -132,6 +152,7 @@ export function maybeInjectContextHandoff(params: {
return {
messages: [...messages.slice(0, pos), handoffMsg, ...messages.slice(pos)],
injected: true,
+ injectedTokens: Math.ceil(handoffContent.length / 4),
};
}