7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
refactor(proxy): tighten context-handoff follow-ups
Tashfeen4a1d3b77abf1
diff --git a/README.md b/README.mdindex fbd5c801..2dd39f26 100644--- a/README.md+++ b/README.md@@ -446,7 +446,7 @@ When FreeLLMAPI falls over to a different model mid-conversation (quota, rate li```FreeLLMAPI context handoff:-You are taking over an ongoing coding-agent conversation from another model (groq:llama-3 → google:gemini-flash).+You are taking over an ongoing conversation from another model (groq:llama-3 → google:gemini-flash).Continue the user's task using the conversation context already provided in this request.Do not restart the task, re-ask already answered setup questions, or discard prior tool results.Respect the user's latest message as the highest-priority instruction.@@ -542,6 +542,7 @@ PRs should include a test, keep the existing test suite green, and match the `.e<a href="https://github.com/hmm183"><img src="https://images.weserv.nl/?url=github.com/hmm183.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@hmm183" /></a><a href="https://github.com/duemilionidieuro-bot"><img src="https://images.weserv.nl/?url=github.com/duemilionidieuro-bot.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@duemilionidieuro-bot" /></a><a href="https://github.com/hjhhoni"><img src="https://images.weserv.nl/?url=github.com/hjhhoni.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@hjhhoni" /></a>+<a href="https://github.com/immanuelsavio"><img src="https://images.weserv.nl/?url=github.com/immanuelsavio.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@immanuelsavio" /></a>## Terms of Service reviewdiff --git a/package-lock.json b/package-lock.jsonindex c6d2e4f4..7f390b1c 100644--- a/package-lock.json+++ b/package-lock.json@@ -1238,6 +1238,7 @@"os": ["aix"],+ "peer": true,"engines": {"node": ">=18"}@@ -1254,6 +1255,7 @@"os": ["android"],+ "peer": true,"engines": {"node": ">=18"}@@ -1270,6 +1272,7 @@"os": ["android"],+ "peer": true,"engines": {"node": ">=18"}@@ -1286,6 +1289,7 @@"os": ["android"],+ "peer": true,"engines": {"node": ">=18"}@@ -1302,6 +1306,7 @@"os": ["darwin"],+ "peer": true,"engines": {"node": ">=18"}@@ -1318,6 +1323,7 @@"os": ["darwin"],+ "peer": true,"engines": {"node": ">=18"}@@ -1334,6 +1340,7 @@"os": ["freebsd"],+ "peer": true,"engines": {"node": ">=18"}@@ -1350,6 +1357,7 @@"os": ["freebsd"],+ "peer": true,"engines": {"node": ">=18"}@@ -1366,6 +1374,7 @@"os": ["linux"],+ "peer": true,"engines": {"node": ">=18"}@@ -1382,6 +1391,7 @@"os": ["linux"],+ "peer": true,"engines": {"node": ">=18"}@@ -1398,6 +1408,7 @@"os": ["linux"],+ "peer": true,"engines": {"node": ">=18"}@@ -1414,6 +1425,7 @@"os": ["linux"],+ "peer": true,"engines": {"node": ">=18"}@@ -1430,6 +1442,7 @@"os": ["linux"],+ "peer": true,"engines": {"node": ">=18"}@@ -1446,6 +1459,7 @@"os": ["linux"],+ "peer": true,"engines": {"node": ">=18"}@@ -1462,6 +1476,7 @@"os": ["linux"],+ "peer": true,"engines": {"node": ">=18"}@@ -1478,6 +1493,7 @@"os": ["linux"],+ "peer": true,"engines": {"node": ">=18"}@@ -1494,6 +1510,7 @@"os": ["linux"],+ "peer": true,"engines": {"node": ">=18"}@@ -1510,6 +1527,7 @@"os": ["netbsd"],+ "peer": true,"engines": {"node": ">=18"}@@ -1526,6 +1544,7 @@"os": ["netbsd"],+ "peer": true,"engines": {"node": ">=18"}@@ -1542,6 +1561,7 @@"os": ["openbsd"],+ "peer": true,"engines": {"node": ">=18"}@@ -1558,6 +1578,7 @@"os": ["openbsd"],+ "peer": true,"engines": {"node": ">=18"}@@ -1574,6 +1595,7 @@"os": ["openharmony"],+ "peer": true,"engines": {"node": ">=18"}@@ -1590,6 +1612,7 @@"os": ["sunos"],+ "peer": true,"engines": {"node": ">=18"}@@ -1606,6 +1629,7 @@"os": ["win32"],+ "peer": true,"engines": {"node": ">=18"}@@ -1622,6 +1646,7 @@"os": ["win32"],+ "peer": true,"engines": {"node": ">=18"}@@ -1638,6 +1663,7 @@"os": ["win32"],+ "peer": true,"engines": {"node": ">=18"}diff --git a/server/src/__tests__/services/context-handoff.test.ts b/server/src/__tests__/services/context-handoff.test.tsindex 209c41ae..9af10f68 100644--- a/server/src/__tests__/services/context-handoff.test.ts+++ b/server/src/__tests__/services/context-handoff.test.ts@@ -4,6 +4,7 @@ import {recordIncomingMessages,maybeInjectContextHandoff,recordSuccessfulModel,+ hasPriorModel,HANDOFF_MAX_TOKENS,_clearStoreForTesting,} from '../../services/context-handoff.js';@@ -310,3 +311,60 @@ describe('HANDOFF_MAX_TOKENS', () => {expect(typeof HANDOFF_MAX_TOKENS).toBe('number');});});++describe('hasPriorModel', () => {+ it('is false for an unknown session', () => {+ expect(hasPriorModel('nope')).toBe(false);+ });++ it('is false for an empty session key', () => {+ expect(hasPriorModel('')).toBe(false);+ });++ it('is false after recordIncomingMessages alone (no model yet)', () => {+ recordIncomingMessages('sess1', messages);+ expect(hasPriorModel('sess1')).toBe(false);+ });++ it('is true once a model has succeeded for the session', () => {+ recordIncomingMessages('sess1', messages);+ recordSuccessfulModel({ sessionKey: 'sess1', modelKey: 'groq:llama-3' });+ expect(hasPriorModel('sess1')).toBe(true);+ });++ it('flips back to false when a fresh conversation clears the prior model', () => {+ recordIncomingMessages('sess1', messages);+ recordSuccessfulModel({ sessionKey: 'sess1', modelKey: 'groq:llama-3' });+ // Reused session ID, brand-new conversation (no assistant turns) → cleared.+ recordIncomingMessages('sess1', [msg('user', 'brand new')]);+ expect(hasPriorModel('sess1')).toBe(false);+ });+});++describe('injectedTokens', () => {+ it('is 0 when no handoff is injected', () => {+ const result = maybeInjectContextHandoff({+ mode: 'on_model_switch',+ sessionKey: 'sess1',+ messages,+ selectedModelKey: 'groq:llama-3',+ });+ expect(result.injected).toBe(false);+ expect(result.injectedTokens).toBe(0);+ });++ it('is a positive estimate when a handoff is injected', () => {+ recordIncomingMessages('sess1', messages);+ recordSuccessfulModel({ sessionKey: 'sess1', modelKey: 'groq:llama-3' });+ const result = maybeInjectContextHandoff({+ mode: 'on_model_switch',+ sessionKey: 'sess1',+ messages,+ selectedModelKey: 'google:gemini-flash',+ });+ expect(result.injected).toBe(true);+ expect(result.injectedTokens).toBeGreaterThan(0);+ // Never exceeds the conservative routing-pad upper bound.+ expect(result.injectedTokens).toBeLessThanOrEqual(HANDOFF_MAX_TOKENS);+ });+});diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 907acf61..06b300fe 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -12,7 +12,7 @@ import { contentToString, messageHasImage, normalizeOutboundContent } from '../limport { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';import { rescueInlineToolCalls, startsWithDialectMarker, couldBecomeDialectMarker, containsDialectMarker } from '../lib/tool-call-rescue.js';-import { getContextHandoffMode, recordIncomingMessages, maybeInjectContextHandoff, recordSuccessfulModel, HANDOFF_MAX_TOKENS } from '../services/context-handoff.js';+import { getContextHandoffMode, recordIncomingMessages, maybeInjectContextHandoff, recordSuccessfulModel, hasPriorModel, HANDOFF_MAX_TOKENS } from '../services/context-handoff.js';export const proxyRouter = Router();@@ -633,6 +633,11 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {if (handoffMode !== 'off' && sessionKey) {recordIncomingMessages(sessionKey, messages);}+ // A handoff can only fire when a prior model is on record for this session.+ // Check after recordIncomingMessages, which clears the prior model on a+ // fresh conversation. Stable across the retry loop (the prior model only+ // changes on a success, which returns), so compute it once here.+ const handoffPossible = handoffMode !== 'off' && !!sessionKey && hasPriorModel(sessionKey);// Explicit `model` field pins routing. If the catalog has no enabled row// matching the requested id, return 400 — silently auto-routing to a@@ -676,13 +681,13 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {let route: RouteResult;try {- // When handoff injection is active, pad the token estimate so the router's+ // When a handoff could fire this turn, pad the token estimate so the router's// context-window and TPM checks account for the extra system message overhead.- // This padding is conservative and fires on every auto-routed request (not just- // on turns where injection will actually occur), because we don't know the- // selected model key until after routeRequest() returns. The trade-off is a- // ~1600-token headroom tax that prevents under-counting on turns that do inject.- const routingEstimate = (handoffMode !== 'off' && sessionKey) ? estimatedTotal + HANDOFF_MAX_TOKENS : estimatedTotal;+ // We don't know the selected model key until after routeRequest() returns, so+ // the padding is conservative on turns where injection is *possible* (a prior+ // model is on record). Turns where injection can't happen — every turn 1, and+ // sessions that never switched — pay no headroom tax.+ const routingEstimate = handoffPossible ? estimatedTotal + HANDOFF_MAX_TOKENS : estimatedTotal;route = routeRequest(routingEstimate, skipKeys.size > 0 ? skipKeys : undefined, preferredModel, hasImage, wantsTools, skipModels.size > 0 ? skipModels : undefined);} catch (err: any) {// No more models available@@ -704,10 +709,16 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {const modelKey = `${route.platform}:${route.modelId}`;let outboundMessages = messages;+ // Extra input tokens the injected handoff adds on this turn (0 when not+ // injected). Folded into the streaming success accounting, where token+ // counts are estimated; the non-stream path uses the provider's usage,+ // which already counts the injected message.+ let injectedHandoffTokens = 0;if (handoffMode !== 'off' && sessionKey) {const handoff = maybeInjectContextHandoff({ mode: handoffMode, sessionKey, messages, selectedModelKey: modelKey });if (handoff.injected) console.log(`[Proxy] Context handoff injected (session ${sessionKey.slice(0, 8)}…, model switch detected)`);outboundMessages = handoff.messages;+ injectedHandoffTokens = handoff.injectedTokens;}try {@@ -909,11 +920,11 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {res.end();recordRequest(route.platform, route.modelId, route.keyId);- recordTokens(route.platform, route.modelId, route.keyId, estimatedInputTokens + totalOutputTokens);+ recordTokens(route.platform, route.modelId, route.keyId, estimatedInputTokens + injectedHandoffTokens + totalOutputTokens);recordSuccess(route.modelDbId);setStickyModel(messages, route.modelDbId, sessionIdHeader);if (handoffMode !== 'off' && sessionKey) recordSuccessfulModel({ sessionKey, modelKey });- logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens, totalOutputTokens, Date.now() - start, null, ttfbMs, pinnedModelId);+ logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens + injectedHandoffTokens, totalOutputTokens, Date.now() - start, null, ttfbMs, pinnedModelId);return;} catch (streamErr: any) {if (headerSent) {diff --git a/server/src/services/context-handoff.ts b/server/src/services/context-handoff.tsindex 7ede5449..d9de756f 100644--- a/server/src/services/context-handoff.ts+++ b/server/src/services/context-handoff.ts@@ -29,9 +29,20 @@ export function getContextHandoffMode(): ContextHandoffMode {return raw === 'on_model_switch' ? 'on_model_switch' : 'off';}+// Slice without cutting through a surrogate pair. A bare String.slice can+// leave a lone high surrogate when it lands mid astral-plane char (emoji etc.),+// and some providers 400 on a lone surrogate. Trim that trailing half-char off.+// (Lib-agnostic — String.prototype.toWellFormed needs the es2024 lib target.)+function safeSlice(text: string, max: number): string {+ if (text.length <= max) return text;+ const lastCode = text.charCodeAt(max - 1);+ const end = lastCode >= 0xd800 && lastCode <= 0xdbff ? max - 1 : max;+ return text.slice(0, end);+}+function trimContent(content: ChatMessage['content']): string {const text = contentToString(content);- return text.length > MAX_CONTENT_PER_MSG ? text.slice(0, MAX_CONTENT_PER_MSG) + '…' : text;+ return text.length > MAX_CONTENT_PER_MSG ? safeSlice(text, MAX_CONTENT_PER_MSG) + '…' : text;}function pruneExpired(): void {@@ -84,22 +95,31 @@ function buildSummary(messages: TrimmedMessage[]): string {const lines = messages.map(m => `${m.role === 'user' ? 'User' : 'Assistant'}: ${m.content}`);const joined = lines.join('\n');return joined.length > MAX_HANDOFF_CHARS- ? joined.slice(0, MAX_HANDOFF_CHARS) + '\n…[truncated]'+ ? safeSlice(joined, MAX_HANDOFF_CHARS) + '\n…[truncated]': joined;}+// True when a prior successful model is on record for this session, i.e. the+// next request *could* trigger a handoff. proxy.ts uses this to skip padding+// the routing token estimate on turns where injection is impossible (every+// turn 1, and sessions that never switched), avoiding a needless headroom tax.+export function hasPriorModel(sessionKey: string): boolean {+ if (!sessionKey) return false;+ return !!store.get(sessionKey)?.lastModelKey;+}+export function maybeInjectContextHandoff(params: {mode: ContextHandoffMode;sessionKey: string;messages: ChatMessage[];selectedModelKey: string;-}): { messages: ChatMessage[]; injected: boolean } {+}): { messages: ChatMessage[]; injected: boolean; injectedTokens: number } {const { mode, sessionKey, messages, selectedModelKey } = params;- if (mode === 'off' || !sessionKey) return { messages, injected: false };+ if (mode === 'off' || !sessionKey) return { messages, injected: false, injectedTokens: 0 };const ctx = store.get(sessionKey);if (!ctx?.lastModelKey || ctx.lastModelKey === selectedModelKey) {- return { messages, injected: false };+ return { messages, injected: false, injectedTokens: 0 };}// Skip if a handoff message is already present — handles both plain strings@@ -109,12 +129,12 @@ export function maybeInjectContextHandoff(params: {const text = typeof m.content === 'string' ? m.content : contentToString(m.content);return text.startsWith('FreeLLMAPI context handoff:');});- if (alreadyPresent) return { messages, injected: false };+ if (alreadyPresent) return { messages, injected: false, injectedTokens: 0 };const summary = buildSummary(ctx.recentMessages);const handoffContent = ['FreeLLMAPI context handoff:',- `You are taking over an ongoing coding-agent conversation from another model (${ctx.lastModelKey} → ${selectedModelKey}).`,+ `You are taking over an ongoing conversation from another model (${ctx.lastModelKey} → ${selectedModelKey}).`,'Continue the user\'s task using the conversation context already provided in this request.','Do not restart the task, re-ask already answered setup questions, or discard prior tool results.','Respect the user\'s latest message as the highest-priority instruction.',@@ -132,6 +152,7 @@ export function maybeInjectContextHandoff(params: {return {messages: [...messages.slice(0, pos), handoffMsg, ...messages.slice(pos)],injected: true,+ injectedTokens: Math.ceil(handoffContent.length / 4),};}