Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(proxy): accept empty assistant content, normalize array deltas, context-aware routing

Tashfeenf7e4318f9f9f
diff --git a/server/src/__tests__/lib/content.test.ts b/server/src/__tests__/lib/content.test.ts
index f6a0648a..791fdc4f 100644
--- a/server/src/__tests__/lib/content.test.ts
+++ b/server/src/__tests__/lib/content.test.ts
@@ -1,5 +1,5 @@
import { describe, it, expect } from 'vitest';
-import { contentToString, flattenMessageContent, messageHasImage } from '../../lib/content.js';
+import { contentToString, flattenMessageContent, messageHasImage, normalizeOutboundContent } from '../../lib/content.js';
describe('contentToString', () => {
it('passes strings through', () => {
@@ -86,3 +86,35 @@ describe('messageHasImage', () => {
expect(messageHasImage([{ role: 'assistant', content: null }])).toBe(false);
});
});
+
+describe('normalizeOutboundContent (#166)', () => {
+ it('coerces array delta.content to a string on streaming chunks', () => {
+ const chunk = { choices: [{ index: 0, delta: { content: [{ type: 'text', text: 'hel' }, { type: 'text', text: 'lo' }] } }] };
+ const out = normalizeOutboundContent(chunk);
+ expect(out.choices[0].delta.content).toBe('hello');
+ });
+
+ it('coerces array message.content to a string on non-stream responses', () => {
+ const result = { choices: [{ index: 0, message: { role: 'assistant', content: [{ type: 'text', text: 'got it' }] } }] };
+ const out = normalizeOutboundContent(result);
+ expect(out.choices[0].message.content).toBe('got it');
+ });
+
+ it('preserves tool_calls even when text content is array-shaped', () => {
+ const chunk = { choices: [{ delta: { content: [{ type: 'text', text: '' }], tool_calls: [{ index: 0, id: 'c1', function: { name: 'f', arguments: '{}' } }] } }] };
+ const out = normalizeOutboundContent(chunk);
+ expect(out.choices[0].delta.content).toBe('');
+ expect(out.choices[0].delta.tool_calls[0].id).toBe('c1');
+ });
+
+ it('leaves string content untouched', () => {
+ const chunk = { choices: [{ delta: { content: 'already a string' } }] };
+ expect(normalizeOutboundContent(chunk).choices[0].delta.content).toBe('already a string');
+ });
+
+ it('tolerates chunks with no choices array (usage/keepalive frames)', () => {
+ expect(() => normalizeOutboundContent({ usage: { prompt_tokens: 1 } })).not.toThrow();
+ expect(() => normalizeOutboundContent(null as unknown)).not.toThrow();
+ expect(() => normalizeOutboundContent({} as unknown)).not.toThrow();
+ });
+});
diff --git a/server/src/__tests__/routes/proxy-array-content.test.ts b/server/src/__tests__/routes/proxy-array-content.test.ts
index 4c8a8eea..172bb5de 100644
--- a/server/src/__tests__/routes/proxy-array-content.test.ts
+++ b/server/src/__tests__/routes/proxy-array-content.test.ts
@@ -139,10 +139,19 @@ describe('OpenAI multimodal array content', () => {
expect(status).toBe(400);
});
- it('rejects an assistant message with neither content nor tool_calls', async () => {
- const { status } = await request(app, 'POST', '/v1/chat/completions', {
- messages: [{ role: 'assistant', content: [] }],
+ it('accepts an assistant message with empty content and no tool_calls (#165)', async () => {
+ // OpenAI accepts empty/null assistant turns in history; we coerce to "" and
+ // forward rather than 400-ing a payload OpenAI would take. The request then
+ // routes (and fails downstream on the fake key) — the point is it is NOT a
+ // 400 schema rejection.
+ const { status, body } = await request(app, 'POST', '/v1/chat/completions', {
+ messages: [
+ { role: 'user', content: 'hi' },
+ { role: 'assistant', content: [] },
+ { role: 'user', content: 'continue' },
+ ],
}, authHeaders());
- expect(status).toBe(400);
+ expect(status).not.toBe(400);
+ if (status === 400) throw new Error(`unexpected 400: ${JSON.stringify(body)}`);
});
});
diff --git a/server/src/__tests__/routes/proxy-retry.test.ts b/server/src/__tests__/routes/proxy-retry.test.ts
index 51a66175..e46d1c24 100644
--- a/server/src/__tests__/routes/proxy-retry.test.ts
+++ b/server/src/__tests__/routes/proxy-retry.test.ts
@@ -32,6 +32,27 @@ describe('isRetryableError', () => {
});
});
+ describe('provider tool-call generation 400s fail over (#168)', () => {
+ // Groq (and every other openai-compat provider) throws its errors as
+ // `${name} API error ${status}: ${msg}`, so a tool-call-generation failure
+ // surfaces as "Groq API error 400: Failed to call a function...". That
+ // matches the "api error 400" rule, so it's ALREADY retryable and fails
+ // over to the next provider — #168 is covered by existing behavior.
+ it('treats a Groq failed_generation 400 as retryable', () => {
+ expect(isRetryableError(new Error(
+ "Groq API error 400: Failed to call a function. Please adjust your prompt. See 'failed_generation' for more details.",
+ ))).toBe(true);
+ });
+
+ it('treats any openai-compat "API error 400" as retryable (one provider rejects params another accepts)', () => {
+ expect(isRetryableError(new Error('Cerebras API error 400: tool schema not supported'))).toBe(true);
+ });
+
+ it('but a bare validation "400 Bad Request" (our own schema) is still NOT retryable', () => {
+ expect(isRetryableError(new Error('400 Bad Request'))).toBe(false);
+ });
+ });
+
describe('existing categories still classify correctly', () => {
it('429 / rate limits are retryable', () => {
expect(isRetryableError(new Error('429 Too Many Requests'))).toBe(true);
diff --git a/server/src/__tests__/services/router.test.ts b/server/src/__tests__/services/router.test.ts
index bf80611f..edee86b4 100644
--- a/server/src/__tests__/services/router.test.ts
+++ b/server/src/__tests__/services/router.test.ts
@@ -100,6 +100,43 @@ describe('Router', () => {
expect(result.platform).toBe('groq');
});
+ it('skips a model whose context window cannot hold the request (#167)', () => {
+ const db = getDb();
+ const groqKey = encrypt('test-groq-key');
+ db.prepare(`
+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)
+ VALUES (?, ?, ?, ?, ?, ?, ?)
+ `).run('groq', 'test', groqKey.encrypted, groqKey.iv, groqKey.authTag, 'healthy', 1);
+
+ // Remove token rate-limit interference so we isolate the context-window
+ // behavior (canUseTokens would otherwise also skip on a large estimate).
+ db.prepare("UPDATE models SET tpm_limit = NULL, tpd_limit = NULL WHERE platform = 'groq'").run();
+
+ // Whatever model a small request lands on, give it a tiny context window.
+ const baseline = routeRequest(5);
+ db.prepare('UPDATE models SET context_window = 10 WHERE id = ?').run(baseline.modelDbId);
+
+ // A small request still lands on it (5 < 10) ...
+ expect(routeRequest(5).modelDbId).toBe(baseline.modelDbId);
+
+ // ... but a request larger than its window is routed elsewhere (2000 > 10).
+ const large = routeRequest(2000);
+ expect(large.modelDbId).not.toBe(baseline.modelDbId);
+ });
+
+ it('still routes a model with an unknown (null) context window (#167)', () => {
+ const db = getDb();
+ const groqKey = encrypt('test-groq-key');
+ db.prepare(`
+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)
+ VALUES (?, ?, ?, ?, ?, ?, ?)
+ `).run('groq', 'test', groqKey.encrypted, groqKey.iv, groqKey.authTag, 'healthy', 1);
+ db.prepare("UPDATE models SET tpm_limit = NULL, tpd_limit = NULL WHERE platform = 'groq'").run();
+ // A null context_window means "unknown" — never filtered out, even for a huge request.
+ db.prepare("UPDATE models SET context_window = NULL WHERE platform = 'groq'").run();
+ expect(() => routeRequest(500000)).not.toThrow();
+ });
+
it('should skip keys that cannot be decrypted and use a valid fallback key', () => {
const db = getDb();
diff --git a/server/src/lib/content.ts b/server/src/lib/content.ts
index b01623ac..f4ef8f46 100644
--- a/server/src/lib/content.ts
+++ b/server/src/lib/content.ts
@@ -45,3 +45,29 @@ export function contentHasImage(content: unknown): boolean {
export function messageHasImage(messages: ChatMessage[]): boolean {
return messages.some((m) => contentHasImage(m.content));
}
+
+// Normalize the OUTBOUND (provider → client) shape so we honor the OpenAI
+// contract on the response path the same way `contentToString` does on the
+// request path. Per spec, `choices[].delta.content` (streaming) and
+// `choices[].message.content` (non-stream) are strings; some providers
+// (e.g. Mistral magistral) return an array of content blocks. Forwarding the
+// array verbatim breaks string-consuming clients ("expected str, got list")
+// and, mid-stream, drops the turn's tool calls. We coerce array content to a
+// string while leaving `tool_calls` and every other field untouched. Mutates
+// and returns the same object (chunks are parsed fresh from JSON per frame, so
+// in-place mutation is safe). Non-array content passes through unchanged. (#166)
+export function normalizeOutboundContent<T>(payload: T): T {
+ const choices = (payload as { choices?: unknown })?.choices;
+ if (!Array.isArray(choices)) return payload;
+ for (const choice of choices) {
+ const delta = (choice as { delta?: { content?: unknown } })?.delta;
+ if (delta && Array.isArray(delta.content)) {
+ delta.content = contentToString(delta.content);
+ }
+ const message = (choice as { message?: { content?: unknown } })?.message;
+ if (message && Array.isArray(message.content)) {
+ message.content = contentToString(message.content);
+ }
+ }
+ return payload;
+}
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 961bad3c..61fbeeed 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -6,7 +6,7 @@ import type { ChatMessage } from '@freellmapi/shared/types.js';
import { routeRequest, recordRateLimitHit, recordSuccess, hasEnabledVisionModel, type RouteResult } from '../services/router.js';
import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit } from '../services/ratelimit.js';
import { getDb, getUnifiedApiKey } from '../db/index.js';
-import { contentToString, messageHasImage } from '../lib/content.js';
+import { contentToString, messageHasImage, normalizeOutboundContent } from '../lib/content.js';
export const proxyRouter = Router();
@@ -145,12 +145,6 @@ const toolCallSchema = z.object({
const contentBlockSchema = z.object({ type: z.string() }).passthrough();
const contentSchema = z.union([z.string(), z.array(contentBlockSchema)]);
-function hasNonEmptyContent(content: unknown): boolean {
- if (typeof content === 'string') return content.length > 0;
- if (Array.isArray(content)) return content.length > 0;
- return false;
-}
-
const systemMessageSchema = z.object({
role: z.literal('system'),
content: contentSchema,
@@ -163,17 +157,17 @@ const userMessageSchema = z.object({
name: z.string().optional(),
});
+// Assistant turns may carry empty/null content and no tool_calls — OpenAI
+// accepts these in conversation history (a turn that produced no visible text,
+// a placeholder, a tool turn whose content was emptied), and clients replay
+// them verbatim. We accept them too and coerce empty/null content to "" before
+// forwarding (see message build below) rather than 400-ing a payload OpenAI
+// would take. (#165)
const assistantMessageSchema = z.object({
role: z.literal('assistant'),
content: z.union([contentSchema, z.null()]).optional(),
name: z.string().optional(),
tool_calls: z.array(toolCallSchema).optional(),
-}).refine((msg) => {
- const hasContent = hasNonEmptyContent(msg.content);
- const hasToolCalls = (msg.tool_calls?.length ?? 0) > 0;
- return hasContent || hasToolCalls;
-}, {
- message: 'assistant messages must include non-empty content or tool_calls',
});
const toolMessageSchema = z.object({
@@ -283,9 +277,19 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
const { model: requestedModel, temperature, max_tokens, top_p, stream, tools, tool_choice, parallel_tool_calls } = parsed.data;
const messages: ChatMessage[] = parsed.data.messages.map((m): ChatMessage => {
if (m.role === 'assistant') {
+ const hasToolCalls = (m.tool_calls?.length ?? 0) > 0;
+ // With tool_calls, content: null is the correct OpenAI shape — keep it.
+ // Without tool_calls, coerce empty/null content to "" so strict upstreams
+ // don't choke on a null-content assistant turn we just accepted. (#165)
+ const isEmptyContent = m.content == null
+ || (typeof m.content === 'string' && m.content.length === 0)
+ || (Array.isArray(m.content) && m.content.length === 0);
+ const assistantContent: ChatMessage['content'] = hasToolCalls
+ ? (m.content ?? null)
+ : (isEmptyContent ? '' : m.content!);
return {
role: 'assistant',
- content: m.content ?? null,
+ content: assistantContent,
...(m.name ? { name: m.name } : {}),
...(m.tool_calls ? { tool_calls: m.tool_calls.map(tc => ({
id: tc.id,
@@ -426,6 +430,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
if (attempt > 0) res.setHeader('X-Fallback-Attempts', String(attempt));
streamStarted = true;
}
+ // Coerce array-shaped delta.content to a string before forwarding,
+ // so spec-conforming clients don't break and tool_calls survive (#166).
+ normalizeOutboundContent(chunk);
const text = streamChunkText(chunk);
totalOutputTokens += Math.ceil(text.length / 4);
res.write(`data: ${JSON.stringify(chunk)}\n\n`);
@@ -473,7 +480,8 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
res.setHeader('X-Routed-Via', `${route.platform}/${route.modelId}`);
if (attempt > 0) res.setHeader('X-Fallback-Attempts', String(attempt));
- res.json(result);
+ // Normalize array-shaped message.content to a string on the way out (#166).
+ res.json(normalizeOutboundContent(result));
logRequest(
route.platform, route.modelId, route.keyId, 'success',
diff --git a/server/src/services/router.ts b/server/src/services/router.ts
index 6349f6dd..20d8d966 100644
--- a/server/src/services/router.ts
+++ b/server/src/services/router.ts
@@ -38,6 +38,7 @@ interface ChainRow {
tpm_limit: number | null;
tpd_limit: number | null;
supports_vision: number;
+ context_window: number | null;
}
export interface RouteResult {
@@ -351,7 +352,8 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre
SELECT fc.model_db_id, fc.priority, fc.enabled,
m.platform, m.model_id, m.display_name, m.intelligence_rank,
m.size_label, m.monthly_token_budget,
- m.rpm_limit, m.rpd_limit, m.tpm_limit, m.tpd_limit, m.supports_vision
+ m.rpm_limit, m.rpd_limit, m.tpm_limit, m.tpd_limit, m.supports_vision,
+ m.context_window
FROM fallback_config fc
JOIN models m ON m.id = fc.model_db_id AND m.enabled = 1
WHERE fc.enabled = 1
@@ -373,6 +375,17 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre
// which is correct: don't pin an image turn to a model that can't see it.
if (requireVision && !entry.supports_vision) continue;
+ // Context-aware routing: skip a model whose context window can't hold the
+ // request, so a large prompt never selects a small-context model and burns
+ // a failover hop on a 413 "request too large" (#167). Only enforced when we
+ // know the model's window; estimatedTokens already includes the reserved
+ // output (max_tokens), so this is the total-context check the model must
+ // satisfy. A 413 that slips through is still retryable downstream, and the
+ // failed model is put on cooldown — so this is a fast-path, not the only
+ // guard. If every model is too small, the loop falls through and the caller
+ // gets the normal "all models exhausted" error rather than a wasted sweep.
+ if (entry.context_window != null && estimatedTokens > entry.context_window) continue;
+
// Check if we have a provider for this platform
const provider = getProvider(entry.platform as any);
if (!provider) continue;
@@ -484,7 +497,8 @@ export function getRoutingScores(): { strategy: RoutingStrategy; weights: Routin
SELECT fc.model_db_id, fc.priority, fc.enabled,
m.platform, m.model_id, m.display_name, m.intelligence_rank,
m.size_label, m.monthly_token_budget,
- m.rpm_limit, m.rpd_limit, m.tpm_limit, m.tpd_limit, m.supports_vision
+ m.rpm_limit, m.rpd_limit, m.tpm_limit, m.tpd_limit, m.supports_vision,
+ m.context_window
FROM fallback_config fc
JOIN models m ON m.id = fc.model_db_id
WHERE m.enabled = 1