7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(proxy): accept empty assistant content, normalize array deltas, context-aware routing
Tashfeenf7e4318f9f9f
diff --git a/server/src/__tests__/lib/content.test.ts b/server/src/__tests__/lib/content.test.tsindex f6a0648a..791fdc4f 100644--- a/server/src/__tests__/lib/content.test.ts+++ b/server/src/__tests__/lib/content.test.ts@@ -1,5 +1,5 @@import { describe, it, expect } from 'vitest';-import { contentToString, flattenMessageContent, messageHasImage } from '../../lib/content.js';+import { contentToString, flattenMessageContent, messageHasImage, normalizeOutboundContent } from '../../lib/content.js';describe('contentToString', () => {it('passes strings through', () => {@@ -86,3 +86,35 @@ describe('messageHasImage', () => {expect(messageHasImage([{ role: 'assistant', content: null }])).toBe(false);});});++describe('normalizeOutboundContent (#166)', () => {+ it('coerces array delta.content to a string on streaming chunks', () => {+ const chunk = { choices: [{ index: 0, delta: { content: [{ type: 'text', text: 'hel' }, { type: 'text', text: 'lo' }] } }] };+ const out = normalizeOutboundContent(chunk);+ expect(out.choices[0].delta.content).toBe('hello');+ });++ it('coerces array message.content to a string on non-stream responses', () => {+ const result = { choices: [{ index: 0, message: { role: 'assistant', content: [{ type: 'text', text: 'got it' }] } }] };+ const out = normalizeOutboundContent(result);+ expect(out.choices[0].message.content).toBe('got it');+ });++ it('preserves tool_calls even when text content is array-shaped', () => {+ const chunk = { choices: [{ delta: { content: [{ type: 'text', text: '' }], tool_calls: [{ index: 0, id: 'c1', function: { name: 'f', arguments: '{}' } }] } }] };+ const out = normalizeOutboundContent(chunk);+ expect(out.choices[0].delta.content).toBe('');+ expect(out.choices[0].delta.tool_calls[0].id).toBe('c1');+ });++ it('leaves string content untouched', () => {+ const chunk = { choices: [{ delta: { content: 'already a string' } }] };+ expect(normalizeOutboundContent(chunk).choices[0].delta.content).toBe('already a string');+ });++ it('tolerates chunks with no choices array (usage/keepalive frames)', () => {+ expect(() => normalizeOutboundContent({ usage: { prompt_tokens: 1 } })).not.toThrow();+ expect(() => normalizeOutboundContent(null as unknown)).not.toThrow();+ expect(() => normalizeOutboundContent({} as unknown)).not.toThrow();+ });+});diff --git a/server/src/__tests__/routes/proxy-array-content.test.ts b/server/src/__tests__/routes/proxy-array-content.test.tsindex 4c8a8eea..172bb5de 100644--- a/server/src/__tests__/routes/proxy-array-content.test.ts+++ b/server/src/__tests__/routes/proxy-array-content.test.ts@@ -139,10 +139,19 @@ describe('OpenAI multimodal array content', () => {expect(status).toBe(400);});- it('rejects an assistant message with neither content nor tool_calls', async () => {- const { status } = await request(app, 'POST', '/v1/chat/completions', {- messages: [{ role: 'assistant', content: [] }],+ it('accepts an assistant message with empty content and no tool_calls (#165)', async () => {+ // OpenAI accepts empty/null assistant turns in history; we coerce to "" and+ // forward rather than 400-ing a payload OpenAI would take. The request then+ // routes (and fails downstream on the fake key) — the point is it is NOT a+ // 400 schema rejection.+ const { status, body } = await request(app, 'POST', '/v1/chat/completions', {+ messages: [+ { role: 'user', content: 'hi' },+ { role: 'assistant', content: [] },+ { role: 'user', content: 'continue' },+ ],}, authHeaders());- expect(status).toBe(400);+ expect(status).not.toBe(400);+ if (status === 400) throw new Error(`unexpected 400: ${JSON.stringify(body)}`);});});diff --git a/server/src/__tests__/routes/proxy-retry.test.ts b/server/src/__tests__/routes/proxy-retry.test.tsindex 51a66175..e46d1c24 100644--- a/server/src/__tests__/routes/proxy-retry.test.ts+++ b/server/src/__tests__/routes/proxy-retry.test.ts@@ -32,6 +32,27 @@ describe('isRetryableError', () => {});});+ describe('provider tool-call generation 400s fail over (#168)', () => {+ // Groq (and every other openai-compat provider) throws its errors as+ // `${name} API error ${status}: ${msg}`, so a tool-call-generation failure+ // surfaces as "Groq API error 400: Failed to call a function...". That+ // matches the "api error 400" rule, so it's ALREADY retryable and fails+ // over to the next provider — #168 is covered by existing behavior.+ it('treats a Groq failed_generation 400 as retryable', () => {+ expect(isRetryableError(new Error(+ "Groq API error 400: Failed to call a function. Please adjust your prompt. See 'failed_generation' for more details.",+ ))).toBe(true);+ });++ it('treats any openai-compat "API error 400" as retryable (one provider rejects params another accepts)', () => {+ expect(isRetryableError(new Error('Cerebras API error 400: tool schema not supported'))).toBe(true);+ });++ it('but a bare validation "400 Bad Request" (our own schema) is still NOT retryable', () => {+ expect(isRetryableError(new Error('400 Bad Request'))).toBe(false);+ });+ });+describe('existing categories still classify correctly', () => {it('429 / rate limits are retryable', () => {expect(isRetryableError(new Error('429 Too Many Requests'))).toBe(true);diff --git a/server/src/__tests__/services/router.test.ts b/server/src/__tests__/services/router.test.tsindex bf80611f..edee86b4 100644--- a/server/src/__tests__/services/router.test.ts+++ b/server/src/__tests__/services/router.test.ts@@ -100,6 +100,43 @@ describe('Router', () => {expect(result.platform).toBe('groq');});+ it('skips a model whose context window cannot hold the request (#167)', () => {+ const db = getDb();+ const groqKey = encrypt('test-groq-key');+ db.prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)+ VALUES (?, ?, ?, ?, ?, ?, ?)+ `).run('groq', 'test', groqKey.encrypted, groqKey.iv, groqKey.authTag, 'healthy', 1);++ // Remove token rate-limit interference so we isolate the context-window+ // behavior (canUseTokens would otherwise also skip on a large estimate).+ db.prepare("UPDATE models SET tpm_limit = NULL, tpd_limit = NULL WHERE platform = 'groq'").run();++ // Whatever model a small request lands on, give it a tiny context window.+ const baseline = routeRequest(5);+ db.prepare('UPDATE models SET context_window = 10 WHERE id = ?').run(baseline.modelDbId);++ // A small request still lands on it (5 < 10) ...+ expect(routeRequest(5).modelDbId).toBe(baseline.modelDbId);++ // ... but a request larger than its window is routed elsewhere (2000 > 10).+ const large = routeRequest(2000);+ expect(large.modelDbId).not.toBe(baseline.modelDbId);+ });++ it('still routes a model with an unknown (null) context window (#167)', () => {+ const db = getDb();+ const groqKey = encrypt('test-groq-key');+ db.prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)+ VALUES (?, ?, ?, ?, ?, ?, ?)+ `).run('groq', 'test', groqKey.encrypted, groqKey.iv, groqKey.authTag, 'healthy', 1);+ db.prepare("UPDATE models SET tpm_limit = NULL, tpd_limit = NULL WHERE platform = 'groq'").run();+ // A null context_window means "unknown" — never filtered out, even for a huge request.+ db.prepare("UPDATE models SET context_window = NULL WHERE platform = 'groq'").run();+ expect(() => routeRequest(500000)).not.toThrow();+ });+it('should skip keys that cannot be decrypted and use a valid fallback key', () => {const db = getDb();diff --git a/server/src/lib/content.ts b/server/src/lib/content.tsindex b01623ac..f4ef8f46 100644--- a/server/src/lib/content.ts+++ b/server/src/lib/content.ts@@ -45,3 +45,29 @@ export function contentHasImage(content: unknown): boolean {export function messageHasImage(messages: ChatMessage[]): boolean {return messages.some((m) => contentHasImage(m.content));}++// Normalize the OUTBOUND (provider → client) shape so we honor the OpenAI+// contract on the response path the same way `contentToString` does on the+// request path. Per spec, `choices[].delta.content` (streaming) and+// `choices[].message.content` (non-stream) are strings; some providers+// (e.g. Mistral magistral) return an array of content blocks. Forwarding the+// array verbatim breaks string-consuming clients ("expected str, got list")+// and, mid-stream, drops the turn's tool calls. We coerce array content to a+// string while leaving `tool_calls` and every other field untouched. Mutates+// and returns the same object (chunks are parsed fresh from JSON per frame, so+// in-place mutation is safe). Non-array content passes through unchanged. (#166)+export function normalizeOutboundContent<T>(payload: T): T {+ const choices = (payload as { choices?: unknown })?.choices;+ if (!Array.isArray(choices)) return payload;+ for (const choice of choices) {+ const delta = (choice as { delta?: { content?: unknown } })?.delta;+ if (delta && Array.isArray(delta.content)) {+ delta.content = contentToString(delta.content);+ }+ const message = (choice as { message?: { content?: unknown } })?.message;+ if (message && Array.isArray(message.content)) {+ message.content = contentToString(message.content);+ }+ }+ return payload;+}diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 961bad3c..61fbeeed 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -6,7 +6,7 @@ import type { ChatMessage } from '@freellmapi/shared/types.js';import { routeRequest, recordRateLimitHit, recordSuccess, hasEnabledVisionModel, type RouteResult } from '../services/router.js';import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit } from '../services/ratelimit.js';import { getDb, getUnifiedApiKey } from '../db/index.js';-import { contentToString, messageHasImage } from '../lib/content.js';+import { contentToString, messageHasImage, normalizeOutboundContent } from '../lib/content.js';export const proxyRouter = Router();@@ -145,12 +145,6 @@ const toolCallSchema = z.object({const contentBlockSchema = z.object({ type: z.string() }).passthrough();const contentSchema = z.union([z.string(), z.array(contentBlockSchema)]);-function hasNonEmptyContent(content: unknown): boolean {- if (typeof content === 'string') return content.length > 0;- if (Array.isArray(content)) return content.length > 0;- return false;-}-const systemMessageSchema = z.object({role: z.literal('system'),content: contentSchema,@@ -163,17 +157,17 @@ const userMessageSchema = z.object({name: z.string().optional(),});+// Assistant turns may carry empty/null content and no tool_calls — OpenAI+// accepts these in conversation history (a turn that produced no visible text,+// a placeholder, a tool turn whose content was emptied), and clients replay+// them verbatim. We accept them too and coerce empty/null content to "" before+// forwarding (see message build below) rather than 400-ing a payload OpenAI+// would take. (#165)const assistantMessageSchema = z.object({role: z.literal('assistant'),content: z.union([contentSchema, z.null()]).optional(),name: z.string().optional(),tool_calls: z.array(toolCallSchema).optional(),-}).refine((msg) => {- const hasContent = hasNonEmptyContent(msg.content);- const hasToolCalls = (msg.tool_calls?.length ?? 0) > 0;- return hasContent || hasToolCalls;-}, {- message: 'assistant messages must include non-empty content or tool_calls',});const toolMessageSchema = z.object({@@ -283,9 +277,19 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {const { model: requestedModel, temperature, max_tokens, top_p, stream, tools, tool_choice, parallel_tool_calls } = parsed.data;const messages: ChatMessage[] = parsed.data.messages.map((m): ChatMessage => {if (m.role === 'assistant') {+ const hasToolCalls = (m.tool_calls?.length ?? 0) > 0;+ // With tool_calls, content: null is the correct OpenAI shape — keep it.+ // Without tool_calls, coerce empty/null content to "" so strict upstreams+ // don't choke on a null-content assistant turn we just accepted. (#165)+ const isEmptyContent = m.content == null+ || (typeof m.content === 'string' && m.content.length === 0)+ || (Array.isArray(m.content) && m.content.length === 0);+ const assistantContent: ChatMessage['content'] = hasToolCalls+ ? (m.content ?? null)+ : (isEmptyContent ? '' : m.content!);return {role: 'assistant',- content: m.content ?? null,+ content: assistantContent,...(m.name ? { name: m.name } : {}),...(m.tool_calls ? { tool_calls: m.tool_calls.map(tc => ({id: tc.id,@@ -426,6 +430,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {if (attempt > 0) res.setHeader('X-Fallback-Attempts', String(attempt));streamStarted = true;}+ // Coerce array-shaped delta.content to a string before forwarding,+ // so spec-conforming clients don't break and tool_calls survive (#166).+ normalizeOutboundContent(chunk);const text = streamChunkText(chunk);totalOutputTokens += Math.ceil(text.length / 4);res.write(`data: ${JSON.stringify(chunk)}\n\n`);@@ -473,7 +480,8 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {res.setHeader('X-Routed-Via', `${route.platform}/${route.modelId}`);if (attempt > 0) res.setHeader('X-Fallback-Attempts', String(attempt));- res.json(result);+ // Normalize array-shaped message.content to a string on the way out (#166).+ res.json(normalizeOutboundContent(result));logRequest(route.platform, route.modelId, route.keyId, 'success',diff --git a/server/src/services/router.ts b/server/src/services/router.tsindex 6349f6dd..20d8d966 100644--- a/server/src/services/router.ts+++ b/server/src/services/router.ts@@ -38,6 +38,7 @@ interface ChainRow {tpm_limit: number | null;tpd_limit: number | null;supports_vision: number;+ context_window: number | null;}export interface RouteResult {@@ -351,7 +352,8 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preSELECT fc.model_db_id, fc.priority, fc.enabled,m.platform, m.model_id, m.display_name, m.intelligence_rank,m.size_label, m.monthly_token_budget,- m.rpm_limit, m.rpd_limit, m.tpm_limit, m.tpd_limit, m.supports_vision+ m.rpm_limit, m.rpd_limit, m.tpm_limit, m.tpd_limit, m.supports_vision,+ m.context_windowFROM fallback_config fcJOIN models m ON m.id = fc.model_db_id AND m.enabled = 1WHERE fc.enabled = 1@@ -373,6 +375,17 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre// which is correct: don't pin an image turn to a model that can't see it.if (requireVision && !entry.supports_vision) continue;+ // Context-aware routing: skip a model whose context window can't hold the+ // request, so a large prompt never selects a small-context model and burns+ // a failover hop on a 413 "request too large" (#167). Only enforced when we+ // know the model's window; estimatedTokens already includes the reserved+ // output (max_tokens), so this is the total-context check the model must+ // satisfy. A 413 that slips through is still retryable downstream, and the+ // failed model is put on cooldown — so this is a fast-path, not the only+ // guard. If every model is too small, the loop falls through and the caller+ // gets the normal "all models exhausted" error rather than a wasted sweep.+ if (entry.context_window != null && estimatedTokens > entry.context_window) continue;+// Check if we have a provider for this platformconst provider = getProvider(entry.platform as any);if (!provider) continue;@@ -484,7 +497,8 @@ export function getRoutingScores(): { strategy: RoutingStrategy; weights: RoutinSELECT fc.model_db_id, fc.priority, fc.enabled,m.platform, m.model_id, m.display_name, m.intelligence_rank,m.size_label, m.monthly_token_budget,- m.rpm_limit, m.rpd_limit, m.tpm_limit, m.tpd_limit, m.supports_vision+ m.rpm_limit, m.rpd_limit, m.tpm_limit, m.tpd_limit, m.supports_vision,+ m.context_windowFROM fallback_config fcJOIN models m ON m.id = fc.model_db_idWHERE m.enabled = 1