Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat(proxy): accept OpenAI multimodal array content envelope (#75, closes #69, credits @jhash)

Tashfeenf2ece370bbfe
diff --git a/README.md b/README.md
index 53bd8c28..cbb58630 100644
--- a/README.md
+++ b/README.md
@@ -299,6 +299,7 @@ PRs should include a test, keep the existing test suite green, and match the `.e
<a href="https://github.com/nordbyte"><img src="https://images.weserv.nl/?url=github.com/nordbyte.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@nordbyte" /></a>
<a href="https://github.com/mybropro"><img src="https://images.weserv.nl/?url=github.com/mybropro.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@mybropro" /></a>
<a href="https://github.com/danscMax"><img src="https://images.weserv.nl/?url=github.com/danscMax.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@danscMax" /></a>
+<a href="https://github.com/jhash"><img src="https://images.weserv.nl/?url=github.com/jhash.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@jhash" /></a>
## Terms of Service review
diff --git a/server/src/__tests__/lib/content.test.ts b/server/src/__tests__/lib/content.test.ts
new file mode 100644
index 00000000..0840d338
--- /dev/null
+++ b/server/src/__tests__/lib/content.test.ts
@@ -0,0 +1,63 @@
+import { describe, it, expect } from 'vitest';
+import { contentToString, flattenMessageContent } from '../../lib/content.js';
+
+describe('contentToString', () => {
+ it('passes strings through', () => {
+ expect(contentToString('hello')).toBe('hello');
+ expect(contentToString('')).toBe('');
+ });
+
+ it('treats null and undefined as empty string', () => {
+ expect(contentToString(null)).toBe('');
+ expect(contentToString(undefined)).toBe('');
+ });
+
+ it('joins text blocks in OpenAI multimodal array envelope', () => {
+ expect(contentToString([
+ { type: 'text', text: 'hello ' },
+ { type: 'text', text: 'world' },
+ ])).toBe('hello world');
+ });
+
+ it('drops non-text blocks silently (image_url etc.) — vision unsupported', () => {
+ expect(contentToString([
+ { type: 'text', text: 'describe ' },
+ { type: 'image_url', image_url: { url: 'https://example.com/x.png' } },
+ { type: 'text', text: 'this' },
+ ])).toBe('describe this');
+ });
+
+ it('handles an array of bare strings (some clients send this)', () => {
+ expect(contentToString(['foo', 'bar'])).toBe('foobar');
+ });
+
+ it('returns empty string for unrecognized types instead of throwing', () => {
+ expect(contentToString(42 as unknown)).toBe('');
+ expect(contentToString({ unknown: true } as unknown)).toBe('');
+ });
+});
+
+describe('flattenMessageContent', () => {
+ it('converts every message content to a string', () => {
+ const out = flattenMessageContent([
+ { role: 'user', content: 'plain' },
+ { role: 'user', content: [{ type: 'text', text: 'array' }] },
+ { role: 'assistant', content: null, tool_calls: [{ id: 'x', type: 'function', function: { name: 'f', arguments: '{}' } }] },
+ ]);
+ expect(out[0].content).toBe('plain');
+ expect(out[1].content).toBe('array');
+ expect(out[2].content).toBe('');
+ });
+
+ it('preserves other message fields (tool_calls, name, tool_call_id)', () => {
+ const out = flattenMessageContent([
+ { role: 'tool', content: 'result', tool_call_id: 'call-1', name: 'fn' },
+ ]);
+ expect(out[0]).toMatchObject({
+ role: 'tool',
+ content: 'result',
+ tool_call_id: 'call-1',
+ name: 'fn',
+ });
+ });
+});
diff --git a/server/src/__tests__/routes/proxy-array-content.test.ts b/server/src/__tests__/routes/proxy-array-content.test.ts
new file mode 100644
index 00000000..c9605769
--- /dev/null
+++ b/server/src/__tests__/routes/proxy-array-content.test.ts
@@ -0,0 +1,144 @@
+import { describe, it, expect, beforeAll, beforeEach, afterEach, vi } from 'vitest';
+import type { Express } from 'express';
+import { createApp } from '../../app.js';
+import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';
+
+async function request(app: Express, method: string, path: string, body?: any, headers: Record<string, string> = {}) {
+ const server = app.listen(0);
+ const addr = server.address() as any;
+ const url = `http://127.0.0.1:${addr.port}${path}`;
+
+ const res = await fetch(url, {
+ method,
+ headers: { ...(body ? { 'Content-Type': 'application/json' } : {}), ...headers },
+ body: body ? JSON.stringify(body) : undefined,
+ });
+
+ const data = await res.text();
+ server.close();
+
+ let json: any = null;
+ try { json = JSON.parse(data); } catch {}
+
+ return { status: res.status, body: json };
+}
+
+function authHeaders() {
+ return { Authorization: `Bearer ${getUnifiedApiKey()}` };
+}
+
+describe('OpenAI multimodal array content', () => {
+ let app: Express;
+
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ app = createApp();
+ });
+
+ beforeEach(async () => {
+ const db = getDb();
+ db.prepare('DELETE FROM api_keys').run();
+ db.prepare('DELETE FROM requests').run();
+
+ const addKey = await request(app, 'POST', '/api/keys', {
+ platform: 'groq',
+ key: 'gsk_array_content_test',
+ label: 'array-content',
+ });
+ expect(addKey.status).toBe(201);
+ });
+
+ afterEach(() => {
+ vi.restoreAllMocks();
+ });
+
+ it('accepts content as a string (the legacy shape)', async () => {
+ // Provider call will fail (no real key), but schema validation must pass —
+ // we assert it isn't rejected with a 400 zod error.
+ const { status, body } = await request(app, 'POST', '/v1/chat/completions', {
+ messages: [{ role: 'user', content: 'hello' }],
+ }, authHeaders());
+ expect(status).not.toBe(400);
+ if (status === 400) {
+ // Diagnostic if regression: show the validation error.
+ throw new Error(`unexpected 400: ${JSON.stringify(body)}`);
+ }
+ });
+
+ it('accepts content as a text-only multimodal array', async () => {
+ const { status, body } = await request(app, 'POST', '/v1/chat/completions', {
+ messages: [{
+ role: 'user',
+ content: [{ type: 'text', text: 'hello from opencode-style client' }],
+ }],
+ }, authHeaders());
+ expect(status).not.toBe(400);
+ if (status === 400) {
+ throw new Error(`unexpected 400: ${JSON.stringify(body)}`);
+ }
+ });
+
+ it('accepts mixed text + image_url blocks (image blocks are silently dropped)', async () => {
+ const { status } = await request(app, 'POST', '/v1/chat/completions', {
+ messages: [{
+ role: 'user',
+ content: [
+ { type: 'text', text: 'describe' },
+ { type: 'image_url', image_url: { url: 'https://example.com/x.png' } },
+ ],
+ }],
+ }, authHeaders());
+ expect(status).not.toBe(400);
+ });
+
+ it('successfully routes an array-content request and gets a 200 (mocked groq)', async () => {
+ const origFetch = global.fetch;
+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {
+ const urlStr = typeof url === 'string' ? url : url.toString();
+ if (urlStr.includes('api.groq.com/openai/v1/chat/completions')) {
+ // Sanity check that the array shape made it through to the upstream call.
+ const body = JSON.parse(String((init as RequestInit).body));
+ expect(Array.isArray(body.messages[0].content)).toBe(true);
+ return {
+ ok: true,
+ json: () => Promise.resolve({
+ id: 'chatcmpl-array', object: 'chat.completion', created: 1, model: 'openai/gpt-oss-120b',
+ choices: [{
+ index: 0,
+ message: { role: 'assistant', content: 'got it' },
+ finish_reason: 'stop',
+ }],
+ usage: { prompt_tokens: 4, completion_tokens: 2, total_tokens: 6 },
+ }),
+ } as any;
+ }
+ return origFetch(url, init);
+ });
+
+ const { status, body } = await request(app, 'POST', '/v1/chat/completions', {
+ model: 'auto',
+ messages: [{
+ role: 'user',
+ content: [{ type: 'text', text: 'hi' }],
+ }],
+ }, authHeaders());
+
+ expect(status).toBe(200);
+ expect(body.choices[0].message.content).toBe('got it');
+ });
+
+ it('rejects an empty array as missing content', async () => {
+ const { status } = await request(app, 'POST', '/v1/chat/completions', {
+ messages: [], // top-level empty messages
+ }, authHeaders());
+ expect(status).toBe(400);
+ });
+
+ it('rejects an assistant message with neither content nor tool_calls', async () => {
+ const { status } = await request(app, 'POST', '/v1/chat/completions', {
+ messages: [{ role: 'assistant', content: [] }],
+ }, authHeaders());
+ expect(status).toBe(400);
+ });
+});
diff --git a/server/src/lib/content.ts b/server/src/lib/content.ts
new file mode 100644
index 00000000..e0a481f8
--- /dev/null
+++ b/server/src/lib/content.ts
@@ -0,0 +1,30 @@
+import type { ChatMessage } from '@freellmapi/shared/types.js';
+
+// OpenAI-spec message content can be one of:
+// - string (plain text)
+// - null (assistant with tool_calls only)
+// - Array<ContentBlock> (multimodal envelope; we extract text only)
+//
+// freellmapi accepts the array envelope so clients like opencode and
+// continue.dev (which always serialize as arrays) don't 400. Non-text blocks
+// are dropped silently — vision/audio aren't supported (see README).
+export type ContentTextBlock = { type: 'text'; text: string };
+export type ContentBlock = ContentTextBlock | { type: string; [key: string]: unknown };
+
+export function contentToString(content: unknown): string {
+ if (typeof content === 'string') return content;
+ if (content == null) return '';
+ if (Array.isArray(content)) {
+ return content
+ .map((b) => (typeof b === 'string' ? b : (b as ContentTextBlock)?.type === 'text' ? (b as ContentTextBlock).text : ''))
+ .join('');
+ }
+ return '';
+}
+
+export function flattenMessageContent(messages: ChatMessage[]): ChatMessage[] {
+ return messages.map((m) => ({
+ ...m,
+ content: contentToString(m.content),
+ }));
+}
diff --git a/server/src/providers/cloudflare.ts b/server/src/providers/cloudflare.ts
index fb3f39d5..ff748cdf 100644
--- a/server/src/providers/cloudflare.ts
+++ b/server/src/providers/cloudflare.ts
@@ -4,6 +4,7 @@ import type {
ChatCompletionChunk,
} from '@freellmapi/shared/types.js';
import { BaseProvider, type CompletionOptions } from './base.js';
+import { contentToString } from '../lib/content.js';
/**
* Cloudflare Workers AI provider.
@@ -20,12 +21,12 @@ export class CloudflareProvider extends BaseProvider {
return { accountId: apiKey.slice(0, sep), token: apiKey.slice(sep + 1) };
}
- // Cloudflare's OpenAI-compat endpoint rejects `content: null` on assistant
- // messages that carry tool_calls, even though the OpenAI spec allows it.
+ // Cloudflare's OpenAI-compat endpoint:
+ // - rejects `content: null` on assistant messages that carry tool_calls,
+ // even though the OpenAI spec allows it (collapse to '');
+ // - doesn't accept the array content envelope, so flatten to string.
private normalizeMessages(messages: ChatMessage[]): ChatMessage[] {
- return messages.map(m =>
- m.content === null ? { ...m, content: '' } : m,
- );
+ return messages.map(m => ({ ...m, content: contentToString(m.content) }));
}
async chatCompletion(
diff --git a/server/src/providers/cohere.ts b/server/src/providers/cohere.ts
index d75ae178..8ef0a2c2 100644
--- a/server/src/providers/cohere.ts
+++ b/server/src/providers/cohere.ts
@@ -4,6 +4,7 @@ import type {
ChatCompletionChunk,
} from '@freellmapi/shared/types.js';
import { BaseProvider, type CompletionOptions } from './base.js';
+import { flattenMessageContent } from '../lib/content.js';
const API_BASE = 'https://api.cohere.ai/compatibility/v1';
@@ -19,7 +20,7 @@ export class CohereProvider extends BaseProvider {
): Promise<ChatCompletionResponse> {
const body: Record<string, unknown> = {
model: modelId,
- messages,
+ messages: flattenMessageContent(messages),
temperature: options?.temperature,
max_tokens: options?.max_tokens,
top_p: options?.top_p,
@@ -54,7 +55,7 @@ export class CohereProvider extends BaseProvider {
): AsyncGenerator<ChatCompletionChunk> {
const body: Record<string, unknown> = {
model: modelId,
- messages,
+ messages: flattenMessageContent(messages),
temperature: options?.temperature,
max_tokens: options?.max_tokens,
top_p: options?.top_p,
diff --git a/server/src/providers/google.ts b/server/src/providers/google.ts
index beb6dfa0..94535948 100644
--- a/server/src/providers/google.ts
+++ b/server/src/providers/google.ts
@@ -8,6 +8,7 @@ import type {
TokenUsage,
} from '@freellmapi/shared/types.js';
import { BaseProvider, type CompletionOptions } from './base.js';
+import { contentToString } from '../lib/content.js';
const API_BASE = 'https://generativelanguage.googleapis.com/v1beta';
@@ -128,11 +129,14 @@ function toGeminiToolConfig(toolChoice?: ChatToolChoice): { functionCallingConfi
};
}
-// Translate OpenAI messages to Gemini format
+// Translate OpenAI messages to Gemini format. Content may arrive as a string,
+// null, or the OpenAI multimodal array envelope — flatten to string first so
+// system/user/tool messages all surface as `parts: [{ text }]` for Gemini.
function toGeminiContents(messages: ChatMessage[]) {
const systemMessages = messages
- .filter(m => m.role === 'system' && typeof m.content === 'string' && m.content.length > 0)
- .map(m => m.content as string);
+ .filter(m => m.role === 'system')
+ .map(m => contentToString(m.content))
+ .filter(s => s.length > 0);
const toolNameByCallId = new Map<string, string>();
for (const m of messages) {
@@ -147,8 +151,9 @@ function toGeminiContents(messages: ChatMessage[]) {
if (m.role === 'assistant') {
const parts: GeminiPart[] = [];
- if (typeof m.content === 'string' && m.content.length > 0) {
- parts.push({ text: m.content });
+ const assistantText = contentToString(m.content);
+ if (assistantText.length > 0) {
+ parts.push({ text: assistantText });
}
for (const call of m.tool_calls ?? []) {
@@ -174,7 +179,7 @@ function toGeminiContents(messages: ChatMessage[]) {
if (!toolCallId) return null;
const toolName = m.name ?? toolNameByCallId.get(toolCallId) ?? 'tool';
- const response = safeParseObject(typeof m.content === 'string' ? m.content : '');
+ const response = safeParseObject(contentToString(m.content));
return {
role: 'user',
@@ -190,7 +195,7 @@ function toGeminiContents(messages: ChatMessage[]) {
return {
role: 'user',
- parts: [{ text: typeof m.content === 'string' ? m.content : '' }],
+ parts: [{ text: contentToString(m.content) }],
};
})
.filter((entry): entry is { role: 'user' | 'model'; parts: GeminiPart[] } => entry !== null);
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index d60a86ed..3c1c9735 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -6,6 +6,7 @@ import type { ChatMessage } from '@freellmapi/shared/types.js';
import { routeRequest, recordRateLimitHit, recordSuccess, type RouteResult } from '../services/router.js';
import { recordRequest, recordTokens, setCooldown } from '../services/ratelimit.js';
import { getDb, getUnifiedApiKey } from '../db/index.js';
+import { contentToString } from '../lib/content.js';
export const proxyRouter = Router();
@@ -120,25 +121,39 @@ const toolCallSchema = z.object({
thought_signature: z.string().optional(),
});
+// OpenAI multimodal envelope. Clients like opencode / continue.dev send
+// content as an array of typed blocks even when only text is present. We
+// accept the envelope on the wire and flatten to string for providers that
+// don't support arrays (Cohere, Cloudflare). Non-text blocks pass z validation
+// but get dropped by contentToString — vision/audio still isn't supported.
+const contentBlockSchema = z.object({ type: z.string() }).passthrough();
+const contentSchema = z.union([z.string(), z.array(contentBlockSchema)]);
+
+function hasNonEmptyContent(content: unknown): boolean {
+ if (typeof content === 'string') return content.length > 0;
+ if (Array.isArray(content)) return content.length > 0;
+ return false;
+}
+
const systemMessageSchema = z.object({
role: z.literal('system'),
- content: z.string(),
+ content: contentSchema,
name: z.string().optional(),
});
const userMessageSchema = z.object({
role: z.literal('user'),
- content: z.string(),
+ content: contentSchema,
name: z.string().optional(),
});
const assistantMessageSchema = z.object({
role: z.literal('assistant'),
- content: z.string().nullable().optional(),
+ content: z.union([contentSchema, z.null()]).optional(),
name: z.string().optional(),
tool_calls: z.array(toolCallSchema).optional(),
}).refine((msg) => {
- const hasContent = typeof msg.content === 'string' && msg.content.length > 0;
+ const hasContent = hasNonEmptyContent(msg.content);
const hasToolCalls = (msg.tool_calls?.length ?? 0) > 0;
return hasContent || hasToolCalls;
}, {
@@ -147,7 +162,7 @@ const assistantMessageSchema = z.object({
const toolMessageSchema = z.object({
role: z.literal('tool'),
- content: z.string(),
+ content: contentSchema,
tool_call_id: z.string().min(1),
name: z.string().optional(),
});
@@ -269,8 +284,8 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
// (see line ~340). Streaming will drift from real consumption — accepted
// tradeoff because per-request usage isn't always returned mid-stream.
const estimatedInputTokens = messages.reduce((sum, m) => {
- if (typeof m.content !== 'string') return sum;
- return sum + Math.ceil(m.content.length / 4);
+ const text = contentToString(m.content);
+ return sum + Math.ceil(text.length / 4);
}, 0);
const estimatedTotal = estimatedInputTokens + (max_tokens ?? 1000);
diff --git a/server/src/scripts/test-all-models.ts b/server/src/scripts/test-all-models.ts
index 463cc09d..b8fc932d 100644
--- a/server/src/scripts/test-all-models.ts
+++ b/server/src/scripts/test-all-models.ts
@@ -46,7 +46,8 @@ for (const row of models) {
const start = Date.now();
try {
const res = await provider.chatCompletion(apiKey, [{ role: 'user', content: 'hi' }], row.model_id, { max_tokens: 5 });
- const reply = res.choices?.[0]?.message?.content?.slice(0, 40) ?? '';
+ const raw = res.choices?.[0]?.message?.content;
+ const reply = typeof raw === 'string' ? raw.slice(0, 40) : '';
results.push({ row, ok: true, ms: Date.now() - start, reply });
} catch (err: any) {
results.push({ row, ok: false, ms: Date.now() - start, error: String(err?.message ?? err).slice(0, 200) });
diff --git a/shared/types.ts b/shared/types.ts
index 3f7c3ee6..065b07fc 100644
--- a/shared/types.ts
+++ b/shared/types.ts
@@ -106,9 +106,16 @@ export type ChatToolChoice =
};
};
+// OpenAI's multimodal envelope: clients like opencode / continue.dev send
+// content as an array of typed blocks even for text-only messages. We accept
+// it on the wire and flatten to string for providers that don't support it
+// (Cohere, Cloudflare). See server/src/lib/content.ts.
+export type ChatContentBlock = { type: string; text?: string; [key: string]: unknown };
+export type ChatContent = string | null | ChatContentBlock[];
+
export interface ChatMessage {
role: 'system' | 'user' | 'assistant' | 'tool';
- content: string | null;
+ content: ChatContent;
name?: string;
tool_call_id?: string;
tool_calls?: ChatToolCall[];