7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(proxy): accept OpenAI multimodal array content envelope (#75, closes #69, credits @jhash)
Tashfeenf2ece370bbfe
diff --git a/README.md b/README.mdindex 53bd8c28..cbb58630 100644--- a/README.md+++ b/README.md@@ -299,6 +299,7 @@ PRs should include a test, keep the existing test suite green, and match the `.e<a href="https://github.com/nordbyte"><img src="https://images.weserv.nl/?url=github.com/nordbyte.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@nordbyte" /></a><a href="https://github.com/mybropro"><img src="https://images.weserv.nl/?url=github.com/mybropro.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@mybropro" /></a><a href="https://github.com/danscMax"><img src="https://images.weserv.nl/?url=github.com/danscMax.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@danscMax" /></a>+<a href="https://github.com/jhash"><img src="https://images.weserv.nl/?url=github.com/jhash.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@jhash" /></a>## Terms of Service reviewdiff --git a/server/src/__tests__/lib/content.test.ts b/server/src/__tests__/lib/content.test.tsnew file mode 100644index 00000000..0840d338--- /dev/null+++ b/server/src/__tests__/lib/content.test.ts@@ -0,0 +1,63 @@+import { describe, it, expect } from 'vitest';+import { contentToString, flattenMessageContent } from '../../lib/content.js';++describe('contentToString', () => {+ it('passes strings through', () => {+ expect(contentToString('hello')).toBe('hello');+ expect(contentToString('')).toBe('');+ });++ it('treats null and undefined as empty string', () => {+ expect(contentToString(null)).toBe('');+ expect(contentToString(undefined)).toBe('');+ });++ it('joins text blocks in OpenAI multimodal array envelope', () => {+ expect(contentToString([+ { type: 'text', text: 'hello ' },+ { type: 'text', text: 'world' },+ ])).toBe('hello world');+ });++ it('drops non-text blocks silently (image_url etc.) — vision unsupported', () => {+ expect(contentToString([+ { type: 'text', text: 'describe ' },+ { type: 'image_url', image_url: { url: 'https://example.com/x.png' } },+ { type: 'text', text: 'this' },+ ])).toBe('describe this');+ });++ it('handles an array of bare strings (some clients send this)', () => {+ expect(contentToString(['foo', 'bar'])).toBe('foobar');+ });++ it('returns empty string for unrecognized types instead of throwing', () => {+ expect(contentToString(42 as unknown)).toBe('');+ expect(contentToString({ unknown: true } as unknown)).toBe('');+ });+});++describe('flattenMessageContent', () => {+ it('converts every message content to a string', () => {+ const out = flattenMessageContent([+ { role: 'user', content: 'plain' },+ { role: 'user', content: [{ type: 'text', text: 'array' }] },+ { role: 'assistant', content: null, tool_calls: [{ id: 'x', type: 'function', function: { name: 'f', arguments: '{}' } }] },+ ]);+ expect(out[0].content).toBe('plain');+ expect(out[1].content).toBe('array');+ expect(out[2].content).toBe('');+ });++ it('preserves other message fields (tool_calls, name, tool_call_id)', () => {+ const out = flattenMessageContent([+ { role: 'tool', content: 'result', tool_call_id: 'call-1', name: 'fn' },+ ]);+ expect(out[0]).toMatchObject({+ role: 'tool',+ content: 'result',+ tool_call_id: 'call-1',+ name: 'fn',+ });+ });+});diff --git a/server/src/__tests__/routes/proxy-array-content.test.ts b/server/src/__tests__/routes/proxy-array-content.test.tsnew file mode 100644index 00000000..c9605769--- /dev/null+++ b/server/src/__tests__/routes/proxy-array-content.test.ts@@ -0,0 +1,144 @@+import { describe, it, expect, beforeAll, beforeEach, afterEach, vi } from 'vitest';+import type { Express } from 'express';+import { createApp } from '../../app.js';+import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';++async function request(app: Express, method: string, path: string, body?: any, headers: Record<string, string> = {}) {+ const server = app.listen(0);+ const addr = server.address() as any;+ const url = `http://127.0.0.1:${addr.port}${path}`;++ const res = await fetch(url, {+ method,+ headers: { ...(body ? { 'Content-Type': 'application/json' } : {}), ...headers },+ body: body ? JSON.stringify(body) : undefined,+ });++ const data = await res.text();+ server.close();++ let json: any = null;+ try { json = JSON.parse(data); } catch {}++ return { status: res.status, body: json };+}++function authHeaders() {+ return { Authorization: `Bearer ${getUnifiedApiKey()}` };+}++describe('OpenAI multimodal array content', () => {+ let app: Express;++ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ app = createApp();+ });++ beforeEach(async () => {+ const db = getDb();+ db.prepare('DELETE FROM api_keys').run();+ db.prepare('DELETE FROM requests').run();++ const addKey = await request(app, 'POST', '/api/keys', {+ platform: 'groq',+ key: 'gsk_array_content_test',+ label: 'array-content',+ });+ expect(addKey.status).toBe(201);+ });++ afterEach(() => {+ vi.restoreAllMocks();+ });++ it('accepts content as a string (the legacy shape)', async () => {+ // Provider call will fail (no real key), but schema validation must pass —+ // we assert it isn't rejected with a 400 zod error.+ const { status, body } = await request(app, 'POST', '/v1/chat/completions', {+ messages: [{ role: 'user', content: 'hello' }],+ }, authHeaders());+ expect(status).not.toBe(400);+ if (status === 400) {+ // Diagnostic if regression: show the validation error.+ throw new Error(`unexpected 400: ${JSON.stringify(body)}`);+ }+ });++ it('accepts content as a text-only multimodal array', async () => {+ const { status, body } = await request(app, 'POST', '/v1/chat/completions', {+ messages: [{+ role: 'user',+ content: [{ type: 'text', text: 'hello from opencode-style client' }],+ }],+ }, authHeaders());+ expect(status).not.toBe(400);+ if (status === 400) {+ throw new Error(`unexpected 400: ${JSON.stringify(body)}`);+ }+ });++ it('accepts mixed text + image_url blocks (image blocks are silently dropped)', async () => {+ const { status } = await request(app, 'POST', '/v1/chat/completions', {+ messages: [{+ role: 'user',+ content: [+ { type: 'text', text: 'describe' },+ { type: 'image_url', image_url: { url: 'https://example.com/x.png' } },+ ],+ }],+ }, authHeaders());+ expect(status).not.toBe(400);+ });++ it('successfully routes an array-content request and gets a 200 (mocked groq)', async () => {+ const origFetch = global.fetch;+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {+ const urlStr = typeof url === 'string' ? url : url.toString();+ if (urlStr.includes('api.groq.com/openai/v1/chat/completions')) {+ // Sanity check that the array shape made it through to the upstream call.+ const body = JSON.parse(String((init as RequestInit).body));+ expect(Array.isArray(body.messages[0].content)).toBe(true);+ return {+ ok: true,+ json: () => Promise.resolve({+ id: 'chatcmpl-array', object: 'chat.completion', created: 1, model: 'openai/gpt-oss-120b',+ choices: [{+ index: 0,+ message: { role: 'assistant', content: 'got it' },+ finish_reason: 'stop',+ }],+ usage: { prompt_tokens: 4, completion_tokens: 2, total_tokens: 6 },+ }),+ } as any;+ }+ return origFetch(url, init);+ });++ const { status, body } = await request(app, 'POST', '/v1/chat/completions', {+ model: 'auto',+ messages: [{+ role: 'user',+ content: [{ type: 'text', text: 'hi' }],+ }],+ }, authHeaders());++ expect(status).toBe(200);+ expect(body.choices[0].message.content).toBe('got it');+ });++ it('rejects an empty array as missing content', async () => {+ const { status } = await request(app, 'POST', '/v1/chat/completions', {+ messages: [], // top-level empty messages+ }, authHeaders());+ expect(status).toBe(400);+ });++ it('rejects an assistant message with neither content nor tool_calls', async () => {+ const { status } = await request(app, 'POST', '/v1/chat/completions', {+ messages: [{ role: 'assistant', content: [] }],+ }, authHeaders());+ expect(status).toBe(400);+ });+});diff --git a/server/src/lib/content.ts b/server/src/lib/content.tsnew file mode 100644index 00000000..e0a481f8--- /dev/null+++ b/server/src/lib/content.ts@@ -0,0 +1,30 @@+import type { ChatMessage } from '@freellmapi/shared/types.js';++// OpenAI-spec message content can be one of:+// - string (plain text)+// - null (assistant with tool_calls only)+// - Array<ContentBlock> (multimodal envelope; we extract text only)+//+// freellmapi accepts the array envelope so clients like opencode and+// continue.dev (which always serialize as arrays) don't 400. Non-text blocks+// are dropped silently — vision/audio aren't supported (see README).+export type ContentTextBlock = { type: 'text'; text: string };+export type ContentBlock = ContentTextBlock | { type: string; [key: string]: unknown };++export function contentToString(content: unknown): string {+ if (typeof content === 'string') return content;+ if (content == null) return '';+ if (Array.isArray(content)) {+ return content+ .map((b) => (typeof b === 'string' ? b : (b as ContentTextBlock)?.type === 'text' ? (b as ContentTextBlock).text : ''))+ .join('');+ }+ return '';+}++export function flattenMessageContent(messages: ChatMessage[]): ChatMessage[] {+ return messages.map((m) => ({+ ...m,+ content: contentToString(m.content),+ }));+}diff --git a/server/src/providers/cloudflare.ts b/server/src/providers/cloudflare.tsindex fb3f39d5..ff748cdf 100644--- a/server/src/providers/cloudflare.ts+++ b/server/src/providers/cloudflare.ts@@ -4,6 +4,7 @@ import type {ChatCompletionChunk,} from '@freellmapi/shared/types.js';import { BaseProvider, type CompletionOptions } from './base.js';+import { contentToString } from '../lib/content.js';/*** Cloudflare Workers AI provider.@@ -20,12 +21,12 @@ export class CloudflareProvider extends BaseProvider {return { accountId: apiKey.slice(0, sep), token: apiKey.slice(sep + 1) };}- // Cloudflare's OpenAI-compat endpoint rejects `content: null` on assistant- // messages that carry tool_calls, even though the OpenAI spec allows it.+ // Cloudflare's OpenAI-compat endpoint:+ // - rejects `content: null` on assistant messages that carry tool_calls,+ // even though the OpenAI spec allows it (collapse to '');+ // - doesn't accept the array content envelope, so flatten to string.private normalizeMessages(messages: ChatMessage[]): ChatMessage[] {- return messages.map(m =>- m.content === null ? { ...m, content: '' } : m,- );+ return messages.map(m => ({ ...m, content: contentToString(m.content) }));}async chatCompletion(diff --git a/server/src/providers/cohere.ts b/server/src/providers/cohere.tsindex d75ae178..8ef0a2c2 100644--- a/server/src/providers/cohere.ts+++ b/server/src/providers/cohere.ts@@ -4,6 +4,7 @@ import type {ChatCompletionChunk,} from '@freellmapi/shared/types.js';import { BaseProvider, type CompletionOptions } from './base.js';+import { flattenMessageContent } from '../lib/content.js';const API_BASE = 'https://api.cohere.ai/compatibility/v1';@@ -19,7 +20,7 @@ export class CohereProvider extends BaseProvider {): Promise<ChatCompletionResponse> {const body: Record<string, unknown> = {model: modelId,- messages,+ messages: flattenMessageContent(messages),temperature: options?.temperature,max_tokens: options?.max_tokens,top_p: options?.top_p,@@ -54,7 +55,7 @@ export class CohereProvider extends BaseProvider {): AsyncGenerator<ChatCompletionChunk> {const body: Record<string, unknown> = {model: modelId,- messages,+ messages: flattenMessageContent(messages),temperature: options?.temperature,max_tokens: options?.max_tokens,top_p: options?.top_p,diff --git a/server/src/providers/google.ts b/server/src/providers/google.tsindex beb6dfa0..94535948 100644--- a/server/src/providers/google.ts+++ b/server/src/providers/google.ts@@ -8,6 +8,7 @@ import type {TokenUsage,} from '@freellmapi/shared/types.js';import { BaseProvider, type CompletionOptions } from './base.js';+import { contentToString } from '../lib/content.js';const API_BASE = 'https://generativelanguage.googleapis.com/v1beta';@@ -128,11 +129,14 @@ function toGeminiToolConfig(toolChoice?: ChatToolChoice): { functionCallingConfi};}-// Translate OpenAI messages to Gemini format+// Translate OpenAI messages to Gemini format. Content may arrive as a string,+// null, or the OpenAI multimodal array envelope — flatten to string first so+// system/user/tool messages all surface as `parts: [{ text }]` for Gemini.function toGeminiContents(messages: ChatMessage[]) {const systemMessages = messages- .filter(m => m.role === 'system' && typeof m.content === 'string' && m.content.length > 0)- .map(m => m.content as string);+ .filter(m => m.role === 'system')+ .map(m => contentToString(m.content))+ .filter(s => s.length > 0);const toolNameByCallId = new Map<string, string>();for (const m of messages) {@@ -147,8 +151,9 @@ function toGeminiContents(messages: ChatMessage[]) {if (m.role === 'assistant') {const parts: GeminiPart[] = [];- if (typeof m.content === 'string' && m.content.length > 0) {- parts.push({ text: m.content });+ const assistantText = contentToString(m.content);+ if (assistantText.length > 0) {+ parts.push({ text: assistantText });}for (const call of m.tool_calls ?? []) {@@ -174,7 +179,7 @@ function toGeminiContents(messages: ChatMessage[]) {if (!toolCallId) return null;const toolName = m.name ?? toolNameByCallId.get(toolCallId) ?? 'tool';- const response = safeParseObject(typeof m.content === 'string' ? m.content : '');+ const response = safeParseObject(contentToString(m.content));return {role: 'user',@@ -190,7 +195,7 @@ function toGeminiContents(messages: ChatMessage[]) {return {role: 'user',- parts: [{ text: typeof m.content === 'string' ? m.content : '' }],+ parts: [{ text: contentToString(m.content) }],};}).filter((entry): entry is { role: 'user' | 'model'; parts: GeminiPart[] } => entry !== null);diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex d60a86ed..3c1c9735 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -6,6 +6,7 @@ import type { ChatMessage } from '@freellmapi/shared/types.js';import { routeRequest, recordRateLimitHit, recordSuccess, type RouteResult } from '../services/router.js';import { recordRequest, recordTokens, setCooldown } from '../services/ratelimit.js';import { getDb, getUnifiedApiKey } from '../db/index.js';+import { contentToString } from '../lib/content.js';export const proxyRouter = Router();@@ -120,25 +121,39 @@ const toolCallSchema = z.object({thought_signature: z.string().optional(),});+// OpenAI multimodal envelope. Clients like opencode / continue.dev send+// content as an array of typed blocks even when only text is present. We+// accept the envelope on the wire and flatten to string for providers that+// don't support arrays (Cohere, Cloudflare). Non-text blocks pass z validation+// but get dropped by contentToString — vision/audio still isn't supported.+const contentBlockSchema = z.object({ type: z.string() }).passthrough();+const contentSchema = z.union([z.string(), z.array(contentBlockSchema)]);++function hasNonEmptyContent(content: unknown): boolean {+ if (typeof content === 'string') return content.length > 0;+ if (Array.isArray(content)) return content.length > 0;+ return false;+}+const systemMessageSchema = z.object({role: z.literal('system'),- content: z.string(),+ content: contentSchema,name: z.string().optional(),});const userMessageSchema = z.object({role: z.literal('user'),- content: z.string(),+ content: contentSchema,name: z.string().optional(),});const assistantMessageSchema = z.object({role: z.literal('assistant'),- content: z.string().nullable().optional(),+ content: z.union([contentSchema, z.null()]).optional(),name: z.string().optional(),tool_calls: z.array(toolCallSchema).optional(),}).refine((msg) => {- const hasContent = typeof msg.content === 'string' && msg.content.length > 0;+ const hasContent = hasNonEmptyContent(msg.content);const hasToolCalls = (msg.tool_calls?.length ?? 0) > 0;return hasContent || hasToolCalls;}, {@@ -147,7 +162,7 @@ const assistantMessageSchema = z.object({const toolMessageSchema = z.object({role: z.literal('tool'),- content: z.string(),+ content: contentSchema,tool_call_id: z.string().min(1),name: z.string().optional(),});@@ -269,8 +284,8 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {// (see line ~340). Streaming will drift from real consumption — accepted// tradeoff because per-request usage isn't always returned mid-stream.const estimatedInputTokens = messages.reduce((sum, m) => {- if (typeof m.content !== 'string') return sum;- return sum + Math.ceil(m.content.length / 4);+ const text = contentToString(m.content);+ return sum + Math.ceil(text.length / 4);}, 0);const estimatedTotal = estimatedInputTokens + (max_tokens ?? 1000);diff --git a/server/src/scripts/test-all-models.ts b/server/src/scripts/test-all-models.tsindex 463cc09d..b8fc932d 100644--- a/server/src/scripts/test-all-models.ts+++ b/server/src/scripts/test-all-models.ts@@ -46,7 +46,8 @@ for (const row of models) {const start = Date.now();try {const res = await provider.chatCompletion(apiKey, [{ role: 'user', content: 'hi' }], row.model_id, { max_tokens: 5 });- const reply = res.choices?.[0]?.message?.content?.slice(0, 40) ?? '';+ const raw = res.choices?.[0]?.message?.content;+ const reply = typeof raw === 'string' ? raw.slice(0, 40) : '';results.push({ row, ok: true, ms: Date.now() - start, reply });} catch (err: any) {results.push({ row, ok: false, ms: Date.now() - start, error: String(err?.message ?? err).slice(0, 200) });diff --git a/shared/types.ts b/shared/types.tsindex 3f7c3ee6..065b07fc 100644--- a/shared/types.ts+++ b/shared/types.ts@@ -106,9 +106,16 @@ export type ChatToolChoice =};};+// OpenAI's multimodal envelope: clients like opencode / continue.dev send+// content as an array of typed blocks even for text-only messages. We accept+// it on the wire and flatten to string for providers that don't support it+// (Cohere, Cloudflare). See server/src/lib/content.ts.+export type ChatContentBlock = { type: string; text?: string; [key: string]: unknown };+export type ChatContent = string | null | ChatContentBlock[];+export interface ChatMessage {role: 'system' | 'user' | 'assistant' | 'tool';- content: string | null;+ content: ChatContent;name?: string;tool_call_id?: string;tool_calls?: ChatToolCall[];