7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(vision): image input support with vision-aware routing (#118, #125)
Tashfeen4b2e6f9bb59b
diff --git a/README.md b/README.mdindex cac8df02..fe7a6742 100644--- a/README.md+++ b/README.md@@ -94,7 +94,6 @@ The scope is deliberately narrow. If a feature isn't on this list and isn't belo- **Embeddings** (`/v1/embeddings`)- **Image generation** (`/v1/images/*`)- **Audio / speech** (`/v1/audio/*`)-- **Vision / multimodal inputs** — message content is text-only- **Legacy completions** (`/v1/completions`) — only the chat endpoint is implemented- **Moderation** (`/v1/moderations`)- **`n > 1`** (multiple completions per request)@@ -265,6 +264,26 @@ final = client.chat.completions.create(print(final.choices[0].message.content)```+**Vision / image input**++Send images with the standard OpenAI `image_url` content blocks (base64 `data:` URLs or `http(s)` URLs). When a request contains an image, the router restricts itself to **vision-capable models** and ignores text-only ones. Vision models are tagged with a **Vision** badge on the Fallback Chain page; the current set includes Gemini (2.5 / 3.x), Llama 4 Scout/Maverick (Groq, NVIDIA, SambaNova), and GitHub's GPT-4o / GPT-4.1.++```python+resp = client.chat.completions.create(+ model="auto", # auto-routes to a vision model+ messages=[{+ "role": "user",+ "content": [+ {"type": "text", "text": "What's in this image?"},+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,<...>"}},+ ],+ }],+)+print(resp.choices[0].message.content)+```++If no vision-capable model is enabled in your Fallback Chain, an image request returns a clear `422` (`code: "no_vision_model"`) rather than silently dropping the image. (Image input on `/v1/responses` isn't supported yet — use `/v1/chat/completions`.)+Works with `stream=True` as well — you'll get `delta.tool_calls` chunks followed by a `finish_reason: "tool_calls"` close. Under the hood, OpenAI-compatible providers (Groq, Cerebras, SambaNova, Mistral, OpenRouter, GitHub Models, HuggingFace, Cloudflare, Cohere compat) get the request passed through; Gemini requests get translated into Google's `functionDeclarations` / `functionResponse` shape and the response is translated back.Every response carries an `X-Routed-Via: <platform>/<model>` header so you can see which provider actually served each call. If a request fell over between providers, you'll also see `X-Fallback-Attempts: N`.diff --git a/client/src/pages/FallbackPage.tsx b/client/src/pages/FallbackPage.tsxindex f18d42b4..d7512e01 100644--- a/client/src/pages/FallbackPage.tsx+++ b/client/src/pages/FallbackPage.tsx@@ -38,6 +38,7 @@ interface FallbackEntry {rpmLimit: number | nullrpdLimit: number | nullmonthlyTokenBudget: string+ supportsVision: booleankeyCount: number}@@ -176,6 +177,14 @@ function SortableModelRow({<div className="flex items-center gap-2 flex-wrap"><span className="font-medium text-sm">{entry.displayName}</span><span className="text-xs text-muted-foreground">{entry.platform}</span>+ {entry.supportsVision && (+ <span+ title="Accepts image input"+ className="text-xs rounded-full px-2 py-0.5 bg-cyan-600/15 text-cyan-700 dark:bg-cyan-400/15 dark:text-cyan-400"+ >+ Vision+ </span>+ )}{entry.penalty > 0 && (<span className="text-xs text-amber-600 dark:text-amber-400">−{entry.penalty} penaltydiff --git a/server/package.json b/server/package.jsonindex a2e4e7c5..fc580996 100644--- a/server/package.json+++ b/server/package.json@@ -7,7 +7,7 @@"dev": "tsx watch src/index.ts","build": "tsc","start": "node dist/index.js",- "test": "vitest run",+ "test": "vitest run --pool=forks --poolOptions.forks.singleFork=true --no-file-parallelism","test:watch": "vitest"},"dependencies": {diff --git a/server/src/__tests__/lib/content.test.ts b/server/src/__tests__/lib/content.test.tsindex 0840d338..f6a0648a 100644--- a/server/src/__tests__/lib/content.test.ts+++ b/server/src/__tests__/lib/content.test.ts@@ -1,5 +1,5 @@import { describe, it, expect } from 'vitest';-import { contentToString, flattenMessageContent } from '../../lib/content.js';+import { contentToString, flattenMessageContent, messageHasImage } from '../../lib/content.js';describe('contentToString', () => {it('passes strings through', () => {@@ -19,7 +19,7 @@ describe('contentToString', () => {])).toBe('hello world');});- it('drops non-text blocks silently (image_url etc.) — vision unsupported', () => {+ it('drops non-text blocks (image_url etc.) — text-only providers flatten this way', () => {expect(contentToString([{ type: 'text', text: 'describe ' },{ type: 'image_url', image_url: { url: 'https://example.com/x.png' } },@@ -61,3 +61,28 @@ describe('flattenMessageContent', () => {});});});++describe('messageHasImage', () => {+ it('detects image_url blocks (OpenAI vision envelope)', () => {+ expect(messageHasImage([+ { role: 'user', content: [+ { type: 'text', text: 'what is this?' },+ { type: 'image_url', image_url: { url: 'data:image/png;base64,AAAA' } },+ ] },+ ])).toBe(true);+ });++ it('detects a bare image block type', () => {+ expect(messageHasImage([+ { role: 'user', content: [{ type: 'image', source: 'x' } as any] },+ ])).toBe(true);+ });++ it('is false for string content and text-only arrays', () => {+ expect(messageHasImage([{ role: 'user', content: 'hello' }])).toBe(false);+ expect(messageHasImage([+ { role: 'user', content: [{ type: 'text', text: 'hello' }] },+ ])).toBe(false);+ expect(messageHasImage([{ role: 'assistant', content: null }])).toBe(false);+ });+});diff --git a/server/src/__tests__/providers/google.test.ts b/server/src/__tests__/providers/google.test.tsindex 31984dc4..caa6aed3 100644--- a/server/src/__tests__/providers/google.test.ts+++ b/server/src/__tests__/providers/google.test.ts@@ -45,6 +45,28 @@ describe('GoogleProvider', () => {expect(result._routed_via?.platform).toBe('google');});+ it('converts an image_url data URL into a Gemini inlineData part (#118)', async () => {+ const fetchSpy = vi.spyOn(global, 'fetch').mockResolvedValueOnce({+ ok: true,+ json: () => Promise.resolve({+ candidates: [{ content: { parts: [{ text: 'a cat' }] }, finishReason: 'STOP' }],+ usageMetadata: { promptTokenCount: 1, candidatesTokenCount: 1, totalTokenCount: 2 },+ }),+ } as any);++ await provider.chatCompletion('test-key', [+ { role: 'user', content: [+ { type: 'text', text: 'what is this?' },+ { type: 'image_url', image_url: { url: 'data:image/png;base64,iVBORw0KGgo=' } },+ ] as any },+ ], 'gemini-2.5-flash');++ const body = JSON.parse((fetchSpy.mock.calls[0][1] as any).body);+ const parts = body.contents[0].parts;+ expect(parts).toContainEqual({ text: 'what is this?' });+ expect(parts).toContainEqual({ inlineData: { mimeType: 'image/png', data: 'iVBORw0KGgo=' } });+ });+it('should throw on API error', async () => {vi.spyOn(global, 'fetch').mockResolvedValueOnce({ok: false,diff --git a/server/src/__tests__/routes/proxy-vision.test.ts b/server/src/__tests__/routes/proxy-vision.test.tsnew file mode 100644index 00000000..c04024ea--- /dev/null+++ b/server/src/__tests__/routes/proxy-vision.test.ts@@ -0,0 +1,87 @@+import { describe, it, expect, beforeAll } from 'vitest';+import type { Express } from 'express';+import { createApp } from '../../app.js';+import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';++async function post(app: Express, path: string, body: any, key: string) {+ const server = app.listen(0);+ const addr = server.address() as any;+ const res = await fetch(`http://127.0.0.1:${addr.port}${path}`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` },+ body: JSON.stringify(body),+ });+ const text = await res.text();+ server.close();+ let json: any = null;+ try { json = JSON.parse(text); } catch {}+ return { status: res.status, body: json };+}++const IMAGE_MESSAGE = {+ messages: [{+ role: 'user',+ content: [+ { type: 'text', text: 'what is in this image?' },+ { type: 'image_url', image_url: { url: 'data:image/png;base64,iVBORw0KGgo=' } },+ ],+ }],+};++describe('Vision-aware routing (#118, #125)', () => {+ let app: Express;+ let key: string;++ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ app = createApp();+ key = getUnifiedApiKey();+ });++ it('seeds supports_vision: true for vision models, false for text-only', () => {+ const db = getDb();+ const vision = db.prepare("SELECT supports_vision FROM models WHERE model_id = 'gemini-2.5-flash'").get() as { supports_vision: number };+ expect(vision.supports_vision).toBe(1);++ // The known vision set is flagged; plenty of text-only models remain at 0.+ const visionCount = (db.prepare('SELECT COUNT(*) c FROM models WHERE supports_vision = 1').get() as { c: number }).c;+ const textCount = (db.prepare('SELECT COUNT(*) c FROM models WHERE supports_vision = 0').get() as { c: number }).c;+ expect(visionCount).toBeGreaterThanOrEqual(4);+ expect(textCount).toBeGreaterThan(0);+ });++ it('lets an image request through routing when a vision model is enabled (no 422)', async () => {+ // Seed has Gemini/Llama-4 vision models enabled but no provider keys, so+ // routing exhausts → 429. The point: it is NOT the 422 "no vision model"+ // error, proving the precheck passed and routing was attempted.+ const { status, body } = await post(app, '/v1/chat/completions', IMAGE_MESSAGE, key);+ expect(status).not.toBe(422);+ expect(body?.error?.code).not.toBe('no_vision_model');+ });++ it('rejects an image request with a clear 422 when no vision model is enabled', async () => {+ // Disable every vision-capable model in the chain.+ getDb().prepare('UPDATE models SET enabled = 0 WHERE supports_vision = 1').run();++ const { status, body } = await post(app, '/v1/chat/completions', IMAGE_MESSAGE, key);+ expect(status).toBe(422);+ expect(body.error.code).toBe('no_vision_model');+ expect(body.error.type).toBe('invalid_request_error');++ // Restore so we don't leak state to other expectations.+ getDb().prepare('UPDATE models SET enabled = 1 WHERE supports_vision = 1').run();+ });++ it('does not apply the vision gate to a text-only request', async () => {+ // Disable all vision models; a plain text request must still route normally+ // (it 429s on exhaustion here, but never the 422 vision error).+ getDb().prepare('UPDATE models SET enabled = 0 WHERE supports_vision = 1').run();+ const { status, body } = await post(app, '/v1/chat/completions', {+ messages: [{ role: 'user', content: 'hello' }],+ }, key);+ expect(status).not.toBe(422);+ expect(body?.error?.code).not.toBe('no_vision_model');+ getDb().prepare('UPDATE models SET enabled = 1 WHERE supports_vision = 1').run();+ });+});diff --git a/server/src/__tests__/routes/responses.test.ts b/server/src/__tests__/routes/responses.test.tsindex d3f8ae5a..c37a2589 100644--- a/server/src/__tests__/routes/responses.test.ts+++ b/server/src/__tests__/routes/responses.test.ts@@ -49,6 +49,22 @@ describe('POST /v1/responses (#96)', () => {expect((await post(app, '/v1/responses', { model: 'auto' }, key)).status).toBe(400);});+ // #118: image input isn't carried through the Responses translation yet, so+ // it must hard-fail clearly rather than silently answer blind to the image.+ it('rejects image input with a clear 422 pointing at /v1/chat/completions', async () => {+ const { status, text } = await post(app, '/v1/responses', {+ input: [{+ role: 'user',+ content: [+ { type: 'input_text', text: 'what is this?' },+ { type: 'input_image', image_url: 'data:image/png;base64,iVBORw0KGgo=' },+ ],+ }],+ }, key);+ expect(status).toBe(422);+ expect(JSON.parse(text).error.code).toBe('no_vision_model');+ });+// #103: the x-api-key header (Anthropic wire format) must authenticate here// too, not just on /v1/chat/completions.it('accepts the unified key via the x-api-key header', async () => {diff --git a/server/src/db/index.ts b/server/src/db/index.tsindex 8d4273df..a7469bea 100644--- a/server/src/db/index.ts+++ b/server/src/db/index.ts@@ -50,6 +50,7 @@ export function initDb(dbPath?: string): Database.Database {migrateModelsV13(db);migrateModelsV14(db);migrateModelsV15(db);+ migrateModelsV16Vision(db);ensureUnifiedKey(db);console.log(`Database initialized at ${resolvedPath}`);@@ -73,6 +74,7 @@ function createTables(db: Database.Database) {monthly_token_budget TEXT NOT NULL DEFAULT '',context_window INTEGER,enabled INTEGER NOT NULL DEFAULT 1,+ supports_vision INTEGER NOT NULL DEFAULT 0,UNIQUE(platform, model_id));@@ -1311,6 +1313,46 @@ function migrateModelsV15(db: Database.Database) {db.prepare(`DELETE FROM models WHERE platform = 'siliconflow'`).run();}+// Adds the supports_vision column to existing DBs and (re)applies the vision+// flags by rule. Rule-based rather than a hardcoded id list because the catalog+// churns through migrations (model ids get renamed/replaced) — rules survive+// that. Two constraints decide a model can accept images:+// 1. The model is genuinely multimodal (every Gemini is; Llama 4 Scout/+// Maverick are; GitHub's GPT-4o/4.1/5 are).+// 2. Its provider adapter forwards image content. OpenAICompat and Google do;+// Cohere and Cloudflare flatten content to text, so models on those+// platforms are excluded even when the underlying model can see.+// Conservative on purpose: with hard-fail routing a false negative is just a+// clear "no vision model" error, while a false positive routes an image to a+// model that chokes. Idempotent — safe on fresh seeds and upgrades alike.+function migrateModelsV16Vision(db: Database.Database) {+ const columns = db.prepare('PRAGMA table_info(models)').all() as { name: string }[];+ if (!columns.some(col => col.name === 'supports_vision')) {+ db.prepare('ALTER TABLE models ADD COLUMN supports_vision INTEGER NOT NULL DEFAULT 0').run();+ }+ const apply = db.transaction(() => {+ // Reset first so de-flagged models (e.g. an id that moved to Cloudflare)+ // don't keep a stale flag across re-runs.+ db.prepare('UPDATE models SET supports_vision = 0').run();+ // Every Gemini is multimodal (the 'google' platform is all Gemini).+ db.prepare("UPDATE models SET supports_vision = 1 WHERE platform = 'google'").run();+ // Llama 4 (Scout/Maverick) is natively multimodal — but only where the+ // adapter forwards images (exclude the text-flattening providers).+ db.prepare(`+ UPDATE models SET supports_vision = 1+ WHERE LOWER(model_id) LIKE '%llama-4%'+ AND platform NOT IN ('cloudflare', 'cohere')+ `).run();+ // GitHub's OpenAI vision models.+ db.prepare(`+ UPDATE models SET supports_vision = 1+ WHERE platform = 'github'+ AND (model_id LIKE '%gpt-4o%' OR model_id LIKE '%gpt-4.1%' OR model_id LIKE '%gpt-5%')+ `).run();+ });+ apply();+}+function ensureUnifiedKey(db: Database.Database) {const existing = db.prepare("SELECT value FROM settings WHERE key = 'unified_api_key'").get() as { value: string } | undefined;if (!existing) {diff --git a/server/src/lib/content.ts b/server/src/lib/content.tsindex e0a481f8..b01623ac 100644--- a/server/src/lib/content.ts+++ b/server/src/lib/content.ts@@ -28,3 +28,20 @@ export function flattenMessageContent(messages: ChatMessage[]): ChatMessage[] {content: contentToString(m.content),}));}++// True if the content array carries an image block. OpenAI's multimodal+// envelope uses `{ type: 'image_url', image_url: { url } }`; some clients send+// a bare `{ type: 'image', ... }`.+export function contentHasImage(content: unknown): boolean {+ if (!Array.isArray(content)) return false;+ return content.some((block) => {+ const type = (block as { type?: string })?.type;+ return type === 'image_url' || type === 'image';+ });+}++// True if any message carries an image content block. Used to route image+// requests only to vision-capable models (#118, #125).+export function messageHasImage(messages: ChatMessage[]): boolean {+ return messages.some((m) => contentHasImage(m.content));+}diff --git a/server/src/providers/google.ts b/server/src/providers/google.tsindex 434925d4..9ef1ff70 100644--- a/server/src/providers/google.ts+++ b/server/src/providers/google.ts@@ -14,6 +14,10 @@ const API_BASE = 'https://generativelanguage.googleapis.com/v1beta';interface GeminiPart {text?: string;+ inlineData?: {+ mimeType: string;+ data: string;+ };thoughtSignature?: string;functionCall?: {id?: string;@@ -130,10 +134,75 @@ function toGeminiToolConfig(toolChoice?: ChatToolChoice): { functionCallingConfi};}+const MAX_IMAGE_BYTES = 8 * 1024 * 1024; // 8 MB cap on fetched/inlined images++// Pull the URL out of an OpenAI image content block. Accepts both the object+// form `{ image_url: { url } }` and the shorthand `{ image_url: '...' }`.+function extractImageUrl(block: unknown): string | undefined {+ const iu = (block as { image_url?: unknown })?.image_url;+ if (typeof iu === 'string') return iu;+ if (iu && typeof (iu as { url?: unknown }).url === 'string') return (iu as { url: string }).url;+ return undefined;+}++// Convert an image URL to a Gemini inlineData part. Handles base64 `data:` URLs+// directly; for `http(s)` URLs we fetch and inline because the Gemini API does+// not fetch external URLs itself. Fetching a user-supplied URL is a minor SSRF+// surface, acceptable for a single-user self-hosted proxy; we still restrict to+// http/https and cap the size. Returns null (part skipped) on any failure.+async function imageUrlToInlineData(url: string): Promise<{ mimeType: string; data: string } | null> {+ const dataMatch = /^data:([^;,]+)?(;base64)?,(.*)$/s.exec(url);+ if (dataMatch) {+ const mimeType = dataMatch[1] || 'application/octet-stream';+ const isBase64 = Boolean(dataMatch[2]);+ const payload = dataMatch[3] ?? '';+ const data = isBase64+ ? payload+ : Buffer.from(decodeURIComponent(payload)).toString('base64');+ return { mimeType, data };+ }+ if (/^https?:\/\//i.test(url)) {+ try {+ const res = await fetch(url);+ if (!res.ok) return null;+ const buf = Buffer.from(await res.arrayBuffer());+ if (buf.length === 0 || buf.length > MAX_IMAGE_BYTES) return null;+ const mimeType = res.headers.get('content-type')?.split(';')[0]?.trim() || 'image/jpeg';+ return { mimeType, data: buf.toString('base64') };+ } catch {+ return null;+ }+ }+ return null;+}++// Build Gemini parts for a user message: joined text first, then any images as+// inlineData. Non-array content (string/null) collapses to a single text part.+async function userContentToParts(content: ChatMessage['content']): Promise<GeminiPart[]> {+ const parts: GeminiPart[] = [];+ const text = contentToString(content);+ if (text.length > 0) parts.push({ text });++ if (Array.isArray(content)) {+ for (const block of content) {+ const type = (block as { type?: string })?.type;+ if (type !== 'image_url' && type !== 'image') continue;+ const url = extractImageUrl(block);+ if (!url) continue;+ const inlineData = await imageUrlToInlineData(url);+ if (inlineData) parts.push({ inlineData });+ }+ }++ // Gemini rejects empty `parts`; keep at least one (possibly empty) text part.+ if (parts.length === 0) parts.push({ text: '' });+ return parts;+}+// Translate OpenAI messages to Gemini format. Content may arrive as a string,-// null, or the OpenAI multimodal array envelope — flatten to string first so-// system/user/tool messages all surface as `parts: [{ text }]` for Gemini.-function toGeminiContents(messages: ChatMessage[]) {+// null, or the OpenAI multimodal array envelope. System/assistant/tool messages+// flatten to text; user messages additionally carry images as inlineData parts.+async function toGeminiContents(messages: ChatMessage[]) {const systemMessages = messages.filter(m => m.role === 'system').map(m => contentToString(m.content))@@ -146,9 +215,9 @@ function toGeminiContents(messages: ChatMessage[]) {}}- const contents = messages+ const contents = (await Promise.all(messages.filter(m => m.role !== 'system')- .map((m): { role: 'user' | 'model'; parts: GeminiPart[] } | null => {+ .map(async (m): Promise<{ role: 'user' | 'model'; parts: GeminiPart[] } | null> => {if (m.role === 'assistant') {const parts: GeminiPart[] = [];@@ -196,9 +265,9 @@ function toGeminiContents(messages: ChatMessage[]) {return {role: 'user',- parts: [{ text: contentToString(m.content) }],+ parts: await userContentToParts(m.content),};- })+ }))).filter((entry): entry is { role: 'user' | 'model'; parts: GeminiPart[] } => entry !== null);return {@@ -250,7 +319,7 @@ export class GoogleProvider extends BaseProvider {modelId: string,options?: CompletionOptions,): Promise<ChatCompletionResponse> {- const { contents, systemInstruction } = toGeminiContents(messages);+ const { contents, systemInstruction } = await toGeminiContents(messages);const body: Record<string, unknown> = {contents,@@ -313,7 +382,7 @@ export class GoogleProvider extends BaseProvider {modelId: string,options?: CompletionOptions,): AsyncGenerator<ChatCompletionChunk> {- const { contents, systemInstruction } = toGeminiContents(messages);+ const { contents, systemInstruction } = await toGeminiContents(messages);const body: Record<string, unknown> = {contents,diff --git a/server/src/routes/fallback.ts b/server/src/routes/fallback.tsindex 7c4e6597..7cbd3b23 100644--- a/server/src/routes/fallback.ts+++ b/server/src/routes/fallback.ts@@ -13,7 +13,7 @@ fallbackRouter.get('/', (_req: Request, res: Response) => {SELECT fc.model_db_id, fc.priority, fc.enabled,m.platform, m.model_id, m.display_name, m.intelligence_rank,m.speed_rank, m.size_label, m.rpm_limit, m.rpd_limit,- m.monthly_token_budget+ m.monthly_token_budget, m.supports_visionFROM fallback_config fcJOIN models m ON m.id = fc.model_db_idORDER BY fc.priority ASC@@ -49,6 +49,7 @@ fallbackRouter.get('/', (_req: Request, res: Response) => {rpmLimit: r.rpm_limit,rpdLimit: r.rpd_limit,monthlyTokenBudget: r.monthly_token_budget,+ supportsVision: r.supports_vision === 1,keyCount: keyCountMap.get(r.platform) ?? 0,};}));diff --git a/server/src/routes/models.ts b/server/src/routes/models.tsindex 474a664d..9a381152 100644--- a/server/src/routes/models.ts+++ b/server/src/routes/models.ts@@ -40,6 +40,7 @@ modelsRouter.get('/', (_req: Request, res: Response) => {monthlyTokenBudget: m.monthly_token_budget,contextWindow: m.context_window,enabled: m.enabled === 1,+ supportsVision: m.supports_vision === 1,priority: m.priority,fallbackEnabled: m.fallback_enabled === 1,hasProvider: hasProvider(m.platform),diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex f8e93488..2cdc8068 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -3,10 +3,10 @@ import { Router } from 'express';import type { Request, Response } from 'express';import { z } from 'zod';import type { ChatMessage } from '@freellmapi/shared/types.js';-import { routeRequest, recordRateLimitHit, recordSuccess, type RouteResult } from '../services/router.js';+import { routeRequest, recordRateLimitHit, recordSuccess, hasEnabledVisionModel, type RouteResult } from '../services/router.js';import { recordRequest, recordTokens, setCooldown, getNextCooldownDuration } from '../services/ratelimit.js';import { getDb, getUnifiedApiKey } from '../db/index.js';-import { contentToString } from '../lib/content.js';+import { contentToString, messageHasImage } from '../lib/content.js';export const proxyRouter = Router();@@ -312,7 +312,27 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {const text = contentToString(m.content);return sum + Math.ceil(text.length / 4);}, 0);- const estimatedTotal = estimatedInputTokens + (max_tokens ?? 1000);++ // Image requests must route to a vision-capable model. Reject up front with a+ // clear message when none is enabled, rather than silently dropping the image+ // or surfacing the generic "all models exhausted" error (#118, #125). Add a+ // rough per-image token cost so budget routing isn't skewed by content the+ // heuristic above (text-only) can't see.+ const hasImage = messageHasImage(messages);+ if (hasImage && !hasEnabledVisionModel()) {+ res.status(422).json({+ error: {+ message: 'This request includes an image, but no vision-capable model is enabled. Enable a vision model (e.g. Gemini 2.5 Flash, Llama 4 Scout) in the Fallback Chain.',+ type: 'invalid_request_error',+ code: 'no_vision_model',+ },+ });+ return;+ }+ const IMAGE_TOKEN_ESTIMATE = 1000;+ const imageCount = messages.reduce((n, m) =>+ n + (Array.isArray(m.content) ? m.content.filter(b => (b as { type?: string })?.type === 'image_url' || (b as { type?: string })?.type === 'image').length : 0), 0);+ const estimatedTotal = estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE + (max_tokens ?? 1000);// Explicit `model` field pins routing. If the catalog has no enabled row// matching the requested id, return 400 — silently auto-routing to a@@ -350,7 +370,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {let route: RouteResult;try {- route = routeRequest(estimatedTotal, skipKeys.size > 0 ? skipKeys : undefined, preferredModel);+ route = routeRequest(estimatedTotal, skipKeys.size > 0 ? skipKeys : undefined, preferredModel, hasImage);} catch (err: any) {// No more models availableif (lastError) {diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.tsindex 8c81ca56..25404539 100644--- a/server/src/routes/responses.ts+++ b/server/src/routes/responses.ts@@ -117,6 +117,24 @@ function partsToString(content: string | Array<{ type: string; text?: unknown }>.join('');}+// Image input via the Responses API isn't carried through translation yet+// (partsToString flattens to text). Detect it so we can hard-fail with a clear+// pointer to /v1/chat/completions rather than silently dropping the image+// (#118, #125). Recognizes the Responses `input_image` part plus the+// chat-style `image_url` / `image` parts some clients reuse here.+export function responsesInputHasImage(req: ResponsesRequest): boolean {+ if (typeof req.input === 'string') return false;+ for (const item of req.input) {+ const content = (item as { content?: unknown }).content;+ if (!Array.isArray(content)) continue;+ if (content.some((p) => {+ const type = (p as { type?: string })?.type;+ return type === 'input_image' || type === 'image_url' || type === 'image';+ })) return true;+ }+ return false;+}+// ── Translate a Responses request → internal chat messages + options ──────export function toChatMessages(req: ResponsesRequest): ChatMessage[] {const messages: ChatMessage[] = [];@@ -250,6 +268,20 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {}const reqData = parsed.data;++ // Vision isn't carried through the Responses translation yet — fail clearly+ // instead of answering blind to a dropped image (#118, #125).+ if (responsesInputHasImage(reqData)) {+ res.status(422).json({+ error: {+ message: 'Image input is not yet supported on /v1/responses. Use /v1/chat/completions with an image_url content part instead.',+ type: 'invalid_request_error',+ code: 'no_vision_model',+ },+ });+ return;+ }+const stream = reqData.stream ?? false;const messages = toChatMessages(reqData);const tools = toChatTools(reqData.tools);diff --git a/server/src/services/router.ts b/server/src/services/router.tsindex 5629f621..3c90a59a 100644--- a/server/src/services/router.ts+++ b/server/src/services/router.ts@@ -13,6 +13,7 @@ interface ModelRow {rpd_limit: number | null;tpm_limit: number | null;tpd_limit: number | null;+ supports_vision: number;}interface KeyRow {@@ -131,8 +132,9 @@ export function getAllPenalties(): Array<{ modelDbId: number; count: number; pen* @param estimatedTokens - estimated total tokens for rate limit check* @param skipKeys - set of "platform:modelId:keyId" to skip (failed on this request)* @param preferredModelDbId - try this model first (sticky session)+ * @param requireVision - only consider models that accept image input (#118)*/-export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preferredModelDbId?: number): RouteResult {+export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preferredModelDbId?: number, requireVision = false): RouteResult {const db = getDb();// Get fallback chain ordered by priority@@ -164,6 +166,10 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preconst model = db.prepare('SELECT * FROM models WHERE id = ? AND enabled = 1').get(entry.model_db_id) as ModelRow | undefined;if (!model) continue;+ // Vision requests skip text-only models — including a sticky/preferred one,+ // which is correct: don't pin an image turn to a model that can't see it.+ if (requireVision && !model.supports_vision) continue;+// Check if we have a provider for this platformconst provider = getProvider(model.platform as any);if (!provider) continue;@@ -243,3 +249,17 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preerr.status = 429;throw err;}++// Whether at least one vision-capable model is enabled in the fallback chain.+// Used to give image requests a clear "enable a vision model" error instead of+// the generic exhaustion message when none is configured (#118, #125).+export function hasEnabledVisionModel(): boolean {+ const db = getDb();+ const row = db.prepare(`+ SELECT COUNT(*) as cnt+ FROM fallback_config fc+ JOIN models m ON m.id = fc.model_db_id+ WHERE fc.enabled = 1 AND m.enabled = 1 AND m.supports_vision = 1+ `).get() as { cnt: number };+ return row.cnt > 0;+}diff --git a/shared/types.ts b/shared/types.tsindex 9982b510..1c917ef5 100644--- a/shared/types.ts+++ b/shared/types.ts@@ -41,6 +41,7 @@ export interface Model {monthlyTokenBudget: string;contextWindow: number | null;enabled: boolean;+ supportsVision: boolean;}export type KeyStatus = 'healthy' | 'rate_limited' | 'invalid' | 'error' | 'unknown';