Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat(vision): image input support with vision-aware routing (#118, #125)

Tashfeen4b2e6f9bb59b
diff --git a/README.md b/README.md
index cac8df02..fe7a6742 100644
--- a/README.md
+++ b/README.md
@@ -94,7 +94,6 @@ The scope is deliberately narrow. If a feature isn't on this list and isn't belo
- **Embeddings** (`/v1/embeddings`)
- **Image generation** (`/v1/images/*`)
- **Audio / speech** (`/v1/audio/*`)
-- **Vision / multimodal inputs** — message content is text-only
- **Legacy completions** (`/v1/completions`) — only the chat endpoint is implemented
- **Moderation** (`/v1/moderations`)
- **`n > 1`** (multiple completions per request)
@@ -265,6 +264,26 @@ final = client.chat.completions.create(
print(final.choices[0].message.content)
```
+**Vision / image input**
+
+Send images with the standard OpenAI `image_url` content blocks (base64 `data:` URLs or `http(s)` URLs). When a request contains an image, the router restricts itself to **vision-capable models** and ignores text-only ones. Vision models are tagged with a **Vision** badge on the Fallback Chain page; the current set includes Gemini (2.5 / 3.x), Llama 4 Scout/Maverick (Groq, NVIDIA, SambaNova), and GitHub's GPT-4o / GPT-4.1.
+
+```python
+resp = client.chat.completions.create(
+ model="auto", # auto-routes to a vision model
+ messages=[{
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "What's in this image?"},
+ {"type": "image_url", "image_url": {"url": "data:image/png;base64,<...>"}},
+ ],
+ }],
+)
+print(resp.choices[0].message.content)
+```
+
+If no vision-capable model is enabled in your Fallback Chain, an image request returns a clear `422` (`code: "no_vision_model"`) rather than silently dropping the image. (Image input on `/v1/responses` isn't supported yet — use `/v1/chat/completions`.)
+
Works with `stream=True` as well — you'll get `delta.tool_calls` chunks followed by a `finish_reason: "tool_calls"` close. Under the hood, OpenAI-compatible providers (Groq, Cerebras, SambaNova, Mistral, OpenRouter, GitHub Models, HuggingFace, Cloudflare, Cohere compat) get the request passed through; Gemini requests get translated into Google's `functionDeclarations` / `functionResponse` shape and the response is translated back.
Every response carries an `X-Routed-Via: <platform>/<model>` header so you can see which provider actually served each call. If a request fell over between providers, you'll also see `X-Fallback-Attempts: N`.
diff --git a/client/src/pages/FallbackPage.tsx b/client/src/pages/FallbackPage.tsx
index f18d42b4..d7512e01 100644
--- a/client/src/pages/FallbackPage.tsx
+++ b/client/src/pages/FallbackPage.tsx
@@ -38,6 +38,7 @@ interface FallbackEntry {
rpmLimit: number | null
rpdLimit: number | null
monthlyTokenBudget: string
+ supportsVision: boolean
keyCount: number
}
@@ -176,6 +177,14 @@ function SortableModelRow({
<div className="flex items-center gap-2 flex-wrap">
<span className="font-medium text-sm">{entry.displayName}</span>
<span className="text-xs text-muted-foreground">{entry.platform}</span>
+ {entry.supportsVision && (
+ <span
+ title="Accepts image input"
+ className="text-xs rounded-full px-2 py-0.5 bg-cyan-600/15 text-cyan-700 dark:bg-cyan-400/15 dark:text-cyan-400"
+ >
+ Vision
+ </span>
+ )}
{entry.penalty > 0 && (
<span className="text-xs text-amber-600 dark:text-amber-400">
−{entry.penalty} penalty
diff --git a/server/package.json b/server/package.json
index a2e4e7c5..fc580996 100644
--- a/server/package.json
+++ b/server/package.json
@@ -7,7 +7,7 @@
"dev": "tsx watch src/index.ts",
"build": "tsc",
"start": "node dist/index.js",
- "test": "vitest run",
+ "test": "vitest run --pool=forks --poolOptions.forks.singleFork=true --no-file-parallelism",
"test:watch": "vitest"
},
"dependencies": {
diff --git a/server/src/__tests__/lib/content.test.ts b/server/src/__tests__/lib/content.test.ts
index 0840d338..f6a0648a 100644
--- a/server/src/__tests__/lib/content.test.ts
+++ b/server/src/__tests__/lib/content.test.ts
@@ -1,5 +1,5 @@
import { describe, it, expect } from 'vitest';
-import { contentToString, flattenMessageContent } from '../../lib/content.js';
+import { contentToString, flattenMessageContent, messageHasImage } from '../../lib/content.js';
describe('contentToString', () => {
it('passes strings through', () => {
@@ -19,7 +19,7 @@ describe('contentToString', () => {
])).toBe('hello world');
});
- it('drops non-text blocks silently (image_url etc.) — vision unsupported', () => {
+ it('drops non-text blocks (image_url etc.) — text-only providers flatten this way', () => {
expect(contentToString([
{ type: 'text', text: 'describe ' },
{ type: 'image_url', image_url: { url: 'https://example.com/x.png' } },
@@ -61,3 +61,28 @@ describe('flattenMessageContent', () => {
});
});
});
+
+describe('messageHasImage', () => {
+ it('detects image_url blocks (OpenAI vision envelope)', () => {
+ expect(messageHasImage([
+ { role: 'user', content: [
+ { type: 'text', text: 'what is this?' },
+ { type: 'image_url', image_url: { url: 'data:image/png;base64,AAAA' } },
+ ] },
+ ])).toBe(true);
+ });
+
+ it('detects a bare image block type', () => {
+ expect(messageHasImage([
+ { role: 'user', content: [{ type: 'image', source: 'x' } as any] },
+ ])).toBe(true);
+ });
+
+ it('is false for string content and text-only arrays', () => {
+ expect(messageHasImage([{ role: 'user', content: 'hello' }])).toBe(false);
+ expect(messageHasImage([
+ { role: 'user', content: [{ type: 'text', text: 'hello' }] },
+ ])).toBe(false);
+ expect(messageHasImage([{ role: 'assistant', content: null }])).toBe(false);
+ });
+});
diff --git a/server/src/__tests__/providers/google.test.ts b/server/src/__tests__/providers/google.test.ts
index 31984dc4..caa6aed3 100644
--- a/server/src/__tests__/providers/google.test.ts
+++ b/server/src/__tests__/providers/google.test.ts
@@ -45,6 +45,28 @@ describe('GoogleProvider', () => {
expect(result._routed_via?.platform).toBe('google');
});
+ it('converts an image_url data URL into a Gemini inlineData part (#118)', async () => {
+ const fetchSpy = vi.spyOn(global, 'fetch').mockResolvedValueOnce({
+ ok: true,
+ json: () => Promise.resolve({
+ candidates: [{ content: { parts: [{ text: 'a cat' }] }, finishReason: 'STOP' }],
+ usageMetadata: { promptTokenCount: 1, candidatesTokenCount: 1, totalTokenCount: 2 },
+ }),
+ } as any);
+
+ await provider.chatCompletion('test-key', [
+ { role: 'user', content: [
+ { type: 'text', text: 'what is this?' },
+ { type: 'image_url', image_url: { url: 'data:image/png;base64,iVBORw0KGgo=' } },
+ ] as any },
+ ], 'gemini-2.5-flash');
+
+ const body = JSON.parse((fetchSpy.mock.calls[0][1] as any).body);
+ const parts = body.contents[0].parts;
+ expect(parts).toContainEqual({ text: 'what is this?' });
+ expect(parts).toContainEqual({ inlineData: { mimeType: 'image/png', data: 'iVBORw0KGgo=' } });
+ });
+
it('should throw on API error', async () => {
vi.spyOn(global, 'fetch').mockResolvedValueOnce({
ok: false,
diff --git a/server/src/__tests__/routes/proxy-vision.test.ts b/server/src/__tests__/routes/proxy-vision.test.ts
new file mode 100644
index 00000000..c04024ea
--- /dev/null
+++ b/server/src/__tests__/routes/proxy-vision.test.ts
@@ -0,0 +1,87 @@
+import { describe, it, expect, beforeAll } from 'vitest';
+import type { Express } from 'express';
+import { createApp } from '../../app.js';
+import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';
+
+async function post(app: Express, path: string, body: any, key: string) {
+ const server = app.listen(0);
+ const addr = server.address() as any;
+ const res = await fetch(`http://127.0.0.1:${addr.port}${path}`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` },
+ body: JSON.stringify(body),
+ });
+ const text = await res.text();
+ server.close();
+ let json: any = null;
+ try { json = JSON.parse(text); } catch {}
+ return { status: res.status, body: json };
+}
+
+const IMAGE_MESSAGE = {
+ messages: [{
+ role: 'user',
+ content: [
+ { type: 'text', text: 'what is in this image?' },
+ { type: 'image_url', image_url: { url: 'data:image/png;base64,iVBORw0KGgo=' } },
+ ],
+ }],
+};
+
+describe('Vision-aware routing (#118, #125)', () => {
+ let app: Express;
+ let key: string;
+
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ app = createApp();
+ key = getUnifiedApiKey();
+ });
+
+ it('seeds supports_vision: true for vision models, false for text-only', () => {
+ const db = getDb();
+ const vision = db.prepare("SELECT supports_vision FROM models WHERE model_id = 'gemini-2.5-flash'").get() as { supports_vision: number };
+ expect(vision.supports_vision).toBe(1);
+
+ // The known vision set is flagged; plenty of text-only models remain at 0.
+ const visionCount = (db.prepare('SELECT COUNT(*) c FROM models WHERE supports_vision = 1').get() as { c: number }).c;
+ const textCount = (db.prepare('SELECT COUNT(*) c FROM models WHERE supports_vision = 0').get() as { c: number }).c;
+ expect(visionCount).toBeGreaterThanOrEqual(4);
+ expect(textCount).toBeGreaterThan(0);
+ });
+
+ it('lets an image request through routing when a vision model is enabled (no 422)', async () => {
+ // Seed has Gemini/Llama-4 vision models enabled but no provider keys, so
+ // routing exhausts → 429. The point: it is NOT the 422 "no vision model"
+ // error, proving the precheck passed and routing was attempted.
+ const { status, body } = await post(app, '/v1/chat/completions', IMAGE_MESSAGE, key);
+ expect(status).not.toBe(422);
+ expect(body?.error?.code).not.toBe('no_vision_model');
+ });
+
+ it('rejects an image request with a clear 422 when no vision model is enabled', async () => {
+ // Disable every vision-capable model in the chain.
+ getDb().prepare('UPDATE models SET enabled = 0 WHERE supports_vision = 1').run();
+
+ const { status, body } = await post(app, '/v1/chat/completions', IMAGE_MESSAGE, key);
+ expect(status).toBe(422);
+ expect(body.error.code).toBe('no_vision_model');
+ expect(body.error.type).toBe('invalid_request_error');
+
+ // Restore so we don't leak state to other expectations.
+ getDb().prepare('UPDATE models SET enabled = 1 WHERE supports_vision = 1').run();
+ });
+
+ it('does not apply the vision gate to a text-only request', async () => {
+ // Disable all vision models; a plain text request must still route normally
+ // (it 429s on exhaustion here, but never the 422 vision error).
+ getDb().prepare('UPDATE models SET enabled = 0 WHERE supports_vision = 1').run();
+ const { status, body } = await post(app, '/v1/chat/completions', {
+ messages: [{ role: 'user', content: 'hello' }],
+ }, key);
+ expect(status).not.toBe(422);
+ expect(body?.error?.code).not.toBe('no_vision_model');
+ getDb().prepare('UPDATE models SET enabled = 1 WHERE supports_vision = 1').run();
+ });
+});
diff --git a/server/src/__tests__/routes/responses.test.ts b/server/src/__tests__/routes/responses.test.ts
index d3f8ae5a..c37a2589 100644
--- a/server/src/__tests__/routes/responses.test.ts
+++ b/server/src/__tests__/routes/responses.test.ts
@@ -49,6 +49,22 @@ describe('POST /v1/responses (#96)', () => {
expect((await post(app, '/v1/responses', { model: 'auto' }, key)).status).toBe(400);
});
+ // #118: image input isn't carried through the Responses translation yet, so
+ // it must hard-fail clearly rather than silently answer blind to the image.
+ it('rejects image input with a clear 422 pointing at /v1/chat/completions', async () => {
+ const { status, text } = await post(app, '/v1/responses', {
+ input: [{
+ role: 'user',
+ content: [
+ { type: 'input_text', text: 'what is this?' },
+ { type: 'input_image', image_url: 'data:image/png;base64,iVBORw0KGgo=' },
+ ],
+ }],
+ }, key);
+ expect(status).toBe(422);
+ expect(JSON.parse(text).error.code).toBe('no_vision_model');
+ });
+
// #103: the x-api-key header (Anthropic wire format) must authenticate here
// too, not just on /v1/chat/completions.
it('accepts the unified key via the x-api-key header', async () => {
diff --git a/server/src/db/index.ts b/server/src/db/index.ts
index 8d4273df..a7469bea 100644
--- a/server/src/db/index.ts
+++ b/server/src/db/index.ts
@@ -50,6 +50,7 @@ export function initDb(dbPath?: string): Database.Database {
migrateModelsV13(db);
migrateModelsV14(db);
migrateModelsV15(db);
+ migrateModelsV16Vision(db);
ensureUnifiedKey(db);
console.log(`Database initialized at ${resolvedPath}`);
@@ -73,6 +74,7 @@ function createTables(db: Database.Database) {
monthly_token_budget TEXT NOT NULL DEFAULT '',
context_window INTEGER,
enabled INTEGER NOT NULL DEFAULT 1,
+ supports_vision INTEGER NOT NULL DEFAULT 0,
UNIQUE(platform, model_id)
);
@@ -1311,6 +1313,46 @@ function migrateModelsV15(db: Database.Database) {
db.prepare(`DELETE FROM models WHERE platform = 'siliconflow'`).run();
}
+// Adds the supports_vision column to existing DBs and (re)applies the vision
+// flags by rule. Rule-based rather than a hardcoded id list because the catalog
+// churns through migrations (model ids get renamed/replaced) — rules survive
+// that. Two constraints decide a model can accept images:
+// 1. The model is genuinely multimodal (every Gemini is; Llama 4 Scout/
+// Maverick are; GitHub's GPT-4o/4.1/5 are).
+// 2. Its provider adapter forwards image content. OpenAICompat and Google do;
+// Cohere and Cloudflare flatten content to text, so models on those
+// platforms are excluded even when the underlying model can see.
+// Conservative on purpose: with hard-fail routing a false negative is just a
+// clear "no vision model" error, while a false positive routes an image to a
+// model that chokes. Idempotent — safe on fresh seeds and upgrades alike.
+function migrateModelsV16Vision(db: Database.Database) {
+ const columns = db.prepare('PRAGMA table_info(models)').all() as { name: string }[];
+ if (!columns.some(col => col.name === 'supports_vision')) {
+ db.prepare('ALTER TABLE models ADD COLUMN supports_vision INTEGER NOT NULL DEFAULT 0').run();
+ }
+ const apply = db.transaction(() => {
+ // Reset first so de-flagged models (e.g. an id that moved to Cloudflare)
+ // don't keep a stale flag across re-runs.
+ db.prepare('UPDATE models SET supports_vision = 0').run();
+ // Every Gemini is multimodal (the 'google' platform is all Gemini).
+ db.prepare("UPDATE models SET supports_vision = 1 WHERE platform = 'google'").run();
+ // Llama 4 (Scout/Maverick) is natively multimodal — but only where the
+ // adapter forwards images (exclude the text-flattening providers).
+ db.prepare(`
+ UPDATE models SET supports_vision = 1
+ WHERE LOWER(model_id) LIKE '%llama-4%'
+ AND platform NOT IN ('cloudflare', 'cohere')
+ `).run();
+ // GitHub's OpenAI vision models.
+ db.prepare(`
+ UPDATE models SET supports_vision = 1
+ WHERE platform = 'github'
+ AND (model_id LIKE '%gpt-4o%' OR model_id LIKE '%gpt-4.1%' OR model_id LIKE '%gpt-5%')
+ `).run();
+ });
+ apply();
+}
+
function ensureUnifiedKey(db: Database.Database) {
const existing = db.prepare("SELECT value FROM settings WHERE key = 'unified_api_key'").get() as { value: string } | undefined;
if (!existing) {
diff --git a/server/src/lib/content.ts b/server/src/lib/content.ts
index e0a481f8..b01623ac 100644
--- a/server/src/lib/content.ts
+++ b/server/src/lib/content.ts
@@ -28,3 +28,20 @@ export function flattenMessageContent(messages: ChatMessage[]): ChatMessage[] {
content: contentToString(m.content),
}));
}
+
+// True if the content array carries an image block. OpenAI's multimodal
+// envelope uses `{ type: 'image_url', image_url: { url } }`; some clients send
+// a bare `{ type: 'image', ... }`.
+export function contentHasImage(content: unknown): boolean {
+ if (!Array.isArray(content)) return false;
+ return content.some((block) => {
+ const type = (block as { type?: string })?.type;
+ return type === 'image_url' || type === 'image';
+ });
+}
+
+// True if any message carries an image content block. Used to route image
+// requests only to vision-capable models (#118, #125).
+export function messageHasImage(messages: ChatMessage[]): boolean {
+ return messages.some((m) => contentHasImage(m.content));
+}
diff --git a/server/src/providers/google.ts b/server/src/providers/google.ts
index 434925d4..9ef1ff70 100644
--- a/server/src/providers/google.ts
+++ b/server/src/providers/google.ts
@@ -14,6 +14,10 @@ const API_BASE = 'https://generativelanguage.googleapis.com/v1beta';
interface GeminiPart {
text?: string;
+ inlineData?: {
+ mimeType: string;
+ data: string;
+ };
thoughtSignature?: string;
functionCall?: {
id?: string;
@@ -130,10 +134,75 @@ function toGeminiToolConfig(toolChoice?: ChatToolChoice): { functionCallingConfi
};
}
+const MAX_IMAGE_BYTES = 8 * 1024 * 1024; // 8 MB cap on fetched/inlined images
+
+// Pull the URL out of an OpenAI image content block. Accepts both the object
+// form `{ image_url: { url } }` and the shorthand `{ image_url: '...' }`.
+function extractImageUrl(block: unknown): string | undefined {
+ const iu = (block as { image_url?: unknown })?.image_url;
+ if (typeof iu === 'string') return iu;
+ if (iu && typeof (iu as { url?: unknown }).url === 'string') return (iu as { url: string }).url;
+ return undefined;
+}
+
+// Convert an image URL to a Gemini inlineData part. Handles base64 `data:` URLs
+// directly; for `http(s)` URLs we fetch and inline because the Gemini API does
+// not fetch external URLs itself. Fetching a user-supplied URL is a minor SSRF
+// surface, acceptable for a single-user self-hosted proxy; we still restrict to
+// http/https and cap the size. Returns null (part skipped) on any failure.
+async function imageUrlToInlineData(url: string): Promise<{ mimeType: string; data: string } | null> {
+ const dataMatch = /^data:([^;,]+)?(;base64)?,(.*)$/s.exec(url);
+ if (dataMatch) {
+ const mimeType = dataMatch[1] || 'application/octet-stream';
+ const isBase64 = Boolean(dataMatch[2]);
+ const payload = dataMatch[3] ?? '';
+ const data = isBase64
+ ? payload
+ : Buffer.from(decodeURIComponent(payload)).toString('base64');
+ return { mimeType, data };
+ }
+ if (/^https?:\/\//i.test(url)) {
+ try {
+ const res = await fetch(url);
+ if (!res.ok) return null;
+ const buf = Buffer.from(await res.arrayBuffer());
+ if (buf.length === 0 || buf.length > MAX_IMAGE_BYTES) return null;
+ const mimeType = res.headers.get('content-type')?.split(';')[0]?.trim() || 'image/jpeg';
+ return { mimeType, data: buf.toString('base64') };
+ } catch {
+ return null;
+ }
+ }
+ return null;
+}
+
+// Build Gemini parts for a user message: joined text first, then any images as
+// inlineData. Non-array content (string/null) collapses to a single text part.
+async function userContentToParts(content: ChatMessage['content']): Promise<GeminiPart[]> {
+ const parts: GeminiPart[] = [];
+ const text = contentToString(content);
+ if (text.length > 0) parts.push({ text });
+
+ if (Array.isArray(content)) {
+ for (const block of content) {
+ const type = (block as { type?: string })?.type;
+ if (type !== 'image_url' && type !== 'image') continue;
+ const url = extractImageUrl(block);
+ if (!url) continue;
+ const inlineData = await imageUrlToInlineData(url);
+ if (inlineData) parts.push({ inlineData });
+ }
+ }
+
+ // Gemini rejects empty `parts`; keep at least one (possibly empty) text part.
+ if (parts.length === 0) parts.push({ text: '' });
+ return parts;
+}
+
// Translate OpenAI messages to Gemini format. Content may arrive as a string,
-// null, or the OpenAI multimodal array envelope — flatten to string first so
-// system/user/tool messages all surface as `parts: [{ text }]` for Gemini.
-function toGeminiContents(messages: ChatMessage[]) {
+// null, or the OpenAI multimodal array envelope. System/assistant/tool messages
+// flatten to text; user messages additionally carry images as inlineData parts.
+async function toGeminiContents(messages: ChatMessage[]) {
const systemMessages = messages
.filter(m => m.role === 'system')
.map(m => contentToString(m.content))
@@ -146,9 +215,9 @@ function toGeminiContents(messages: ChatMessage[]) {
}
}
- const contents = messages
+ const contents = (await Promise.all(messages
.filter(m => m.role !== 'system')
- .map((m): { role: 'user' | 'model'; parts: GeminiPart[] } | null => {
+ .map(async (m): Promise<{ role: 'user' | 'model'; parts: GeminiPart[] } | null> => {
if (m.role === 'assistant') {
const parts: GeminiPart[] = [];
@@ -196,9 +265,9 @@ function toGeminiContents(messages: ChatMessage[]) {
return {
role: 'user',
- parts: [{ text: contentToString(m.content) }],
+ parts: await userContentToParts(m.content),
};
- })
+ })))
.filter((entry): entry is { role: 'user' | 'model'; parts: GeminiPart[] } => entry !== null);
return {
@@ -250,7 +319,7 @@ export class GoogleProvider extends BaseProvider {
modelId: string,
options?: CompletionOptions,
): Promise<ChatCompletionResponse> {
- const { contents, systemInstruction } = toGeminiContents(messages);
+ const { contents, systemInstruction } = await toGeminiContents(messages);
const body: Record<string, unknown> = {
contents,
@@ -313,7 +382,7 @@ export class GoogleProvider extends BaseProvider {
modelId: string,
options?: CompletionOptions,
): AsyncGenerator<ChatCompletionChunk> {
- const { contents, systemInstruction } = toGeminiContents(messages);
+ const { contents, systemInstruction } = await toGeminiContents(messages);
const body: Record<string, unknown> = {
contents,
diff --git a/server/src/routes/fallback.ts b/server/src/routes/fallback.ts
index 7c4e6597..7cbd3b23 100644
--- a/server/src/routes/fallback.ts
+++ b/server/src/routes/fallback.ts
@@ -13,7 +13,7 @@ fallbackRouter.get('/', (_req: Request, res: Response) => {
SELECT fc.model_db_id, fc.priority, fc.enabled,
m.platform, m.model_id, m.display_name, m.intelligence_rank,
m.speed_rank, m.size_label, m.rpm_limit, m.rpd_limit,
- m.monthly_token_budget
+ m.monthly_token_budget, m.supports_vision
FROM fallback_config fc
JOIN models m ON m.id = fc.model_db_id
ORDER BY fc.priority ASC
@@ -49,6 +49,7 @@ fallbackRouter.get('/', (_req: Request, res: Response) => {
rpmLimit: r.rpm_limit,
rpdLimit: r.rpd_limit,
monthlyTokenBudget: r.monthly_token_budget,
+ supportsVision: r.supports_vision === 1,
keyCount: keyCountMap.get(r.platform) ?? 0,
};
}));
diff --git a/server/src/routes/models.ts b/server/src/routes/models.ts
index 474a664d..9a381152 100644
--- a/server/src/routes/models.ts
+++ b/server/src/routes/models.ts
@@ -40,6 +40,7 @@ modelsRouter.get('/', (_req: Request, res: Response) => {
monthlyTokenBudget: m.monthly_token_budget,
contextWindow: m.context_window,
enabled: m.enabled === 1,
+ supportsVision: m.supports_vision === 1,
priority: m.priority,
fallbackEnabled: m.fallback_enabled === 1,
hasProvider: hasProvider(m.platform),
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index f8e93488..2cdc8068 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -3,10 +3,10 @@ import { Router } from 'express';
import type { Request, Response } from 'express';
import { z } from 'zod';
import type { ChatMessage } from '@freellmapi/shared/types.js';
-import { routeRequest, recordRateLimitHit, recordSuccess, type RouteResult } from '../services/router.js';
+import { routeRequest, recordRateLimitHit, recordSuccess, hasEnabledVisionModel, type RouteResult } from '../services/router.js';
import { recordRequest, recordTokens, setCooldown, getNextCooldownDuration } from '../services/ratelimit.js';
import { getDb, getUnifiedApiKey } from '../db/index.js';
-import { contentToString } from '../lib/content.js';
+import { contentToString, messageHasImage } from '../lib/content.js';
export const proxyRouter = Router();
@@ -312,7 +312,27 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
const text = contentToString(m.content);
return sum + Math.ceil(text.length / 4);
}, 0);
- const estimatedTotal = estimatedInputTokens + (max_tokens ?? 1000);
+
+ // Image requests must route to a vision-capable model. Reject up front with a
+ // clear message when none is enabled, rather than silently dropping the image
+ // or surfacing the generic "all models exhausted" error (#118, #125). Add a
+ // rough per-image token cost so budget routing isn't skewed by content the
+ // heuristic above (text-only) can't see.
+ const hasImage = messageHasImage(messages);
+ if (hasImage && !hasEnabledVisionModel()) {
+ res.status(422).json({
+ error: {
+ message: 'This request includes an image, but no vision-capable model is enabled. Enable a vision model (e.g. Gemini 2.5 Flash, Llama 4 Scout) in the Fallback Chain.',
+ type: 'invalid_request_error',
+ code: 'no_vision_model',
+ },
+ });
+ return;
+ }
+ const IMAGE_TOKEN_ESTIMATE = 1000;
+ const imageCount = messages.reduce((n, m) =>
+ n + (Array.isArray(m.content) ? m.content.filter(b => (b as { type?: string })?.type === 'image_url' || (b as { type?: string })?.type === 'image').length : 0), 0);
+ const estimatedTotal = estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE + (max_tokens ?? 1000);
// Explicit `model` field pins routing. If the catalog has no enabled row
// matching the requested id, return 400 — silently auto-routing to a
@@ -350,7 +370,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
let route: RouteResult;
try {
- route = routeRequest(estimatedTotal, skipKeys.size > 0 ? skipKeys : undefined, preferredModel);
+ route = routeRequest(estimatedTotal, skipKeys.size > 0 ? skipKeys : undefined, preferredModel, hasImage);
} catch (err: any) {
// No more models available
if (lastError) {
diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.ts
index 8c81ca56..25404539 100644
--- a/server/src/routes/responses.ts
+++ b/server/src/routes/responses.ts
@@ -117,6 +117,24 @@ function partsToString(content: string | Array<{ type: string; text?: unknown }>
.join('');
}
+// Image input via the Responses API isn't carried through translation yet
+// (partsToString flattens to text). Detect it so we can hard-fail with a clear
+// pointer to /v1/chat/completions rather than silently dropping the image
+// (#118, #125). Recognizes the Responses `input_image` part plus the
+// chat-style `image_url` / `image` parts some clients reuse here.
+export function responsesInputHasImage(req: ResponsesRequest): boolean {
+ if (typeof req.input === 'string') return false;
+ for (const item of req.input) {
+ const content = (item as { content?: unknown }).content;
+ if (!Array.isArray(content)) continue;
+ if (content.some((p) => {
+ const type = (p as { type?: string })?.type;
+ return type === 'input_image' || type === 'image_url' || type === 'image';
+ })) return true;
+ }
+ return false;
+}
+
// ── Translate a Responses request → internal chat messages + options ──────
export function toChatMessages(req: ResponsesRequest): ChatMessage[] {
const messages: ChatMessage[] = [];
@@ -250,6 +268,20 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
}
const reqData = parsed.data;
+
+ // Vision isn't carried through the Responses translation yet — fail clearly
+ // instead of answering blind to a dropped image (#118, #125).
+ if (responsesInputHasImage(reqData)) {
+ res.status(422).json({
+ error: {
+ message: 'Image input is not yet supported on /v1/responses. Use /v1/chat/completions with an image_url content part instead.',
+ type: 'invalid_request_error',
+ code: 'no_vision_model',
+ },
+ });
+ return;
+ }
+
const stream = reqData.stream ?? false;
const messages = toChatMessages(reqData);
const tools = toChatTools(reqData.tools);
diff --git a/server/src/services/router.ts b/server/src/services/router.ts
index 5629f621..3c90a59a 100644
--- a/server/src/services/router.ts
+++ b/server/src/services/router.ts
@@ -13,6 +13,7 @@ interface ModelRow {
rpd_limit: number | null;
tpm_limit: number | null;
tpd_limit: number | null;
+ supports_vision: number;
}
interface KeyRow {
@@ -131,8 +132,9 @@ export function getAllPenalties(): Array<{ modelDbId: number; count: number; pen
* @param estimatedTokens - estimated total tokens for rate limit check
* @param skipKeys - set of "platform:modelId:keyId" to skip (failed on this request)
* @param preferredModelDbId - try this model first (sticky session)
+ * @param requireVision - only consider models that accept image input (#118)
*/
-export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preferredModelDbId?: number): RouteResult {
+export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preferredModelDbId?: number, requireVision = false): RouteResult {
const db = getDb();
// Get fallback chain ordered by priority
@@ -164,6 +166,10 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre
const model = db.prepare('SELECT * FROM models WHERE id = ? AND enabled = 1').get(entry.model_db_id) as ModelRow | undefined;
if (!model) continue;
+ // Vision requests skip text-only models — including a sticky/preferred one,
+ // which is correct: don't pin an image turn to a model that can't see it.
+ if (requireVision && !model.supports_vision) continue;
+
// Check if we have a provider for this platform
const provider = getProvider(model.platform as any);
if (!provider) continue;
@@ -243,3 +249,17 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre
err.status = 429;
throw err;
}
+
+// Whether at least one vision-capable model is enabled in the fallback chain.
+// Used to give image requests a clear "enable a vision model" error instead of
+// the generic exhaustion message when none is configured (#118, #125).
+export function hasEnabledVisionModel(): boolean {
+ const db = getDb();
+ const row = db.prepare(`
+ SELECT COUNT(*) as cnt
+ FROM fallback_config fc
+ JOIN models m ON m.id = fc.model_db_id
+ WHERE fc.enabled = 1 AND m.enabled = 1 AND m.supports_vision = 1
+ `).get() as { cnt: number };
+ return row.cnt > 0;
+}
diff --git a/shared/types.ts b/shared/types.ts
index 9982b510..1c917ef5 100644
--- a/shared/types.ts
+++ b/shared/types.ts
@@ -41,6 +41,7 @@ export interface Model {
monthlyTokenBudget: string;
contextWindow: number | null;
enabled: boolean;
+ supportsVision: boolean;
}
export type KeyStatus = 'healthy' | 'rate_limited' | 'invalid' | 'error' | 'unknown';