Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat(fusion): support vision (image) requests via vision-capable panel (#884)

Icesenator7b54a7729ae9
diff --git a/server/src/__tests__/lib/content.test.ts b/server/src/__tests__/lib/content.test.ts
index d2ff2d6e..8b875642 100644
--- a/server/src/__tests__/lib/content.test.ts
+++ b/server/src/__tests__/lib/content.test.ts
@@ -1,5 +1,5 @@
import { describe, it, expect } from 'vitest';
-import { contentToString, flattenMessageContent, messageHasImage, normalizeOutboundContent, sanitizeResponse } from '../../lib/content.js';
+import { contentToString, flattenMessageContent, messageHasImage, normalizeOutboundContent, sanitizeResponse, stripImagesFromMessages } from '../../lib/content.js';
describe('contentToString', () => {
it('passes strings through', () => {
@@ -93,6 +93,35 @@ describe('messageHasImage', () => {
});
});
+describe('stripImagesFromMessages', () => {
+ it('drops image blocks but keeps text blocks', () => {
+ const out = stripImagesFromMessages([
+ { role: 'user', content: [
+ { type: 'text', text: 'hello' },
+ { type: 'image_url', image_url: { url: 'data:image/png;base64,AA==' } },
+ ] },
+ ]);
+ expect(out[0].content).toEqual([{ type: 'text', text: 'hello' }]);
+ });
+
+ it('replaces an image-only message with an empty string', () => {
+ const out = stripImagesFromMessages([
+ { role: 'user', content: [
+ { type: 'image_url', image_url: { url: 'data:image/png;base64,AA==' } },
+ ] },
+ ]);
+ expect(out[0].content).toBe('');
+ });
+
+ it('leaves string content and plain messages untouched', () => {
+ const m1 = { role: 'user', content: 'plain text' };
+ const m2 = { role: 'assistant', content: null };
+ const out = stripImagesFromMessages([m1 as any, m2 as any]);
+ expect(out[0]).toBe(m1); // identity preserved (no unnecessary copy)
+ expect(out[1]).toBe(m2);
+ });
+});
+
describe('normalizeOutboundContent (#166)', () => {
it('coerces array delta.content to a string on streaming chunks', () => {
const chunk = { choices: [{ index: 0, delta: { content: [{ type: 'text', text: 'hel' }, { type: 'text', text: 'lo' }] } }] };
diff --git a/server/src/__tests__/routes/proxy-vision.test.ts b/server/src/__tests__/routes/proxy-vision.test.ts
index 3f8b21f1..92819f05 100644
--- a/server/src/__tests__/routes/proxy-vision.test.ts
+++ b/server/src/__tests__/routes/proxy-vision.test.ts
@@ -85,4 +85,12 @@ describe('Vision-aware routing (#118, #125)', () => {
expect(body?.error?.code).not.toBe('no_vision_model');
getDb().prepare('UPDATE models SET enabled = 1 WHERE supports_vision = 1').run();
});
+
+ it('lets a fusion image request through routing (no 422 fusion_no_vision)', async () => {
+ // Same seed as the auto case: vision models enabled but no provider keys,
+ // so routing exhausts. The point is that fusion is NOT rejected up front.
+ const { status, body } = await post(app, '/v1/chat/completions', { ...IMAGE_MESSAGE, model: 'fusion' }, key);
+ expect(status).not.toBe(422);
+ expect(body?.error?.code).not.toBe('fusion_no_vision');
+ });
});
diff --git a/server/src/__tests__/services/fusion.test.ts b/server/src/__tests__/services/fusion.test.ts
new file mode 100644
index 00000000..fc170bca
--- /dev/null
+++ b/server/src/__tests__/services/fusion.test.ts
@@ -0,0 +1,90 @@
+import { describe, it, expect, beforeAll, beforeEach } from 'vitest';
+import { initDb, getDb } from '../../db/index.js';
+import { encrypt } from '../../lib/crypto.js';
+import { setRoutingStrategy } from '../../services/router.js';
+
+// A fusion unit-test fixture: seed the standard catalog, then add one healthy
+// key per platform so the auto-panel chain has servable members (the seed ships
+// no keys by default). Priority strategy keeps the panel order deterministic.
+function addKey(platform: string): void {
+ const { encrypted, iv, authTag } = encrypt(`test-${platform}-key`);
+ getDb().prepare(`
+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)
+ VALUES (?, 'fusion-vision-test', ?, ?, ?, 'healthy', 1)
+ `).run(platform, encrypted, iv, authTag);
+}
+
+describe('selectPanel vision filtering', () => {
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ setRoutingStrategy('priority');
+ });
+
+ beforeEach(() => {
+ const db = getDb();
+ db.prepare('DELETE FROM api_keys').run();
+ db.prepare("DELETE FROM settings WHERE key = 'active_profile_id'").run();
+ const models = db.prepare('SELECT id, intelligence_rank FROM models ORDER BY intelligence_rank ASC').all() as any[];
+ const update = db.prepare('UPDATE fallback_config SET priority = ? WHERE model_db_id = ?');
+ for (let i = 0; i < models.length; i++) update.run(i + 1, models[i].id);
+ addKey('google');
+ addKey('groq');
+ });
+
+ it('drops explicit panel models without vision support when requireVision', async () => {
+ const { selectPanel } = await import('../../services/fusion.js');
+ const config = { models: ['gemini-2.5-flash', 'llama-3.1-8b-instant'], k: 2, judge: null, strategy: 'synthesize' as const, expose_panel: false };
+ const { panel, dropped } = selectPanel(config, { requireVision: true, estimatedTokens: 100 });
+ expect(panel.every((c) => c.supportsVision === 1)).toBe(true);
+ expect(panel.some((c) => c.modelId === 'gemini-2.5-flash')).toBe(true);
+ expect(panel.some((c) => c.modelId === 'llama-3.1-8b-instant')).toBe(false);
+ expect(dropped).toContain('llama-3.1-8b-instant (no vision support)');
+ });
+
+ it('auto panel only contains vision models when requireVision', async () => {
+ const { selectPanel } = await import('../../services/fusion.js');
+ const config = { k: 4, judge: null, strategy: 'synthesize' as const, expose_panel: false };
+ const { panel } = selectPanel(config, { requireVision: true, estimatedTokens: 100 });
+ expect(panel.length).toBeGreaterThan(0);
+ expect(panel.every((c) => c.supportsVision === 1)).toBe(true);
+ });
+});
+
+describe('fusion vision judge', () => {
+ it('buildJudgeMessages strips image blocks when vision is requested', async () => {
+ const { buildJudgeMessages } = await import('../../services/fusion.js');
+ const original = [
+ { role: 'user' as const, content: [
+ { type: 'text', text: 'what is in this image?' },
+ { type: 'image_url', image_url: { url: 'data:image/png;base64,AAAA' } },
+ ] },
+ ];
+ const answers = [
+ { status: 'ok' as const, content: 'a red square', usage: undefined } as any,
+ ];
+ const judgeMessages = buildJudgeMessages(original, answers, true);
+ const hasImage = judgeMessages.some((m) =>
+ Array.isArray(m.content) && m.content.some((b: any) => b?.type === 'image_url' || b?.type === 'image'),
+ );
+ expect(hasImage).toBe(false);
+ });
+
+ it('keeps image blocks by default (no strip flag)', async () => {
+ const { buildJudgeMessages } = await import('../../services/fusion.js');
+ const original = [
+ { role: 'user' as const, content: [
+ { type: 'text', text: 'what is in this image?' },
+ { type: 'image_url', image_url: { url: 'data:image/png;base64,AAAA' } },
+ ] },
+ ];
+ const answers = [
+ { status: 'ok' as const, content: 'a red square', usage: undefined } as any,
+ ];
+ const judgeMessages = buildJudgeMessages(original, answers);
+ const hasImage = judgeMessages.some((m) =>
+ Array.isArray(m.content) && m.content.some((b: any) => b?.type === 'image_url' || b?.type === 'image'),
+ );
+ expect(hasImage).toBe(true);
+ });
+});
diff --git a/server/src/lib/content.ts b/server/src/lib/content.ts
index 50e1f1b0..696eaf47 100644
--- a/server/src/lib/content.ts
+++ b/server/src/lib/content.ts
@@ -57,6 +57,20 @@ export function messageHasImage(messages: ChatMessage[]): boolean {
return messages.some((m) => contentHasImage(m.content));
}
+// Drop image blocks from every message so a text-only model (e.g. the fusion
+// judge, which synthesizes from panel text answers) never receives pixels.
+export function stripImagesFromMessages(messages: ChatMessage[]): ChatMessage[] {
+ return messages.map((m) => {
+ if (!Array.isArray(m.content)) return m;
+ const kept = m.content.filter((block) => {
+ const type = (block as { type?: string })?.type;
+ return type !== 'image_url' && type !== 'image';
+ });
+ if (kept.length === m.content.length) return m;
+ return { ...m, content: kept.length > 0 ? kept : '' };
+ });
+}
+
// Harden the OUTBOUND envelope so strict OpenAI clients don't choke on the
// shape variations free-tier providers emit. Complements normalizeOutboundContent
// (which fixes array content) by fixing the *frame* fields, not the content:
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 1e2730db..1c52ad85 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -1481,13 +1481,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
// in parallel, then a judge synthesizes one answer. It routes each panel/judge
// sub-call through the normal path (cooldowns, quotas, analytics), so it
// behaves like a normal model from the client's side — just K+1x the tokens.
- // Vision is still rejected up front; tool requests run on tool-capable panel
- // members and return the first structured tool call directly.
+ // Image requests run on vision-capable panel members; tool requests run on
+ // tool-capable members and return the first structured tool call directly.
if (isFusionModel(requestedModel)) {
- if (hasImage) {
- res.status(422).json({ error: { message: 'Fusion does not support image input yet. Use a vision model directly.', type: 'invalid_request_error', code: 'fusion_no_vision' } });
- return;
- }
const fusionOptions = { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, ...samplingParams };
const fusionConfig = parsed.data.fusion ?? {};
@@ -1511,6 +1507,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
config: fusionConfig,
options: fusionOptions,
estimatedTokens: estimatedTotal,
+ vision: hasImage,
hooks: {
// `a` already carries a sanitized error for failed slots; content is
// the model's own answer and is forwarded as-is.
@@ -1564,6 +1561,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
config: fusionConfig,
options: fusionOptions,
estimatedTokens: estimatedTotal,
+ vision: hasImage,
});
// Structured-output enforcement for fusion (#516 scope gap): the panel/
// judge output got no format check, so model:"fusion" could hand back
diff --git a/server/src/services/fusion.ts b/server/src/services/fusion.ts
index 66fb9355..faf4f2c9 100644
--- a/server/src/services/fusion.ts
+++ b/server/src/services/fusion.ts
@@ -14,7 +14,7 @@ import {
isRetryableError, isRateLimitSignal, isPaymentRequiredError,
isModelNotFoundError, isModelAccessForbiddenError,
} from '../lib/error-classify.js';
-import { contentToString } from '../lib/content.js';
+import { contentToString, stripImagesFromMessages } from '../lib/content.js';
import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';
import { getSetting, setSetting } from '../db/index.js';
import type { CompletionOptions } from '../providers/base.js';
@@ -446,7 +446,7 @@ export function diversifyChain(ordered: FusionCandidate[]): FusionCandidate[] {
* both the panel and its refills span genuinely different perspectives before
* doubling up on either axis.
*/
-function selectPanel(config: FusionConfig, requirements: { requireTools?: boolean; estimatedTokens: number }): { panel: FusionCandidate[]; overflow: FusionCandidate[]; dropped: string[] } {
+export function selectPanel(config: FusionConfig, requirements: { requireTools?: boolean; requireVision?: boolean; estimatedTokens: number }): { panel: FusionCandidate[]; overflow: FusionCandidate[]; dropped: string[] } {
const maxK = panelMaxK();
if (config.models && config.models.length > 0) {
@@ -458,6 +458,7 @@ function selectPanel(config: FusionConfig, requirements: { requireTools?: boolea
const cand = resolveFusionCandidate(id);
if (!cand) { dropped.push(`${id} (unknown or disabled)`); continue; }
if (requirements.requireTools && !cand.supportsTools) { dropped.push(`${id} (no tool-calling support)`); continue; }
+ if (requirements.requireVision && !cand.supportsVision) { dropped.push(`${id} (no vision support)`); continue; }
if (seen.has(cand.modelDbId)) continue; // de-dup repeats
seen.add(cand.modelDbId);
panel.push(cand);
@@ -470,7 +471,8 @@ function selectPanel(config: FusionConfig, requirements: { requireTools?: boolea
// Size-aware: the chain excludes models that cannot hold a prompt this large,
// so a too-small model never claims a slot it is guaranteed to fail.
const ordered = getOrderedFusionChain(requirements.estimatedTokens)
- .filter(c => !requirements.requireTools || c.supportsTools);
+ .filter(c => !requirements.requireTools || c.supportsTools)
+ .filter(c => !requirements.requireVision || c.supportsVision);
// Diversity-first ordering of the whole servable chain along provider AND
// model family (see diversifyChain). The first K are the panel; the rest are
@@ -494,7 +496,7 @@ const JUDGE_SYSTEM_PROMPT =
'Then REWRITE it all from scratch, in your own words, as one clear, well-structured, self-contained answer that makes complete sense by itself. ' +
'Do not mention that other answers exist, do not refer to "Response 1/2/3", do not compare the responses, and do not describe your process — just deliver the final, authoritative answer directly to the user.';
-function buildJudgeMessages(original: ChatMessage[], answers: PanelAnswer[]): ChatMessage[] {
+export function buildJudgeMessages(original: ChatMessage[], answers: PanelAnswer[], stripImages = false): ChatMessage[] {
const ok = answers.filter(a => a.status === 'ok' && a.content);
const panelBlock = ok
.map((a, i) => `--- Response ${i + 1} ---\n${a.content}`)
@@ -505,7 +507,7 @@ function buildJudgeMessages(original: ChatMessage[], answers: PanelAnswer[]): Ch
// prompt leads so it frames everything that follows.
return [
{ role: 'system', content: JUDGE_SYSTEM_PROMPT },
- ...original,
+ ...(stripImages ? stripImagesFromMessages(original) : original),
{
role: 'user',
content:
@@ -553,18 +555,22 @@ export async function runFusion(params: {
options: CompletionOptions;
estimatedTokens: number;
hooks?: FusionHooks;
+ vision?: boolean;
}): Promise<FusionResult> {
- const { messages, options, estimatedTokens, hooks } = params;
+ const { messages, options, estimatedTokens, hooks, vision = false } = params;
// Apply the dashboard-saved default; the request's inline fusion field (if
// any) has already-merged precedence field-by-field.
const config = resolveEffectiveConfig(params.config);
const strategy = config.strategy ?? 'synthesize';
const requireTools = (options.tools?.length ?? 0) > 0;
- const { panel, overflow, dropped } = selectPanel(config, { requireTools, estimatedTokens });
+ const { panel, overflow, dropped } = selectPanel(config, { requireTools, requireVision: vision, estimatedTokens });
if (panel.length === 0) {
+ const hint = vision
+ ? 'No vision-capable model is servable for the panel. Enable a vision model in the Fallback Chain or pass `fusion.models` with vision-capable model ids.'
+ : 'Provide `fusion.models` with enabled model ids, or enable models in the Fallback Chain.';
throw new FusionError(
- 'fusion: no usable models for the panel. Provide `fusion.models` with enabled model ids, or enable models in the Fallback Chain.',
+ `fusion: no usable models for the panel. ${hint}`,
400,
);
}
@@ -695,7 +701,7 @@ export async function runFusion(params: {
// (longest as a cheap proxy for completeness) — no judge call.
finalText = textSurvivors.slice().sort((a, b) => (b.content!.length - a.content!.length))[0].content!;
} else {
- const judgeMessages = buildJudgeMessages(messages, textSurvivors);
+ const judgeMessages = buildJudgeMessages(messages, textSurvivors, vision);
// The judge prompt carries every panel answer, so its input is much larger
// than the original — size the routing estimate accordingly.
const judgeEstimate = estimatedTokens + textSurvivors.reduce((n, a) => n + Math.ceil((a.content?.length ?? 0) / 4), 0);