7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(fusion): support vision (image) requests via vision-capable panel (#884)
Icesenator7b54a7729ae9
diff --git a/server/src/__tests__/lib/content.test.ts b/server/src/__tests__/lib/content.test.tsindex d2ff2d6e..8b875642 100644--- a/server/src/__tests__/lib/content.test.ts+++ b/server/src/__tests__/lib/content.test.ts@@ -1,5 +1,5 @@import { describe, it, expect } from 'vitest';-import { contentToString, flattenMessageContent, messageHasImage, normalizeOutboundContent, sanitizeResponse } from '../../lib/content.js';+import { contentToString, flattenMessageContent, messageHasImage, normalizeOutboundContent, sanitizeResponse, stripImagesFromMessages } from '../../lib/content.js';describe('contentToString', () => {it('passes strings through', () => {@@ -93,6 +93,35 @@ describe('messageHasImage', () => {});});+describe('stripImagesFromMessages', () => {+ it('drops image blocks but keeps text blocks', () => {+ const out = stripImagesFromMessages([+ { role: 'user', content: [+ { type: 'text', text: 'hello' },+ { type: 'image_url', image_url: { url: 'data:image/png;base64,AA==' } },+ ] },+ ]);+ expect(out[0].content).toEqual([{ type: 'text', text: 'hello' }]);+ });++ it('replaces an image-only message with an empty string', () => {+ const out = stripImagesFromMessages([+ { role: 'user', content: [+ { type: 'image_url', image_url: { url: 'data:image/png;base64,AA==' } },+ ] },+ ]);+ expect(out[0].content).toBe('');+ });++ it('leaves string content and plain messages untouched', () => {+ const m1 = { role: 'user', content: 'plain text' };+ const m2 = { role: 'assistant', content: null };+ const out = stripImagesFromMessages([m1 as any, m2 as any]);+ expect(out[0]).toBe(m1); // identity preserved (no unnecessary copy)+ expect(out[1]).toBe(m2);+ });+});+describe('normalizeOutboundContent (#166)', () => {it('coerces array delta.content to a string on streaming chunks', () => {const chunk = { choices: [{ index: 0, delta: { content: [{ type: 'text', text: 'hel' }, { type: 'text', text: 'lo' }] } }] };diff --git a/server/src/__tests__/routes/proxy-vision.test.ts b/server/src/__tests__/routes/proxy-vision.test.tsindex 3f8b21f1..92819f05 100644--- a/server/src/__tests__/routes/proxy-vision.test.ts+++ b/server/src/__tests__/routes/proxy-vision.test.ts@@ -85,4 +85,12 @@ describe('Vision-aware routing (#118, #125)', () => {expect(body?.error?.code).not.toBe('no_vision_model');getDb().prepare('UPDATE models SET enabled = 1 WHERE supports_vision = 1').run();});++ it('lets a fusion image request through routing (no 422 fusion_no_vision)', async () => {+ // Same seed as the auto case: vision models enabled but no provider keys,+ // so routing exhausts. The point is that fusion is NOT rejected up front.+ const { status, body } = await post(app, '/v1/chat/completions', { ...IMAGE_MESSAGE, model: 'fusion' }, key);+ expect(status).not.toBe(422);+ expect(body?.error?.code).not.toBe('fusion_no_vision');+ });});diff --git a/server/src/__tests__/services/fusion.test.ts b/server/src/__tests__/services/fusion.test.tsnew file mode 100644index 00000000..fc170bca--- /dev/null+++ b/server/src/__tests__/services/fusion.test.ts@@ -0,0 +1,90 @@+import { describe, it, expect, beforeAll, beforeEach } from 'vitest';+import { initDb, getDb } from '../../db/index.js';+import { encrypt } from '../../lib/crypto.js';+import { setRoutingStrategy } from '../../services/router.js';++// A fusion unit-test fixture: seed the standard catalog, then add one healthy+// key per platform so the auto-panel chain has servable members (the seed ships+// no keys by default). Priority strategy keeps the panel order deterministic.+function addKey(platform: string): void {+ const { encrypted, iv, authTag } = encrypt(`test-${platform}-key`);+ getDb().prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)+ VALUES (?, 'fusion-vision-test', ?, ?, ?, 'healthy', 1)+ `).run(platform, encrypted, iv, authTag);+}++describe('selectPanel vision filtering', () => {+ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ setRoutingStrategy('priority');+ });++ beforeEach(() => {+ const db = getDb();+ db.prepare('DELETE FROM api_keys').run();+ db.prepare("DELETE FROM settings WHERE key = 'active_profile_id'").run();+ const models = db.prepare('SELECT id, intelligence_rank FROM models ORDER BY intelligence_rank ASC').all() as any[];+ const update = db.prepare('UPDATE fallback_config SET priority = ? WHERE model_db_id = ?');+ for (let i = 0; i < models.length; i++) update.run(i + 1, models[i].id);+ addKey('google');+ addKey('groq');+ });++ it('drops explicit panel models without vision support when requireVision', async () => {+ const { selectPanel } = await import('../../services/fusion.js');+ const config = { models: ['gemini-2.5-flash', 'llama-3.1-8b-instant'], k: 2, judge: null, strategy: 'synthesize' as const, expose_panel: false };+ const { panel, dropped } = selectPanel(config, { requireVision: true, estimatedTokens: 100 });+ expect(panel.every((c) => c.supportsVision === 1)).toBe(true);+ expect(panel.some((c) => c.modelId === 'gemini-2.5-flash')).toBe(true);+ expect(panel.some((c) => c.modelId === 'llama-3.1-8b-instant')).toBe(false);+ expect(dropped).toContain('llama-3.1-8b-instant (no vision support)');+ });++ it('auto panel only contains vision models when requireVision', async () => {+ const { selectPanel } = await import('../../services/fusion.js');+ const config = { k: 4, judge: null, strategy: 'synthesize' as const, expose_panel: false };+ const { panel } = selectPanel(config, { requireVision: true, estimatedTokens: 100 });+ expect(panel.length).toBeGreaterThan(0);+ expect(panel.every((c) => c.supportsVision === 1)).toBe(true);+ });+});++describe('fusion vision judge', () => {+ it('buildJudgeMessages strips image blocks when vision is requested', async () => {+ const { buildJudgeMessages } = await import('../../services/fusion.js');+ const original = [+ { role: 'user' as const, content: [+ { type: 'text', text: 'what is in this image?' },+ { type: 'image_url', image_url: { url: 'data:image/png;base64,AAAA' } },+ ] },+ ];+ const answers = [+ { status: 'ok' as const, content: 'a red square', usage: undefined } as any,+ ];+ const judgeMessages = buildJudgeMessages(original, answers, true);+ const hasImage = judgeMessages.some((m) =>+ Array.isArray(m.content) && m.content.some((b: any) => b?.type === 'image_url' || b?.type === 'image'),+ );+ expect(hasImage).toBe(false);+ });++ it('keeps image blocks by default (no strip flag)', async () => {+ const { buildJudgeMessages } = await import('../../services/fusion.js');+ const original = [+ { role: 'user' as const, content: [+ { type: 'text', text: 'what is in this image?' },+ { type: 'image_url', image_url: { url: 'data:image/png;base64,AAAA' } },+ ] },+ ];+ const answers = [+ { status: 'ok' as const, content: 'a red square', usage: undefined } as any,+ ];+ const judgeMessages = buildJudgeMessages(original, answers);+ const hasImage = judgeMessages.some((m) =>+ Array.isArray(m.content) && m.content.some((b: any) => b?.type === 'image_url' || b?.type === 'image'),+ );+ expect(hasImage).toBe(true);+ });+});diff --git a/server/src/lib/content.ts b/server/src/lib/content.tsindex 50e1f1b0..696eaf47 100644--- a/server/src/lib/content.ts+++ b/server/src/lib/content.ts@@ -57,6 +57,20 @@ export function messageHasImage(messages: ChatMessage[]): boolean {return messages.some((m) => contentHasImage(m.content));}+// Drop image blocks from every message so a text-only model (e.g. the fusion+// judge, which synthesizes from panel text answers) never receives pixels.+export function stripImagesFromMessages(messages: ChatMessage[]): ChatMessage[] {+ return messages.map((m) => {+ if (!Array.isArray(m.content)) return m;+ const kept = m.content.filter((block) => {+ const type = (block as { type?: string })?.type;+ return type !== 'image_url' && type !== 'image';+ });+ if (kept.length === m.content.length) return m;+ return { ...m, content: kept.length > 0 ? kept : '' };+ });+}+// Harden the OUTBOUND envelope so strict OpenAI clients don't choke on the// shape variations free-tier providers emit. Complements normalizeOutboundContent// (which fixes array content) by fixing the *frame* fields, not the content:diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 1e2730db..1c52ad85 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -1481,13 +1481,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {// in parallel, then a judge synthesizes one answer. It routes each panel/judge// sub-call through the normal path (cooldowns, quotas, analytics), so it// behaves like a normal model from the client's side — just K+1x the tokens.- // Vision is still rejected up front; tool requests run on tool-capable panel- // members and return the first structured tool call directly.+ // Image requests run on vision-capable panel members; tool requests run on+ // tool-capable members and return the first structured tool call directly.if (isFusionModel(requestedModel)) {- if (hasImage) {- res.status(422).json({ error: { message: 'Fusion does not support image input yet. Use a vision model directly.', type: 'invalid_request_error', code: 'fusion_no_vision' } });- return;- }const fusionOptions = { temperature, max_tokens, top_p, stop, tools, tool_choice, parallel_tool_calls, ...samplingParams };const fusionConfig = parsed.data.fusion ?? {};@@ -1511,6 +1507,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {config: fusionConfig,options: fusionOptions,estimatedTokens: estimatedTotal,+ vision: hasImage,hooks: {// `a` already carries a sanitized error for failed slots; content is// the model's own answer and is forwarded as-is.@@ -1564,6 +1561,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {config: fusionConfig,options: fusionOptions,estimatedTokens: estimatedTotal,+ vision: hasImage,});// Structured-output enforcement for fusion (#516 scope gap): the panel/// judge output got no format check, so model:"fusion" could hand backdiff --git a/server/src/services/fusion.ts b/server/src/services/fusion.tsindex 66fb9355..faf4f2c9 100644--- a/server/src/services/fusion.ts+++ b/server/src/services/fusion.ts@@ -14,7 +14,7 @@ import {isRetryableError, isRateLimitSignal, isPaymentRequiredError,isModelNotFoundError, isModelAccessForbiddenError,} from '../lib/error-classify.js';-import { contentToString } from '../lib/content.js';+import { contentToString, stripImagesFromMessages } from '../lib/content.js';import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';import { getSetting, setSetting } from '../db/index.js';import type { CompletionOptions } from '../providers/base.js';@@ -446,7 +446,7 @@ export function diversifyChain(ordered: FusionCandidate[]): FusionCandidate[] {* both the panel and its refills span genuinely different perspectives before* doubling up on either axis.*/-function selectPanel(config: FusionConfig, requirements: { requireTools?: boolean; estimatedTokens: number }): { panel: FusionCandidate[]; overflow: FusionCandidate[]; dropped: string[] } {+export function selectPanel(config: FusionConfig, requirements: { requireTools?: boolean; requireVision?: boolean; estimatedTokens: number }): { panel: FusionCandidate[]; overflow: FusionCandidate[]; dropped: string[] } {const maxK = panelMaxK();if (config.models && config.models.length > 0) {@@ -458,6 +458,7 @@ function selectPanel(config: FusionConfig, requirements: { requireTools?: booleaconst cand = resolveFusionCandidate(id);if (!cand) { dropped.push(`${id} (unknown or disabled)`); continue; }if (requirements.requireTools && !cand.supportsTools) { dropped.push(`${id} (no tool-calling support)`); continue; }+ if (requirements.requireVision && !cand.supportsVision) { dropped.push(`${id} (no vision support)`); continue; }if (seen.has(cand.modelDbId)) continue; // de-dup repeatsseen.add(cand.modelDbId);panel.push(cand);@@ -470,7 +471,8 @@ function selectPanel(config: FusionConfig, requirements: { requireTools?: boolea// Size-aware: the chain excludes models that cannot hold a prompt this large,// so a too-small model never claims a slot it is guaranteed to fail.const ordered = getOrderedFusionChain(requirements.estimatedTokens)- .filter(c => !requirements.requireTools || c.supportsTools);+ .filter(c => !requirements.requireTools || c.supportsTools)+ .filter(c => !requirements.requireVision || c.supportsVision);// Diversity-first ordering of the whole servable chain along provider AND// model family (see diversifyChain). The first K are the panel; the rest are@@ -494,7 +496,7 @@ const JUDGE_SYSTEM_PROMPT ='Then REWRITE it all from scratch, in your own words, as one clear, well-structured, self-contained answer that makes complete sense by itself. ' +'Do not mention that other answers exist, do not refer to "Response 1/2/3", do not compare the responses, and do not describe your process — just deliver the final, authoritative answer directly to the user.';-function buildJudgeMessages(original: ChatMessage[], answers: PanelAnswer[]): ChatMessage[] {+export function buildJudgeMessages(original: ChatMessage[], answers: PanelAnswer[], stripImages = false): ChatMessage[] {const ok = answers.filter(a => a.status === 'ok' && a.content);const panelBlock = ok.map((a, i) => `--- Response ${i + 1} ---\n${a.content}`)@@ -505,7 +507,7 @@ function buildJudgeMessages(original: ChatMessage[], answers: PanelAnswer[]): Ch// prompt leads so it frames everything that follows.return [{ role: 'system', content: JUDGE_SYSTEM_PROMPT },- ...original,+ ...(stripImages ? stripImagesFromMessages(original) : original),{role: 'user',content:@@ -553,18 +555,22 @@ export async function runFusion(params: {options: CompletionOptions;estimatedTokens: number;hooks?: FusionHooks;+ vision?: boolean;}): Promise<FusionResult> {- const { messages, options, estimatedTokens, hooks } = params;+ const { messages, options, estimatedTokens, hooks, vision = false } = params;// Apply the dashboard-saved default; the request's inline fusion field (if// any) has already-merged precedence field-by-field.const config = resolveEffectiveConfig(params.config);const strategy = config.strategy ?? 'synthesize';const requireTools = (options.tools?.length ?? 0) > 0;- const { panel, overflow, dropped } = selectPanel(config, { requireTools, estimatedTokens });+ const { panel, overflow, dropped } = selectPanel(config, { requireTools, requireVision: vision, estimatedTokens });if (panel.length === 0) {+ const hint = vision+ ? 'No vision-capable model is servable for the panel. Enable a vision model in the Fallback Chain or pass `fusion.models` with vision-capable model ids.'+ : 'Provide `fusion.models` with enabled model ids, or enable models in the Fallback Chain.';throw new FusionError(- 'fusion: no usable models for the panel. Provide `fusion.models` with enabled model ids, or enable models in the Fallback Chain.',+ `fusion: no usable models for the panel. ${hint}`,400,);}@@ -695,7 +701,7 @@ export async function runFusion(params: {// (longest as a cheap proxy for completeness) — no judge call.finalText = textSurvivors.slice().sort((a, b) => (b.content!.length - a.content!.length))[0].content!;} else {- const judgeMessages = buildJudgeMessages(messages, textSurvivors);+ const judgeMessages = buildJudgeMessages(messages, textSurvivors, vision);// The judge prompt carries every panel answer, so its input is much larger// than the original — size the routing estimate accordingly.const judgeEstimate = estimatedTokens + textSurvivors.reduce((n, a) => n + Math.ceil((a.content?.length ?? 0) / 4), 0);