Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Picture input for a server on this network only when its own model list says it reads pictures (a server without a vision encoder refused whole requests)

Niko12211fe683e951c5
diff --git a/apps/host/server/agent/profile.mjs b/apps/host/server/agent/profile.mjs
index 8879aff89..768bf0f2a 100644
--- a/apps/host/server/agent/profile.mjs
+++ b/apps/host/server/agent/profile.mjs
@@ -233,7 +233,8 @@ export function buildProfile({ local = [], connections = [], gate = true, tools
models: conn.models.map(m => ({
id: m.id, name: m.name || m.id,
contextWindow: m.contextMax ?? ASSUMED_API_CONTEXT, maxTokens: reply(m),
- input: ownServer ? ['text', 'image'] : ['text'],
+ // Pictures only when the server itself says it reads them (the host asks its model list): a server started without a vision encoder refuses the whole request when a picture is in it.
+ input: ownServer && m.vision === true ? ['text', 'image'] : ['text'],
...(ownServer ? { reasoning: true, reasoningEfforts: LOCAL_EFFORTS } : {}),
})),
};
diff --git a/apps/host/server/index.mjs b/apps/host/server/index.mjs
index e9377f29b..0cb65c2f6 100644
--- a/apps/host/server/index.mjs
+++ b/apps/host/server/index.mjs
@@ -48,6 +48,7 @@ import { CONTEXT_TOKENS, VARIANTS, localModels, variant } from './catalog.mjs';
import { browseLocalModels, inspectLocalModel } from './local-models.mjs';
import { EventHub, error, json, readBody, readRaw, uid } from './util.mjs';
import { createUploads } from './chat-uploads.mjs';
+import { createVisionProbe } from './vision-probe.mjs';
import { isPrivateHost } from './agent/llm-shape.mjs';
import { RequestQueue } from './request-queue.mjs';
import { selectSkills, skillInstructions } from './skill-selection.mjs';
@@ -118,6 +119,7 @@ const native=createNativeControl({control,appUpdater,hardwareProbe:()=>(hardware
const llmProxy = createLlmProxy({ targetOf: key => agent.llmTarget(key), samplingNow: () => samplingFromExpert(store.get().expert) });
const agentTools = createAgentTools({ read: () => store.get().agentTools, write: value => store.update({ agentTools: value }), dataDir: store.DATA_DIR, browser: () => ({ ...findBrowser(), cli: findPlaywrightMcp(agent.harnessDir()) }) });
agentTools.ensureFile();
+const visionProbe = createVisionProbe();
const agent = createAgentBridge({
tools: () => agentTools.profileView(),
skillsDir: () => AGENT_SKILLS,
@@ -139,7 +141,7 @@ const agent = createAgentBridge({
// The tool servers set up for this model (Settings, Tools): the agent connects to the same ones.
mcpServers: i.mcpServers,
})),
- connections: () => modelProviders.withSecrets(),
+ connections: () => modelProviders.withSecrets().map(c => ({ ...c, models: c.models.map(m => ({ ...m, vision: visionProbe.known(c, m.id) })) })),
log: line => console.log(line),
});
let storageChanging = false;
@@ -523,7 +525,12 @@ async function runAgentTurn({ res, controller, chatId, body, message, workspaceI
const write = frame => { if (!res.writableEnded && !res.destroyed) res.write(`${JSON.stringify(frame)}\n`); };
const cwd = chatFolders.get(chatId) ?? (workspaceId ? workspaces.find(w => w.id === workspaceId)?.path : null) ?? WORKSPACE;
let choice, instance = null;
- if (body.remoteModel) choice = { connectionId: String(body.remoteModel.connectionId), modelId: String(body.remoteModel.modelId) };
+ if (body.remoteModel) {
+ choice = { connectionId: String(body.remoteModel.connectionId), modelId: String(body.remoteModel.modelId) };
+ // Does this server read pictures? Asked before the agent is built, so a picture is never sent to one that cannot.
+ const own = modelProviders.withSecrets().find(c => c.id === choice.connectionId);
+ if (own && own.protocol === 'openai-chat' && isPrivateHost(own.baseUrl)) await visionProbe.check(own, choice.modelId);
+ }
else {
try { instance = native.resolveInstance(body.nativeModelId ?? native.state().defaultId); }
catch (e) { write({ type: 'error', message: e.message }); return; }
@@ -1031,7 +1038,8 @@ async function api(req, res, path, method, url) {
if (resolve(cwd) === resolve(ROOT) || resolve(cwd).startsWith(resolve(ROOT) + sep)) return error(res, 400, 'Time-skip works on a folder of your own. Choose a workspace folder for this chat, then try again.');
if (measureFolder(cwd).tooBig) return error(res, 400, 'This folder is too big to try a change on a copy. Open a smaller folder for this chat, then try again.');
let choice, slots = 1;
- if (body.remoteModel?.connectionId && body.remoteModel?.modelId) { choice = { connectionId: String(body.remoteModel.connectionId), modelId: String(body.remoteModel.modelId) }; slots = Number(body.slots) >= 1 ? Math.floor(Number(body.slots)) : 1; }
+ if (body.remoteModel?.connectionId && body.remoteModel?.modelId) { choice = { connectionId: String(body.remoteModel.connectionId), modelId: String(body.remoteModel.modelId) };
+ { const own = modelProviders.withSecrets().find(c => c.id === choice.connectionId); if (own && own.protocol === 'openai-chat' && isPrivateHost(own.baseUrl)) await visionProbe.check(own, choice.modelId); } slots = Number(body.slots) >= 1 ? Math.floor(Number(body.slots)) : 1; }
else { try { const inst = native.resolveInstance(native.state().defaultId); choice = { instanceId: inst.id }; slots = inst.metrics?.live?.slots?.length || inst.parallel || 1; } catch { return error(res, 503, 'Load a model first, then try Time-skip again.'); } }
const chatId = typeof body.sessionId === 'string' && body.sessionId ? body.sessionId : null;
if (chatId && forkTs.byChat(chatId)) return error(res, 409, 'A Time-skip is already running in this chat.');
diff --git a/apps/host/server/vision-probe.mjs b/apps/host/server/vision-probe.mjs
new file mode 100644
index 000000000..9d66a51b5
--- /dev/null
+++ b/apps/host/server/vision-probe.mjs
@@ -0,0 +1,31 @@
+/**
+ * Does a model server on this network read pictures? Asked of its own model list (`/models`, `architecture.input_modalities`), cached for a
+ * few minutes. Unknown or unreachable counts as "no": a picture sent to a server without a vision encoder makes the whole request fail.
+ */
+const TTL_MS = 5 * 60_000;
+
+export function createVisionProbe({ fetcher = fetch, now = Date.now } = {}) {
+ const cache = new Map(); // key -> { at, value }
+ const keyOf = (conn, modelId) => `${conn.id}|${modelId}`;
+ return {
+ /** Last known answer for a model (true / false), or undefined when never asked. */
+ known: (conn, modelId) => { const e = cache.get(keyOf(conn, modelId)); return e ? e.value : undefined; },
+ /** Ask now (at most once per TTL per model). */
+ async check(conn, modelId) {
+ const k = keyOf(conn, modelId); const hit = cache.get(k);
+ if (hit && now() - hit.at < TTL_MS) return hit.value;
+ let value = false;
+ try {
+ const res = await fetcher(String(conn.baseUrl).replace(/\/+$/, '') + '/models', { headers: conn.apiKey ? { authorization: `Bearer ${conn.apiKey}` } : {}, signal: AbortSignal.timeout(4000) });
+ if (res.ok) {
+ const list = (await res.json())?.data ?? [];
+ const row = list.find(m => m?.id === modelId) ?? (list.length === 1 ? list[0] : null);
+ const modes = row?.architecture?.input_modalities ?? row?.input_modalities ?? [];
+ value = Array.isArray(modes) && modes.includes('image');
+ }
+ } catch { value = false; }
+ cache.set(k, { at: now(), value });
+ return value;
+ },
+ };
+}
diff --git a/apps/host/tests/agent-shape.test.mjs b/apps/host/tests/agent-shape.test.mjs
index 0094aa933..37d818624 100644
--- a/apps/host/tests/agent-shape.test.mjs
+++ b/apps/host/tests/agent-shape.test.mjs
@@ -7,6 +7,7 @@ import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { BUDGETS, createLlmProxy, isPrivateHost, nearestBudget, samplingFromExpert, shapeBody } from '../server/agent/llm-shape.mjs';
import { buildProfile } from '../server/agent/profile.mjs';
+import { createVisionProbe } from '../server/vision-probe.mjs';
import { startScriptedWorker } from './helpers/scripted-worker.mjs';
import { createAgentBridge } from '../server/agent/agent-bridge.mjs';
import { resolveHarness } from '../server/agent/harness-runtime.mjs';
@@ -126,3 +127,16 @@ test('a turn uses the response-settings route only when a setting is changed: th
await run('s8', { thinking: 'Medium' });
assert.equal(last().temperature, undefined, 'back to defaults: straight to the model again');
});
+
+test('a model server on this network gets picture input only when its own model list says it reads pictures', async () => {
+ const lists = { yes: { data: [{ id: 'm', architecture: { input_modalities: ['text', 'image'] } }] }, no: { data: [{ id: 'm', architecture: { input_modalities: ['text'] } }] } };
+ let which = 'yes';
+ const probe = createVisionProbe({ fetcher: async () => ({ ok: true, json: async () => lists[which] }) });
+ const conn = { id: 'c', baseUrl: 'http://127.0.0.1:1/v1', apiKey: '' };
+ assert.equal(probe.known(conn, 'm'), undefined);
+ assert.equal(await probe.check(conn, 'm'), true); assert.equal(probe.known(conn, 'm'), true);
+ which = 'no'; assert.equal(await probe.check(conn, 'm'), true, 'cached for a few minutes');
+ assert.equal(await createVisionProbe({ fetcher: async () => { throw new Error('down'); } }).check(conn, 'm'), false, 'unreachable counts as no');
+ const build = vision => JSON.parse(buildProfile({ local: [], connections: [{ id: 'own', name: 'Own', baseUrl: 'http://127.0.0.1:18798/v1', protocol: 'openai-chat', apiKey: '', models: [{ id: 'm', contextMax: 32768, vision }] }] }).patch.slice(buildProfile({ local: [], connections: [] }).patch.indexOf('['))).find(e => e.id === 'llm-pi-ai').config.providers['api-own'].models[0].input;
+ assert.deepEqual(build(true), ['text', 'image']); assert.deepEqual(build(false), ['text']); assert.deepEqual(build(undefined), ['text']);
+});