Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Picture input for a server on this network only when its own model list says it reads pictures (a server without a vision encoder refused whole requests)
Niko12211fe683e951c5
diff --git a/apps/host/server/agent/profile.mjs b/apps/host/server/agent/profile.mjsindex 8879aff89..768bf0f2a 100644--- a/apps/host/server/agent/profile.mjs+++ b/apps/host/server/agent/profile.mjs@@ -233,7 +233,8 @@ export function buildProfile({ local = [], connections = [], gate = true, toolsmodels: conn.models.map(m => ({id: m.id, name: m.name || m.id,contextWindow: m.contextMax ?? ASSUMED_API_CONTEXT, maxTokens: reply(m),- input: ownServer ? ['text', 'image'] : ['text'],+ // Pictures only when the server itself says it reads them (the host asks its model list): a server started without a vision encoder refuses the whole request when a picture is in it.+ input: ownServer && m.vision === true ? ['text', 'image'] : ['text'],...(ownServer ? { reasoning: true, reasoningEfforts: LOCAL_EFFORTS } : {}),})),};diff --git a/apps/host/server/index.mjs b/apps/host/server/index.mjsindex e9377f29b..0cb65c2f6 100644--- a/apps/host/server/index.mjs+++ b/apps/host/server/index.mjs@@ -48,6 +48,7 @@ import { CONTEXT_TOKENS, VARIANTS, localModels, variant } from './catalog.mjs';import { browseLocalModels, inspectLocalModel } from './local-models.mjs';import { EventHub, error, json, readBody, readRaw, uid } from './util.mjs';import { createUploads } from './chat-uploads.mjs';+import { createVisionProbe } from './vision-probe.mjs';import { isPrivateHost } from './agent/llm-shape.mjs';import { RequestQueue } from './request-queue.mjs';import { selectSkills, skillInstructions } from './skill-selection.mjs';@@ -118,6 +119,7 @@ const native=createNativeControl({control,appUpdater,hardwareProbe:()=>(hardwareconst llmProxy = createLlmProxy({ targetOf: key => agent.llmTarget(key), samplingNow: () => samplingFromExpert(store.get().expert) });const agentTools = createAgentTools({ read: () => store.get().agentTools, write: value => store.update({ agentTools: value }), dataDir: store.DATA_DIR, browser: () => ({ ...findBrowser(), cli: findPlaywrightMcp(agent.harnessDir()) }) });agentTools.ensureFile();+const visionProbe = createVisionProbe();const agent = createAgentBridge({tools: () => agentTools.profileView(),skillsDir: () => AGENT_SKILLS,@@ -139,7 +141,7 @@ const agent = createAgentBridge({// The tool servers set up for this model (Settings, Tools): the agent connects to the same ones.mcpServers: i.mcpServers,})),- connections: () => modelProviders.withSecrets(),+ connections: () => modelProviders.withSecrets().map(c => ({ ...c, models: c.models.map(m => ({ ...m, vision: visionProbe.known(c, m.id) })) })),log: line => console.log(line),});let storageChanging = false;@@ -523,7 +525,12 @@ async function runAgentTurn({ res, controller, chatId, body, message, workspaceIconst write = frame => { if (!res.writableEnded && !res.destroyed) res.write(`${JSON.stringify(frame)}\n`); };const cwd = chatFolders.get(chatId) ?? (workspaceId ? workspaces.find(w => w.id === workspaceId)?.path : null) ?? WORKSPACE;let choice, instance = null;- if (body.remoteModel) choice = { connectionId: String(body.remoteModel.connectionId), modelId: String(body.remoteModel.modelId) };+ if (body.remoteModel) {+ choice = { connectionId: String(body.remoteModel.connectionId), modelId: String(body.remoteModel.modelId) };+ // Does this server read pictures? Asked before the agent is built, so a picture is never sent to one that cannot.+ const own = modelProviders.withSecrets().find(c => c.id === choice.connectionId);+ if (own && own.protocol === 'openai-chat' && isPrivateHost(own.baseUrl)) await visionProbe.check(own, choice.modelId);+ }else {try { instance = native.resolveInstance(body.nativeModelId ?? native.state().defaultId); }catch (e) { write({ type: 'error', message: e.message }); return; }@@ -1031,7 +1038,8 @@ async function api(req, res, path, method, url) {if (resolve(cwd) === resolve(ROOT) || resolve(cwd).startsWith(resolve(ROOT) + sep)) return error(res, 400, 'Time-skip works on a folder of your own. Choose a workspace folder for this chat, then try again.');if (measureFolder(cwd).tooBig) return error(res, 400, 'This folder is too big to try a change on a copy. Open a smaller folder for this chat, then try again.');let choice, slots = 1;- if (body.remoteModel?.connectionId && body.remoteModel?.modelId) { choice = { connectionId: String(body.remoteModel.connectionId), modelId: String(body.remoteModel.modelId) }; slots = Number(body.slots) >= 1 ? Math.floor(Number(body.slots)) : 1; }+ if (body.remoteModel?.connectionId && body.remoteModel?.modelId) { choice = { connectionId: String(body.remoteModel.connectionId), modelId: String(body.remoteModel.modelId) };+ { const own = modelProviders.withSecrets().find(c => c.id === choice.connectionId); if (own && own.protocol === 'openai-chat' && isPrivateHost(own.baseUrl)) await visionProbe.check(own, choice.modelId); } slots = Number(body.slots) >= 1 ? Math.floor(Number(body.slots)) : 1; }else { try { const inst = native.resolveInstance(native.state().defaultId); choice = { instanceId: inst.id }; slots = inst.metrics?.live?.slots?.length || inst.parallel || 1; } catch { return error(res, 503, 'Load a model first, then try Time-skip again.'); } }const chatId = typeof body.sessionId === 'string' && body.sessionId ? body.sessionId : null;if (chatId && forkTs.byChat(chatId)) return error(res, 409, 'A Time-skip is already running in this chat.');diff --git a/apps/host/server/vision-probe.mjs b/apps/host/server/vision-probe.mjsnew file mode 100644index 000000000..9d66a51b5--- /dev/null+++ b/apps/host/server/vision-probe.mjs@@ -0,0 +1,31 @@+/**+ * Does a model server on this network read pictures? Asked of its own model list (`/models`, `architecture.input_modalities`), cached for a+ * few minutes. Unknown or unreachable counts as "no": a picture sent to a server without a vision encoder makes the whole request fail.+ */+const TTL_MS = 5 * 60_000;++export function createVisionProbe({ fetcher = fetch, now = Date.now } = {}) {+ const cache = new Map(); // key -> { at, value }+ const keyOf = (conn, modelId) => `${conn.id}|${modelId}`;+ return {+ /** Last known answer for a model (true / false), or undefined when never asked. */+ known: (conn, modelId) => { const e = cache.get(keyOf(conn, modelId)); return e ? e.value : undefined; },+ /** Ask now (at most once per TTL per model). */+ async check(conn, modelId) {+ const k = keyOf(conn, modelId); const hit = cache.get(k);+ if (hit && now() - hit.at < TTL_MS) return hit.value;+ let value = false;+ try {+ const res = await fetcher(String(conn.baseUrl).replace(/\/+$/, '') + '/models', { headers: conn.apiKey ? { authorization: `Bearer ${conn.apiKey}` } : {}, signal: AbortSignal.timeout(4000) });+ if (res.ok) {+ const list = (await res.json())?.data ?? [];+ const row = list.find(m => m?.id === modelId) ?? (list.length === 1 ? list[0] : null);+ const modes = row?.architecture?.input_modalities ?? row?.input_modalities ?? [];+ value = Array.isArray(modes) && modes.includes('image');+ }+ } catch { value = false; }+ cache.set(k, { at: now(), value });+ return value;+ },+ };+}diff --git a/apps/host/tests/agent-shape.test.mjs b/apps/host/tests/agent-shape.test.mjsindex 0094aa933..37d818624 100644--- a/apps/host/tests/agent-shape.test.mjs+++ b/apps/host/tests/agent-shape.test.mjs@@ -7,6 +7,7 @@ import { tmpdir } from 'node:os';import { join } from 'node:path';import { BUDGETS, createLlmProxy, isPrivateHost, nearestBudget, samplingFromExpert, shapeBody } from '../server/agent/llm-shape.mjs';import { buildProfile } from '../server/agent/profile.mjs';+import { createVisionProbe } from '../server/vision-probe.mjs';import { startScriptedWorker } from './helpers/scripted-worker.mjs';import { createAgentBridge } from '../server/agent/agent-bridge.mjs';import { resolveHarness } from '../server/agent/harness-runtime.mjs';@@ -126,3 +127,16 @@ test('a turn uses the response-settings route only when a setting is changed: thawait run('s8', { thinking: 'Medium' });assert.equal(last().temperature, undefined, 'back to defaults: straight to the model again');});++test('a model server on this network gets picture input only when its own model list says it reads pictures', async () => {+ const lists = { yes: { data: [{ id: 'm', architecture: { input_modalities: ['text', 'image'] } }] }, no: { data: [{ id: 'm', architecture: { input_modalities: ['text'] } }] } };+ let which = 'yes';+ const probe = createVisionProbe({ fetcher: async () => ({ ok: true, json: async () => lists[which] }) });+ const conn = { id: 'c', baseUrl: 'http://127.0.0.1:1/v1', apiKey: '' };+ assert.equal(probe.known(conn, 'm'), undefined);+ assert.equal(await probe.check(conn, 'm'), true); assert.equal(probe.known(conn, 'm'), true);+ which = 'no'; assert.equal(await probe.check(conn, 'm'), true, 'cached for a few minutes');+ assert.equal(await createVisionProbe({ fetcher: async () => { throw new Error('down'); } }).check(conn, 'm'), false, 'unreachable counts as no');+ const build = vision => JSON.parse(buildProfile({ local: [], connections: [{ id: 'own', name: 'Own', baseUrl: 'http://127.0.0.1:18798/v1', protocol: 'openai-chat', apiKey: '', models: [{ id: 'm', contextMax: 32768, vision }] }] }).patch.slice(buildProfile({ local: [], connections: [] }).patch.indexOf('['))).find(e => e.id === 'llm-pi-ai').config.providers['api-own'].models[0].input;+ assert.deepEqual(build(true), ['text', 'image']); assert.deepEqual(build(false), ['text']); assert.deepEqual(build(undefined), ['text']);+});