Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Pictures reach models: catalog marks picture-capable models, API models take image_url, import knows picture readers

Niko1221a3cf8a647bda
diff --git a/apps/desktop/scripts/pics-e2e.mjs b/apps/desktop/scripts/pics-e2e.mjs
new file mode 100644
index 000000000..7bd08f678
--- /dev/null
+++ b/apps/desktop/scripts/pics-e2e.mjs
@@ -0,0 +1,172 @@
+#!/usr/bin/env node
+/**
+ * Attached pictures reaching a model, end to end on a test box, through the installed app layout (packaged host + bundled Control + an engine
+ * component that carries strata-vision):
+ * 1. the model catalog marks the picture-capable models (images: true) when the picture reader is installed;
+ * 2. API models: a llama-server with a small picture model (--mmproj) is saved as a Custom model API; the host works out that it reads pictures,
+ * and a real picture goes out as image_url and is answered. A server without a picture reader, a model set "not to read pictures", and an API
+ * of another style each get a plain refusal (409) that says how to fix it; ticking "This model reads pictures" lets the picture through;
+ * 3. a local model (unless --skip-local): Download and prepare with Read pictures ON, load, and a picture through the app's own chat.
+ *
+ * node pics-e2e.mjs --install <node/ host/ control/> --components <engine components> --work <scratch> --image shapes.png \
+ * --vlm http://127.0.0.1:18090/v1 --vlm-model smolvlm --text-only http://127.0.0.1:18091/v1 [--skip-local --family qwen --model IQ3_XXS --data <folder> --device cpu]
+ */
+import { spawn } from 'node:child_process';
+import { existsSync, mkdirSync, readFileSync, readdirSync } from 'node:fs';
+import { createServer } from 'node:net';
+import { join, resolve } from 'node:path';
+
+const args = Object.fromEntries(process.argv.slice(2).reduce((acc, a, i, all) => (a.startsWith('--') ? [...acc, [a.slice(2), all[i + 1]?.startsWith('--') || all[i + 1] === undefined ? true : all[i + 1]]] : acc), []));
+for (const need of ['install', 'components', 'work', 'image', 'vlm']) if (!args[need]) { console.error('missing --' + need); process.exit(2); }
+const install = resolve(args.install), work = resolve(args.work), components = resolve(args.components);
+const skipLocal = args['skip-local'] === true, device = String(args.device ?? 'cpu'), family = String(args.family ?? 'qwen'), modelId = String(args.model ?? 'IQ3_XXS');
+const node = join(install, 'node', 'node'), hostEntry = join(install, 'host', 'server', 'index.mjs');
+const dirs = { models: join(work, 'models'), profiles: join(work, 'profiles'), runtime: join(work, 'runtime'), home: join(work, 'home'), xdg: join(work, 'xdg'), appdata: join(work, 'appdata') };
+for (const d of [dirs.models, dirs.home, dirs.xdg, dirs.appdata]) mkdirSync(d, { recursive: true });
+const delay = ms => new Promise(r => setTimeout(r, ms));
+let failed = false;
+const record = (name, ok, detail = '') => { if (!ok) failed = true; console.log(`${ok ? 'ok ' : 'FAIL'} ${name}${detail ? ' ' + String(detail).slice(0, 600) : ''}`); };
+const freePort = async () => { const s = createServer(); await new Promise(r => s.listen(0, '127.0.0.1', r)); const p = s.address().port; await new Promise(r => s.close(r)); return p; };
+const env = { ...process.env, STRATA_COMPONENTS_DIR: components, STRATA_PROFILE_DIR: dirs.profiles, STRATA_RUNTIME_DIR: dirs.runtime, STRATA_INSTALL_DIR: dirs.models,
+ STRATA_HOME: dirs.home, STRATA_MODELS_DIR: join(work, 'models-unused'), APPDATA: dirs.appdata, XDG_CONFIG_HOME: dirs.xdg, PYTHONDONTWRITEBYTECODE: '1' };
+delete env.STRATA_API_KEY;
+const port = await freePort();
+const base = `http://127.0.0.1:${port}`;
+const host = spawn(node, [hostEntry], { cwd: join(install, 'host'), stdio: ['ignore', 'pipe', 'pipe'],
+ env: { ...env, STRATA_PORT: String(port), STRATA_DIST: join(install, 'host', 'dist'), NODE_ENV: 'production', STRATA_PACKAGED: '1', STRATA_REAL_ENGINE: '1', STRATA_NO_ACQUIRE: '1',
+ STRATA_SHARE_PORT: String(await freePort()), STRATA_TUNNEL_PORT: String(await freePort()) } });
+let hostLog = '';
+host.stdout.on('data', c => { hostLog += c; }); host.stderr.on('data', c => { hostLog += c; });
+const call = async (path, method = 'GET', body) => {
+ const response = await fetch(base + path, { method, headers: { 'content-type': 'application/json' }, body: body === undefined ? undefined : JSON.stringify(body) });
+ return { status: response.status, body: await response.json().catch(() => null) };
+};
+const png = 'data:image/png;base64,' + readFileSync(resolve(args.image)).toString('base64');
+const QUESTION = 'Name each shape in this picture with its colour, and read out any text.';
+
+/** One chat turn through the app's own endpoint: the plain-text answer, or the refusal (status + message). */
+async function chat(extra, message = QUESTION) {
+ const response = await fetch(base + '/api/chat/stream', { method: 'POST', headers: { 'content-type': 'application/json' },
+ body: JSON.stringify({ message, agent: false, images: [{ url: png, name: 'shapes.png' }], ...extra }) });
+ if (response.status !== 200) { const b = await response.json().catch(() => ({})); return { status: response.status, message: b.error ?? b.message ?? '' }; }
+ let text = '', error = '';
+ for (const line of (await response.text()).split('\n')) {
+ if (!line.trim()) continue;
+ let f; try { f = JSON.parse(line); } catch { continue; }
+ if (f.type === 'error') error = f.message ?? 'error';
+ else if (typeof f.text === 'string' && f.type !== 'reasoning' && f.type !== 'thinking') text += f.text;
+ }
+ return { status: 200, text, error };
+}
+const saveApi = async (name, baseUrl, models, protocol = 'openai-chat') => {
+ const existing = (await call('/api/model-providers')).body.connections.find(c => c.name === name);
+ return (await call('/api/model-providers', 'POST', { ...(existing ? { id: existing.id } : {}), kind: 'custom', provider: null, name, baseUrl, protocol, apiKey: '', models })).body;
+};
+async function readsAfter(connectionName, id, want, seconds = 20) {
+ for (let i = 0; i < seconds * 2; i++) {
+ const c = (await call('/api/model-providers')).body.connections.find(x => x.name === connectionName);
+ const m = c?.models.find(x => x.id === id);
+ if (m && m.reads === want) return { c, m };
+ await delay(500);
+ }
+ const c = (await call('/api/model-providers')).body.connections.find(x => x.name === connectionName);
+ return { c, m: c?.models.find(x => x.id === id) };
+}
+async function waitJob(id, seconds) {
+ let last = '';
+ for (let i = 0; i < seconds * 2; i++) {
+ const state = (await call('/api/native')).body;
+ const job = state.jobs.find(j => j.id === id);
+ if (job && job.summary !== last) { last = job.summary; console.log(' ... ' + String(last).slice(0, 120)); }
+ if (job && job.state !== 'running') return { job, state };
+ await delay(500);
+ }
+ throw new Error('operation did not finish');
+}
+const names = low => /(red|blue|green|circle|square|triangle|strata|42)/.test(low);
+
+async function main() {
+ let up = false;
+ for (let i = 0; i < 200 && !up; i++) { try { up = (await fetch(base + '/api/health')).ok; } catch { /* not yet */ } if (!up) await delay(150); }
+ record('host healthy', up, up ? base : hostLog.slice(-500));
+ if (!up) return;
+
+ // ---- 1. the catalog
+ const packages = existsSync(join(components, 'engine')) ? readdirSync(join(components, 'engine')) : [];
+ const reader = packages.some(p => existsSync(join(components, 'engine', p, 'strata-vision')));
+ const native = (await call('/api/native?refresh=1')).body;
+ record('the engine component carries the picture reader and the host knows it (visionReader)', reader && native.visionReader === true, `component=${reader} host=${native.visionReader}`);
+ const rows = (native.catalog?.families ?? []).flatMap(f => (f.sizes ?? []).map(s => ({ family: f.title ?? f.family, size: s.id, images: s.images, capable: s.imagesCapable })));
+ console.log(' catalog (family / size / images / imagesCapable):');
+ for (const r of rows) console.log(` ${String(r.family).padEnd(26)} ${String(r.size).padEnd(30)} images=${r.images} capable=${r.capable}`);
+ record('some models in the catalog read pictures', rows.some(r => r.images === true), `${rows.filter(r => r.images).length} of ${rows.length}`);
+ record('every picture-capable model reads pictures when the reader is installed', rows.every(r => r.capable === r.images), '');
+ const flash = rows.filter(r => /flash-next/i.test(`${r.family} ${r.size}`) && !/gyro/i.test(`${r.family} ${r.size}`));
+ record('the Qwen3.8-Flash-Next models are marked images: true', flash.length > 0 && flash.every(r => r.images === true), flash.map(r => r.size + '=' + r.images).join(' '));
+
+ // ---- 2. API models
+ const smol = args['vlm-model'] ?? 'smolvlm';
+ await saveApi('Small picture model', args.vlm, [{ id: smol, name: 'SmolVLM' }]);
+ const found = await readsAfter('Small picture model', smol, true);
+ record('the host works out that the llama-server reads pictures (no tick needed)', found.m?.reads === true, JSON.stringify(found.m));
+ const conn = found.c;
+ const sent = await chat({ remoteModel: { connectionId: conn?.id, modelId: smol } });
+ console.log(` answer from the API model: ${JSON.stringify((sent.text ?? sent.message ?? '').slice(0, 300))}`);
+ record('the picture goes to the API model and is answered', sent.status === 200 && !sent.error && (sent.text ?? '').trim().length > 0 && names((sent.text ?? '').toLowerCase()), `status=${sent.status} err=${sent.error ?? ''}`);
+
+ if (args['text-only']) {
+ await saveApi('Text only server', args['text-only'], [{ id: 'plain', name: 'Plain model' }]);
+ const t = await readsAfter('Text only server', 'plain', false);
+ record('a server without a picture reader is known not to read pictures', t.m?.reads === false, JSON.stringify(t.m));
+ const refused = await chat({ remoteModel: { connectionId: t.c.id, modelId: 'plain' } });
+ console.log(` refusal: ${JSON.stringify(refused.message)}`);
+ record('a picture for it is refused in plain words that say how to fix it', refused.status === 409 && /This model reads pictures/.test(refused.message ?? '') && /Settings/.test(refused.message ?? ''), `${refused.status}`);
+ const ticked = await saveApi('Text only server', args['text-only'], [{ id: 'plain', name: 'Plain model', vision: true }]);
+ const after = (await call('/api/model-providers')).body.connections.find(x => x.name === 'Text only server');
+ record('ticking "This model reads pictures" is saved and makes the model read pictures', after?.models[0]?.vision === true && after?.models[0]?.reads === true, JSON.stringify(after?.models[0]));
+ const through = await chat({ remoteModel: { connectionId: after.id, modelId: 'plain' } });
+ record('with the tick the picture is sent on (the server itself decides what to do with it)', through.status === 200, `${through.status} ${through.message ?? through.error ?? ''}`);
+ }
+
+ await saveApi('Small picture model', args.vlm, [{ id: smol, name: 'SmolVLM', vision: false }]);
+ const off = await readsAfter('Small picture model', smol, false);
+ const notRead = await chat({ remoteModel: { connectionId: off.c.id, modelId: smol } });
+ record('a model set not to read pictures refuses them and says where to change it', notRead.status === 409 && /set not to read pictures/.test(notRead.message ?? ''), `${notRead.status} ${notRead.message ?? ''}`);
+ await saveApi('Small picture model', args.vlm, [{ id: smol, name: 'SmolVLM', vision: null }]);
+
+ await saveApi('Other style', 'https://api.anthropic.com/v1', [{ id: 'claude-x', name: 'Claude' }], 'anthropic');
+ const other = (await call('/api/model-providers')).body.connections.find(x => x.name === 'Other style');
+ const style = await chat({ remoteModel: { connectionId: other.id, modelId: 'claude-x' } });
+ console.log(` other style: ${JSON.stringify(style.message)}`);
+ record('an API of another style says plainly why pictures stay in the preview', style.status === 409 && /OpenAI/.test(style.message ?? ''), `${style.status}`);
+
+ if (skipLocal) return;
+
+ // ---- 3. a local model
+ const values = { family, model: modelId, context: 32768, vision: 'yes', loadVision: device, ...(args.data ? { data_dir: resolve(args.data) } : {}) };
+ const started = await call('/api/native/operations', 'POST', { kind: 'install', args: values });
+ record('the install operation starts', started.status === 200 || started.status === 202, JSON.stringify(started.body).slice(0, 200));
+ const done = await waitJob(started.body.id, 3600);
+ record('Download and prepare finishes', done.job.state === 'done', `${done.job.state} ${done.job.error ?? done.job.summary}`);
+ const refreshed = (await call('/api/native?refresh=1')).body;
+ const model = refreshed.installed?.models?.find(m => [m.id, m.model, m.tag].some(x => String(x ?? '').toLowerCase() === modelId.toLowerCase()));
+ record('the model is prepared with its picture reader', !!model?.visionPrepared, JSON.stringify({ ready: model?.ready, visionPrepared: model?.visionPrepared }));
+ const load = await call('/api/native/operations', 'POST', { kind: 'load', args: { model: model?.model ?? modelId.toLowerCase(), vision: device } });
+ const loaded = await waitJob(load.body.id, 900);
+ const instance = loaded.state.instances[0];
+ record('the model loads with the picture reader', loaded.job.state === 'done' && instance?.status === 'ready' && instance?.vision === true, `${loaded.job.state} ${loaded.job.error ?? ''} vision=${instance?.vision}`);
+ const t0 = Date.now();
+ let local = { status: 0 };
+ for (let i = 0; i < 2 && !(local.text ?? '').trim(); i++) local = await chat({ nativeModelId: instance?.id, maxTokens: 4096 });
+ console.log(` answer from the local model (${Math.round((Date.now() - t0) / 1000)} s): ${JSON.stringify((local.text ?? local.message ?? '').slice(0, 300))}`);
+ const low = (local.text ?? '').toLowerCase();
+ record('the local model answers about the picture (names the shapes)', local.status === 200 && /red|circle/.test(low) && /blue|square/.test(low) && /green|triangle/.test(low), `status=${local.status} err=${local.error ?? ''}`);
+ record('the local model reads STRATA 42', low.includes('42'));
+ const unloaded = await call(`/api/native/instances/${instance?.id}/unload`, 'POST', {});
+ record('unload', unloaded.status === 200, unloaded.status);
+}
+try { await main(); } catch (e) { record('run', false, e.stack ?? e); }
+host.kill();
+await delay(1500);
+console.log(failed ? 'PICS-E2E FAIL' : 'PICS-E2E PASS');
+process.exit(failed ? 1 : 0);
diff --git a/apps/host/server/agent/profile.mjs b/apps/host/server/agent/profile.mjs
index 768bf0f2a..b70fb63ae 100644
--- a/apps/host/server/agent/profile.mjs
+++ b/apps/host/server/agent/profile.mjs
@@ -234,7 +234,8 @@ export function buildProfile({ local = [], connections = [], gate = true, tools
id: m.id, name: m.name || m.id,
contextWindow: m.contextMax ?? ASSUMED_API_CONTEXT, maxTokens: reply(m),
// Pictures only when the server itself says it reads them (the host asks its model list): a server started without a vision encoder refuses the whole request when a picture is in it.
- input: ownServer && m.vision === true ? ['text', 'image'] : ['text'],
+ // (any OpenAI-style API: a cloud model the person marked, or whose model list says it reads pictures, takes image_url the same way)
+ input: protocol === 'openai-completions' && m.vision === true ? ['text', 'image'] : ['text'],
...(ownServer ? { reasoning: true, reasoningEfforts: LOCAL_EFFORTS } : {}),
})),
};
diff --git a/apps/host/server/index.mjs b/apps/host/server/index.mjs
index 63210c21f..fd2ec429b 100644
--- a/apps/host/server/index.mjs
+++ b/apps/host/server/index.mjs
@@ -48,8 +48,7 @@ import { CONTEXT_TOKENS, VARIANTS, localModels, variant } from './catalog.mjs';
import { browseLocalModels, inspectLocalModel } from './local-models.mjs';
import { EventHub, error, json, readBody, readRaw, uid } from './util.mjs';
import { createUploads } from './chat-uploads.mjs';
-import { createVisionProbe } from './vision-probe.mjs';
-import { isPrivateHost } from './agent/llm-shape.mjs';
+import { createVisionProbe, effectiveVision } from './vision-probe.mjs';
import { RequestQueue } from './request-queue.mjs';
import { selectSkills, skillInstructions } from './skill-selection.mjs';
import { createModelProviders } from './model-providers.mjs';
@@ -104,7 +103,7 @@ const WORKSPACE_NAME = process.env.STRATA_WORKSPACE ? basename(WORKSPACE) : 'Str
const insideApp = path => { const rel = relative(ROOT, resolve(path)); return rel === '' || (!rel.startsWith('..') && !isAbsolute(rel)); };
const hub = new EventHub();
-const modelProviders = createModelProviders({ directory: store.DATA_DIR, changed: value => hub.send('providers', value) });
+const modelProviders = createModelProviders({ directory: store.DATA_DIR, changed: value => hub.send('providers', withPictureAnswers(value)) });
const imageModels = createImageModels({ dataDir: store.DATA_DIR, modelsDir: () => store.MODELS_DIR, real: models.REAL, changed: value => hub.send('image-models', value) });
/* The installed app ships Strata Control (python + strata_control) next to the host; a source checkout has none and keeps its older paths. */
const CONTROL = findControl();
@@ -127,6 +126,26 @@ const llmProxy = createLlmProxy({ targetOf: key => agent.llmTarget(key), samplin
const agentTools = createAgentTools({ read: () => store.get().agentTools, write: value => store.update({ agentTools: value }), dataDir: store.DATA_DIR, browser: () => ({ ...findBrowser(), cli: findPlaywrightMcp(agent.harnessDir()) }) });
agentTools.ensureFile();
const visionProbe = createVisionProbe();
+/** The model APIs as the chat picker sees them: each model says whether it reads pictures (what the person ticked, else what the server answered). Unanswered models are asked in the background. */
+function withPictureAnswers(list) {
+ const secrets = new Map(modelProviders.withSecrets().map(c => [c.id, c]));
+ const connections = (list.connections ?? []).map(c => ({ ...c, models: c.models.map(m => {
+ const asked = visionProbe.known(c, m.id);
+ if (m.vision == null && asked === undefined && c.protocol === 'openai-chat' && secrets.has(c.id)) void visionProbe.check(secrets.get(c.id), m.id).catch(() => {});
+ return { ...m, reads: effectiveVision(m, asked) ?? null };
+ }) }));
+ return { ...list, connections };
+}
+/** Why a picture cannot go to this API model, in plain words and with the way to fix it; null when it can. */
+async function pictureRefusal(connection, model) {
+ if (!connection || !model) return 'This API model is no longer available. Choose another model in the chat picker.';
+ if (connection.protocol !== 'openai-chat') return `Pictures can only be sent to model APIs that use the OpenAI chat style. "${connection.name}" uses a different style, so the picture stays in the preview. Add the model through an OpenAI-compatible address (a /v1 address) to send pictures.`;
+ const answer = effectiveVision(model, await visionProbe.check(connection, model.id));
+ if (answer === true) return null;
+ return answer === false
+ ? `"${model.name}" is set not to read pictures. If it does, open Settings \u2192 Models, edit "${connection.name}" and tick "This model reads pictures".`
+ : `Strata cannot tell that "${model.name}" reads pictures. If it is a picture-reading model, open Settings \u2192 Models, edit "${connection.name}" and tick "This model reads pictures". Otherwise choose a model that reads pictures.`;
+}
const agent = createAgentBridge({
tools: () => agentTools.profileView(),
skillsDir: () => AGENT_SKILLS,
@@ -148,7 +167,7 @@ const agent = createAgentBridge({
// The tool servers set up for this model (Settings, Tools): the agent connects to the same ones.
mcpServers: i.mcpServers,
})),
- connections: () => modelProviders.withSecrets().map(c => ({ ...c, models: c.models.map(m => ({ ...m, vision: visionProbe.known(c, m.id) })) })),
+ connections: () => modelProviders.withSecrets().map(c => ({ ...c, models: c.models.map(m => ({ ...m, vision: effectiveVision(m, visionProbe.known(c, m.id)) })) })),
log: line => console.log(line),
});
let storageChanging = false;
@@ -536,7 +555,7 @@ async function runAgentTurn({ res, controller, chatId, body, message, workspaceI
choice = { connectionId: String(body.remoteModel.connectionId), modelId: String(body.remoteModel.modelId) };
// Does this server read pictures? Asked before the agent is built, so a picture is never sent to one that cannot.
const own = modelProviders.withSecrets().find(c => c.id === choice.connectionId);
- if (own && own.protocol === 'openai-chat' && isPrivateHost(own.baseUrl)) await visionProbe.check(own, choice.modelId);
+ if (own && own.protocol === 'openai-chat') await visionProbe.check(own, choice.modelId);
}
else {
try { instance = native.resolveInstance(body.nativeModelId ?? native.state().defaultId); }
@@ -959,7 +978,7 @@ async function api(req, res, path, method, url) {
// Credentials are handled only on the host. Responses and events expose a key-set flag.
return error(res,404,'Unknown native control action.');}catch(e){return error(res,/Finish|Unload|running|current operation/.test(e.message)?409:400,e.message);}}
- if (path === '/api/model-providers' && method === 'GET') return json(res, 200, modelProviders.list());
+ if (path === '/api/model-providers' && method === 'GET') return json(res, 200, withPictureAnswers(modelProviders.list()));
if (path === '/api/model-providers/discover' && method === 'POST') {
const body = await readBody(req);
const controller = new AbortController();
@@ -968,11 +987,11 @@ async function api(req, res, path, method, url) {
catch (err) { return error(res, 400, err.name === 'TimeoutError' ? 'The model API timed out. Check the address and try again.' : err.message); }
}
if (path === '/api/model-providers' && method === 'POST') {
- try { return json(res, 200, modelProviders.save(await readBody(req))); }
+ try { return json(res, 200, withPictureAnswers(modelProviders.save(await readBody(req)))); }
catch (err) { return error(res, 400, err.message); }
}
const providerMatch = /^\/api\/model-providers\/([a-zA-Z0-9-]+)$/.exec(path);
- if (providerMatch && method === 'DELETE') return json(res, 200, modelProviders.remove(providerMatch[1]));
+ if (providerMatch && method === 'DELETE') return json(res, 200, withPictureAnswers(modelProviders.remove(providerMatch[1])));
if(path === '/api/artifacts/open' && method === 'POST') {
const body=await readBody(req, 30*1024*1024);
@@ -1047,7 +1066,7 @@ async function api(req, res, path, method, url) {
if (measureFolder(cwd).tooBig) return error(res, 400, 'This folder is too big to try a change on a copy. Open a smaller folder for this chat, then try again.');
let choice, slots = 1;
if (body.remoteModel?.connectionId && body.remoteModel?.modelId) { choice = { connectionId: String(body.remoteModel.connectionId), modelId: String(body.remoteModel.modelId) };
- { const own = modelProviders.withSecrets().find(c => c.id === choice.connectionId); if (own && own.protocol === 'openai-chat' && isPrivateHost(own.baseUrl)) await visionProbe.check(own, choice.modelId); } slots = Number(body.slots) >= 1 ? Math.floor(Number(body.slots)) : 1; }
+ { const own = modelProviders.withSecrets().find(c => c.id === choice.connectionId); if (own && own.protocol === 'openai-chat') await visionProbe.check(own, choice.modelId); } slots = Number(body.slots) >= 1 ? Math.floor(Number(body.slots)) : 1; }
else { try { const inst = native.resolveInstance(native.state().defaultId); choice = { instanceId: inst.id }; slots = inst.metrics?.live?.slots?.length || inst.parallel || 1; } catch { return error(res, 503, 'Load a model first, then try Time-skip again.'); } }
const chatId = typeof body.sessionId === 'string' && body.sessionId ? body.sessionId : null;
if (chatId && forkTs.byChat(chatId)) return error(res, 409, 'A Time-skip is already running in this chat.');
@@ -1794,7 +1813,7 @@ async function api(req, res, path, method, url) {
const body = await readBody(req,12*1024*1024);
if (path === '/api/plan/start-goal') { body.message = '/plan start'; if (typeof body.planId !== 'string') return error(res, 400, 'planId is required'); }
let images;try{const named=Array.isArray(body.imageIds)&&body.imageIds.length?uploads.dataUrls(typeof body.sessionId==='string'?body.sessionId:'',body.imageIds):[];images=validateChatImages([...(Array.isArray(body.images)?body.images:[]),...named]);}catch(e){return error(res,400,e.message);}
- if(images.length&&body.remoteModel){const own=modelProviders.withSecrets().find(c=>c.id===body.remoteModel.connectionId);if(!(own&&own.protocol==='openai-chat'&&isPrivateHost(own.baseUrl)))return error(res,409,'Choose a local image-reading model before sending attached pictures.');}
+ if(images.length&&body.remoteModel){const own=modelProviders.withSecrets().find(c=>c.id===body.remoteModel.connectionId);const row=own?.models.find(m=>m.id===body.remoteModel.modelId);const refused=await pictureRefusal(own,row);if(refused)return error(res,409,refused);}
else if(images.length){let selected;try{selected=native.resolveInstance(body.nativeModelId??native.state().defaultId);}catch(e){return error(res,409,e.message);}if(!selected.vision)return error(res,409,'Enable Read pictures for this model before sending images.');}
// The goal bar's Resume has no words of its own: it is the same turn as "/goal resume".
const message = body.goalAction === 'resume' ? '/goal resume' : String(body.message ?? '').trim();
diff --git a/apps/host/server/local-models.mjs b/apps/host/server/local-models.mjs
index bef57a464..0bc6dc3f3 100644
--- a/apps/host/server/local-models.mjs
+++ b/apps/host/server/local-models.mjs
@@ -14,7 +14,8 @@ export function browseLocalModels(input){
return {directory,parent:dirname(directory)===directory?null:dirname(directory),entries:entries.slice(0,1000),truncated:entries.length>1000};
}
export function inspectLocalModel(input){
- let file=fullPath(input);if(extname(file).toLowerCase()!=='.gguf')throw new Error('Select a .gguf model file. Folders, adapters, and projectors cannot be loaded as a chat model.');
+ let file=fullPath(input);if(extname(file).toLowerCase()!=='.gguf')throw new Error('Select a .gguf model file. Folders and adapters cannot be loaded as a chat model.');
+ if(/^mmproj/i.test(basename(file)))throw new Error('This is a picture reader (mmproj), not a model. Choose the model’s own .gguf file; a model that reads pictures needs its reader in the same folder.');
if(!statSync(file).isFile())throw new Error('Select the model’s .gguf file, rather than its folder.');
accessSync(file,constants.R_OK);
let selected=readGGUF(file);const split=basename(file).match(/^(.*)-(\d{5})-of-(\d{5})\.gguf$/i);
@@ -25,9 +26,9 @@ export function inspectLocalModel(input){
try{selected=readGGUF(file);}catch{throw new Error('The first shard is missing or invalid. Keep all model shards in the same folder.');}
}
const m=selected.metadata,arch=m['general.architecture'];
- if(!ARCHITECTURES.has(arch))throw new Error(`The declared architecture ${clean(arch)||'(missing)'} is not supported by this text-model importer. Choose a complete text GGUF model.`);
+ if(!ARCHITECTURES.has(arch))throw new Error(`The declared architecture ${clean(arch)||'(missing)'} is not supported by this importer. This model type isn’t supported by the Strata engine; add it as a Custom model API (llama-server) instead.`);
if(m['general.type']&&m['general.type']!=='model')throw new Error('This is an adapter or auxiliary file. Choose the full model GGUF.');
- if(!m['tokenizer.ggml.model']||!m['tokenizer.ggml.tokens']?.count)throw new Error('This GGUF does not include its tokenizer. Choose the full model, not a projector or adapter.');
+ if(!m['tokenizer.ggml.model']||!m['tokenizer.ggml.tokens']?.count)throw new Error('This GGUF does not include its tokenizer. Choose the full model, not an adapter. A picture reader (mmproj) goes next to its model, not in its place.');
const integer=(key,min,max)=>{const value=m[key];if(!Number.isInteger(value)||value<min||value>max)throw new Error(`Missing or invalid ${key} metadata. This model cannot be configured safely.`);return value;};
const context=integer(`${arch}.context_length`,4096,100_000_000),layers=integer(`${arch}.block_count`,1,4096),embedding=integer(`${arch}.embedding_length`,1,1_000_000),heads=integer(`${arch}.attention.head_count`,1,8192);
const kvHeads=m[`${arch}.attention.head_count_kv`]??heads,keyLength=m[`${arch}.attention.key_length`]??embedding/heads,valueLength=m[`${arch}.attention.value_length`]??keyLength,experts=m[`${arch}.expert_count`]??0;
diff --git a/apps/host/server/model-providers.mjs b/apps/host/server/model-providers.mjs
index ecd7acc87..cd4ea8824 100644
--- a/apps/host/server/model-providers.mjs
+++ b/apps/host/server/model-providers.mjs
@@ -1,4 +1,5 @@
/** Saved API connections. Secrets never leave the host's private credentials file. */
+import { modelReadsPictures } from './vision-probe.mjs';
import { existsSync, readFileSync, writeFileSync, renameSync, mkdirSync } from 'node:fs';
import { join } from 'node:path';
import { randomUUID } from 'node:crypto';
@@ -75,7 +76,7 @@ export function createModelProviders({ directory, changed = () => {}, fetchImpl
if (rows.length > 100) fail('Add up to 100 models to one connection.');
const models = rows.map(row => {
const id = text(row.id, 256); if (!id || /[\r\n]/.test(id)) fail('Every model needs a model ID.');
- return { id, name: text(row.name) || id, contextMax: numeric(row.contextMax), maxOutput: numeric(row.maxOutput) };
+ return { id, name: text(row.name) || id, contextMax: numeric(row.contextMax), maxOutput: numeric(row.maxOutput), vision: row.vision === true ? true : row.vision === false ? false : null };
});
if (new Set(models.map(m => m.id)).size !== models.length) fail('Each model ID must be unique within this connection.');
return { id: existing?.id ?? randomUUID(), kind, provider: catalog?.id ?? null, name, baseUrl, protocol, apiKey, models };
@@ -128,7 +129,7 @@ export function createModelProviders({ directory, changed = () => {}, fetchImpl
if (connection.protocol !== 'anthropic' || !data.has_more) break;
after = data.last_id; if (!after) break;
}
- const models = [...new Map(rows.filter(m => text(m.id, 256)).map(m => [m.id, { id: text(m.id, 256), name: text(m.display_name ?? m.name) || text(m.id, 256), contextMax: numeric(m.context_length ?? m.max_context_length ?? m.context_window), maxOutput: numeric(m.max_output_tokens ?? m.max_completion_tokens ?? m.top_provider?.max_completion_tokens) }])).values()];
+ const models = [...new Map(rows.filter(m => text(m.id, 256)).map(m => [m.id, { id: text(m.id, 256), name: text(m.display_name ?? m.name) || text(m.id, 256), contextMax: numeric(m.context_length ?? m.max_context_length ?? m.context_window), maxOutput: numeric(m.max_output_tokens ?? m.max_completion_tokens ?? m.top_provider?.max_completion_tokens), vision: modelReadsPictures(m) }])).values()];
return { models: models.sort((a, b) => a.name.localeCompare(b.name)) };
}
async function stream({ selection, message, history = [], systemPrompt = '', images = [], signal, onFrame }) {
diff --git a/apps/host/server/native-control.mjs b/apps/host/server/native-control.mjs
index f4b114c30..08254fc9d 100644
--- a/apps/host/server/native-control.mjs
+++ b/apps/host/server/native-control.mjs
@@ -111,7 +111,7 @@ export function createNativeControl({directory,requests=()=>[],changed=()=>{},sp
/* A read that is not forced never makes the window wait for the (slow) status of Control: it gets the last state at once, and a stale one
is refreshed in the background. Only the first read, and a forced one after an operation, waits. */
if(refreshing)return !force&&snapshot.available?state():refreshing;if(!force&&Date.now()-lastRead<45000)return state();
- refreshing=(async()=>{try{const [catalog,status]=packaged?await (async()=>{const s=await tool('strata_status',{hardware:true});const c=await tool('strata_models').catch(error=>({families:[],contexts:[],recommendation:{family:null,model:null},context_note:'',error:error.message}));return [c,s];})():await Promise.all([tool('strata_models'),tool('strata_status',{hardware:true})]);const noImages=c=>c&&Array.isArray(c.families)?{...c,families:c.families.map(f=>({...f,images:false,sizes:(f.sizes??[]).map(s=>({...s,images:false}))}))}:c;snapshot={available:true,repo:packaged?status.installDir:repo(),version:packaged?(control.version??null):await execute('git',['describe','--tags','--always'],repo()).then(s=>s.trim()).catch(()=>null),summary:status.summary,catalog:packaged?noImages(catalog):catalog,installed:installedInfo(status.installed),hardware:status.hardware,error:null,install:status.install,external:status.running??null,...(packaged?{packaged:true,installDir:status.installDir,engine:status.engine,installerError:status.installerError}:{})};lastRead=Date.now();if(status.install?.running&&![...jobs.values()].some(j=>j.state==='running'))begin('install',j=>watchInstall(j,status.install));}catch(error){snapshot={...snapshot,available:false,error:error.message};lastRead=Date.now();}emit();return state();})().finally(()=>{refreshing=null;});const run=refreshing;if(!force&&snapshot.available){run.catch(()=>{});return state();}return run;
+ refreshing=(async()=>{try{const [catalog,status]=packaged?await (async()=>{const s=await tool('strata_status',{hardware:true});const c=await tool('strata_models').catch(error=>({families:[],contexts:[],recommendation:{family:null,model:null},context_note:'',error:error.message}));return [c,s];})():await Promise.all([tool('strata_models'),tool('strata_status',{hardware:true})]);let visionReader=false;if(packaged){try{const e=await control.engine();const root=e?.engine?.root??(e?.engine?.exe?dirname(e.engine.exe):'');visionReader=!!root&&['strata-vision','strata-vision.exe'].some(n=>existsSync(join(root,n)));}catch{visionReader=false;}}const markImages=c=>c&&Array.isArray(c.families)?{...c,families:c.families.map(f=>({...f,imagesCapable:!!f.images,images:!!f.images&&visionReader,sizes:(f.sizes??[]).map(s=>({...s,imagesCapable:!!s.images,images:!!s.images&&visionReader}))}))}:c;snapshot={available:true,repo:packaged?status.installDir:repo(),version:packaged?(control.version??null):await execute('git',['describe','--tags','--always'],repo()).then(s=>s.trim()).catch(()=>null),summary:status.summary,catalog:packaged?markImages(catalog):catalog,visionReader:packaged?visionReader:true,installed:installedInfo(status.installed),hardware:status.hardware,error:null,install:status.install,external:status.running??null,...(packaged?{packaged:true,installDir:status.installDir,engine:status.engine,installerError:status.installerError}:{})};lastRead=Date.now();if(status.install?.running&&![...jobs.values()].some(j=>j.state==='running'))begin('install',j=>watchInstall(j,status.install));}catch(error){snapshot={...snapshot,available:false,error:error.message};lastRead=Date.now();}emit();return state();})().finally(()=>{refreshing=null;});const run=refreshing;if(!force&&snapshot.available){run.catch(()=>{});return state();}return run;
}
function configure({repository}){if(packaged){if(instances.size||[...jobs.values()].some(j=>j.state==='running'))throw new Error('Unload models and finish operations before changing the installation.');const next=realpathSync(String(repository));if(!statSync(next).isDirectory())throw new Error('Choose the folder that holds your Strata model configurations.');preferences={...preferences,installDir:next};packagedInstall=null;disconnect();writeFileSync(settingsPath,JSON.stringify(preferences,null,2),{mode:0o600});lastRead=0;return refresh(true);}if(instances.size||[...jobs.values()].some(j=>j.state==='running'))throw new Error('Unload models and finish operations before changing the installation.');const next=realpathSync(String(repository));const old=preferences;preferences={...preferences,repo:next};try{repo();}catch(e){preferences=old;throw e;}disconnect();writeFileSync(settingsPath,JSON.stringify(preferences,null,2),{mode:0o600});lastRead=0;return refresh(true);}
function setDownloadDirectory(input){if([...jobs.values()].some(j=>j.state==='running'))throw new Error('Finish the current operation before changing the folder.');if(typeof input!=='string'||!input.trim()||/[\r\n\0]/.test(input))throw new Error('Choose a model folder.');const path=realpathSync(input.trim());if(!statSync(path).isDirectory())throw new Error('Choose a folder, not a file.');accessSync(path,constants.R_OK|constants.W_OK);preferences={...preferences,downloadDirectory:path};writeFileSync(settingsPath,JSON.stringify(preferences,null,2),{mode:0o600});emit();return state();}
diff --git a/apps/host/server/vision-probe.mjs b/apps/host/server/vision-probe.mjs
index 9d66a51b5..e81f79cc6 100644
--- a/apps/host/server/vision-probe.mjs
+++ b/apps/host/server/vision-probe.mjs
@@ -1,9 +1,31 @@
/**
- * Does a model server on this network read pictures? Asked of its own model list (`/models`, `architecture.input_modalities`), cached for a
- * few minutes. Unknown or unreachable counts as "no": a picture sent to a server without a vision encoder makes the whole request fail.
+ * Does a model server read pictures? Asked of the server itself, cached for a few minutes:
+ * - its model list (`/models`): `architecture.input_modalities`, `input_modalities`, `modalities`, or a `capabilities` list that names image / vision / multimodal;
+ * - a llama-server's `/props` (`modalities.vision`), which is how `llama-server --mmproj` says it can read pictures (its `/v1/models` does not).
+ * Unknown or unreachable counts as "no" (a picture sent to a server without a vision encoder makes the whole request fail), but the person can
+ * always tick "This model reads pictures" in the model's settings: that answer wins over anything asked here.
*/
const TTL_MS = 5 * 60_000;
+/** True / false / null (the row does not say) from one entry of a server's model list. */
+export function modelReadsPictures(row) {
+ if (!row || typeof row !== 'object') return null;
+ const lists = [row.architecture?.input_modalities, row.input_modalities, row.modalities?.input, Array.isArray(row.modalities) ? row.modalities : null];
+ for (const list of lists) if (Array.isArray(list)) return list.some(x => /^(image|vision|images)$/i.test(String(x)));
+ if (row.modalities && typeof row.modalities === 'object' && typeof row.modalities.vision === 'boolean') return row.modalities.vision;
+ const caps = row.capabilities ?? row.meta?.capabilities;
+ if (Array.isArray(caps)) return caps.some(x => /vision|image|multimodal/i.test(String(x)));
+ if (caps && typeof caps === 'object' && typeof caps.vision === 'boolean') return caps.vision;
+ return null;
+}
+
+/** The effective answer for a saved model: what the person ticked, else what the server said (undefined: never asked). */
+export function effectiveVision(model, asked) {
+ if (model?.vision === true) return true;
+ if (model?.vision === false) return false;
+ return asked;
+}
+
export function createVisionProbe({ fetcher = fetch, now = Date.now } = {}) {
const cache = new Map(); // key -> { at, value }
const keyOf = (conn, modelId) => `${conn.id}|${modelId}`;
@@ -15,13 +37,18 @@ export function createVisionProbe({ fetcher = fetch, now = Date.now } = {}) {
const k = keyOf(conn, modelId); const hit = cache.get(k);
if (hit && now() - hit.at < TTL_MS) return hit.value;
let value = false;
+ const base = String(conn.baseUrl).replace(/\/+$/, '');
+ const headers = conn.apiKey ? { authorization: `Bearer ${conn.apiKey}` } : {};
+ const get = async path => { const res = await fetcher(base + path, { headers, signal: AbortSignal.timeout(4000) }); return res.ok ? res.json() : null; };
try {
- const res = await fetcher(String(conn.baseUrl).replace(/\/+$/, '') + '/models', { headers: conn.apiKey ? { authorization: `Bearer ${conn.apiKey}` } : {}, signal: AbortSignal.timeout(4000) });
- if (res.ok) {
- const list = (await res.json())?.data ?? [];
- const row = list.find(m => m?.id === modelId) ?? (list.length === 1 ? list[0] : null);
- const modes = row?.architecture?.input_modalities ?? row?.input_modalities ?? [];
- value = Array.isArray(modes) && modes.includes('image');
+ const list = (await get('/models'))?.data ?? [];
+ const row = list.find(m => m?.id === modelId) ?? (list.length === 1 ? list[0] : null);
+ value = modelReadsPictures(row) === true;
+ if (!value && modelReadsPictures(row) !== false) { // the list did not say: a llama-server tells in /props (next to /v1, not under it)
+ try {
+ const props = await (async () => { const res = await fetcher(base.replace(/\/v1$/, '') + '/props', { headers, signal: AbortSignal.timeout(4000) }); return res.ok ? res.json() : null; })();
+ value = props?.modalities?.vision === true;
+ } catch { /* no /props: not a llama-server */ }
}
} catch { value = false; }
cache.set(k, { at: now(), value });
diff --git a/apps/host/src/components/ComposerControls.tsx b/apps/host/src/components/ComposerControls.tsx
index fd680d8e6..b8bbe1405 100644
--- a/apps/host/src/components/ComposerControls.tsx
+++ b/apps/host/src/components/ComposerControls.tsx
@@ -83,14 +83,14 @@ export function ComposerControls({ session, onPreview, onAttach, onError, onSlas
skipped++;
continue;
}
- if (!directory && photoTypes.test(file.name) && nativeModel?.vision && !remoteModel) {
+ if (!directory && photoTypes.test(file.name) && (remoteModel ? apiModel?.reads === true : nativeModel?.vision)) {
await attachChatPhoto(file, session?.id);
continue;
}
if (!directory && (photoTypes.test(file.name) || /\.pdf$/i.test(file.name))) {
await openFile(file);
if (photoTypes.test(file.name))
- onError(remoteModel ? 'Pictures are open in preview. Choose a local model with Read pictures enabled to send them.' : 'Enable Read pictures in Models to send this image to the model. It is open in preview.');
+ onError(remoteModel ? `Strata cannot tell that "${apiModel?.name ?? 'this model'}" reads pictures, so the picture is only open in preview. If it does, open Settings > Models, edit its connection and tick "This model reads pictures". Otherwise choose a model that reads pictures.` : 'Enable Read pictures in Models to send this image to the model. It is open in preview.');
continue;
}
if (!textTypes.test(file.name) || bytes + file.size > 256 * 1024) {
diff --git a/apps/host/src/components/NativeLocalModel.tsx b/apps/host/src/components/NativeLocalModel.tsx
index 09f9750bd..7c68d038a 100644
--- a/apps/host/src/components/NativeLocalModel.tsx
+++ b/apps/host/src/components/NativeLocalModel.tsx
@@ -8,14 +8,16 @@ import { nativeCatalogModels, catalogVersionLabel } from '../lib/native-catalog'
import { sizeGb } from '../lib/format';
import { NativeProgress } from './NativeModels';
-interface Preview { path: string; preview: { file_count: number; total_bytes: number; file_set_status: string; issues: string[]; candidate_selection: { family: string; model: string } | null } }
+interface Preview { path: string; preview: { file_count: number; total_bytes: number; file_set_status: string; issues: string[]; messages?: string[]; projectors?: { name: string; status: string; pinned: boolean; plain: string }[]; reads_pictures?: boolean; candidate_selection: { family: string; model: string } | null } }
/** The words for why a folder of GGUF files cannot be used: a sentence and what to do, the checker's own codes under Details. */
function problem(issues: string[]): { message: string; next: string } {
const has = (re: RegExp) => issues.some(i => re.test(i));
if (has(/incomplete_or_invalid_shard_set|multiple_model_groups/))
return { message: 'Some parts of this model are missing, or the folder holds more than one model.', next: 'Put every part of one model in one folder, then check again.' };
- if (has(/unrecognized_or_unsupported_model_variant|unsupported_architecture|unsupported_or_missing_architecture|expert_quantization|tokenizer/))
+ if (has(/unsupported_architecture|unsupported_or_missing_architecture/))
+ return { message: 'This model type isn’t supported by the Strata engine.', next: 'Add it as a Custom model API instead: start it with llama-server, then open Settings > Models > Add model provider > Custom model API.' };
+ if (has(/unrecognized_or_unsupported_model_variant|expert_quantization|tokenizer/))
return { message: 'Strata can’t run these files yet.', next: 'Strata runs the Qwen3.8-Flash-Next models. Download one from Models, or choose a folder with its files.' };
return { message: 'These files could not be checked.', next: 'Check that the folder holds complete GGUF files, then try again.' };
}
@@ -51,7 +53,7 @@ export function NativeLocalModel({ onBack }: { onBack: () => void }) {
setPending(true);
setError('');
try {
- const result = await nativeAction<NativeJob>('operations', { kind: 'install', args: { family: selection.family, model: selection.model, gguf_dir: checked.path, context: native?.catalog?.recommendation?.context ?? 32768, vision: 'no', kv: 'int8', loadAfter: true } });
+ const result = await nativeAction<NativeJob>('operations', { kind: 'install', args: { family: selection.family, model: selection.model, gguf_dir: checked.path, context: native?.catalog?.recommendation?.context ?? 32768, vision: checked.preview.projectors?.some(p => p.status === 'ok' && p.pinned) && native?.visionReader ? 'yes' : 'no', kv: 'int8', loadAfter: true } });
setJobId(result.id);
}
catch (e) { setError(e instanceof Error ? e.message : String(e)); }
@@ -70,6 +72,7 @@ export function NativeLocalModel({ onBack }: { onBack: () => void }) {
<h2 className="h3">{tr(family ? family.title : 'Model found')}</h2>
<p><strong>{tr(version ? catalogVersionLabel(version) : selection?.model ?? '')}</strong> · {tr(selection?.model ?? '')}</p>
<p className="small muted">{tr("{0} files · {1}", [checked.preview.file_count, sizeGb(checked.preview.total_bytes / 1024 ** 3)])}</p>
+ {tr(checked.preview.messages?.map(m => <p className="small muted" key={m}>{tr(m)}</p>))}
<div className="native-model-actions"><Btn variant="primary" icon="models" disabled={pending || !!active} onClick={() => void prepare()}>{tr(pending ? 'Checking…' : 'Prepare and load')}</Btn></div>
</> : <>
<h2 className="h3">{tr(found!.message)}</h2>
diff --git a/apps/host/src/components/NativeModels.tsx b/apps/host/src/components/NativeModels.tsx
index 0e3489b77..0b9d01d9f 100644
--- a/apps/host/src/components/NativeModels.tsx
+++ b/apps/host/src/components/NativeModels.tsx
@@ -135,6 +135,7 @@ export function NativeModels({ onOtherModels, onReady, onImport, onConnectEngine
<div className="native-choice__col"><span className="native-choice__label">{tr("Computer use")}</span><div className="native-resource-choice"><button aria-pressed={resourceProfile === 'work'} data-selected={resourceProfile === 'work'} onClick={() => setResourceProfile('work')}><Icon name="computer" size={20}/><span><strong>{tr("Leave 20% for my work")}</strong><small>{tr("Headroom for your other apps")}</small></span>{tr(resourceProfile === 'work' ? <Icon name="check" size={15}/> : null)}</button><button aria-pressed={resourceProfile === 'performance'} data-selected={resourceProfile === 'performance'} onClick={() => setResourceProfile('performance')}><Icon name="bolt" size={20}/><span><strong>{tr("Full performance")}</strong><small>{tr("Give Strata priority")}</small></span>{tr(resourceProfile === 'performance' ? <Icon name="check" size={15}/> : null)}</button></div></div>
</div>
{tr(cardOptions ? <div className="native-cards"><span className="native-choice__label">{tr("Graphics cards")}</span><div className="native-cards__list" role="radiogroup" aria-label={tr("Graphics cards")}>{cardOptions.options.map(o => { const on = activeCards?.key === o.key; return <button key={o.key} type="button" role="radio" aria-checked={on} data-selected={on} disabled={!!active} className="native-cards__opt" onClick={() => setCardsKey(o.key)}><span className="native-cards__dot" aria-hidden="true"/><span><strong>{o.label}</strong><small data-warn={o.verdict === 'spills' || o.verdict === 'tight'}>{[o.words, ...o.notes].join('. ')}.</small></span></button>; })}{cardOptions.ignored.length ? <p className="small muted native-cards__note">{`Not used: ${cardOptions.ignored.map(g => `${g.name ?? 'a card'} (${g.why_ignored})`).join('; ')}.`}</p> : null}</div></div> : null)}
+ {tr(selectedSize?.imagesCapable && !selectedSize.images ? <p className="small muted native-spill-note">{tr("This model can read pictures, but the engine installed here does not include the picture reader yet, so Read pictures is off. Update the engine in Settings → About to turn it on.")}</p> : null)}
{tr(selectedSize?.images ? <><div className="native-image-reading"><label><input type="checkbox" checked={vision !== 'no'} disabled={!selectedSize?.images || !!active} onChange={e => setVision(e.target.checked ? 'yes' : 'no')}/><span><strong>{tr("Read pictures ")}<small>{tr("Optional")}</small></strong><small>{tr("Understand photos, screenshots and charts. Vision files download automatically. Enable them when loading this model.")}</small></span></label>{tr(expert && vision !== 'no' ? <Select ariaLabel={tr("Image encoder device")} value={vision === 'yes' ? 'yes' : vision} options={[{ value: 'yes', label: 'Automatic' }, { value: 'gpu', label: 'GPU' }, { value: 'cpu', label: 'CPU' }]} onChange={setVision}/> : null)}</div>{tr(vision !== 'no' && installed?.ready && installed.visionPrepared === false ? <div className="native-vision-missing" role="status"><span>{tr("Picture reading needs a few more files that are not on this computer yet.")}</span><Btn size="sm" disabled={pending || !!active} onClick={() => void download()}>{tr("Download picture-reading files")}</Btn></div> : null)}</> : null)}
</> : null)}
{tr(active ? <NativeProgress job={active}/> : null)}
diff --git a/apps/host/src/components/SettingsModels.tsx b/apps/host/src/components/SettingsModels.tsx
index 36d530989..671225f1c 100644
--- a/apps/host/src/components/SettingsModels.tsx
+++ b/apps/host/src/components/SettingsModels.tsx
@@ -119,7 +119,7 @@ export function SettingsModels() {
<div className="provider-models__head"><h3 className="h4">{tr("Models in chat")}</h3><Btn size="sm" icon="refresh" disabled={fetching || !draft.baseUrl.trim() || (draft.kind === 'provider' && !draft.apiKey.trim() && (!saved?.apiKeySet || !!draft.clearKey))} onClick={() => void fetchModels()}>{tr(fetching ? 'Fetching…' : 'Fetch available models')}</Btn></div>
{tr(notice ? <p className="small muted" role="status">{tr(notice)}</p> : null)}
{tr(available.length ? <><input className="st-input" type="search" aria-label={tr("Filter available models")} placeholder={tr("Filter models…")} value={filter} onChange={e => setFilter(e.target.value)}/><div className="provider-model-list">{tr(available.filter(m => `${m.name} ${m.id}`.toLowerCase().includes(filter.toLowerCase())).map(m => <label className="provider-model-row" key={m.id}><input type="checkbox" checked={draft.models.some(v => v.id === m.id)} onChange={e => update({ models: e.target.checked ? [...draft.models, m] : draft.models.filter(v => v.id !== m.id) })}/><span><strong>{m.name}</strong><small className="muted">{m.id}</small></span></label>))}</div></> : null)}
- {tr(draft.models.length ? <div className="stack stack--sm">{tr(draft.models.map(m => <div className="provider-selected" key={m.id}><Icon name="check" size={15}/><span><strong>{m.name}</strong><small className="muted">{m.id}</small></span><button type="button" className="st-iconbtn" aria-label={tr("Remove {0} from chat picker", [m.name])} onClick={() => update({ models: draft.models.filter(v => v.id !== m.id) })}><Icon name="x" size={16}/></button></div>))}</div> : <p className="small muted">{tr("No models selected yet. Fetch the list or add a model ID.")}</p>)}
+ {tr(draft.models.length ? <div className="stack stack--sm">{tr(draft.models.map(m => <div className="provider-selected" key={m.id}><Icon name="check" size={15}/><span><strong>{m.name}</strong><small className="muted">{m.id}</small>{tr(draft.protocol === 'openai-chat' ? <label className="row xs"><input type="checkbox" checked={m.vision === true} onChange={e => update({ models: draft.models.map(v => v.id === m.id ? { ...v, vision: e.target.checked ? true : null } : v) })}/>{tr("This model reads pictures")}{tr(m.vision == null && m.reads === true ? <span className="muted"> {tr("(the server says it does)")}</span> : null)}</label> : null)}</span><button type="button" className="st-iconbtn" aria-label={tr("Remove {0} from chat picker", [m.name])} onClick={() => update({ models: draft.models.filter(v => v.id !== m.id) })}><Icon name="x" size={16}/></button></div>))}</div> : <p className="small muted">{tr("No models selected yet. Fetch the list or add a model ID.")}</p>)}
<div className="provider-manual"><label className="input-field"><span>{tr("Model ID")}</span><input className="st-input" value={modelId} placeholder={tr("Model ID from your provider or server")} spellCheck={false} onChange={e => setModelId(e.target.value)} onKeyDown={e => {
if (e.key === 'Enter') {
e.preventDefault();
diff --git a/apps/host/src/lib/native.ts b/apps/host/src/lib/native.ts
index 059f95423..c0cc221e4 100644
--- a/apps/host/src/lib/native.ts
+++ b/apps/host/src/lib/native.ts
@@ -3,12 +3,14 @@ import { api } from './client';
import type { Hardware, RuntimeModels } from './types';
export interface GpuChoiceOption {key:string;cards:number[];label:string;verdict:'fits'|'tight'|'spills'|'unknown';words:string;notes:string[];recommended:boolean}
export interface GpuChoice {options:GpuChoiceOption[];recommended:string|null;ignored:{name?:string;why_ignored:string}[];cards:number}
-export interface NativeSize {id:string;model:string;about:string;download_gb:number;ram_needed_gb:number;experts_gb:number;on_this_pc:string|null;installed:boolean;gpu_options?:GpuChoice|null}
+export interface NativeSize {id:string;model:string;about:string;download_gb:number;ram_needed_gb:number;experts_gb:number;on_this_pc:string|null;installed:boolean;gpu_options?:GpuChoice|null;images?:boolean;imagesCapable?:boolean}
export interface NativeFamily {family:string;title:string;about:string;experimental:boolean;images:boolean;sizes:NativeSize[]}
export interface NativeField {id:string;label:string;group:string;type:'number'|'boolean'|'string'|'enum'|'array';min?:number;max?:number;choices?:string[];hint?:string;supported?:boolean}
export interface NativeJob {id:string;kind:string;guided?:boolean;model?:string;state:'running'|'done'|'error'|'cancelled';startedAt:string;summary:string;error:string|null;details?:string;result:Record<string,any>|null;progress?:Record<string,any>|null}
export interface NativeInstance {id:string;apiModelId:string;model:string;label:string;aliases?:string[];vision?:boolean;resourceProfile?:'work'|'performance'|null;context?:number;port:number;gpus:number[];status:'loading'|'ready'|'unloading'|'error';phase:string;loadedAt:string;error:string|null;metrics:Record<string,any>|null}
-export interface NativeState {available:boolean;repo:string;downloadDirectory?:string;version?:string|null;summary:string;error:string|null;catalog:{families:NativeFamily[];contexts:number[];recommendation:{family:string|null;model:string|null;context?:number;why?:string};context_note:string}|null;installed:{models:{model:string;config:string;model_name:string;ready:boolean;/** The picture-reading files are on this computer (notes-split-agent8.md, item 8). */visionPrepared?:boolean;context?:number;kv?:string;images?:string;visionEnabled?:boolean;visionDevice?:string;lowRamMode?:string;resourceProfile?:'work'|'performance';dataFolder?:string;diskGb?:number;diskBytes?:number;missing_files?:string[]}[];data_folder:string;engine:{version:string|null}}|null;hardware:{ram_gb:number;gpus:{index:number;name:string;vram_gb:number;usable:boolean}[]}|null;fields:NativeField[];instances:NativeInstance[];defaultId:string|null;jobs:NativeJob[];capabilities:{responses?:boolean;updates:boolean;rollback:boolean;calibration:boolean;componentUpdates?:boolean}}
+export interface NativeState {
+ /** The picture reader (strata-vision) is installed with the engine, so models marked for pictures can read them here. */
+ visionReader?: boolean;available:boolean;repo:string;downloadDirectory?:string;version?:string|null;summary:string;error:string|null;catalog:{families:NativeFamily[];contexts:number[];recommendation:{family:string|null;model:string|null;context?:number;why?:string};context_note:string}|null;installed:{models:{model:string;config:string;model_name:string;ready:boolean;/** The picture-reading files are on this computer (notes-split-agent8.md, item 8). */visionPrepared?:boolean;context?:number;kv?:string;images?:string;visionEnabled?:boolean;visionDevice?:string;lowRamMode?:string;resourceProfile?:'work'|'performance';dataFolder?:string;diskGb?:number;diskBytes?:number;missing_files?:string[]}[];data_folder:string;engine:{version:string|null}}|null;hardware:{ram_gb:number;gpus:{index:number;name:string;vram_gb:number;usable:boolean}[]}|null;fields:NativeField[];instances:NativeInstance[];defaultId:string|null;jobs:NativeJob[];capabilities:{responses?:boolean;updates:boolean;rollback:boolean;calibration:boolean;componentUpdates?:boolean}}
let state:NativeState|null=null,timer:ReturnType<typeof setInterval>|null=null,reading:Promise<void>|null=null;
const listeners=new Set<()=>void>();
let again=false;
diff --git a/apps/host/src/lib/store.ts b/apps/host/src/lib/store.ts
index a0b322241..d0369bd9d 100644
--- a/apps/host/src/lib/store.ts
+++ b/apps/host/src/lib/store.ts
@@ -1525,8 +1525,8 @@ export async function sendMessage(text: string, options: { fromLoop?: boolean; g
else startTimeSkipDemo(demoPrompt);
return;
}
- const imageTarget=nativeState()?.instances.find(i=>i.id===nativeState()?.defaultId);const canSendPictures=!state.remoteModel&&imageTarget?.status==='ready'&&imageTarget.vision;
- if(pendingChatPhotos(state.activeSession).length&&!canSendPictures){pushToast('error','Choose an image-reading model','Switch to a local model with Read pictures enabled, or remove the attachments.');return;}
+ const imageTarget=nativeState()?.instances.find(i=>i.id===nativeState()?.defaultId);const remoteApi=state.remoteModel?state.providers?.connections.find(c=>c.id===state.remoteModel!.connectionId)?.models.find(m=>m.id===state.remoteModel!.modelId):null;const canSendPictures=state.remoteModel?remoteApi?.reads===true:imageTarget?.status==='ready'&&imageTarget.vision;
+ if(pendingChatPhotos(state.activeSession).length&&!canSendPictures){pushToast('error','This model cannot read pictures',state.remoteModel?'Open Settings > Models, edit this connection and tick "This model reads pictures" if it does, or choose another model. You can also remove the attachments.':'Switch to a model that reads pictures (turn on Read pictures in Models), or remove the attachments.');return;}
const pictures=takeChatPhotos(state.activeSession);
const body=text.trim()||(pictures.length?'Describe this picture.':'');
if(!body&&!resume&&!planStart)return;
diff --git a/apps/host/src/lib/types.ts b/apps/host/src/lib/types.ts
index bfc8613a4..3fd645047 100644
--- a/apps/host/src/lib/types.ts
+++ b/apps/host/src/lib/types.ts
@@ -125,7 +125,7 @@ export interface ImageModelLibrary {
}
export type ModelProtocol = 'openai-chat' | 'openai-responses' | 'anthropic';
-export interface ApiModel { id: string; name: string; contextMax?: number | null }
+export interface ApiModel { id: string; name: string; contextMax?: number | null; /** What the person ticked ("This model reads pictures"): true, false, or null for not set. */ vision?: boolean | null; /** The answer in use: the tick, else what the server said; null when nobody knows. */ reads?: boolean | null }
export interface ModelProviderPreset { id: string; name: string; baseUrl: string; protocol: ModelProtocol; baseUrlHint?: string }
export interface ModelConnection {
id: string; kind: 'provider' | 'custom'; provider: string | null; name: string;
diff --git a/apps/host/tests/vision-probe.test.mjs b/apps/host/tests/vision-probe.test.mjs
new file mode 100644
index 000000000..86418e6fa
--- /dev/null
+++ b/apps/host/tests/vision-probe.test.mjs
@@ -0,0 +1,66 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { createVisionProbe, effectiveVision, modelReadsPictures } from '../server/vision-probe.mjs';
+import { createModelProviders } from '../server/model-providers.mjs';
+
+const answer = (routes, seen = []) => async (url, init) => {
+ const path = new URL(url).pathname;
+ seen.push(path);
+ const body = routes[path];
+ return body === undefined ? { ok: false, status: 404, json: async () => ({}) } : { ok: true, status: 200, json: async () => body };
+};
+
+test('a model list row says whether the model reads pictures in every common shape', () => {
+ assert.equal(modelReadsPictures({ architecture: { input_modalities: ['text', 'image'] } }), true);
+ assert.equal(modelReadsPictures({ architecture: { input_modalities: ['text'] } }), false);
+ assert.equal(modelReadsPictures({ modalities: { input: ['text', 'image'] } }), true);
+ assert.equal(modelReadsPictures({ capabilities: ['completion', 'vision'] }), true);
+ assert.equal(modelReadsPictures({ capabilities: { vision: false } }), false);
+ assert.equal(modelReadsPictures({ id: 'x' }), null);
+ assert.equal(modelReadsPictures(null), null);
+});
+
+test('what the person ticked wins over what the server said', () => {
+ assert.equal(effectiveVision({ vision: true }, false), true);
+ assert.equal(effectiveVision({ vision: false }, true), false);
+ assert.equal(effectiveVision({ vision: null }, true), true);
+ assert.equal(effectiveVision({}, undefined), undefined);
+});
+
+test('a llama-server started with --mmproj says so in /props, next to /v1, and the answer is cached', async () => {
+ const seen = [];
+ const probe = createVisionProbe({ fetcher: answer({ '/v1/models': { data: [{ id: 'smol' }] }, '/props': { modalities: { vision: true } } }, seen) });
+ const conn = { id: 'c1', baseUrl: 'http://127.0.0.1:8080/v1' };
+ assert.equal(probe.known(conn, 'smol'), undefined);
+ assert.equal(await probe.check(conn, 'smol'), true);
+ assert.equal(await probe.check(conn, 'smol'), true);
+ assert.equal(probe.known(conn, 'smol'), true);
+ assert.deepEqual(seen, ['/v1/models', '/props']);
+});
+
+test('a text-only server, and an unreachable one, count as not reading pictures', async () => {
+ const text = createVisionProbe({ fetcher: answer({ '/v1/models': { data: [{ id: 'm' }] }, '/props': { modalities: { vision: false } } }) });
+ assert.equal(await text.check({ id: 'c', baseUrl: 'http://h/v1' }, 'm'), false);
+ const down = createVisionProbe({ fetcher: async () => { throw new Error('refused'); } });
+ assert.equal(await down.check({ id: 'c', baseUrl: 'http://h/v1' }, 'm'), false);
+});
+
+test('a provider list that names image input is believed without asking /props', async () => {
+ const seen = [];
+ const probe = createVisionProbe({ fetcher: answer({ '/v1/models': { data: [{ id: 'gpt', architecture: { input_modalities: ['text', 'image'] } }] } }, seen) });
+ assert.equal(await probe.check({ id: 'c', baseUrl: 'https://api.example.com/v1' }, 'gpt'), true);
+ assert.deepEqual(seen, ['/v1/models']);
+});
+
+test('the tick "This model reads pictures" is saved with the model and survives a reload', t => {
+ const directory = mkdtempSync(join(tmpdir(), 'strata-vision-'));
+ t.after(() => rmSync(directory, { recursive: true, force: true }));
+ const service = createModelProviders({ directory });
+ const saved = service.save({ kind: 'custom', name: 'Other PC', baseUrl: 'http://192.168.1.20:8080/v1', protocol: 'openai-chat', apiKey: '', models: [{ id: 'smol', name: 'SmolVLM', vision: true }, { id: 'text', name: 'Text only' }] }).connections[0];
+ assert.deepEqual(saved.models.map(m => m.vision), [true, null]);
+ const again = createModelProviders({ directory }).list().connections[0];
+ assert.deepEqual(again.models.map(m => m.vision), [true, null]);
+});
diff --git a/strata_cli/launch/runner.py b/strata_cli/launch/runner.py
index a26a60c35..d0ffeb260 100644
--- a/strata_cli/launch/runner.py
+++ b/strata_cli/launch/runner.py
@@ -748,7 +748,7 @@ ISSUE_WORDS = {
"multiple_model_groups": "the folder holds more than one model",
"incomplete_or_invalid_shard_set": "some shards are missing",
"unrecognized_or_unsupported_model_variant": "this is not a model size Strata knows",
- "unsupported_or_missing_architecture": "this is not a model Strata can run",
+ "unsupported_or_missing_architecture": "this model type is not supported by the Strata engine (add it as a Custom model API with llama-server instead)",
"unsupported_or_incomplete_tokenizer_metadata": "the file has no usable tokenizer",
"missing_or_unexpected_expert_quantization": "its quantization is not one Strata knows",
}
@@ -764,7 +764,7 @@ def plain_issue(code: str) -> str:
if "does not begin with a gguf header" in low or "header version" in low:
return "a file is not a GGUF model file"
if "unsupported_architecture" in low:
- return "this is not a model Strata can run"
+ return "this model type is not supported by the Strata engine (add it as a Custom model API with llama-server instead)"
if "symlink" in low:
return "a file is a shortcut (symbolic link); point at the real file"
return code.replace("_", " ")
diff --git a/strata_control/gguf_preflight.py b/strata_control/gguf_preflight.py
index 189bc7fb3..b8e037485 100644
--- a/strata_control/gguf_preflight.py
+++ b/strata_control/gguf_preflight.py
@@ -22,7 +22,7 @@ MAX_ARRAY_ITEMS = 500_000
MAX_TENSORS = 100_000
SCALAR_BYTES = {0: 1, 1: 1, 2: 2, 3: 2, 4: 4, 5: 4, 6: 4, 7: 1,
10: 8, 11: 8, 12: 8}
-CAPTURE_STRINGS = {"general.architecture", "tokenizer.ggml.model"}
+CAPTURE_STRINGS = {"general.architecture", "tokenizer.ggml.model", "clip.projector_type"}
PRESENCE_KEYS = {"tokenizer.ggml.tokens", "tokenizer.ggml.merges"}
@@ -164,6 +164,7 @@ def inspect_metadata(path: Path) -> dict:
if tensors == 1 and details.st_size - (data_start + required_payload_bytes) >= 64:
raise ProfileError("single-tensor GGUF shard has unexplained trailing bytes", 5)
return {"architecture": found.get("general.architecture"),
+ "projector_type": found.get("clip.projector_type"),
"tokenizer_model": found.get("tokenizer.ggml.model"),
"tokenizer_tokens_present": found.get("tokenizer.ggml.tokens") is True,
"tokenizer_merges_present": found.get("tokenizer.ggml.merges") is True,
diff --git a/strata_control/models.py b/strata_control/models.py
index c5daeab0e..9dc12a2ad 100644
--- a/strata_control/models.py
+++ b/strata_control/models.py
@@ -250,6 +250,46 @@ def _candidate_selection(shards: list[Path]) -> dict | None:
return None
+UNSUPPORTED_MODEL_TYPE = ("This model type isn't supported by the Strata engine. "
+ "Add it as a Custom model API (llama-server) instead.")
+# The picture readers Strata ships a pinned copy of (setup downloads and checks one of these next to the model).
+# (name, size in bytes) as setup.py pins them: setup would delete a file of that name with another size, so only an exact
+# match is called "the official one".
+PINNED_PROJECTORS = {"mmproj-qwen3.8-flash-next-bf16.gguf": 907543008, "mmproj-f16.gguf": 899283584}
+OUR_PROJECTOR_TYPES = {"qwen3vl_merger"}
+
+
+def _assess_projector(path: Path) -> dict:
+ """What a picture-reader file (mmproj*.gguf) next to a model is, in words; never blocks the import."""
+ try:
+ size = path.stat().st_size
+ except OSError:
+ size = -1
+ row = {"name": path.name, "status": "unreadable", "pinned": PINNED_PROJECTORS.get(path.name.lower()) == size}
+ try:
+ info = inspect_metadata(path)
+ except ProfileError:
+ row["plain"] = f"{path.name} could not be read as a picture reader, so it is not used."
+ return row
+ row["architecture"], row["projector_type"] = info["architecture"], info.get("projector_type")
+ if info["architecture"] != "clip":
+ row["status"] = "not_a_projector"
+ row["plain"] = f"{path.name} is not a picture reader (mmproj) file, so it is not used."
+ elif info.get("projector_type") not in OUR_PROJECTOR_TYPES:
+ row["status"] = "unsupported"
+ row["plain"] = (f"{path.name} reads pictures for a different kind of model. "
+ "Strata cannot use it; add that model as a Custom model API (llama-server) instead.")
+ elif row["pinned"]:
+ row["status"] = "ok"
+ row["plain"] = f"{path.name} is the picture reader Strata uses for this model family; pictures will work."
+ else:
+ row["status"] = "other_name"
+ row["plain"] = (f"{path.name} is a picture reader of the right kind but not the file Strata checks; "
+ "Strata uses the official one (mmproj-Qwen3.8-Flash-Next-BF16.gguf) and "
+ "downloads it when it is not in the folder.")
+ return row
+
+
def plan_import(raw_path: str) -> dict:
"""Inspect a local candidate without copying, parsing tensors or adopting it."""
path = Path(raw_path).expanduser()
@@ -277,6 +317,14 @@ def plan_import(raw_path: str) -> dict:
raise ProfileError("cannot inspect GGUF directory", 5) from None
elif stat.S_ISREG(details.st_mode) and path.name.lower().endswith(".gguf"):
ignored_auxiliary = []
+ try: # the picture reader that sits next to a chosen model file
+ ignored_auxiliary = [c.name for c in path.parent.iterdir()
+ if c.name.lower().startswith("mmproj") and c.name.lower().endswith(".gguf")][:8]
+ except OSError:
+ pass
+ if path.name.lower().startswith("mmproj"):
+ raise ProfileError("this is a picture reader (mmproj), not a model. Choose the model's own "
+ ".gguf file; Strata finds the reader next to it", 5)
match = SHARD.fullmatch(path.name)
if match:
total = int(match.group(3))
@@ -334,6 +382,13 @@ def plan_import(raw_path: str) -> dict:
if metadata["tokenizer_model"] != "gpt2" or not metadata["tokenizer_tokens_present"] or \
not metadata["tokenizer_merges_present"]:
issues.append("unsupported_or_incomplete_tokenizer_metadata")
+ projectors = [_assess_projector(path.parent / n if path.is_file() else path / n)
+ for n in sorted(ignored_auxiliary)[:8]]
+ messages = []
+ if any(i.endswith("unsupported_architecture") or i == "unsupported_or_missing_architecture" for i in issues):
+ messages.append(UNSUPPORTED_MODEL_TYPE)
+ elif projectors:
+ messages.extend(p["plain"] for p in projectors)
candidate = _candidate_selection(shards)
if candidate is None:
issues.append("unrecognized_or_unsupported_model_variant")
@@ -347,6 +402,9 @@ def plan_import(raw_path: str) -> dict:
"observed_expert_types": sorted(observed_experts),
"observed_tensor_count": observed_tensor_count,
"ignored_auxiliary_files": sorted(ignored_auxiliary),
+ "projectors": projectors, "messages": messages,
+ "reads_pictures": any(p["status"] in ("ok", "other_name") for p in projectors) and
+ not any(i.endswith("unsupported_architecture") for i in issues),
"file_set_status": "plausible" if not issues else "invalid_or_incomplete",
"issues": issues, "adoption_status": "requires_architecture_quantization_tokenizer_and_engine_checks",
"execution_compatibility": "unverified", "would_write": False,
diff --git a/tests/terminal/test_import_projector.py b/tests/terminal/test_import_projector.py
new file mode 100644
index 000000000..f17bda8cb
--- /dev/null
+++ b/tests/terminal/test_import_projector.py
@@ -0,0 +1,107 @@
+"""Importing a model together with its picture reader (mmproj), and the plain words when a model type is not ours."""
+
+import io
+import json
+import struct
+import tempfile
+import unittest
+from contextlib import redirect_stdout
+from pathlib import Path
+
+from unittest import mock
+
+from strata_cli.cli import main
+from strata_control import models
+from tests.terminal.test_models import gguf_fixture
+
+
+def projector_fixture(projector_type="qwen3vl_merger", architecture="clip") -> bytes:
+ def string(value):
+ raw = value.encode()
+ return struct.pack("<Q", len(raw)) + raw
+
+ pairs = [("general.architecture", 8, string(architecture)), ("clip.projector_type", 8, string(projector_type))]
+ header = (b"GGUF" + struct.pack("<IQQ", 3, 1, len(pairs)) +
+ b"".join(string(key) + struct.pack("<I", kind) + value for key, kind, value in pairs))
+ header += string("v.blk.0.attn_q.weight") + struct.pack("<IQIQ", 1, 64, 1, 0)
+ return header + b"\x00" * (-len(header) % 32) + b"\x00" * 128
+
+
+class ImportProjectorTests(unittest.TestCase):
+ def setUp(self):
+ self.temp = tempfile.TemporaryDirectory()
+ self.addCleanup(self.temp.cleanup)
+ self.source = Path(self.temp.name) / "model"
+ self.source.mkdir()
+
+ def plan(self, path):
+ output = io.StringIO()
+ with redirect_stdout(output):
+ main(["models", "import", str(path), "--plan", "--output", "json"])
+ return json.loads(output.getvalue())["plan"]
+
+ def model(self, **kwargs):
+ first = self.source / "Qwen3.8-Flash-Next-GSQ-RCO-Q2_0-00001-of-00001.gguf"
+ first.write_bytes(gguf_fixture(**kwargs))
+ return first
+
+ def test_pinned_projector_next_to_a_model_means_pictures(self):
+ self.model()
+ file = self.source / "mmproj-Qwen3.8-Flash-Next-BF16.gguf"
+ file.write_bytes(projector_fixture())
+ with mock.patch.dict(models.PINNED_PROJECTORS, {file.name.lower(): file.stat().st_size}):
+ plan = self.plan(self.source)
+ self.assertTrue(plan["reads_pictures"])
+ self.assertEqual(plan["projectors"][0]["status"], "ok")
+ self.assertIn("pictures will work", plan["messages"][0])
+
+ def test_same_name_but_another_size_is_not_called_official(self):
+ self.model()
+ (self.source / "mmproj-Qwen3.8-Flash-Next-BF16.gguf").write_bytes(projector_fixture())
+ plan = self.plan(self.source)
+ self.assertEqual(plan["projectors"][0]["status"], "other_name")
+
+ def test_projector_beside_a_chosen_model_file_is_found(self):
+ first = self.model()
+ (self.source / "mmproj-F16.gguf").write_bytes(projector_fixture())
+ plan = self.plan(first)
+ self.assertEqual([p["name"] for p in plan["projectors"]], ["mmproj-F16.gguf"])
+ self.assertTrue(plan["reads_pictures"])
+
+ def test_projector_of_another_kind_is_not_used(self):
+ self.model()
+ (self.source / "mmproj-other.gguf").write_bytes(projector_fixture("mlp"))
+ plan = self.plan(self.source)
+ self.assertFalse(plan["reads_pictures"])
+ self.assertEqual(plan["projectors"][0]["status"], "unsupported")
+ self.assertIn("Custom model API", plan["projectors"][0]["plain"])
+
+ def test_other_name_projector_of_our_kind_is_explained(self):
+ self.model()
+ (self.source / "mmproj-mine.gguf").write_bytes(projector_fixture())
+ plan = self.plan(self.source)
+ self.assertEqual(plan["projectors"][0]["status"], "other_name")
+ self.assertTrue(plan["reads_pictures"])
+
+ def test_unsupported_model_type_says_custom_api(self):
+ first = self.source / "SmolVLM-256M-Q8_0.gguf"
+ first.write_bytes(gguf_fixture(architecture="llama"))
+ (self.source / "mmproj-SmolVLM-256M-Instruct-Q8_0.gguf").write_bytes(projector_fixture("idefics3"))
+ plan = self.plan(self.source)
+ self.assertEqual(plan["file_set_status"], "invalid_or_incomplete")
+ self.assertFalse(plan["reads_pictures"])
+ self.assertEqual(plan["messages"], ["This model type isn't supported by the Strata engine. "
+ "Add it as a Custom model API (llama-server) instead."])
+
+ def test_choosing_the_projector_itself_says_so(self):
+ projector = self.source / "mmproj-F16.gguf"
+ projector.write_bytes(projector_fixture())
+ output = io.StringIO()
+ with redirect_stdout(output):
+ code = main(["models", "import", str(projector), "--plan", "--output", "json"])
+ self.assertEqual(code, 5)
+ self.assertIn("picture reader", json.dumps(json.loads(output.getvalue())))
+
+
+if __name__ == "__main__":
+ unittest.main()