Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Vision end to end: vision files are downloaded when asked, pictures through the agent and the plain path, uploads by multipart/data address/raw bytes, dropped files named in the next message

Niko12217af6b6c15204
diff --git a/apps/host/server/agent/agent-bridge.mjs b/apps/host/server/agent/agent-bridge.mjs
index 5a2dbf32a..849f789cd 100644
--- a/apps/host/server/agent/agent-bridge.mjs
+++ b/apps/host/server/agent/agent-bridge.mjs
@@ -44,6 +44,14 @@ export function plainAgentFailure(raw) {
return text;
}
+/** Attached pictures (data addresses) as the Harness's image prompt parts. */
+export function imageParts(images) {
+ return (Array.isArray(images) ? images : []).map(i => {
+ const m = /^data:(image\/(?:png|jpeg|webp));base64,(.+)$/.exec(String(i?.url ?? ''));
+ return m ? { type: 'image', mediaType: m[1], data: m[2], name: String(i.name ?? 'Picture').slice(0, 120) } : null;
+ }).filter(Boolean);
+}
+
/** A timer that never keeps the process alive by itself. */
const after = (ms, fn) => { const t = setTimeout(fn, ms); t.unref?.(); return t; };
const deferred = () => { let resolve; const promise = new Promise(r => { resolve = r; }); return { promise, resolve }; };
@@ -260,7 +268,7 @@ export function createAgentBridge({ dataDir, componentsDir = null, bundledDir =
try { return await runTurnInner(c, options); } finally { c.starting = false; }
}
- async function runTurnInner(c, { chatId, message, workspacePath, choice = null, mode = null, history = [], onFrame, requestId = `req-${randomUUID()}`, signal, goal = null, thinking = null, thinkingBudget = 0, planTask = null, planRun = null, researchTopic = null }) {
+ async function runTurnInner(c, { chatId, message, workspacePath, choice = null, mode = null, history = [], onFrame, requestId = `req-${randomUUID()}`, signal, goal = null, thinking = null, thinkingBudget = 0, images = [], planTask = null, planRun = null, researchTopic = null }) {
let send = onFrame; let attached = true;
// A research run gathers first; what the assistant writes while gathering is notes, not the answer, so it is not shown.
let hideDeltas = !!researchTopic;
@@ -683,7 +691,7 @@ export function createAgentBridge({ dataDir, componentsDir = null, bundledDir =
let prompt = message;
if (fresh && history.length) prompt = withEarlierConversation(history, message);
try {
- await runtime.gw.rpc('session/prompt', { request: { requestId, sessionId, mode: 'queue', content: [{ type: 'text', text: prompt }] } });
+ await runtime.gw.rpc('session/prompt', { request: { requestId, sessionId, mode: 'queue', content: [{ type: 'text', text: prompt }, ...imageParts(images)] } });
} catch (e) {
finishTurn(`The agent did not accept the message: ${e.message}`);
return { error: true };
diff --git a/apps/host/server/agent/profile.mjs b/apps/host/server/agent/profile.mjs
index c6b2d0b0b..0bed92ac9 100644
--- a/apps/host/server/agent/profile.mjs
+++ b/apps/host/server/agent/profile.mjs
@@ -229,7 +229,7 @@ export function buildProfile({ local = [], connections = [], gate = true, tools
models: conn.models.map(m => ({
id: m.id, name: m.name || m.id,
contextWindow: m.contextMax ?? ASSUMED_API_CONTEXT, maxTokens: outputBudget(m.contextMax ?? ASSUMED_API_CONTEXT, m.maxOutput),
- input: ['text'],
+ input: ownServer ? ['text', 'image'] : ['text'],
...(ownServer ? { reasoning: true, reasoningEfforts: LOCAL_EFFORTS } : {}),
})),
};
diff --git a/apps/host/server/chat-images.mjs b/apps/host/server/chat-images.mjs
index 59428a7f7..d2738c3ae 100644
--- a/apps/host/server/chat-images.mjs
+++ b/apps/host/server/chat-images.mjs
@@ -1 +1 @@
-export function validateChatImages(images){if(images===undefined)return [];if(!Array.isArray(images)||images.length>3)throw new Error('Attach up to three pictures per message.');return images.map(i=>{if(!i||typeof i.url!=='string'||!/^data:image\/(png|jpeg|webp);base64,[A-Za-z0-9+/]+={0,2}$/.test(i.url))throw new Error('Pictures must be attached PNG, JPEG or WebP files.');if(Buffer.byteLength(i.url.split(',')[1],'base64')>2*1024*1024)throw new Error('Choose pictures smaller than 2 MB.');return {url:i.url,name:String(i.name??'Picture').slice(0,256)};});}
+export function validateChatImages(images){if(images===undefined)return [];if(!Array.isArray(images)||images.length>4)throw new Error('Attach up to four pictures per message.');return images.map(i=>{if(!i||typeof i.url!=='string'||!/^data:image\/(png|jpeg|webp);base64,[A-Za-z0-9+/]+={0,2}$/.test(i.url))throw new Error('Pictures must be attached PNG, JPEG or WebP files.');if(Buffer.byteLength(i.url.split(',')[1],'base64')>6*1024*1024)throw new Error('Choose pictures smaller than 6 MB.');return {url:i.url,name:String(i.name??'Picture').slice(0,256)};});}
diff --git a/apps/host/server/chat-uploads.mjs b/apps/host/server/chat-uploads.mjs
new file mode 100644
index 000000000..6dd5f9a5d
--- /dev/null
+++ b/apps/host/server/chat-uploads.mjs
@@ -0,0 +1,125 @@
+/**
+ * Files and pictures dropped or pasted into a chat.
+ *
+ * `POST /api/chat/upload?sessionId=&name=` takes the bytes in any of three forms: multipart/form-data (one or more files), a JSON body
+ * { name, dataUrl } (a pasted or read picture), or the raw bytes with the file's type. A picture is kept as an upload the next message
+ * can name (imageIds) and is shown again by its id; any other file is saved inside the chat's folder under .strata/attachments and named
+ * in the next message (@path), so the assistant reads it. Nothing is ever written outside those two places.
+ */
+
+import { randomUUID } from 'node:crypto';
+import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, unlinkSync, writeFileSync } from 'node:fs';
+import { basename, extname, join } from 'node:path';
+
+export const IMAGE_TYPES = { 'image/png': 'png', 'image/jpeg': 'jpg', 'image/webp': 'webp' };
+export const MAX_UPLOAD_BYTES = 12 * 1024 * 1024;
+export const MAX_IMAGE_BYTES = 6 * 1024 * 1024;
+export const MAX_IMAGES = 4;
+
+const safeName = name => String(name ?? 'file').replace(/[\\/:*?"<>|\x00-\x1f]/g, '_').replace(/^\.+/, '').slice(0, 120) || 'file';
+
+/** The files of a multipart/form-data body: [{ name, type, bytes }]. */
+export function parseMultipart(body, contentType) {
+ const m = /boundary=(?:"([^"]+)"|([^;]+))/i.exec(String(contentType ?? ''));
+ if (!m) throw new Error('This upload is not readable. Try again.');
+ const boundary = Buffer.from('--' + (m[1] ?? m[2]).trim());
+ const files = [];
+ let at = body.indexOf(boundary);
+ while (at >= 0) {
+ const next = body.indexOf(boundary, at + boundary.length);
+ if (next < 0) break;
+ let part = body.subarray(at + boundary.length, next);
+ if (part.subarray(0, 2).toString() === '\r\n') part = part.subarray(2);
+ const head = part.indexOf('\r\n\r\n');
+ if (head > 0) {
+ const headers = part.subarray(0, head).toString('utf8');
+ let data = part.subarray(head + 4);
+ if (data.subarray(-2).toString() === '\r\n') data = data.subarray(0, -2);
+ const file = /filename="([^"]*)"/i.exec(headers);
+ if (file) files.push({ name: file[1] || 'file', type: (/content-type:\s*([^\r\n;]+)/i.exec(headers)?.[1] ?? 'application/octet-stream').trim().toLowerCase(), bytes: Buffer.from(data) });
+ }
+ at = next;
+ }
+ if (!files.length) throw new Error('No file was in this upload.');
+ return files;
+}
+
+const typeFromBytes = b => (b.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])) ? 'image/png' : b.subarray(0, 3).equals(Buffer.from([255, 216, 255])) ? 'image/jpeg' : b.subarray(0, 4).toString() === 'RIFF' && b.subarray(8, 12).toString() === 'WEBP' ? 'image/webp' : null);
+
+export function createUploads({ dir, folderOf, attachments }) {
+ mkdirSync(dir, { recursive: true });
+ const index = new Map(); // id -> { id, chatId, name, type, size, file }
+ const metaFile = id => join(dir, id + '.json');
+ const load = id => {
+ if (index.has(id)) return index.get(id);
+ try { const m = JSON.parse(readFileSync(metaFile(id), 'utf8')); index.set(id, m); return m; } catch { return null; }
+ };
+
+ /** Store one file; returns what the page needs to show it. */
+ function saveOne(chatId, f) {
+ if (f.bytes.length > MAX_UPLOAD_BYTES) throw Object.assign(new Error(`"${safeName(f.name)}" is bigger than ${MAX_UPLOAD_BYTES / 1024 / 1024} MB.`), { code: 'too-big' });
+ // A picture is recognised by its first bytes, not by the name or the type the sender claimed.
+ const real = typeFromBytes(f.bytes);
+ if (real) {
+ if (f.bytes.length > MAX_IMAGE_BYTES) throw Object.assign(new Error(`The picture "${safeName(f.name)}" is bigger than ${MAX_IMAGE_BYTES / 1024 / 1024} MB. Choose a smaller one.`), { code: 'too-big' });
+ const id = randomUUID();
+ const file = join(dir, id + '.' + IMAGE_TYPES[real]);
+ writeFileSync(file, f.bytes);
+ const meta = { id, chatId, name: safeName(f.name), type: real, size: f.bytes.length, file, kind: 'image' };
+ writeFileSync(metaFile(id), JSON.stringify(meta)); index.set(id, meta);
+ return { id, kind: 'image', name: meta.name, type: real, size: meta.size, url: `/api/chat/upload/${id}` };
+ }
+ if (/^image\//i.test(f.type)) throw Object.assign(new Error(`"${safeName(f.name)}" is a picture type the model cannot read. Use PNG, JPEG or WebP.`), { code: 'bad-image' });
+ const root = folderOf(chatId);
+ const target = join(root, '.strata', 'attachments');
+ mkdirSync(target, { recursive: true });
+ let name = safeName(f.name), n = 1;
+ while (existsSync(join(target, name))) name = `${basename(f.name, extname(f.name))}-${++n}${extname(f.name)}`.replace(/[\\/:*?"<>|]/g, '_');
+ const path = join(target, name);
+ writeFileSync(path, f.bytes);
+ attachments.add(chatId, path);
+ return { id: null, kind: 'file', name, type: f.type, size: f.bytes.length, path, attached: true };
+ }
+
+ return {
+ /** Accept a request body in any of the three forms. */
+ accept(chatId, { body, contentType, name }) {
+ const ct = String(contentType ?? '');
+ let files;
+ if (/^multipart\/form-data/i.test(ct)) files = parseMultipart(body, ct);
+ else if (/^application\/json/i.test(ct)) {
+ const j = JSON.parse(body.toString('utf8') || '{}');
+ const list = Array.isArray(j.files) ? j.files : [j];
+ files = list.map(x => {
+ const m = /^data:([^;,]+);base64,([A-Za-z0-9+/=]+)$/.exec(String(x.dataUrl ?? x.url ?? ''));
+ if (!m) throw new Error('A picture must be sent as a data address (data:image/png;base64,...).');
+ return { name: x.name ?? 'Picture', type: m[1], bytes: Buffer.from(m[2], 'base64') };
+ });
+ } else files = [{ name: name ?? 'file', type: ct || 'application/octet-stream', bytes: body }];
+ if (files.length > 10) throw new Error('Drop up to ten files at a time.');
+ return files.map(f => saveOne(chatId, f));
+ },
+ /** The picture of an upload as { bytes, type, name }, for the page and for the model. */
+ image(id) {
+ const m = load(String(id));
+ if (!m || m.kind !== 'image') return null;
+ try { return { bytes: readFileSync(m.file), type: m.type, name: m.name, chatId: m.chatId }; } catch { return null; }
+ },
+ /** The pictures a message names (imageIds), as data addresses the engine and the agent take. */
+ dataUrls(chatId, ids) {
+ const list = Array.isArray(ids) ? ids : [];
+ if (list.length > MAX_IMAGES) throw new Error(`Attach up to ${MAX_IMAGES} pictures per message.`);
+ return list.map(id => {
+ const img = this.image(id);
+ if (!img || (img.chatId && chatId && img.chatId !== chatId)) throw new Error('A picture is no longer available. Attach it again.');
+ return { url: `data:${img.type};base64,${img.bytes.toString('base64')}`, name: img.name };
+ });
+ },
+ /** Remove uploads older than a day (the folder never grows without end). */
+ sweep(maxAgeMs = 24 * 3600_000) {
+ let removed = 0;
+ for (const f of readdirSync(dir)) { const p = join(dir, f); try { if (Date.now() - statSync(p).mtimeMs > maxAgeMs) { unlinkSync(p); removed += 1; } } catch { /* gone already */ } }
+ return removed;
+ },
+ };
+}
diff --git a/apps/host/server/index.mjs b/apps/host/server/index.mjs
index e89668a64..3645dcf61 100644
--- a/apps/host/server/index.mjs
+++ b/apps/host/server/index.mjs
@@ -46,7 +46,9 @@ import * as engineInstall from './engine-install.mjs';
import { detectBackend, currentBackend, streamChat } from './inference.mjs';
import { CONTEXT_TOKENS, VARIANTS, localModels, variant } from './catalog.mjs';
import { browseLocalModels, inspectLocalModel } from './local-models.mjs';
-import { EventHub, error, json, readBody, uid } from './util.mjs';
+import { EventHub, error, json, readBody, readRaw, uid } from './util.mjs';
+import { createUploads } from './chat-uploads.mjs';
+import { isPrivateHost } from './agent/llm-shape.mjs';
import { RequestQueue } from './request-queue.mjs';
import { selectSkills, skillInstructions } from './skill-selection.mjs';
import { createModelProviders } from './model-providers.mjs';
@@ -426,6 +428,8 @@ function loadWorkspaces() {
let workspaces = loadWorkspaces();
/* The folder each chat works in ("Select a folder"): the agent's working folder and sandbox for that chat. */
const chatAttachments = createAttachments();
+const uploads = createUploads({ dir: join(store.DATA_DIR, 'uploads'), folderOf: chatId => chatFolders.get(chatId) ?? workspaces[0]?.path ?? WORKSPACE, attachments: chatAttachments });
+try { uploads.sweep(); } catch { /* best effort */ }
const chatFolders = createChatFolders({ file: join(store.DATA_DIR, 'chat-folders.json') });
function saveWorkspaces() {
@@ -487,7 +491,7 @@ function clientName(req) {
return 'Local client';
}
-async function runRemoteTurn({ selection, message, history, onFrame, signal, workspaceId }) {
+async function runRemoteTurn({ selection, message, history, images = [], onFrame, signal, workspaceId }) {
const controller = new AbortController();
const requestSignal = signal ? AbortSignal.any([signal, controller.signal]) : controller.signal;
let entry;
@@ -497,7 +501,7 @@ async function runRemoteTurn({ selection, message, history, onFrame, signal, wor
const appliedSkills = selectSkills(skillState, message, workspaceId);
entry = serving.open({ client: 'Strata chat', endpoint: connection.protocol === 'anthropic' ? '/messages' : connection.protocol === 'openai-responses' ? '/responses' : '/chat/completions', model: `${connection.name} · ${model.name}`, cancel: () => controller.abort(new DOMException('Cancelled by the user', 'AbortError')) });
serving.mark(entry.id, { state: 'generating' });
- const result = await modelProviders.stream({ selection, message, history, signal: requestSignal, systemPrompt: ['You are Strata, a helpful assistant. Answer clearly. Do not claim to have read files or performed actions unless their results are included in the conversation.', skillInstructions(appliedSkills)].filter(Boolean).join('\n\n'), onFrame: frame => {
+ const result = await modelProviders.stream({ selection, message, history, images, signal: requestSignal, systemPrompt: ['You are Strata, a helpful assistant. Answer clearly. Do not claim to have read files or performed actions unless their results are included in the conversation.', skillInstructions(appliedSkills)].filter(Boolean).join('\n\n'), onFrame: frame => {
if (frame.type === 'start') noteSkillUse(appliedSkills);
if (frame.type === 'done') serving.mark(entry.id, { tokensIn: frame.tokensIn, tokensOut: frame.tokensOut, usageSource: frame.usageSource });
onFrame({ ...frame, requestId: entry.id, model: model.name, contextMax: model.contextMax, remoteModel: true, ...(frame.type === 'start' ? { skills: appliedSkills.map(s => s.name) } : {}) });
@@ -514,7 +518,7 @@ async function runRemoteTurn({ selection, message, history, onFrame, signal, wor
}
/** One agent turn for a chat, streamed to the caller as the NDJSON frames the chat already renders. */
-async function runAgentTurn({ res, controller, chatId, body, message, workspaceId }) {
+async function runAgentTurn({ res, controller, chatId, body, message, workspaceId, images = [] }) {
const write = frame => { if (!res.writableEnded && !res.destroyed) res.write(`${JSON.stringify(frame)}\n`); };
const cwd = chatFolders.get(chatId) ?? (workspaceId ? workspaces.find(w => w.id === workspaceId)?.path : null) ?? WORKSPACE;
let choice, instance = null;
@@ -536,7 +540,7 @@ async function runAgentTurn({ res, controller, chatId, body, message, workspaceI
const goalLine = /^\/goal(?:\s+([\s\S]*))?$/i.exec(message.trim());
const planLine = /^\/plan(?:\s+([\s\S]*))?$/i.exec(message.trim());
const researchLine = /^\/research(?:\s+([\s\S]*))?$/i.exec(message.trim());
- const turnOptions = { chatId, message, workspacePath: cwd, choice, mode: typeof body.mode === 'string' ? body.mode : null, thinking: typeof body.thinking === 'string' ? body.thinking : null, thinkingBudget: Number.isFinite(body.thinkingBudget) ? body.thinkingBudget : 0, history: Array.isArray(body.history) ? body.history : [], onFrame: write, signal: controller.signal };
+ const turnOptions = { chatId, message, workspacePath: cwd, choice, images, mode: typeof body.mode === 'string' ? body.mode : null, thinking: typeof body.thinking === 'string' ? body.thinking : null, thinkingBudget: Number.isFinite(body.thinkingBudget) ? body.thinkingBudget : 0, history: Array.isArray(body.history) ? body.history : [], onFrame: write, signal: controller.signal };
const result = typeof body.planId === 'string' && body.message === '/plan start' ? await agent.planGoal({ ...turnOptions, planId: body.planId })
: researchLine ? await agent.researchCommand({ ...turnOptions, line: researchLine[1] ?? '' })
: planLine ? await agent.planCommand({ ...turnOptions, line: planLine[1] ?? '' })
@@ -550,7 +554,7 @@ async function runAgentTurn({ res, controller, chatId, body, message, workspaceI
}
async function runTurn({ message, history = [], onFrame, signal, client, workspaceId = null, sessionId = null, remoteModel = null, runtimeModelId = undefined, nativeModelId, images=[], useTools=false, thinking='medium', thinkingBudget=0 }) {
- if (remoteModel) return runRemoteTurn({ selection: remoteModel, message, history, onFrame, signal, workspaceId });
+ if (remoteModel) return runRemoteTurn({ selection: remoteModel, message, history, images, onFrame, signal, workspaceId });
if(native.hasLive()||nativeModelId){
let instance;try{instance=native.resolveInstance(nativeModelId??native.state().defaultId);}catch(e){onFrame({type:'error',message:e.message});return {error:e.message};}const started=Date.now(),cancel=new AbortController();
const entry=serving.open({client:client??'Strata chat',endpoint:'/chat/completions',model:instance.apiModelId,instanceId:instance.id,gpuIds:instance.gpus.map(String),cancel:()=>cancel.abort()});
@@ -1537,6 +1541,19 @@ async function api(req, res, path, method, url) {
try { return json(res, 200, openFile(chatFolders.get(String(body.sessionId ?? '')), body.path, String(body.how ?? 'app'))); }
catch (e) { return error(res, e.code === 'outside' ? 403 : 400, e.message); }
}
+ /* Files and pictures dropped or pasted into a chat: multipart, a JSON data address, or raw bytes. A picture comes back with an id the next message names (imageIds). */
+ if (path === '/api/chat/upload' && method === 'POST') {
+ const sid = url.searchParams.get('sessionId') ?? '';
+ if (!sid) return error(res, 400, 'sessionId is required');
+ try { const body = await readRaw(req); return json(res, 200, { files: uploads.accept(sid, { body, contentType: req.headers['content-type'], name: url.searchParams.get('name') }) }); }
+ catch (e) { return error(res, e.code === 'too-big' ? 413 : 400, e.message); }
+ }
+ const uploadGet = /^\/api\/chat\/upload\/([0-9a-f-]{36})$/.exec(path);
+ if (uploadGet && method === 'GET') {
+ const img = uploads.image(uploadGet[1]);
+ if (!img) return error(res, 404, 'That picture is no longer available.');
+ res.writeHead(200, { 'content-type': img.type, 'content-length': img.bytes.length, 'cache-control': 'no-store', 'x-content-type-options': 'nosniff' }); res.end(img.bytes); return undefined;
+ }
if (path === '/api/files/preview' && method === 'GET') {
const sid = url.searchParams.get('sessionId') ?? '';
try { return json(res, 200, previewFile(chatFolders.get(sid), url.searchParams.get('path'), sid)); }
@@ -1732,8 +1749,9 @@ async function api(req, res, path, method, url) {
if ((path === '/api/chat/stream' || path === '/api/plan/start-goal') && method === 'POST') {
const body = await readBody(req,12*1024*1024);
if (path === '/api/plan/start-goal') { body.message = '/plan start'; if (typeof body.planId !== 'string') return error(res, 400, 'planId is required'); }
- let images;try{images=validateChatImages(body.images);}catch(e){return error(res,400,e.message);}
- if(images.length){if(body.remoteModel)return error(res,409,'Choose a local image-reading model before sending attached pictures.');let selected;try{selected=native.resolveInstance(body.nativeModelId??native.state().defaultId);}catch(e){return error(res,409,e.message);}if(!selected.vision)return error(res,409,'Enable Read pictures for this model before sending images.');}
+ let images;try{const named=Array.isArray(body.imageIds)&&body.imageIds.length?uploads.dataUrls(typeof body.sessionId==='string'?body.sessionId:'',body.imageIds):[];images=validateChatImages([...(Array.isArray(body.images)?body.images:[]),...named]);}catch(e){return error(res,400,e.message);}
+ if(images.length&&body.remoteModel){const own=modelProviders.withSecrets().find(c=>c.id===body.remoteModel.connectionId);if(!(own&&own.protocol==='openai-chat'&&isPrivateHost(own.baseUrl)))return error(res,409,'Choose a local image-reading model before sending attached pictures.');}
+ else if(images.length){let selected;try{selected=native.resolveInstance(body.nativeModelId??native.state().defaultId);}catch(e){return error(res,409,e.message);}if(!selected.vision)return error(res,409,'Enable Read pictures for this model before sending images.');}
// The goal bar's Resume has no words of its own: it is the same turn as "/goal resume".
const message = body.goalAction === 'resume' ? '/goal resume' : String(body.message ?? '').trim();
if (!message) return error(res, 400, 'message is required');
@@ -1755,13 +1773,12 @@ async function api(req, res, path, method, url) {
const agentSkip = process.env.STRATA_AGENT === 'off' ? 'The agent is turned off (STRATA_AGENT=off).'
: body.agent === false || body.tools === false ? 'This chat runs without the agent.'
: !chatId ? 'This request did not name a chat.'
- : images.length ? 'The agent does not take attached pictures yet; this message was sent directly.'
: !(body.remoteModel || native.hasLive()) ? 'The agent needs a loaded Strata model or an API model.'
: null;
if (chatId && !agentSkip) {
const capability = agent.status();
if (capability.state !== 'unavailable') {
- await runAgentTurn({ res, controller, chatId, body, message, workspaceId });
+ await runAgentTurn({ res, controller, chatId, body, message, workspaceId, images });
res.end();
return undefined;
}
diff --git a/apps/host/server/model-providers.mjs b/apps/host/server/model-providers.mjs
index e71c46df0..ecd7acc87 100644
--- a/apps/host/server/model-providers.mjs
+++ b/apps/host/server/model-providers.mjs
@@ -131,11 +131,13 @@ export function createModelProviders({ directory, changed = () => {}, fetchImpl
const models = [...new Map(rows.filter(m => text(m.id, 256)).map(m => [m.id, { id: text(m.id, 256), name: text(m.display_name ?? m.name) || text(m.id, 256), contextMax: numeric(m.context_length ?? m.max_context_length ?? m.context_window), maxOutput: numeric(m.max_output_tokens ?? m.max_completion_tokens ?? m.top_provider?.max_completion_tokens) }])).values()];
return { models: models.sort((a, b) => a.name.localeCompare(b.name)) };
}
- async function stream({ selection, message, history = [], systemPrompt = '', signal, onFrame }) {
+ async function stream({ selection, message, history = [], systemPrompt = '', images = [], signal, onFrame }) {
const { connection, model } = resolve(selection);
const started = Date.now();
const messages = history.filter(m => ['user', 'assistant'].includes(m.role) && typeof m.content === 'string').slice(-40).map(m => ({ role: m.role, content: m.content }));
- messages.push({ role: 'user', content: message });
+ // Pictures only go to OpenAI-style servers (the one protocol every local server shares); other protocols keep the text.
+ const pics = connection.protocol === 'openai-chat' ? images : [];
+ messages.push({ role: 'user', content: pics.length ? [{ type: 'text', text: message }, ...pics.map(i => ({ type: 'image_url', image_url: { url: i.url } }))] : message });
const protocol = connection.protocol;
const payload = protocol === 'anthropic'
? { model: model.id, messages, system: systemPrompt, max_tokens: 8192, stream: true }
diff --git a/apps/host/server/native-control.mjs b/apps/host/server/native-control.mjs
index 1d6f247b4..ee78fd77d 100644
--- a/apps/host/server/native-control.mjs
+++ b/apps/host/server/native-control.mjs
@@ -62,7 +62,7 @@ export function createNativeControl({directory,requests=()=>[],changed=()=>{},sp
for(const name of names){const path=join(dir,name),config=readJson(path,null);if(!config||typeof config!=='object')continue;
const next={...config,...(config.cwd!==undefined?{cwd:fix(config.cwd)}:{}),...(Array.isArray(config.args)?{args:config.args.map(fix)}:{})};
if(JSON.stringify(next)!==JSON.stringify(config)){try{writeModelConfig(path,next);}catch{/* read-only folder: the load reports it */}}}}
- function installedInfo(info){if(!info)return info;return {...info,models:(info.models??[]).map(entry=>{const cfg=/^strata-[a-z0-9_-]+\.json$/i.test(entry.config??'')?readJson(entry.configPath??join(repo(),entry.config)):{};const at=cfg.args?.indexOf('--native')??-1,nativeFile=at>=0?cfg.args[at+1]:null;const dataFolder=typeof nativeFile==='string'&&basename(dirname(dirname(nativeFile)))==='models'?dirname(dirname(dirname(nativeFile))):info.data_folder;return {...entry,diskGb:entry.diskGb??entry.disk_gb,diskBytes:entry.diskBytes??entry.disk_bytes,dataFolder,visionEnabled:cfg.ui_vision_enabled??[...instances.values()].some(i=>i.model===entry.model&&i.status==='ready'&&i.vision),visionDevice:cfg.ui_vision_device,resourceProfile:cfg.ui_resource_profile,lowRamMode:cfg.args?.includes('--resident-experts')?'resident':cfg.args?.includes('--mmap-experts')?'mmap':'auto'};})};}
+ function installedInfo(info){if(!info)return info;return {...info,models:(info.models??[]).map(entry=>{const cfg=/^strata-[a-z0-9_-]+\.json$/i.test(entry.config??'')?readJson(entry.configPath??join(repo(),entry.config)):{};const at=cfg.args?.indexOf('--native')??-1,nativeFile=at>=0?cfg.args[at+1]:null;const dataFolder=typeof nativeFile==='string'&&basename(dirname(dirname(nativeFile)))==='models'?dirname(dirname(dirname(nativeFile))):info.data_folder;return {...entry,diskGb:entry.diskGb??entry.disk_gb,diskBytes:entry.diskBytes??entry.disk_bytes,dataFolder,visionPrepared:!!cfg.vision,visionEnabled:cfg.ui_vision_enabled??[...instances.values()].some(i=>i.model===entry.model&&i.status==='ready'&&i.vision),visionDevice:cfg.ui_vision_device,resourceProfile:cfg.ui_resource_profile,lowRamMode:cfg.args?.includes('--resident-experts')?'resident':cfg.args?.includes('--mmap-experts')?'mmap':'auto'};})};}
function installRoot(){const found=[preferences.installDir,packagedInstall,snapshot.installDir].find(d=>d&&existsSync(d));if(found)return realpathSync(found);mkdirSync(defaultInstall(),{recursive:true});return realpathSync(defaultInstall());}
function repo(){if(packaged)return installRoot();const path=realpathSync(preferences.repo);if(!existsSync(join(path,'setup.py'))||!existsSync(join(path,'tools/strata_mcp.py'))||!existsSync(join(path,'serve/server.py')))throw new Error('Choose the Strata repository folder containing setup.py and tools/strata_mcp.py.');return path;}
function python(){return resolvePython({repo:preferences.repo});}
@@ -118,7 +118,7 @@ export function createNativeControl({directory,requests=()=>[],changed=()=>{},sp
function save(model,{values,mcp}){if([...jobs.values()].some(j=>j.state==='running'))throw new Error('Finish the current operation before saving native settings.');const {path,config}=record(model);if(busy(requests().filter(r=>instances.get(r.instanceId)?.model===model)))throw new Error('Finish this model’s requests before saving its settings.');const patch=validateNativeSettings(values??{}),next={...config};for(const field of nativeFields())if(field.supported===false&&patch[field.id]!=null&&JSON.stringify(patch[field.id])!==JSON.stringify(config[field.id]))throw new Error(`${field.label} needs a newer native Strata installation. Update it in Settings → About.`);for(const [key,value] of Object.entries(patch)){if(key==='env')next.env=restoreNativeEnv(value,config.env??{});else if(value===null)delete next[key];else next[key]=value;}if(mcp!==undefined){next.mcp_servers=validateMcpServers(mcp,config.mcp_servers??config.mcpServers??{});delete next.mcpServers;}copyFileSync(path,path+'.bak');chmodSync(path+'.bak',0o600);const temp=path+'.ui-tmp';writeFileSync(temp,JSON.stringify(next,null,2)+'\n',{mode:0o600});renameSync(temp,path);return settings(model);}
function previousEngine(){if(packaged)return !!snapshot.engine?.previous;try{const data=snapshot.installed?.data_dir??snapshot.installed?.data_folder;return [join(preferences.repo,'engine/.previous'),data&&join(data,'engine/.previous')].filter(Boolean).some(existsSync)&&readFileSync(join(preferences.repo,'setup.py'),'utf8').includes('--rollback-engine');}catch{return false;}}
async function plan(args){await refresh();if(!snapshot.available)throw new Error(snapshot.error);return tool('strata_install',installArgs(args,false));}
- function installArgs(args,confirm){const allowed=['family','model','context','vision','kv','gpu','gpus','backend','low_ram','port','gguf_dir'];const size=snapshot.catalog?.families?.find(f=>f.family===args.family)?.sizes.find(s=>s.model===args.model);const existing=snapshot.installed?.models?.find(m=>m.model===size?.id);/* A model adopted from the person's own GGUF folder reports that folder's parent as its data folder: it is theirs, not Strata's, so a second preparation never uses it for the packs. */const data=nativeDataDirectory(args.data_dir??(args.gguf_dir?undefined:existing?.dataFolder)??preferences.downloadDirectory??modelsDirectory?.());return {...(data?{data_dir:data}:{}),...Object.fromEntries(allowed.filter(k=>args[k]!==undefined).map(k=>[k,args[k]])),...(packaged?{vision:'no'}:{}),confirm};}
+ function installArgs(args,confirm){const allowed=['family','model','context','vision','kv','gpu','gpus','backend','low_ram','port','gguf_dir'];const size=snapshot.catalog?.families?.find(f=>f.family===args.family)?.sizes.find(s=>s.model===args.model);const existing=snapshot.installed?.models?.find(m=>m.model===size?.id);/* A model adopted from the person's own GGUF folder reports that folder's parent as its data folder: it is theirs, not Strata's, so a second preparation never uses it for the packs. */const data=nativeDataDirectory(args.data_dir??(args.gguf_dir?undefined:existing?.dataFolder)??preferences.downloadDirectory??modelsDirectory?.());return {...(data?{data_dir:data}:{}),...Object.fromEntries(allowed.filter(k=>args[k]!==undefined).map(k=>[k,args[k]])),/* Picture reading is only downloaded when it was asked for (Read pictures); the default stays the smaller download. */...(packaged&&args.vision===undefined?{vision:'no'}:{}),confirm};}
function begin(kind,run,metadata={}){if([...jobs.values()].some(j=>j.state==='running'))throw new Error('Finish or cancel the current engine operation first.');const job={id:randomUUID(),kind,state:'running',startedAt:new Date().toISOString(),summary:'Starting…',result:null,error:null,...metadata};for(const [id,old] of jobs)if(jobs.size>=12&&old.state!=='running')jobs.delete(id);jobs.set(job.id,job);emit();void run(job).then(result=>{if(job.state==='running'){Object.assign(job,{state:'done',summary:result?.summary??'Complete',result,finishedAt:new Date().toISOString()});}}).catch(error=>{if(job.state==='running')Object.assign(job,{state:'error',error:error.message,summary:error.message,...(error.details?{details:error.details}:{}),finishedAt:new Date().toISOString()});}).finally(()=>{if(job.kind.startsWith('update')||job.kind==='rollback')disconnect();lastRead=0;emit();void refresh(true);});return state().jobs.find(j=>j.id===job.id);}
async function watchInstall(job,current){job.summary=current.step??'Resuming preparation status…';job.result={install:current};while(job.state==='running'){await new Promise(r=>setTimeout(r,1500));const result=await tool('strata_install');job.summary=result.summary;job.result=result;emit();if(!result.install?.running){if(result.install?.exit_code!==0)throw new Error(result.install?.error??'Preparation did not finish. Retry to reuse completed files.');return {summary:'Model preparation complete. Load it from Models.'};}}}
diff --git a/apps/host/server/util.mjs b/apps/host/server/util.mjs
index 434d4ac0a..9e97f1cde 100644
--- a/apps/host/server/util.mjs
+++ b/apps/host/server/util.mjs
@@ -93,3 +93,15 @@ export class EventHub {
return this.#clients.size;
}
}
+
+/** The raw bytes of a request body (an upload), up to `limit`. */
+export async function readRaw(req, limit = 16 * 1024 * 1024) {
+ const chunks = [];
+ let size = 0;
+ for await (const chunk of req) {
+ size += chunk.length;
+ if (size > limit) throw new Error('That upload is too big.');
+ chunks.push(chunk);
+ }
+ return Buffer.concat(chunks);
+}
diff --git a/apps/host/tests/chat-uploads.test.mjs b/apps/host/tests/chat-uploads.test.mjs
new file mode 100644
index 000000000..48b5b83c8
--- /dev/null
+++ b/apps/host/tests/chat-uploads.test.mjs
@@ -0,0 +1,51 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtempSync, mkdirSync, rmSync, readFileSync, existsSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { createUploads, parseMultipart } from '../server/chat-uploads.mjs';
+import { createAttachments } from '../server/chat-files.mjs';
+import { imageParts } from '../server/agent/agent-bridge.mjs';
+
+const PNG = Buffer.concat([Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), Buffer.alloc(40, 1)]);
+const multipart = (parts, boundary = 'XBOUND') => ({
+ contentType: 'multipart/form-data; boundary=' + boundary,
+ body: Buffer.concat([...parts.flatMap(p => [Buffer.from('--' + boundary + '\r\nContent-Disposition: form-data; name="file"; filename="' + p.name + '"\r\nContent-Type: ' + p.type + '\r\n\r\n'), p.bytes, Buffer.from('\r\n')]), Buffer.from('--' + boundary + '--\r\n')]),
+});
+
+test('pictures and files arrive as multipart, a data address or raw bytes; pictures get an id, files land in the chat folder', () => {
+ const dir = mkdtempSync(join(tmpdir(), 'strata-up-'));
+ try {
+ const proj = join(dir, 'proj'); mkdirSync(proj);
+ const att = createAttachments();
+ const up = createUploads({ dir: join(dir, 'uploads'), folderOf: () => proj, attachments: att });
+ const m = multipart([{ name: 'a.png', type: 'image/png', bytes: PNG }, { name: 'notes.md', type: 'text/markdown', bytes: Buffer.from('# hi\r\nthere') }]);
+ assert.equal(parseMultipart(m.body, m.contentType).length, 2);
+ const out = up.accept('c1', m);
+ assert.equal(out[0].kind, 'image'); assert.match(out[0].url, /^\/api\/chat\/upload\/[0-9a-f-]{36}$/);
+ assert.equal(out[1].kind, 'file'); assert.ok(existsSync(join(proj, '.strata', 'attachments', 'notes.md'))); assert.deepEqual(att.list('c1'), [out[1].path]);
+ assert.equal(readFileSync(out[1].path, 'utf8'), '# hi\r\nthere', 'the bytes are kept exactly');
+ const j = up.accept('c1', { body: Buffer.from(JSON.stringify({ name: 'p.png', dataUrl: 'data:image/png;base64,' + PNG.toString('base64') })), contentType: 'application/json' });
+ assert.equal(j[0].kind, 'image');
+ const raw = up.accept('c1', { body: PNG, contentType: 'image/png', name: 'x.png' });
+ assert.equal(up.image(raw[0].id).type, 'image/png');
+ assert.equal(up.dataUrls('c1', [raw[0].id])[0].url.slice(0, 22), 'data:image/png;base64,');
+ assert.throws(() => up.dataUrls('other-chat', [raw[0].id]), /no longer available/, 'a picture belongs to its chat');
+ assert.throws(() => up.dataUrls('c1', ['1', '2', '3', '4', '5']), /up to 4/);
+ // the first bytes decide: a renamed non-picture is a file, a picture type the model cannot read is refused
+ assert.equal(up.accept('c1', { body: Buffer.from('plain'), contentType: 'text/plain', name: 'renamed.png' })[0].kind, 'file');
+ assert.throws(() => up.accept('c1', { body: Buffer.from('plain'), contentType: 'image/png', name: 'fake.png' }), /cannot read/);
+ assert.throws(() => up.accept('c1', { body: Buffer.from('GIF89a....'), contentType: 'image/gif', name: 'a.gif' }), /PNG, JPEG or WebP/);
+ const bigPng = Buffer.concat([PNG, Buffer.alloc(7 * 1024 * 1024)]);
+ assert.throws(() => up.accept('c1', { body: bigPng, contentType: 'image/png', name: 'big.png' }), /bigger than 6 MB/);
+ assert.throws(() => up.accept('c1', { body: Buffer.alloc(13 * 1024 * 1024), contentType: 'application/pdf', name: 'huge.pdf' }), /bigger than 12 MB/);
+ const evil = up.accept('c1', { body: Buffer.from('x'), contentType: 'text/plain', name: '..' + String.fromCharCode(92) + '..' + String.fromCharCode(92) + 'evil.txt' });
+ assert.ok(evil[0].path.startsWith(join(proj, '.strata', 'attachments')), 'a file name cannot climb out of the attachments folder');
+ } finally { rmSync(dir, { recursive: true, force: true }); }
+});
+
+test('pictures become the Harness image parts; anything else is dropped', () => {
+ const parts = imageParts([{ url: 'data:image/png;base64,AAAA', name: 'a.png' }, { url: 'data:image/gif;base64,AAAA' }, { url: 'nope' }, null]);
+ assert.deepEqual(parts, [{ type: 'image', mediaType: 'image/png', data: 'AAAA', name: 'a.png' }]);
+ assert.deepEqual(imageParts(undefined), []);
+});