Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Pictures on by default everywhere: paste, shrink big pictures, one status for composer and Send, plain tool error, per-model off switch wording

Niko1221af2605356545
diff --git a/apps/host/server/agent/events.mjs b/apps/host/server/agent/events.mjs
index 749d4b2e7..cbf158238 100644
--- a/apps/host/server/agent/events.mjs
+++ b/apps/host/server/agent/events.mjs
@@ -146,6 +146,7 @@ export function plainToolError(text) {
if (/cannot modify|must (?:first )?be read|has not been read|changed since/i.test(t)) return 'The file could not be changed. It may need to be read again first, or it changed in the meantime.';
if (/sandbox|access denied|permission denied|eacces|eperm/i.test(t)) return 'The computer did not allow this.';
if (/timed out|timeout/i.test(t)) return 'This took too long and was stopped.';
+ if (/does not declare image input|image input/i.test(t)) return 'Picture reading is off for this model. Open Models and turn on Read pictures for it (load it again if it is a local model), or choose a model that reads pictures.';
if (/extension|not a supported image|cannot read/i.test(t)) return 'This file could not be read.';
return null;
}
diff --git a/apps/host/server/agent/preset.mjs b/apps/host/server/agent/preset.mjs
index cf85bfd26..6088b4f93 100644
--- a/apps/host/server/agent/preset.mjs
+++ b/apps/host/server/agent/preset.mjs
@@ -44,7 +44,7 @@ export const AGENT_PROMPT = [
'Look before you change: read a file before you edit or overwrite it, and prefer a small edit to a rewrite. Search with the glob and grep tools and read files with the read tool; use the shell for running programs.',
'When you start a program or open a folder or file for the person, the command returning without an error means it worked: say plainly that it is open. Do not say you could not confirm it. If a command fails, say what went wrong in one sentence and what you will try next.',
'To open a page or a file for the person (a browser, an app, a folder), start it so the command returns at once and never wait for the program to close: on Windows use Start-Process, on macOS open, on Linux xdg-open. Then say plainly that it is open.',
- 'When the person asks you to ask a subagent, a helper or another model ("ask a subagent", "ask DeepSeek", "use the local model"), start it with the subagent tool, naming the model with its model field. Never look for a model with shell commands or in the environment, and never wait for a helper with sleep: end your turn and its answer arrives as a new message.',
+ 'When the person asks you to ask a subagent, a helper or another model ("ask a subagent", "ask model X"), use the subagent tool and name the model in its model field. Never look for a model with shell commands, and never wait for a helper with sleep: end your turn; its answer arrives as a new message.',
'Tokens prefixed with @ are files or folders the person pointed at. A failed or refused action changed nothing; say so, and do not claim work you did not do.',
].join('\n\n');
diff --git a/apps/host/server/index.mjs b/apps/host/server/index.mjs
index a4f8d67fb..b5e48f11e 100644
--- a/apps/host/server/index.mjs
+++ b/apps/host/server/index.mjs
@@ -1843,7 +1843,7 @@ async function api(req, res, path, method, url) {
if (path === '/api/plan/start-goal') { body.message = '/plan start'; if (typeof body.planId !== 'string') return error(res, 400, 'planId is required'); }
let images;try{const named=Array.isArray(body.imageIds)&&body.imageIds.length?uploads.dataUrls(typeof body.sessionId==='string'?body.sessionId:'',body.imageIds):[];images=validateChatImages([...(Array.isArray(body.images)?body.images:[]),...named]);}catch(e){return error(res,400,e.message);}
if(images.length&&body.remoteModel){const own=modelProviders.withSecrets().find(c=>c.id===body.remoteModel.connectionId);const row=own?.models.find(m=>m.id===body.remoteModel.modelId);const refused=await pictureRefusal(own,row);if(refused)return error(res,409,refused);}
- else if(images.length){let selected;try{selected=native.resolveInstance(body.nativeModelId??native.state().defaultId);}catch(e){return error(res,409,e.message);}if(!selected.vision)return error(res,409,'Enable Read pictures for this model before sending images.');}
+ else if(images.length){let selected;try{selected=native.resolveInstance(body.nativeModelId??native.state().defaultId);}catch(e){return error(res,409,e.message);}if(!selected.vision)return error(res,409,'This model was started without picture reading. Open Models, turn on Read pictures and load it again, or choose a model that reads pictures.');}
// The goal bar's Resume has no words of its own: it is the same turn as "/goal resume".
const message = body.goalAction === 'resume' ? '/goal resume' : String(body.message ?? '').trim();
if (!message) return error(res, 400, 'message is required');
diff --git a/apps/host/src/components/ComposerControls.tsx b/apps/host/src/components/ComposerControls.tsx
index 2a9f5fb00..193e35af0 100644
--- a/apps/host/src/components/ComposerControls.tsx
+++ b/apps/host/src/components/ComposerControls.tsx
@@ -10,6 +10,7 @@ import { artifactFromUrl, type PreviewArtifact } from '../lib/artifacts';
import type { ChatMode, Session } from '../lib/types';
import { attachChatPhoto } from '../lib/chat-photos';
import { useNative } from '../lib/native';
+import { pictureStatus } from '../lib/pictures';
const modes: {
value: ChatMode;
label: string;
@@ -51,6 +52,7 @@ export function ComposerControls({ session, onPreview, onAttach, onError, onSlas
const addFilesLatest = useRef<(files: File[]) => Promise<void>>(async () => { });
// A model on an API takes a reasoning effort when its provider says it can think (OpenRouter's model list); otherwise the provider decides.
const apiOnlyDefault = !!apiModel && apiModel.thinks !== true;
+ const pictures = pictureStatus(native, remoteModel, providers);
const mode = modes.find(m => m.value === session?.preferences?.mode) ?? modes[0];
const effort = efforts.includes(session?.preferences?.thinking ?? '') ? session!.preferences!.thinking! : 'Medium';
const budget = session?.preferences?.thinkingBudget ?? 0;
@@ -85,14 +87,14 @@ export function ComposerControls({ session, onPreview, onAttach, onError, onSlas
skipped++;
continue;
}
- if (!directory && photoTypes.test(file.name) && (remoteModel ? apiModel?.reads === true : nativeModel?.vision)) {
+ if (!directory && photoTypes.test(file.name) && pictures.ok) {
await attachChatPhoto(file, session?.id);
continue;
}
if (!directory && (photoTypes.test(file.name) || /\.pdf$/i.test(file.name))) {
await openFile(file);
if (photoTypes.test(file.name))
- onError(remoteModel ? `Strata cannot tell that "${apiModel?.name ?? 'this model'}" reads pictures, so the picture is only open in preview. If it does, open Settings > Models, edit its connection and tick "This model reads pictures". Otherwise choose a model that reads pictures.` : 'Enable Read pictures in Models to send this image to the model. It is open in preview.');
+ onError(`${pictures.ok ? '' : pictures.message} It is open in preview.`);
continue;
}
if (!textTypes.test(file.name) || bytes + file.size > 256 * 1024) {
diff --git a/apps/host/src/components/DropZone.tsx b/apps/host/src/components/DropZone.tsx
index 44c935a64..2c8fce5b0 100644
--- a/apps/host/src/components/DropZone.tsx
+++ b/apps/host/src/components/DropZone.tsx
@@ -4,6 +4,16 @@ import { Icon } from './ui';
/** Files dropped on the chat go where the attach button would put them: the composer reads the 'strata:drop-files' event. */
export const DROP_EVENT = 'strata:drop-files';
+/** The pictures on the clipboard, named so each one is told apart in the composer (a pasted screenshot arrives as "image.png"). */
+export function pastedFiles(data: DataTransfer | null | undefined): File[] {
+ const out: File[] = [];
+ for (const file of Array.from(data?.files ?? [])) {
+ if (!file.type.startsWith('image/')) continue;
+ const extension = file.type.split('/')[1]?.replace('jpeg', 'jpg').replace(/[+].*/, '') || 'png';
+ out.push(new File([file], `Pasted picture ${new Date().toLocaleTimeString().replace(/[^0-9]+/g, '-').replace(/-$/, '')}${out.length ? `-${out.length + 1}` : ''}.${extension}`, { type: file.type }));
+ }
+ return out;
+}
const hasFiles = (e: React.DragEvent) => Array.from(e.dataTransfer?.types ?? []).includes('Files');
/** The conversation column, with a calm highlight while something is dragged over it. */
diff --git a/apps/host/src/components/GuidedSetup.tsx b/apps/host/src/components/GuidedSetup.tsx
index 4bbe713e6..f8fdf2039 100644
--- a/apps/host/src/components/GuidedSetup.tsx
+++ b/apps/host/src/components/GuidedSetup.tsx
@@ -27,7 +27,7 @@ export function GuidedSetup() {
const initialFamily = families.find(f => f.family === family || f.sizes.some(size => size.sourceFamily === family));
const [size, setSize] = useState(saved?.size ?? initialFamily?.sizes.find(s => s.model === native?.catalog?.recommendation?.model)?.id ?? initialFamily?.sizes[0]?.id ?? '');
const [context, setContext] = useState(saved?.context ?? native?.catalog?.recommendation?.context ?? 32768);
- const [vision, setVision] = useState(saved?.vision ?? false), [directory, setDirectory] = useState(saved?.directory ?? '');
+ const [vision, setVision] = useState(saved?.vision ?? true), [directory, setDirectory] = useState(saved?.directory ?? '');
const [profile, setProfile] = useState<'work' | 'performance'>(saved?.profile ?? 'work');
const [installJobId, setInstallJobId] = useState(saved?.installJobId ?? ''), [tuneJobId, setTuneJobId] = useState(saved?.tuneJobId ?? '');
const [plan, setPlan] = useState<Record<string, any> | null>(null), [tour, setTour] = useState(0), [folderOpen, setFolderOpen] = useState(false);
@@ -93,7 +93,7 @@ export function GuidedSetup() {
setError(e instanceof Error ? e.message : String(e));
}
}
- function chooseNative(family: string, size: string) { setFamily(family); setSize(size); setVision(false); setPlan(null); setError(''); }
+ function chooseNative(family: string, size: string) { setFamily(family); setSize(size); setVision(true); setPlan(null); setError(''); }
useEffect(() => { titleRef.current?.focus({ preventScroll: true }); titleRef.current?.closest('.setupmodal__surface')?.scrollTo({ top: 0 }); }, [step]);
useEffect(() => {
if (step < 2 || leaving.current)
diff --git a/apps/host/src/components/NativeModels.tsx b/apps/host/src/components/NativeModels.tsx
index 0b9d01d9f..4f5e10a59 100644
--- a/apps/host/src/components/NativeModels.tsx
+++ b/apps/host/src/components/NativeModels.tsx
@@ -30,7 +30,7 @@ export function NativeModels({ onOtherModels, onReady, onImport, onConnectEngine
onReady?: () => void;
}) {
const native = useNative(), imageLibrary = useStore(s => s.imageModels), hardware = useStore(s => s.hardware);
- const [family, setFamily] = useState(''), [size, setSize] = useState(''), [context, setContext] = useState(32768), [vision, setVision] = useState('no');
+ const [family, setFamily] = useState(''), [size, setSize] = useState(''), [context, setContext] = useState(32768), [vision, setVision] = useState('yes');
const manual = useStore(s => s.setup?.choices?.onboarding?.manual ?? false);
const [expert, setExpert] = useState(manual), [multiple, setMultiple] = useState(false), [gpus, setGpus] = useState<number[]>([]), [kv, setKv] = useState('int8'), [lowRam, setLowRam] = useState('auto'), [split, setSplit] = useState('auto'), [deviceChoice, setDeviceChoice] = useState('auto');
const [cardsKey, setCardsKey] = useState<string | null>(null);
@@ -92,7 +92,7 @@ export function NativeModels({ onOtherModels, onReady, onImport, onConnectEngine
return;
} if (installed) {
setContext(installed.context ?? 32768);
- setVision(installed.visionEnabled === false || installed.images === 'off' ? 'no' : installed.visionDevice ?? installed.images ?? 'no');
+ setVision(installed.visionEnabled === false || installed.images === 'off' ? 'no' : installed.visionDevice ?? installed.images ?? 'yes');
setKv(installed.kv ?? 'int8');
setLowRam(installed.lowRamMode ?? 'auto');
// A model never tuned has no saved choice: keep the page's own (Leave 20%) rather than flipping to Full performance.
@@ -120,7 +120,7 @@ export function NativeModels({ onOtherModels, onReady, onImport, onConnectEngine
else
setPlan(await api.post('/api/native/plan', downloadValues)); }); }
async function download() { await perform(async () => { const result = await nativeAction<NativeJob>('operations', { kind: 'install', args: downloadValues }); setJobId(result.id); setPlan(null); }); }
- return <Screen plainHeader title={<span className="sr-only">{tr("Models")}</span>} titleDetail={<FamilySwitch families={families} selected={selectedFamily?.family} onPick={f => { setFamily(f.family); setSize((f.sizes.find(s => native?.installed?.models.some(m => m.model === s.id && m.ready)) ?? f.sizes[0])?.id ?? ''); setVision('no'); setError(''); }}/>} actions={<Btn icon="sliders" aria-pressed={expert} aria-expanded={expert} aria-controls="native-expert-settings" onClick={() => { setExpert(v => !v); setMultiple(false); setGpus([]); setDeviceChoice('auto'); }}>{tr(expert ? 'Close expert mode' : 'Expert mode')}</Btn>}>
+ return <Screen plainHeader title={<span className="sr-only">{tr("Models")}</span>} titleDetail={<FamilySwitch families={families} selected={selectedFamily?.family} onPick={f => { setFamily(f.family); setSize((f.sizes.find(s => native?.installed?.models.some(m => m.model === s.id && m.ready)) ?? f.sizes[0])?.id ?? ''); setVision('yes'); setError(''); }}/>} actions={<Btn icon="sliders" aria-pressed={expert} aria-expanded={expert} aria-controls="native-expert-settings" onClick={() => { setExpert(v => !v); setMultiple(false); setGpus([]); setDeviceChoice('auto'); }}>{tr(expert ? 'Close expert mode' : 'Expert mode')}</Btn>}>
<div className="native-model-layout native-model-layout--fit">
<div className="stack">
{tr(native?.instances.length && expert ? <section className="st-card native-running"><h2 className="h4">{tr(native.instances.length === 1 ? 'Loaded model' : 'Loaded models')}</h2>{tr(native.instances.map(i => <div key={i.id}><span><strong>{tr(i.label)}</strong><small>{tr(i.gpus.length ? i.gpus.map(g => `GPU ${g + 1}`).join(' + ') : 'Automatic GPU')} · {tr(i.status === 'ready' ? i.phase : i.status)}{tr(native.defaultId === i.id ? ' · Chat default' : '')}</small>{tr(i.error ? <small className="inline-error">{tr(plainFailureText(i.error).message)}</small> : null)}</span>{tr(expert && multiple && native.defaultId !== i.id ? <Btn size="sm" variant="ghost" onClick={() => void perform(() => nativeAction(`instances/${i.id}/default`))}>{tr("Use in chat")}</Btn> : null)}<Btn size="sm" disabled={pending || !!active} onClick={() => void perform(() => nativeAction(`instances/${i.id}/unload`))}>{tr("Unload")}</Btn></div>))}</section> : null)}
@@ -136,7 +136,7 @@ export function NativeModels({ onOtherModels, onReady, onImport, onConnectEngine
</div>
{tr(cardOptions ? <div className="native-cards"><span className="native-choice__label">{tr("Graphics cards")}</span><div className="native-cards__list" role="radiogroup" aria-label={tr("Graphics cards")}>{cardOptions.options.map(o => { const on = activeCards?.key === o.key; return <button key={o.key} type="button" role="radio" aria-checked={on} data-selected={on} disabled={!!active} className="native-cards__opt" onClick={() => setCardsKey(o.key)}><span className="native-cards__dot" aria-hidden="true"/><span><strong>{o.label}</strong><small data-warn={o.verdict === 'spills' || o.verdict === 'tight'}>{[o.words, ...o.notes].join('. ')}.</small></span></button>; })}{cardOptions.ignored.length ? <p className="small muted native-cards__note">{`Not used: ${cardOptions.ignored.map(g => `${g.name ?? 'a card'} (${g.why_ignored})`).join('; ')}.`}</p> : null}</div></div> : null)}
{tr(selectedSize?.imagesCapable && !selectedSize.images ? <p className="small muted native-spill-note">{tr("This model can read pictures, but the engine installed here does not include the picture reader yet, so Read pictures is off. Update the engine in Settings → About to turn it on.")}</p> : null)}
- {tr(selectedSize?.images ? <><div className="native-image-reading"><label><input type="checkbox" checked={vision !== 'no'} disabled={!selectedSize?.images || !!active} onChange={e => setVision(e.target.checked ? 'yes' : 'no')}/><span><strong>{tr("Read pictures ")}<small>{tr("Optional")}</small></strong><small>{tr("Understand photos, screenshots and charts. Vision files download automatically. Enable them when loading this model.")}</small></span></label>{tr(expert && vision !== 'no' ? <Select ariaLabel={tr("Image encoder device")} value={vision === 'yes' ? 'yes' : vision} options={[{ value: 'yes', label: 'Automatic' }, { value: 'gpu', label: 'GPU' }, { value: 'cpu', label: 'CPU' }]} onChange={setVision}/> : null)}</div>{tr(vision !== 'no' && installed?.ready && installed.visionPrepared === false ? <div className="native-vision-missing" role="status"><span>{tr("Picture reading needs a few more files that are not on this computer yet.")}</span><Btn size="sm" disabled={pending || !!active} onClick={() => void download()}>{tr("Download picture-reading files")}</Btn></div> : null)}</> : null)}
+ {tr(selectedSize?.images ? <><div className="native-image-reading"><label><input type="checkbox" checked={vision !== 'no'} disabled={!selectedSize?.images || !!active} onChange={e => setVision(e.target.checked ? 'yes' : 'no')}/><span><strong>{tr("Read pictures ")}<small>{tr("On by default")}</small></strong><small>{tr("Understand photos, screenshots and charts. The picture files download automatically. Turn it off to save memory.")}</small></span></label>{tr(expert && vision !== 'no' ? <Select ariaLabel={tr("Image encoder device")} value={vision === 'yes' ? 'yes' : vision} options={[{ value: 'yes', label: 'Automatic' }, { value: 'gpu', label: 'GPU' }, { value: 'cpu', label: 'CPU' }]} onChange={setVision}/> : null)}</div>{tr(vision !== 'no' && installed?.ready && installed.visionPrepared === false ? <div className="native-vision-missing" role="status"><span>{tr("Picture reading needs a few more files that are not on this computer yet.")}</span><Btn size="sm" disabled={pending || !!active} onClick={() => void download()}>{tr("Download picture-reading files")}</Btn></div> : null)}</> : null)}
</> : null)}
{tr(active ? <NativeProgress job={active}/> : null)}
{tr(job?.state === 'error' ? <ErrorNotice message={job.error ?? job.summary} action={tr("Retry")} onRetry={() => void load()}/> : null)}
diff --git a/apps/host/src/components/SettingsModels.tsx b/apps/host/src/components/SettingsModels.tsx
index 840294051..d1e1b003c 100644
--- a/apps/host/src/components/SettingsModels.tsx
+++ b/apps/host/src/components/SettingsModels.tsx
@@ -119,7 +119,7 @@ export function SettingsModels() {
<div className="provider-models__head"><h3 className="h4">{tr("Models in chat")}</h3><Btn size="sm" icon="refresh" disabled={fetching || !draft.baseUrl.trim() || (draft.kind === 'provider' && !draft.apiKey.trim() && (!saved?.apiKeySet || !!draft.clearKey))} onClick={() => void fetchModels()}>{tr(fetching ? 'Fetching…' : 'Fetch available models')}</Btn></div>
{tr(notice ? <p className="small muted" role="status">{tr(notice)}</p> : null)}
{tr(available.length ? <><input className="st-input" type="search" aria-label={tr("Filter available models")} placeholder={tr("Filter models…")} value={filter} onChange={e => setFilter(e.target.value)}/><div className="provider-model-list">{tr(available.filter(m => `${m.name} ${m.id}`.toLowerCase().includes(filter.toLowerCase())).map(m => <label className="provider-model-row" key={m.id}><input type="checkbox" checked={draft.models.some(v => v.id === m.id)} onChange={e => update({ models: e.target.checked ? [...draft.models, m] : draft.models.filter(v => v.id !== m.id) })}/><span><strong>{m.name}</strong><small className="muted">{m.id}</small></span></label>))}</div></> : null)}
- {tr(draft.models.length ? <div className="stack stack--sm">{tr(draft.models.map(m => <div className="provider-selected" key={m.id}><Icon name="check" size={15}/><span><strong>{m.name}</strong><small className="muted">{m.id}</small>{tr(draft.protocol === 'openai-chat' ? <label className="row xs"><input type="checkbox" checked={m.vision === true || (m.vision == null && m.reads === true)} onChange={e => update({ models: draft.models.map(v => v.id === m.id ? { ...v, vision: e.target.checked } : v) })}/>{tr("This model reads pictures")}{tr(m.vision == null && m.reads === true ? <span className="muted"> {tr("(the server says it does)")}</span> : null)}</label> : null)}</span><button type="button" className="st-iconbtn" aria-label={tr("Remove {0} from chat picker", [m.name])} onClick={() => update({ models: draft.models.filter(v => v.id !== m.id) })}><Icon name="x" size={16}/></button></div>))}</div> : <p className="small muted">{tr("No models selected yet. Fetch the list or add a model ID.")}</p>)}
+ {tr(draft.models.length ? <div className="stack stack--sm">{tr(draft.models.map(m => <div className="provider-selected" key={m.id}><Icon name="check" size={15}/><span><strong>{m.name}</strong><small className="muted">{m.id}</small>{tr(draft.protocol === 'openai-chat' ? <label className="row xs"><input type="checkbox" checked={m.vision ?? m.reads ?? true} onChange={e => update({ models: draft.models.map(v => v.id === m.id ? { ...v, vision: e.target.checked } : v) })}/>{tr("Read pictures with this model")}{tr(m.vision != null ? null : <span className="muted"> {tr(m.reads === false ? "(its server says it cannot)" : "(on by default)")}</span>)}</label> : null)}</span><button type="button" className="st-iconbtn" aria-label={tr("Remove {0} from chat picker", [m.name])} onClick={() => update({ models: draft.models.filter(v => v.id !== m.id) })}><Icon name="x" size={16}/></button></div>))}</div> : <p className="small muted">{tr("No models selected yet. Fetch the list or add a model ID.")}</p>)}
<div className="provider-manual"><label className="input-field"><span>{tr("Model ID")}</span><input className="st-input" value={modelId} placeholder={tr("Model ID from your provider or server")} spellCheck={false} onChange={e => setModelId(e.target.value)} onKeyDown={e => {
if (e.key === 'Enter') {
e.preventDefault();
diff --git a/apps/host/src/lib/chat-photos.ts b/apps/host/src/lib/chat-photos.ts
index 684eebee8..4390a3a3e 100644
--- a/apps/host/src/lib/chat-photos.ts
+++ b/apps/host/src/lib/chat-photos.ts
@@ -4,7 +4,9 @@ const queues=new Map<string,ChatPhoto[]>(),listeners=new Set<()=>void>(),empty:C
const key=(id?:string|null)=>id??'new';
function database(){return new Promise<IDBDatabase>((resolve,reject)=>{const request=indexedDB.open('strata-photos',1);request.onupgradeneeded=()=>request.result.createObjectStore('photos');request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(new Error('Could not store this picture. Check browser storage.'));});}
export function useChatPhotos(id?:string|null){return useSyncExternalStore(fn=>{listeners.add(fn);return()=>listeners.delete(fn);},()=>queues.get(key(id))??empty);}
-export async function attachChatPhoto(file:File,id?:string|null){if(!['image/png','image/jpeg','image/webp'].includes(file.type))throw new Error('Attach a PNG, JPEG or WebP picture. Other images can be opened in preview.');if(file.size>2*1024*1024)throw new Error('Choose a picture smaller than 2 MB.');const current=queues.get(key(id))??[];if(current.length>=3)throw new Error('Attach up to three pictures per message.');const photo={id:crypto.randomUUID(),name:file.name};const db=await database();await new Promise<void>((resolve,reject)=>{const tx=db.transaction('photos','readwrite');tx.objectStore('photos').put(file,photo.id);tx.oncomplete=()=>resolve();tx.onerror=()=>reject(new Error('This picture could not be saved.'));});db.close();queues.set(key(id),[...current,photo]);listeners.forEach(fn=>fn());return photo;}
+/** A big picture (a screenshot of a large screen) is made smaller so it can be sent: at most 2048 pixels on the long side, as a JPEG, under 2 MB. */
+export async function shrinkPicture(file:File,limit=2*1024*1024):Promise<File>{if(file.size<=limit)return file;if(typeof createImageBitmap!=='function'||typeof document==='undefined')throw new Error('Choose a picture smaller than 2 MB.');const bitmap=await createImageBitmap(file);let side=Math.min(1,2048/Math.max(bitmap.width,bitmap.height));for(let round=0;round<5;round++){const canvas=document.createElement('canvas');canvas.width=Math.max(1,Math.round(bitmap.width*side));canvas.height=Math.max(1,Math.round(bitmap.height*side));const context=canvas.getContext('2d');if(!context)break;context.fillStyle='#fff';context.fillRect(0,0,canvas.width,canvas.height);context.drawImage(bitmap,0,0,canvas.width,canvas.height);const blob=await new Promise<Blob|null>(resolve=>canvas.toBlob(resolve,'image/jpeg',round<2?0.9:0.75));if(blob&&blob.size<=limit){bitmap.close?.();return new File([blob],file.name.replace(/[.][a-z0-9]+$/i,'')+'.jpg',{type:'image/jpeg'});}side*=0.75;}bitmap.close?.();throw new Error('Choose a picture smaller than 2 MB.');}
+export async function attachChatPhoto(file:File,id?:string|null){if(!['image/png','image/jpeg','image/webp'].includes(file.type))throw new Error('Attach a PNG, JPEG or WebP picture. Other images can be opened in preview.');file=await shrinkPicture(file);const current=queues.get(key(id))??[];if(current.length>=3)throw new Error('Attach up to three pictures per message.');const photo={id:crypto.randomUUID(),name:file.name};const db=await database();await new Promise<void>((resolve,reject)=>{const tx=db.transaction('photos','readwrite');tx.objectStore('photos').put(file,photo.id);tx.oncomplete=()=>resolve();tx.onerror=()=>reject(new Error('This picture could not be saved.'));});db.close();queues.set(key(id),[...current,photo]);listeners.forEach(fn=>fn());return photo;}
export function removeChatPhoto(photo:string,id?:string|null){queues.set(key(id),(queues.get(key(id))??[]).filter(i=>i.id!==photo));listeners.forEach(fn=>fn());}
export function pendingChatPhotos(id?:string|null){return queues.get(key(id))??empty;}
export function takeChatPhotos(id?:string|null){const value=queues.get(key(id))??[];queues.delete(key(id));listeners.forEach(fn=>fn());return value;}
diff --git a/apps/host/src/lib/pictures.ts b/apps/host/src/lib/pictures.ts
new file mode 100644
index 000000000..613f30f59
--- /dev/null
+++ b/apps/host/src/lib/pictures.ts
@@ -0,0 +1,30 @@
+import type { NativeState } from './native';
+import type { ModelProviders, RemoteModelSelection } from './types';
+
+/**
+ * Can the model that answers this chat take a picture, and if not, what does the person do next?
+ *
+ * One answer for the composer (the "+" button, paste and drop) and for Send, so they always agree. Pictures are on by default for every
+ * model that can read them (owner rule); this says "no" only when the model cannot, or when it was loaded or set with pictures off.
+ * - a local model: it was loaded with picture reading (`vision`); the catalog says whether this model could (and the engine has the reader);
+ * - an API model: `reads` from the host (the person's switch, else what its server said, else on).
+ */
+export type PictureStatus =
+ | { ok: true }
+ | { ok: false; why: 'none' | 'local-off' | 'local-cannot' | 'api-off' | 'api-style'; message: string; openModels: boolean };
+
+export function pictureStatus(native: NativeState | null, remote: RemoteModelSelection | null, providers: ModelProviders | null): PictureStatus {
+ if (remote) {
+ const connection = providers?.connections.find(c => c.id === remote.connectionId);
+ const model = connection?.models.find(m => m.id === remote.modelId);
+ if (connection && connection.protocol !== 'openai-chat') return { ok: false, why: 'api-style', openModels: false, message: 'Pictures can only be sent to model APIs that use the OpenAI chat style. Choose another model, or remove the picture.' };
+ if (model?.reads === false) return { ok: false, why: 'api-off', openModels: true, message: 'Picture reading is turned off for this model. Open Settings, Models, edit the connection and turn on Read pictures with this model, or choose another model.' };
+ return { ok: true };
+ }
+ const instance = native?.instances.find(i => i.id === native.defaultId && i.status === 'ready');
+ if (!instance) return { ok: false, why: 'none', openModels: true, message: 'Load a model that reads pictures first, then add the picture.' };
+ if (instance.vision) return { ok: true };
+ const size = native?.catalog?.families.flatMap(f => f.sizes.map(s => ({ ...s, images: f.images }))).find(s => s.id === instance.model);
+ if (size?.images) return { ok: false, why: 'local-off', openModels: true, message: 'This model was started without picture reading. Open Models, turn on Read pictures and load it again.' };
+ return { ok: false, why: 'local-cannot', openModels: false, message: 'This model cannot read pictures. Choose a model that reads pictures, or remove the picture.' };
+}
diff --git a/apps/host/src/lib/store.ts b/apps/host/src/lib/store.ts
index d0369bd9d..a2a1ed80f 100644
--- a/apps/host/src/lib/store.ts
+++ b/apps/host/src/lib/store.ts
@@ -12,6 +12,7 @@ import { lifecyclePolicy } from './lifecycle';
import { newestServingState } from './serving-state';
import { photoData, uploadChatPhotos, photoHistorySlots, pendingChatPhotos, takeChatPhotos } from './chat-photos';
import { nativeState } from './native';
+import { pictureStatus } from './pictures';
import { postWithFolderFallback } from './folder-pick';
import { setTranslationLanguage, tr, validLanguage, type Language } from './i18n';
/**
@@ -1525,8 +1526,8 @@ export async function sendMessage(text: string, options: { fromLoop?: boolean; g
else startTimeSkipDemo(demoPrompt);
return;
}
- const imageTarget=nativeState()?.instances.find(i=>i.id===nativeState()?.defaultId);const remoteApi=state.remoteModel?state.providers?.connections.find(c=>c.id===state.remoteModel!.connectionId)?.models.find(m=>m.id===state.remoteModel!.modelId):null;const canSendPictures=state.remoteModel?remoteApi?.reads===true:imageTarget?.status==='ready'&&imageTarget.vision;
- if(pendingChatPhotos(state.activeSession).length&&!canSendPictures){pushToast('error','This model cannot read pictures',state.remoteModel?'Open Settings > Models, edit this connection and tick "This model reads pictures" if it does, or choose another model. You can also remove the attachments.':'Switch to a model that reads pictures (turn on Read pictures in Models), or remove the attachments.');return;}
+ const pictureCheck=pictureStatus(nativeState(),state.remoteModel,state.providers);
+ if(pendingChatPhotos(state.activeSession).length&&!pictureCheck.ok){pushToast('error','This model cannot read pictures',pictureCheck.message);return;}
const pictures=takeChatPhotos(state.activeSession);
const body=text.trim()||(pictures.length?'Describe this picture.':'');
if(!body&&!resume&&!planStart)return;
@@ -1593,7 +1594,7 @@ export async function sendMessage(text: string, options: { fromLoop?: boolean; g
sessionId,
...(planStart ? { planId: planStart } : {}),
...(resume ? { goalAction: 'resume' } : planStart ? {} : { message: options.shown ? body : body }),
- history:await Promise.all(prior.map(async(m,index)=>{const included=canSendPictures?(m.photos??[]).filter((_,p)=>imageSlots.has(`${index}:${p}`)):[];const omitted=(m.photos??[]).filter(p=>!included.includes(p));const text=m.content+(omitted.length?`\n[Earlier pictures: ${omitted.map(p=>p.name).join(', ')}. Their pixels are outside this model's image context; ask to reattach them if needed.]`:'');return {role:m.role,content:included.length?[{type:'text',text},...await Promise.all(included.map(async photo=>({type:'image_url',image_url:{url:(await photoData(photo)).url}})))]:text};})),
+ history:await Promise.all(prior.map(async(m,index)=>{const included=pictureCheck.ok?(m.photos??[]).filter((_,p)=>imageSlots.has(`${index}:${p}`)):[];const omitted=(m.photos??[]).filter(p=>!included.includes(p));const text=m.content+(omitted.length?`\n[Earlier pictures: ${omitted.map(p=>p.name).join(', ')}. Their pixels are outside this model's image context; ask to reattach them if needed.]`:'');return {role:m.role,content:included.length?[{type:'text',text},...await Promise.all(included.map(async photo=>({type:'image_url',image_url:{url:(await photoData(photo)).url}})))]:text};})),
model: state.library?.loaded ?? state.library?.current ?? null,
remoteModel: state.remoteModel,
nativeModelId:nativeState()?.instances.length?nativeState()?.defaultId:undefined,
diff --git a/apps/host/src/screens/Chat.tsx b/apps/host/src/screens/Chat.tsx
index a28f437f9..93c40ef8c 100644
--- a/apps/host/src/screens/Chat.tsx
+++ b/apps/host/src/screens/Chat.tsx
@@ -22,7 +22,7 @@ import { StatusLine } from '../components/StatusLine';
import { GoalBar } from '../components/GoalBar';
import { PlanCard, PlanPanel, QueuedMessages } from '../components/PlanCard';
import { FolderChip } from '../components/FolderChip';
-import { DropZone } from '../components/DropZone';
+import { DropZone, DROP_EVENT, pastedFiles } from '../components/DropZone';
import { ActivityLine, EditedFiles, FileCard } from '../components/Deliverables';
import { queueMessage, usePlan, usePlanQuiet } from '../lib/plan';
import { plainFailureText } from '../lib/plain-errors';
@@ -505,7 +505,14 @@ function Composer({ draft, setDraft, loaded, streaming, onSubmit, onKey, session
<GoalBar chatId={session?.id}/>
<div className="st-composer" style={{ boxShadow: 'var(--st-sh-bar)' }}>
{tr(photos.length ? <ChatPhotos photos={photos} onRemove={id => removeChatPhoto(id, session?.id)}/> : null)}
- <textarea ref={textareaRef} rows={1} value={draft} aria-label={tr("Message Strata")} placeholder={tr(canChat ? 'Message Strata…' : 'Try /time-skip, or load a model to chat…')} onFocus={() => setSlashDismissed(false)} onChange={(e) => { setSlashOpen(false); setSlashDismissed(false); setDraft(e.target.value); e.currentTarget.style.height = "auto"; e.currentTarget.style.height = `${Math.min(180, e.currentTarget.scrollHeight)}px`; }} onKeyDown={(e) => {
+ <textarea ref={textareaRef} rows={1} value={draft} aria-label={tr("Message Strata")} placeholder={tr(canChat ? 'Message Strata…' : 'Try /time-skip, or load a model to chat…')} onFocus={() => setSlashDismissed(false)} onPaste={(e) => {
+ // A picture on the clipboard (a screenshot, a copied image) is attached like a dropped file; text pastes as usual.
+ const files = pastedFiles(e.clipboardData);
+ if (!files.length)
+ return;
+ e.preventDefault();
+ window.dispatchEvent(new CustomEvent(DROP_EVENT, { detail: files }));
+ }} onChange={(e) => { setSlashOpen(false); setSlashDismissed(false); setDraft(e.target.value); e.currentTarget.style.height = "auto"; e.currentTarget.style.height = `${Math.min(180, e.currentTarget.scrollHeight)}px`; }} onKeyDown={(e) => {
// The command menu owns the keyboard while it is open.
if (slashQuery !== null && e.key === 'Escape') {
e.preventDefault();