Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

App: a start that stopped for no VRAM left for the expert cache is tried once more with less kept free (#3192); a model on the person's own GGUF folder keeps that folder, so a reload does not download it (#3199)

Niko12218441cfd0e2b0
diff --git a/apps/host/server/native-control.mjs b/apps/host/server/native-control.mjs
index ae4ec6485..b0e610802 100644
--- a/apps/host/server/native-control.mjs
+++ b/apps/host/server/native-control.mjs
@@ -69,7 +69,7 @@ export function createNativeControl({directory,requests=()=>[],changed=()=>{},sp
for(const name of names){const path=join(dir,name),config=readJson(path,null);if(!config||typeof config!=='object')continue;
const next={...config,...(config.cwd!==undefined?{cwd:fix(config.cwd)}:{}),...(Array.isArray(config.args)?{args:config.args.map(fix)}:{})};
if(JSON.stringify(next)!==JSON.stringify(config)){try{writeModelConfig(path,next);}catch{/* read-only folder: the load reports it */}}}}
- function installedInfo(info){if(!info)return info;return {...info,models:(info.models??[]).map(entry=>{const cfg=/^strata-[a-z0-9_-]+\.json$/i.test(entry.config??'')?readJson(entry.configPath??join(repo(),entry.config)):{};const at=cfg.args?.indexOf('--native')??-1,nativeFile=at>=0?cfg.args[at+1]:null;const dataFolder=typeof nativeFile==='string'&&basename(dirname(dirname(nativeFile)))==='models'?dirname(dirname(dirname(nativeFile))):info.data_folder;return {...entry,diskGb:entry.diskGb??entry.disk_gb,diskBytes:entry.diskBytes??entry.disk_bytes,sharedBytes:entry.sharedBytes??entry.shared_bytes,removableGb:entry.removableGb??entry.removable_gb,dataFolder,visionPrepared:!!cfg.vision,/* The picture files are prepared when the run config has a vision section: the Models screen then loads straight away instead of running setup again (it used to read 'off' for every model, which sent every load of a picture-capable model through "Apply settings and load", and setup writes the arguments, the draft layer and the reader back each time). */images:cfg.vision?'yes':'off',draftLayer:cfg.mtp!==false,/* Picture reading is ON unless the person turned it off: no saved choice and not loaded is undefined, not false (owner rule: pictures on by default). */visionEnabled:cfg.ui_vision_enabled??(()=>{const running=[...instances.values()].find(i=>i.model===entry.model&&i.status==='ready');return running?!!running.vision:undefined;})(),visionDevice:cfg.ui_vision_device,resourceProfile:cfg.ui_resource_profile,lowRamMode:cfg.args?.includes('--resident-experts')?'resident':cfg.args?.includes('--mmap-experts')?'mmap':'auto'};})};}
+ function installedInfo(info){if(!info)return info;return {...info,models:(info.models??[]).map(entry=>{const cfg=/^strata-[a-z0-9_-]+\.json$/i.test(entry.config??'')?readJson(entry.configPath??join(repo(),entry.config)):{};const at=cfg.args?.indexOf('--native')??-1,nativeFile=at>=0?cfg.args[at+1]:null;const dataFolder=typeof nativeFile==='string'&&basename(dirname(dirname(nativeFile)))==='models'?dirname(dirname(dirname(nativeFile))):info.data_folder;/* A model the person pointed at their own GGUF folder (not <data>/models/<tag>): remembered, so a later load runs setup with that folder again instead of downloading the files (#3199). */const ownGgufDir=typeof nativeFile==='string'&&basename(dirname(dirname(nativeFile)))!=='models'&&existsSync(nativeFile)?dirname(nativeFile):undefined;return {...entry,ownGgufDir,diskGb:entry.diskGb??entry.disk_gb,diskBytes:entry.diskBytes??entry.disk_bytes,sharedBytes:entry.sharedBytes??entry.shared_bytes,removableGb:entry.removableGb??entry.removable_gb,dataFolder,visionPrepared:!!cfg.vision,/* The picture files are prepared when the run config has a vision section: the Models screen then loads straight away instead of running setup again (it used to read 'off' for every model, which sent every load of a picture-capable model through "Apply settings and load", and setup writes the arguments, the draft layer and the reader back each time). */images:cfg.vision?'yes':'off',draftLayer:cfg.mtp!==false,/* Picture reading is ON unless the person turned it off: no saved choice and not loaded is undefined, not false (owner rule: pictures on by default). */visionEnabled:cfg.ui_vision_enabled??(()=>{const running=[...instances.values()].find(i=>i.model===entry.model&&i.status==='ready');return running?!!running.vision:undefined;})(),visionDevice:cfg.ui_vision_device,resourceProfile:cfg.ui_resource_profile,lowRamMode:cfg.args?.includes('--resident-experts')?'resident':cfg.args?.includes('--mmap-experts')?'mmap':'auto'};})};}
function installRoot(){const found=[preferences.installDir].find(d=>d&&existsSync(d));if(found)return realpathSync(found);mkdirSync(defaultInstall(),{recursive:true});return realpathSync(defaultInstall());}
function repo(){if(packaged)return installRoot();const path=realpathSync(preferences.repo);if(!existsSync(join(path,'setup.py'))||!existsSync(join(path,'tools/strata_mcp.py'))||!existsSync(join(path,'serve/server.py')))throw new Error('Choose the Strata repository folder containing setup.py and tools/strata_mcp.py.');return path;}
function python(){return resolvePython({repo:preferences.repo});}
@@ -132,7 +132,7 @@ export function createNativeControl({directory,requests=()=>[],changed=()=>{},sp
function save(model,{values,mcp}){if([...jobs.values()].some(j=>j.state==='running'))throw new Error('Finish the current operation before saving native settings.');const {path,config}=record(model);if(busy(requests().filter(r=>instances.get(r.instanceId)?.model===model)))throw new Error('Finish this model’s requests before saving its settings.');const patch=validateNativeSettings(values??{}),next={...config};for(const field of nativeFields())if(field.supported===false&&patch[field.id]!=null&&JSON.stringify(patch[field.id])!==JSON.stringify(config[field.id]))throw new Error(`${field.label} needs a newer native Strata installation. Update it in Settings → About.`);for(const [key,value] of Object.entries(patch)){if(key==='env')next.env=restoreNativeEnv(value,config.env??{});else if(value===null)delete next[key];else next[key]=value;}if(mcp!==undefined){next.mcp_servers=validateMcpServers(mcp,config.mcp_servers??config.mcpServers??{});delete next.mcpServers;}copyFileSync(path,path+'.bak');chmodSync(path+'.bak',0o600);const temp=path+'.ui-tmp';writeFileSync(temp,JSON.stringify(next,null,2)+'\n',{mode:0o600});renameSync(temp,path);return settings(model);}
function previousEngine(){if(packaged)return !!snapshot.engine?.previous;try{const data=snapshot.installed?.data_dir??snapshot.installed?.data_folder;return [join(preferences.repo,'engine/.previous'),data&&join(data,'engine/.previous')].filter(Boolean).some(existsSync)&&readFileSync(join(preferences.repo,'setup.py'),'utf8').includes('--rollback-engine');}catch{return false;}}
async function plan(args){await refresh();if(!snapshot.available)throw new Error(snapshot.error);return tool('strata_install',installArgs(args,false));}
- function installArgs(args,confirm){const allowed=['family','model','context','vision','kv','gpu','gpus','backend','low_ram','port','gguf_dir'];const size=snapshot.catalog?.families?.find(f=>f.family===args.family)?.sizes.find(s=>s.model===args.model);const existing=snapshot.installed?.models?.find(m=>m.model===size?.id);/* A model adopted from the person's own GGUF folder reports that folder's parent as its data folder: it is theirs, not Strata's, so a second preparation never uses it for the packs. */const data=nativeDataDirectory(args.data_dir??(args.gguf_dir?undefined:existing?.dataFolder)??preferences.downloadDirectory??modelsDirectory?.());return {...(data?{data_dir:data}:{}),...Object.fromEntries(allowed.filter(k=>args[k]!==undefined).map(k=>[k,args[k]])),/* Picture reading is only downloaded when it was asked for (Read pictures); the default stays the smaller download. */...(packaged&&args.vision===undefined?{vision:'no'}:{}),confirm};}
+ function installArgs(args,confirm){const allowed=['family','model','context','vision','kv','gpu','gpus','backend','low_ram','port','gguf_dir'];const size=snapshot.catalog?.families?.find(f=>f.family===args.family)?.sizes.find(s=>s.model===args.model);const existing=snapshot.installed?.models?.find(m=>m.model===size?.id);/* A model adopted from the person's own GGUF folder reports that folder's parent as its data folder: it is theirs, not Strata's, so a second preparation never uses it for the packs. */if(args.gguf_dir===undefined&&existing?.ownGgufDir)args={...args,gguf_dir:existing.ownGgufDir};const data=nativeDataDirectory(args.data_dir??(args.gguf_dir?undefined:existing?.dataFolder)??preferences.downloadDirectory??modelsDirectory?.());return {...(data?{data_dir:data}:{}),...Object.fromEntries(allowed.filter(k=>args[k]!==undefined).map(k=>[k,args[k]])),/* Picture reading is only downloaded when it was asked for (Read pictures); the default stays the smaller download. */...(packaged&&args.vision===undefined?{vision:'no'}:{}),confirm};}
function begin(kind,run,metadata={}){if([...jobs.values()].some(j=>j.state==='running'))throw new Error('Finish or cancel the current engine operation first.');const job={id:randomUUID(),kind,state:'running',startedAt:new Date().toISOString(),summary:'Starting…',result:null,error:null,...metadata};for(const [id,old] of jobs)if(jobs.size>=12&&old.state!=='running')jobs.delete(id);jobs.set(job.id,job);emit();void run(job).then(result=>{if(job.state==='running'){Object.assign(job,{state:'done',summary:result?.summary??'Complete',result,finishedAt:new Date().toISOString()});}}).catch(error=>{if(job.state==='running')Object.assign(job,{state:'error',error:error.message,summary:error.message,...(error.details?{details:error.details}:{}),finishedAt:new Date().toISOString()});}).finally(()=>{if(job.kind.startsWith('update')||job.kind==='rollback')disconnect();lastRead=0;emit();void refresh(true);});return state().jobs.find(j=>j.id===job.id);}
async function watchInstall(job,current){job.summary=current.step??'Resuming preparation status…';job.result={install:current};while(job.state==='running'){await new Promise(r=>setTimeout(r,1500));const result=await tool('strata_install');job.summary=result.summary;job.result=result;emit();if(!result.install?.running){if(result.install?.exit_code!==0)throw new Error(result.install?.error??'Preparation did not finish. Retry to reuse completed files.');return {summary:'Model preparation complete. Load it from Models.'};}}}
diff --git a/apps/host/server/native-resources.mjs b/apps/host/server/native-resources.mjs
index 5dcf7baa5..542cf3fe6 100644
--- a/apps/host/server/native-resources.mjs
+++ b/apps/host/server/native-resources.mjs
@@ -49,6 +49,23 @@ export function saferStart(config,hardware,model,failure){
if(/0xC0000135|0xC000007B|0xC0000142|0xC000001D|0xC0000005|0xC0000409|0xC0000022|error while loading shared libraries|cannot open shared object file|\bENOENT\b|exit code 127|exit code 139|illegal instruction/i.test(text))return null;
const ranOut=/out of memory|memory guard|\boom\b|\bkilled\b|exit code 137|cannot allocate|not enough memory|bad_alloc|insufficient memory/i.test(text),first=ranOut?'The first start ran out of memory':'The first start did not finish';
if(!/server_exited|exited before|died before|engine stopped|pinned-memory-exhausted|mapped staging|out of memory|memory guard|\bkilled\b/i.test(text))return null;
+ // The engine stopped because the GPU's expert cache got no room (a 10 GB card with the app's 20% reserve and pictures on): the engine can shrink its own
+ // reserve only when none was given, and the app always gives one. Ask for less kept free (never under 1024 MiB: below that the driver pages
+ // graphics memory to RAM, #2065) and, when that is not enough, for a shorter context. Never saved.
+ if(/no VRAM (?:is|was) left for the expert cache|needs the profile-filled VRAM expert tier/i.test(text)){
+ const args=[...(config.args??[])],at=args.indexOf('--vram-reserve-mib'),cur=at>=0?Number(args[at+1])||0:Number(config.vram_reserve_mib)||0;
+ const short=Number((text.match(/about (\d+) MiB more than this card has free/)??[])[1])||700,need=short+256;
+ const notes=[];let left=need;
+ if(cur>1024){const next=Math.max(1024,cur-need);left=need-(cur-next);if(next<cur){if(at>=0)args[at+1]=String(next);else args.push('--vram-reserve-mib',String(next));notes.push(`${next} MiB of graphics memory kept free for other programs (was ${cur})`);}}
+ const cx=args.indexOf('--max-context'),ctx=cx>=0?Number(args[cx+1])||0:0,sess=text.match(/takes (\d+) MiB at (\d+) tokens/);
+ if(left>0&&ctx>16384&&sess){
+ const session=Number(sess[1]),tokens=Number(sess[2]);let next=ctx;
+ while(next>16384&&session*(ctx-next)/tokens<left)next=Math.max(16384,Math.floor(next/2));
+ if(next<ctx){args[cx+1]=String(next);notes.push(`a shorter conversation context (${next} tokens, was ${ctx})`);}
+ }
+ if(!notes.length)return null;
+ return {config:{...config,args},words:`This graphics card had no room left for the model's expert cache, so Strata started again with ${notes.join(' and ')}. Nothing is saved; change it in the model's settings if you want it permanently.`,detail:notes.join('; ')};
+ }
const args=[...(config.args??[])],env={...(config.env??{})};
const ram=Number(hardware?.ram_gb)||0,arena=Number(model?.arenaGb)||0;
// Windows lets a program lock only about half the RAM for the graphics card; the engine pins the model's experts up to that.
diff --git a/apps/host/tests/native-control.test.mjs b/apps/host/tests/native-control.test.mjs
index d70604b65..b6c6b23ff 100644
--- a/apps/host/tests/native-control.test.mjs
+++ b/apps/host/tests/native-control.test.mjs
@@ -1,6 +1,6 @@
import test from 'node:test';
import assert from 'node:assert/strict';
-import {readFileSync,readdirSync} from 'node:fs';
+import {readFileSync,readdirSync,mkdirSync,writeFileSync} from 'node:fs';
import {join} from 'node:path';
import {nativeFixture,finishJob} from './helpers/native-fixture.mjs';
test('native instances isolate devices, ports, IDs and private keys; default loading switches one model',async()=>{
@@ -35,3 +35,11 @@ test('native update refuses local edits; calibration targets the chosen installe
test('cancelling a native load before dispatch preserves the running instance',async()=>{const f=nativeFixture();try{await f.control.refresh();const first=await finishJob(f.control,f.control.operation('load',{model:'iq2_xs',gpus:[0]}).id);const job=f.control.operation('load',{model:'iq3_s',gpus:[1]});await f.control.cancel(job.id);await new Promise(r=>setTimeout(r,20));assert.equal(f.control.state().instances.length,1);assert.equal(f.control.state().instances[0].id,first.result.instanceId);assert.equal(f.control.state().jobs.find(j=>j.id===job.id).state,'cancelled');}finally{await f.cleanup();}});
test('retuning or repairing an installed native model reuses its original data folder',async()=>{const f=nativeFixture();try{await f.control.refresh();await f.control.plan({family:'qwen',model:'IQ2_XS',context:131072});const call=f.rpcCalls.find(c=>c.name==='strata_install');assert.equal(call.arguments.data_dir,f.installed.data_folder);assert.equal(call.arguments.confirm,false);}finally{await f.cleanup();}});
+
+test('a model the person pointed at their own GGUF folder keeps that folder for the next load: setup is not asked to download it again (#3199)',async()=>{const f=nativeFixture();try{
+ const own=join(f.directory,'my-ggufs');mkdirSync(own,{recursive:true});const shard=join(own,'Qwen3.8-Flash-Next-IQ2_XS-00001-of-00002.gguf');writeFileSync(shard,'x');
+ const path=join(f.repo,'strata-iq2_xs.json'),config=JSON.parse(readFileSync(path));config.args=['--native',shard,'--max-context','32768'];writeFileSync(path,JSON.stringify(config));
+ await f.control.refresh(true);await f.control.plan({family:'qwen',model:'IQ2_XS',context:131072});
+ const call=f.rpcCalls.filter(c=>c.name==='strata_install').pop();assert.equal(call.arguments.gguf_dir,own);assert.equal(call.arguments.data_dir,undefined,'the person\'s folder is not the data folder');
+ await f.control.plan({family:'qwen',model:'IQ3_S',context:131072});const other=f.rpcCalls.filter(c=>c.name==='strata_install').pop();assert.equal(other.arguments.gguf_dir,undefined,'a model Strata downloaded itself keeps its normal folders');
+}finally{await f.cleanup();}});
diff --git a/apps/host/tests/resource-ram.test.mjs b/apps/host/tests/resource-ram.test.mjs
index 0f98e8c3a..86e29750b 100644
--- a/apps/host/tests/resource-ram.test.mjs
+++ b/apps/host/tests/resource-ram.test.mjs
@@ -74,3 +74,19 @@ test('a start that died gets ONE safer second try: pin less when the locked memo
assert.equal(saferStart(config, pc64, iq3s, 'Some model files are missing'), null, 'not a memory failure: no second try');
assert.equal(config.args.includes('--resident-budget-gib'), false, 'the input config is not changed');
});
+
+test('an engine that stopped because the GPU expert cache got no room is started again with less VRAM kept free, then a shorter context (#3192)', () => {
+ const tenGb = { cpu: { cores: 16 }, ram_gb: 64, gpus: [{ index: 0, name: 'NVIDIA GeForce RTX 3080', vram_gb: 10 }] };
+ const cfg = { gpu: 0, args: ['--native', 'x.gguf', '--max-context', '131072', '--vision', '--vram-reserve-mib', '2048'] };
+ const log = "server_exited (exit code 1): strata generate: no VRAM is left for the expert cache: it needs at least 288 slots (639 MiB), about 727 MiB more than this card has free. To make room: a smaller --max-context (the session, mostly its KV cache, takes 1931 MiB at 131072 tokens), --kv q4_0 | strata serve: verify: needs the profile-filled VRAM expert tier";
+ const safer = saferStart(cfg, tenGb, iq3s, log);
+ assert.equal(flag(safer.config.args, '--vram-reserve-mib'), '1065', '2048 - (727 + 256 margin)');
+ assert.equal(flag(safer.config.args, '--max-context'), '131072', 'the reserve alone was enough: the context stays');
+ assert.match(safer.words, /no room left/);
+ const tight = saferStart({ ...cfg, args: ['--max-context', '131072', '--vram-reserve-mib', '1100'] }, tenGb, iq3s, log);
+ assert.equal(flag(tight.config.args, '--vram-reserve-mib'), '1024', 'never below 1024 MiB (#2065)');
+ assert.ok(Number(flag(tight.config.args, '--max-context')) < 131072, 'the rest comes from the context');
+ assert.equal(saferStart({ ...cfg, args: ['--max-context', '16384', '--vram-reserve-mib', '1024'] }, tenGb, iq3s, log), null, 'nothing left to give');
+ assert.equal(flag(saferStart(cfg, tenGb, undefined, 'server_exited: strata serve: verify: needs the profile-filled VRAM expert tier').config.args, '--vram-reserve-mib'), '1092', 'without the numbers: a fixed step, no model size needed');
+ assert.equal(cfg.args[cfg.args.indexOf('--vram-reserve-mib') + 1], '2048', 'the input config is not changed');
+});