Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
host: one automatic safer second try when the engine dies while loading (smaller pin / RAM budget), said in plain words and not saved; Leave 20% also sets the engine's own RAM headroom
Niko12214e67f1f045ea
diff --git a/apps/host/server/native-control.mjs b/apps/host/server/native-control.mjsindex 02e0bf1f7..7cfa62780 100644--- a/apps/host/server/native-control.mjs+++ b/apps/host/server/native-control.mjs@@ -6,7 +6,7 @@ import { join, resolve, dirname, basename } from 'node:path';import { fileURLToPath } from 'node:url';import { createServer } from 'node:net';import { NATIVE_FIELDS, validateNativeSettings, validateMcpServers, publicMcpServers, publicNativeEnv, restoreNativeEnv } from './native-settings.mjs';-import { resourceSettings } from './native-resources.mjs';+import { resourceSettings, saferStart } from './native-resources.mjs';import { resolvePython, venvPython } from './python-resolver.mjs';import os from 'node:os';import { runtimeVision } from './native-vision.mjs';@@ -153,14 +153,21 @@ export function createNativeControl({directory,requests=()=>[],changed=()=>{},spif(job.state!=='running')return;writeFileSync(configPath,JSON.stringify(cfg,null,2),{mode:0o600});const instance={id,apiModelId,fingerprint,context:entry.context??0,aliases:cfg.aliases??[],vision:!!cfg.vision,model:args.model,label:entry.model_name??args.model,port:cfg.port,gpus:Array.isArray(cfg.gpu)?cfg.gpu:cfg.gpu===undefined?[]:[cfg.gpu],status:'loading',phase:'loading',key,configPath,loadedAt:new Date().toISOString(),metrics:null,error:null};- let proc=null,started=null;+ let proc=null,started=null,retried=null;if(packaged){/* The engine starts under Control's owner runtime, not as our child: a saved profile per slot, then `runtime start`. */const used=new Set([...instances.values()].map(i=>i.profile));let slot=1;while(used.has('app-'+slot))slot++;instance.profile='app-'+slot;instances.set(id,instance);if(!defaultId||!args.multiple||wasDefault)defaultId=id;emit();- started=control.stop(instance.profile).catch(()=>{}).then(()=>control.saveProfile(instance.profile,configPath)).then(()=>control.start(instance.profile,{waitSeconds:300}));+ const startWith=value=>{writeFileSync(configPath,JSON.stringify(value,null,2),{mode:0o600});return control.stop(instance.profile).catch(()=>{}).then(()=>control.saveProfile(instance.profile,configPath)).then(()=>control.start(instance.profile,{waitSeconds:300}));};+ started=startWith(cfg);started.then(()=>append(id,'Strata Control reports the engine is ready.'),error=>{append(id,error.message);});- try{await started;}catch(error){const said=humanStartError(error);instance.status='error';instance.error=said;emit();throw new Error(said);}+ try{await started;}catch(error){const said=humanStartError(error);+ /* A start that died gets ONE automatic second try with a safer memory setting (never saved, never repeated), unless the caller says no. */+ const safer=args.retrySafer===false||job.state!=='running'?null:saferStart(cfg,snapshot.hardware,modelArena(args.model),said);+ if(!safer){instance.status='error';instance.error=said;emit();throw new Error(said);}+ retried=safer;append(id,`The first start stopped (${said}). Starting once more: ${safer.detail}`);job.summary=safer.words;job.details=[said,`Second try: ${safer.detail}`];instance.notice=safer.words;emit();+ try{await startWith(safer.config);append(id,'Strata Control reports the engine is ready (second try).');}+ catch(second){const said2=humanStartError(second);instance.status='error';instance.error=`${said2} (after one automatic second try with a safer setting: ${safer.detail}; the first stop was: ${said})`;emit();throw new Error(instance.error);}}} else {proc=spawnProcess(py,[join(repo(),'serve/server.py'),'--engine','strata','--config',configPath,'--port',String(cfg.port),'--host','127.0.0.1'],{cwd:repo(),detached:process.platform!=='win32',stdio:['ignore','pipe','pipe'],env:{...process.env,PYTHONUNBUFFERED:'1',PYTHONIOENCODING:'utf-8',STRATA_API_KEY:''}});instances.set(id,instance);processes.set(id,proc);if(!defaultId||!args.multiple||wasDefault)defaultId=id;@@ -169,7 +176,7 @@ export function createNativeControl({directory,requests=()=>[],changed=()=>{},sp}const deadline=Date.now()+300000;while(Date.now()<deadline&&job.state==='running'){if(instance.status==='error')throw new Error(instance.error);- try{const response=await fetcher(`http://127.0.0.1:${instance.port}/health`,{headers:{authorization:`Bearer ${key}`},signal:AbortSignal.timeout(2000)});if(job.state!=='running'){await unload(id);return;}if(response.ok){instance.status='ready';instance.phase='idle';if(resources||args.vision!==undefined)writeModelConfig(record(args.model).path,{...config,...(resources?{...resources,ui_resource_profile:args.resourceProfile}:{}),...visionPreferences});emit();ensurePoll();return {summary:`${instance.label} is ready.`,instanceId:id};}}catch{}+ try{const response=await fetcher(`http://127.0.0.1:${instance.port}/health`,{headers:{authorization:`Bearer ${key}`},signal:AbortSignal.timeout(2000)});if(job.state!=='running'){await unload(id);return;}if(response.ok){instance.status='ready';instance.phase='idle';if(resources||args.vision!==undefined)writeModelConfig(record(args.model).path,{...config,...(resources?{...resources,ui_resource_profile:args.resourceProfile}:{}),...visionPreferences});emit();ensurePoll();return {summary:retried?`${instance.label} is ready, after a second try: ${retried.words}`:`${instance.label} is ready.`,instanceId:id};}}catch{}await new Promise(resolve=>setTimeout(resolve,1200));}if(job.state!=='running'){await unload(id);return;}diff --git a/apps/host/server/native-resources.mjs b/apps/host/server/native-resources.mjsindex eda84f7f2..519d020cb 100644--- a/apps/host/server/native-resources.mjs+++ b/apps/host/server/native-resources.mjs@@ -27,9 +27,44 @@ export function resourceSettings(config,hardware,profile,model={}){// setup rule: experts plus 10 GB) keeps only that many GiB of them in RAM and reads the rest from the model files// (--resident-budget-gib, which implies --mmap-experts). Full performance takes the budget out again.if(baseline.ramBudget)remove('--resident-budget-gib');+ let env=null; // the engine's own "leave RAM free" setting travels in the run config's env+ if(baseline.ramHeadroom&&config.env){env={...config.env};delete env.STRATA_RESIDENT_HEADROOM_GIB;}const ram=Number(hardware?.ram_gb)||0,arena=Number(model?.arenaGb)||0;let ramBudget=false;if(profile==='work'&&ram>0&&arena>0&&!['--resident-experts','--mmap-experts','--resident-budget-gib'].some(f=>args.includes(f))&&ram*.8<arena+10){args.push('--resident-budget-gib',String(Math.max(8,Math.min(Math.floor(ram*.8)-10,Math.floor(arena/1.073741824)))));ramBudget=true;}- return {args,vram_reserve_mib:reserve,ui_resource_baseline:{...baseline,ramBudget}};+ // The budget caps what the engine keeps in RAM. "Leave 20%" is also asked of the engine itself: at its start it leaves 20% of the RAM+ // available for other programs (it keeps fewer experts if the PC is busy), and it said nothing about this before.+ let ramHeadroom=false;+ if(ramBudget&&!(config.env?.STRATA_RESIDENT_HEADROOM_GIB)){env={...(env??config.env??{}),STRATA_RESIDENT_HEADROOM_GIB:String(Math.ceil(ram*.2))};ramHeadroom=true;}+ return {args,vram_reserve_mib:reserve,...(env?{env}:{}),ui_resource_baseline:{...baseline,ramBudget,ramHeadroom}};+}++/* One automatic second try after a start that died. Returns null when there is nothing safer to try (the caller then reports the first+ failure as it was) or {config, words, detail}: the same run config with ONE safer memory setting, a sentence for the user and the+ technical line for Details. The caller tries once and never again, and does not save this config. */+export function saferStart(config,hardware,model,failure){+ const text=String(failure??'');+ if(!/server_exited|exited before|died before|engine stopped|pinned-memory-exhausted|mapped staging|out of memory|memory guard|\bkilled\b/i.test(text))return null;+ const args=[...(config.args??[])],env={...(config.env??{})};+ const ram=Number(hardware?.ram_gb)||0,arena=Number(model?.arenaGb)||0;+ // Windows lets a program lock only about half the RAM for the graphics card; the engine pins the model's experts up to that.+ // When it ran out of locked memory, pin less of them: they stay in RAM, a little slower to reach.+ if(/pinned-memory-exhausted|mapped staging/i.test(text)&&!env.STRATA_ARENA_PIN_GIB){+ const hint=Number((text.match(/STRATA_ARENA_PIN_GIB=(\d+)/)??[])[1]);+ const gib=hint>0?hint:ram>0?Math.max(4,Math.floor(ram/2)-8):16;+ env.STRATA_ARENA_PIN_GIB=String(gib);+ return {config:{...config,env},words:`Windows would not lock as much memory as the model asked for, so Strata started again and locks ${gib} GB of the model instead of all of it. Replies may start a little slower.`,detail:`STRATA_ARENA_PIN_GIB=${gib}`};+ }+ if(['--resident-experts','--mmap-experts'].some(f=>args.includes(f)))return null;+ const at=args.indexOf('--resident-budget-gib');+ if(at>=0){+ const next=Math.floor((Number(args[at+1])||0)*.7);if(next<8)return null;+ args[at+1]=String(next);+ return {config:{...config,args},words:`The first start ran out of memory, so Strata started again and keeps less of the model in RAM (${next} GB); more of it is read from the SSD, which is slower.`,detail:`--resident-budget-gib ${next}`};+ }+ if(!(ram>0&&arena>0))return null;+ const budget=Math.max(8,Math.floor(ram*.5)-4);if(budget>=arena)return null;+ args.push('--resident-budget-gib',String(budget));+ return {config:{...config,args},words:`The first start ran out of memory, so Strata started again in low-memory mode: ${budget} GB of the model stays in RAM and the rest is read from the SSD. It is slower, but it should load.`,detail:`--resident-budget-gib ${budget}`};}diff --git a/apps/host/tests/packaged-control.test.mjs b/apps/host/tests/packaged-control.test.mjsindex f4dd41b63..57fd7448d 100644--- a/apps/host/tests/packaged-control.test.mjs+++ b/apps/host/tests/packaged-control.test.mjs@@ -97,7 +97,7 @@ function packagedFixture({ engineUsable = true } = {}) {const calls = [];const spawned = [];const rpc = [];- let startFailure = null;+ let startFailure = null, startFailureLeft = Infinity;const control = {control: { dir: join(root, 'control'), python: join(root, 'control', 'python', 'python.exe') }, version: '0.1.45',async installedModels(options) { calls.push(['installedModels', options?.installDir ?? null]); return { install_dir: install, models: configs.map(c => ({ id: c.id, name: `Qwen ${c.id}`, source_config: c.path, component_status: 'present', context_tokens: 32768, backend: 'cuda' })) }; },@@ -108,7 +108,7 @@ function packagedFixture({ engineUsable = true } = {}) {: { usable: false, notice: 'No engine for this PC was found in E:\\c: the component folder holds no engine package. The app never downloads an engine while it starts or serves.', backend: 'cuda', engine: null, components_dir: 'E:\\c' };},async saveProfile(name, path) { calls.push(['saveProfile', name, basename(path)]); return {}; },- async start(name, options) { calls.push(['start', name, options?.waitSeconds]); if (startFailure) { const e = new Error(startFailure.message); e.output = startFailure.output; throw e; } return { ok: true, state: 'ready' }; },+ async start(name, options) { calls.push(['start', name, options?.waitSeconds]); if (startFailure && startFailureLeft-- > 0) { const e = new Error(startFailure.message); e.output = startFailure.output; throw e; } return { ok: true, state: 'ready' }; },async stop(name) { calls.push(['stop', name]); return { ok: true, state: 'stopped' }; },};const requests = [];@@ -144,7 +144,7 @@ function packagedFixture({ engineUsable = true } = {}) {};const native = createNativeControl({ control, directory: data, spawnProcess, fetcher, requests: () => requests });const others = () => spawned.filter(s => !String(s.args[0]).endsWith('strata_mcp.py'));- return { native, calls, spawned, others, rpc, failStart: v => { startFailure = v; }, catalog, hardware, install, root, cleanup: () => { rmSync(root, { recursive: true, force: true }); } };+ return { native, calls, spawned, others, rpc, failStart: (v, times = Infinity) => { startFailure = v; startFailureLeft = times; }, catalog, hardware, install, root, cleanup: () => { rmSync(root, { recursive: true, force: true }); } };}const finish = async (native, id) => { for (let i = 0; i < 400; i++) { const job = native.state().jobs.find(j => j.id === id); if (job.state !== 'running') return job; await new Promise(r => setTimeout(r, 10)); } throw new Error('job did not finish'); };@@ -308,6 +308,48 @@ test('packaged: an engine that stops while loading keeps the code and the last e} finally { await f.native.close(); f.cleanup(); }});+test('packaged: a start that died because Windows ran out of locked memory is tried once more with a smaller pin, said in plain words, and the saved settings stay as they were', async () => {+ const f = packagedFixture();+ try {+ await f.native.refresh(true);+ f.failStart({ message: 'server_exited', output: { log_tail: 'verify: mapped staging allocation failed [pinned-memory-exhausted] the OS gave no more page-locked memory; start again with STRATA_ARENA_PIN_GIB=26 (pin less of the expert arena)' } }, 1);+ const job = await finish(f.native, f.native.operation('load', { model: 'iq2_xs' }).id);+ assert.equal(job.state, 'done', job.error);+ assert.equal(f.calls.filter(c => c[0] === 'start').length, 2, 'one try and exactly one more');+ assert.match(job.result.summary, /ready, after a second try: Windows would not lock as much memory/);+ assert.match(job.details[0], /pinned-memory-exhausted/);+ assert.match(job.details[1], /STRATA_ARENA_PIN_GIB=26/);+ const instance = f.native.state().instances[0];+ assert.equal(instance.status, 'ready');+ const saved = JSON.parse(readFileSync(join(f.install, 'strata-iq2_xs.json'), 'utf8'));+ assert.equal(saved.env, undefined, 'the safer setting is for this start only; the model config is not changed');+ } finally { await f.native.close(); f.cleanup(); }+});++test('packaged: the second try is the last one: a second failure is reported with both, and nothing loops', async () => {+ const f = packagedFixture();+ try {+ await f.native.refresh(true);+ f.failStart({ message: 'server_exited', output: { log_tail: 'mapped staging allocation failed [pinned-memory-exhausted]' } });+ const job = await finish(f.native, f.native.operation('load', { model: 'iq2_xs' }).id);+ assert.equal(job.state, 'error');+ assert.equal(f.calls.filter(c => c[0] === 'start').length, 2);+ assert.match(job.error, /after one automatic second try/);+ assert.match(job.error, /the first stop was/);+ } finally { await f.native.close(); f.cleanup(); }+});++test('packaged: retrySafer false keeps the single try', async () => {+ const f = packagedFixture();+ try {+ await f.native.refresh(true);+ f.failStart({ message: 'server_exited', output: { log_tail: 'mapped staging allocation failed [pinned-memory-exhausted]' } });+ const job = await finish(f.native, f.native.operation('load', { model: 'iq2_xs', retrySafer: false }).id);+ assert.equal(job.state, 'error');+ assert.equal(f.calls.filter(c => c[0] === 'start').length, 1);+ } finally { await f.native.close(); f.cleanup(); }+});+test('packaged: a run config that points into an old Control folder is repaired to this Control (a reinstall or an update moved it)', async () => {const f = packagedFixture();try {diff --git a/apps/host/tests/resource-ram.test.mjs b/apps/host/tests/resource-ram.test.mjsindex 3965ea479..0f98e8c3a 100644--- a/apps/host/tests/resource-ram.test.mjs+++ b/apps/host/tests/resource-ram.test.mjs@@ -1,6 +1,6 @@import test from 'node:test';import assert from 'node:assert/strict';-import { resourceSettings } from '../server/native-resources.mjs';+import { resourceSettings, saferStart } from '../server/native-resources.mjs';const pc64 = { cpu: { cores: 12 }, ram_gb: 63, gpus: [{ index: 0, name: 'NVIDIA GeForce RTX 5070', vram_gb: 11.9 }] };const iq3s = { arenaGb: 50.3 };@@ -43,3 +43,34 @@ test('neither profile reserves less VRAM than the engine\'s safe floor (the drivassert.equal(resourceSettings({ gpu: 0, args: [] }, card24, 'performance').vram_reserve_mib, 1024);assert.equal(resourceSettings({ gpu: 0, args: [], vram_reserve_mib: 2437 }, card12, 'performance').vram_reserve_mib, 2437, 'a larger reserve of the installation is kept');});++test('"leave 20% for my work" also asks the engine to leave 20% of the RAM free at its start, and full performance takes that out again', () => {+ const work = resourceSettings(config, pc64, 'work', iq3s);+ assert.equal(work.env.STRATA_RESIDENT_HEADROOM_GIB, '13', '20% of 63 GB, rounded up');+ assert.equal(work.ui_resource_baseline.ramHeadroom, true);+ const back = resourceSettings({ ...config, args: work.args, env: work.env, ui_resource_profile: 'work', ui_resource_baseline: work.ui_resource_baseline }, pc64, 'performance', iq3s);+ assert.equal(back.env.STRATA_RESIDENT_HEADROOM_GIB, undefined);+ const own = resourceSettings({ ...config, env: { STRATA_RESIDENT_HEADROOM_GIB: '6' } }, pc64, 'work', iq3s);+ assert.equal(own.env, undefined, 'a value the user set is not overwritten');+ assert.equal(resourceSettings(config, { ...pc64, ram_gb: 128 }, 'work', iq3s).env, undefined, 'no budget, no headroom setting');+});++test('a start that died gets ONE safer second try: pin less when the locked memory ran out, else a RAM budget with the rest on the SSD', () => {+ const pinned = saferStart(config, pc64, iq3s, 'server_exited: mapped staging allocation failed [pinned-memory-exhausted] start again with STRATA_ARENA_PIN_GIB=26 (pin less)');+ assert.equal(pinned.config.env.STRATA_ARENA_PIN_GIB, '26');+ assert.match(pinned.words, /would not lock as much memory/);+ assert.equal(flag(saferStart(pinned.config, pc64, iq3s, 'server_exited: mapped staging allocation failed [pinned-memory-exhausted]').config.args, '--resident-budget-gib'), String(Math.floor(63 * .5) - 4), 'already pinned less: the next safer setting is the RAM budget');+ const noHint = saferStart(config, pc64, iq3s, 'mapped staging allocation failed');+ assert.equal(noHint.config.env.STRATA_ARENA_PIN_GIB, String(Math.floor(63 / 2) - 8));+ const low = saferStart(config, pc64, iq3s, 'server_exited: memory guard: 1.9 GiB free');+ assert.equal(flag(low.config.args, '--resident-budget-gib'), String(Math.floor(63 * .5) - 4));+ assert.match(low.words, /low-memory mode/);+ const already = saferStart({ ...config, args: [...config.args, '--resident-budget-gib', '40'] }, pc64, iq3s, 'server_exited');+ assert.equal(flag(already.config.args, '--resident-budget-gib'), '28', '70% of the budget it had');+ assert.equal(saferStart({ ...config, args: [...config.args, '--resident-budget-gib', '10'] }, pc64, iq3s, 'server_exited'), null, 'below 8 GB there is nothing left to give');+ assert.equal(saferStart({ ...config, args: [...config.args, '--resident-experts'] }, pc64, iq3s, 'server_exited'), null, 'already the SSD mode');+ assert.equal(saferStart(config, pc64, { arenaGb: 20 }, 'server_exited'), null, 'a model that fits RAM gains nothing from the SSD mode');+ assert.equal(saferStart(config, pc64, undefined, 'server_exited'), null, 'unknown model size: no guess');+ assert.equal(saferStart(config, pc64, iq3s, 'Some model files are missing'), null, 'not a memory failure: no second try');+ assert.equal(config.args.includes('--resident-budget-gib'), false, 'the input config is not changed');+});