Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
request slots: fit to the expert cache the card would have (measured), hardware from the host probe, safe default with no read
Niko1221ec53cae11fa9
diff --git a/apps/host/server/index.mjs b/apps/host/server/index.mjsindex b9b394a91..06e008435 100644--- a/apps/host/server/index.mjs+++ b/apps/host/server/index.mjs@@ -80,7 +80,7 @@ const imageModels = createImageModels({ dataDir: store.DATA_DIR, modelsDir: () =/* The installed app ships Strata Control (python + strata_control) next to the host; a source checkout has none and keeps its older paths. */const CONTROL = findControl();const control = createControl({ control: CONTROL });-const native=createNativeControl({control,directory:store.DATA_DIR,modelsDirectory:()=>store.MODELS_DIR,nativeDataDirectory:store.nativeDataDirectory,beforeLoad:async()=>{if(models.REAL&&models.library().loaded){if(serving.state().requests.some(r=>runtimeModels.state().instances.some(i=>i.id===r.instanceId)&&["queued","generating"].includes(r.state)))throw new Error("Finish requests before switching engines.");await models.unload();runtimeState();}},requests:()=>serving.state().requests,changed:value=>hub.send('native',value)});+const native=createNativeControl({control,hardwareProbe:()=>(hardware.cachedHardware()?.gpus??[]).map((g,n)=>({index:g.index??n,vram_gb:g.vramGb})),directory:store.DATA_DIR,modelsDirectory:()=>store.MODELS_DIR,nativeDataDirectory:store.nativeDataDirectory,beforeLoad:async()=>{if(models.REAL&&models.library().loaded){if(serving.state().requests.some(r=>runtimeModels.state().instances.some(i=>i.id===r.instanceId)&&["queued","generating"].includes(r.state)))throw new Error("Finish requests before switching engines.");await models.unload();runtimeState();}},requests:()=>serving.state().requests,changed:value=>hub.send('native',value)});/* The agent: chats become DeepSeek Harness sessions when that component is installed. It is a separatepinned component (STRATA_HARNESS_DIR), not part of this host, so its absence is a visible capabilitystate and the chat falls back to the direct path with a notice, never silently to another model. */diff --git a/apps/host/server/native-control.mjs b/apps/host/server/native-control.mjsindex bb7ad4b73..c3a7baed0 100644--- a/apps/host/server/native-control.mjs+++ b/apps/host/server/native-control.mjs@@ -17,7 +17,7 @@ const safeId=value=>typeof value==='string'&&/^[a-z0-9_-]{1,80}$/i.test(value);const busy=requests=>requests.some(r=>['queued','generating'].includes(r.state));function readJson(path,fallback={}){try{return JSON.parse(readFileSync(path,'utf8').replace(/^\uFEFF/,''));}catch{return fallback;}}-export function createNativeControl({directory,requests=()=>[],changed=()=>{},spawnProcess=spawn,fetcher=fetch,modelsDirectory,nativeDataDirectory=value=>value,beforeLoad=async()=>{},control=null,defaultRepo=process.env.STRATA_NATIVE_REPO??resolve(ROOT,'../../Strata')}={}){+export function createNativeControl({directory,requests=()=>[],changed=()=>{},spawnProcess=spawn,fetcher=fetch,modelsDirectory,nativeDataDirectory=value=>value,beforeLoad=async()=>{},hardwareProbe=()=>[],control=null,defaultRepo=process.env.STRATA_NATIVE_REPO??resolve(ROOT,'../../Strata')}={}){mkdirSync(directory,{recursive:true});const settingsPath=join(directory,'native-host.json');let preferences=readJson(settingsPath,{repo:defaultRepo});@@ -101,7 +101,7 @@ export function createNativeControl({directory,requests=()=>[],changed=()=>{},spif(cfg.vram_reserve_mib!==undefined&&cfg.vram_reserve_mib!==null){cfg.args=[...(cfg.args??[])];const at=cfg.args.indexOf('--vram-reserve-mib');if(at>=0)cfg.args.splice(at,2);cfg.args.push('--vram-reserve-mib',String(cfg.vram_reserve_mib));}if(cfg.vision){if(cfg.vision_tokens!=null)cfg.vision={...cfg.vision,max_tokens:cfg.vision_tokens};if(cfg.vision_min_tokens!=null)cfg.vision={...cfg.vision,min_tokens:cfg.vision_min_tokens};}cfg.mcp={...(cfg.mcp??{}),...(cfg.mcp_timeout_s!=null?{timeout_s:cfg.mcp_timeout_s}:{}),...(cfg.mcp_max_rounds!=null?{max_rounds:cfg.mcp_max_rounds}:{}),...(cfg.mcp_max_result_chars!=null?{max_result_chars:cfg.mcp_max_result_chars}:{})};for(const arg of cfg.args??[])if(arg.endsWith('.gguf')&&!existsSync(resolve(repo(),arg)))throw new Error('An engine argument names a missing GGUF. Restore the saved arguments or repair the files.');/* Installed app: several requests at once by default (4, fewer where the card has no room), unless a number was chosen. */- const parallelChoice=packaged?chooseParallel({config:cfg,values:args.values??{},hardware:snapshot.hardware}):null;+ const parallelChoice=packaged?chooseParallel({config:cfg,values:args.values??{},hardware:snapshot.hardware?.gpus?.length?snapshot.hardware:{gpus:hardwareProbe()??[]}}):null;let parallelNote=null;if(parallelChoice){const applied=applyParallel(cfg,parallelChoice);for(const k of Object.keys(cfg))if(!(k in applied.cfg))delete cfg[k];Object.assign(cfg,applied.cfg);parallelNote=applied.note;}const fingerprint=createHash('sha256').update(JSON.stringify({...cfg,api_key:undefined,port:undefined})).digest('hex');diff --git a/apps/host/server/native-parallel.mjs b/apps/host/server/native-parallel.mjsindex 2543112cb..7b340d56b 100644--- a/apps/host/server/native-parallel.mjs+++ b/apps/host/server/native-parallel.mjs@@ -23,13 +23,25 @@ export const DEFAULT_SLOTS = 4;export const MAX_SLOTS = 8;/*** THE one-line policy.- * 'fit' the default aims for 4 and takes fewer where the slots would take more than SLOT_SHARE of the card's memory- * (measured on a 12 GB RTX 3060, 32K: 4 slots took 3.7 GiB, left 0.35 GiB free and cost a single chat 37%)+ * 'fit' the default aims for 4 and takes fewer where the slots would eat more than CACHE_LOSS_SHARE of the expert cache+ * the card would otherwise have (the owner's choice: "fit to the card")* 'fixed' the default is always DEFAULT_SLOTS, whatever the card (the engine still fits what it can)+ *+ * Why a cache budget: on a 12 GB RTX 3060 (IQ3_XXS, 32K) the slots' sessions come out of the expert cache. Measured, one request+ * alone: no slots 61 tok/s (cache 5.1 GiB); 2 slots 41 (cache 3.25 GiB); 4 slots 38 (cache 1.4 GiB). Several requests together+ * were SLOWER in total than queueing them (52-57 tok/s total queued, against 33-42 in slots). Slots start everyone at once;+ * they pay off when the cache stays mostly intact.*/export const PARALLEL_POLICY = 'fit';-/** Share of ALL the graphics memory the slots' sessions may take together under the 'fit' policy. */-export const SLOT_SHARE = 0.2;+/** The share of the solo expert cache the slots' sessions may take together under the 'fit' policy. */+export const CACHE_LOSS_SHARE = 0.2;+/**+ * The expert cache a card has with one request at a time is what remains of its memory after the dense weights, the draft layer,+ * one session, the safety reserve and the prompt path (measured: a 12 GB card ends with 5.1 GiB of cache and 0.35 GiB free, so+ * about 6.9 GiB goes to everything else; a second card in a layer split about 2 GiB).+ */+export const FIRST_CARD_OTHER_GB = 6.9;+export const NEXT_CARD_OTHER_GB = 2.0;/*** `--batch-mtp` lets each slot verify draft tokens. Measured on a 12 GB RTX 3060 (docs/BATCHING.md says it pays on larger* cards): a lone chat 35.6 tok/s with it against 37.8 without, 2 chats 37.2 against 33.8 together, 4 chats 42.5 against 38.4 -@@ -81,16 +93,15 @@ export function chooseParallel({ config = {}, values = {}, hardware = null, requreturn { slots: slots ?? 0, batchMtp: false, explicit: true, requested, note: null, slotGb: slot };}const cards = (hardware?.gpus ?? []).filter(g => facts.gpus.includes(g.index)).map(g => Number(g.vram_gb)).filter(n => Number.isFinite(n) && n > 0);- // Without a hardware read the engine's own fitting is all there is: ask for the default and let it say what it can run.- if (!cards.length) return { slots: requested, batchMtp: BATCH_MTP_POLICY === 'on' && facts.mtp && facts.gpus.length === 1, explicit: false, requested, note: null, slotGb: slot };+ // Without a hardware read nothing can be sized: one at a time is the safe default (a number can still be chosen).+ if (!cards.length) return { slots: PARALLEL_POLICY === 'fixed' ? requested : 1, batchMtp: false, explicit: false, requested, note: null, slotGb: slot };const total = cards.reduce((a, b) => a + b, 0);+ const soloCache = Math.max(0, total - FIRST_CARD_OTHER_GB - NEXT_CARD_OTHER_GB * (cards.length - 1));let slots = requested;- if (PARALLEL_POLICY === 'fit') while (slots > 1 && slots * slot > SLOT_SHARE * total) slots -= 1;- const note = slots < requested- ? (slots > 1- ? `Your graphics card has room for ${slots} requests at once with this context size, so Strata set that. A smaller context size leaves room for more.`- : 'Your graphics card has no spare room for several requests at once with this context size, so Strata set it to one at a time. A smaller context size leaves room for more.')- : null;+ if (PARALLEL_POLICY === 'fit') while (slots > 1 && slots * slot > CACHE_LOSS_SHARE * soloCache) slots -= 1;+ const note = slots >= requested ? null+ : slots > 1 ? `Your graphics card has room for ${slots} requests at once with this context size, so Strata set that. A smaller context size leaves room for more.`+ : 'Strata set this model to one request at a time: your graphics card is best used that way, because more at once would make every answer slower. Helper agents and a second chat take their turn instead. A smaller context size leaves room for more at once.';return { slots, batchMtp: BATCH_MTP_POLICY === 'on' && slots > 1 && facts.mtp && !facts.batchMtp && facts.gpus.length === 1, explicit: false, requested, note, slotGb: slot };}diff --git a/apps/host/tests/parallel-default.test.mjs b/apps/host/tests/parallel-default.test.mjsindex 39a3bf34c..fe04e4f46 100644--- a/apps/host/tests/parallel-default.test.mjs+++ b/apps/host/tests/parallel-default.test.mjs@@ -15,31 +15,31 @@ import { buildProfile } from '../server/agent/profile.mjs';const hw = (gb, n = 1) => ({ gpus: Array.from({ length: n }, (_, i) => ({ index: i, vram_gb: gb })) });const cfg = (context, extra = {}) => ({ args: ['--max-context', String(context), '--mtp', '/m', '--spec', '4'], gpu: 0, ...extra });-test('the default is 4 request slots, and the policy is one switch', () => {+test('the default aims for 4 request slots; the policy is one switch; with no hardware read it is one at a time', () => {assert.equal(DEFAULT_SLOTS, 4);assert.ok(['fit', 'fixed'].includes(PARALLEL_POLICY));- assert.equal(chooseParallel({ config: cfg(32768) }).slots, 4, 'no hardware read: ask for the default, the engine fits what it can');+ const unknown = chooseParallel({ config: cfg(32768) });+ assert.equal(unknown.slots, PARALLEL_POLICY === 'fixed' ? 4 : 1, 'nothing to size against: the safe default');});-test('sizing: 4 where the card has room, fewer where the slots would crowd out the expert cache, said in plain words', () => {- // the slot cost was measured on the 0.1.45 engine: 0.92 GiB per slot at 32K- assert.ok(Math.abs(slotGb({ context: 32768 }) * 0.931 - 0.92) < 0.1);+test('sizing: the slots may take at most a fifth of the expert cache the card would have; said in plain words', () => {+ // measured on the 0.1.45 engine: 0.92 GiB per slot at 32K; a 12 GB card ends with 5.1 GiB of expert cache+ assert.ok(Math.abs(slotGb({ context: 32768 }) * 0.931 - 0.92) < 0.03);if (PARALLEL_POLICY !== 'fit') return;- assert.equal(chooseParallel({ config: cfg(32768), hardware: hw(24) }).slots, 4);- assert.equal(chooseParallel({ config: cfg(32768), hardware: hw(16) }).slots, 3);- const small = chooseParallel({ config: cfg(32768), hardware: hw(12) });- assert.equal(small.slots, 2);- assert.match(small.note, /room for 2 requests at once/);- assert.match(small.note, /smaller context size leaves room for more/);- const tight = chooseParallel({ config: cfg(65536), hardware: hw(12) });- assert.equal(tight.slots, 1); assert.match(tight.note, /one at a time/);- // two cards share the load- assert.equal(chooseParallel({ config: cfg(65536, { gpu: [0, 1] }), hardware: hw(24, 2) }).slots, 4);- assert.doesNotMatch(small.note + tight.note, /--|batch|VRAM|GiB|tok/i, 'no engine words');+ const slots = (gb, context, extra = {}, cards = 1) => chooseParallel({ config: cfg(context, extra), hardware: hw(gb, cards) });+ assert.deepEqual([12, 16, 20, 24, 32].map(gb => slots(gb, 32768).slots), [1, 1, 2, 3, 4]);+ assert.deepEqual([12, 24, 32, 48].map(gb => slots(gb, 65536).slots), [1, 1, 2, 4], 'the context is per slot: doubling it halves the slots');+ assert.equal(slots(24, 65536, { gpu: [0, 1] }, 2).slots, 4, 'two cards share the load');+ const small = slots(12, 32768);+ assert.equal(small.slots, 1); assert.match(small.note, /one request at a time/); assert.match(small.note, /Helper agents and a second chat take their turn/);+ const some = slots(20, 32768);+ assert.match(some.note, /room for 2 requests at once/); assert.match(some.note, /smaller context size leaves room for more/);+ assert.equal(slots(32, 32768).note, null, 'no note when the default fits');+ assert.doesNotMatch(small.note + some.note, /--|batch|VRAM|GiB|tok|cache/i, 'no engine words');});test('batch-mtp is off by default (measured: it does not pay on a 12 GB card); where the policy is on it applies to one card with a draft layer only', () => {- const mtp = c => chooseParallel({ config: c, hardware: hw(24) });+ const mtp = c => chooseParallel({ config: c, hardware: hw(32) });assert.equal(BATCH_MTP_POLICY, 'off');assert.equal(mtp(cfg(32768)).batchMtp, false);const applied = applyParallel(cfg(32768), mtp(cfg(32768)));@@ -93,7 +93,7 @@ function fixture({ failStarts = 0, config = {} } = {}) {async stop(name) { calls.push(['stop', name]); return { ok: true }; },};const fetcher = async () => Response.json({ status: 'ok' });- const native = createNativeControl({ control, directory: data, spawnProcess: () => { throw new Error('the host starts no process'); }, fetcher, requests: () => [] });+ const native = createNativeControl({ control, hardwareProbe: () => [{ index: 0, vram_gb: 32 }], directory: data, spawnProcess: () => { throw new Error('the host starts no process'); }, fetcher, requests: () => [] });return { native, saved, calls, cleanup: () => rmSync(root, { recursive: true, force: true }) };}const finish = async (native, id) => { for (let i = 0; i < 500; i++) { const job = native.state().jobs.find(j => j.id === id); if (job.state !== 'running') return job; await new Promise(r => setTimeout(r, 10)); } throw new Error('job did not finish'); };