Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

request slots: fit to the expert cache the card would have (measured), hardware from the host probe, safe default with no read

Niko1221ec53cae11fa9
diff --git a/apps/host/server/index.mjs b/apps/host/server/index.mjs
index b9b394a91..06e008435 100644
--- a/apps/host/server/index.mjs
+++ b/apps/host/server/index.mjs
@@ -80,7 +80,7 @@ const imageModels = createImageModels({ dataDir: store.DATA_DIR, modelsDir: () =
/* The installed app ships Strata Control (python + strata_control) next to the host; a source checkout has none and keeps its older paths. */
const CONTROL = findControl();
const control = createControl({ control: CONTROL });
-const native=createNativeControl({control,directory:store.DATA_DIR,modelsDirectory:()=>store.MODELS_DIR,nativeDataDirectory:store.nativeDataDirectory,beforeLoad:async()=>{if(models.REAL&&models.library().loaded){if(serving.state().requests.some(r=>runtimeModels.state().instances.some(i=>i.id===r.instanceId)&&["queued","generating"].includes(r.state)))throw new Error("Finish requests before switching engines.");await models.unload();runtimeState();}},requests:()=>serving.state().requests,changed:value=>hub.send('native',value)});
+const native=createNativeControl({control,hardwareProbe:()=>(hardware.cachedHardware()?.gpus??[]).map((g,n)=>({index:g.index??n,vram_gb:g.vramGb})),directory:store.DATA_DIR,modelsDirectory:()=>store.MODELS_DIR,nativeDataDirectory:store.nativeDataDirectory,beforeLoad:async()=>{if(models.REAL&&models.library().loaded){if(serving.state().requests.some(r=>runtimeModels.state().instances.some(i=>i.id===r.instanceId)&&["queued","generating"].includes(r.state)))throw new Error("Finish requests before switching engines.");await models.unload();runtimeState();}},requests:()=>serving.state().requests,changed:value=>hub.send('native',value)});
/* The agent: chats become DeepSeek Harness sessions when that component is installed. It is a separate
pinned component (STRATA_HARNESS_DIR), not part of this host, so its absence is a visible capability
state and the chat falls back to the direct path with a notice, never silently to another model. */
diff --git a/apps/host/server/native-control.mjs b/apps/host/server/native-control.mjs
index bb7ad4b73..c3a7baed0 100644
--- a/apps/host/server/native-control.mjs
+++ b/apps/host/server/native-control.mjs
@@ -17,7 +17,7 @@ const safeId=value=>typeof value==='string'&&/^[a-z0-9_-]{1,80}$/i.test(value);
const busy=requests=>requests.some(r=>['queued','generating'].includes(r.state));
function readJson(path,fallback={}){try{return JSON.parse(readFileSync(path,'utf8').replace(/^\uFEFF/,''));}catch{return fallback;}}
-export function createNativeControl({directory,requests=()=>[],changed=()=>{},spawnProcess=spawn,fetcher=fetch,modelsDirectory,nativeDataDirectory=value=>value,beforeLoad=async()=>{},control=null,defaultRepo=process.env.STRATA_NATIVE_REPO??resolve(ROOT,'../../Strata')}={}){
+export function createNativeControl({directory,requests=()=>[],changed=()=>{},spawnProcess=spawn,fetcher=fetch,modelsDirectory,nativeDataDirectory=value=>value,beforeLoad=async()=>{},hardwareProbe=()=>[],control=null,defaultRepo=process.env.STRATA_NATIVE_REPO??resolve(ROOT,'../../Strata')}={}){
mkdirSync(directory,{recursive:true});
const settingsPath=join(directory,'native-host.json');
let preferences=readJson(settingsPath,{repo:defaultRepo});
@@ -101,7 +101,7 @@ export function createNativeControl({directory,requests=()=>[],changed=()=>{},sp
if(cfg.vram_reserve_mib!==undefined&&cfg.vram_reserve_mib!==null){cfg.args=[...(cfg.args??[])];const at=cfg.args.indexOf('--vram-reserve-mib');if(at>=0)cfg.args.splice(at,2);cfg.args.push('--vram-reserve-mib',String(cfg.vram_reserve_mib));}if(cfg.vision){if(cfg.vision_tokens!=null)cfg.vision={...cfg.vision,max_tokens:cfg.vision_tokens};if(cfg.vision_min_tokens!=null)cfg.vision={...cfg.vision,min_tokens:cfg.vision_min_tokens};}cfg.mcp={...(cfg.mcp??{}),...(cfg.mcp_timeout_s!=null?{timeout_s:cfg.mcp_timeout_s}:{}),...(cfg.mcp_max_rounds!=null?{max_rounds:cfg.mcp_max_rounds}:{}),...(cfg.mcp_max_result_chars!=null?{max_result_chars:cfg.mcp_max_result_chars}:{})};
for(const arg of cfg.args??[])if(arg.endsWith('.gguf')&&!existsSync(resolve(repo(),arg)))throw new Error('An engine argument names a missing GGUF. Restore the saved arguments or repair the files.');
/* Installed app: several requests at once by default (4, fewer where the card has no room), unless a number was chosen. */
- const parallelChoice=packaged?chooseParallel({config:cfg,values:args.values??{},hardware:snapshot.hardware}):null;
+ const parallelChoice=packaged?chooseParallel({config:cfg,values:args.values??{},hardware:snapshot.hardware?.gpus?.length?snapshot.hardware:{gpus:hardwareProbe()??[]}}):null;
let parallelNote=null;
if(parallelChoice){const applied=applyParallel(cfg,parallelChoice);for(const k of Object.keys(cfg))if(!(k in applied.cfg))delete cfg[k];Object.assign(cfg,applied.cfg);parallelNote=applied.note;}
const fingerprint=createHash('sha256').update(JSON.stringify({...cfg,api_key:undefined,port:undefined})).digest('hex');
diff --git a/apps/host/server/native-parallel.mjs b/apps/host/server/native-parallel.mjs
index 2543112cb..7b340d56b 100644
--- a/apps/host/server/native-parallel.mjs
+++ b/apps/host/server/native-parallel.mjs
@@ -23,13 +23,25 @@ export const DEFAULT_SLOTS = 4;
export const MAX_SLOTS = 8;
/**
* THE one-line policy.
- * 'fit' the default aims for 4 and takes fewer where the slots would take more than SLOT_SHARE of the card's memory
- * (measured on a 12 GB RTX 3060, 32K: 4 slots took 3.7 GiB, left 0.35 GiB free and cost a single chat 37%)
+ * 'fit' the default aims for 4 and takes fewer where the slots would eat more than CACHE_LOSS_SHARE of the expert cache
+ * the card would otherwise have (the owner's choice: "fit to the card")
* 'fixed' the default is always DEFAULT_SLOTS, whatever the card (the engine still fits what it can)
+ *
+ * Why a cache budget: on a 12 GB RTX 3060 (IQ3_XXS, 32K) the slots' sessions come out of the expert cache. Measured, one request
+ * alone: no slots 61 tok/s (cache 5.1 GiB); 2 slots 41 (cache 3.25 GiB); 4 slots 38 (cache 1.4 GiB). Several requests together
+ * were SLOWER in total than queueing them (52-57 tok/s total queued, against 33-42 in slots). Slots start everyone at once;
+ * they pay off when the cache stays mostly intact.
*/
export const PARALLEL_POLICY = 'fit';
-/** Share of ALL the graphics memory the slots' sessions may take together under the 'fit' policy. */
-export const SLOT_SHARE = 0.2;
+/** The share of the solo expert cache the slots' sessions may take together under the 'fit' policy. */
+export const CACHE_LOSS_SHARE = 0.2;
+/**
+ * The expert cache a card has with one request at a time is what remains of its memory after the dense weights, the draft layer,
+ * one session, the safety reserve and the prompt path (measured: a 12 GB card ends with 5.1 GiB of cache and 0.35 GiB free, so
+ * about 6.9 GiB goes to everything else; a second card in a layer split about 2 GiB).
+ */
+export const FIRST_CARD_OTHER_GB = 6.9;
+export const NEXT_CARD_OTHER_GB = 2.0;
/**
* `--batch-mtp` lets each slot verify draft tokens. Measured on a 12 GB RTX 3060 (docs/BATCHING.md says it pays on larger
* cards): a lone chat 35.6 tok/s with it against 37.8 without, 2 chats 37.2 against 33.8 together, 4 chats 42.5 against 38.4 -
@@ -81,16 +93,15 @@ export function chooseParallel({ config = {}, values = {}, hardware = null, requ
return { slots: slots ?? 0, batchMtp: false, explicit: true, requested, note: null, slotGb: slot };
}
const cards = (hardware?.gpus ?? []).filter(g => facts.gpus.includes(g.index)).map(g => Number(g.vram_gb)).filter(n => Number.isFinite(n) && n > 0);
- // Without a hardware read the engine's own fitting is all there is: ask for the default and let it say what it can run.
- if (!cards.length) return { slots: requested, batchMtp: BATCH_MTP_POLICY === 'on' && facts.mtp && facts.gpus.length === 1, explicit: false, requested, note: null, slotGb: slot };
+ // Without a hardware read nothing can be sized: one at a time is the safe default (a number can still be chosen).
+ if (!cards.length) return { slots: PARALLEL_POLICY === 'fixed' ? requested : 1, batchMtp: false, explicit: false, requested, note: null, slotGb: slot };
const total = cards.reduce((a, b) => a + b, 0);
+ const soloCache = Math.max(0, total - FIRST_CARD_OTHER_GB - NEXT_CARD_OTHER_GB * (cards.length - 1));
let slots = requested;
- if (PARALLEL_POLICY === 'fit') while (slots > 1 && slots * slot > SLOT_SHARE * total) slots -= 1;
- const note = slots < requested
- ? (slots > 1
- ? `Your graphics card has room for ${slots} requests at once with this context size, so Strata set that. A smaller context size leaves room for more.`
- : 'Your graphics card has no spare room for several requests at once with this context size, so Strata set it to one at a time. A smaller context size leaves room for more.')
- : null;
+ if (PARALLEL_POLICY === 'fit') while (slots > 1 && slots * slot > CACHE_LOSS_SHARE * soloCache) slots -= 1;
+ const note = slots >= requested ? null
+ : slots > 1 ? `Your graphics card has room for ${slots} requests at once with this context size, so Strata set that. A smaller context size leaves room for more.`
+ : 'Strata set this model to one request at a time: your graphics card is best used that way, because more at once would make every answer slower. Helper agents and a second chat take their turn instead. A smaller context size leaves room for more at once.';
return { slots, batchMtp: BATCH_MTP_POLICY === 'on' && slots > 1 && facts.mtp && !facts.batchMtp && facts.gpus.length === 1, explicit: false, requested, note, slotGb: slot };
}
diff --git a/apps/host/tests/parallel-default.test.mjs b/apps/host/tests/parallel-default.test.mjs
index 39a3bf34c..fe04e4f46 100644
--- a/apps/host/tests/parallel-default.test.mjs
+++ b/apps/host/tests/parallel-default.test.mjs
@@ -15,31 +15,31 @@ import { buildProfile } from '../server/agent/profile.mjs';
const hw = (gb, n = 1) => ({ gpus: Array.from({ length: n }, (_, i) => ({ index: i, vram_gb: gb })) });
const cfg = (context, extra = {}) => ({ args: ['--max-context', String(context), '--mtp', '/m', '--spec', '4'], gpu: 0, ...extra });
-test('the default is 4 request slots, and the policy is one switch', () => {
+test('the default aims for 4 request slots; the policy is one switch; with no hardware read it is one at a time', () => {
assert.equal(DEFAULT_SLOTS, 4);
assert.ok(['fit', 'fixed'].includes(PARALLEL_POLICY));
- assert.equal(chooseParallel({ config: cfg(32768) }).slots, 4, 'no hardware read: ask for the default, the engine fits what it can');
+ const unknown = chooseParallel({ config: cfg(32768) });
+ assert.equal(unknown.slots, PARALLEL_POLICY === 'fixed' ? 4 : 1, 'nothing to size against: the safe default');
});
-test('sizing: 4 where the card has room, fewer where the slots would crowd out the expert cache, said in plain words', () => {
- // the slot cost was measured on the 0.1.45 engine: 0.92 GiB per slot at 32K
- assert.ok(Math.abs(slotGb({ context: 32768 }) * 0.931 - 0.92) < 0.1);
+test('sizing: the slots may take at most a fifth of the expert cache the card would have; said in plain words', () => {
+ // measured on the 0.1.45 engine: 0.92 GiB per slot at 32K; a 12 GB card ends with 5.1 GiB of expert cache
+ assert.ok(Math.abs(slotGb({ context: 32768 }) * 0.931 - 0.92) < 0.03);
if (PARALLEL_POLICY !== 'fit') return;
- assert.equal(chooseParallel({ config: cfg(32768), hardware: hw(24) }).slots, 4);
- assert.equal(chooseParallel({ config: cfg(32768), hardware: hw(16) }).slots, 3);
- const small = chooseParallel({ config: cfg(32768), hardware: hw(12) });
- assert.equal(small.slots, 2);
- assert.match(small.note, /room for 2 requests at once/);
- assert.match(small.note, /smaller context size leaves room for more/);
- const tight = chooseParallel({ config: cfg(65536), hardware: hw(12) });
- assert.equal(tight.slots, 1); assert.match(tight.note, /one at a time/);
- // two cards share the load
- assert.equal(chooseParallel({ config: cfg(65536, { gpu: [0, 1] }), hardware: hw(24, 2) }).slots, 4);
- assert.doesNotMatch(small.note + tight.note, /--|batch|VRAM|GiB|tok/i, 'no engine words');
+ const slots = (gb, context, extra = {}, cards = 1) => chooseParallel({ config: cfg(context, extra), hardware: hw(gb, cards) });
+ assert.deepEqual([12, 16, 20, 24, 32].map(gb => slots(gb, 32768).slots), [1, 1, 2, 3, 4]);
+ assert.deepEqual([12, 24, 32, 48].map(gb => slots(gb, 65536).slots), [1, 1, 2, 4], 'the context is per slot: doubling it halves the slots');
+ assert.equal(slots(24, 65536, { gpu: [0, 1] }, 2).slots, 4, 'two cards share the load');
+ const small = slots(12, 32768);
+ assert.equal(small.slots, 1); assert.match(small.note, /one request at a time/); assert.match(small.note, /Helper agents and a second chat take their turn/);
+ const some = slots(20, 32768);
+ assert.match(some.note, /room for 2 requests at once/); assert.match(some.note, /smaller context size leaves room for more/);
+ assert.equal(slots(32, 32768).note, null, 'no note when the default fits');
+ assert.doesNotMatch(small.note + some.note, /--|batch|VRAM|GiB|tok|cache/i, 'no engine words');
});
test('batch-mtp is off by default (measured: it does not pay on a 12 GB card); where the policy is on it applies to one card with a draft layer only', () => {
- const mtp = c => chooseParallel({ config: c, hardware: hw(24) });
+ const mtp = c => chooseParallel({ config: c, hardware: hw(32) });
assert.equal(BATCH_MTP_POLICY, 'off');
assert.equal(mtp(cfg(32768)).batchMtp, false);
const applied = applyParallel(cfg(32768), mtp(cfg(32768)));
@@ -93,7 +93,7 @@ function fixture({ failStarts = 0, config = {} } = {}) {
async stop(name) { calls.push(['stop', name]); return { ok: true }; },
};
const fetcher = async () => Response.json({ status: 'ok' });
- const native = createNativeControl({ control, directory: data, spawnProcess: () => { throw new Error('the host starts no process'); }, fetcher, requests: () => [] });
+ const native = createNativeControl({ control, hardwareProbe: () => [{ index: 0, vram_gb: 32 }], directory: data, spawnProcess: () => { throw new Error('the host starts no process'); }, fetcher, requests: () => [] });
return { native, saved, calls, cleanup: () => rmSync(root, { recursive: true, force: true }) };
}
const finish = async (native, id) => { for (let i = 0; i < 500; i++) { const job = native.state().jobs.find(j => j.id === id); if (job.state !== 'running') return job; await new Promise(r => setTimeout(r, 10)); } throw new Error('job did not finish'); };