Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
batch-mtp on automatically for one big card (24 GB+, several slots, MTP on); off on small cards and layer splits; Expert mode MTP per request overrides
Niko1221bdb04dd8d6d8
diff --git a/apps/host/server/native-parallel.mjs b/apps/host/server/native-parallel.mjsindex 7b340d56b..28b6964ed 100644--- a/apps/host/server/native-parallel.mjs+++ b/apps/host/server/native-parallel.mjs@@ -43,12 +43,21 @@ export const CACHE_LOSS_SHARE = 0.2;export const FIRST_CARD_OTHER_GB = 6.9;export const NEXT_CARD_OTHER_GB = 2.0;/**- * `--batch-mtp` lets each slot verify draft tokens. Measured on a 12 GB RTX 3060 (docs/BATCHING.md says it pays on larger- * cards): a lone chat 35.6 tok/s with it against 37.8 without, 2 chats 37.2 against 33.8 together, 4 chats 42.5 against 38.4 -- * a small gain for several chats, a small loss for one, and 400 MiB less expert cache. Off until a card shows it pays; Expert- * mode adds it by hand (`--batch-mtp` in the engine arguments).+ * `--batch-mtp` lets each slot verify draft tokens. Where it pays and where it does not (all measured):+ * - RTX PRO 5000 72 GB, 32K (FP4-Local): 1 stream 140 tok/s either way; 2 streams 129 -> 160 (+24%) with it; 4 streams 177 either way;+ * the text is exactly the batch text.+ * - 12 GB RTX 3060: a lone chat 35.6 tok/s with it against 37.8 without, 2 chats 37.2 against 33.8, 4 chats 42.5 against 38.4, and+ * 400 MiB less expert cache: not worth it on a small card.+ * - 2 cards in a layer split with all experts in VRAM: -22% (docs/BATCHING.md): never on a layer split.+ * 'auto' turns it on for one card of at least BATCH_MTP_MIN_VRAM_GB with a draft layer, 2 or more slots, and room for the slot drafters+ * with a margin. 'off' never adds it. Expert mode decides either way: `--batch-mtp` in the engine arguments, or "batchMtp": true / false+ * in the model's settings (or the load's values).*/-export const BATCH_MTP_POLICY = 'off';+export const BATCH_MTP_POLICY = 'auto';+export const BATCH_MTP_MIN_VRAM_GB = 24;+/** Memory the slot drafters need per slot (their K/V ring and buffers; measured about 0.1 GiB, rounded up) and the cache that must remain after them. */+export const BATCH_MTP_DRAFTER_GB = 0.25;+export const BATCH_MTP_MARGIN_GB = 2;const argValue = (args, flag) => { const at = args.indexOf(flag); return at >= 0 ? args[at + 1] : undefined; };@@ -90,19 +99,25 @@ export function chooseParallel({ config = {}, values = {}, hardware = null, requ// A number someone chose (including 1) is the person's decision, and so is an explicit --batch in the engine arguments.if (facts.explicitBatch || (chosen !== null && Number.isFinite(chosen))) {const slots = facts.explicitBatch ? null : Math.max(1, Math.min(1024, Math.trunc(chosen)));- return { slots: slots ?? 0, batchMtp: false, explicit: true, requested, note: null, slotGb: slot };+ const wantMtp = (values.batchMtp ?? config.batchMtp) === true && !facts.batchMtp && facts.mtp && facts.gpus.length === 1 && (slots ?? 0) > 1;+ return { slots: slots ?? 0, batchMtp: wantMtp, explicit: true, requested, note: null, slotGb: slot };}const cards = (hardware?.gpus ?? []).filter(g => facts.gpus.includes(g.index)).map(g => Number(g.vram_gb)).filter(n => Number.isFinite(n) && n > 0);// Without a hardware read nothing can be sized: one at a time is the safe default (a number can still be chosen).if (!cards.length) return { slots: PARALLEL_POLICY === 'fixed' ? requested : 1, batchMtp: false, explicit: false, requested, note: null, slotGb: slot };const total = cards.reduce((a, b) => a + b, 0);+ const oneCard = facts.gpus.length === 1 && cards.length === 1;const soloCache = Math.max(0, total - FIRST_CARD_OTHER_GB - NEXT_CARD_OTHER_GB * (cards.length - 1));let slots = requested;if (PARALLEL_POLICY === 'fit') while (slots > 1 && slots * slot > CACHE_LOSS_SHARE * soloCache) slots -= 1;const note = slots >= requested ? null: slots > 1 ? `Your graphics card has room for ${slots} requests at once with this context size, so Strata set that. A smaller context size leaves room for more.`: 'Strata set this model to one request at a time: your graphics card is best used that way, because more at once would make every answer slower. Helper agents and a second chat take their turn instead. A smaller context size leaves room for more at once.';- return { slots, batchMtp: BATCH_MTP_POLICY === 'on' && slots > 1 && facts.mtp && !facts.batchMtp && facts.gpus.length === 1, explicit: false, requested, note, slotGb: slot };+ const wish = values.batchMtp ?? config.batchMtp; // Expert mode: true adds it by hand, false keeps it off+ const roomLeft = soloCache - slots * (slot + BATCH_MTP_DRAFTER_GB);+ const auto = BATCH_MTP_POLICY === 'auto' && oneCard && cards[0] >= BATCH_MTP_MIN_VRAM_GB && roomLeft >= BATCH_MTP_MARGIN_GB;+ const batchMtp = slots > 1 && facts.mtp && !facts.batchMtp && wish !== false && (wish === true ? oneCard : auto);+ return { slots, batchMtp, explicit: false, requested, note, slotGb: slot };}/**@@ -110,8 +125,11 @@ export function chooseParallel({ config = {}, values = {}, hardware = null, requ* Returns the configuration and the plain-words note (or null).*/export function applyParallel(cfg, choice) {- if (choice.explicit) return { cfg, note: null };- const next = { ...cfg };+ // `batchMtp` is the app's own setting (Expert mode); the engine's config does not know it, the argument is what it reads.+ const { batchMtp: _wish, ...clean } = cfg;+ if (choice.explicit && !choice.batchMtp) return { cfg: 'batchMtp' in cfg ? clean : cfg, note: null };+ if (choice.explicit) return { cfg: { ...clean, args: [...(clean.args ?? []), '--batch-mtp'] }, note: null };+ const next = { ...clean };if (choice.slots > 1) next.parallel = choice.slots; else delete next.parallel;if (choice.batchMtp) next.args = [...(next.args ?? []), '--batch-mtp'];return { cfg: next, note: choice.note };diff --git a/apps/host/server/native-settings.mjs b/apps/host/server/native-settings.mjsindex be3a9d44c..7a4ed7d21 100644--- a/apps/host/server/native-settings.mjs+++ b/apps/host/server/native-settings.mjs@@ -7,6 +7,7 @@ export const NATIVE_FIELDS=[{id:'aliases',label:'API model aliases',group:'Serving',type:'array',hint:'Comma-separated names clients may use for this model.'},{id:'idle_unload_s',label:'Unload after idle seconds',group:'Serving',type:'number',min:0,max:86400,hint:'Zero keeps the model loaded.'},{id:'lazy_load',label:'Load on first request',group:'Serving',type:'boolean'},+ {id:'batchMtp',label:'MTP per request',group:'Serving',type:'boolean',hint:'Each concurrent request uses the draft layer. Left alone, Strata turns it on for one graphics card with 24 GB or more and several requests at once, and off otherwise; choose on or off here to decide yourself.'},{id:'parallel',label:'Concurrent requests',group:'Serving',type:'number',min:1,max:8,hint:'More slots need more memory and can slow each response.'},{id:'engine_silence_s',label:'Engine stall timeout (seconds)',group:'Recovery',type:'number',min:10,max:3600},{id:'api_monitor',label:'Keep API request metadata',group:'Recovery',type:'boolean'},diff --git a/apps/host/tests/parallel-default.test.mjs b/apps/host/tests/parallel-default.test.mjsindex fe04e4f46..79a2cafe9 100644--- a/apps/host/tests/parallel-default.test.mjs+++ b/apps/host/tests/parallel-default.test.mjs@@ -38,17 +38,29 @@ test('sizing: the slots may take at most a fifth of the expert cache the card woassert.doesNotMatch(small.note + some.note, /--|batch|VRAM|GiB|tok|cache/i, 'no engine words');});-test('batch-mtp is off by default (measured: it does not pay on a 12 GB card); where the policy is on it applies to one card with a draft layer only', () => {- const mtp = c => chooseParallel({ config: c, hardware: hw(32) });- assert.equal(BATCH_MTP_POLICY, 'off');- assert.equal(mtp(cfg(32768)).batchMtp, false);- const applied = applyParallel(cfg(32768), mtp(cfg(32768)));- assert.equal(applied.cfg.parallel, 4); assert.ok(!applied.cfg.args.includes('--batch-mtp'));- // the rule itself, for the day the policy is turned on- if (BATCH_MTP_POLICY === 'on') {- assert.equal(mtp(cfg(32768)).batchMtp, true);- assert.equal(mtp(cfg(32768, { gpu: [0, 1] })).batchMtp, false);- }+test('batch-mtp is automatic on big single cards only: 24 GB and up with room for the drafters, never on small cards or a layer split; Expert mode overrides either way', () => {+ assert.equal(BATCH_MTP_POLICY, 'auto');+ const mtp = (gb, extra = {}, cards = 1, values = {}, context = 32768) => chooseParallel({ config: cfg(context, extra), hardware: hw(gb, cards), values });+ assert.deepEqual([12, 16, 20].map(gb => mtp(gb).batchMtp), [false, false, false], 'small cards: it did not pay (3060: -6% for one chat, 400 MiB less cache)');+ assert.deepEqual([24, 32, 48, 72].map(gb => mtp(gb).batchMtp), [true, true, true, true]);+ assert.equal(mtp(72).slots, 4);+ assert.equal(mtp(24, {}, 1, {}, 131072).batchMtp, false, 'one slot only at this context: nothing to batch');+ assert.equal(mtp(72, { gpu: [0, 1] }, 2).batchMtp, false, 'a layer split: -22% with all experts in VRAM');+ assert.equal(chooseParallel({ config: { args: ['--max-context', '32768'], gpu: 0 }, hardware: hw(72) }).batchMtp, false, 'no draft layer, nothing to batch');+ assert.equal(chooseParallel({ config: cfg(32768, { args: ['--max-context', '32768', '--mtp', '/m', '--batch-mtp'] }), hardware: hw(72) }).batchMtp, false, 'already there: not added twice');+ const applied = applyParallel(cfg(32768), mtp(72));+ assert.ok(applied.cfg.args.includes('--batch-mtp')); assert.equal(applied.cfg.parallel, 4);+ // Expert mode: off wins on a big card; on wins on a small one-card setup; an explicit slot count can still ask for it+ assert.equal(mtp(72, {}, 1, { batchMtp: false }).batchMtp, false);+ assert.equal(mtp(72, { batchMtp: false }).batchMtp, false);+ assert.equal(mtp(20, {}, 1, { batchMtp: true }).batchMtp, true, '20 GB: 2 slots, asked for by hand');+ assert.equal(mtp(72, { gpu: [0, 1] }, 2, { batchMtp: true }).batchMtp, false, 'never forced on a layer split');+ const manual = chooseParallel({ config: cfg(32768), values: { parallel: 2, batchMtp: true }, hardware: hw(12) });+ assert.deepEqual([manual.slots, manual.explicit, manual.batchMtp], [2, true, true]);+ assert.ok(applyParallel(cfg(32768), manual).cfg.args.includes('--batch-mtp'));+ assert.ok(!applyParallel(cfg(32768, { parallel: 2 }), chooseParallel({ config: cfg(32768, { parallel: 2 }), hardware: hw(72) })).cfg.args.includes('--batch-mtp'), 'a number the person chose alone does not add it');+ // the retry with one request at a time takes it away again+ assert.ok(!withoutParallel(applied.cfg).args.includes('--batch-mtp'));});test("a number the person chose is kept exactly: Expert mode's value, the saved settings, an explicit --batch", () => {@@ -106,7 +118,7 @@ test('the installed app loads with 4 request slots by default', async () => {assert.equal(job.state, 'done', job.error);assert.equal(f.saved.length, 1);assert.equal(f.saved[0].parallel, 4);- assert.ok(!f.saved[0].args.includes('--batch-mtp'));+ assert.ok(f.saved[0].args.includes('--batch-mtp'), 'a 32 GB card: MTP per request is on automatically');assert.equal(f.native.state().instances[0].parallel, 4, 'the agent reads the slot count from here');} finally { await f.native.close(); f.cleanup(); }});