Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Fit checks and slot sizing count the draft layer's ~1 GB of graphics memory while MTP is on
Niko1221671f4573da1e
diff --git a/apps/host/server/native-parallel.mjs b/apps/host/server/native-parallel.mjsindex 28b6964ed..dfee0f6ec 100644--- a/apps/host/server/native-parallel.mjs+++ b/apps/host/server/native-parallel.mjs@@ -41,6 +41,12 @@ export const CACHE_LOSS_SHARE = 0.2;* about 6.9 GiB goes to everything else; a second card in a layer split about 2 GiB).*/export const FIRST_CARD_OTHER_GB = 6.9;+/**+ * The draft layer (MTP) is part of that 6.9: about 1 GB of graphics memory (786 MiB of files and the draft head's ~218 MiB,+ * phaseA-tests/notes-021-settings.md). With "Draft layer (MTP)" off in Expert settings (`"mtp": false`) the engine does not load it, so+ * the card has that much more cache and the slots may use it.+ */+export const MTP_VRAM_GB = 1.0;export const NEXT_CARD_OTHER_GB = 2.0;/*** `--batch-mtp` lets each slot verify draft tokens. Where it pays and where it does not (all measured):@@ -77,7 +83,7 @@ export function engineFacts(config = {}) {const gpus = Array.isArray(config.gpu) ? config.gpu : config.gpu === undefined || config.gpu === null ? [0] : [config.gpu];return {args, context, kv, family, gpus,- mtp: args.includes('--mtp'),+ mtp: args.includes('--mtp') && config.mtp !== false,batchMtp: args.includes('--batch-mtp'),explicitBatch: args.includes('--batch') || args.includes('--slots'),};@@ -107,7 +113,7 @@ export function chooseParallel({ config = {}, values = {}, hardware = null, requif (!cards.length) return { slots: PARALLEL_POLICY === 'fixed' ? requested : 1, batchMtp: false, explicit: false, requested, note: null, slotGb: slot };const total = cards.reduce((a, b) => a + b, 0);const oneCard = facts.gpus.length === 1 && cards.length === 1;- const soloCache = Math.max(0, total - FIRST_CARD_OTHER_GB - NEXT_CARD_OTHER_GB * (cards.length - 1));+ const soloCache = Math.max(0, total - (FIRST_CARD_OTHER_GB - (facts.mtp ? 0 : MTP_VRAM_GB)) - NEXT_CARD_OTHER_GB * (cards.length - 1));let slots = requested;if (PARALLEL_POLICY === 'fit') while (slots > 1 && slots * slot > CACHE_LOSS_SHARE * soloCache) slots -= 1;const note = slots >= requested ? nulldiff --git a/apps/host/tests/parallel-default.test.mjs b/apps/host/tests/parallel-default.test.mjsindex 0253c010d..59db83bc5 100644--- a/apps/host/tests/parallel-default.test.mjs+++ b/apps/host/tests/parallel-default.test.mjs@@ -168,3 +168,12 @@ test('a load that fails even with one slot reports the failure, and a chosen numassert.equal(chosen.saved.length, 1);} finally { await chosen.native.close(); chosen.cleanup(); }});++test('the draft layer (MTP) is part of what the card is sized for; switched off, its 1 GB goes to the slots', () => {+ if (PARALLEL_POLICY !== 'fit') return;+ const slots = (gb, extra = {}, args) => chooseParallel({ config: { ...cfg(32768, extra), ...(args ? { args } : {}) }, hardware: hw(gb) }).slots;+ assert.equal(slots(26.5), 3, 'MTP on (the default): the measured sizing stands');+ assert.equal(slots(26.5, { mtp: false }), 4, 'a card with the draft layer off has 1 GB more cache');+ assert.equal(slots(26.5, {}, ['--max-context', '32768', '--spec', '4']), 4, 'no draft layer in the arguments');+ assert.equal(slots(12, { mtp: false }), 1, 'a 12 GB card still gets one request at a time');+});diff --git a/setup.py b/setup.pyindex 53d54f418..328aa5b1f 100644--- a/setup.py+++ b/setup.py@@ -4961,32 +4961,37 @@ def low_ram_needed(model, ram) -> bool:return ram < MODELS[model]["arena_gb"] + LOW_RAM_HEADROOM_GB-def low_ram_gpu_gb(model, vram_gb, ctx=32768, kv="int8") -> float:+#: The draft layer (MTP) takes about 1 GB of graphics memory: its 786 MiB of files plus the draft head's ~218 MiB+#: (phaseA-tests/notes-021-settings.md). It comes out of the expert cache, so every fit check counts it while MTP is on.+MTP_VRAM_GB = 1.0+++def low_ram_gpu_gb(model, vram_gb, ctx=32768, kv="int8", mtp=True) -> float:"""About how many GB of the model's experts the GPU's cache holds: its VRAM minus ~5 GB for the dense weights,buffers and a 32K context's KV cache, minus the KV cache of a longer context (in VRAM in the low-RAM mode: its RAM- has no room for KV streaming)."""+ has no room for KV streaming), minus ~1 GB for the draft layer (MTP) when it is on."""kv_tok = 13 * KV_CELL_BYTES.get(kv, 1056) # bytes per context token: 12 QSA layers + the draft layerlonger = max(0, ctx - 32768) * kv_tok / 1e9- return max(0.0, min(MODELS[model]["arena_gb"], vram_gb - 5 - longer))+ return max(0.0, min(MODELS[model]["arena_gb"], vram_gb - 5 - (MTP_VRAM_GB if mtp else 0.0) - longer))-def low_ram_gpu_share(model, vram_gb, ctx=32768, kv="int8") -> float:+def low_ram_gpu_share(model, vram_gb, ctx=32768, kv="int8", mtp=True) -> float:"""About how much of the model's experts the GPU holds."""- return low_ram_gpu_gb(model, vram_gb, ctx, kv) / MODELS[model]["arena_gb"]+ return low_ram_gpu_gb(model, vram_gb, ctx, kv, mtp) / MODELS[model]["arena_gb"]-def low_ram_resident(model, ram, vram_gb, ctx=32768, kv="int8") -> bool:+def low_ram_resident(model, ram, vram_gb, ctx=32768, kv="int8", mtp=True) -> bool:"""In the low-RAM mode: the experts the GPU does not hold fit the RAM with the usual room beside them, so they arecopied into RAM once (the resident variant, `--resident-experts`) instead of being read through the OS file cache(plain `--mmap-experts`, which a PC this short of RAM keeps re-reading from the SSD)."""- rest = MODELS[model]["arena_gb"] - low_ram_gpu_gb(model, vram_gb, ctx, kv)+ rest = MODELS[model]["arena_gb"] - low_ram_gpu_gb(model, vram_gb, ctx, kv, mtp)return ram >= rest + LOW_RAM_HEADROOM_GB-def low_ram_fits(model, ram, vram_gb) -> bool:+def low_ram_fits(model, ram, vram_gb, mtp=True) -> bool:"""In the low-RAM mode: the experts the GPU does not hold fit the RAM left beside the rest (as file cache)."""arena = MODELS[model]["arena_gb"]- return ram - 6 + max(0.0, vram_gb - 5) >= arena+ return ram - 6 + max(0.0, vram_gb - 5 - (MTP_VRAM_GB if mtp else 0.0)) >= arenadef low_ram_wanted(model, ram, choice="auto") -> bool:@@ -7711,12 +7716,13 @@ def main() -> int:if low_ram and not fam.get("ready"): # NVFP4's packs: the RAM-budget mode instead (step 7)arena = MODELS[model]["arena_gb"]vram = low_ram_vram(gpu) - (VISION[vision]["reserve_mib"] / 1024 if vision != "none" else 0)- share = low_ram_gpu_share(model, vram, ctx, kv)- rest = arena - low_ram_gpu_gb(model, vram, ctx, kv)- resident = a.low_ram == "resident" or (a.low_ram != "mmap" and low_ram_resident(model, ram, vram, ctx, kv))+ with_mtp = a.mtp != "off" # the draft layer's ~1 GB comes out of the expert cache+ share = low_ram_gpu_share(model, vram, ctx, kv, with_mtp)+ rest = arena - low_ram_gpu_gb(model, vram, ctx, kv, with_mtp)+ resident = a.low_ram == "resident" or (a.low_ram != "mmap" and low_ram_resident(model, ram, vram, ctx, kv, with_mtp))if multi: # every chosen card's share (the image encoder on the main one); #364 #384: the mapped variant- held = min(arena, low_ram_gpu_gb(model, vram, ctx, kv) +- sum(low_ram_gpu_gb(model, low_ram_vram(x), ctx, kv) for x in chosen[1:]))+ held = min(arena, low_ram_gpu_gb(model, vram, ctx, kv, with_mtp) ++ sum(low_ram_gpu_gb(model, low_ram_vram(x), ctx, kv, with_mtp) for x in chosen[1:]))share, rest = held / arena, arena - held# #642: the resident variant on a split, by the same rule as on one card - what no card holds fits the RAMresident = resident_split() and (a.low_ram == "resident" ordiff --git a/tools/strata_mcp.py b/tools/strata_mcp.pyindex a4fa29b8f..8780210ac 100644--- a/tools/strata_mcp.py+++ b/tools/strata_mcp.py@@ -1013,7 +1013,7 @@ class Strata:def low_fits(m):try:- return S.low_ram_fits(m, ram, vram) if S else ram - 6 + max(0.0, vram - 5) >= models[m]["arena_gb"]+ return S.low_ram_fits(m, ram, vram) if S else ram - 6 + max(0.0, vram - 5 - 1.0) >= models[m]["arena_gb"]except Exception: # noqa: BLE001return Falsenotes = []diff --git a/tools/test_setup_lowram.py b/tools/test_setup_lowram.pyindex b7240b82d..403c00cff 100644--- a/tools/test_setup_lowram.py+++ b/tools/test_setup_lowram.py@@ -80,5 +80,23 @@ class PagingQuestion(unittest.TestCase):self.assertIn("[y]" if choice == "off" else "[n]", asked[0])+class DraftLayerCountsInTheFit(unittest.TestCase):+ """The draft layer (MTP) takes about 1 GB of graphics memory out of the expert cache: every fit check counts it while it is on."""++ def test_the_card_holds_one_gb_less_with_mtp_on(self):+ on = setup.low_ram_gpu_gb("IQ3_XXS", 12.0)+ off = setup.low_ram_gpu_gb("IQ3_XXS", 12.0, mtp=False)+ self.assertAlmostEqual(off - on, setup.MTP_VRAM_GB)+ self.assertEqual(setup.MTP_VRAM_GB, 1.0)+ self.assertAlmostEqual(setup.low_ram_gpu_share("IQ3_XXS", 12.0, mtp=False) - setup.low_ram_gpu_share("IQ3_XXS", 12.0),+ 1.0 / setup.MODELS["IQ3_XXS"]["arena_gb"])++ def test_the_edge_of_what_fits_moves_by_the_draft_layer(self):+ arena = setup.MODELS["IQ1_M"]["arena_gb"]+ ram = arena + 6 - 7.0 + 0.5 # a card of 12 GB holds 7 GB beside the dense part; the draft layer needs one of them+ self.assertFalse(setup.low_ram_fits("IQ1_M", ram, 12.0))+ self.assertTrue(setup.low_ram_fits("IQ1_M", ram, 12.0, mtp=False))++if __name__ == "__main__":unittest.main()