Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
setup: --ple-io ram on an SSD too when the RAM can spare the n-gram table (Linux)
Niko1221b25b30970340
diff --git a/setup.py b/setup.pyindex 3dd2ca51a..edc8969ca 100644--- a/setup.py+++ b/setup.py@@ -4799,6 +4799,12 @@ def is_wsl() -> bool:return sys.platform.startswith("linux") and "microsoft" in platform.uname().release.lower()+def ple_ram_platform_ok() -> bool:+ """147-verify: --ple-io ram for a PC with RAM to spare was measured on Linux only (Windows locks pages differently and+ WSL pins little), so only there."""+ return not WIN and not is_wsl()++def rotational_disk(path) -> str | None:"""#605 (Linux): the disk's name when `path` is on a rotational disk (sysfs queue/rotational), else None."""if WIN:@@ -6887,6 +6893,24 @@ def main() -> int:warn(f"the model is on a rotational disk ({disk}): its n-gram table is read from it at random, which can "f"stall prompts for minutes (#605). An SSD is recommended; with ~{table_gb:.0f} GB more RAM, ""--ple-io ram in the config's args keeps the table in RAM instead")+ elif not low_ram and ple_ram_platform_ok():+ # 147-verify: on an SSD too, a PC with RAM to spare keeps the n-gram table in RAM: a new prompt's first chunk then+ # does not wait for its table rows (65,536 random SSD reads at 4K tokens). RTX PRO 5000 (Linux, NVMe, fresh engine+ # per prompt, 3 rounds, the same bits): prompt 4K 1298 -> 1059 ms (-18%), 20K 4644 -> 4223 ms (-9%), 98K 20447 ->+ # 20027 ms (-2%), decode +0.8%. On a card whose prompt is slow (R9700) the read hides behind layer 0 and nothing+ # changes. It costs the table's size in RAM, so only with 16 GB to spare beyond the model's own need; a smaller PC+ # keeps reading from the SSD as before.+ try:+ tensor = None if ready else next((t for t in GGUFFile(ple).tensors if t.name == "per_layer_token_embd.weight"), None)+ size = ple.stat().st_size if ready else getattr(tensor, "expected_bytes", lambda: None)()+ except Exception:+ size = None+ table_gb = size / 1e9 if size else 28.8+ if ram >= MODELS[model]["ram_gb"] + table_gb + 16:+ args += ["--ple-io", "ram"]+ ok(f"{ram:.0f} GB of RAM: the {table_gb:.0f} GB n-gram table is kept in RAM (--ple-io ram), so a new prompt does "+ "not wait for its rows from the SSD (a 4K prompt about 18% sooner on a fast card); remove --ple-io ram from "+ "the config's args to read it from the SSD instead")# KV streaming: from 64K up the whole KV cache lives in RAM and only the part the attention reads (32K positions# per layer) stays in VRAM; the VRAM it frees holds more experts (+6% at 128K, +23% at 262K with Q2_0). It# costs ~13.7 KB of RAM per context token with 8-bit KV (1.7 GB at 128K), 10.6 KB with K8V4, 7.5 KB with 4-bit, so onlydiff --git a/tools/test_setup_choices.py b/tools/test_setup_choices.pyindex 0b048c99d..2e7540cfd 100644--- a/tools/test_setup_choices.py+++ b/tools/test_setup_choices.py@@ -371,11 +371,12 @@ class HipVision(unittest.TestCase):class RotationalDisk(unittest.TestCase):"""#605: a model on a rotational disk gets --ple-io ram when the n-gram table fits the RAM, else a warning."""- def install(self, ram, disk):+ def install(self, ram, disk, platform_ok=False, model="Q2_0"):from test_setup_golden import PROFILES, install_, found = PROFILES["64GB-1x32GB"]- return install(ram, found, ["--family", "qwen", "--model", "Q2_0", "--no-start"],- extra=[mock.patch.object(setup, "rotational_disk", lambda p: disk)])+ return install(ram, found, ["--family", "qwen", "--model", model, "--no-start"],+ extra=[mock.patch.object(setup, "rotational_disk", lambda p: disk),+ mock.patch.object(setup, "ple_ram_platform_ok", lambda: platform_ok)])def test_fits(self):code, out, cfg, _ = self.install(127.8, "sdb")@@ -395,6 +396,29 @@ class RotationalDisk(unittest.TestCase):self.assertNotIn("--ple-io", cfg["args"])self.assertNotIn("rotational", out)+ def test_ssd_ram_to_spare(self):+ # 147-verify (Linux): an SSD and RAM beyond the model's need + the table + 16 GB: the table in RAM too+ code, out, cfg, _ = self.install(127.8, None, platform_ok=True)+ self.assertEqual(code, 0, out)+ a = cfg["args"]+ self.assertEqual(a[a.index("--ple-io") + 1], "ram")+ self.assertIn("kept in RAM (--ple-io ram)", out)+ self.assertNotIn("rotational", out)++ def test_ssd_little_ram_keeps_the_ssd(self):+ # Q2_0 needs 48 GB + the 28.8 GB table + 16: a 64 GB / 92 GB PC keeps the SSD reads+ for ram in (63.7, 92.0):+ code, out, cfg, _ = self.install(ram, None, platform_ok=True)+ self.assertEqual(code, 0, out)+ self.assertNotIn("--ple-io", cfg["args"])++ def test_ssd_ram_rule_follows_the_model(self):+ # IQ3_S needs 62 GB: 96 GB is not enough to spare the table, 128 GB is+ for ram, want in ((95.8, False), (127.8, True)):+ code, out, cfg, _ = self.install(ram, None, platform_ok=True, model="IQ3_S")+ self.assertEqual(code, 0, out)+ self.assertEqual("--ple-io" in cfg["args"], want, (ram, cfg["args"]))+def test_sysfs(self):if not sys.platform.startswith("linux"):self.assertIsNone(setup.rotational_disk(__file__)) # Windows: neverdiff --git a/tools/test_setup_golden.py b/tools/test_setup_golden.pyindex af0777a45..a1062065c 100644--- a/tools/test_setup_golden.py+++ b/tools/test_setup_golden.py@@ -105,6 +105,7 @@ def install(ram, found, argv, answers=None, extra=(), avx512=False, configs=()):mock.patch.object(setup, "page_file_gb", lambda: 16.0),mock.patch.object(setup, "is_wsl", lambda: False),mock.patch.object(setup, "rotational_disk", lambda p: None), # #605: the test PC's own disk+ mock.patch.object(setup, "ple_ram_platform_ok", lambda: False), # 147-verify: the golden configs as on Windowsmock.patch.object(setup, "free_gb", lambda p: 900.0),mock.patch.object(setup, "pip_install", lambda *a, **k: None),mock.patch.object(setup, "get_llama_cpp", lambda: t / "llama.cpp"),