Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

setup: --ple-io ram on an SSD too when the RAM can spare the n-gram table (Linux)

Niko1221b25b30970340
diff --git a/setup.py b/setup.py
index 3dd2ca51a..edc8969ca 100644
--- a/setup.py
+++ b/setup.py
@@ -4799,6 +4799,12 @@ def is_wsl() -> bool:
return sys.platform.startswith("linux") and "microsoft" in platform.uname().release.lower()
+def ple_ram_platform_ok() -> bool:
+ """147-verify: --ple-io ram for a PC with RAM to spare was measured on Linux only (Windows locks pages differently and
+ WSL pins little), so only there."""
+ return not WIN and not is_wsl()
+
+
def rotational_disk(path) -> str | None:
"""#605 (Linux): the disk's name when `path` is on a rotational disk (sysfs queue/rotational), else None."""
if WIN:
@@ -6887,6 +6893,24 @@ def main() -> int:
warn(f"the model is on a rotational disk ({disk}): its n-gram table is read from it at random, which can "
f"stall prompts for minutes (#605). An SSD is recommended; with ~{table_gb:.0f} GB more RAM, "
"--ple-io ram in the config's args keeps the table in RAM instead")
+ elif not low_ram and ple_ram_platform_ok():
+ # 147-verify: on an SSD too, a PC with RAM to spare keeps the n-gram table in RAM: a new prompt's first chunk then
+ # does not wait for its table rows (65,536 random SSD reads at 4K tokens). RTX PRO 5000 (Linux, NVMe, fresh engine
+ # per prompt, 3 rounds, the same bits): prompt 4K 1298 -> 1059 ms (-18%), 20K 4644 -> 4223 ms (-9%), 98K 20447 ->
+ # 20027 ms (-2%), decode +0.8%. On a card whose prompt is slow (R9700) the read hides behind layer 0 and nothing
+ # changes. It costs the table's size in RAM, so only with 16 GB to spare beyond the model's own need; a smaller PC
+ # keeps reading from the SSD as before.
+ try:
+ tensor = None if ready else next((t for t in GGUFFile(ple).tensors if t.name == "per_layer_token_embd.weight"), None)
+ size = ple.stat().st_size if ready else getattr(tensor, "expected_bytes", lambda: None)()
+ except Exception:
+ size = None
+ table_gb = size / 1e9 if size else 28.8
+ if ram >= MODELS[model]["ram_gb"] + table_gb + 16:
+ args += ["--ple-io", "ram"]
+ ok(f"{ram:.0f} GB of RAM: the {table_gb:.0f} GB n-gram table is kept in RAM (--ple-io ram), so a new prompt does "
+ "not wait for its rows from the SSD (a 4K prompt about 18% sooner on a fast card); remove --ple-io ram from "
+ "the config's args to read it from the SSD instead")
# KV streaming: from 64K up the whole KV cache lives in RAM and only the part the attention reads (32K positions
# per layer) stays in VRAM; the VRAM it frees holds more experts (+6% at 128K, +23% at 262K with Q2_0). It
# costs ~13.7 KB of RAM per context token with 8-bit KV (1.7 GB at 128K), 10.6 KB with K8V4, 7.5 KB with 4-bit, so only
diff --git a/tools/test_setup_choices.py b/tools/test_setup_choices.py
index 0b048c99d..2e7540cfd 100644
--- a/tools/test_setup_choices.py
+++ b/tools/test_setup_choices.py
@@ -371,11 +371,12 @@ class HipVision(unittest.TestCase):
class RotationalDisk(unittest.TestCase):
"""#605: a model on a rotational disk gets --ple-io ram when the n-gram table fits the RAM, else a warning."""
- def install(self, ram, disk):
+ def install(self, ram, disk, platform_ok=False, model="Q2_0"):
from test_setup_golden import PROFILES, install
_, found = PROFILES["64GB-1x32GB"]
- return install(ram, found, ["--family", "qwen", "--model", "Q2_0", "--no-start"],
- extra=[mock.patch.object(setup, "rotational_disk", lambda p: disk)])
+ return install(ram, found, ["--family", "qwen", "--model", model, "--no-start"],
+ extra=[mock.patch.object(setup, "rotational_disk", lambda p: disk),
+ mock.patch.object(setup, "ple_ram_platform_ok", lambda: platform_ok)])
def test_fits(self):
code, out, cfg, _ = self.install(127.8, "sdb")
@@ -395,6 +396,29 @@ class RotationalDisk(unittest.TestCase):
self.assertNotIn("--ple-io", cfg["args"])
self.assertNotIn("rotational", out)
+ def test_ssd_ram_to_spare(self):
+ # 147-verify (Linux): an SSD and RAM beyond the model's need + the table + 16 GB: the table in RAM too
+ code, out, cfg, _ = self.install(127.8, None, platform_ok=True)
+ self.assertEqual(code, 0, out)
+ a = cfg["args"]
+ self.assertEqual(a[a.index("--ple-io") + 1], "ram")
+ self.assertIn("kept in RAM (--ple-io ram)", out)
+ self.assertNotIn("rotational", out)
+
+ def test_ssd_little_ram_keeps_the_ssd(self):
+ # Q2_0 needs 48 GB + the 28.8 GB table + 16: a 64 GB / 92 GB PC keeps the SSD reads
+ for ram in (63.7, 92.0):
+ code, out, cfg, _ = self.install(ram, None, platform_ok=True)
+ self.assertEqual(code, 0, out)
+ self.assertNotIn("--ple-io", cfg["args"])
+
+ def test_ssd_ram_rule_follows_the_model(self):
+ # IQ3_S needs 62 GB: 96 GB is not enough to spare the table, 128 GB is
+ for ram, want in ((95.8, False), (127.8, True)):
+ code, out, cfg, _ = self.install(ram, None, platform_ok=True, model="IQ3_S")
+ self.assertEqual(code, 0, out)
+ self.assertEqual("--ple-io" in cfg["args"], want, (ram, cfg["args"]))
+
def test_sysfs(self):
if not sys.platform.startswith("linux"):
self.assertIsNone(setup.rotational_disk(__file__)) # Windows: never
diff --git a/tools/test_setup_golden.py b/tools/test_setup_golden.py
index af0777a45..a1062065c 100644
--- a/tools/test_setup_golden.py
+++ b/tools/test_setup_golden.py
@@ -105,6 +105,7 @@ def install(ram, found, argv, answers=None, extra=(), avx512=False, configs=()):
mock.patch.object(setup, "page_file_gb", lambda: 16.0),
mock.patch.object(setup, "is_wsl", lambda: False),
mock.patch.object(setup, "rotational_disk", lambda p: None), # #605: the test PC's own disk
+ mock.patch.object(setup, "ple_ram_platform_ok", lambda: False), # 147-verify: the golden configs as on Windows
mock.patch.object(setup, "free_gb", lambda p: 900.0),
mock.patch.object(setup, "pip_install", lambda *a, **k: None),
mock.patch.object(setup, "get_llama_cpp", lambda: t / "llama.cpp"),