Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
dsv41 plan: four or more AMD cards run the IST-DAS sets with --tp 1 (8K prompt 370 against 196 tokens a second on 4 x R9700)
Niko122167d5f44bf83a
diff --git a/strata_control/dsv41_fit.py b/strata_control/dsv41_fit.pyindex b5dc68572..9eaf5c0aa 100644--- a/strata_control/dsv41_fit.py+++ b/strata_control/dsv41_fit.py@@ -122,12 +122,22 @@ def split_layers(weights: list[float], layers: int = LAYERS) -> list[tuple[int,return out-def tp_for(n_cards: int) -> int:- """--tp for this many cards: pairs of cards share each layer's matrices (measured 2x faster prompts than a plain layer split).- The engine's own check refuses 4 (576 is not a multiple of 128) and drops to 2, so 2 is what is written for two or more cards."""+def tp_for(n_cards: int, cards: list[dict] | None = None) -> int:+ """--tp for this many cards. Pairs of cards sharing each layer's matrices (--tp 2, the engine's default for two or more cards) were+ measured 2x faster for prompts than a plain layer split on the first rig; on four AMD Radeon AI PRO R9700 cards the plain layer split+ (--tp 1) is faster for both: 8K prompt 370 against 196 tokens a second and 32 against 29 for answers (2026-10-23, r9700), so the IST-DAS plans give four+ or more AMD cards 1 (cards passed); the Unsloth plan is unmeasured and keeps 2. Two and three cards keep 2 (two R9700: prompts 220 against 140, answers slower; not changed).+ (The engine's own check refuses 4 - 576 is not a multiple of 128 - and drops to 2.)"""+ if n_cards >= 4 and cards and all(str(c.get("vendor") or "").lower() == "amd" for c in cards):+ return 1return 2 if n_cards >= 2 else 1+def _tp_args(n_cards: int, tp: int) -> list[str]:+ """The engine's --tp when it is not the engine's own pick: it takes 2 with two or more cards when told nothing, so 1 is written out."""+ return ["--tp", str(tp)] if tp > 1 or n_cards >= 2 else []++def _plan_ista(model: str, cards: list[dict] | None, ram_gb: float | None, disk_free_gb: float | None, context: int,vendor: str | None) -> dict[str, Any]:"""plan() for an official IST-DAS set (ISTA_SETS): the same keys, the measured speeds in plain words. RAM classes: with RAM for every@@ -144,7 +154,7 @@ def _plan_ista(model: str, cards: list[dict] | None, ram_gb: float | None, disk_out: dict[str, Any] = {"variant": model, "cards": [], "experts_total": EXPERTS, "experts_in_vram": 0, "vram_experts_gb": 0.0,"cpu_experts_gb": v["experts_gb"], "weights_gb": _round1(set_gb), "experts_gb": _round1(v["experts_gb"]),"dense_gb": _round1(ISTA_DENSE_GB), "engram_gb": _round1(ISTA_ENGRAM_BYTES / GB), "download_gb": _round1(download),- "tp": tp_for(n), "args": [], "measured": ISTA_MEASURED, "lines": [], "warnings": [], "ram_gb": ram_gb,+ "tp": tp_for(n, keep), "args": [], "measured": ISTA_MEASURED, "lines": [], "warnings": [], "ram_gb": ram_gb,"context": context, "disk_total_gb": v["disk_total"]}need_disk = download + DISK_HEADROOM_GBout["disk_need_gb"] = _round1(need_disk)@@ -173,13 +183,13 @@ def _plan_ista(model: str, cards: list[dict] | None, ram_gb: float | None, disk_out["experts_in_vram"] = in_vramout["vram_experts_gb"] = _round1(in_vram * expert_gb)out["cpu_experts_gb"] = _round1(v["experts_gb"] - in_vram * expert_gb)- out["tp"] = tp_for(n)- out["args"] = ["--tp", str(out["tp"])] if out["tp"] > 1 else []+ out["tp"] = tp_for(n, keep)+ out["args"] = _tp_args(n, out["tp"])names = " + ".join(f"{c['name'] or 'card ' + str(c['index'])} ({c['vram_gb']:.0f} GB)" for c in out["cards"])lines.append(f"Graphics memory: {names}. About {in_vram:,} of the {EXPERTS:,} experts ({out['vram_experts_gb']:.0f} GB) stay on "f"the card{'s' if n > 1 else ''}; the other {EXPERTS - in_vram:,} ({out['cpu_experts_gb']:.0f} GB) are computed on the ""CPU from RAM, and the ones RAM cannot hold are read from the SSD when a word needs them.")- if n >= 2:+ if n >= 2 and out["tp"] == 2:lines.append("Two cards share each layer's matrices (--tp 2): measured about twice as fast at reading prompts.")if ram_gb:all_cached = out["cpu_experts_gb"] + REST_RAM_GBdiff --git a/tools/test_setup_dsv41.py b/tools/test_setup_dsv41.pyindex c7b7446b4..ee851408b 100644--- a/tools/test_setup_dsv41.py+++ b/tools/test_setup_dsv41.py@@ -84,6 +84,19 @@ class Fit(unittest.TestCase):self.assertAlmostEqual(dsv41_fit.DOWNLOAD_GB, 508.4, delta=0.1)self.assertEqual(dsv41_fit.EXPERTS, 15360)+ def test_four_amd_cards_run_the_ista_sets_with_a_plain_layer_split(self):+ # r9700, 2026-10-23: 8K prompt 370 tokens a second with --tp 1 against 196 with the engine's default (2)+ for model in ("IST-DAS-2.00bpw", "IST-DAS-2.25bpw"):+ p = dsv41_fit.plan(FOUR, 251.0, 900.0, model=model)+ self.assertEqual((p["tp"], p["args"]), (1, ["--tp", "1"]), model)+ # two and three cards keep pairs; other makes keep pairs; one card needs no flag; the Unsloth set is not changed+ self.assertEqual(dsv41_fit.plan(FOUR[:2], 251.0, 900.0, model="IST-DAS-2.00bpw")["args"], ["--tp", "2"])+ self.assertEqual(dsv41_fit.plan(FOUR[:3], 251.0, 900.0, model="IST-DAS-2.00bpw")["args"], ["--tp", "2"])+ nv = [{"name": "NVIDIA GeForce RTX 3090", "vram_gb": 24.0, "vendor": "nvidia", "index": i} for i in range(4)]+ self.assertEqual(dsv41_fit.plan(nv, 251.0, 900.0, model="IST-DAS-2.00bpw")["args"], ["--tp", "2"])+ self.assertEqual(dsv41_fit.plan(FOUR[:1], 251.0, 900.0, model="IST-DAS-2.00bpw")["args"], [])+ self.assertEqual(dsv41_fit.plan(FOUR, 251.0, 900.0)["args"], ["--tp", "2"])+def test_four_r9700_match_the_measured_run(self):# r9700, 2026-10-20: 1,558 + 1,559 + 1,575 + 1,505 = 6,197 experts in graphics memory, 9,163 (172.3 GB) on the CPU lanep = dsv41_fit.plan(FOUR, 251.0, 900.0)