Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
setup: the multi-GPU default follows the fit rule (largest card + RAM first, the split recommended only when the model does not fit one card), goldens re-recorded, setup tests
Niko12211d8d830afa65
diff --git a/setup.py b/setup.pyindex 60663fc3e..9c81fd296 100644--- a/setup.py+++ b/setup.py@@ -1436,23 +1436,30 @@ def choose_gpus(a, found) -> list:can = together_ok(found)if not can:return [single[0]["index"]]+ # the owner's rule (strata_control/gpu_fit.py, shared with the app and `strata launch`): the model goes on the LARGEST card + RAM first;+ # the second card is offered as a layer split and recommended only when the model does not fit one card (with RAM) or would be too tight+ from strata_control import gpu_fit+ mem = None+ d = MODELS.get(getattr(a, "model", None) or "")+ if d:+ mem = float(d["ram_gb"]) if d.get("mem_total") else float(d.get("arena_gb") or d["ram_gb"]) + 1.0+ cards = [{"index": g["index"], "name": g["name"], "vram_gb": g["vram_gb"], "usable": gpu_problem(g) is None} for g in found]+ res = gpu_fit.options(mem if mem is not None else 0.0, ram_gb(), cards)+ by_key = {o["key"]: o for o in res["options"]}say()- say(f" Strata can run the model on one GPU, or share it across {'these' if len(can) > 2 else 'both'}: then each"- " card holds the")- say(" experts of its own layers, so together they hold about twice as many, and prompts are read about 20%")- say(" faster (details: docs/MULTI_GPU.md). A much slower extra card can also make it slower.")- opts = [can[:2]] + ([can] if len(can) > 2 else []) + [[g] for g in single]- # #448: a pair whose second card cannot lend a 4096-token chunk recommends the first card alone (still offered)- short = split_short(can[:2])- rec = next(i for i, o in enumerate(opts, 1) if o == [can[0]]) if short else 1- for i, o in enumerate(opts, 1):- label = (" + ".join(gpu_name(g) for g in o) + " together") if len(o) > 1 else gpu_name(o[0]) + " only"- say(f" {i}) {label}" + (" (recommended)" if i == rec else ""))+ say(" Strata first tries the model on your largest GPU plus the PC's memory; a second GPU is offered as a layer split")+ say(" (each card holds the experts of its own layers: more room, prompts read about 20% faster; details: docs/MULTI_GPU.md).")+ if mem is None:+ say(" (the model is chosen below; the numbers here are for the recommended size when you name one: --model)")+ opts = [[g for g in found if g["index"] in o["cards"]] for o in res["options"]]+ rec = next((i for i, o in enumerate(res["options"], 1) if o["recommended"]), 1) if mem is not None else 1+ for i, o in enumerate(res["options"], 1):+ say(f" {i}) {o['summary'] if mem is not None else o['label']}" + (" (recommended)" if i == rec else ""))+ for n in o["notes"]:+ say(f" {n}")for g in found:if gpu_problem(g, together=True) is not None:say(f" (GPU {g['index']}, {g['name']}: {gpu_problem(g, together=True)})")- for g in short:- say(f" ({split_short_note(g)})")pick = opts[int(ask("Which GPUs?", [str(i) for i in range(1, len(opts) + 1)], str(rec), a.yes or a.check)) - 1]return [g["index"] for g in pick]diff --git a/tools/test_setup_golden.json b/tools/test_setup_golden.jsonindex ad762c162..e9c250b5b 100644--- a/tools/test_setup_golden.json+++ b/tools/test_setup_golden.json@@ -341,12 +341,8 @@],"cwd": "<T>","exe": "<T>/engine/<EXE>",- "gpu": [- 0,- 1- ],+ "gpu": 0,"gpus_asked": true,- "layer_split": "auto","lib_dirs": ["<cuda>"],@@ -386,12 +382,8 @@],"cwd": "<T>","exe": "<T>/engine/<EXE>",- "gpu": [- 0,- 1- ],+ "gpu": 0,"gpus_asked": true,- "layer_split": "auto","lib_dirs": ["<cuda>"],@@ -529,12 +521,8 @@],"cwd": "<T>","exe": "<T>/engine/<EXE>",- "gpu": [- 0,- 1- ],+ "gpu": 0,"gpus_asked": true,- "layer_split": "auto","lib_dirs": ["<cuda>"],@@ -573,12 +561,8 @@],"cwd": "<T>","exe": "<T>/engine/<EXE>",- "gpu": [- 0,- 1- ],+ "gpu": 0,"gpus_asked": true,- "layer_split": "auto","lib_dirs": ["<cuda>"],@@ -617,12 +601,8 @@],"cwd": "<T>","exe": "<T>/engine/<EXE>",- "gpu": [- 0,- 1- ],+ "gpu": 0,"gpus_asked": true,- "layer_split": "auto","lib_dirs": ["<cuda>"],diff --git a/tools/test_setup_gpu_fit.py b/tools/test_setup_gpu_fit.pynew file mode 100644index 000000000..869678fd2--- /dev/null+++ b/tools/test_setup_gpu_fit.py@@ -0,0 +1,64 @@+"""The owner's multi-GPU rule in setup (strata_control/gpu_fit.py): the model goes on the LARGEST card + RAM first; the second card is+offered as a layer split and is the default only when the model does not fit one card (with RAM) or would be too tight.++ python tools/test_setup_gpu_fit.py+"""+from __future__ import annotations++import sys+import unittest+from pathlib import Path++ROOT = Path(__file__).resolve().parents[1]+sys.path.insert(0, str(ROOT))+sys.path.insert(0, str(ROOT / "tools"))+import setup # noqa: E402+from test_setup_golden import card, install # noqa: E402+++def run(ram, found, model, answers=None, extra=()):+ argv = ["--family", "qwen", "--no-start", "--model", model, "--context", "32768", *extra]+ return install(ram, found, argv, answers=answers or {})+++class SetupFitRule(unittest.TestCase):+ def test_one_card_has_no_question(self):+ code, out, cfg, asked = run(62.0, [card(0, "NVIDIA GeForce RTX 3070 Ti", 8.0, "86")], "IQ3_XXS")+ self.assertEqual(code, 0, out)+ self.assertEqual(cfg["gpu"], 0) if "gpu" in cfg else None+ self.assertFalse(any("Which GPUs" in q for q in asked), asked)++ def test_two_unequal_cards_and_the_model_fits_one_defaults_to_the_largest_card_and_offers_the_split(self):+ found = [card(0, "NVIDIA GeForce RTX 3060 Ti", 8.0, "86"), card(1, "NVIDIA GeForce RTX 3070 Ti", 16.0, "86")]+ code, out, cfg, asked = run(62.0, found, "IQ3_XXS")+ self.assertEqual(code, 0, out)+ self.assertEqual(cfg["gpu"], 1) # the 16 GB card, alone+ self.assertNotIn("layer_split", cfg)+ self.assertIn("layer split", out) # the split was offered+ self.assertIn("1) RTX 3070 Ti + RAM", out.replace("NVIDIA GeForce ", ""))+ self.assertTrue(any("Which GPUs" in q and "[1]" in q for q in asked), asked)+ # choosing the split is one answer away+ code, out, cfg, _ = run(62.0, found, "IQ3_XXS", answers={"Which GPUs?": "2"})+ self.assertEqual(code, 0, out)+ self.assertEqual(sorted(cfg["gpu"]), [0, 1])+ self.assertEqual(cfg["layer_split"], "auto")++ def test_two_small_cards_and_the_model_does_not_fit_one_defaults_to_the_split(self):+ found = [card(0, "NVIDIA GeForce RTX 3060", 12.0, "86"), card(1, "NVIDIA GeForce RTX 3060", 12.0, "86")]+ code, out, cfg, asked = run(26.0, found, "IQ3_XXS")+ self.assertEqual(code, 0, out)+ self.assertEqual(sorted(cfg["gpu"]), [0, 1])+ self.assertEqual(cfg["layer_split"], "auto")+ self.assertTrue(any("Which GPUs" in q and "[2]" in q for q in asked), asked)++ def test_naming_the_cards_still_wins(self):+ found = [card(0, "NVIDIA GeForce RTX 3070 Ti", 16.0, "86"), card(1, "NVIDIA GeForce RTX 3060 Ti", 8.0, "86")]+ code, out, cfg, _ = run(62.0, found, "IQ3_XXS", extra=("--gpus", "0,1"))+ self.assertEqual(code, 0, out)+ self.assertEqual(sorted(cfg["gpu"]), [0, 1])+ code, out, cfg, _ = run(62.0, found, "IQ3_XXS", extra=("--gpu", "1"))+ self.assertEqual(cfg["gpu"], 1)+++if __name__ == "__main__":+ unittest.main()diff --git a/tools/test_setup_remote_opt.py b/tools/test_setup_remote_opt.pyindex 8d797a2ae..5fb8ad6be 100644--- a/tools/test_setup_remote_opt.py+++ b/tools/test_setup_remote_opt.py@@ -81,7 +81,7 @@ class Install(unittest.TestCase):def test_install_on_two_gpus(self):"""#1447: a plain two-GPU install has no helper cache, so no --remote-expert-opt is written."""ram, found = PROFILES["47GB-2x16GB"]- args = ["--family", "qwen", "--no-start", "--model", "Q2_0"]+ args = ["--family", "qwen", "--no-start", "--model", "Q2_0", "--gpus", "0,1"] # named: since the fit rule the default is the largest card + RAMcode, text, cfg, _ = install(ram, found, args)self.assertEqual(code, 0, text)self.assertIsInstance(cfg.get("gpu"), list)