Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

setup: the multi-GPU default follows the fit rule (largest card + RAM first, the split recommended only when the model does not fit one card), goldens re-recorded, setup tests

Niko12211d8d830afa65
diff --git a/setup.py b/setup.py
index 60663fc3e..9c81fd296 100644
--- a/setup.py
+++ b/setup.py
@@ -1436,23 +1436,30 @@ def choose_gpus(a, found) -> list:
can = together_ok(found)
if not can:
return [single[0]["index"]]
+ # the owner's rule (strata_control/gpu_fit.py, shared with the app and `strata launch`): the model goes on the LARGEST card + RAM first;
+ # the second card is offered as a layer split and recommended only when the model does not fit one card (with RAM) or would be too tight
+ from strata_control import gpu_fit
+ mem = None
+ d = MODELS.get(getattr(a, "model", None) or "")
+ if d:
+ mem = float(d["ram_gb"]) if d.get("mem_total") else float(d.get("arena_gb") or d["ram_gb"]) + 1.0
+ cards = [{"index": g["index"], "name": g["name"], "vram_gb": g["vram_gb"], "usable": gpu_problem(g) is None} for g in found]
+ res = gpu_fit.options(mem if mem is not None else 0.0, ram_gb(), cards)
+ by_key = {o["key"]: o for o in res["options"]}
say()
- say(f" Strata can run the model on one GPU, or share it across {'these' if len(can) > 2 else 'both'}: then each"
- " card holds the")
- say(" experts of its own layers, so together they hold about twice as many, and prompts are read about 20%")
- say(" faster (details: docs/MULTI_GPU.md). A much slower extra card can also make it slower.")
- opts = [can[:2]] + ([can] if len(can) > 2 else []) + [[g] for g in single]
- # #448: a pair whose second card cannot lend a 4096-token chunk recommends the first card alone (still offered)
- short = split_short(can[:2])
- rec = next(i for i, o in enumerate(opts, 1) if o == [can[0]]) if short else 1
- for i, o in enumerate(opts, 1):
- label = (" + ".join(gpu_name(g) for g in o) + " together") if len(o) > 1 else gpu_name(o[0]) + " only"
- say(f" {i}) {label}" + (" (recommended)" if i == rec else ""))
+ say(" Strata first tries the model on your largest GPU plus the PC's memory; a second GPU is offered as a layer split")
+ say(" (each card holds the experts of its own layers: more room, prompts read about 20% faster; details: docs/MULTI_GPU.md).")
+ if mem is None:
+ say(" (the model is chosen below; the numbers here are for the recommended size when you name one: --model)")
+ opts = [[g for g in found if g["index"] in o["cards"]] for o in res["options"]]
+ rec = next((i for i, o in enumerate(res["options"], 1) if o["recommended"]), 1) if mem is not None else 1
+ for i, o in enumerate(res["options"], 1):
+ say(f" {i}) {o['summary'] if mem is not None else o['label']}" + (" (recommended)" if i == rec else ""))
+ for n in o["notes"]:
+ say(f" {n}")
for g in found:
if gpu_problem(g, together=True) is not None:
say(f" (GPU {g['index']}, {g['name']}: {gpu_problem(g, together=True)})")
- for g in short:
- say(f" ({split_short_note(g)})")
pick = opts[int(ask("Which GPUs?", [str(i) for i in range(1, len(opts) + 1)], str(rec), a.yes or a.check)) - 1]
return [g["index"] for g in pick]
diff --git a/tools/test_setup_golden.json b/tools/test_setup_golden.json
index ad762c162..e9c250b5b 100644
--- a/tools/test_setup_golden.json
+++ b/tools/test_setup_golden.json
@@ -341,12 +341,8 @@
],
"cwd": "<T>",
"exe": "<T>/engine/<EXE>",
- "gpu": [
- 0,
- 1
- ],
+ "gpu": 0,
"gpus_asked": true,
- "layer_split": "auto",
"lib_dirs": [
"<cuda>"
],
@@ -386,12 +382,8 @@
],
"cwd": "<T>",
"exe": "<T>/engine/<EXE>",
- "gpu": [
- 0,
- 1
- ],
+ "gpu": 0,
"gpus_asked": true,
- "layer_split": "auto",
"lib_dirs": [
"<cuda>"
],
@@ -529,12 +521,8 @@
],
"cwd": "<T>",
"exe": "<T>/engine/<EXE>",
- "gpu": [
- 0,
- 1
- ],
+ "gpu": 0,
"gpus_asked": true,
- "layer_split": "auto",
"lib_dirs": [
"<cuda>"
],
@@ -573,12 +561,8 @@
],
"cwd": "<T>",
"exe": "<T>/engine/<EXE>",
- "gpu": [
- 0,
- 1
- ],
+ "gpu": 0,
"gpus_asked": true,
- "layer_split": "auto",
"lib_dirs": [
"<cuda>"
],
@@ -617,12 +601,8 @@
],
"cwd": "<T>",
"exe": "<T>/engine/<EXE>",
- "gpu": [
- 0,
- 1
- ],
+ "gpu": 0,
"gpus_asked": true,
- "layer_split": "auto",
"lib_dirs": [
"<cuda>"
],
diff --git a/tools/test_setup_gpu_fit.py b/tools/test_setup_gpu_fit.py
new file mode 100644
index 000000000..869678fd2
--- /dev/null
+++ b/tools/test_setup_gpu_fit.py
@@ -0,0 +1,64 @@
+"""The owner's multi-GPU rule in setup (strata_control/gpu_fit.py): the model goes on the LARGEST card + RAM first; the second card is
+offered as a layer split and is the default only when the model does not fit one card (with RAM) or would be too tight.
+
+ python tools/test_setup_gpu_fit.py
+"""
+from __future__ import annotations
+
+import sys
+import unittest
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+sys.path.insert(0, str(ROOT / "tools"))
+import setup # noqa: E402
+from test_setup_golden import card, install # noqa: E402
+
+
+def run(ram, found, model, answers=None, extra=()):
+ argv = ["--family", "qwen", "--no-start", "--model", model, "--context", "32768", *extra]
+ return install(ram, found, argv, answers=answers or {})
+
+
+class SetupFitRule(unittest.TestCase):
+ def test_one_card_has_no_question(self):
+ code, out, cfg, asked = run(62.0, [card(0, "NVIDIA GeForce RTX 3070 Ti", 8.0, "86")], "IQ3_XXS")
+ self.assertEqual(code, 0, out)
+ self.assertEqual(cfg["gpu"], 0) if "gpu" in cfg else None
+ self.assertFalse(any("Which GPUs" in q for q in asked), asked)
+
+ def test_two_unequal_cards_and_the_model_fits_one_defaults_to_the_largest_card_and_offers_the_split(self):
+ found = [card(0, "NVIDIA GeForce RTX 3060 Ti", 8.0, "86"), card(1, "NVIDIA GeForce RTX 3070 Ti", 16.0, "86")]
+ code, out, cfg, asked = run(62.0, found, "IQ3_XXS")
+ self.assertEqual(code, 0, out)
+ self.assertEqual(cfg["gpu"], 1) # the 16 GB card, alone
+ self.assertNotIn("layer_split", cfg)
+ self.assertIn("layer split", out) # the split was offered
+ self.assertIn("1) RTX 3070 Ti + RAM", out.replace("NVIDIA GeForce ", ""))
+ self.assertTrue(any("Which GPUs" in q and "[1]" in q for q in asked), asked)
+ # choosing the split is one answer away
+ code, out, cfg, _ = run(62.0, found, "IQ3_XXS", answers={"Which GPUs?": "2"})
+ self.assertEqual(code, 0, out)
+ self.assertEqual(sorted(cfg["gpu"]), [0, 1])
+ self.assertEqual(cfg["layer_split"], "auto")
+
+ def test_two_small_cards_and_the_model_does_not_fit_one_defaults_to_the_split(self):
+ found = [card(0, "NVIDIA GeForce RTX 3060", 12.0, "86"), card(1, "NVIDIA GeForce RTX 3060", 12.0, "86")]
+ code, out, cfg, asked = run(26.0, found, "IQ3_XXS")
+ self.assertEqual(code, 0, out)
+ self.assertEqual(sorted(cfg["gpu"]), [0, 1])
+ self.assertEqual(cfg["layer_split"], "auto")
+ self.assertTrue(any("Which GPUs" in q and "[2]" in q for q in asked), asked)
+
+ def test_naming_the_cards_still_wins(self):
+ found = [card(0, "NVIDIA GeForce RTX 3070 Ti", 16.0, "86"), card(1, "NVIDIA GeForce RTX 3060 Ti", 8.0, "86")]
+ code, out, cfg, _ = run(62.0, found, "IQ3_XXS", extra=("--gpus", "0,1"))
+ self.assertEqual(code, 0, out)
+ self.assertEqual(sorted(cfg["gpu"]), [0, 1])
+ code, out, cfg, _ = run(62.0, found, "IQ3_XXS", extra=("--gpu", "1"))
+ self.assertEqual(cfg["gpu"], 1)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tools/test_setup_remote_opt.py b/tools/test_setup_remote_opt.py
index 8d797a2ae..5fb8ad6be 100644
--- a/tools/test_setup_remote_opt.py
+++ b/tools/test_setup_remote_opt.py
@@ -81,7 +81,7 @@ class Install(unittest.TestCase):
def test_install_on_two_gpus(self):
"""#1447: a plain two-GPU install has no helper cache, so no --remote-expert-opt is written."""
ram, found = PROFILES["47GB-2x16GB"]
- args = ["--family", "qwen", "--no-start", "--model", "Q2_0"]
+ args = ["--family", "qwen", "--no-start", "--model", "Q2_0", "--gpus", "0,1"] # named: since the fit rule the default is the largest card + RAM
code, text, cfg, _ = install(ram, found, args)
self.assertEqual(code, 0, text)
self.assertIsInstance(cfg.get("gpu"), list)