Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
launch: the installed config must carry the chosen cards (a forced split re-runs setup), unified-memory chips count when they are the only device; setup tests follow the fit rule
Niko122131abd50a60cb
diff --git a/setup.py b/setup.pyindex 9c81fd296..736cc0e26 100644--- a/setup.py+++ b/setup.py@@ -1460,6 +1460,8 @@ def choose_gpus(a, found) -> list:for g in found:if gpu_problem(g, together=True) is not None:say(f" (GPU {g['index']}, {g['name']}: {gpu_problem(g, together=True)})")+ for g in split_short(can[:2]): # #448: the note for a second card that cannot lend a 4096-token chunk+ say(f" ({split_short_note(g)})")pick = opts[int(ask("Which GPUs?", [str(i) for i in range(1, len(opts) + 1)], str(rec), a.yes or a.check)) - 1]return [g["index"] for g in pick]diff --git a/strata_cli/launch/runner.py b/strata_cli/launch/runner.pyindex 3d249bcb2..b6fed20ec 100644--- a/strata_cli/launch/runner.py+++ b/strata_cli/launch/runner.py@@ -94,8 +94,20 @@ def read_config(path: Path) -> dict:return json.loads(Path(path).read_text(encoding="utf-8-sig"))-def desc_matches(desc: dict, s: S.LaunchSettings) -> bool:- """Does the installed model's config already carry the context, memory format and picture choice of these settings?"""+def gpus_of(desc_gpu, largest: int | None) -> list[int] | None:+ """The card numbers an installed config uses: a list, one number, or nothing written (setup then took the largest card)."""+ if isinstance(desc_gpu, list):+ return sorted(int(x) for x in desc_gpu)+ if isinstance(desc_gpu, int):+ return [desc_gpu]+ return [largest] if largest is not None else None+++def desc_matches(desc: dict, s: S.LaunchSettings, gpus: list[int] | None = None, largest: int | None = None) -> bool:+ """Does the installed model's config already carry the context, memory format, picture choice and graphics cards of these settings?+ `gpus`: the cards these settings mean (None: this computer has one card that counts, nothing to compare)."""+ if gpus is not None and gpus_of(desc.get("gpu"), largest) != sorted(gpus):+ return Falseif s.context and desc.get("context") != s.context:return Falsewant_kv = s.kv or ("int8" if (s.context or 32768) > 8192 else "fp16")@@ -370,6 +382,24 @@ class Runner:desc = self.catalog.strata.describe_config(path)return path, desc+ def wanted_gpus(self) -> tuple[list[int] | None, int | None]:+ """(the cards these settings mean, the largest card): the cards named, else the recommended ones by the shared fit rule+ (strata_control/gpu_fit.py); (None, None) when this computer has one card that counts."""+ try:+ from strata_control import gpu_fit+ comp = self.catalog.computer()+ keep = comp.usable_gpus+ if len(keep) < 2:+ return None, None+ largest = keep[0].get("index")+ if self.s.gpus:+ return sorted(int(g) for g in self.s.gpus), largest+ row = next((r for r in self.catalog.rows() if (r.family, r.model) == (self.s.family, self.s.model)), None)+ mem = float(row.memory_gb) if row is not None and row.memory_gb else 44.0+ return sorted(gpu_fit.recommended_cards(mem, comp.ram_gb, comp.gpus)), largest+ except Exception: # noqa: BLE001 - never block a launch over this comparison+ return None, None+def download_gb(self) -> float:"""How much would be downloaded for this launch right now: 0 when the model is installed with these choices (or is the person's own files)."""s = self.s@@ -381,7 +411,7 @@ class Runner:if lost and all("mmproj" in Path(x).name.lower() for x in lost):return 0.9 # only the picture reader's file is goneif path is not None and desc is not None and not lost:- if desc_matches(desc, s):+ if desc_matches(desc, s, *self.wanted_gpus()):return 0.0# the model is on disk: what differs is the context / memory format (nothing to download) or the pictures (the 0.9 GB reader file)return 0.9 if (s.vision != "off" and (desc.get("images") or "off") == "off") else 0.0@@ -397,7 +427,7 @@ class Runner:if not (s.family and s.model):raise LaunchError("No model was chosen.", "Choose a model, or use --model NAME (strata launch --help lists them).", 2)path, desc = self.installed_state()- if path is not None and desc is not None and desc_matches(desc, s) and not files_present(read_config(path)):+ if path is not None and desc is not None and desc_matches(desc, s, *self.wanted_gpus()) and not files_present(read_config(path)):self.emit("model", f"{s.family} {s.model} is already installed with these choices.", 100)return pathtitle = f"{s.family} {s.model}"diff --git a/strata_control/gpu_fit.py b/strata_control/gpu_fit.pyindex dfcca9e55..25ad2d177 100644--- a/strata_control/gpu_fit.py+++ b/strata_control/gpu_fit.py@@ -68,23 +68,34 @@ def verdict(f: dict) -> str:return "fits" if f["spare_ram_gb"] >= FITS_SPARE_RAM_GB or f["ram_use_gb"] <= 0 else "tight"+def _shares_memory(c: dict) -> bool:+ """An integrated chip or a unified-memory chip (Apple, Strix Halo): its memory is the system RAM."""+ return bool(c.get("integrated") or c.get("uma") or c.get("unified_memory") or c.get("unified_gb") or str(c.get("vendor")) == "apple")++def usable_cards(cards: list[dict]) -> tuple[list[dict], list[dict]]:- """(cards that count, cards ignored with a plain reason in 'why_ignored'). Largest memory first, then the card number."""+ """(cards that count, cards ignored with a plain reason in 'why_ignored'). Largest memory first, then the card number.++ A chip that shares the system memory is ignored when the computer has a card with memory of its own (it adds nothing to it); when it is+ the only device (a Mac, a Strix Halo PC) it is the one that counts. A card under MIN_CARD_GB cannot hold the dense weights."""+ flagged = [c for c in (cards or []) if c.get("usable") is not False]+ has_real = any(not _shares_memory(c) and float(c.get("vram_gb") or 0) >= MIN_CARD_GB for c in flagged)keep, ignored = [], []for c in cards or []:vram = float(c.get("vram_gb") or 0)why = Noneif c.get("usable") is False:why = c.get("problem") or "Strata cannot use this card"- elif c.get("integrated") or c.get("uma") or c.get("unified_memory"):- why = "it shares the system memory (an integrated graphics chip), so it adds nothing"+ elif _shares_memory(c):+ if has_real:+ why = "it shares the system memory (an integrated graphics chip), so it adds nothing"elif vram < MIN_CARD_GB:why = f"only {vram:.0f} GB of graphics memory"if why:ignored.append({**c, "why_ignored": why})else:keep.append(c)- keep.sort(key=lambda c: (-round(float(c.get("vram_gb") or 0)), c.get("index", 0)))+ keep.sort(key=lambda c: (-round(float(c.get("vram_gb") or c.get("unified_gb") or 0)), c.get("index", 0)))return keep, ignoreddiff --git a/tests/terminal/test_gpu_fit.py b/tests/terminal/test_gpu_fit.pyindex 425027a84..e788af89e 100644--- a/tests/terminal/test_gpu_fit.py+++ b/tests/terminal/test_gpu_fit.py@@ -58,6 +58,16 @@ class GpuFitRule(unittest.TestCase):self.assertTrue(all(c["why_ignored"] for c in r["ignored"]))self.assertEqual(G.recommended_cards(44.0, 62.0, cards), [0])+ def test_a_unified_memory_chip_counts_when_it_is_the_only_device(self):+ mac = {"index": 0, "name": "Apple M4", "vendor": "apple", "vram_gb": 0.0, "unified_gb": 32.0, "usable": True}+ keep, ignored = G.usable_cards([mac])+ self.assertEqual((len(keep), ignored), (1, []))+ igpu = {"index": 0, "name": "AMD Radeon 8060S", "vram_gb": 64.0, "uma": True, "usable": True}+ self.assertEqual(len(G.usable_cards([igpu])[0]), 1) # a Strix Halo PC: the integrated chip IS the device+ keep, ignored = G.usable_cards([card(0, "RTX 3070 Ti", 8), igpu]) # beside a real card it adds nothing+ self.assertEqual([c["index"] for c in keep], [0] if keep[0]["name"] == "RTX 3070 Ti" else [])+ self.assertEqual(len(ignored), 1)+def test_an_unusable_card_is_ignored_with_its_own_reason(self):r = G.options(44.0, 62.0, [card(0, "RTX 3070 Ti", 8), card(1, "GTX 1060", 6, usable=False, problem="older than the RTX 20 series")])self.assertEqual([o["key"] for o in r["options"]], ["single"])diff --git a/tests/terminal/test_launch.py b/tests/terminal/test_launch.pyindex 9c41faba4..9ada36d86 100644--- a/tests/terminal/test_launch.py+++ b/tests/terminal/test_launch.py@@ -455,6 +455,24 @@ class AdoptRules(unittest.TestCase):with mock.patch.object(r, "installed_state", return_value=(cfgfile, {"context": 32768, "kv": "int8", "images": "cpu"})):self.assertAlmostEqual(r.download_gb(), 0.9)+ def test_the_installed_config_must_carry_the_chosen_cards(self):+ from tests.terminal.launch_support import HW_2X16+ with Home():+ cat = catalog(HW_2X16)+ base = {"context": 32768, "kv": "int8", "images": "off"}+ auto = Runner(S.LaunchSettings(family="qwen", model="IQ3_XXS", context=32768), catalog=cat)+ self.assertEqual(auto.wanted_gpus(), ([0], 0)) # the recommended: the largest card + RAM+ both = Runner(S.LaunchSettings(family="qwen", model="IQ3_XXS", context=32768, gpus=[0, 1]), catalog=cat)+ self.assertEqual(both.wanted_gpus(), ([0, 1], 0))+ # a config written for one card (no "gpu" key: setup took the largest) is right for the default and wrong for the split+ self.assertTrue(desc_matches({**base, "gpu": None}, auto.s, *auto.wanted_gpus()))+ self.assertFalse(desc_matches({**base, "gpu": None}, both.s, *both.wanted_gpus()))+ self.assertTrue(desc_matches({**base, "gpu": [1, 0]}, both.s, *both.wanted_gpus()))+ self.assertFalse(desc_matches({**base, "gpu": [0, 1]}, auto.s, *auto.wanted_gpus()))+ with Home():+ one = Runner(S.LaunchSettings(family="qwen", model="IQ3_XXS", context=32768), catalog=catalog(HW_12))+ self.assertEqual(one.wanted_gpus(), (None, None)) # one card: nothing to compare+def test_pictures_asked_but_still_off_is_said_in_plain_words(self):with Home() as h:cat = catalog(HW_12)diff --git a/tools/test_setup_risk.py b/tools/test_setup_risk.pyindex f6689c49a..be0cf34c9 100644--- a/tools/test_setup_risk.py+++ b/tools/test_setup_risk.py@@ -238,7 +238,7 @@ class LowRamGpus(unittest.TestCase):def test_asked_and_answered_two(self):ram, found = PROFILES["32GB-2x24GB"]code, out, cfg, asked = install(ram, found, ["--family", "qwen", "--model", "Q2_0", "--no-start"],- answers={"Low-RAM mode: which GPUs?": "2"})+ answers={"Which GPUs?": "2", "Low-RAM mode: which GPUs?": "2"}) # the split asked for (the default is the largest card + RAM)self.assertEqual(code, 0, out)self.assertTrue(any("Low-RAM mode: which GPUs? [1]" in q for q in asked), asked)self.assertEqual(cfg["gpu"], [0, 1])@@ -276,7 +276,7 @@ class LowRamGpus(unittest.TestCase):ram, found = PROFILES["32GB-2x24GB"]with mock.patch.object(setup, "MIN_ENGINE", setup.RESIDENT_SPLIT_ENGINE):code, out, cfg, asked = install(ram, found, ["--family", "qwen", "--model", "Q2_0", "--no-start"],- answers="")+ answers={"Which GPUs?": "2"}) # the split chosen (the default is the largest card + RAM)self.assertEqual(code, 0, out)self.assertEqual(cfg["gpu"], [0, 1])self.assertIn("--resident-experts", cfg["args"])diff --git a/tools/test_setup_unsloth.py b/tools/test_setup_unsloth.pyindex 775794851..0a30687d4 100644--- a/tools/test_setup_unsloth.py+++ b/tools/test_setup_unsloth.py@@ -372,7 +372,7 @@ class Main(Base):self.assertAlmostEqual(need, setup.MODELS[M]["download_gb"] + setup.UNSLOTH_RAM_LEFT_GB)code, out, cfg = self.main(["--context", "8192"], n_gpus=2, ram=127.8) # no --gpus: stays on one GPUself.assertEqual(code, 0, out)- self.assertIn("uses them together anyway", out)+ self.assertNotIn("uses them together anyway", out) # one card + RAM is the default now: nothing to warn aboutself.assertNotIn("layer_split", cfg)self.assertEqual(cfg["args"][cfg["args"].index("--resident-budget-gib") + 1], "71") # all of themcode, out, cfg = self.main(["--context", "8192", "--gpus", "0,1"], n_gpus=2, ram=127.8) # asked for by name@@ -428,8 +428,8 @@ class Main(Base):def test_yes_alone_keeps_one_gpu(self):code, out, cfg = self.main(["--context", "8192"], n_gpus=2, ram=165.0)self.assertEqual(code, 0, out)- self.assertIn("runs on one GPU: using", out)- self.assertIn("--gpus 0,1 shares it", out)+ self.assertIn("layer split", out) # the second card is offered as a split+ self.assertIn("(recommended)", out)self.assertNotIn("layer_split", cfg)self.assertIn("--resident-budget-gib", cfg["args"])@@ -629,10 +629,10 @@ class LayerSplit(unittest.TestCase):argv = ["--family", "unsloth", "--model", M, "--context", "32768", "--no-start"]code, out, cfg, asked = install(165.0, self.found, argv, answers={"which GPUs?": "1", "Which GPUs?": "1"})self.assertEqual(code, 0, out)- self.assertTrue(any(f"{M}: which GPUs?" in q and "[1]" in q for q in asked), asked)+ self.assertTrue(any("Which GPUs? [1]" in q for q in asked), asked) # the fit rule's question: the largest card + RAM is the defaultself.assertEqual(cfg["gpu"], 0)self.assertIn("--resident-budget-gib", cfg["args"])- code, out, cfg, asked = install(165.0, self.found, argv, answers={f"{M}: which GPUs?": "2"})+ code, out, cfg, asked = install(165.0, self.found, argv, answers={"Which GPUs?": "2", f"{M}: which GPUs?": "2"})self.assertEqual(code, 0, out)self.assertEqual(cfg["gpu"], [0, 1])self.assertEqual(cfg["layer_split"], "auto")@@ -791,7 +791,7 @@ class Resident(Base):def test_two_gpus_split_with_every_expert_in_ram(self):"""Where the experts fit, two cards are the GGUF models' layer split (no budget question, no #498 note)."""- code, out, cfg = self.main(["--context", "8192"], n_gpus=2, ram=127.8)+ code, out, cfg = self.main(["--context", "8192"], n_gpus=2, ram=127.8, answers={"Which GPUs?": "2"}) # the split chosenself.assertEqual(code, 0, out)self.assertEqual(cfg["gpu"], [0, 1])self.assertEqual(cfg["layer_split"], "auto")