Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

launch: the installed config must carry the chosen cards (a forced split re-runs setup), unified-memory chips count when they are the only device; setup tests follow the fit rule

Niko122131abd50a60cb
diff --git a/setup.py b/setup.py
index 9c81fd296..736cc0e26 100644
--- a/setup.py
+++ b/setup.py
@@ -1460,6 +1460,8 @@ def choose_gpus(a, found) -> list:
for g in found:
if gpu_problem(g, together=True) is not None:
say(f" (GPU {g['index']}, {g['name']}: {gpu_problem(g, together=True)})")
+ for g in split_short(can[:2]): # #448: the note for a second card that cannot lend a 4096-token chunk
+ say(f" ({split_short_note(g)})")
pick = opts[int(ask("Which GPUs?", [str(i) for i in range(1, len(opts) + 1)], str(rec), a.yes or a.check)) - 1]
return [g["index"] for g in pick]
diff --git a/strata_cli/launch/runner.py b/strata_cli/launch/runner.py
index 3d249bcb2..b6fed20ec 100644
--- a/strata_cli/launch/runner.py
+++ b/strata_cli/launch/runner.py
@@ -94,8 +94,20 @@ def read_config(path: Path) -> dict:
return json.loads(Path(path).read_text(encoding="utf-8-sig"))
-def desc_matches(desc: dict, s: S.LaunchSettings) -> bool:
- """Does the installed model's config already carry the context, memory format and picture choice of these settings?"""
+def gpus_of(desc_gpu, largest: int | None) -> list[int] | None:
+ """The card numbers an installed config uses: a list, one number, or nothing written (setup then took the largest card)."""
+ if isinstance(desc_gpu, list):
+ return sorted(int(x) for x in desc_gpu)
+ if isinstance(desc_gpu, int):
+ return [desc_gpu]
+ return [largest] if largest is not None else None
+
+
+def desc_matches(desc: dict, s: S.LaunchSettings, gpus: list[int] | None = None, largest: int | None = None) -> bool:
+ """Does the installed model's config already carry the context, memory format, picture choice and graphics cards of these settings?
+ `gpus`: the cards these settings mean (None: this computer has one card that counts, nothing to compare)."""
+ if gpus is not None and gpus_of(desc.get("gpu"), largest) != sorted(gpus):
+ return False
if s.context and desc.get("context") != s.context:
return False
want_kv = s.kv or ("int8" if (s.context or 32768) > 8192 else "fp16")
@@ -370,6 +382,24 @@ class Runner:
desc = self.catalog.strata.describe_config(path)
return path, desc
+ def wanted_gpus(self) -> tuple[list[int] | None, int | None]:
+ """(the cards these settings mean, the largest card): the cards named, else the recommended ones by the shared fit rule
+ (strata_control/gpu_fit.py); (None, None) when this computer has one card that counts."""
+ try:
+ from strata_control import gpu_fit
+ comp = self.catalog.computer()
+ keep = comp.usable_gpus
+ if len(keep) < 2:
+ return None, None
+ largest = keep[0].get("index")
+ if self.s.gpus:
+ return sorted(int(g) for g in self.s.gpus), largest
+ row = next((r for r in self.catalog.rows() if (r.family, r.model) == (self.s.family, self.s.model)), None)
+ mem = float(row.memory_gb) if row is not None and row.memory_gb else 44.0
+ return sorted(gpu_fit.recommended_cards(mem, comp.ram_gb, comp.gpus)), largest
+ except Exception: # noqa: BLE001 - never block a launch over this comparison
+ return None, None
+
def download_gb(self) -> float:
"""How much would be downloaded for this launch right now: 0 when the model is installed with these choices (or is the person's own files)."""
s = self.s
@@ -381,7 +411,7 @@ class Runner:
if lost and all("mmproj" in Path(x).name.lower() for x in lost):
return 0.9 # only the picture reader's file is gone
if path is not None and desc is not None and not lost:
- if desc_matches(desc, s):
+ if desc_matches(desc, s, *self.wanted_gpus()):
return 0.0
# the model is on disk: what differs is the context / memory format (nothing to download) or the pictures (the 0.9 GB reader file)
return 0.9 if (s.vision != "off" and (desc.get("images") or "off") == "off") else 0.0
@@ -397,7 +427,7 @@ class Runner:
if not (s.family and s.model):
raise LaunchError("No model was chosen.", "Choose a model, or use --model NAME (strata launch --help lists them).", 2)
path, desc = self.installed_state()
- if path is not None and desc is not None and desc_matches(desc, s) and not files_present(read_config(path)):
+ if path is not None and desc is not None and desc_matches(desc, s, *self.wanted_gpus()) and not files_present(read_config(path)):
self.emit("model", f"{s.family} {s.model} is already installed with these choices.", 100)
return path
title = f"{s.family} {s.model}"
diff --git a/strata_control/gpu_fit.py b/strata_control/gpu_fit.py
index dfcca9e55..25ad2d177 100644
--- a/strata_control/gpu_fit.py
+++ b/strata_control/gpu_fit.py
@@ -68,23 +68,34 @@ def verdict(f: dict) -> str:
return "fits" if f["spare_ram_gb"] >= FITS_SPARE_RAM_GB or f["ram_use_gb"] <= 0 else "tight"
+def _shares_memory(c: dict) -> bool:
+ """An integrated chip or a unified-memory chip (Apple, Strix Halo): its memory is the system RAM."""
+ return bool(c.get("integrated") or c.get("uma") or c.get("unified_memory") or c.get("unified_gb") or str(c.get("vendor")) == "apple")
+
+
def usable_cards(cards: list[dict]) -> tuple[list[dict], list[dict]]:
- """(cards that count, cards ignored with a plain reason in 'why_ignored'). Largest memory first, then the card number."""
+ """(cards that count, cards ignored with a plain reason in 'why_ignored'). Largest memory first, then the card number.
+
+ A chip that shares the system memory is ignored when the computer has a card with memory of its own (it adds nothing to it); when it is
+ the only device (a Mac, a Strix Halo PC) it is the one that counts. A card under MIN_CARD_GB cannot hold the dense weights."""
+ flagged = [c for c in (cards or []) if c.get("usable") is not False]
+ has_real = any(not _shares_memory(c) and float(c.get("vram_gb") or 0) >= MIN_CARD_GB for c in flagged)
keep, ignored = [], []
for c in cards or []:
vram = float(c.get("vram_gb") or 0)
why = None
if c.get("usable") is False:
why = c.get("problem") or "Strata cannot use this card"
- elif c.get("integrated") or c.get("uma") or c.get("unified_memory"):
- why = "it shares the system memory (an integrated graphics chip), so it adds nothing"
+ elif _shares_memory(c):
+ if has_real:
+ why = "it shares the system memory (an integrated graphics chip), so it adds nothing"
elif vram < MIN_CARD_GB:
why = f"only {vram:.0f} GB of graphics memory"
if why:
ignored.append({**c, "why_ignored": why})
else:
keep.append(c)
- keep.sort(key=lambda c: (-round(float(c.get("vram_gb") or 0)), c.get("index", 0)))
+ keep.sort(key=lambda c: (-round(float(c.get("vram_gb") or c.get("unified_gb") or 0)), c.get("index", 0)))
return keep, ignored
diff --git a/tests/terminal/test_gpu_fit.py b/tests/terminal/test_gpu_fit.py
index 425027a84..e788af89e 100644
--- a/tests/terminal/test_gpu_fit.py
+++ b/tests/terminal/test_gpu_fit.py
@@ -58,6 +58,16 @@ class GpuFitRule(unittest.TestCase):
self.assertTrue(all(c["why_ignored"] for c in r["ignored"]))
self.assertEqual(G.recommended_cards(44.0, 62.0, cards), [0])
+ def test_a_unified_memory_chip_counts_when_it_is_the_only_device(self):
+ mac = {"index": 0, "name": "Apple M4", "vendor": "apple", "vram_gb": 0.0, "unified_gb": 32.0, "usable": True}
+ keep, ignored = G.usable_cards([mac])
+ self.assertEqual((len(keep), ignored), (1, []))
+ igpu = {"index": 0, "name": "AMD Radeon 8060S", "vram_gb": 64.0, "uma": True, "usable": True}
+ self.assertEqual(len(G.usable_cards([igpu])[0]), 1) # a Strix Halo PC: the integrated chip IS the device
+ keep, ignored = G.usable_cards([card(0, "RTX 3070 Ti", 8), igpu]) # beside a real card it adds nothing
+ self.assertEqual([c["index"] for c in keep], [0] if keep[0]["name"] == "RTX 3070 Ti" else [])
+ self.assertEqual(len(ignored), 1)
+
def test_an_unusable_card_is_ignored_with_its_own_reason(self):
r = G.options(44.0, 62.0, [card(0, "RTX 3070 Ti", 8), card(1, "GTX 1060", 6, usable=False, problem="older than the RTX 20 series")])
self.assertEqual([o["key"] for o in r["options"]], ["single"])
diff --git a/tests/terminal/test_launch.py b/tests/terminal/test_launch.py
index 9c41faba4..9ada36d86 100644
--- a/tests/terminal/test_launch.py
+++ b/tests/terminal/test_launch.py
@@ -455,6 +455,24 @@ class AdoptRules(unittest.TestCase):
with mock.patch.object(r, "installed_state", return_value=(cfgfile, {"context": 32768, "kv": "int8", "images": "cpu"})):
self.assertAlmostEqual(r.download_gb(), 0.9)
+ def test_the_installed_config_must_carry_the_chosen_cards(self):
+ from tests.terminal.launch_support import HW_2X16
+ with Home():
+ cat = catalog(HW_2X16)
+ base = {"context": 32768, "kv": "int8", "images": "off"}
+ auto = Runner(S.LaunchSettings(family="qwen", model="IQ3_XXS", context=32768), catalog=cat)
+ self.assertEqual(auto.wanted_gpus(), ([0], 0)) # the recommended: the largest card + RAM
+ both = Runner(S.LaunchSettings(family="qwen", model="IQ3_XXS", context=32768, gpus=[0, 1]), catalog=cat)
+ self.assertEqual(both.wanted_gpus(), ([0, 1], 0))
+ # a config written for one card (no "gpu" key: setup took the largest) is right for the default and wrong for the split
+ self.assertTrue(desc_matches({**base, "gpu": None}, auto.s, *auto.wanted_gpus()))
+ self.assertFalse(desc_matches({**base, "gpu": None}, both.s, *both.wanted_gpus()))
+ self.assertTrue(desc_matches({**base, "gpu": [1, 0]}, both.s, *both.wanted_gpus()))
+ self.assertFalse(desc_matches({**base, "gpu": [0, 1]}, auto.s, *auto.wanted_gpus()))
+ with Home():
+ one = Runner(S.LaunchSettings(family="qwen", model="IQ3_XXS", context=32768), catalog=catalog(HW_12))
+ self.assertEqual(one.wanted_gpus(), (None, None)) # one card: nothing to compare
+
def test_pictures_asked_but_still_off_is_said_in_plain_words(self):
with Home() as h:
cat = catalog(HW_12)
diff --git a/tools/test_setup_risk.py b/tools/test_setup_risk.py
index f6689c49a..be0cf34c9 100644
--- a/tools/test_setup_risk.py
+++ b/tools/test_setup_risk.py
@@ -238,7 +238,7 @@ class LowRamGpus(unittest.TestCase):
def test_asked_and_answered_two(self):
ram, found = PROFILES["32GB-2x24GB"]
code, out, cfg, asked = install(ram, found, ["--family", "qwen", "--model", "Q2_0", "--no-start"],
- answers={"Low-RAM mode: which GPUs?": "2"})
+ answers={"Which GPUs?": "2", "Low-RAM mode: which GPUs?": "2"}) # the split asked for (the default is the largest card + RAM)
self.assertEqual(code, 0, out)
self.assertTrue(any("Low-RAM mode: which GPUs? [1]" in q for q in asked), asked)
self.assertEqual(cfg["gpu"], [0, 1])
@@ -276,7 +276,7 @@ class LowRamGpus(unittest.TestCase):
ram, found = PROFILES["32GB-2x24GB"]
with mock.patch.object(setup, "MIN_ENGINE", setup.RESIDENT_SPLIT_ENGINE):
code, out, cfg, asked = install(ram, found, ["--family", "qwen", "--model", "Q2_0", "--no-start"],
- answers="")
+ answers={"Which GPUs?": "2"}) # the split chosen (the default is the largest card + RAM)
self.assertEqual(code, 0, out)
self.assertEqual(cfg["gpu"], [0, 1])
self.assertIn("--resident-experts", cfg["args"])
diff --git a/tools/test_setup_unsloth.py b/tools/test_setup_unsloth.py
index 775794851..0a30687d4 100644
--- a/tools/test_setup_unsloth.py
+++ b/tools/test_setup_unsloth.py
@@ -372,7 +372,7 @@ class Main(Base):
self.assertAlmostEqual(need, setup.MODELS[M]["download_gb"] + setup.UNSLOTH_RAM_LEFT_GB)
code, out, cfg = self.main(["--context", "8192"], n_gpus=2, ram=127.8) # no --gpus: stays on one GPU
self.assertEqual(code, 0, out)
- self.assertIn("uses them together anyway", out)
+ self.assertNotIn("uses them together anyway", out) # one card + RAM is the default now: nothing to warn about
self.assertNotIn("layer_split", cfg)
self.assertEqual(cfg["args"][cfg["args"].index("--resident-budget-gib") + 1], "71") # all of them
code, out, cfg = self.main(["--context", "8192", "--gpus", "0,1"], n_gpus=2, ram=127.8) # asked for by name
@@ -428,8 +428,8 @@ class Main(Base):
def test_yes_alone_keeps_one_gpu(self):
code, out, cfg = self.main(["--context", "8192"], n_gpus=2, ram=165.0)
self.assertEqual(code, 0, out)
- self.assertIn("runs on one GPU: using", out)
- self.assertIn("--gpus 0,1 shares it", out)
+ self.assertIn("layer split", out) # the second card is offered as a split
+ self.assertIn("(recommended)", out)
self.assertNotIn("layer_split", cfg)
self.assertIn("--resident-budget-gib", cfg["args"])
@@ -629,10 +629,10 @@ class LayerSplit(unittest.TestCase):
argv = ["--family", "unsloth", "--model", M, "--context", "32768", "--no-start"]
code, out, cfg, asked = install(165.0, self.found, argv, answers={"which GPUs?": "1", "Which GPUs?": "1"})
self.assertEqual(code, 0, out)
- self.assertTrue(any(f"{M}: which GPUs?" in q and "[1]" in q for q in asked), asked)
+ self.assertTrue(any("Which GPUs? [1]" in q for q in asked), asked) # the fit rule's question: the largest card + RAM is the default
self.assertEqual(cfg["gpu"], 0)
self.assertIn("--resident-budget-gib", cfg["args"])
- code, out, cfg, asked = install(165.0, self.found, argv, answers={f"{M}: which GPUs?": "2"})
+ code, out, cfg, asked = install(165.0, self.found, argv, answers={"Which GPUs?": "2", f"{M}: which GPUs?": "2"})
self.assertEqual(code, 0, out)
self.assertEqual(cfg["gpu"], [0, 1])
self.assertEqual(cfg["layer_split"], "auto")
@@ -791,7 +791,7 @@ class Resident(Base):
def test_two_gpus_split_with_every_expert_in_ram(self):
"""Where the experts fit, two cards are the GGUF models' layer split (no budget question, no #498 note)."""
- code, out, cfg = self.main(["--context", "8192"], n_gpus=2, ram=127.8)
+ code, out, cfg = self.main(["--context", "8192"], n_gpus=2, ram=127.8, answers={"Which GPUs?": "2"}) # the split chosen
self.assertEqual(code, 0, out)
self.assertEqual(cfg["gpu"], [0, 1])
self.assertEqual(cfg["layer_split"], "auto")