Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
gpu_fit: one rule for which graphics cards a model uses (largest card + RAM first, the split offered when it does not fit, an iGPU ignored), with tests
Niko1221ac38793caa2e
diff --git a/strata_control/gpu_fit.py b/strata_control/gpu_fit.pynew file mode 100644index 000000000..887d74c8a--- /dev/null+++ b/strata_control/gpu_fit.py@@ -0,0 +1,162 @@+"""Which graphics cards a model should use: ONE place for the app, `strata launch` and setup.++The rule (owner decision):+1. First try the model on ONE card (the largest) plus system RAM, the normal hybrid mode.+2. Only when that does not fit, or would be too tight, offer to add the second card as a layer split.+3. Let the person choose: every option comes with plain numbers and a recommended one is pre-selected (recommend, never force).++The memory arithmetic is native-fit.ts `fitOn` (apps/host/src/lib/native-fit.ts) with the same constants (a test keeps them equal):+the experts are held first in the graphics memory beyond about DENSE_GB (the dense weights, buffers and a 32K cache), then in RAM beyond+about SYSTEM_ROOM_GB left for the system; whatever is left would be read from the SSD (`spill_gb`). A split adds the memory of the extra+cards but every extra card holds its own dense layers and prompt buffers, so each one costs SPLIT_CARD_OVERHEAD_GB of that (an estimate:+docs/MULTI_GPU.md says each card "holds the dense weights of its layers and its own prompt buffers").++Cards that count: not integrated (an iGPU shares the system RAM and is ignored), at least MIN_CARD_GB of graphics memory, flagged usable by+the caller's own driver checks (NVIDIA RTX 20 or newer, a supported AMD card, ...). A card needs SPLIT_MIN_VRAM_GB to share a model.+Pure functions, no I/O: setup.py, strata_cli/launch and tools/strata_mcp.py import it.+"""++from __future__ import annotations++from typing import Any++DENSE_GB = 5.0 # = native-fit.ts DENSE_GB+SYSTEM_ROOM_GB = 10.0 # = native-fit.ts SYSTEM_ROOM_GB+DENSE_EXTRA_GB = 1.0 # = native-fit.ts DENSE_EXTRA_GB+SPLIT_CARD_OVERHEAD_GB = 2.0 # each extra card of a split: its own dense layers and prompt buffers+SPLIT_MIN_VRAM_GB = 8.0 # setup.py SPLIT_MIN_VRAM_GB: a card sharing a model needs this much+SPLIT_PROMPT_VRAM_GB = 12.0 # setup.py SPLIT_PROMPT_VRAM_GB (#448): a smaller second card caps the prompt chunk+MIN_CARD_GB = 4.0 # under this a card cannot hold the dense weights: ignored (an iGPU, a display card)+FITS_SPARE_RAM_GB = 4.0 # RAM left beyond the experts below which "fits" becomes "tight"+SPLIT_PROMPT_GAIN = "about 20% faster" # setup.py's own words for prompts read on two cards (docs/MULTI_GPU.md)+++def _round1(x: float) -> float:+ return round(float(x) * 10) / 10+++def memory_needed(variant: dict) -> float:+ """RAM + VRAM the model asks for (native-fit.ts memoryNeeded)."""+ if isinstance(variant.get("memory_gb"), (int, float)):+ return float(variant["memory_gb"])+ if variant.get("mem_total") and isinstance(variant.get("ram_needed_gb"), (int, float)):+ return float(variant["ram_needed_gb"])+ return _round1(float(variant.get("experts_gb") or variant.get("ram_needed_gb") or 0) + DENSE_EXTRA_GB)+++def fit(memory_gb: float, ram_gb: float | None, vram_gb: float) -> dict[str, Any]:+ """native-fit.ts `fitOn` for a given memory need: where the weights go. Keys: vram_use_gb, ram_use_gb, spill_gb, spare_ram_gb, known."""+ vram = float(vram_gb or 0)+ if not ram_gb or ram_gb <= 0:+ return {"vram_use_gb": 0.0, "ram_use_gb": 0.0, "spill_gb": 0.0, "spare_ram_gb": 0.0, "known": False}+ experts = max(0.0, memory_gb - DENSE_EXTRA_GB)+ on_gpu = max(0.0, min(experts, vram - DENSE_GB))+ ram_room = max(0.0, float(ram_gb) - SYSTEM_ROOM_GB)+ spill = max(0.0, experts - on_gpu - ram_room)+ in_ram = max(0.0, experts - on_gpu - spill)+ return {"vram_use_gb": _round1(min(vram, DENSE_GB + on_gpu) if vram > 0 else 0.0),+ "ram_use_gb": _round1(in_ram + (DENSE_EXTRA_GB if vram <= 0 else 0)), "spill_gb": _round1(spill),+ "spare_ram_gb": _round1(max(0.0, ram_room - in_ram)), "known": True}+++def verdict(f: dict) -> str:+ """fits / tight / spills: all in memory with room to spare / all in memory but little RAM left / part of it is read from the SSD."""+ if not f.get("known"):+ return "unknown"+ if f["spill_gb"] >= 1.0:+ return "spills"+ return "fits" if f["spare_ram_gb"] >= FITS_SPARE_RAM_GB or f["ram_use_gb"] <= 0 else "tight"+++def usable_cards(cards: list[dict]) -> tuple[list[dict], list[dict]]:+ """(cards that count, cards ignored with a plain reason in 'why_ignored'). Largest memory first, then the card number."""+ keep, ignored = [], []+ for c in cards or []:+ vram = float(c.get("vram_gb") or 0)+ why = None+ if c.get("usable") is False:+ why = c.get("problem") or "Strata cannot use this card"+ elif c.get("integrated") or c.get("uma") or c.get("unified_memory"):+ why = "it shares the system memory (an integrated graphics chip), so it adds nothing"+ elif vram < MIN_CARD_GB:+ why = f"only {vram:.0f} GB of graphics memory"+ if why:+ ignored.append({**c, "why_ignored": why})+ else:+ keep.append(c)+ keep.sort(key=lambda c: (-round(float(c.get("vram_gb") or 0)), c.get("index", 0)))+ return keep, ignored+++def _name(c: dict) -> str:+ n = str(c.get("name") or f"card {c.get('index')}")+ for junk in ("NVIDIA GeForce ", "NVIDIA ", "AMD Radeon ", "AMD "):+ n = n.replace(junk, "")+ return n+++def _option(key: str, cards: list[dict], memory_gb: float, ram_gb: float | None) -> dict[str, Any]:+ n = len(cards)+ total = sum(float(c.get("vram_gb") or 0) for c in cards)+ eff = total - SPLIT_CARD_OVERHEAD_GB * (n - 1)+ f = fit(memory_gb, ram_gb, eff)+ v = verdict(f)+ names = " + ".join(_name(c) for c in cards)+ label = f"{names} + RAM" if n == 1 else f"{names} + RAM (layer split)"+ have = (eff + max(0.0, float(ram_gb or 0) - SYSTEM_ROOM_GB))+ if v == "unknown":+ words = "memory of this computer unknown"+ elif v == "spills":+ words = f"does not fit in memory: about {f['spill_gb']:.0f} GB is read from the SSD while answering (slower)"+ elif v == "tight":+ words = f"fits, but tight: needs {memory_gb:.0f} GB of the {have:.0f} GB available"+ else:+ words = f"fits: needs {memory_gb:.0f} GB of the {have:.0f} GB available"+ notes = []+ if n >= 2:+ notes.append(f"prompts are read {SPLIT_PROMPT_GAIN}; the cards share the layers (docs/MULTI_GPU.md)")+ small = [c for c in cards[1:] if float(c.get("vram_gb") or 0) < SPLIT_PROMPT_VRAM_GB - 0.5]+ if small and float(cards[0].get("vram_gb") or 0) >= SPLIT_PROMPT_VRAM_GB - 0.5:+ notes.append(f"{_name(small[0])} is small: it can make prompts slower than the first card alone")+ return {"key": key, "cards": [c.get("index") for c in cards], "label": label, "verdict": v, "summary": f"{label}: {words}",+ "words": words, "vram_gb": _round1(total), "spill_gb": f["spill_gb"], "ram_use_gb": f["ram_use_gb"],+ "spare_ram_gb": f["spare_ram_gb"], "notes": notes, "recommended": False}+++_RANK = {"fits": 0, "tight": 1, "unknown": 2, "spills": 3}+++def options(memory_gb: float, ram_gb: float | None, cards: list[dict]) -> dict[str, Any]:+ """The choices for this model on this computer: {"options": [...], "recommended": key, "ignored": [...], "cards": n counted}.++ Options (best known first): the largest card alone; with two or more cards that can share a model, the two largest as a layer split+ (and all of them when there are more than two); then every other card alone. Exactly one option is recommended:+ the single largest card when the model fits it (with RAM); otherwise the split if it fits better (a smaller verdict or less SSD use);+ otherwise the single largest card again (nothing better to offer)."""+ keep, ignored = usable_cards(cards)+ out: dict[str, Any] = {"options": [], "recommended": None, "ignored": ignored, "cards": len(keep)}+ if not keep:+ return out+ opts = [_option("single", [keep[0]], memory_gb, ram_gb)]+ share = [c for c in keep if float(c.get("vram_gb") or 0) >= SPLIT_MIN_VRAM_GB - 0.5]+ if len(share) >= 2:+ opts.append(_option("split", share[:2], memory_gb, ram_gb))+ if len(share) > 2:+ opts.append(_option("split-all", share, memory_gb, ram_gb))+ for c in keep[1:]:+ opts.append(_option(f"single-{c.get('index')}", [c], memory_gb, ram_gb))+ single = opts[0]+ best = single+ if single["verdict"] != "fits":+ for o in opts[1:]:+ if len(o["cards"]) > 1 and (_RANK[o["verdict"]], o["spill_gb"]) < (_RANK[best["verdict"]], best["spill_gb"]):+ best = o+ best["recommended"] = True+ out["options"], out["recommended"] = opts, best["key"]+ return out+++def recommended_cards(memory_gb: float, ram_gb: float | None, cards: list[dict]) -> list[int]:+ """The card numbers of the recommended option (empty: no usable card)."""+ res = options(memory_gb, ram_gb, cards)+ return next((o["cards"] for o in res["options"] if o["recommended"]), [])diff --git a/tests/terminal/test_gpu_fit.py b/tests/terminal/test_gpu_fit.pynew file mode 100644index 000000000..b4b099581--- /dev/null+++ b/tests/terminal/test_gpu_fit.py@@ -0,0 +1,110 @@+"""strata_control/gpu_fit.py: the one rule for which graphics cards a model uses (app, strata launch and setup share it)."""++from __future__ import annotations++import re+import unittest+from pathlib import Path++from strata_control import gpu_fit as G++ROOT = Path(__file__).resolve().parents[2]+++def card(index, name, vram, **kw):+ return {"index": index, "name": name, "vram_gb": vram, "usable": True, **kw}+++class GpuFitRule(unittest.TestCase):+ def test_one_card_is_the_only_option(self):+ r = G.options(44.0, 62.0, [card(0, "NVIDIA GeForce RTX 3070 Ti", 8)])+ self.assertEqual([o["key"] for o in r["options"]], ["single"])+ self.assertEqual(r["recommended"], "single")+ self.assertEqual(r["options"][0]["verdict"], "fits")+ self.assertIn("RTX 3070 Ti + RAM", r["options"][0]["label"])++ def test_two_unequal_cards_and_the_model_fits_one_defaults_to_single_and_offers_the_split(self):+ r = G.options(44.0, 62.0, [card(1, "NVIDIA GeForce RTX 3060 Ti", 8), card(0, "NVIDIA GeForce RTX 3070 Ti", 16)])+ keys = [o["key"] for o in r["options"]]+ self.assertEqual(keys[:2], ["single", "split"]) # the largest card first, then the split+ self.assertEqual(r["recommended"], "single")+ self.assertEqual(r["options"][0]["cards"], [0]) # the largest card is the 16 GB one+ split = r["options"][1]+ self.assertEqual(split["cards"], [0, 1])+ self.assertIn("layer split", split["label"])+ self.assertTrue(any("20% faster" in n for n in split["notes"]))+ self.assertEqual(sum(1 for o in r["options"] if o["recommended"]), 1)++ def test_two_small_cards_and_the_model_does_not_fit_one_defaults_to_the_split(self):+ r = G.options(44.0, 32.0, [card(0, "NVIDIA GeForce RTX 3060", 10), card(1, "NVIDIA GeForce RTX 3060", 10)])+ single, split = r["options"][0], r["options"][1]+ self.assertEqual(single["verdict"], "spills")+ self.assertLess(split["spill_gb"], single["spill_gb"]) # the second card takes part of the SSD reading away+ self.assertEqual(r["recommended"], "split")+ self.assertEqual(split["cards"], [0, 1])++ def test_a_tight_single_card_moves_the_default_to_the_split_when_that_fits_with_room(self):+ # 44 GB model, 46 GB of RAM: one 12 GB card + RAM holds it but leaves no RAM spare (tight); two cards leave room+ r = G.options(44.0, 46.0, [card(0, "RTX 4070", 12), card(1, "RTX 4070", 12)])+ self.assertEqual(r["options"][0]["verdict"], "tight")+ self.assertEqual(r["options"][1]["verdict"], "fits")+ self.assertEqual(r["recommended"], "split")++ def test_an_integrated_chip_and_a_tiny_card_are_ignored(self):+ cards = [card(0, "NVIDIA GeForce RTX 3070 Ti", 8), card(1, "AMD Radeon Graphics", 16, integrated=True), card(2, "Old display card", 2)]+ r = G.options(44.0, 62.0, cards)+ self.assertEqual([o["key"] for o in r["options"]], ["single"])+ self.assertEqual({c["index"] for c in r["ignored"]}, {1, 2})+ self.assertTrue(all(c["why_ignored"] for c in r["ignored"]))+ self.assertEqual(G.recommended_cards(44.0, 62.0, cards), [0])++ def test_an_unusable_card_is_ignored_with_its_own_reason(self):+ r = G.options(44.0, 62.0, [card(0, "RTX 3070 Ti", 8), card(1, "GTX 1060", 6, usable=False, problem="older than the RTX 20 series")])+ self.assertEqual([o["key"] for o in r["options"]], ["single"])+ self.assertIn("older than the RTX 20 series", r["ignored"][0]["why_ignored"])++ def test_a_card_under_the_split_minimum_is_offered_alone_but_not_in_the_split(self):+ r = G.options(44.0, 32.0, [card(0, "RTX 3060", 12), card(1, "GTX 1660", 6)])+ keys = [o["key"] for o in r["options"]]+ self.assertNotIn("split", keys)+ self.assertEqual(r["recommended"], "single")++ def test_three_cards_offer_the_pair_and_all_of_them(self):+ r = G.options(70.0, 32.0, [card(i, "RTX 3060", 12) for i in range(3)])+ keys = [o["key"] for o in r["options"]]+ self.assertEqual(keys[:3], ["single", "split", "split-all"])++ def test_no_card_gives_no_options(self):+ r = G.options(44.0, 62.0, [])+ self.assertEqual((r["options"], r["recommended"]), ([], None))++ def test_unknown_ram_is_unknown_not_a_guess(self):+ r = G.options(44.0, None, [card(0, "RTX 3070 Ti", 8), card(1, "RTX 3060 Ti", 8)])+ self.assertEqual(r["options"][0]["verdict"], "unknown")+ self.assertEqual(r["recommended"], "single")++ def test_plain_numbers_are_in_the_words(self):+ r = G.options(44.0, 62.0, [card(0, "NVIDIA GeForce RTX 3070 Ti", 8)])+ self.assertIn("needs 44 GB", r["options"][0]["words"])+ self.assertIn("RTX 3070 Ti + RAM", r["options"][0]["summary"])+ r2 = G.options(44.0, 32.0, [card(0, "RTX 3060", 10)])+ self.assertIn("read from the SSD", r2["options"][0]["words"])++ def test_the_constants_equal_the_apps(self):+ text = (ROOT / "apps" / "host" / "src" / "lib" / "native-fit.ts").read_text(encoding="utf-8")+ m = re.search(r"const DENSE_GB = (\d+), SYSTEM_ROOM_GB = (\d+), DENSE_EXTRA_GB = (\d+)", text)+ self.assertIsNotNone(m)+ self.assertEqual((G.DENSE_GB, G.SYSTEM_ROOM_GB, G.DENSE_EXTRA_GB), tuple(float(x) for x in m.groups()))++ def test_fit_matches_the_launch_catalog_port_of_native_fit(self):+ from strata_cli.launch.catalog import fit_on+ for ram, vram in ((62, 16), (32, 10), (16, 0), (128, 24)):+ a = fit_on({"memory_gb": 44.0}, ram, vram)+ b = G.fit(44.0, ram, vram)+ self.assertAlmostEqual(a["spillGb"], b["spill_gb"])+ self.assertAlmostEqual(a["vramUseGb"], b["vram_use_gb"])+ self.assertAlmostEqual(a["ramUseGb"], b["ram_use_gb"])+++if __name__ == "__main__":+ unittest.main()