Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

gpu_fit: one rule for which graphics cards a model uses (largest card + RAM first, the split offered when it does not fit, an iGPU ignored), with tests

Niko1221ac38793caa2e
diff --git a/strata_control/gpu_fit.py b/strata_control/gpu_fit.py
new file mode 100644
index 000000000..887d74c8a
--- /dev/null
+++ b/strata_control/gpu_fit.py
@@ -0,0 +1,162 @@
+"""Which graphics cards a model should use: ONE place for the app, `strata launch` and setup.
+
+The rule (owner decision):
+1. First try the model on ONE card (the largest) plus system RAM, the normal hybrid mode.
+2. Only when that does not fit, or would be too tight, offer to add the second card as a layer split.
+3. Let the person choose: every option comes with plain numbers and a recommended one is pre-selected (recommend, never force).
+
+The memory arithmetic is native-fit.ts `fitOn` (apps/host/src/lib/native-fit.ts) with the same constants (a test keeps them equal):
+the experts are held first in the graphics memory beyond about DENSE_GB (the dense weights, buffers and a 32K cache), then in RAM beyond
+about SYSTEM_ROOM_GB left for the system; whatever is left would be read from the SSD (`spill_gb`). A split adds the memory of the extra
+cards but every extra card holds its own dense layers and prompt buffers, so each one costs SPLIT_CARD_OVERHEAD_GB of that (an estimate:
+docs/MULTI_GPU.md says each card "holds the dense weights of its layers and its own prompt buffers").
+
+Cards that count: not integrated (an iGPU shares the system RAM and is ignored), at least MIN_CARD_GB of graphics memory, flagged usable by
+the caller's own driver checks (NVIDIA RTX 20 or newer, a supported AMD card, ...). A card needs SPLIT_MIN_VRAM_GB to share a model.
+Pure functions, no I/O: setup.py, strata_cli/launch and tools/strata_mcp.py import it.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+DENSE_GB = 5.0 # = native-fit.ts DENSE_GB
+SYSTEM_ROOM_GB = 10.0 # = native-fit.ts SYSTEM_ROOM_GB
+DENSE_EXTRA_GB = 1.0 # = native-fit.ts DENSE_EXTRA_GB
+SPLIT_CARD_OVERHEAD_GB = 2.0 # each extra card of a split: its own dense layers and prompt buffers
+SPLIT_MIN_VRAM_GB = 8.0 # setup.py SPLIT_MIN_VRAM_GB: a card sharing a model needs this much
+SPLIT_PROMPT_VRAM_GB = 12.0 # setup.py SPLIT_PROMPT_VRAM_GB (#448): a smaller second card caps the prompt chunk
+MIN_CARD_GB = 4.0 # under this a card cannot hold the dense weights: ignored (an iGPU, a display card)
+FITS_SPARE_RAM_GB = 4.0 # RAM left beyond the experts below which "fits" becomes "tight"
+SPLIT_PROMPT_GAIN = "about 20% faster" # setup.py's own words for prompts read on two cards (docs/MULTI_GPU.md)
+
+
+def _round1(x: float) -> float:
+ return round(float(x) * 10) / 10
+
+
+def memory_needed(variant: dict) -> float:
+ """RAM + VRAM the model asks for (native-fit.ts memoryNeeded)."""
+ if isinstance(variant.get("memory_gb"), (int, float)):
+ return float(variant["memory_gb"])
+ if variant.get("mem_total") and isinstance(variant.get("ram_needed_gb"), (int, float)):
+ return float(variant["ram_needed_gb"])
+ return _round1(float(variant.get("experts_gb") or variant.get("ram_needed_gb") or 0) + DENSE_EXTRA_GB)
+
+
+def fit(memory_gb: float, ram_gb: float | None, vram_gb: float) -> dict[str, Any]:
+ """native-fit.ts `fitOn` for a given memory need: where the weights go. Keys: vram_use_gb, ram_use_gb, spill_gb, spare_ram_gb, known."""
+ vram = float(vram_gb or 0)
+ if not ram_gb or ram_gb <= 0:
+ return {"vram_use_gb": 0.0, "ram_use_gb": 0.0, "spill_gb": 0.0, "spare_ram_gb": 0.0, "known": False}
+ experts = max(0.0, memory_gb - DENSE_EXTRA_GB)
+ on_gpu = max(0.0, min(experts, vram - DENSE_GB))
+ ram_room = max(0.0, float(ram_gb) - SYSTEM_ROOM_GB)
+ spill = max(0.0, experts - on_gpu - ram_room)
+ in_ram = max(0.0, experts - on_gpu - spill)
+ return {"vram_use_gb": _round1(min(vram, DENSE_GB + on_gpu) if vram > 0 else 0.0),
+ "ram_use_gb": _round1(in_ram + (DENSE_EXTRA_GB if vram <= 0 else 0)), "spill_gb": _round1(spill),
+ "spare_ram_gb": _round1(max(0.0, ram_room - in_ram)), "known": True}
+
+
+def verdict(f: dict) -> str:
+ """fits / tight / spills: all in memory with room to spare / all in memory but little RAM left / part of it is read from the SSD."""
+ if not f.get("known"):
+ return "unknown"
+ if f["spill_gb"] >= 1.0:
+ return "spills"
+ return "fits" if f["spare_ram_gb"] >= FITS_SPARE_RAM_GB or f["ram_use_gb"] <= 0 else "tight"
+
+
+def usable_cards(cards: list[dict]) -> tuple[list[dict], list[dict]]:
+ """(cards that count, cards ignored with a plain reason in 'why_ignored'). Largest memory first, then the card number."""
+ keep, ignored = [], []
+ for c in cards or []:
+ vram = float(c.get("vram_gb") or 0)
+ why = None
+ if c.get("usable") is False:
+ why = c.get("problem") or "Strata cannot use this card"
+ elif c.get("integrated") or c.get("uma") or c.get("unified_memory"):
+ why = "it shares the system memory (an integrated graphics chip), so it adds nothing"
+ elif vram < MIN_CARD_GB:
+ why = f"only {vram:.0f} GB of graphics memory"
+ if why:
+ ignored.append({**c, "why_ignored": why})
+ else:
+ keep.append(c)
+ keep.sort(key=lambda c: (-round(float(c.get("vram_gb") or 0)), c.get("index", 0)))
+ return keep, ignored
+
+
+def _name(c: dict) -> str:
+ n = str(c.get("name") or f"card {c.get('index')}")
+ for junk in ("NVIDIA GeForce ", "NVIDIA ", "AMD Radeon ", "AMD "):
+ n = n.replace(junk, "")
+ return n
+
+
+def _option(key: str, cards: list[dict], memory_gb: float, ram_gb: float | None) -> dict[str, Any]:
+ n = len(cards)
+ total = sum(float(c.get("vram_gb") or 0) for c in cards)
+ eff = total - SPLIT_CARD_OVERHEAD_GB * (n - 1)
+ f = fit(memory_gb, ram_gb, eff)
+ v = verdict(f)
+ names = " + ".join(_name(c) for c in cards)
+ label = f"{names} + RAM" if n == 1 else f"{names} + RAM (layer split)"
+ have = (eff + max(0.0, float(ram_gb or 0) - SYSTEM_ROOM_GB))
+ if v == "unknown":
+ words = "memory of this computer unknown"
+ elif v == "spills":
+ words = f"does not fit in memory: about {f['spill_gb']:.0f} GB is read from the SSD while answering (slower)"
+ elif v == "tight":
+ words = f"fits, but tight: needs {memory_gb:.0f} GB of the {have:.0f} GB available"
+ else:
+ words = f"fits: needs {memory_gb:.0f} GB of the {have:.0f} GB available"
+ notes = []
+ if n >= 2:
+ notes.append(f"prompts are read {SPLIT_PROMPT_GAIN}; the cards share the layers (docs/MULTI_GPU.md)")
+ small = [c for c in cards[1:] if float(c.get("vram_gb") or 0) < SPLIT_PROMPT_VRAM_GB - 0.5]
+ if small and float(cards[0].get("vram_gb") or 0) >= SPLIT_PROMPT_VRAM_GB - 0.5:
+ notes.append(f"{_name(small[0])} is small: it can make prompts slower than the first card alone")
+ return {"key": key, "cards": [c.get("index") for c in cards], "label": label, "verdict": v, "summary": f"{label}: {words}",
+ "words": words, "vram_gb": _round1(total), "spill_gb": f["spill_gb"], "ram_use_gb": f["ram_use_gb"],
+ "spare_ram_gb": f["spare_ram_gb"], "notes": notes, "recommended": False}
+
+
+_RANK = {"fits": 0, "tight": 1, "unknown": 2, "spills": 3}
+
+
+def options(memory_gb: float, ram_gb: float | None, cards: list[dict]) -> dict[str, Any]:
+ """The choices for this model on this computer: {"options": [...], "recommended": key, "ignored": [...], "cards": n counted}.
+
+ Options (best known first): the largest card alone; with two or more cards that can share a model, the two largest as a layer split
+ (and all of them when there are more than two); then every other card alone. Exactly one option is recommended:
+ the single largest card when the model fits it (with RAM); otherwise the split if it fits better (a smaller verdict or less SSD use);
+ otherwise the single largest card again (nothing better to offer)."""
+ keep, ignored = usable_cards(cards)
+ out: dict[str, Any] = {"options": [], "recommended": None, "ignored": ignored, "cards": len(keep)}
+ if not keep:
+ return out
+ opts = [_option("single", [keep[0]], memory_gb, ram_gb)]
+ share = [c for c in keep if float(c.get("vram_gb") or 0) >= SPLIT_MIN_VRAM_GB - 0.5]
+ if len(share) >= 2:
+ opts.append(_option("split", share[:2], memory_gb, ram_gb))
+ if len(share) > 2:
+ opts.append(_option("split-all", share, memory_gb, ram_gb))
+ for c in keep[1:]:
+ opts.append(_option(f"single-{c.get('index')}", [c], memory_gb, ram_gb))
+ single = opts[0]
+ best = single
+ if single["verdict"] != "fits":
+ for o in opts[1:]:
+ if len(o["cards"]) > 1 and (_RANK[o["verdict"]], o["spill_gb"]) < (_RANK[best["verdict"]], best["spill_gb"]):
+ best = o
+ best["recommended"] = True
+ out["options"], out["recommended"] = opts, best["key"]
+ return out
+
+
+def recommended_cards(memory_gb: float, ram_gb: float | None, cards: list[dict]) -> list[int]:
+ """The card numbers of the recommended option (empty: no usable card)."""
+ res = options(memory_gb, ram_gb, cards)
+ return next((o["cards"] for o in res["options"] if o["recommended"]), [])
diff --git a/tests/terminal/test_gpu_fit.py b/tests/terminal/test_gpu_fit.py
new file mode 100644
index 000000000..b4b099581
--- /dev/null
+++ b/tests/terminal/test_gpu_fit.py
@@ -0,0 +1,110 @@
+"""strata_control/gpu_fit.py: the one rule for which graphics cards a model uses (app, strata launch and setup share it)."""
+
+from __future__ import annotations
+
+import re
+import unittest
+from pathlib import Path
+
+from strata_control import gpu_fit as G
+
+ROOT = Path(__file__).resolve().parents[2]
+
+
+def card(index, name, vram, **kw):
+ return {"index": index, "name": name, "vram_gb": vram, "usable": True, **kw}
+
+
+class GpuFitRule(unittest.TestCase):
+ def test_one_card_is_the_only_option(self):
+ r = G.options(44.0, 62.0, [card(0, "NVIDIA GeForce RTX 3070 Ti", 8)])
+ self.assertEqual([o["key"] for o in r["options"]], ["single"])
+ self.assertEqual(r["recommended"], "single")
+ self.assertEqual(r["options"][0]["verdict"], "fits")
+ self.assertIn("RTX 3070 Ti + RAM", r["options"][0]["label"])
+
+ def test_two_unequal_cards_and_the_model_fits_one_defaults_to_single_and_offers_the_split(self):
+ r = G.options(44.0, 62.0, [card(1, "NVIDIA GeForce RTX 3060 Ti", 8), card(0, "NVIDIA GeForce RTX 3070 Ti", 16)])
+ keys = [o["key"] for o in r["options"]]
+ self.assertEqual(keys[:2], ["single", "split"]) # the largest card first, then the split
+ self.assertEqual(r["recommended"], "single")
+ self.assertEqual(r["options"][0]["cards"], [0]) # the largest card is the 16 GB one
+ split = r["options"][1]
+ self.assertEqual(split["cards"], [0, 1])
+ self.assertIn("layer split", split["label"])
+ self.assertTrue(any("20% faster" in n for n in split["notes"]))
+ self.assertEqual(sum(1 for o in r["options"] if o["recommended"]), 1)
+
+ def test_two_small_cards_and_the_model_does_not_fit_one_defaults_to_the_split(self):
+ r = G.options(44.0, 32.0, [card(0, "NVIDIA GeForce RTX 3060", 10), card(1, "NVIDIA GeForce RTX 3060", 10)])
+ single, split = r["options"][0], r["options"][1]
+ self.assertEqual(single["verdict"], "spills")
+ self.assertLess(split["spill_gb"], single["spill_gb"]) # the second card takes part of the SSD reading away
+ self.assertEqual(r["recommended"], "split")
+ self.assertEqual(split["cards"], [0, 1])
+
+ def test_a_tight_single_card_moves_the_default_to_the_split_when_that_fits_with_room(self):
+ # 44 GB model, 46 GB of RAM: one 12 GB card + RAM holds it but leaves no RAM spare (tight); two cards leave room
+ r = G.options(44.0, 46.0, [card(0, "RTX 4070", 12), card(1, "RTX 4070", 12)])
+ self.assertEqual(r["options"][0]["verdict"], "tight")
+ self.assertEqual(r["options"][1]["verdict"], "fits")
+ self.assertEqual(r["recommended"], "split")
+
+ def test_an_integrated_chip_and_a_tiny_card_are_ignored(self):
+ cards = [card(0, "NVIDIA GeForce RTX 3070 Ti", 8), card(1, "AMD Radeon Graphics", 16, integrated=True), card(2, "Old display card", 2)]
+ r = G.options(44.0, 62.0, cards)
+ self.assertEqual([o["key"] for o in r["options"]], ["single"])
+ self.assertEqual({c["index"] for c in r["ignored"]}, {1, 2})
+ self.assertTrue(all(c["why_ignored"] for c in r["ignored"]))
+ self.assertEqual(G.recommended_cards(44.0, 62.0, cards), [0])
+
+ def test_an_unusable_card_is_ignored_with_its_own_reason(self):
+ r = G.options(44.0, 62.0, [card(0, "RTX 3070 Ti", 8), card(1, "GTX 1060", 6, usable=False, problem="older than the RTX 20 series")])
+ self.assertEqual([o["key"] for o in r["options"]], ["single"])
+ self.assertIn("older than the RTX 20 series", r["ignored"][0]["why_ignored"])
+
+ def test_a_card_under_the_split_minimum_is_offered_alone_but_not_in_the_split(self):
+ r = G.options(44.0, 32.0, [card(0, "RTX 3060", 12), card(1, "GTX 1660", 6)])
+ keys = [o["key"] for o in r["options"]]
+ self.assertNotIn("split", keys)
+ self.assertEqual(r["recommended"], "single")
+
+ def test_three_cards_offer_the_pair_and_all_of_them(self):
+ r = G.options(70.0, 32.0, [card(i, "RTX 3060", 12) for i in range(3)])
+ keys = [o["key"] for o in r["options"]]
+ self.assertEqual(keys[:3], ["single", "split", "split-all"])
+
+ def test_no_card_gives_no_options(self):
+ r = G.options(44.0, 62.0, [])
+ self.assertEqual((r["options"], r["recommended"]), ([], None))
+
+ def test_unknown_ram_is_unknown_not_a_guess(self):
+ r = G.options(44.0, None, [card(0, "RTX 3070 Ti", 8), card(1, "RTX 3060 Ti", 8)])
+ self.assertEqual(r["options"][0]["verdict"], "unknown")
+ self.assertEqual(r["recommended"], "single")
+
+ def test_plain_numbers_are_in_the_words(self):
+ r = G.options(44.0, 62.0, [card(0, "NVIDIA GeForce RTX 3070 Ti", 8)])
+ self.assertIn("needs 44 GB", r["options"][0]["words"])
+ self.assertIn("RTX 3070 Ti + RAM", r["options"][0]["summary"])
+ r2 = G.options(44.0, 32.0, [card(0, "RTX 3060", 10)])
+ self.assertIn("read from the SSD", r2["options"][0]["words"])
+
+ def test_the_constants_equal_the_apps(self):
+ text = (ROOT / "apps" / "host" / "src" / "lib" / "native-fit.ts").read_text(encoding="utf-8")
+ m = re.search(r"const DENSE_GB = (\d+), SYSTEM_ROOM_GB = (\d+), DENSE_EXTRA_GB = (\d+)", text)
+ self.assertIsNotNone(m)
+ self.assertEqual((G.DENSE_GB, G.SYSTEM_ROOM_GB, G.DENSE_EXTRA_GB), tuple(float(x) for x in m.groups()))
+
+ def test_fit_matches_the_launch_catalog_port_of_native_fit(self):
+ from strata_cli.launch.catalog import fit_on
+ for ram, vram in ((62, 16), (32, 10), (16, 0), (128, 24)):
+ a = fit_on({"memory_gb": 44.0}, ram, vram)
+ b = G.fit(44.0, ram, vram)
+ self.assertAlmostEqual(a["spillGb"], b["spill_gb"])
+ self.assertAlmostEqual(a["vramUseGb"], b["vram_use_gb"])
+ self.assertAlmostEqual(a["ramUseGb"], b["ram_use_gb"])
+
+
+if __name__ == "__main__":
+ unittest.main()