Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
AMD: built-in Radeon graphics (Strix Halo) is never the last stage of a layer split, the picture reader leaves it, --gpus all leaves it out beside two cards, engine nice +5 on Linux with such a chip, split note
Niko1221034422c4d307
diff --git a/serve/server.py b/serve/server.pyindex 969651106..b5b3c1033 100644--- a/serve/server.py+++ b/serve/server.py@@ -1014,6 +1014,12 @@ class StrataEngine:pin = {}if cpus and hasattr(os, "sched_setaffinity"): # replicas: this engine's own physical cores (serve/replicas.py)pin["preexec_fn"] = lambda: os.sched_setaffinity(0, cpus)+ nice = engine_nice(env) # Linux with built-in Radeon graphics: the desktop keeps its share of the CPU+ if nice:+ say_nice = pin.get("preexec_fn")+ pin["preexec_fn"] = lambda: (say_nice() if say_nice else None, os.nice(nice))[0]+ print(f"[strata] engine priority: nice +{nice} (this PC has built-in Radeon graphics that usually draw the desktop; "+ "STRATA_ENGINE_NICE=0 turns it off)", flush=True)end_leftover_engines(exe) # one an earlier run lost still holds the GPU and the memoryspawned_at = time.monotonic()try:@@ -3145,6 +3151,47 @@ def hip_speed_scores(indices: list[int], root: str = "/sys/class/kfd/kfd/topologreturn got if all(i in got for i in indices) else None+def engine_nice(env: dict | None = None, archs: dict[int, str] | None = None, platform: str | None = None) -> int:+ """Linux only: the nice value the engine starts with. The engine's CPU pool spins at 100% on every physical core but one and its host+ thread on another; with the desktop drawn by a built-in Radeon (HIP_IGPU_ARCHS: Strix Halo and the other APUs) the compositor then+ competes with them for the CPU, which reads as a frozen screen. +5 gives the desktop its share and costs nothing when nothing else+ runs. STRATA_ENGINE_NICE=N sets it (0 = off, also for any PC without such a chip). Not on Windows / macOS."""+ if not (platform or sys.platform).startswith("linux"):+ return 0+ asked = (env or os.environ).get("STRATA_ENGINE_NICE")+ if asked is not None and str(asked).strip() != "":+ try:+ return max(0, min(19, int(str(asked).strip())))+ except ValueError:+ return 0+ ar = archs if archs is not None else hip_gpu_archs()+ return 5 if ar and any(a in HIP_IGPU_ARCHS for a in ar.values()) else 0+++def hip_gpu_archs(root: str = "/sys/class/kfd/kfd/topology/nodes") -> dict[int, str] | None:+ """AMD on Linux: {HIP ordinal: "gfx1151"} from the KFD topology (GPU nodes in node order, as hip_speed_scores and setup number them).+ None when the files are not there (Windows, no ROCm driver)."""+ try:+ nodes = sorted(int(n) for n in os.listdir(root) if n.isdigit())+ except OSError:+ return None+ out: dict[int, str] = {}+ for n in nodes:+ try:+ props = dict(line.split() for line in open(os.path.join(root, str(n), "properties")) if len(line.split()) == 2)+ ver, simd = int(props.get("gfx_target_version", 0)), int(props.get("simd_count", 0))+ except (OSError, ValueError):+ continue+ if simd > 0 and ver > 0:+ out[len(out)] = f"gfx{ver // 10000}{(ver // 100) % 100:x}{ver % 100:x}"+ return out or None+++def igpu_cards(indices: list[int], archs: dict[int, str] | None) -> set[int]:+ """The cards of `indices` that are integrated (their memory is the system RAM, and they usually draw the desktop)."""+ return {i for i in indices if archs and archs.get(i) in HIP_IGPU_ARCHS}++def gpu_vram_mib(indices: list[int], hip: bool = False) -> dict[int, int] | None:"""Total VRAM of each card in MiB (NVML / amdgpu sysfs); None when any of them cannot be read."""try:@@ -3163,7 +3210,7 @@ def gpu_vram_mib(indices: list[int], hip: bool = False) -> dict[int, int] | Nonereturn out-def ordered_gpus(cfg: dict, scores=None, vrams=None) -> list[int]:+def ordered_gpus(cfg: dict, scores=None, vrams=None, archs=None) -> list[int]:"""#1352: the cards of a layer split in the order the engine stages them. With "layer_split": "auto" (thedefault) the faster card goes LAST - the last stage runs the head, the draft layer and the verify, and a promptchunk waits on it (the reporter's 4070 Ti SUPER + 5060 Ti: a 6K prompt took 50 s one way round and 15 s the@@ -3180,6 +3227,24 @@ def ordered_gpus(cfg: dict, scores=None, vrams=None) -> list[int]:if "--peer-device" in args or "--layer-split" in args or layer_split_value(cfg) != "auto":return glhip = cfg.get("backend") == "hip"+ if hip:+ # An integrated Radeon (Strix Halo beside an R9700) is never the LAST stage: that stage runs the head, the draft layer and the+ # check of the guesses, and the iGPU usually draws the desktop. Its "VRAM" is the BIOS carve-out (up to 96 GB), which made the+ # #1576 memory guard below keep it last. It goes first, the cards with memory of their own follow in speed order.+ # archs: {index: "gfx1151"} (tests); None asks the KFD topology, unless scores were injected (a test seam).+ ar = archs if archs is not None else (hip_gpu_archs() if scores is None else None)+ ig = igpu_cards(gl, ar)+ if ig and len(ig) < len(gl):+ sc0 = scores if scores is not None else hip_speed_scores(gl)+ rest = [i for i in gl if i not in ig]+ if sc0 and all(i in sc0 for i in rest):+ rest = sorted(rest, key=lambda i: sc0[i]) if max(sc0[i] for i in rest) >= 1.05 * min(sc0[i] for i in rest) else rest+ out = [i for i in gl if i in ig] + rest+ if out != gl:+ print(f"[strata] layer split: card order {','.join(map(str, out))} (GPU {','.join(str(i) for i in sorted(ig))} is built-in "+ f"graphics: it takes the first layers, never the last stage; \"gpu_order\": \"as_given\" keeps {','.join(map(str, gl))})",+ flush=True)+ return outsc = scores if scores is not None else (hip_speed_scores(gl) if hip else gpu_speed_scores(gl))if not sc or any(i not in sc for i in gl):return gl@@ -3361,6 +3426,23 @@ def hip_env_guard(env: dict, cfg: dict) -> list[str]:return said+def igpu_split_warning(cfg: dict, archs: dict[int, str] | None = None) -> str | None:+ """A plain line when the layers are split over built-in Radeon graphics AND a card with memory of its own. A recommendation, not a+ refusal: the split runs as asked. The iGPU's memory is the RAM, it usually draws the desktop, and compute on it can freeze the screen."""+ if cfg.get("backend") != "hip":+ return None+ gl = gpu_list(cfg)+ if len(gl) < 2:+ return None+ ar = archs if archs is not None else hip_gpu_archs()+ ig = igpu_cards(gl, ar)+ if not ig or len(ig) == len(gl):+ return None+ return ("[strata] note: GPU " + ", ".join(str(i) for i in sorted(ig)) + " is built-in graphics (its memory is the system RAM, and it "+ "usually draws the desktop). Running part of the model on it can freeze the screen and uses RAM twice; the cards with memory of "+ "their own are faster alone (docs/STRIX_HALO.md). Turn it Off in the card roles, or name only the other card, to leave it out")++def card_note(cfg: dict, env: dict) -> str | None:"""One plain line that ties the engine's own numbering to the person's: the engine counts the cards it is given from 0 ("GPU 0","CUDA0" in its log), so a person who chose GPU 1 would read "GPU 0" and think the choice was lost. Said only when the two differ@@ -3377,11 +3459,32 @@ def card_note(cfg: dict, env: dict) -> str | None:f"{', '.join(names)} - the first runs the first layers and the picture reader, the last runs the head and the draft layer")-def vision_env(cfg: dict, env: dict) -> dict:+def vision_card_off_igpu(env: dict, archs: dict[int, str] | None = None) -> int | None:+ """AMD: the picture reader is a separate program that sees the engine's HIP_VISIBLE_DEVICES and runs on the FIRST visible card. With a+ layer split that starts on a built-in Radeon (Strix Halo beside an R9700) that is the iGPU: the card that draws the desktop and whose+ memory is the RAM the experts live in. Returns the HIP ordinal of the last visible card with memory of its own (the Main stage), or+ None when nothing needs changing (one card, no integrated chip among them, or the topology is unknown)."""+ vis = [x.strip() for x in str(env.get("HIP_VISIBLE_DEVICES") or "").split(",") if x.strip().isdigit()]+ if len(vis) < 2:+ return None+ idx = [int(x) for x in vis]+ ar = archs if archs is not None else hip_gpu_archs()+ ig = igpu_cards(idx, ar)+ own = [i for i in idx if i not in ig and ar and i in ar]+ if not ig or not own:+ return None+ print(f"[strata] the picture reader runs on GPU {own[-1]}, not on the built-in graphics (GPU {','.join(str(i) for i in sorted(ig))}) "+ "that draws the desktop; \"vision\": {\"cuda_device\": N} chooses another card", flush=True)+ return own[-1]+++def vision_env(cfg: dict, env: dict, archs: dict[int, str] | None = None) -> dict:"""The image encoder's environment: the engine's, unless the config's vision section names its own "cuda_device"(numbered like nvidia-smi) - then the encoder runs on that card alone, so a spare GPU can hold it while the enginekeeps all of its own cards' VRAM (#408, Efs-O). Without it, nothing changes."""dev = (cfg.get("vision") or {}).get("cuda_device")+ if dev is None and cfg.get("backend") == "hip":+ dev = vision_card_off_igpu(env, archs)if dev is None:return envenv = dict(env)@@ -8047,6 +8150,9 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> innote = card_note(cfg, env)if note:print(note, flush=True)+ igpu_line = igpu_split_warning(cfg)+ if igpu_line:+ print(igpu_line, flush=True)if layer_split_of(cfg):try:split = layer_split_value(cfg) # #644: before the (minutes-long) startdiff --git a/serve/test_server.py b/serve/test_server.pyindex 4249585d4..c85643e41 100644--- a/serve/test_server.py+++ b/serve/test_server.py@@ -1772,6 +1772,71 @@ class GpuChoice(unittest.TestCase):self.assertEqual(child_env({"gpu": [0, 2], "args": [], "gpu_order": "as_given"})["CUDA_VISIBLE_DEVICES"], "0,2")self.assertIsNone(hip_speed_scores([0], root="/nonexistent/kfd"))+ def test_igpu_is_never_the_last_stage(self):+ """Strix Halo (gfx1151, BIOS carve-out up to 96 GB) beside an R9700 (gfx1201, 32 GB): the #1576 memory guard used to keep the+ iGPU LAST (head, draft layer, verify). Built-in graphics goes first, the cards with memory of their own follow."""+ from serve.server import ordered_gpus+ cfg = {"gpu": [0, 1], "backend": "hip", "args": []}+ arch = {0: "gfx1151", 1: "gfx1201"}+ speeds = {0: 80 * 2900.0, 1: 128 * 2350.0}+ self.assertEqual(ordered_gpus(cfg, speeds, {0: 65536, 1: 32768}, arch), [0, 1]) # carve-out 64 GB > 32 GB: still first+ self.assertEqual(ordered_gpus({**cfg, "gpu": [1, 0]}, speeds, {0: 65536, 1: 32768}, arch), [0, 1]) # given R9700 first: iGPU moves first+ self.assertEqual(ordered_gpus({**cfg, "gpu": [1, 0]}, speeds, None, arch), [0, 1])+ self.assertEqual(ordered_gpus({**cfg, "gpu": [1, 0], "gpu_order": "as_given"}, speeds, None, arch), [1, 0]) # the person's order stays+ self.assertEqual(ordered_gpus({**cfg, "gpu": [1, 0], "layer_split": "24"}, speeds, None, arch), [1, 0])+ # two cards of their own plus the iGPU: iGPU first, then the two by speed (faster last)+ three = {"gpu": [1, 0, 2], "backend": "hip", "args": []}+ arch3 = {0: "gfx1151", 1: "gfx1201", 2: "gfx1100"}+ self.assertEqual(ordered_gpus(three, {0: 10.0, 1: 30.0, 2: 20.0}, None, arch3), [0, 2, 1])+ self.assertEqual(ordered_gpus(three, {0: 10.0, 1: 30.0, 2: 30.0}, None, arch3), [0, 1, 2]) # equal cards keep their order+ # no integrated chip, or only integrated chips: the old rule+ self.assertEqual(ordered_gpus({"gpu": [0, 1], "backend": "hip", "args": []}, {0: 1.0, 1: 2.0}, None, {0: "gfx1201", 1: "gfx1201"}), [0, 1])+ self.assertEqual(ordered_gpus({"gpu": [0, 1], "backend": "hip", "args": []}, {0: 2.0, 1: 1.0}, None, {0: "gfx1151", 1: "gfx1151"}), [1, 0])++ def test_igpu_split_warning(self):+ from serve.server import igpu_split_warning+ arch = {0: "gfx1151", 1: "gfx1201"}+ line = igpu_split_warning({"backend": "hip", "gpu": [0, 1]}, arch)+ self.assertIn("GPU 0 is built-in graphics", line)+ self.assertIsNone(igpu_split_warning({"backend": "hip", "gpu": [1]}, arch)) # one card+ self.assertIsNone(igpu_split_warning({"backend": "hip", "gpu": [0]}, arch))+ self.assertIsNone(igpu_split_warning({"backend": "hip", "gpu": [1, 2]}, {1: "gfx1201", 2: "gfx1201"}))+ self.assertIsNone(igpu_split_warning({"backend": "cuda", "gpu": [0, 1]}, arch)) # NVIDIA: not this rule+ self.assertIsNone(igpu_split_warning({"backend": "hip", "gpu": [0, 1]}, {0: "gfx1151", 1: "gfx1103"})) # only integrated chips++ def test_kfd_archs_and_engine_nice(self):+ import tempfile+ from serve.server import engine_nice, hip_gpu_archs+ with tempfile.TemporaryDirectory() as d:+ for n, simd, ver in ((0, 0, 0), (1, 80, 110501), (2, 128, 120001)):+ os.makedirs(os.path.join(d, str(n)))+ with open(os.path.join(d, str(n), "properties"), "w") as f:+ f.write(f"simd_count {simd}\nmax_engine_clk_fcompute 2000\ngfx_target_version {ver}\n")+ self.assertEqual(hip_gpu_archs(root=d), {0: "gfx1151", 1: "gfx1201"})+ self.assertIsNone(hip_gpu_archs(root="/nonexistent/kfd"))+ arch = {0: "gfx1151", 1: "gfx1201"}+ self.assertEqual(engine_nice({}, arch, "linux"), 5) # built-in Radeon on the PC+ self.assertEqual(engine_nice({}, {0: "gfx1201"}, "linux"), 0) # cards only: unchanged+ self.assertEqual(engine_nice({}, arch, "win32"), 0)+ self.assertEqual(engine_nice({"STRATA_ENGINE_NICE": "0"}, arch, "linux"), 0) # the off switch+ self.assertEqual(engine_nice({"STRATA_ENGINE_NICE": "9"}, {0: "gfx1201"}, "linux"), 9)+ self.assertEqual(engine_nice({"STRATA_ENGINE_NICE": "x"}, arch, "linux"), 0)++ def test_picture_reader_not_on_the_igpu(self):+ """The picture reader runs on the first visible card; with a split that starts on the iGPU it must use a card of its own."""+ from serve.server import vision_env+ arch = {0: "gfx1151", 1: "gfx1201"}+ cfg = {"backend": "hip", "gpu": [0, 1], "vision": {"exe": "v"}}+ env = {"HIP_VISIBLE_DEVICES": "0,1", "PATH": "x"}+ self.assertEqual(vision_env(cfg, env, arch)["HIP_VISIBLE_DEVICES"], "1")+ self.assertEqual(env["HIP_VISIBLE_DEVICES"], "0,1") # the engine's own environment is not touched+ self.assertEqual(vision_env({**cfg, "vision": {"exe": "v", "cuda_device": 0}}, env, arch)["HIP_VISIBLE_DEVICES"], "0") # asked: kept+ for e in ({"HIP_VISIBLE_DEVICES": "1"}, {"HIP_VISIBLE_DEVICES": "0"}, {}):+ self.assertIs(vision_env(cfg, e, arch), e) # one card: unchanged+ two = {"HIP_VISIBLE_DEVICES": "0,1"}+ self.assertIs(vision_env(cfg, two, {0: "gfx1201", 1: "gfx1201"}), two) # no integrated chip+ self.assertIs(vision_env(cfg, two, {0: "gfx1151", 1: "gfx1103"}), two) # nothing with memory of its own+def test_kfd_scores(self):"""AMD on Linux: the GPU nodes of the KFD topology, SIMDs x clock, node order = the config's index."""import tempfilediff --git a/setup.py b/setup.pyindex 047b00316..514290d99 100644--- a/setup.py+++ b/setup.py@@ -3764,7 +3764,14 @@ def amd_parse_gpus(text, amd) -> list:architectures: the engine is compiled for each). Returns the cards, the main one first."""usable = [g for g in amd if amd_problem(g) is None]if str(text).strip().lower() == "all":- sel = [g["index"] for g in sorted(usable, key=amd_rank)]+ # "all" means the cards that help: a Strix Halo / APU beside a card with memory of its own adds no memory (its memory is the RAM),+ # draws the desktop, and as a stage it only slows the split down. Named by number (--gpus 1,0) it is still taken as asked.+ own = [g for g in usable if not g.get("uma")]+ sel = [g["index"] for g in sorted(own if len(own) >= 2 else usable, key=amd_rank)]+ left = [g for g in usable if g["index"] not in sel]+ if left and len(own) >= 2:+ say(" --gpus all leaves out " + ", ".join(f"GPU {g['index']} ({g['name']})" for g in left)+ + ": it shares the system memory and draws the desktop. Name it (--gpus ...) to use it anyway")else:try:sel = [int(x) for x in str(text).split(",") if x.strip()]diff --git a/tools/test_setup_amd.py b/tools/test_setup_amd.pyindex bd557b648..f612246a0 100644--- a/tools/test_setup_amd.py+++ b/tools/test_setup_amd.py@@ -136,6 +136,16 @@ class GpuLists(unittest.TestCase):def test_all_is_every_supported_card_most_vram_first(self):self.assertEqual([g["index"] for g in setup.amd_parse_gpus("all", self.AMD)], [1, 3, 0])+ def test_all_leaves_out_the_apu_beside_two_cards(self):+ """Strix Halo beside two discrete cards: "all" is the two cards (the APU shares the RAM and draws the desktop); with ONE card it is both+ (two are needed for a split) and a number list still takes the APU as asked."""+ apu = {"index": 0, "name": "AMD Radeon 8060S", "vram_gb": 64.0, "arch": "gfx1151", "uma": True}+ r9700 = {"index": 1, "name": "AMD Radeon AI PRO R9700", "vram_gb": 32.0, "arch": "gfx1201"}+ r9070 = {"index": 2, "name": "AMD Radeon RX 9070 XT", "vram_gb": 16.0, "arch": "gfx1201"}+ self.assertEqual([g["index"] for g in setup.amd_parse_gpus("all", [apu, r9700, r9070])], [1, 2])+ self.assertEqual([g["index"] for g in setup.amd_parse_gpus("all", [apu, r9700])], [1, 0])+ self.assertEqual([g["index"] for g in setup.amd_parse_gpus("0,1", [apu, r9700, r9070])], [0, 1])+def test_refusals(self):for text in ("1,2", "1,7", "1", "1,1", "x,y"):with self.assertRaises(SystemExit, msg=text):