Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

AMD: built-in Radeon graphics (Strix Halo) is never the last stage of a layer split, the picture reader leaves it, --gpus all leaves it out beside two cards, engine nice +5 on Linux with such a chip, split note

Niko1221034422c4d307
diff --git a/serve/server.py b/serve/server.py
index 969651106..b5b3c1033 100644
--- a/serve/server.py
+++ b/serve/server.py
@@ -1014,6 +1014,12 @@ class StrataEngine:
pin = {}
if cpus and hasattr(os, "sched_setaffinity"): # replicas: this engine's own physical cores (serve/replicas.py)
pin["preexec_fn"] = lambda: os.sched_setaffinity(0, cpus)
+ nice = engine_nice(env) # Linux with built-in Radeon graphics: the desktop keeps its share of the CPU
+ if nice:
+ say_nice = pin.get("preexec_fn")
+ pin["preexec_fn"] = lambda: (say_nice() if say_nice else None, os.nice(nice))[0]
+ print(f"[strata] engine priority: nice +{nice} (this PC has built-in Radeon graphics that usually draw the desktop; "
+ "STRATA_ENGINE_NICE=0 turns it off)", flush=True)
end_leftover_engines(exe) # one an earlier run lost still holds the GPU and the memory
spawned_at = time.monotonic()
try:
@@ -3145,6 +3151,47 @@ def hip_speed_scores(indices: list[int], root: str = "/sys/class/kfd/kfd/topolog
return got if all(i in got for i in indices) else None
+def engine_nice(env: dict | None = None, archs: dict[int, str] | None = None, platform: str | None = None) -> int:
+ """Linux only: the nice value the engine starts with. The engine's CPU pool spins at 100% on every physical core but one and its host
+ thread on another; with the desktop drawn by a built-in Radeon (HIP_IGPU_ARCHS: Strix Halo and the other APUs) the compositor then
+ competes with them for the CPU, which reads as a frozen screen. +5 gives the desktop its share and costs nothing when nothing else
+ runs. STRATA_ENGINE_NICE=N sets it (0 = off, also for any PC without such a chip). Not on Windows / macOS."""
+ if not (platform or sys.platform).startswith("linux"):
+ return 0
+ asked = (env or os.environ).get("STRATA_ENGINE_NICE")
+ if asked is not None and str(asked).strip() != "":
+ try:
+ return max(0, min(19, int(str(asked).strip())))
+ except ValueError:
+ return 0
+ ar = archs if archs is not None else hip_gpu_archs()
+ return 5 if ar and any(a in HIP_IGPU_ARCHS for a in ar.values()) else 0
+
+
+def hip_gpu_archs(root: str = "/sys/class/kfd/kfd/topology/nodes") -> dict[int, str] | None:
+ """AMD on Linux: {HIP ordinal: "gfx1151"} from the KFD topology (GPU nodes in node order, as hip_speed_scores and setup number them).
+ None when the files are not there (Windows, no ROCm driver)."""
+ try:
+ nodes = sorted(int(n) for n in os.listdir(root) if n.isdigit())
+ except OSError:
+ return None
+ out: dict[int, str] = {}
+ for n in nodes:
+ try:
+ props = dict(line.split() for line in open(os.path.join(root, str(n), "properties")) if len(line.split()) == 2)
+ ver, simd = int(props.get("gfx_target_version", 0)), int(props.get("simd_count", 0))
+ except (OSError, ValueError):
+ continue
+ if simd > 0 and ver > 0:
+ out[len(out)] = f"gfx{ver // 10000}{(ver // 100) % 100:x}{ver % 100:x}"
+ return out or None
+
+
+def igpu_cards(indices: list[int], archs: dict[int, str] | None) -> set[int]:
+ """The cards of `indices` that are integrated (their memory is the system RAM, and they usually draw the desktop)."""
+ return {i for i in indices if archs and archs.get(i) in HIP_IGPU_ARCHS}
+
+
def gpu_vram_mib(indices: list[int], hip: bool = False) -> dict[int, int] | None:
"""Total VRAM of each card in MiB (NVML / amdgpu sysfs); None when any of them cannot be read."""
try:
@@ -3163,7 +3210,7 @@ def gpu_vram_mib(indices: list[int], hip: bool = False) -> dict[int, int] | None
return out
-def ordered_gpus(cfg: dict, scores=None, vrams=None) -> list[int]:
+def ordered_gpus(cfg: dict, scores=None, vrams=None, archs=None) -> list[int]:
"""#1352: the cards of a layer split in the order the engine stages them. With "layer_split": "auto" (the
default) the faster card goes LAST - the last stage runs the head, the draft layer and the verify, and a prompt
chunk waits on it (the reporter's 4070 Ti SUPER + 5060 Ti: a 6K prompt took 50 s one way round and 15 s the
@@ -3180,6 +3227,24 @@ def ordered_gpus(cfg: dict, scores=None, vrams=None) -> list[int]:
if "--peer-device" in args or "--layer-split" in args or layer_split_value(cfg) != "auto":
return gl
hip = cfg.get("backend") == "hip"
+ if hip:
+ # An integrated Radeon (Strix Halo beside an R9700) is never the LAST stage: that stage runs the head, the draft layer and the
+ # check of the guesses, and the iGPU usually draws the desktop. Its "VRAM" is the BIOS carve-out (up to 96 GB), which made the
+ # #1576 memory guard below keep it last. It goes first, the cards with memory of their own follow in speed order.
+ # archs: {index: "gfx1151"} (tests); None asks the KFD topology, unless scores were injected (a test seam).
+ ar = archs if archs is not None else (hip_gpu_archs() if scores is None else None)
+ ig = igpu_cards(gl, ar)
+ if ig and len(ig) < len(gl):
+ sc0 = scores if scores is not None else hip_speed_scores(gl)
+ rest = [i for i in gl if i not in ig]
+ if sc0 and all(i in sc0 for i in rest):
+ rest = sorted(rest, key=lambda i: sc0[i]) if max(sc0[i] for i in rest) >= 1.05 * min(sc0[i] for i in rest) else rest
+ out = [i for i in gl if i in ig] + rest
+ if out != gl:
+ print(f"[strata] layer split: card order {','.join(map(str, out))} (GPU {','.join(str(i) for i in sorted(ig))} is built-in "
+ f"graphics: it takes the first layers, never the last stage; \"gpu_order\": \"as_given\" keeps {','.join(map(str, gl))})",
+ flush=True)
+ return out
sc = scores if scores is not None else (hip_speed_scores(gl) if hip else gpu_speed_scores(gl))
if not sc or any(i not in sc for i in gl):
return gl
@@ -3361,6 +3426,23 @@ def hip_env_guard(env: dict, cfg: dict) -> list[str]:
return said
+def igpu_split_warning(cfg: dict, archs: dict[int, str] | None = None) -> str | None:
+ """A plain line when the layers are split over built-in Radeon graphics AND a card with memory of its own. A recommendation, not a
+ refusal: the split runs as asked. The iGPU's memory is the RAM, it usually draws the desktop, and compute on it can freeze the screen."""
+ if cfg.get("backend") != "hip":
+ return None
+ gl = gpu_list(cfg)
+ if len(gl) < 2:
+ return None
+ ar = archs if archs is not None else hip_gpu_archs()
+ ig = igpu_cards(gl, ar)
+ if not ig or len(ig) == len(gl):
+ return None
+ return ("[strata] note: GPU " + ", ".join(str(i) for i in sorted(ig)) + " is built-in graphics (its memory is the system RAM, and it "
+ "usually draws the desktop). Running part of the model on it can freeze the screen and uses RAM twice; the cards with memory of "
+ "their own are faster alone (docs/STRIX_HALO.md). Turn it Off in the card roles, or name only the other card, to leave it out")
+
+
def card_note(cfg: dict, env: dict) -> str | None:
"""One plain line that ties the engine's own numbering to the person's: the engine counts the cards it is given from 0 ("GPU 0",
"CUDA0" in its log), so a person who chose GPU 1 would read "GPU 0" and think the choice was lost. Said only when the two differ
@@ -3377,11 +3459,32 @@ def card_note(cfg: dict, env: dict) -> str | None:
f"{', '.join(names)} - the first runs the first layers and the picture reader, the last runs the head and the draft layer")
-def vision_env(cfg: dict, env: dict) -> dict:
+def vision_card_off_igpu(env: dict, archs: dict[int, str] | None = None) -> int | None:
+ """AMD: the picture reader is a separate program that sees the engine's HIP_VISIBLE_DEVICES and runs on the FIRST visible card. With a
+ layer split that starts on a built-in Radeon (Strix Halo beside an R9700) that is the iGPU: the card that draws the desktop and whose
+ memory is the RAM the experts live in. Returns the HIP ordinal of the last visible card with memory of its own (the Main stage), or
+ None when nothing needs changing (one card, no integrated chip among them, or the topology is unknown)."""
+ vis = [x.strip() for x in str(env.get("HIP_VISIBLE_DEVICES") or "").split(",") if x.strip().isdigit()]
+ if len(vis) < 2:
+ return None
+ idx = [int(x) for x in vis]
+ ar = archs if archs is not None else hip_gpu_archs()
+ ig = igpu_cards(idx, ar)
+ own = [i for i in idx if i not in ig and ar and i in ar]
+ if not ig or not own:
+ return None
+ print(f"[strata] the picture reader runs on GPU {own[-1]}, not on the built-in graphics (GPU {','.join(str(i) for i in sorted(ig))}) "
+ "that draws the desktop; \"vision\": {\"cuda_device\": N} chooses another card", flush=True)
+ return own[-1]
+
+
+def vision_env(cfg: dict, env: dict, archs: dict[int, str] | None = None) -> dict:
"""The image encoder's environment: the engine's, unless the config's vision section names its own "cuda_device"
(numbered like nvidia-smi) - then the encoder runs on that card alone, so a spare GPU can hold it while the engine
keeps all of its own cards' VRAM (#408, Efs-O). Without it, nothing changes."""
dev = (cfg.get("vision") or {}).get("cuda_device")
+ if dev is None and cfg.get("backend") == "hip":
+ dev = vision_card_off_igpu(env, archs)
if dev is None:
return env
env = dict(env)
@@ -8047,6 +8150,9 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> in
note = card_note(cfg, env)
if note:
print(note, flush=True)
+ igpu_line = igpu_split_warning(cfg)
+ if igpu_line:
+ print(igpu_line, flush=True)
if layer_split_of(cfg):
try:
split = layer_split_value(cfg) # #644: before the (minutes-long) start
diff --git a/serve/test_server.py b/serve/test_server.py
index 4249585d4..c85643e41 100644
--- a/serve/test_server.py
+++ b/serve/test_server.py
@@ -1772,6 +1772,71 @@ class GpuChoice(unittest.TestCase):
self.assertEqual(child_env({"gpu": [0, 2], "args": [], "gpu_order": "as_given"})["CUDA_VISIBLE_DEVICES"], "0,2")
self.assertIsNone(hip_speed_scores([0], root="/nonexistent/kfd"))
+ def test_igpu_is_never_the_last_stage(self):
+ """Strix Halo (gfx1151, BIOS carve-out up to 96 GB) beside an R9700 (gfx1201, 32 GB): the #1576 memory guard used to keep the
+ iGPU LAST (head, draft layer, verify). Built-in graphics goes first, the cards with memory of their own follow."""
+ from serve.server import ordered_gpus
+ cfg = {"gpu": [0, 1], "backend": "hip", "args": []}
+ arch = {0: "gfx1151", 1: "gfx1201"}
+ speeds = {0: 80 * 2900.0, 1: 128 * 2350.0}
+ self.assertEqual(ordered_gpus(cfg, speeds, {0: 65536, 1: 32768}, arch), [0, 1]) # carve-out 64 GB > 32 GB: still first
+ self.assertEqual(ordered_gpus({**cfg, "gpu": [1, 0]}, speeds, {0: 65536, 1: 32768}, arch), [0, 1]) # given R9700 first: iGPU moves first
+ self.assertEqual(ordered_gpus({**cfg, "gpu": [1, 0]}, speeds, None, arch), [0, 1])
+ self.assertEqual(ordered_gpus({**cfg, "gpu": [1, 0], "gpu_order": "as_given"}, speeds, None, arch), [1, 0]) # the person's order stays
+ self.assertEqual(ordered_gpus({**cfg, "gpu": [1, 0], "layer_split": "24"}, speeds, None, arch), [1, 0])
+ # two cards of their own plus the iGPU: iGPU first, then the two by speed (faster last)
+ three = {"gpu": [1, 0, 2], "backend": "hip", "args": []}
+ arch3 = {0: "gfx1151", 1: "gfx1201", 2: "gfx1100"}
+ self.assertEqual(ordered_gpus(three, {0: 10.0, 1: 30.0, 2: 20.0}, None, arch3), [0, 2, 1])
+ self.assertEqual(ordered_gpus(three, {0: 10.0, 1: 30.0, 2: 30.0}, None, arch3), [0, 1, 2]) # equal cards keep their order
+ # no integrated chip, or only integrated chips: the old rule
+ self.assertEqual(ordered_gpus({"gpu": [0, 1], "backend": "hip", "args": []}, {0: 1.0, 1: 2.0}, None, {0: "gfx1201", 1: "gfx1201"}), [0, 1])
+ self.assertEqual(ordered_gpus({"gpu": [0, 1], "backend": "hip", "args": []}, {0: 2.0, 1: 1.0}, None, {0: "gfx1151", 1: "gfx1151"}), [1, 0])
+
+ def test_igpu_split_warning(self):
+ from serve.server import igpu_split_warning
+ arch = {0: "gfx1151", 1: "gfx1201"}
+ line = igpu_split_warning({"backend": "hip", "gpu": [0, 1]}, arch)
+ self.assertIn("GPU 0 is built-in graphics", line)
+ self.assertIsNone(igpu_split_warning({"backend": "hip", "gpu": [1]}, arch)) # one card
+ self.assertIsNone(igpu_split_warning({"backend": "hip", "gpu": [0]}, arch))
+ self.assertIsNone(igpu_split_warning({"backend": "hip", "gpu": [1, 2]}, {1: "gfx1201", 2: "gfx1201"}))
+ self.assertIsNone(igpu_split_warning({"backend": "cuda", "gpu": [0, 1]}, arch)) # NVIDIA: not this rule
+ self.assertIsNone(igpu_split_warning({"backend": "hip", "gpu": [0, 1]}, {0: "gfx1151", 1: "gfx1103"})) # only integrated chips
+
+ def test_kfd_archs_and_engine_nice(self):
+ import tempfile
+ from serve.server import engine_nice, hip_gpu_archs
+ with tempfile.TemporaryDirectory() as d:
+ for n, simd, ver in ((0, 0, 0), (1, 80, 110501), (2, 128, 120001)):
+ os.makedirs(os.path.join(d, str(n)))
+ with open(os.path.join(d, str(n), "properties"), "w") as f:
+ f.write(f"simd_count {simd}\nmax_engine_clk_fcompute 2000\ngfx_target_version {ver}\n")
+ self.assertEqual(hip_gpu_archs(root=d), {0: "gfx1151", 1: "gfx1201"})
+ self.assertIsNone(hip_gpu_archs(root="/nonexistent/kfd"))
+ arch = {0: "gfx1151", 1: "gfx1201"}
+ self.assertEqual(engine_nice({}, arch, "linux"), 5) # built-in Radeon on the PC
+ self.assertEqual(engine_nice({}, {0: "gfx1201"}, "linux"), 0) # cards only: unchanged
+ self.assertEqual(engine_nice({}, arch, "win32"), 0)
+ self.assertEqual(engine_nice({"STRATA_ENGINE_NICE": "0"}, arch, "linux"), 0) # the off switch
+ self.assertEqual(engine_nice({"STRATA_ENGINE_NICE": "9"}, {0: "gfx1201"}, "linux"), 9)
+ self.assertEqual(engine_nice({"STRATA_ENGINE_NICE": "x"}, arch, "linux"), 0)
+
+ def test_picture_reader_not_on_the_igpu(self):
+ """The picture reader runs on the first visible card; with a split that starts on the iGPU it must use a card of its own."""
+ from serve.server import vision_env
+ arch = {0: "gfx1151", 1: "gfx1201"}
+ cfg = {"backend": "hip", "gpu": [0, 1], "vision": {"exe": "v"}}
+ env = {"HIP_VISIBLE_DEVICES": "0,1", "PATH": "x"}
+ self.assertEqual(vision_env(cfg, env, arch)["HIP_VISIBLE_DEVICES"], "1")
+ self.assertEqual(env["HIP_VISIBLE_DEVICES"], "0,1") # the engine's own environment is not touched
+ self.assertEqual(vision_env({**cfg, "vision": {"exe": "v", "cuda_device": 0}}, env, arch)["HIP_VISIBLE_DEVICES"], "0") # asked: kept
+ for e in ({"HIP_VISIBLE_DEVICES": "1"}, {"HIP_VISIBLE_DEVICES": "0"}, {}):
+ self.assertIs(vision_env(cfg, e, arch), e) # one card: unchanged
+ two = {"HIP_VISIBLE_DEVICES": "0,1"}
+ self.assertIs(vision_env(cfg, two, {0: "gfx1201", 1: "gfx1201"}), two) # no integrated chip
+ self.assertIs(vision_env(cfg, two, {0: "gfx1151", 1: "gfx1103"}), two) # nothing with memory of its own
+
def test_kfd_scores(self):
"""AMD on Linux: the GPU nodes of the KFD topology, SIMDs x clock, node order = the config's index."""
import tempfile
diff --git a/setup.py b/setup.py
index 047b00316..514290d99 100644
--- a/setup.py
+++ b/setup.py
@@ -3764,7 +3764,14 @@ def amd_parse_gpus(text, amd) -> list:
architectures: the engine is compiled for each). Returns the cards, the main one first."""
usable = [g for g in amd if amd_problem(g) is None]
if str(text).strip().lower() == "all":
- sel = [g["index"] for g in sorted(usable, key=amd_rank)]
+ # "all" means the cards that help: a Strix Halo / APU beside a card with memory of its own adds no memory (its memory is the RAM),
+ # draws the desktop, and as a stage it only slows the split down. Named by number (--gpus 1,0) it is still taken as asked.
+ own = [g for g in usable if not g.get("uma")]
+ sel = [g["index"] for g in sorted(own if len(own) >= 2 else usable, key=amd_rank)]
+ left = [g for g in usable if g["index"] not in sel]
+ if left and len(own) >= 2:
+ say(" --gpus all leaves out " + ", ".join(f"GPU {g['index']} ({g['name']})" for g in left)
+ + ": it shares the system memory and draws the desktop. Name it (--gpus ...) to use it anyway")
else:
try:
sel = [int(x) for x in str(text).split(",") if x.strip()]
diff --git a/tools/test_setup_amd.py b/tools/test_setup_amd.py
index bd557b648..f612246a0 100644
--- a/tools/test_setup_amd.py
+++ b/tools/test_setup_amd.py
@@ -136,6 +136,16 @@ class GpuLists(unittest.TestCase):
def test_all_is_every_supported_card_most_vram_first(self):
self.assertEqual([g["index"] for g in setup.amd_parse_gpus("all", self.AMD)], [1, 3, 0])
+ def test_all_leaves_out_the_apu_beside_two_cards(self):
+ """Strix Halo beside two discrete cards: "all" is the two cards (the APU shares the RAM and draws the desktop); with ONE card it is both
+ (two are needed for a split) and a number list still takes the APU as asked."""
+ apu = {"index": 0, "name": "AMD Radeon 8060S", "vram_gb": 64.0, "arch": "gfx1151", "uma": True}
+ r9700 = {"index": 1, "name": "AMD Radeon AI PRO R9700", "vram_gb": 32.0, "arch": "gfx1201"}
+ r9070 = {"index": 2, "name": "AMD Radeon RX 9070 XT", "vram_gb": 16.0, "arch": "gfx1201"}
+ self.assertEqual([g["index"] for g in setup.amd_parse_gpus("all", [apu, r9700, r9070])], [1, 2])
+ self.assertEqual([g["index"] for g in setup.amd_parse_gpus("all", [apu, r9700])], [1, 0])
+ self.assertEqual([g["index"] for g in setup.amd_parse_gpus("0,1", [apu, r9700, r9070])], [0, 1])
+
def test_refusals(self):
for text in ("1,2", "1,7", "1", "1,1", "x,y"):
with self.assertRaises(SystemExit, msg=text):