Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Server: gpu_roles (Main last, Off excluded, hand-edited gpu wins), roles log line, every-card-Off handling

Niko1221b6825a493cb3
diff --git a/serve/server.py b/serve/server.py
index 1cfadc6af..c212ae249 100644
--- a/serve/server.py
+++ b/serve/server.py
@@ -70,6 +70,7 @@ from strata_control import hipblaslt_tuning # noqa: E402
from strata_control import vision_reader # noqa: E402
from strata_control import draft_layer # noqa: E402
from strata_control import user_args # noqa: E402
+from strata_control import gpu_roles # noqa: E402
from serve import ready as ready_file # noqa: E402
from serve import owner_guard # noqa: E402
from serve import launch_identity # noqa: E402
@@ -2812,13 +2813,50 @@ class Vision:
def gpu_list(cfg: dict) -> list[int]:
"""The config's "gpu": one card (2), or several for a layer split ([0, 2] or "0,2"), numbered as nvidia-smi
numbers them; [] when it names none."""
- g = cfg.get("gpu")
+ roles, _ = gpu_roles.saved_roles(cfg) # "gpu_roles" (Main / Helper / Off per card) wins unless "gpu" was edited by hand
+ if roles is not None:
+ legacy = _gpu_numbers(cfg.get("gpu"))
+ return legacy if legacy else gpu_roles.engine_order(roles) # Main LAST; Off cards left out; [] when every card is Off
+ return _gpu_numbers(cfg.get("gpu"))
+
+
+def _gpu_numbers(g) -> list[int]:
if g is None or g == "":
return []
items = g if isinstance(g, (list, tuple)) else str(g).split(",")
return [int(str(x).strip()) for x in items if str(x).strip() != ""]
+def gpu_roles_active(cfg: dict) -> bool:
+ """True when the run settings carry usable "gpu_roles": the card order is the person's (Main last), never re-sorted by speed."""
+ return gpu_roles.saved_roles(cfg)[0] is not None
+
+
+def gpu_all_off(cfg: dict) -> bool:
+ """Every card is set to Off ("gpu_roles" with no card on, or "gpu_none": true)."""
+ roles, _ = gpu_roles.saved_roles(cfg)
+ return roles is not None and not any(r != "off" for r in roles.values())
+
+
+def card_roles_lines(cfg: dict) -> list[str]:
+ """The plain lines printed at start: the roles (cards numbered as nvidia-smi numbers them), a hand-edited "gpu" that beat them, and the
+ all-Off case. SystemExit when every card is Off and the engine cannot start without one (the CUDA, HIP and SYCL engines)."""
+ roles, problem = gpu_roles.saved_roles(cfg)
+ out = [f"[strata] {problem}"] if problem else []
+ if roles is None:
+ return out
+ if gpu_all_off(cfg):
+ if cfg.get("backend") in ("cpu", "metal") or sys.platform == "darwin":
+ out.append("[strata] every graphics card is Off: " + gpu_roles.CPU_ONLY_LINE)
+ return out
+ raise SystemExit("[strata] every graphics card is set to Off, and this engine cannot start without one. Turn a card on "
+ "(Main or Helper) in the app or with strata launch --gpu-roles.")
+ word = {"main": "Main", "helper": "Helper", "off": "Off"}
+ out.append("[strata] card roles: " + ", ".join(f"{word[r]} GPU {i}" for r in ("main", "helper", "off")
+ for i in sorted(roles) if roles[i] == r) + " (numbers as nvidia-smi gives them)")
+ return out
+
+
def engine_binary(exe: str, cfg: dict | None = None) -> str:
"""The file that holds the engine's own option strings, for a feature check that reads it. The Intel launcher
(sycl/serve/strata-sycl.sh) is a shell script that runs its repo's STRATA_SYCL_BIN (default build-sycl-aot/strata)
@@ -3109,7 +3147,7 @@ def ordered_gpus(cfg: dict, scores=None, vrams=None) -> list[int]:
draft and verify have a floor the smaller card cannot afford. scores/vrams: {index: value} (tests); None asks
the driver - injected scores without injected vrams means no VRAM check (a test seam is a test seam)."""
gl = gpu_list(cfg)
- if len(gl) < 2 or cfg.get("gpu_order") == "as_given":
+ if len(gl) < 2 or cfg.get("gpu_order") == "as_given" or gpu_roles_active(cfg):
return gl
args = cfg.get("args") or []
if "--peer-device" in args or "--layer-split" in args or layer_split_value(cfg) != "auto":
@@ -3189,7 +3227,7 @@ def hip_auto_devices(cfg: dict, env: dict) -> list[int] | None:
"""HIP_VISIBLE_DEVICES for a start that names no card (see pick_hip_devices), asked of the engine package's own `strata-device`.
None leaves everything as it was: a card the config or the environment names, no probe next to the engine, or a probe that fails.
SystemExit with the plain line when no card of this computer can run."""
- if cfg.get("backend") != "hip" or gpu_list(cfg) or str(cfg.get("hip_ordinal") if cfg.get("hip_ordinal") is not None else "").strip() \
+ if cfg.get("backend") != "hip" or gpu_list(cfg) or gpu_all_off(cfg) or str(cfg.get("hip_ordinal") if cfg.get("hip_ordinal") is not None else "").strip() \
or env.get("HIP_VISIBLE_DEVICES") or env.get("ROCR_VISIBLE_DEVICES"):
return None
exe = cfg.get("exe")
@@ -3220,7 +3258,10 @@ def child_env(cfg: dict, *, offline: bool = False) -> dict:
"""The engine's environment: the CUDA libraries setup installed (pip's nvidia packages, or the toolkit that
compiled it) first on the library search path."""
env = dict(os.environ)
- if hip_visible(cfg) and cfg.get("backend") == "hip": # AMD: numbered as HIP numbers them (hip_visible)
+ if gpu_all_off(cfg): # every card Off: the engine sees none (a CPU engine runs; the others say so at start)
+ env["CUDA_VISIBLE_DEVICES"] = ""
+ env["HIP_VISIBLE_DEVICES"] = ""
+ elif hip_visible(cfg) and cfg.get("backend") == "hip": # AMD: numbered as HIP numbers them (hip_visible)
env["HIP_VISIBLE_DEVICES"] = ",".join(str(i) for i in hip_visible(cfg))
elif gpu_list(cfg): # issue #51: the GPU(s) to run on, numbered as nvidia-smi does; CUDA's
env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # own default order (fastest first) can number the cards otherwise
@@ -7974,6 +8015,8 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> in
print("model unloaded; the first request loads it ..." if lazy else
"loading the model (the first start takes a minute or two) ...", flush=True)
warn_budget_over_ram(engine_args(cfg) if "args" in cfg else []) # #1080
+ for line in card_roles_lines(cfg):
+ print(line, flush=True)
note = card_note(cfg, env)
if note:
print(note, flush=True)
diff --git a/serve/test_gpu_roles_server.py b/serve/test_gpu_roles_server.py
new file mode 100644
index 000000000..ef2add116
--- /dev/null
+++ b/serve/test_gpu_roles_server.py
@@ -0,0 +1,63 @@
+"""serve/server.py: the run settings' "gpu_roles" (Main / Helper / Off) become the engine's card list, Main LAST, Off left out."""
+from __future__ import annotations
+
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from serve.server import card_roles_lines, child_env, gpu_list, ordered_gpus # noqa: E402
+
+
+class Roles(unittest.TestCase):
+ def test_roles_alone_give_the_order_main_last_and_off_left_out(self):
+ cfg = {"gpu_roles": {"0": "helper", "1": "main", "2": "off"}, "args": []}
+ self.assertEqual(gpu_list(cfg), [0, 1])
+ self.assertEqual(gpu_list({"gpu_roles": {"0": "main", "1": "helper"}, "args": []}), [1, 0])
+ env = child_env({"gpu_roles": {"0": "main", "1": "helper", "2": "off"}, "args": []})
+ self.assertEqual(env["CUDA_VISIBLE_DEVICES"], "1,0")
+
+ def test_the_mirror_is_used_and_never_re_sorted_by_speed(self):
+ cfg = {"gpu_roles": {"0": "main", "1": "helper"}, "gpu": [1, 0], "gpu_order": "as_given", "args": []}
+ self.assertEqual(ordered_gpus(cfg, {0: 1.0, 1: 9.0}), [1, 0]) # the faster card is Helper on purpose
+ cfg2 = {"gpu_roles": {"0": "main", "1": "helper"}, "gpu": [1, 0], "args": []} # even without gpu_order
+ self.assertEqual(ordered_gpus(cfg2, {0: 1.0, 1: 9.0}), [1, 0])
+
+ def test_one_card_on(self):
+ cfg = {"gpu_roles": {"0": "off", "1": "main"}, "gpu": 1, "gpu_order": "as_given", "args": []}
+ self.assertEqual(gpu_list(cfg), [1])
+ self.assertEqual(child_env(cfg)["CUDA_VISIBLE_DEVICES"], "1")
+
+ def test_a_hand_edited_gpu_wins(self):
+ cfg = {"gpu_roles": {"0": "helper", "1": "main"}, "gpu": [1, 0], "gpu_order": "as_given", "args": []}
+ self.assertEqual(gpu_list(cfg), [1, 0])
+ self.assertTrue(any("edited by hand" in l for l in card_roles_lines(cfg)))
+ self.assertEqual(ordered_gpus({**cfg, "gpu_order": None}, {0: 1.0, 1: 9.0}), [0, 1]) # the legacy rule again
+
+ def test_bad_roles_are_ignored(self):
+ for bad in ({"0": "boss"}, {"0": "main", "1": "main"}, "x", {}):
+ self.assertEqual(gpu_list({"gpu_roles": bad, "gpu": 1, "args": []}), [1])
+
+ def test_legacy_settings_are_untouched(self):
+ self.assertEqual(gpu_list({"gpu": "0,2", "args": []}), [0, 2])
+ self.assertEqual(gpu_list({"args": []}), [])
+ self.assertEqual(card_roles_lines({"gpu": [1, 0], "args": []}), [])
+
+ def test_the_log_line_names_every_role(self):
+ lines = card_roles_lines({"gpu_roles": {"0": "helper", "1": "main", "2": "off"}, "gpu": [0, 1], "args": []})
+ self.assertEqual(lines, ["[strata] card roles: Main GPU 1, Helper GPU 0, Off GPU 2 (numbers as nvidia-smi gives them)"])
+
+ def test_every_card_off(self):
+ cfg = {"gpu_roles": {"0": "off", "1": "off"}, "gpu": [], "gpu_none": True, "args": []}
+ self.assertEqual(gpu_list(cfg), [])
+ env = child_env(cfg)
+ self.assertEqual((env["CUDA_VISIBLE_DEVICES"], env["HIP_VISIBLE_DEVICES"]), ("", ""))
+ with self.assertRaises(SystemExit) as e:
+ card_roles_lines(cfg) # a CUDA engine cannot start without a card: said plainly
+ self.assertIn("Turn a card on", str(e.exception))
+ cpu = card_roles_lines({**cfg, "backend": "cpu"})
+ self.assertIn("Performance is degraded on CPU only.", cpu[0])
+
+
+if __name__ == "__main__":
+ unittest.main()