Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Server: gpu_roles (Main last, Off excluded, hand-edited gpu wins), roles log line, every-card-Off handling
Niko1221b6825a493cb3
diff --git a/serve/server.py b/serve/server.pyindex 1cfadc6af..c212ae249 100644--- a/serve/server.py+++ b/serve/server.py@@ -70,6 +70,7 @@ from strata_control import hipblaslt_tuning # noqa: E402from strata_control import vision_reader # noqa: E402from strata_control import draft_layer # noqa: E402from strata_control import user_args # noqa: E402+from strata_control import gpu_roles # noqa: E402from serve import ready as ready_file # noqa: E402from serve import owner_guard # noqa: E402from serve import launch_identity # noqa: E402@@ -2812,13 +2813,50 @@ class Vision:def gpu_list(cfg: dict) -> list[int]:"""The config's "gpu": one card (2), or several for a layer split ([0, 2] or "0,2"), numbered as nvidia-sminumbers them; [] when it names none."""- g = cfg.get("gpu")+ roles, _ = gpu_roles.saved_roles(cfg) # "gpu_roles" (Main / Helper / Off per card) wins unless "gpu" was edited by hand+ if roles is not None:+ legacy = _gpu_numbers(cfg.get("gpu"))+ return legacy if legacy else gpu_roles.engine_order(roles) # Main LAST; Off cards left out; [] when every card is Off+ return _gpu_numbers(cfg.get("gpu"))+++def _gpu_numbers(g) -> list[int]:if g is None or g == "":return []items = g if isinstance(g, (list, tuple)) else str(g).split(",")return [int(str(x).strip()) for x in items if str(x).strip() != ""]+def gpu_roles_active(cfg: dict) -> bool:+ """True when the run settings carry usable "gpu_roles": the card order is the person's (Main last), never re-sorted by speed."""+ return gpu_roles.saved_roles(cfg)[0] is not None+++def gpu_all_off(cfg: dict) -> bool:+ """Every card is set to Off ("gpu_roles" with no card on, or "gpu_none": true)."""+ roles, _ = gpu_roles.saved_roles(cfg)+ return roles is not None and not any(r != "off" for r in roles.values())+++def card_roles_lines(cfg: dict) -> list[str]:+ """The plain lines printed at start: the roles (cards numbered as nvidia-smi numbers them), a hand-edited "gpu" that beat them, and the+ all-Off case. SystemExit when every card is Off and the engine cannot start without one (the CUDA, HIP and SYCL engines)."""+ roles, problem = gpu_roles.saved_roles(cfg)+ out = [f"[strata] {problem}"] if problem else []+ if roles is None:+ return out+ if gpu_all_off(cfg):+ if cfg.get("backend") in ("cpu", "metal") or sys.platform == "darwin":+ out.append("[strata] every graphics card is Off: " + gpu_roles.CPU_ONLY_LINE)+ return out+ raise SystemExit("[strata] every graphics card is set to Off, and this engine cannot start without one. Turn a card on "+ "(Main or Helper) in the app or with strata launch --gpu-roles.")+ word = {"main": "Main", "helper": "Helper", "off": "Off"}+ out.append("[strata] card roles: " + ", ".join(f"{word[r]} GPU {i}" for r in ("main", "helper", "off")+ for i in sorted(roles) if roles[i] == r) + " (numbers as nvidia-smi gives them)")+ return out++def engine_binary(exe: str, cfg: dict | None = None) -> str:"""The file that holds the engine's own option strings, for a feature check that reads it. The Intel launcher(sycl/serve/strata-sycl.sh) is a shell script that runs its repo's STRATA_SYCL_BIN (default build-sycl-aot/strata)@@ -3109,7 +3147,7 @@ def ordered_gpus(cfg: dict, scores=None, vrams=None) -> list[int]:draft and verify have a floor the smaller card cannot afford. scores/vrams: {index: value} (tests); None asksthe driver - injected scores without injected vrams means no VRAM check (a test seam is a test seam)."""gl = gpu_list(cfg)- if len(gl) < 2 or cfg.get("gpu_order") == "as_given":+ if len(gl) < 2 or cfg.get("gpu_order") == "as_given" or gpu_roles_active(cfg):return glargs = cfg.get("args") or []if "--peer-device" in args or "--layer-split" in args or layer_split_value(cfg) != "auto":@@ -3189,7 +3227,7 @@ def hip_auto_devices(cfg: dict, env: dict) -> list[int] | None:"""HIP_VISIBLE_DEVICES for a start that names no card (see pick_hip_devices), asked of the engine package's own `strata-device`.None leaves everything as it was: a card the config or the environment names, no probe next to the engine, or a probe that fails.SystemExit with the plain line when no card of this computer can run."""- if cfg.get("backend") != "hip" or gpu_list(cfg) or str(cfg.get("hip_ordinal") if cfg.get("hip_ordinal") is not None else "").strip() \+ if cfg.get("backend") != "hip" or gpu_list(cfg) or gpu_all_off(cfg) or str(cfg.get("hip_ordinal") if cfg.get("hip_ordinal") is not None else "").strip() \or env.get("HIP_VISIBLE_DEVICES") or env.get("ROCR_VISIBLE_DEVICES"):return Noneexe = cfg.get("exe")@@ -3220,7 +3258,10 @@ def child_env(cfg: dict, *, offline: bool = False) -> dict:"""The engine's environment: the CUDA libraries setup installed (pip's nvidia packages, or the toolkit thatcompiled it) first on the library search path."""env = dict(os.environ)- if hip_visible(cfg) and cfg.get("backend") == "hip": # AMD: numbered as HIP numbers them (hip_visible)+ if gpu_all_off(cfg): # every card Off: the engine sees none (a CPU engine runs; the others say so at start)+ env["CUDA_VISIBLE_DEVICES"] = ""+ env["HIP_VISIBLE_DEVICES"] = ""+ elif hip_visible(cfg) and cfg.get("backend") == "hip": # AMD: numbered as HIP numbers them (hip_visible)env["HIP_VISIBLE_DEVICES"] = ",".join(str(i) for i in hip_visible(cfg))elif gpu_list(cfg): # issue #51: the GPU(s) to run on, numbered as nvidia-smi does; CUDA'senv["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" # own default order (fastest first) can number the cards otherwise@@ -7974,6 +8015,8 @@ def main(argv=None, *, lifecycle_event=None, interrupt_exit_code: int = 0) -> inprint("model unloaded; the first request loads it ..." if lazy else"loading the model (the first start takes a minute or two) ...", flush=True)warn_budget_over_ram(engine_args(cfg) if "args" in cfg else []) # #1080+ for line in card_roles_lines(cfg):+ print(line, flush=True)note = card_note(cfg, env)if note:print(note, flush=True)diff --git a/serve/test_gpu_roles_server.py b/serve/test_gpu_roles_server.pynew file mode 100644index 000000000..ef2add116--- /dev/null+++ b/serve/test_gpu_roles_server.py@@ -0,0 +1,63 @@+"""serve/server.py: the run settings' "gpu_roles" (Main / Helper / Off) become the engine's card list, Main LAST, Off left out."""+from __future__ import annotations++import sys+import unittest+from pathlib import Path++sys.path.insert(0, str(Path(__file__).resolve().parents[1]))+from serve.server import card_roles_lines, child_env, gpu_list, ordered_gpus # noqa: E402+++class Roles(unittest.TestCase):+ def test_roles_alone_give_the_order_main_last_and_off_left_out(self):+ cfg = {"gpu_roles": {"0": "helper", "1": "main", "2": "off"}, "args": []}+ self.assertEqual(gpu_list(cfg), [0, 1])+ self.assertEqual(gpu_list({"gpu_roles": {"0": "main", "1": "helper"}, "args": []}), [1, 0])+ env = child_env({"gpu_roles": {"0": "main", "1": "helper", "2": "off"}, "args": []})+ self.assertEqual(env["CUDA_VISIBLE_DEVICES"], "1,0")++ def test_the_mirror_is_used_and_never_re_sorted_by_speed(self):+ cfg = {"gpu_roles": {"0": "main", "1": "helper"}, "gpu": [1, 0], "gpu_order": "as_given", "args": []}+ self.assertEqual(ordered_gpus(cfg, {0: 1.0, 1: 9.0}), [1, 0]) # the faster card is Helper on purpose+ cfg2 = {"gpu_roles": {"0": "main", "1": "helper"}, "gpu": [1, 0], "args": []} # even without gpu_order+ self.assertEqual(ordered_gpus(cfg2, {0: 1.0, 1: 9.0}), [1, 0])++ def test_one_card_on(self):+ cfg = {"gpu_roles": {"0": "off", "1": "main"}, "gpu": 1, "gpu_order": "as_given", "args": []}+ self.assertEqual(gpu_list(cfg), [1])+ self.assertEqual(child_env(cfg)["CUDA_VISIBLE_DEVICES"], "1")++ def test_a_hand_edited_gpu_wins(self):+ cfg = {"gpu_roles": {"0": "helper", "1": "main"}, "gpu": [1, 0], "gpu_order": "as_given", "args": []}+ self.assertEqual(gpu_list(cfg), [1, 0])+ self.assertTrue(any("edited by hand" in l for l in card_roles_lines(cfg)))+ self.assertEqual(ordered_gpus({**cfg, "gpu_order": None}, {0: 1.0, 1: 9.0}), [0, 1]) # the legacy rule again++ def test_bad_roles_are_ignored(self):+ for bad in ({"0": "boss"}, {"0": "main", "1": "main"}, "x", {}):+ self.assertEqual(gpu_list({"gpu_roles": bad, "gpu": 1, "args": []}), [1])++ def test_legacy_settings_are_untouched(self):+ self.assertEqual(gpu_list({"gpu": "0,2", "args": []}), [0, 2])+ self.assertEqual(gpu_list({"args": []}), [])+ self.assertEqual(card_roles_lines({"gpu": [1, 0], "args": []}), [])++ def test_the_log_line_names_every_role(self):+ lines = card_roles_lines({"gpu_roles": {"0": "helper", "1": "main", "2": "off"}, "gpu": [0, 1], "args": []})+ self.assertEqual(lines, ["[strata] card roles: Main GPU 1, Helper GPU 0, Off GPU 2 (numbers as nvidia-smi gives them)"])++ def test_every_card_off(self):+ cfg = {"gpu_roles": {"0": "off", "1": "off"}, "gpu": [], "gpu_none": True, "args": []}+ self.assertEqual(gpu_list(cfg), [])+ env = child_env(cfg)+ self.assertEqual((env["CUDA_VISIBLE_DEVICES"], env["HIP_VISIBLE_DEVICES"]), ("", ""))+ with self.assertRaises(SystemExit) as e:+ card_roles_lines(cfg) # a CUDA engine cannot start without a card: said plainly+ self.assertIn("Turn a card on", str(e.exception))+ cpu = card_roles_lines({**cfg, "backend": "cpu"})+ self.assertIn("Performance is degraded on CPU only.", cpu[0])+++if __name__ == "__main__":+ unittest.main()