Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Four-card AMD serving profile: 32 slots in 4 groups, split 13,26,39, chunk 2048 (setup and launcher)
Niko12214fdb563a3475
diff --git a/serve/server.py b/serve/server.pyindex 8c5213b1f..633339dee 100644--- a/serve/server.py+++ b/serve/server.py@@ -3201,7 +3201,7 @@ def engine_args(cfg: dict) -> list[str]:return learned_profile_args(cfg, args)-PARALLEL_MAX = 8 # the engine's batch window holds at most 8 rows (kVerifyMaxT); --batch-mtp waves more through it+PARALLEL_MAX = 8 # without a layer split the engine's batch window holds at most 8 rows (kVerifyNarrowT); --batch-mtp waves more through itdef parallel_args(cfg: dict, args: list[str]) -> list[str]:@@ -3219,7 +3219,10 @@ def parallel_args(cfg: dict, args: list[str]) -> list[str]:return []if n <= 1:return []- if n > PARALLEL_MAX and "--batch-mtp" not in args and os.environ.get("STRATA_BATCH_MTP", "0") in ("", "0"):+ # (on a layer split the pipelined groups each hold up to 8 slots - 16 in a HIP engine with wide windows - so more than 8 is+ # not a limit there: the engine says how many slots it runs, INFO batch_slots)+ if (n > PARALLEL_MAX and "--batch-mtp" not in args and os.environ.get("STRATA_BATCH_MTP", "0") in ("", "0")+ and not layer_split_of(cfg)):print(f'[strata] "parallel": {n} - the engine runs at most {PARALLEL_MAX} requests together; it will use 'f"{PARALLEL_MAX}", flush=True)return ["--batch", str(n)]diff --git a/setup.py b/setup.pyindex bf5cd2e69..de1c38886 100644--- a/setup.py+++ b/setup.py@@ -10039,6 +10039,14 @@ def main() -> int:cfg["gpu_order"] = "as_given"ok(f"layer split across GPUs {multi} ({cfg['layer_split']})")recommend_remote_expert_opt(cfg, off=a.no_remote_expert_opt)+ if hip and a.layer_split in (None, "auto"): # four Radeon AI PRO R9700 cards and Qwen3.8 IQ3_S: the measured serving profile+ try: # (strata_control/amd_four_card.py); what the person chose always wins+ from strata_control import amd_four_card+ for line in amd_four_card.apply([{"name": c.get("name"), "vram_gb": c.get("vram_gb"), "vendor": "amd"} for c in chosen],+ cfg, parallel_given=a.parallel is not None):+ ok(line)+ except Exception as e: # noqa: BLE001 - a profile is never a reason to stop setup+ say(f" (the four-card profile was not applied: {e})")if getattr(a, "_roles", None): # the roles themselves (strata_control/gpu_roles.py); "gpu" above is their mirrorfinal = cfg["gpu"] if isinstance(cfg.get("gpu"), list) else [cfg["gpu"]] # what the checks above left (a model that cannot share keeps one card)main = final[-1]diff --git a/strata_cli/launch/runner.py b/strata_cli/launch/runner.pyindex cd323caa6..d109fef17 100644--- a/strata_cli/launch/runner.py+++ b/strata_cli/launch/runner.py@@ -1409,6 +1409,7 @@ class Runner:base = read_config(base_path)cfg = derive_config(base, self.s, bool_flags=tune_knobs.BOOL_FLAGS,log=str(launch_dir() / f"{self.name}.engine.log"), gpus=cards_for_launch(self.s, self.catalog, base=base))+ self.apply_four_card(cfg)self.use_own_dsv41_engine(cfg)apply_experts(cfg, self.experts_flag(cfg.get("exe"))) # after the engine is settled: only a 0.1.50+ engine is given --expertswait = self.start_wait()@@ -1419,10 +1420,29 @@ class Runner:self.config_path = outreturn out+ def apply_four_card(self, cfg: dict) -> None:+ """Four Radeon AI PRO R9700 cards and Qwen3.8-Flash-Next IQ3_S: the measured serving profile (strata_control/amd_four_card.py), filled+ in where the person chose nothing. Never stops a launch."""+ self.four_card_parallel = None+ try:+ from strata_control import amd_four_card+ picked = cfg.get("gpu")+ if not (isinstance(picked, list) and len(picked) == amd_four_card.CARDS):+ return+ want = {int(i) for i in picked}+ cards = [c for c in self.catalog.computer().usable_gpus if int(c.get("index", -1)) in want]+ before = cfg.get("parallel")+ notes = amd_four_card.apply(cards, cfg, parallel_given=bool(self.s.parallel))+ if notes:+ self.four_card_parallel = cfg.get("parallel") if before is None and cfg.get("parallel") else None # (only a number the profile set)+ self.emit("model", "Four " + str(cards[0].get("name", "AMD")).replace("AMD Radeon ", "") + " cards: " + "; ".join(notes) + ".", None)+ except Exception: # noqa: BLE001 - a profile is never a reason to stop a launch+ self.four_card_parallel = None+def ensure_profile(self, config: Path) -> None:from strata_control import profilesname = self.profile- parallel = self.s.parallel if self.s.parallel else self.catalog_parallel()+ parallel = self.s.parallel if self.s.parallel else (getattr(self, "four_card_parallel", None) or self.catalog_parallel())sets = [f"host={acc.host_for(self.s.access)}", f"port={int(self.s.port)}"] + ([f"parallel={parallel}"] if parallel else [])try:current, _rev = profiles.load(name)diff --git a/strata_control/amd_four_card.py b/strata_control/amd_four_card.pynew file mode 100644index 000000000..b040fba06--- /dev/null+++ b/strata_control/amd_four_card.py@@ -0,0 +1,92 @@+"""The serving profile of a four-card AMD box: 4 x Radeon AI PRO R9700 (32 GB, gfx1201) running Qwen3.8-Flash-Next IQ3_S.++Measured on that box (docs/BATCHING.md, "Four R9700 cards: the serving profile"): the cards hold the whole model, so what limits+them is how many requests decode together and how the four stages are fed, not memory. The profile sets++* 32 slots in 4 pipelined groups (one group per card), instead of the launcher's 4 requests at a time;+* the layer split 13,26,39: the output head costs about four layers on the last card, so it gets 9 layers and the others 13;+* a 2048-token prompt chunk: a smaller chunk reads long prompts faster beside the decodes of the other slots;+* the three batch switches STRATA_QSA_SLOTS, STRATA_BATCH_SELF_COMMIT and STRATA_SLOT_COPY_DIRECT, which change how the work is+ scheduled and copied, not what is computed (tools/batch_test.py: every slot decodes the tokens it decodes alone).++Recommend, never force: a setting the person made (the config, `--parallel`, `--layer-split`, an advanced flag, an environment+variable) always wins, the profile only fills what is not set, and `STRATA_FOUR_CARD_PROFILE=0` (or "four_card_profile": false in+the run config) turns it off. Only this hardware class is touched; every other computer gets exactly what it got before.++Pure functions, no I/O (the callers pass the cards and the run config).+"""++from __future__ import annotations++import os++PARALLEL = 32 # slots (requests decoding together)+GROUPS = 4 # pipelined groups: one per card+LAYER_SPLIT = "13,26,39" # Qwen3.8-Flash-Next has 48 layers: 13 / 13 / 13 / 9 (the output head sits on the last card)+PREFILL_CHUNK = "2048"+BATCH_ENV = {"STRATA_QSA_SLOTS": "1", "STRATA_BATCH_SELF_COMMIT": "1", "STRATA_SLOT_COPY_DIRECT": "1"}+PROMPT_CACHE_OFF = False # --prompt-cache 0 is not part of the default: see docs/BATCHING.md+MIN_CARD_GB = 28.0 # a 32 GB card reports a little less than its name+CARDS = 4+OFF_VALUES = ("0", "off", "no", "false")+++def disabled(cfg: dict | None = None) -> bool:+ """The person turned the profile off: STRATA_FOUR_CARD_PROFILE=0 in the environment, or "four_card_profile": false in the config."""+ if str(os.environ.get("STRATA_FOUR_CARD_PROFILE", "")).strip().lower() in OFF_VALUES:+ return True+ return bool(cfg) and cfg.get("four_card_profile") is False+++def card_class(cards: list[dict]) -> bool:+ """Exactly four AMD cards of one kind, each with at least MIN_CARD_GB: the Radeon AI PRO R9700 (the 32 GB gfx12 card this was+ measured on)."""+ if len(cards) != CARDS:+ return False+ names = {str(c.get("name") or "") for c in cards}+ if len(names) != 1 or "R9700" not in next(iter(names)).upper().replace(" ", ""):+ return False+ return all(str(c.get("vendor") or "amd").lower() == "amd" and float(c.get("vram_gb") or 0.0) >= MIN_CARD_GB for c in cards)+++def model_ok(cfg: dict) -> bool:+ """The measured model: Qwen3.8-Flash-Next at IQ3_S (48 layers; the pack that fits whole on four 32 GB cards)."""+ text = " ".join([str(cfg.get("model_name") or "")] + [str(a) for a in cfg.get("args") or []]).lower().replace("-", "_").replace(".", "_")+ return "qwen3_8" in text and "iq3_s" in text+++def applies(cards: list[dict], cfg: dict) -> bool:+ args = [str(a) for a in cfg.get("args") or []]+ return (not disabled(cfg)) and card_class(cards) and cfg.get("backend", "hip") == "hip" and model_ok(cfg) \+ and "--peer-device" not in args and not cfg.get("replicas") and cfg.get("arch") != "deepseek41"+++def apply(cards: list[dict], cfg: dict, *, parallel_given: bool = False) -> list[str]:+ """Fill the profile into the run config `cfg` (in place) where the person has not chosen; returns the plain lines to show.+ `parallel_given`: the person named a number of requests (--parallel): then only the split, the chunk and the switches are+ filled (the engine groups the slots itself)."""+ if not applies(cards, cfg):+ return []+ args = [str(a) for a in cfg.get("args") or []]+ env = dict(cfg.get("env") or {})+ notes: list[str] = []+ if str(cfg.get("layer_split") or "auto") == "auto" and "--layer-split" not in args:+ cfg["layer_split"] = LAYER_SPLIT+ notes.append(f"layer split {LAYER_SPLIT}: the last card also holds the output head, so it takes fewer layers")+ if "--prefill" not in args:+ args.extend(["--prefill", PREFILL_CHUNK])+ elif args[args.index("--prefill") + 1:args.index("--prefill") + 2] == ["auto"]: # "auto" is what setup writes, not a choice+ args[args.index("--prefill") + 1] = PREFILL_CHUNK+ if PROMPT_CACHE_OFF and "--prompt-cache" not in args:+ args.extend(["--prompt-cache", "0"])+ for k, v in BATCH_ENV.items():+ if k not in os.environ: # an environment variable of the person's own wins+ env.setdefault(k, v)+ if not parallel_given and "parallel" not in cfg and not {"--batch", "--slots"} & set(args):+ cfg["parallel"] = PARALLEL+ if "--batch-groups" not in args:+ args.extend(["--batch-groups", str(GROUPS)])+ notes.append(f"{PARALLEL} requests at once in {GROUPS} groups, one group per card (--parallel N changes it)")+ cfg["args"] = args+ cfg["env"] = env+ return notesdiff --git a/tests/terminal/test_amd_four_card.py b/tests/terminal/test_amd_four_card.pynew file mode 100644index 000000000..d313fc0c0--- /dev/null+++ b/tests/terminal/test_amd_four_card.py@@ -0,0 +1,158 @@+"""strata_control/amd_four_card.py: the serving profile of four Radeon AI PRO R9700 cards, and the places that apply it."""++from __future__ import annotations++import os+import unittest+from unittest import mock++from strata_control import amd_four_card as F++R9700 = {"name": "AMD Radeon AI PRO R9700", "vram_gb": 31.9, "vendor": "amd", "usable": True}+CARDS = [{**R9700, "index": i} for i in range(4)]+CFG = {"model_name": "qwen3.8-flash-next-iq3_s", "backend": "hip", "gpu": [0, 1, 2, 3], "layer_split": "auto",+ "args": ["--pack", "/p/iq3_s", "--spec", "4", "--max-context", "32768"]}+++def fresh(**kw):+ c = {**CFG, "args": list(CFG["args"]), **kw}+ return c+++class Class(unittest.TestCase):+ def test_four_r9700_are_the_class(self):+ self.assertTrue(F.card_class(CARDS))++ def test_other_hardware_is_not(self):+ self.assertFalse(F.card_class(CARDS[:3])) # three cards+ self.assertFalse(F.card_class(CARDS + [CARDS[0]])) # five+ self.assertFalse(F.card_class([{**c, "name": "AMD Radeon RX 7900 XTX", "vram_gb": 24.0} for c in CARDS]))+ self.assertFalse(F.card_class([{**c, "vendor": "nvidia"} for c in CARDS]))+ self.assertFalse(F.card_class([{**c, "vram_gb": 16.0} for c in CARDS])) # a 16 GB card with the same name+ self.assertFalse(F.card_class([*CARDS[:3], {**R9700, "name": "AMD Radeon RX 9070"}]))++ def test_only_the_measured_model(self):+ for name in ("qwen3.8-flash-next-iq3_xxs", "qwen3.6-35b-a3b-q4", "deepseek-v4.1-flash-2.00bpw"):+ with self.subTest(name):+ self.assertFalse(F.applies(CARDS, fresh(model_name=name, args=["--pack", "/p/x"])))+ self.assertTrue(F.applies(CARDS, fresh()))+ self.assertFalse(F.applies(CARDS, fresh(backend="cuda")))+ self.assertFalse(F.applies(CARDS, fresh(arch="deepseek41")))+ self.assertFalse(F.applies(CARDS, fresh(args=CFG["args"] + ["--peer-device", "1"])))+++class Apply(unittest.TestCase):+ def test_fills_the_profile_on_a_default_config(self):+ c = fresh()+ with mock.patch.dict(os.environ, {}, clear=False):+ for k in list(F.BATCH_ENV) + ["STRATA_FOUR_CARD_PROFILE"]:+ os.environ.pop(k, None)+ notes = F.apply(CARDS, c)+ self.assertTrue(notes)+ self.assertEqual(c["layer_split"], "13,26,39")+ self.assertEqual(c["parallel"], 32)+ self.assertEqual(c["args"][c["args"].index("--batch-groups") + 1], "4")+ self.assertEqual(c["args"][c["args"].index("--prefill") + 1], "2048")+ self.assertEqual(c["env"], F.BATCH_ENV)+ self.assertNotIn("--prompt-cache", c["args"])++ def test_prefill_auto_from_setup_is_not_a_choice(self):+ c = fresh(args=CFG["args"] + ["--prefill", "auto"])+ F.apply(CARDS, c)+ self.assertEqual(c["args"][c["args"].index("--prefill") + 1], "2048")+ self.assertEqual(c["args"].count("--prefill"), 1)+ c = fresh(args=CFG["args"] + ["--prefill", "auto:32768"])+ F.apply(CARDS, c)+ self.assertEqual(c["args"][c["args"].index("--prefill") + 1], "auto:32768")++ def test_the_persons_choices_win(self):+ c = fresh(layer_split="12,25,37", parallel=8, args=CFG["args"] + ["--prefill", "8192", "--batch-groups", "2"],+ env={"STRATA_QSA_SLOTS": "0"})+ F.apply(CARDS, c)+ self.assertEqual(c["layer_split"], "12,25,37")+ self.assertEqual(c["parallel"], 8)+ self.assertEqual(c["args"][c["args"].index("--prefill") + 1], "8192")+ self.assertEqual(c["args"].count("--batch-groups"), 1)+ self.assertEqual(c["env"]["STRATA_QSA_SLOTS"], "0")++ def test_a_number_of_requests_asked_for_keeps_the_groups_to_the_engine(self):+ c = fresh()+ F.apply(CARDS, c, parallel_given=True)+ self.assertNotIn("parallel", c)+ self.assertNotIn("--batch-groups", c["args"])+ self.assertEqual(c["layer_split"], "13,26,39")++ def test_an_explicit_batch_in_the_args_is_kept(self):+ c = fresh(args=CFG["args"] + ["--batch", "6"])+ F.apply(CARDS, c)+ self.assertNotIn("parallel", c)+ self.assertNotIn("--batch-groups", c["args"])++ def test_an_environment_variable_of_the_persons_own_wins(self):+ c = fresh()+ with mock.patch.dict(os.environ, {"STRATA_BATCH_SELF_COMMIT": "0"}):+ F.apply(CARDS, c)+ self.assertNotIn("STRATA_BATCH_SELF_COMMIT", c["env"])+ self.assertEqual(c["env"]["STRATA_QSA_SLOTS"], "1")++ def test_off_switches(self):+ c = fresh()+ with mock.patch.dict(os.environ, {"STRATA_FOUR_CARD_PROFILE": "0"}):+ self.assertEqual(F.apply(CARDS, c), [])+ self.assertEqual(c, fresh())+ c = fresh(four_card_profile=False)+ self.assertEqual(F.apply(CARDS, c), [])+ self.assertNotIn("parallel", c)++ def test_other_hardware_is_untouched(self):+ for cards in (CARDS[:2], [{**x, "name": "NVIDIA RTX A4000", "vendor": "nvidia"} for x in CARDS]):+ c = fresh()+ self.assertEqual(F.apply(cards, c), [])+ self.assertEqual(c, fresh())+++class Launcher(unittest.TestCase):+ def test_the_runner_fills_the_profile_and_the_profile_override(self):+ from strata_cli.launch import runner as R, settings as S+ from tests.terminal.launch_support import HW_12, Home, catalog+ hw = {**HW_12, "gpus": [{**x, "usable": True} for x in CARDS]}+ with Home():+ r = R.Runner(S.LaunchSettings(), catalog=catalog(hw))+ cfg = fresh()+ r.apply_four_card(cfg)+ self.assertEqual(cfg["layer_split"], "13,26,39")+ self.assertEqual(r.four_card_parallel, 32)+ r2 = R.Runner(S.LaunchSettings(parallel=3), catalog=catalog(hw))+ cfg2 = fresh()+ r2.apply_four_card(cfg2)+ self.assertNotIn("parallel", cfg2)+ self.assertIsNone(r2.four_card_parallel)++ def test_the_runner_leaves_other_computers_alone(self):+ from strata_cli.launch import runner as R, settings as S+ from tests.terminal.launch_support import HW_2X16, Home, catalog+ with Home():+ r = R.Runner(S.LaunchSettings(), catalog=catalog(HW_2X16))+ cfg = {**fresh(), "gpu": [0, 1]}+ r.apply_four_card(cfg)+ self.assertEqual(cfg, {**fresh(), "gpu": [0, 1]})+ self.assertIsNone(r.four_card_parallel)++++class ServerMessage(unittest.TestCase):+ def test_more_than_eight_slots_on_a_layer_split_is_not_cut_in_the_message(self):+ import contextlib+ import io+ from serve import server as S+ split = {"parallel": 32, "gpu": [0, 1, 2, 3], "layer_split": "13,26,39", "args": ["--pack", "/p"]}+ one = {"parallel": 32, "gpu": 0, "args": ["--pack", "/p"]}+ for cfg, said in ((split, False), (one, True)):+ with self.subTest(gpus=len(S.gpu_list(cfg))):+ out = io.StringIO()+ with contextlib.redirect_stdout(out):+ self.assertEqual(S.parallel_args(cfg, cfg["args"]), ["--batch", "32"])+ self.assertEqual("runs at most" in out.getvalue(), said)++if __name__ == "__main__":+ unittest.main()