Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Four-card AMD serving profile: 32 slots in 4 groups, split 13,26,39, chunk 2048 (setup and launcher)

Niko12214fdb563a3475
diff --git a/serve/server.py b/serve/server.py
index 8c5213b1f..633339dee 100644
--- a/serve/server.py
+++ b/serve/server.py
@@ -3201,7 +3201,7 @@ def engine_args(cfg: dict) -> list[str]:
return learned_profile_args(cfg, args)
-PARALLEL_MAX = 8 # the engine's batch window holds at most 8 rows (kVerifyMaxT); --batch-mtp waves more through it
+PARALLEL_MAX = 8 # without a layer split the engine's batch window holds at most 8 rows (kVerifyNarrowT); --batch-mtp waves more through it
def parallel_args(cfg: dict, args: list[str]) -> list[str]:
@@ -3219,7 +3219,10 @@ def parallel_args(cfg: dict, args: list[str]) -> list[str]:
return []
if n <= 1:
return []
- if n > PARALLEL_MAX and "--batch-mtp" not in args and os.environ.get("STRATA_BATCH_MTP", "0") in ("", "0"):
+ # (on a layer split the pipelined groups each hold up to 8 slots - 16 in a HIP engine with wide windows - so more than 8 is
+ # not a limit there: the engine says how many slots it runs, INFO batch_slots)
+ if (n > PARALLEL_MAX and "--batch-mtp" not in args and os.environ.get("STRATA_BATCH_MTP", "0") in ("", "0")
+ and not layer_split_of(cfg)):
print(f'[strata] "parallel": {n} - the engine runs at most {PARALLEL_MAX} requests together; it will use '
f"{PARALLEL_MAX}", flush=True)
return ["--batch", str(n)]
diff --git a/setup.py b/setup.py
index bf5cd2e69..de1c38886 100644
--- a/setup.py
+++ b/setup.py
@@ -10039,6 +10039,14 @@ def main() -> int:
cfg["gpu_order"] = "as_given"
ok(f"layer split across GPUs {multi} ({cfg['layer_split']})")
recommend_remote_expert_opt(cfg, off=a.no_remote_expert_opt)
+ if hip and a.layer_split in (None, "auto"): # four Radeon AI PRO R9700 cards and Qwen3.8 IQ3_S: the measured serving profile
+ try: # (strata_control/amd_four_card.py); what the person chose always wins
+ from strata_control import amd_four_card
+ for line in amd_four_card.apply([{"name": c.get("name"), "vram_gb": c.get("vram_gb"), "vendor": "amd"} for c in chosen],
+ cfg, parallel_given=a.parallel is not None):
+ ok(line)
+ except Exception as e: # noqa: BLE001 - a profile is never a reason to stop setup
+ say(f" (the four-card profile was not applied: {e})")
if getattr(a, "_roles", None): # the roles themselves (strata_control/gpu_roles.py); "gpu" above is their mirror
final = cfg["gpu"] if isinstance(cfg.get("gpu"), list) else [cfg["gpu"]] # what the checks above left (a model that cannot share keeps one card)
main = final[-1]
diff --git a/strata_cli/launch/runner.py b/strata_cli/launch/runner.py
index cd323caa6..d109fef17 100644
--- a/strata_cli/launch/runner.py
+++ b/strata_cli/launch/runner.py
@@ -1409,6 +1409,7 @@ class Runner:
base = read_config(base_path)
cfg = derive_config(base, self.s, bool_flags=tune_knobs.BOOL_FLAGS,
log=str(launch_dir() / f"{self.name}.engine.log"), gpus=cards_for_launch(self.s, self.catalog, base=base))
+ self.apply_four_card(cfg)
self.use_own_dsv41_engine(cfg)
apply_experts(cfg, self.experts_flag(cfg.get("exe"))) # after the engine is settled: only a 0.1.50+ engine is given --experts
wait = self.start_wait()
@@ -1419,10 +1420,29 @@ class Runner:
self.config_path = out
return out
+ def apply_four_card(self, cfg: dict) -> None:
+ """Four Radeon AI PRO R9700 cards and Qwen3.8-Flash-Next IQ3_S: the measured serving profile (strata_control/amd_four_card.py), filled
+ in where the person chose nothing. Never stops a launch."""
+ self.four_card_parallel = None
+ try:
+ from strata_control import amd_four_card
+ picked = cfg.get("gpu")
+ if not (isinstance(picked, list) and len(picked) == amd_four_card.CARDS):
+ return
+ want = {int(i) for i in picked}
+ cards = [c for c in self.catalog.computer().usable_gpus if int(c.get("index", -1)) in want]
+ before = cfg.get("parallel")
+ notes = amd_four_card.apply(cards, cfg, parallel_given=bool(self.s.parallel))
+ if notes:
+ self.four_card_parallel = cfg.get("parallel") if before is None and cfg.get("parallel") else None # (only a number the profile set)
+ self.emit("model", "Four " + str(cards[0].get("name", "AMD")).replace("AMD Radeon ", "") + " cards: " + "; ".join(notes) + ".", None)
+ except Exception: # noqa: BLE001 - a profile is never a reason to stop a launch
+ self.four_card_parallel = None
+
def ensure_profile(self, config: Path) -> None:
from strata_control import profiles
name = self.profile
- parallel = self.s.parallel if self.s.parallel else self.catalog_parallel()
+ parallel = self.s.parallel if self.s.parallel else (getattr(self, "four_card_parallel", None) or self.catalog_parallel())
sets = [f"host={acc.host_for(self.s.access)}", f"port={int(self.s.port)}"] + ([f"parallel={parallel}"] if parallel else [])
try:
current, _rev = profiles.load(name)
diff --git a/strata_control/amd_four_card.py b/strata_control/amd_four_card.py
new file mode 100644
index 000000000..b040fba06
--- /dev/null
+++ b/strata_control/amd_four_card.py
@@ -0,0 +1,92 @@
+"""The serving profile of a four-card AMD box: 4 x Radeon AI PRO R9700 (32 GB, gfx1201) running Qwen3.8-Flash-Next IQ3_S.
+
+Measured on that box (docs/BATCHING.md, "Four R9700 cards: the serving profile"): the cards hold the whole model, so what limits
+them is how many requests decode together and how the four stages are fed, not memory. The profile sets
+
+* 32 slots in 4 pipelined groups (one group per card), instead of the launcher's 4 requests at a time;
+* the layer split 13,26,39: the output head costs about four layers on the last card, so it gets 9 layers and the others 13;
+* a 2048-token prompt chunk: a smaller chunk reads long prompts faster beside the decodes of the other slots;
+* the three batch switches STRATA_QSA_SLOTS, STRATA_BATCH_SELF_COMMIT and STRATA_SLOT_COPY_DIRECT, which change how the work is
+ scheduled and copied, not what is computed (tools/batch_test.py: every slot decodes the tokens it decodes alone).
+
+Recommend, never force: a setting the person made (the config, `--parallel`, `--layer-split`, an advanced flag, an environment
+variable) always wins, the profile only fills what is not set, and `STRATA_FOUR_CARD_PROFILE=0` (or "four_card_profile": false in
+the run config) turns it off. Only this hardware class is touched; every other computer gets exactly what it got before.
+
+Pure functions, no I/O (the callers pass the cards and the run config).
+"""
+
+from __future__ import annotations
+
+import os
+
+PARALLEL = 32 # slots (requests decoding together)
+GROUPS = 4 # pipelined groups: one per card
+LAYER_SPLIT = "13,26,39" # Qwen3.8-Flash-Next has 48 layers: 13 / 13 / 13 / 9 (the output head sits on the last card)
+PREFILL_CHUNK = "2048"
+BATCH_ENV = {"STRATA_QSA_SLOTS": "1", "STRATA_BATCH_SELF_COMMIT": "1", "STRATA_SLOT_COPY_DIRECT": "1"}
+PROMPT_CACHE_OFF = False # --prompt-cache 0 is not part of the default: see docs/BATCHING.md
+MIN_CARD_GB = 28.0 # a 32 GB card reports a little less than its name
+CARDS = 4
+OFF_VALUES = ("0", "off", "no", "false")
+
+
+def disabled(cfg: dict | None = None) -> bool:
+ """The person turned the profile off: STRATA_FOUR_CARD_PROFILE=0 in the environment, or "four_card_profile": false in the config."""
+ if str(os.environ.get("STRATA_FOUR_CARD_PROFILE", "")).strip().lower() in OFF_VALUES:
+ return True
+ return bool(cfg) and cfg.get("four_card_profile") is False
+
+
+def card_class(cards: list[dict]) -> bool:
+ """Exactly four AMD cards of one kind, each with at least MIN_CARD_GB: the Radeon AI PRO R9700 (the 32 GB gfx12 card this was
+ measured on)."""
+ if len(cards) != CARDS:
+ return False
+ names = {str(c.get("name") or "") for c in cards}
+ if len(names) != 1 or "R9700" not in next(iter(names)).upper().replace(" ", ""):
+ return False
+ return all(str(c.get("vendor") or "amd").lower() == "amd" and float(c.get("vram_gb") or 0.0) >= MIN_CARD_GB for c in cards)
+
+
+def model_ok(cfg: dict) -> bool:
+ """The measured model: Qwen3.8-Flash-Next at IQ3_S (48 layers; the pack that fits whole on four 32 GB cards)."""
+ text = " ".join([str(cfg.get("model_name") or "")] + [str(a) for a in cfg.get("args") or []]).lower().replace("-", "_").replace(".", "_")
+ return "qwen3_8" in text and "iq3_s" in text
+
+
+def applies(cards: list[dict], cfg: dict) -> bool:
+ args = [str(a) for a in cfg.get("args") or []]
+ return (not disabled(cfg)) and card_class(cards) and cfg.get("backend", "hip") == "hip" and model_ok(cfg) \
+ and "--peer-device" not in args and not cfg.get("replicas") and cfg.get("arch") != "deepseek41"
+
+
+def apply(cards: list[dict], cfg: dict, *, parallel_given: bool = False) -> list[str]:
+ """Fill the profile into the run config `cfg` (in place) where the person has not chosen; returns the plain lines to show.
+ `parallel_given`: the person named a number of requests (--parallel): then only the split, the chunk and the switches are
+ filled (the engine groups the slots itself)."""
+ if not applies(cards, cfg):
+ return []
+ args = [str(a) for a in cfg.get("args") or []]
+ env = dict(cfg.get("env") or {})
+ notes: list[str] = []
+ if str(cfg.get("layer_split") or "auto") == "auto" and "--layer-split" not in args:
+ cfg["layer_split"] = LAYER_SPLIT
+ notes.append(f"layer split {LAYER_SPLIT}: the last card also holds the output head, so it takes fewer layers")
+ if "--prefill" not in args:
+ args.extend(["--prefill", PREFILL_CHUNK])
+ elif args[args.index("--prefill") + 1:args.index("--prefill") + 2] == ["auto"]: # "auto" is what setup writes, not a choice
+ args[args.index("--prefill") + 1] = PREFILL_CHUNK
+ if PROMPT_CACHE_OFF and "--prompt-cache" not in args:
+ args.extend(["--prompt-cache", "0"])
+ for k, v in BATCH_ENV.items():
+ if k not in os.environ: # an environment variable of the person's own wins
+ env.setdefault(k, v)
+ if not parallel_given and "parallel" not in cfg and not {"--batch", "--slots"} & set(args):
+ cfg["parallel"] = PARALLEL
+ if "--batch-groups" not in args:
+ args.extend(["--batch-groups", str(GROUPS)])
+ notes.append(f"{PARALLEL} requests at once in {GROUPS} groups, one group per card (--parallel N changes it)")
+ cfg["args"] = args
+ cfg["env"] = env
+ return notes
diff --git a/tests/terminal/test_amd_four_card.py b/tests/terminal/test_amd_four_card.py
new file mode 100644
index 000000000..d313fc0c0
--- /dev/null
+++ b/tests/terminal/test_amd_four_card.py
@@ -0,0 +1,158 @@
+"""strata_control/amd_four_card.py: the serving profile of four Radeon AI PRO R9700 cards, and the places that apply it."""
+
+from __future__ import annotations
+
+import os
+import unittest
+from unittest import mock
+
+from strata_control import amd_four_card as F
+
+R9700 = {"name": "AMD Radeon AI PRO R9700", "vram_gb": 31.9, "vendor": "amd", "usable": True}
+CARDS = [{**R9700, "index": i} for i in range(4)]
+CFG = {"model_name": "qwen3.8-flash-next-iq3_s", "backend": "hip", "gpu": [0, 1, 2, 3], "layer_split": "auto",
+ "args": ["--pack", "/p/iq3_s", "--spec", "4", "--max-context", "32768"]}
+
+
+def fresh(**kw):
+ c = {**CFG, "args": list(CFG["args"]), **kw}
+ return c
+
+
+class Class(unittest.TestCase):
+ def test_four_r9700_are_the_class(self):
+ self.assertTrue(F.card_class(CARDS))
+
+ def test_other_hardware_is_not(self):
+ self.assertFalse(F.card_class(CARDS[:3])) # three cards
+ self.assertFalse(F.card_class(CARDS + [CARDS[0]])) # five
+ self.assertFalse(F.card_class([{**c, "name": "AMD Radeon RX 7900 XTX", "vram_gb": 24.0} for c in CARDS]))
+ self.assertFalse(F.card_class([{**c, "vendor": "nvidia"} for c in CARDS]))
+ self.assertFalse(F.card_class([{**c, "vram_gb": 16.0} for c in CARDS])) # a 16 GB card with the same name
+ self.assertFalse(F.card_class([*CARDS[:3], {**R9700, "name": "AMD Radeon RX 9070"}]))
+
+ def test_only_the_measured_model(self):
+ for name in ("qwen3.8-flash-next-iq3_xxs", "qwen3.6-35b-a3b-q4", "deepseek-v4.1-flash-2.00bpw"):
+ with self.subTest(name):
+ self.assertFalse(F.applies(CARDS, fresh(model_name=name, args=["--pack", "/p/x"])))
+ self.assertTrue(F.applies(CARDS, fresh()))
+ self.assertFalse(F.applies(CARDS, fresh(backend="cuda")))
+ self.assertFalse(F.applies(CARDS, fresh(arch="deepseek41")))
+ self.assertFalse(F.applies(CARDS, fresh(args=CFG["args"] + ["--peer-device", "1"])))
+
+
+class Apply(unittest.TestCase):
+ def test_fills_the_profile_on_a_default_config(self):
+ c = fresh()
+ with mock.patch.dict(os.environ, {}, clear=False):
+ for k in list(F.BATCH_ENV) + ["STRATA_FOUR_CARD_PROFILE"]:
+ os.environ.pop(k, None)
+ notes = F.apply(CARDS, c)
+ self.assertTrue(notes)
+ self.assertEqual(c["layer_split"], "13,26,39")
+ self.assertEqual(c["parallel"], 32)
+ self.assertEqual(c["args"][c["args"].index("--batch-groups") + 1], "4")
+ self.assertEqual(c["args"][c["args"].index("--prefill") + 1], "2048")
+ self.assertEqual(c["env"], F.BATCH_ENV)
+ self.assertNotIn("--prompt-cache", c["args"])
+
+ def test_prefill_auto_from_setup_is_not_a_choice(self):
+ c = fresh(args=CFG["args"] + ["--prefill", "auto"])
+ F.apply(CARDS, c)
+ self.assertEqual(c["args"][c["args"].index("--prefill") + 1], "2048")
+ self.assertEqual(c["args"].count("--prefill"), 1)
+ c = fresh(args=CFG["args"] + ["--prefill", "auto:32768"])
+ F.apply(CARDS, c)
+ self.assertEqual(c["args"][c["args"].index("--prefill") + 1], "auto:32768")
+
+ def test_the_persons_choices_win(self):
+ c = fresh(layer_split="12,25,37", parallel=8, args=CFG["args"] + ["--prefill", "8192", "--batch-groups", "2"],
+ env={"STRATA_QSA_SLOTS": "0"})
+ F.apply(CARDS, c)
+ self.assertEqual(c["layer_split"], "12,25,37")
+ self.assertEqual(c["parallel"], 8)
+ self.assertEqual(c["args"][c["args"].index("--prefill") + 1], "8192")
+ self.assertEqual(c["args"].count("--batch-groups"), 1)
+ self.assertEqual(c["env"]["STRATA_QSA_SLOTS"], "0")
+
+ def test_a_number_of_requests_asked_for_keeps_the_groups_to_the_engine(self):
+ c = fresh()
+ F.apply(CARDS, c, parallel_given=True)
+ self.assertNotIn("parallel", c)
+ self.assertNotIn("--batch-groups", c["args"])
+ self.assertEqual(c["layer_split"], "13,26,39")
+
+ def test_an_explicit_batch_in_the_args_is_kept(self):
+ c = fresh(args=CFG["args"] + ["--batch", "6"])
+ F.apply(CARDS, c)
+ self.assertNotIn("parallel", c)
+ self.assertNotIn("--batch-groups", c["args"])
+
+ def test_an_environment_variable_of_the_persons_own_wins(self):
+ c = fresh()
+ with mock.patch.dict(os.environ, {"STRATA_BATCH_SELF_COMMIT": "0"}):
+ F.apply(CARDS, c)
+ self.assertNotIn("STRATA_BATCH_SELF_COMMIT", c["env"])
+ self.assertEqual(c["env"]["STRATA_QSA_SLOTS"], "1")
+
+ def test_off_switches(self):
+ c = fresh()
+ with mock.patch.dict(os.environ, {"STRATA_FOUR_CARD_PROFILE": "0"}):
+ self.assertEqual(F.apply(CARDS, c), [])
+ self.assertEqual(c, fresh())
+ c = fresh(four_card_profile=False)
+ self.assertEqual(F.apply(CARDS, c), [])
+ self.assertNotIn("parallel", c)
+
+ def test_other_hardware_is_untouched(self):
+ for cards in (CARDS[:2], [{**x, "name": "NVIDIA RTX A4000", "vendor": "nvidia"} for x in CARDS]):
+ c = fresh()
+ self.assertEqual(F.apply(cards, c), [])
+ self.assertEqual(c, fresh())
+
+
+class Launcher(unittest.TestCase):
+ def test_the_runner_fills_the_profile_and_the_profile_override(self):
+ from strata_cli.launch import runner as R, settings as S
+ from tests.terminal.launch_support import HW_12, Home, catalog
+ hw = {**HW_12, "gpus": [{**x, "usable": True} for x in CARDS]}
+ with Home():
+ r = R.Runner(S.LaunchSettings(), catalog=catalog(hw))
+ cfg = fresh()
+ r.apply_four_card(cfg)
+ self.assertEqual(cfg["layer_split"], "13,26,39")
+ self.assertEqual(r.four_card_parallel, 32)
+ r2 = R.Runner(S.LaunchSettings(parallel=3), catalog=catalog(hw))
+ cfg2 = fresh()
+ r2.apply_four_card(cfg2)
+ self.assertNotIn("parallel", cfg2)
+ self.assertIsNone(r2.four_card_parallel)
+
+ def test_the_runner_leaves_other_computers_alone(self):
+ from strata_cli.launch import runner as R, settings as S
+ from tests.terminal.launch_support import HW_2X16, Home, catalog
+ with Home():
+ r = R.Runner(S.LaunchSettings(), catalog=catalog(HW_2X16))
+ cfg = {**fresh(), "gpu": [0, 1]}
+ r.apply_four_card(cfg)
+ self.assertEqual(cfg, {**fresh(), "gpu": [0, 1]})
+ self.assertIsNone(r.four_card_parallel)
+
+
+
+class ServerMessage(unittest.TestCase):
+ def test_more_than_eight_slots_on_a_layer_split_is_not_cut_in_the_message(self):
+ import contextlib
+ import io
+ from serve import server as S
+ split = {"parallel": 32, "gpu": [0, 1, 2, 3], "layer_split": "13,26,39", "args": ["--pack", "/p"]}
+ one = {"parallel": 32, "gpu": 0, "args": ["--pack", "/p"]}
+ for cfg, said in ((split, False), (one, True)):
+ with self.subTest(gpus=len(S.gpu_list(cfg))):
+ out = io.StringIO()
+ with contextlib.redirect_stdout(out):
+ self.assertEqual(S.parallel_args(cfg, cfg["args"]), ["--batch", "32"])
+ self.assertEqual("runs at most" in out.getvalue(), said)
+
+if __name__ == "__main__":
+ unittest.main()