Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

AMD on Windows: the shared-expert fork with KV streaming is not measured, never picked, and warned about when set by hand (#3895)

Niko12215f8145403854
diff --git a/serve/server.py b/serve/server.py
index adf9f552e..25202d32f 100644
--- a/serve/server.py
+++ b/serve/server.py
@@ -3531,6 +3531,9 @@ def child_env(cfg: dict, *, offline: bool = False) -> dict:
env[var] = os.pathsep.join(dirs + ([env[var]] if env.get(var) else []))
if cfg.get("backend") == "hip":
hip_env_guard(env, cfg)
+ note = sh_stream_warning(env, cfg)
+ if note:
+ print(note, flush=True)
auto = hip_auto_devices(cfg, env)
if auto:
env["HIP_VISIBLE_DEVICES"] = ",".join(str(i) for i in auto)
@@ -3545,6 +3548,20 @@ def _dir_writable(p: str) -> bool:
return False
+def sh_stream_warning(env: dict, cfg: dict) -> str | None:
+ """#3895: the shared-expert fork (STRATA_SH_STREAM=1) together with KV streaming (--kv-resident) made one Windows RX 6900 XT fail on the first
+ request ("verify: unspecified launch failure"); each alone, or Linux, did not. A warning, not a refusal: the person set the variable, the
+ engine runs as asked, and the line says what to take out."""
+ if os.name != "nt" or str(env.get("STRATA_SH_STREAM", "")).strip() in ("", "0"):
+ return None
+ args = [str(a) for a in cfg.get("args") or []]
+ if "--kv-resident" not in args:
+ return None
+ return ("[strata] WARNING: STRATA_SH_STREAM=1 with --kv-resident (KV streaming) failed on the first request on one Windows RX 6000 PC "
+ "(\"verify: unspecified launch failure\", #3895). If the engine stops that way, take STRATA_SH_STREAM out of the config's env "
+ "(AMD runs without it by default) or run without --kv-resident, and tell us the card and the driver.")
+
+
def hip_env_guard(env: dict, cfg: dict) -> list[str]:
"""#654: two things in an AMD user's environment that crash the HIP runtime before it prints a line. (1) HIP_PATH,
HIP_DEVICE_LIB_PATH or LLVM_PATH naming a folder that is gone (a ROCm build that was deleted): the bundled runtime
diff --git a/strata_control/tune_knobs.py b/strata_control/tune_knobs.py
index 54f27f723..0ee3842a5 100644
--- a/strata_control/tune_knobs.py
+++ b/strata_control/tune_knobs.py
@@ -16,6 +16,7 @@ bit-exact run to run). Only ``exact`` is picked on its own; the others need the
from __future__ import annotations
+import sys
from dataclasses import dataclass, field
from typing import Any, Callable, Sequence
@@ -152,6 +153,7 @@ BOUNDS: dict[str, dict[str, Any]] = {
#: in plain words. ``when`` names facts from `condition_holds`; empty = always.
NEVER: tuple[dict[str, Any], ...] = (
{"env": "STRATA_SH_STREAM", "when": ("gfx12",), "reason": "it halves decode speed on this graphics card"},
+ {"env": "STRATA_SH_STREAM", "when": ("windows_amd_kv_stream",), "reason": "with KV streaming on Windows it made the engine fail on the first request on one RX 6000 PC (#3895)"},
{"env": "STRATA_RDNA2_SGEMM", "value": "all", "when": ("gfx12",), "reason": "it makes long prompts five times slower"},
{"env": "STRATA_SELECT_SIMT", "reason": "its gain is below the 3% margin"},
{"env": "STRATA_HIP_WMMA", "reason": "it changes the numbers (a quality setting)"},
@@ -220,6 +222,7 @@ def condition_holds(name: str, ctx: Context) -> bool:
"vram_ge_24g": ctx.vram_mib >= 24000,
"vram_le_8g": 0 < ctx.vram_mib <= 8400,
"gfx12": bool(ctx.arch and ctx.arch.startswith("gfx12")),
+ "windows_amd_kv_stream": ctx.backend == "hip" and sys.platform == "win32" and "--kv-resident" in ctx.base_args,
"pcie_bound": 0 < ctx.vram_mib < 24000,
"ram_bound": bool(ctx.ram_gib and ctx.model_gib and ctx.ram_gib - ctx.model_gib < 30),
"ctx_ge_128k": ctx.context_tokens >= 98304,
diff --git a/tests/terminal/test_sh_stream_windows.py b/tests/terminal/test_sh_stream_windows.py
new file mode 100644
index 000000000..5921710b2
--- /dev/null
+++ b/tests/terminal/test_sh_stream_windows.py
@@ -0,0 +1,97 @@
+"""#3895: STRATA_SH_STREAM=1 (the shared expert on its own stream) together with --kv-resident failed the engine on its first request on one
+Windows RX 6900 XT ("verify: unspecified launch failure"). AMD runs without the fork by default. Calibrate does not measure the combination, the
+tuner never picks it, and a person who set it by hand gets one warning line at the start. Nothing is refused.
+"""
+from __future__ import annotations
+
+import sys
+import unittest
+from pathlib import Path
+from unittest import mock
+
+ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(ROOT / "tools"))
+sys.path.insert(0, str(ROOT))
+import calibrate as CAL # noqa: E402
+from serve import server # noqa: E402
+from strata_control import tune_knobs # noqa: E402
+from tools.test_calibrate import BASE, FakeEngine # noqa: E402
+
+
+class CalibrateSkipsTheCombination(unittest.TestCase):
+ def measure(self, platform, args):
+ starts, envs, said = [], [], []
+
+ def start(a, env=None):
+ envs.append(dict(env or {}))
+ return FakeEngine(a, lambda f, p, w: 50.0, 6, starts)
+
+ with mock.patch.object(CAL.sys, "platform", platform):
+ res = CAL.measure(args, [[1, 2, 3]] * 3, start, say=said.append, hip=True)
+ return res, envs, said
+
+ def test_windows_with_kv_streaming_does_not_measure_the_fork(self):
+ args = CAL.with_arg(BASE, "--kv-resident", "32768")
+ res, envs, said = self.measure("win32", args)
+ self.assertEqual(res["env"], {})
+ self.assertNotIn("sh_stream", res["report"])
+ self.assertEqual(res["report"]["sh_stream_skipped"], "windows with --kv-resident")
+ self.assertTrue(all(CAL.SH_STREAM not in e for e in envs))
+ self.assertTrue(any("#3895" in line and "STRATA_SH_STREAM=1" in line for line in said))
+
+ def test_linux_with_kv_streaming_and_windows_without_it_still_measure_it(self):
+ for platform, args in (("linux", CAL.with_arg(BASE, "--kv-resident", "32768")), ("win32", BASE)):
+ with self.subTest(platform=platform):
+ res, envs, _ = self.measure(platform, args)
+ self.assertEqual(set(res["report"]["sh_stream"]), {"0", "1"})
+ self.assertNotIn("sh_stream_skipped", res["report"])
+
+
+class TunerNeverPicksIt(unittest.TestCase):
+ def ctx(self, platform, backend="hip", args=("--kv-resident", "32768")):
+ context = tune_knobs.Context(backend=backend, arch="gfx1030", gpu_count=1, vram_mib=16384, ram_gib=64, model_gib=40.0, base_args=list(args))
+ return context, mock.patch.object(tune_knobs.sys, "platform", platform)
+
+ def test_windows_amd_with_kv_streaming(self):
+ ctx, patch = self.ctx("win32")
+ with patch:
+ problems = tune_knobs.check_setting({}, {"STRATA_SH_STREAM": "1"}, ctx)
+ self.assertEqual(len(problems), 1)
+ self.assertIn("#3895", problems[0])
+
+ def test_other_cases_stay_allowed(self):
+ for platform, backend, args in (("linux", "hip", ("--kv-resident", "32768")), ("win32", "hip", ()), ("win32", "cuda", ("--kv-resident", "32768"))):
+ ctx, patch = self.ctx(platform, backend, args)
+ with patch, self.subTest(platform=platform, backend=backend, args=args):
+ self.assertEqual(tune_knobs.check_setting({}, {"STRATA_SH_STREAM": "1"}, ctx), [])
+
+
+class StartWarning(unittest.TestCase):
+ CFG = {"backend": "hip", "args": ["--kv", "int8", "--kv-resident", "32768"]}
+
+ def test_a_hand_set_fork_with_kv_streaming_on_windows_warns_once_per_start(self):
+ with mock.patch.object(server.os, "name", "nt"):
+ note = server.sh_stream_warning({"STRATA_SH_STREAM": "1"}, self.CFG)
+ self.assertIn("#3895", note)
+ self.assertIn("WARNING", note)
+ self.assertIn("take STRATA_SH_STREAM out", note)
+
+ def test_no_warning_without_the_combination(self):
+ with mock.patch.object(server.os, "name", "nt"):
+ self.assertIsNone(server.sh_stream_warning({}, self.CFG))
+ self.assertIsNone(server.sh_stream_warning({"STRATA_SH_STREAM": "0"}, self.CFG))
+ self.assertIsNone(server.sh_stream_warning({"STRATA_SH_STREAM": "1"}, {"backend": "hip", "args": ["--kv", "int8"]}))
+ with mock.patch.object(server.os, "name", "posix"):
+ self.assertIsNone(server.sh_stream_warning({"STRATA_SH_STREAM": "1"}, self.CFG))
+
+ def test_child_env_prints_it_and_still_passes_the_variable(self):
+ cfg = {**self.CFG, "env": {"STRATA_SH_STREAM": "1"}}
+ with mock.patch.object(server.os, "name", "nt"), mock.patch("builtins.print") as shown, \
+ mock.patch.object(server, "hip_env_guard", return_value=[]), mock.patch.object(server, "hip_auto_devices", return_value=None):
+ env = server.child_env(cfg)
+ self.assertEqual(env["STRATA_SH_STREAM"], "1") # nothing is taken away
+ self.assertTrue(any("#3895" in str(call.args[0]) for call in shown.call_args_list if call.args))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tools/calibrate.py b/tools/calibrate.py
index 182c5a4d2..56819e30b 100644
--- a/tools/calibrate.py
+++ b/tools/calibrate.py
@@ -426,8 +426,14 @@ def measure(base_args: list[str], ids_list, start_engine, say=print, extra_worke
# 6. HIP: the shared-expert stream fork (a restart each), with everything chosen so far. Both values are set
# explicitly, so a value the config carried before does not decide the baseline.
env = {}
- if hip:
- tuned = apply(base_args, settings)
+ tuned = apply(base_args, settings)
+ if hip and fork_is_unsafe(tuned):
+ # #3895: one Windows RX 6000 PC faulted the engine on the first request with the fork and KV streaming together. The fork is off by
+ # default on AMD, so skipping the measurement changes nothing; whoever wants it sets the variable by hand.
+ say(" Not measuring the shared expert on its own stream: on Windows with KV streaming (--kv-resident) it made the engine fail on the first "
+ f"request on one PC (#3895). To try it anyway, put {SH_STREAM}=1 in the config's env.")
+ report["sh_stream_skipped"] = "windows with --kv-resident"
+ elif hip:
by_fork = {}
for v in ("0", "1"):
_check_cancel(cancel_event)
@@ -455,6 +461,12 @@ def measure(base_args: list[str], ids_list, start_engine, say=print, extra_worke
return {"settings": settings, "env": env, "report": report}
+def fork_is_unsafe(args) -> bool:
+ """The shared-expert fork (STRATA_SH_STREAM) with KV streaming on Windows: the one combination that has made an engine fail on its first
+ request (#3895, an RX 6900 XT). Not measured there; a person can still set the variable."""
+ return sys.platform == "win32" and arg_value(args, "--kv-resident") is not None
+
+
def restart(start_engine, args, what, say, report):
"""#1337: a later step's engine start that fails is a candidate that loses, not a failed run: what the earlier
steps measured (and the settings they chose) must survive it. Returns the engine, or None after saying why and