Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

picture reader choice for AMD engine folders and the server's fallback from the GPU reader to the CPU reader (one log line)

Niko1221f82fffbe25be
diff --git a/serve/server.py b/serve/server.py
index 1b82c1023..bce59b6f6 100644
--- a/serve/server.py
+++ b/serve/server.py
@@ -66,6 +66,7 @@ from serve.frontend import (THINK_END, ChatTemplate, Event, OutputParser, anthro
from serve.mcp import McpCancelled, hub_from_config # noqa: E402
from serve import runconfig # noqa: E402
from strata_control import engine_component # noqa: E402
+from strata_control import vision_reader # noqa: E402
from serve import ready as ready_file # noqa: E402
from serve import owner_guard # noqa: E402
from serve import launch_identity # noqa: E402
@@ -2487,18 +2488,13 @@ class Vision:
return Path(tempfile.mkdtemp(prefix="strata-vision-"))
def __init__(self, cfg: dict, log=None, env: dict | None = None, lazy: bool = False):
- absolute = lambda p: os.path.abspath(p) if os.path.dirname(str(p)) else p # the encoder runs in its own dir (#480) # noqa: E731
- args = [absolute(cfg["exe"]), "--mmproj", absolute(cfg["mmproj"]), "--model", absolute(cfg["model"])]
- if cfg.get("gpu"):
- args.append("--gpu")
- if cfg.get("threads"):
- args += ["--threads", str(cfg["threads"])]
- if cfg.get("max_tokens"):
- args += ["--max-tokens", str(cfg["max_tokens"])]
- if cfg.get("min_tokens"): # #767: mtmd's image_min_tokens (a hand-edited key)
- args += ["--min-tokens", str(cfg["min_tokens"])]
+ args = self.reader_args(cfg)
self.dir = self.work_dir()
self.spawn = (args, log, env) # to start it again after an unload
+ # AMD: the GPU reader (strata-vision-hip) has the CPU reader (strata-vision, same program, no GPU runtime) to
+ # fall back on when it does not start; _start switches to it by itself
+ cpu = vision_reader.cpu_fallback(cfg)
+ self.fallback = (self.reader_args(cpu), log, env) if cpu else None
self.proc = None
self.stopped = True
self.lock = threading.Lock()
@@ -2512,7 +2508,36 @@ class Vision:
if not lazy and not self.elastic:
self._start()
+ @staticmethod
+ def reader_args(cfg: dict) -> list:
+ absolute = lambda p: os.path.abspath(p) if os.path.dirname(str(p)) else p # the encoder runs in its own dir (#480) # noqa: E731
+ args = [absolute(cfg["exe"]), "--mmproj", absolute(cfg["mmproj"]), "--model", absolute(cfg["model"])]
+ if cfg.get("gpu"):
+ args.append("--gpu")
+ if cfg.get("threads"):
+ args += ["--threads", str(cfg["threads"])]
+ if cfg.get("max_tokens"):
+ args += ["--max-tokens", str(cfg["max_tokens"])]
+ if cfg.get("min_tokens"): # #767: mtmd's image_min_tokens (a hand-edited key)
+ args += ["--min-tokens", str(cfg["min_tokens"])]
+ return args
+
def _start(self):
+ """Start the reader. When a GPU reader that has a CPU reader to fall back on does not start (the GPU runtime does not
+ load, the card is not in its build, the card is busy), the CPU reader is started instead, for good, with one log line
+ and no question."""
+ try:
+ self._start_reader()
+ except (RuntimeError, OSError) as e:
+ fallback = getattr(self, "fallback", None)
+ if fallback is None:
+ raise
+ why = " ".join(str(e).split())[:240]
+ print(f"[strata] the GPU picture reader did not start ({why}); pictures are read on the CPU", flush=True)
+ self.spawn, self.fallback = fallback, None
+ self._start_reader()
+
+ def _start_reader(self):
args, log, env = self.spawn
# cwd is the encoder's directory: the ENC lines name the files relative to it (#480: no space in the line)
self.proc = popen("the image encoder", args, role="vision", stdin=subprocess.PIPE, stdout=subprocess.PIPE,
diff --git a/strata_control/vision_reader.py b/strata_control/vision_reader.py
new file mode 100644
index 000000000..cd0001941
--- /dev/null
+++ b/strata_control/vision_reader.py
@@ -0,0 +1,101 @@
+"""Which picture reader an engine folder gets: the GPU one (`strata-vision-hip`, AMD) or the CPU one (`strata-vision`).
+
+An AMD engine package carries two programs for pictures. `strata-vision` is the CPU reader: it needs nothing from the GPU
+runtime, so it starts on every PC. `strata-vision-hip` is the same reader on the card (llama.cpp's HIP backend), a
+fraction of a second per picture instead of seconds. Setup writes the run config with the GPU reader first and the CPU
+reader as its fallback (`vision.cpu_exe`); the server (serve/server.py, Vision) starts the CPU reader by itself, with one
+log line, when the GPU reader does not start: the HIP runtime does not load, the card is not one the build has code for
+(the reader says so before it runs a kernel), or the card is busy. Nothing is asked of the user.
+
+`choose` is the whole decision; setup.py calls it, and tools/test_vision_reader.py checks every branch.
+"""
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+
+CPU_READER = "strata-vision"
+GPU_READER = "strata-vision-hip"
+
+GFX = re.compile(r"gfx[0-9a-f]+", re.IGNORECASE)
+
+
+def reader_name(base: str, windows: bool) -> str:
+ return base + (".exe" if windows else "")
+
+
+def find(folder: Path | str, windows: bool) -> dict:
+ """The readers in an engine folder: {"cpu": Path | None, "gpu": Path | None}."""
+ folder = Path(folder)
+ found = {}
+ for key, base in (("cpu", CPU_READER), ("gpu", GPU_READER)):
+ path = folder / reader_name(base, windows)
+ found[key] = path if path.is_file() else None
+ return found
+
+
+def gfx_of(text: object) -> str:
+ """"gfx1201" out of a gcnArchName ("gfx1201:sramecc-:xnack-"), a device line or a plain arch; "" when there is none."""
+ m = GFX.search(str(text or ""))
+ return m.group(0).lower() if m else ""
+
+
+@dataclass
+class Reader:
+ mode: str # "gpu" | "cpu" | "none"
+ exe: Path | None = None # the program the run config starts
+ cpu_exe: Path | None = None # the CPU reader to fall back on when a GPU reader does not start (mode "gpu" only)
+ note: str = "" # one plain sentence for the setup output when it is not what was asked for
+
+
+def choose(folder: Path | str, asked: str, *, windows: bool, gpu_arch: str = "", archs: list | None = None) -> Reader:
+ """The reader for `asked` ("yes"/"on"/"gpu" = the best the folder has, "cpu", "no"/"none"/"off").
+
+ gpu_arch: the card's architecture when setup knows it ("gfx1201"); archs: what the GPU reader was built for
+ (BUILD.json `vision_archs`, else the package's `archs`). A card the build does not list gets the CPU reader here;
+ when the card is unknown the GPU reader is tried and falls back by itself (see the module doc)."""
+ asked = (asked or "").lower()
+ if asked in ("no", "none", "off", ""):
+ return Reader("none")
+ have = find(folder, windows)
+ if asked == "cpu":
+ if have["cpu"]:
+ return Reader("cpu", have["cpu"])
+ return Reader("none", note="this engine has no picture reader: pictures off")
+ # yes / on / gpu
+ gfx = gfx_of(gpu_arch)
+ built = [gfx_of(a) for a in (archs or []) if gfx_of(a)]
+ if have["gpu"] is None:
+ if have["cpu"]:
+ return Reader("cpu", have["cpu"], note="this engine has no GPU picture reader: pictures are read on the CPU "
+ "(a few seconds each)" if asked == "gpu" else "")
+ return Reader("none", note="this engine has no picture reader: pictures off")
+ if gfx and built and gfx not in built:
+ if have["cpu"]:
+ return Reader("cpu", have["cpu"], note=f"the GPU picture reader has no code for your card ({gfx}): pictures "
+ "are read on the CPU (a few seconds each)")
+ return Reader("none", note=f"the GPU picture reader has no code for your card ({gfx}): pictures off")
+ return Reader("gpu", have["gpu"], have["cpu"])
+
+
+def vision_section(reader: Reader, base: dict) -> dict:
+ """The run config's "vision" section for `reader` on top of `base` (mmproj, model, max_tokens ...): exe, gpu, and for the
+ GPU reader the CPU reader to fall back on."""
+ out = dict(base)
+ out["exe"] = str(reader.exe)
+ out["gpu"] = reader.mode == "gpu"
+ if reader.mode == "gpu" and reader.cpu_exe:
+ out["cpu_exe"] = str(reader.cpu_exe)
+ else:
+ out.pop("cpu_exe", None)
+ return out
+
+
+def cpu_fallback(vision_cfg: dict) -> dict | None:
+ """The same section for the CPU reader, when the GPU reader has one to fall back on; None otherwise."""
+ if not vision_cfg.get("gpu") or not vision_cfg.get("cpu_exe"):
+ return None
+ out = {k: v for k, v in vision_cfg.items() if k not in ("cpu_exe", "elastic", "elastic_mib", "elastic_idle_s")}
+ out["exe"], out["gpu"] = vision_cfg["cpu_exe"], False
+ return out
diff --git a/tools/test_vision_reader.py b/tools/test_vision_reader.py
new file mode 100644
index 000000000..1f77b0c91
--- /dev/null
+++ b/tools/test_vision_reader.py
@@ -0,0 +1,202 @@
+"""The picture reader choice for an AMD engine folder (strata_control/vision_reader.py) and the server's fallback from the GPU
+reader to the CPU reader (serve/server.py, Vision._start): hip present, hip missing, hip start failure, then cpu.
+
+ python -m unittest tools.test_vision_reader
+"""
+from __future__ import annotations
+
+import io
+import tempfile
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest import mock
+
+from strata_control import vision_reader as VR
+import serve.server as server
+
+
+def folder(*names: str) -> Path:
+ d = Path(tempfile.mkdtemp(prefix="vr-"))
+ for n in names:
+ (d / n).write_bytes(b"x")
+ return d
+
+
+class Choose(unittest.TestCase):
+ def test_hip_present_gives_the_gpu_reader_with_the_cpu_one_to_fall_back_on(self):
+ d = folder("strata-vision", "strata-vision-hip")
+ for ask in ("yes", "on", "gpu"):
+ r = VR.choose(d, ask, windows=False, gpu_arch="gfx1201", archs=["gfx1200", "gfx1201"])
+ self.assertEqual((r.mode, r.exe.name, r.cpu_exe.name), ("gpu", "strata-vision-hip", "strata-vision"), ask)
+ self.assertEqual(r.note, "")
+
+ def test_hip_missing_gives_the_cpu_reader(self):
+ d = folder("strata-vision")
+ r = VR.choose(d, "yes", windows=False, gpu_arch="gfx1201")
+ self.assertEqual((r.mode, r.exe.name, r.cpu_exe), ("cpu", "strata-vision", None))
+ self.assertEqual(r.note, "") # "yes" never asked for the GPU in particular
+ self.assertIn("no GPU picture reader", VR.choose(d, "gpu", windows=False).note) # --vision gpu: said once
+
+ def test_card_not_in_the_build_gives_the_cpu_reader(self):
+ d = folder("strata-vision", "strata-vision-hip")
+ r = VR.choose(d, "yes", windows=False, gpu_arch="gfx1100", archs=["gfx1200", "gfx1201"])
+ self.assertEqual((r.mode, r.exe.name), ("cpu", "strata-vision"))
+ self.assertIn("gfx1100", r.note)
+
+ def test_unknown_card_tries_the_gpu_reader(self):
+ d = folder("strata-vision", "strata-vision-hip")
+ self.assertEqual(VR.choose(d, "yes", windows=False, gpu_arch="", archs=["gfx1201"]).mode, "gpu")
+ self.assertEqual(VR.choose(d, "yes", windows=False, gpu_arch="gfx1100", archs=None).mode, "gpu")
+
+ def test_gcn_arch_names_with_features_match(self):
+ self.assertEqual(VR.gfx_of("gfx1201:sramecc-:xnack-"), "gfx1201")
+ d = folder("strata-vision", "strata-vision-hip")
+ self.assertEqual(VR.choose(d, "yes", windows=False, gpu_arch="gfx1151:sramecc+:xnack-", archs=["gfx1151"]).mode, "gpu")
+
+ def test_cpu_asked_and_off(self):
+ d = folder("strata-vision", "strata-vision-hip")
+ self.assertEqual(VR.choose(d, "cpu", windows=False).exe.name, "strata-vision")
+ for ask in ("no", "none", "off", None):
+ self.assertEqual(VR.choose(d, ask, windows=False).mode, "none")
+
+ def test_no_reader_at_all(self):
+ d = folder("strata")
+ for ask in ("yes", "cpu", "gpu"):
+ r = VR.choose(d, ask, windows=False)
+ self.assertEqual(r.mode, "none", ask)
+ self.assertIn("no picture reader", r.note)
+
+ def test_windows_names(self):
+ d = folder("strata-vision.exe", "strata-vision-hip.exe")
+ r = VR.choose(d, "yes", windows=True, gpu_arch="gfx1201", archs=["gfx1201"])
+ self.assertEqual((r.mode, r.exe.name, r.cpu_exe.name), ("gpu", "strata-vision-hip.exe", "strata-vision.exe"))
+ self.assertEqual(VR.choose(folder("strata-vision-hip"), "yes", windows=True).mode, "none") # a Linux name on Windows
+
+ def test_run_config_section(self):
+ d = folder("strata-vision", "strata-vision-hip")
+ base = {"mmproj": "m.gguf", "model": "t.gguf", "max_tokens": 1024}
+ sec = VR.vision_section(VR.choose(d, "yes", windows=False), base)
+ self.assertTrue(sec["gpu"])
+ self.assertEqual(Path(sec["exe"]).name, "strata-vision-hip")
+ self.assertEqual(Path(sec["cpu_exe"]).name, "strata-vision")
+ cpu = VR.cpu_fallback(sec)
+ self.assertFalse(cpu["gpu"])
+ self.assertEqual(Path(cpu["exe"]).name, "strata-vision")
+ self.assertNotIn("cpu_exe", cpu)
+ self.assertEqual(cpu["mmproj"], "m.gguf")
+ cpu_only = VR.vision_section(VR.choose(folder("strata-vision"), "yes", windows=False), base)
+ self.assertFalse(cpu_only["gpu"])
+ self.assertNotIn("cpu_exe", cpu_only)
+ self.assertIsNone(VR.cpu_fallback(cpu_only))
+ self.assertIsNone(VR.cpu_fallback({**sec, "cpu_exe": ""}))
+ self.assertIsNone(VR.cpu_fallback({"exe": "strata-vision", "gpu": True})) # an NVIDIA config has no fallback
+
+
+class FakeProc:
+ def __init__(self, first_line: str):
+ self.stdin, self.stdout = io.StringIO(), io.StringIO(first_line)
+ self.killed = False
+
+ def kill(self):
+ self.killed = True
+
+ def wait(self, timeout=None):
+ return 0
+
+ def poll(self):
+ return None
+
+
+def make_vision(cfg: dict) -> "server.Vision":
+ with mock.patch.object(server.Vision, "work_dir", staticmethod(lambda: Path(tempfile.gettempdir()))):
+ return server.Vision(cfg, lazy=True)
+
+
+class Fallback(unittest.TestCase):
+ CFG = {"exe": "/e/strata-vision-hip", "cpu_exe": "/e/strata-vision", "gpu": True, "mmproj": "/m/mm.gguf",
+ "model": "/m/t.gguf", "max_tokens": 1024, "threads": 6}
+
+ def run_start(self, answers: dict, cfg: dict | None = None):
+ """answers: exe name -> its first stdout line, or an exception to raise from popen."""
+ started = []
+
+ def popen(what, args, **kw):
+ name = Path(args[0]).name
+ started.append(args)
+ a = answers[name]
+ if isinstance(a, Exception):
+ raise a
+ return FakeProc(a)
+
+ v = make_vision(cfg or self.CFG)
+ out = io.StringIO()
+ with mock.patch.object(server, "popen", popen), mock.patch("builtins.print", lambda *a, **k: out.write(" ".join(map(str, a)) + "\n")):
+ err = None
+ try:
+ v._start()
+ except RuntimeError as e:
+ err = e
+ return v, started, out.getvalue(), err
+
+ def test_hip_present_and_starts(self):
+ v, started, log, err = self.run_start({"strata-vision-hip": "READY 2048\n"})
+ self.assertIsNone(err)
+ self.assertEqual([Path(a[0]).name for a in started], ["strata-vision-hip"])
+ self.assertIn("--gpu", started[0])
+ self.assertEqual(log, "")
+ self.assertFalse(v.stopped)
+
+ def test_hip_start_failure_falls_back_to_cpu_with_one_line(self):
+ v, started, log, err = self.run_start({"strata-vision-hip": "ERR this GPU (gfx1100) is not in the picture reader's build (gfx1200,gfx1201)\n",
+ "strata-vision": "READY 2048\n"})
+ self.assertIsNone(err)
+ self.assertEqual([Path(a[0]).name for a in started], ["strata-vision-hip", "strata-vision"])
+ self.assertNotIn("--gpu", started[1])
+ self.assertIn("--threads", started[1])
+ self.assertEqual(log.count("\n"), 1)
+ self.assertIn("gfx1100", log)
+ self.assertIn("CPU", log)
+ self.assertFalse(v.stopped)
+ self.assertIsNone(v.fallback)
+
+ def test_hip_process_that_dies_silently_falls_back(self): # a GPU runtime that aborts: stdout closes empty
+ v, started, log, err = self.run_start({"strata-vision-hip": "", "strata-vision": "READY 2048\n"})
+ self.assertIsNone(err)
+ self.assertEqual(len(started), 2)
+ self.assertEqual(log.count("\n"), 1)
+
+ def test_hip_missing_file_falls_back(self): # popen raises (exe gone, not executable)
+ v, started, log, err = self.run_start({"strata-vision-hip": FileNotFoundError("strata-vision-hip"),
+ "strata-vision": "READY 2048\n"})
+ self.assertIsNone(err)
+ self.assertEqual(len(started), 2)
+
+ def test_restart_after_a_fallback_stays_on_the_cpu_reader(self):
+ v, started, log, err = self.run_start({"strata-vision-hip": "ERR nope\n", "strata-vision": "READY 2048\n"})
+ started.clear()
+ with mock.patch.object(server, "popen", lambda what, args, **kw: started.append(args) or FakeProc("READY 2048\n")):
+ v.restart()
+ self.assertEqual([Path(a[0]).name for a in started], ["strata-vision"])
+
+ def test_cpu_failure_after_the_fallback_is_an_error(self):
+ v, started, log, err = self.run_start({"strata-vision-hip": "ERR nope\n", "strata-vision": "ERR cannot load the vision encoder\n"})
+ self.assertIsNotNone(err)
+ self.assertIn("cannot load", str(err))
+
+ def test_a_config_without_cpu_exe_has_no_fallback(self): # NVIDIA: the error is shown as before
+ cfg = {k: v for k, v in self.CFG.items() if k != "cpu_exe"}
+ v, started, log, err = self.run_start({"strata-vision-hip": "ERR nope\n"}, cfg)
+ self.assertIsNotNone(err)
+ self.assertEqual(len(started), 1)
+ self.assertEqual(log, "")
+
+ def test_cpu_config_is_unchanged(self):
+ cfg = {"exe": "/e/strata-vision", "gpu": False, "mmproj": "/m/mm.gguf", "model": "/m/t.gguf"}
+ v, started, log, err = self.run_start({"strata-vision": "READY 2048\n"}, cfg)
+ self.assertIsNone(err)
+ self.assertNotIn("--gpu", started[0])
+
+
+if __name__ == "__main__":
+ unittest.main()