Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
picture reader choice for AMD engine folders and the server's fallback from the GPU reader to the CPU reader (one log line)
Niko1221f82fffbe25be
diff --git a/serve/server.py b/serve/server.pyindex 1b82c1023..bce59b6f6 100644--- a/serve/server.py+++ b/serve/server.py@@ -66,6 +66,7 @@ from serve.frontend import (THINK_END, ChatTemplate, Event, OutputParser, anthrofrom serve.mcp import McpCancelled, hub_from_config # noqa: E402from serve import runconfig # noqa: E402from strata_control import engine_component # noqa: E402+from strata_control import vision_reader # noqa: E402from serve import ready as ready_file # noqa: E402from serve import owner_guard # noqa: E402from serve import launch_identity # noqa: E402@@ -2487,18 +2488,13 @@ class Vision:return Path(tempfile.mkdtemp(prefix="strata-vision-"))def __init__(self, cfg: dict, log=None, env: dict | None = None, lazy: bool = False):- absolute = lambda p: os.path.abspath(p) if os.path.dirname(str(p)) else p # the encoder runs in its own dir (#480) # noqa: E731- args = [absolute(cfg["exe"]), "--mmproj", absolute(cfg["mmproj"]), "--model", absolute(cfg["model"])]- if cfg.get("gpu"):- args.append("--gpu")- if cfg.get("threads"):- args += ["--threads", str(cfg["threads"])]- if cfg.get("max_tokens"):- args += ["--max-tokens", str(cfg["max_tokens"])]- if cfg.get("min_tokens"): # #767: mtmd's image_min_tokens (a hand-edited key)- args += ["--min-tokens", str(cfg["min_tokens"])]+ args = self.reader_args(cfg)self.dir = self.work_dir()self.spawn = (args, log, env) # to start it again after an unload+ # AMD: the GPU reader (strata-vision-hip) has the CPU reader (strata-vision, same program, no GPU runtime) to+ # fall back on when it does not start; _start switches to it by itself+ cpu = vision_reader.cpu_fallback(cfg)+ self.fallback = (self.reader_args(cpu), log, env) if cpu else Noneself.proc = Noneself.stopped = Trueself.lock = threading.Lock()@@ -2512,7 +2508,36 @@ class Vision:if not lazy and not self.elastic:self._start()+ @staticmethod+ def reader_args(cfg: dict) -> list:+ absolute = lambda p: os.path.abspath(p) if os.path.dirname(str(p)) else p # the encoder runs in its own dir (#480) # noqa: E731+ args = [absolute(cfg["exe"]), "--mmproj", absolute(cfg["mmproj"]), "--model", absolute(cfg["model"])]+ if cfg.get("gpu"):+ args.append("--gpu")+ if cfg.get("threads"):+ args += ["--threads", str(cfg["threads"])]+ if cfg.get("max_tokens"):+ args += ["--max-tokens", str(cfg["max_tokens"])]+ if cfg.get("min_tokens"): # #767: mtmd's image_min_tokens (a hand-edited key)+ args += ["--min-tokens", str(cfg["min_tokens"])]+ return args+def _start(self):+ """Start the reader. When a GPU reader that has a CPU reader to fall back on does not start (the GPU runtime does not+ load, the card is not in its build, the card is busy), the CPU reader is started instead, for good, with one log line+ and no question."""+ try:+ self._start_reader()+ except (RuntimeError, OSError) as e:+ fallback = getattr(self, "fallback", None)+ if fallback is None:+ raise+ why = " ".join(str(e).split())[:240]+ print(f"[strata] the GPU picture reader did not start ({why}); pictures are read on the CPU", flush=True)+ self.spawn, self.fallback = fallback, None+ self._start_reader()++ def _start_reader(self):args, log, env = self.spawn# cwd is the encoder's directory: the ENC lines name the files relative to it (#480: no space in the line)self.proc = popen("the image encoder", args, role="vision", stdin=subprocess.PIPE, stdout=subprocess.PIPE,diff --git a/strata_control/vision_reader.py b/strata_control/vision_reader.pynew file mode 100644index 000000000..cd0001941--- /dev/null+++ b/strata_control/vision_reader.py@@ -0,0 +1,101 @@+"""Which picture reader an engine folder gets: the GPU one (`strata-vision-hip`, AMD) or the CPU one (`strata-vision`).++An AMD engine package carries two programs for pictures. `strata-vision` is the CPU reader: it needs nothing from the GPU+runtime, so it starts on every PC. `strata-vision-hip` is the same reader on the card (llama.cpp's HIP backend), a+fraction of a second per picture instead of seconds. Setup writes the run config with the GPU reader first and the CPU+reader as its fallback (`vision.cpu_exe`); the server (serve/server.py, Vision) starts the CPU reader by itself, with one+log line, when the GPU reader does not start: the HIP runtime does not load, the card is not one the build has code for+(the reader says so before it runs a kernel), or the card is busy. Nothing is asked of the user.++`choose` is the whole decision; setup.py calls it, and tools/test_vision_reader.py checks every branch.+"""+from __future__ import annotations++import re+from dataclasses import dataclass+from pathlib import Path++CPU_READER = "strata-vision"+GPU_READER = "strata-vision-hip"++GFX = re.compile(r"gfx[0-9a-f]+", re.IGNORECASE)+++def reader_name(base: str, windows: bool) -> str:+ return base + (".exe" if windows else "")+++def find(folder: Path | str, windows: bool) -> dict:+ """The readers in an engine folder: {"cpu": Path | None, "gpu": Path | None}."""+ folder = Path(folder)+ found = {}+ for key, base in (("cpu", CPU_READER), ("gpu", GPU_READER)):+ path = folder / reader_name(base, windows)+ found[key] = path if path.is_file() else None+ return found+++def gfx_of(text: object) -> str:+ """"gfx1201" out of a gcnArchName ("gfx1201:sramecc-:xnack-"), a device line or a plain arch; "" when there is none."""+ m = GFX.search(str(text or ""))+ return m.group(0).lower() if m else ""+++@dataclass+class Reader:+ mode: str # "gpu" | "cpu" | "none"+ exe: Path | None = None # the program the run config starts+ cpu_exe: Path | None = None # the CPU reader to fall back on when a GPU reader does not start (mode "gpu" only)+ note: str = "" # one plain sentence for the setup output when it is not what was asked for+++def choose(folder: Path | str, asked: str, *, windows: bool, gpu_arch: str = "", archs: list | None = None) -> Reader:+ """The reader for `asked` ("yes"/"on"/"gpu" = the best the folder has, "cpu", "no"/"none"/"off").++ gpu_arch: the card's architecture when setup knows it ("gfx1201"); archs: what the GPU reader was built for+ (BUILD.json `vision_archs`, else the package's `archs`). A card the build does not list gets the CPU reader here;+ when the card is unknown the GPU reader is tried and falls back by itself (see the module doc)."""+ asked = (asked or "").lower()+ if asked in ("no", "none", "off", ""):+ return Reader("none")+ have = find(folder, windows)+ if asked == "cpu":+ if have["cpu"]:+ return Reader("cpu", have["cpu"])+ return Reader("none", note="this engine has no picture reader: pictures off")+ # yes / on / gpu+ gfx = gfx_of(gpu_arch)+ built = [gfx_of(a) for a in (archs or []) if gfx_of(a)]+ if have["gpu"] is None:+ if have["cpu"]:+ return Reader("cpu", have["cpu"], note="this engine has no GPU picture reader: pictures are read on the CPU "+ "(a few seconds each)" if asked == "gpu" else "")+ return Reader("none", note="this engine has no picture reader: pictures off")+ if gfx and built and gfx not in built:+ if have["cpu"]:+ return Reader("cpu", have["cpu"], note=f"the GPU picture reader has no code for your card ({gfx}): pictures "+ "are read on the CPU (a few seconds each)")+ return Reader("none", note=f"the GPU picture reader has no code for your card ({gfx}): pictures off")+ return Reader("gpu", have["gpu"], have["cpu"])+++def vision_section(reader: Reader, base: dict) -> dict:+ """The run config's "vision" section for `reader` on top of `base` (mmproj, model, max_tokens ...): exe, gpu, and for the+ GPU reader the CPU reader to fall back on."""+ out = dict(base)+ out["exe"] = str(reader.exe)+ out["gpu"] = reader.mode == "gpu"+ if reader.mode == "gpu" and reader.cpu_exe:+ out["cpu_exe"] = str(reader.cpu_exe)+ else:+ out.pop("cpu_exe", None)+ return out+++def cpu_fallback(vision_cfg: dict) -> dict | None:+ """The same section for the CPU reader, when the GPU reader has one to fall back on; None otherwise."""+ if not vision_cfg.get("gpu") or not vision_cfg.get("cpu_exe"):+ return None+ out = {k: v for k, v in vision_cfg.items() if k not in ("cpu_exe", "elastic", "elastic_mib", "elastic_idle_s")}+ out["exe"], out["gpu"] = vision_cfg["cpu_exe"], False+ return outdiff --git a/tools/test_vision_reader.py b/tools/test_vision_reader.pynew file mode 100644index 000000000..1f77b0c91--- /dev/null+++ b/tools/test_vision_reader.py@@ -0,0 +1,202 @@+"""The picture reader choice for an AMD engine folder (strata_control/vision_reader.py) and the server's fallback from the GPU+reader to the CPU reader (serve/server.py, Vision._start): hip present, hip missing, hip start failure, then cpu.++ python -m unittest tools.test_vision_reader+"""+from __future__ import annotations++import io+import tempfile+import unittest+from pathlib import Path+from types import SimpleNamespace+from unittest import mock++from strata_control import vision_reader as VR+import serve.server as server+++def folder(*names: str) -> Path:+ d = Path(tempfile.mkdtemp(prefix="vr-"))+ for n in names:+ (d / n).write_bytes(b"x")+ return d+++class Choose(unittest.TestCase):+ def test_hip_present_gives_the_gpu_reader_with_the_cpu_one_to_fall_back_on(self):+ d = folder("strata-vision", "strata-vision-hip")+ for ask in ("yes", "on", "gpu"):+ r = VR.choose(d, ask, windows=False, gpu_arch="gfx1201", archs=["gfx1200", "gfx1201"])+ self.assertEqual((r.mode, r.exe.name, r.cpu_exe.name), ("gpu", "strata-vision-hip", "strata-vision"), ask)+ self.assertEqual(r.note, "")++ def test_hip_missing_gives_the_cpu_reader(self):+ d = folder("strata-vision")+ r = VR.choose(d, "yes", windows=False, gpu_arch="gfx1201")+ self.assertEqual((r.mode, r.exe.name, r.cpu_exe), ("cpu", "strata-vision", None))+ self.assertEqual(r.note, "") # "yes" never asked for the GPU in particular+ self.assertIn("no GPU picture reader", VR.choose(d, "gpu", windows=False).note) # --vision gpu: said once++ def test_card_not_in_the_build_gives_the_cpu_reader(self):+ d = folder("strata-vision", "strata-vision-hip")+ r = VR.choose(d, "yes", windows=False, gpu_arch="gfx1100", archs=["gfx1200", "gfx1201"])+ self.assertEqual((r.mode, r.exe.name), ("cpu", "strata-vision"))+ self.assertIn("gfx1100", r.note)++ def test_unknown_card_tries_the_gpu_reader(self):+ d = folder("strata-vision", "strata-vision-hip")+ self.assertEqual(VR.choose(d, "yes", windows=False, gpu_arch="", archs=["gfx1201"]).mode, "gpu")+ self.assertEqual(VR.choose(d, "yes", windows=False, gpu_arch="gfx1100", archs=None).mode, "gpu")++ def test_gcn_arch_names_with_features_match(self):+ self.assertEqual(VR.gfx_of("gfx1201:sramecc-:xnack-"), "gfx1201")+ d = folder("strata-vision", "strata-vision-hip")+ self.assertEqual(VR.choose(d, "yes", windows=False, gpu_arch="gfx1151:sramecc+:xnack-", archs=["gfx1151"]).mode, "gpu")++ def test_cpu_asked_and_off(self):+ d = folder("strata-vision", "strata-vision-hip")+ self.assertEqual(VR.choose(d, "cpu", windows=False).exe.name, "strata-vision")+ for ask in ("no", "none", "off", None):+ self.assertEqual(VR.choose(d, ask, windows=False).mode, "none")++ def test_no_reader_at_all(self):+ d = folder("strata")+ for ask in ("yes", "cpu", "gpu"):+ r = VR.choose(d, ask, windows=False)+ self.assertEqual(r.mode, "none", ask)+ self.assertIn("no picture reader", r.note)++ def test_windows_names(self):+ d = folder("strata-vision.exe", "strata-vision-hip.exe")+ r = VR.choose(d, "yes", windows=True, gpu_arch="gfx1201", archs=["gfx1201"])+ self.assertEqual((r.mode, r.exe.name, r.cpu_exe.name), ("gpu", "strata-vision-hip.exe", "strata-vision.exe"))+ self.assertEqual(VR.choose(folder("strata-vision-hip"), "yes", windows=True).mode, "none") # a Linux name on Windows++ def test_run_config_section(self):+ d = folder("strata-vision", "strata-vision-hip")+ base = {"mmproj": "m.gguf", "model": "t.gguf", "max_tokens": 1024}+ sec = VR.vision_section(VR.choose(d, "yes", windows=False), base)+ self.assertTrue(sec["gpu"])+ self.assertEqual(Path(sec["exe"]).name, "strata-vision-hip")+ self.assertEqual(Path(sec["cpu_exe"]).name, "strata-vision")+ cpu = VR.cpu_fallback(sec)+ self.assertFalse(cpu["gpu"])+ self.assertEqual(Path(cpu["exe"]).name, "strata-vision")+ self.assertNotIn("cpu_exe", cpu)+ self.assertEqual(cpu["mmproj"], "m.gguf")+ cpu_only = VR.vision_section(VR.choose(folder("strata-vision"), "yes", windows=False), base)+ self.assertFalse(cpu_only["gpu"])+ self.assertNotIn("cpu_exe", cpu_only)+ self.assertIsNone(VR.cpu_fallback(cpu_only))+ self.assertIsNone(VR.cpu_fallback({**sec, "cpu_exe": ""}))+ self.assertIsNone(VR.cpu_fallback({"exe": "strata-vision", "gpu": True})) # an NVIDIA config has no fallback+++class FakeProc:+ def __init__(self, first_line: str):+ self.stdin, self.stdout = io.StringIO(), io.StringIO(first_line)+ self.killed = False++ def kill(self):+ self.killed = True++ def wait(self, timeout=None):+ return 0++ def poll(self):+ return None+++def make_vision(cfg: dict) -> "server.Vision":+ with mock.patch.object(server.Vision, "work_dir", staticmethod(lambda: Path(tempfile.gettempdir()))):+ return server.Vision(cfg, lazy=True)+++class Fallback(unittest.TestCase):+ CFG = {"exe": "/e/strata-vision-hip", "cpu_exe": "/e/strata-vision", "gpu": True, "mmproj": "/m/mm.gguf",+ "model": "/m/t.gguf", "max_tokens": 1024, "threads": 6}++ def run_start(self, answers: dict, cfg: dict | None = None):+ """answers: exe name -> its first stdout line, or an exception to raise from popen."""+ started = []++ def popen(what, args, **kw):+ name = Path(args[0]).name+ started.append(args)+ a = answers[name]+ if isinstance(a, Exception):+ raise a+ return FakeProc(a)++ v = make_vision(cfg or self.CFG)+ out = io.StringIO()+ with mock.patch.object(server, "popen", popen), mock.patch("builtins.print", lambda *a, **k: out.write(" ".join(map(str, a)) + "\n")):+ err = None+ try:+ v._start()+ except RuntimeError as e:+ err = e+ return v, started, out.getvalue(), err++ def test_hip_present_and_starts(self):+ v, started, log, err = self.run_start({"strata-vision-hip": "READY 2048\n"})+ self.assertIsNone(err)+ self.assertEqual([Path(a[0]).name for a in started], ["strata-vision-hip"])+ self.assertIn("--gpu", started[0])+ self.assertEqual(log, "")+ self.assertFalse(v.stopped)++ def test_hip_start_failure_falls_back_to_cpu_with_one_line(self):+ v, started, log, err = self.run_start({"strata-vision-hip": "ERR this GPU (gfx1100) is not in the picture reader's build (gfx1200,gfx1201)\n",+ "strata-vision": "READY 2048\n"})+ self.assertIsNone(err)+ self.assertEqual([Path(a[0]).name for a in started], ["strata-vision-hip", "strata-vision"])+ self.assertNotIn("--gpu", started[1])+ self.assertIn("--threads", started[1])+ self.assertEqual(log.count("\n"), 1)+ self.assertIn("gfx1100", log)+ self.assertIn("CPU", log)+ self.assertFalse(v.stopped)+ self.assertIsNone(v.fallback)++ def test_hip_process_that_dies_silently_falls_back(self): # a GPU runtime that aborts: stdout closes empty+ v, started, log, err = self.run_start({"strata-vision-hip": "", "strata-vision": "READY 2048\n"})+ self.assertIsNone(err)+ self.assertEqual(len(started), 2)+ self.assertEqual(log.count("\n"), 1)++ def test_hip_missing_file_falls_back(self): # popen raises (exe gone, not executable)+ v, started, log, err = self.run_start({"strata-vision-hip": FileNotFoundError("strata-vision-hip"),+ "strata-vision": "READY 2048\n"})+ self.assertIsNone(err)+ self.assertEqual(len(started), 2)++ def test_restart_after_a_fallback_stays_on_the_cpu_reader(self):+ v, started, log, err = self.run_start({"strata-vision-hip": "ERR nope\n", "strata-vision": "READY 2048\n"})+ started.clear()+ with mock.patch.object(server, "popen", lambda what, args, **kw: started.append(args) or FakeProc("READY 2048\n")):+ v.restart()+ self.assertEqual([Path(a[0]).name for a in started], ["strata-vision"])++ def test_cpu_failure_after_the_fallback_is_an_error(self):+ v, started, log, err = self.run_start({"strata-vision-hip": "ERR nope\n", "strata-vision": "ERR cannot load the vision encoder\n"})+ self.assertIsNotNone(err)+ self.assertIn("cannot load", str(err))++ def test_a_config_without_cpu_exe_has_no_fallback(self): # NVIDIA: the error is shown as before+ cfg = {k: v for k, v in self.CFG.items() if k != "cpu_exe"}+ v, started, log, err = self.run_start({"strata-vision-hip": "ERR nope\n"}, cfg)+ self.assertIsNotNone(err)+ self.assertEqual(len(started), 1)+ self.assertEqual(log, "")++ def test_cpu_config_is_unchanged(self):+ cfg = {"exe": "/e/strata-vision", "gpu": False, "mmproj": "/m/mm.gguf", "model": "/m/t.gguf"}+ v, started, log, err = self.run_start({"strata-vision": "READY 2048\n"}, cfg)+ self.assertIsNone(err)+ self.assertNotIn("--gpu", started[0])+++if __name__ == "__main__":+ unittest.main()