Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
setup: AMD pictures on by default (the HIP reader with the CPU reader behind it, else the CPU reader), on Windows too; source build compiles the HIP reader; package_engine carries strata-vision-hip
Niko1221de9101935d3b
diff --git a/packaging/package_engine.py b/packaging/package_engine.pyindex 158202944..319f3cab8 100644--- a/packaging/package_engine.py+++ b/packaging/package_engine.py@@ -53,7 +53,7 @@ def sha256(path: Path) -> str:def wanted(path: Path) -> bool:name = path.name.lower()return (name.endswith((".dll", ".so", ".dylib", ".metallib")) or ".so." in name or name == "build.json"- or name.startswith("license") or name in ("strata-vision", "strata-vision.exe")) # the picture reader beside the engine (pictures need it)+ or name.startswith("license") or name in ("strata-vision", "strata-vision.exe", "strata-vision-hip", "strata-vision-hip.exe")) # the picture readers (CPU, AMD GPU) beside the enginedef git_build_id(repo: Path = ROOT) -> str:diff --git a/setup.py b/setup.pyindex 0c7ab554f..cdb4fb15c 100644--- a/setup.py+++ b/setup.py@@ -574,6 +574,7 @@ VISION = {"gpu": {"max_tokens": 1024, "reserve_mib": 700},"cpu": {"max_tokens": 300, "reserve_mib": 700}}EXE = "strata.exe" if WIN else "strata"VEXE = "strata-vision.exe" if WIN else "strata-vision"+HVEXE = "strata-vision-hip.exe" if WIN else "strata-vision-hip" # AMD: the picture reader on the GPU (tools/vision, STRATA_VISION_HIP)# MODEL-UPDATE Phase 1: the K-quant prompt kernels (Unsloth's UD-Q4/Q5/Q6_K_XL experts) are a library next to the CUDA# engine, opened only for those models; an engine without it runs them on the FP16 prompt pathKQ_LIB = "strata_kq.dll" if WIN else "libstrata_kq.so"@@ -3340,12 +3341,13 @@ def build_engine_hip(gpu, llama, vision="none") -> Path:floor = cpu_floor(cpu_info()[1]) # "" on an AVX2 CPU: the normal engineengine_ok = meta.get("backend") == "hip" and (eng / EXE).exists() and meta.get("src") == src and has_archs and \(meta.get("isa_floor") or "") == floor and not stale_binary(eng / EXE)- vision_ok = vision == "none" or ((eng / VEXE).exists() and meta.get("vision_src") == vsrc)+ vision_ok = vision == "none" or ((eng / VEXE).exists() and meta.get("vision_src") == vsrc and+ (vision != "gpu" or ((eng / HVEXE).exists() and set(archs) <= set(meta.get("vision_archs", [])))))if engine_ok and vision_ok:ok("engine already built for this PC")return engif engine_ok:- return build_vision_cpu(eng, stamp, meta, llama, vsrc)+ return build_vision_hip(eng, stamp, meta, llama, vsrc, archs, vision)if not (shutil.which("c++") or shutil.which("g++")) or not shutil.which("git"):fail("a C++ compiler and git are needed to compile the AMD engine","Ubuntu/Debian: sudo apt install build-essential git Fedora: sudo dnf install gcc-c++ git")@@ -3376,23 +3378,80 @@ def build_engine_hip(gpu, llama, vision="none") -> Path:meta = {"source": "local-hip", "backend": "hip", "version": source_version(), "archs": archs, "vision": "none","lib_dirs": dirs, "src": src, **({"isa_floor": floor} if floor else {})}if vision != "none":- return build_vision_cpu(eng, stamp, meta, llama, vsrc)+ return build_vision_hip(eng, stamp, meta, llama, vsrc, archs, vision)stamp.write_text(json.dumps(meta, indent=1))ok(f"engine compiled: {eng / EXE}")return engdef hip_vision(asked) -> str:- """The image encoder with the AMD backend (--vision): the CPU one when asked for (#304); a HIP (GPU) encoder build- is a later step, so `yes`/`gpu` leave images off, as before, and say how to get them."""- if asked in ("yes", "gpu"):- warn("the AMD backend has no GPU image encoder yet: images off"- + ("" if WIN else " (--vision cpu reads them on the CPU)"))- if asked == "cpu" and WIN:- warn("images on the CPU with an AMD card are Linux-only for now (the ready-made Windows AMD engine has no "- "image encoder): images off")+ """The picture reader wanted with the AMD backend (--vision): "gpu" for yes/gpu and when nothing was asked (pictures are on+ by default), "cpu" when asked for (#304), "none" for no. "gpu" is a wish: hip_reader() settles it against what the+ engine folder carries (the HIP reader, else the CPU one) once the engine is known."""+ if asked in ("no", "none"):return "none"- return "cpu" if asked == "cpu" else "none"+ return "cpu" if asked == "cpu" else "gpu"+++def reader_archs(eng: Path) -> list:+ """The GPU architectures the engine folder's HIP picture reader was built for: its BUILD.json (vision_archs, else archs),+ else the component manifest's archs; [] when neither says (the reader then checks the card itself when it starts)."""+ for name, keys in (("BUILD.json", ("vision_archs", "archs")), ("engine-component.json", ("archs",))):+ try:+ data = json.loads((eng / name).read_text(encoding="utf-8-sig"))+ except (OSError, ValueError):+ continue+ for key in keys:+ if isinstance(data, dict) and isinstance(data.get(key), list) and data[key]:+ return [str(x) for x in data[key]]+ return []+++def hip_reader(eng: Path, vision: str, gpu: dict):+ """AMD: the picture reader this engine folder gets for `vision` ("gpu" or "cpu"; strata_control/vision_reader.py): the HIP+ reader with the CPU reader to fall back on, else the CPU reader. Returns (vision, Reader); a Reader of mode "none" means+ the folder has no reader. The one plain note (a card the HIP reader has no code for, no HIP reader in this engine) is said."""+ from strata_control import vision_reader as VR+ r = VR.choose(eng, vision, windows=WIN, gpu_arch=str(gpu.get("arch") or ""), archs=reader_archs(eng))+ if r.note:+ warn(r.note)+ return r.mode, r+++def build_vision_hip(eng: Path, stamp: Path, meta: dict, llama, vsrc, archs: list, vision: str) -> Path:+ """The picture readers beside the HIP engine: the CPU reader always (the fallback), and for "gpu" the HIP reader+ (tools/vision, STRATA_VISION_HIP) for the same architectures. The HIP reader needs the ROCm environment the engine build+ sets; a build that fails leaves the CPU reader and says so (pictures still work, on the CPU)."""+ if vision == "gpu" and not (shutil.which("hipconfig") or os.environ.get("ROCM_PATH")):+ root, dirs = rocm_root(archs) # the engine was built earlier: set the same environment again+ os.environ.update({"HIP_PLATFORM": "amd", "HIP_COMPILER": "clang", "HIP_RUNTIME": "rocclr", "ROCM_PATH": str(root),+ "HIP_PATH": str(root)})+ os.environ["LD_LIBRARY_PATH"] = os.pathsep.join(dirs + [os.environ.get("LD_LIBRARY_PATH", "")]).rstrip(os.pathsep)+ os.environ["PATH"] = os.pathsep.join([str(root / "bin"), str(root / "llvm" / "bin"), os.environ.get("PATH", "")])+ build_vision_cpu(eng, stamp, meta, llama, vsrc)+ meta = json.loads(stamp.read_text(encoding="utf-8"))+ if vision != "gpu":+ return eng+ root, dirs = rocm_root(archs)+ libs = [str(Path(d).parent) for d in dirs[1:]]+ bitcode = next((p for p in (root / "lib" / "llvm" / "amdgcn" / "bitcode", root / "amdgcn" / "bitcode") if p.is_dir()),+ root / "amdgcn" / "bitcode")+ say(" Compiling the picture reader for your AMD GPU (a few minutes, once) ...")+ try:+ cmake_build(ROOT / "tools" / "vision", ROOT / "build-vision-hip", "strata-vision",+ [f"-DLLAMA_DIR={llama}", "-DSTRATA_VISION_HIP=ON", "-DSTRATA_VISION_HIP_ARCHS=" + ";".join(archs),+ "-DSTRATA_PORTABLE=OFF", f"-DCMAKE_HIP_COMPILER={root / 'llvm' / 'bin' / 'clang++'}",+ f"-DCMAKE_HIP_COMPILER_ROCM_ROOT={root}", "-DCMAKE_PREFIX_PATH=" + ";".join([str(root), *libs]),+ f"-DCMAKE_HIP_FLAGS=--rocm-path={root} --rocm-device-lib-path={bitcode}"], None, "")+ shutil.copy2(ROOT / "build-vision-hip" / "bin" / HVEXE, eng / HVEXE)+ except SystemExit:+ raise+ except Exception as e: # noqa: BLE001 - the CPU reader is there; pictures work without the GPU one+ warn(f"the GPU picture reader did not compile ({e}): pictures are read on the CPU")+ return eng+ stamp.write_text(json.dumps({**meta, "vision": "gpu", "vision_archs": archs, "vision_gpu": HVEXE, "vision_cpu": VEXE}, indent=1))+ ok(f"picture reader (GPU): {eng / HVEXE}, (CPU fallback): {eng / VEXE}")+ return engdef build_vision_cpu(eng: Path, stamp: Path, meta: dict, llama, vsrc) -> Path:@@ -4841,7 +4900,7 @@ SETUP_ENV = frozenset({"STRATA_HIPBLASLT_TUNING", "STRATA_ROCBLAS_TUNING", "STRA# user's own (STRATA_ROUTE_TAIL_SKIP=0 to turn the tail skip off, STRATA_MMQ_KQ=1 ...): a setup run again keeps them there# (0.1.44 dropped them with the rest of SETUP_ENV).SETUP_ENV_UNSLOTH = frozenset({"STRATA_MMQ_KQ", "STRATA_HC_Q8", "STRATA_ROUTE_TAIL_SKIP"})-SETUP_VISION = frozenset({"exe", "mmproj", "model", "gpu", "max_tokens", "threads"})+SETUP_VISION = frozenset({"exe", "cpu_exe", "mmproj", "model", "gpu", "max_tokens", "threads"})def carry_over(old: dict, cfg: dict) -> list[str]:@@ -6029,22 +6088,16 @@ def own_path_vision(a, fam: dict, gpu: dict, hip: bool) -> str:say()say(f" Images: {fam['title']} can also read pictures (screenshots, photos, scanned pages). This adds a "f"{fam['sha256'][fam['mmproj']][0] / 1e9:.1f} GB download" + (- "; the encoder runs on the CPU (no VRAM), a few seconds per picture." if hip else+ "; the encoder runs on the GPU when the engine has the AMD one (about 1 GB of VRAM), else on the CPU." if hip else" and the image encoder runs on the GPU (~1.4 GB of VRAM kept free, 0.1-0.5 s per picture)." if big else"; the encoder runs on the CPU (a few seconds per picture), because the GPU encoder would take ~1.4 GB ""of VRAM from the experts on this card."))- default = "n" if (hip and WIN) else "y" if (big or hip) else "n"+ default = "y" if (big or hip) else "n"if ask("Do you want images?", ["y", "n"], default, a.yes) != "y":return "none"- asked = "cpu" if (hip or not big) else "gpu"+ asked = "gpu" if hip else "cpu" if not big else "gpu"if hip:- if asked == "gpu":- say(" The AMD backend has no GPU image encoder yet: the encoder runs on the CPU.")- if WIN:- warn("images on the CPU with an AMD card are Linux-only for now (the ready-made Windows AMD engine has no "- "image encoder): images off")- return "none"- return "cpu"+ return hip_vision(asked) # settled against the engine's readers in hip_reader() (GPU reader, else CPU)if asked == "yes":asked = "gpu" if big else "cpu"if asked == "gpu" and not big and vram > 0:@@ -6207,7 +6260,10 @@ def install_own_path(a, family, gpu, hip, amd, cuda_tk, multi, port, ram, adoptestep(4, "the Strata engine")llama, eng, meta, lib_dirs, gpu, vision = engine_step(a, gpu, hip, amd, cuda_tk, vision)- if vision != "none" and not (eng / VEXE).exists():+ reader = None+ if hip and vision != "none":+ vision, reader = hip_reader(eng, vision, gpu)+ elif vision != "none" and not (eng / VEXE).exists():warn("this engine has no image encoder: images off (run setup again with --build to compile it)")vision = "none"if fam["arch"] not in engine_families(eng):@@ -6276,6 +6332,9 @@ def install_own_path(a, family, gpu, hip, amd, cuda_tk, multi, port, ram, adoptecfg["args"].append("--vision")cfg["vision"] = {"exe": str(eng / VEXE), "mmproj": str(mmproj), "model": str(gguf), "gpu": vision == "gpu","max_tokens": vt} # the text model, opened vocab-only+ if reader is not None: # AMD: the HIP reader with the CPU reader behind it, or the CPU reader+ from strata_control import vision_reader as VR+ cfg["vision"] = VR.vision_section(reader, cfg["vision"])if vision == "cpu":cfg["vision"]["threads"] = vision_threads()elif getattr(a, "vision_tokens", None) is not None:@@ -7261,6 +7320,9 @@ def main() -> int:# ---- 4. the enginestep(4, "the Strata engine")llama, eng, meta, lib_dirs, gpu, vision = engine_step(a, gpu, hip, amd, cuda_tk, vision)+ reader = None+ if hip and vision != "none": # AMD: the engine folder's reader (GPU, else CPU), settled now the engine is known+ vision, reader = hip_reader(eng, vision, gpu)if (ready or family == "gyro") and meta.get("source") != "local" and \tuple(int(x) for x in str(meta.get("version", "0")).split(".")[:4] if x.isdigit()) < MODELS[model]["engine"]:# a ready-made engine from before NVFP4 / Gyro cannot read these experts: one compiled from this source can@@ -7668,6 +7730,9 @@ def main() -> int:cfg["vision"] = {"exe": str(eng / VEXE), "mmproj": str(mmproj), # the text model, opened vocab-only:"model": str(ready_paths(model, models_dir)["dense"] if ready else shards[0]),"gpu": vision == "gpu", "max_tokens": vt}+ if reader is not None: # AMD: the HIP reader with the CPU reader behind it, or the CPU reader+ from strata_control import vision_reader as VR+ cfg["vision"] = VR.vision_section(reader, cfg["vision"])if vision == "cpu":cfg["vision"]["threads"] = vision_threads()elif a.vision_tokens is not None:diff --git a/tools/test_setup_amd.py b/tools/test_setup_amd.pyindex 6c0253e45..e8a75ecdc 100644--- a/tools/test_setup_amd.py+++ b/tools/test_setup_amd.py@@ -429,13 +429,37 @@ class CalibrationKey(unittest.TestCase):self.assertIsNone(setup.setup_calibration(cfg, hip=True)) # Windows HIP: defaults for now-class WindowsHipVision(unittest.TestCase):- def test_no_cpu_encoder_on_windows(self):- with mock.patch.object(setup, "WIN", True), mock.patch.object(setup, "warn", lambda *a: None):- self.assertEqual(setup.hip_vision("cpu"), "none")- self.assertEqual(setup.hip_vision("yes"), "none")- with mock.patch.object(setup, "WIN", False), mock.patch.object(setup, "warn", lambda *a: None):- self.assertEqual(setup.hip_vision("cpu"), "cpu")+class HipVisionWish(unittest.TestCase):+ """AMD pictures are on by default, on Linux and Windows: the wish is "gpu" (hip_reader() settles it against the readers the+ engine folder carries: the HIP one, else the CPU one), "cpu" when asked for, "none" for no."""++ def test_wish(self):+ for win in (True, False):+ with mock.patch.object(setup, "WIN", win):+ self.assertEqual(setup.hip_vision(None), "gpu")+ self.assertEqual(setup.hip_vision("yes"), "gpu")+ self.assertEqual(setup.hip_vision("gpu"), "gpu")+ self.assertEqual(setup.hip_vision("cpu"), "cpu")+ self.assertEqual(setup.hip_vision("no"), "none")+ self.assertEqual(setup.hip_vision("none"), "none")++ def test_reader_settled_against_the_engine_folder(self):+ for win, suffix in ((False, ""), (True, ".exe")):+ with tempfile.TemporaryDirectory() as d, mock.patch.object(setup, "WIN", win), mock.patch.object(setup, "warn", lambda *a: None):+ eng = Path(d)+ (eng / ("strata-vision" + suffix)).write_bytes(b"x")+ self.assertEqual(setup.hip_reader(eng, "gpu", {"arch": "gfx1201"})[0], "cpu") # no HIP reader: the CPU one+ (eng / ("strata-vision-hip" + suffix)).write_bytes(b"x")+ mode, r = setup.hip_reader(eng, "gpu", {"arch": "gfx1201"})+ self.assertEqual((mode, r.exe.name, r.cpu_exe.name),+ ("gpu", "strata-vision-hip" + suffix, "strata-vision" + suffix))+ (eng / "BUILD.json").write_text('{"vision_archs": ["gfx1100"]}')+ self.assertEqual(setup.hip_reader(eng, "gpu", {"arch": "gfx1201"})[0], "cpu") # the card is not in its build+ self.assertEqual(setup.hip_reader(eng, "gpu", {"arch": "gfx1100"})[0], "gpu")+ self.assertEqual(setup.hip_reader(eng, "cpu", {"arch": "gfx1100"})[0], "cpu")+ (eng / ("strata-vision" + suffix)).unlink()+ (eng / ("strata-vision-hip" + suffix)).unlink()+ self.assertEqual(setup.hip_reader(eng, "gpu", {"arch": "gfx1100"})[0], "none")class HipRuntimeBesideExe(unittest.TestCase):diff --git a/tools/test_setup_choices.py b/tools/test_setup_choices.pyindex 2e7540cfd..d2d431599 100644--- a/tools/test_setup_choices.py+++ b/tools/test_setup_choices.py@@ -318,15 +318,13 @@ class ExperimentalSm60(unittest.TestCase):class HipVision(unittest.TestCase):"""#304: --vision cpu with --backend hip builds the CPU image encoder beside the HIP engine."""- @mock.patch.object(setup, "WIN", False) # Linux: compiled here (Windows has no AMD image encoder yet)+ @mock.patch.object(setup, "WIN", False)def test_the_choice(self):self.assertEqual(quiet(setup.hip_vision, "cpu"), ("cpu", ""))- for asked in (None, "no", "none"):+ for asked in ("no", "none"):self.assertEqual(quiet(setup.hip_vision, asked), ("none", ""))- for asked in ("yes", "gpu"):- got, out = quiet(setup.hip_vision, asked)- self.assertEqual(got, "none")- self.assertIn("--vision cpu", out)+ for asked in (None, "yes", "gpu"): # pictures are on by default; the engine folder's readers settle gpu vs cpu+ self.assertEqual(quiet(setup.hip_vision, asked), ("gpu", ""))def build(self, meta, vision, vexe=False):"""build_engine_hip with an engine that is already built: only the encoder can be missing."""diff --git a/tools/test_setup_qwen36.py b/tools/test_setup_qwen36.pyindex 3106e6165..69bdd7268 100644--- a/tools/test_setup_qwen36.py+++ b/tools/test_setup_qwen36.py@@ -88,6 +88,7 @@ class InstallOwnPath(unittest.TestCase):self.root = Path(self.tmp.name)self.calls = []self.families = {"qwen4exp", "qwen35moe"}+ self.hip_reader = Trueeng = self.root / "engine"self.cfg = {}@@ -96,6 +97,8 @@ class InstallOwnPath(unittest.TestCase):if vision != "none": # the image encoder beside the engineeng.mkdir(parents=True, exist_ok=True)(eng / setup.VEXE).write_text("x")+ if hip and self.hip_reader: # an AMD engine package carries the GPU reader too+ (eng / setup.HVEXE).write_text("x")return self.root / "llama", eng, {"version": "0.1.44"}, ["/libs"], gpu, visiondef write_cfg(path, cfg, source=None):@@ -219,16 +222,36 @@ class InstallOwnPath(unittest.TestCase):self.assertTrue(self.cfg["vision"]["gpu"])self.assertIn("kept as you chose", out)- def test_amd_reads_pictures_on_the_cpu(self):- with mock.patch.object(setup, "WIN", False):+ def test_amd_reads_pictures_on_the_gpu_with_the_cpu_reader_behind_it(self):+ for win, suffix in ((False, ""), (True, ".exe")):+ with mock.patch.object(setup, "WIN", win), mock.patch.object(setup, "VEXE", "strata-vision" + suffix), mock.patch.object(setup, "HVEXE", "strata-vision-hip" + suffix):+ rc, out = self.install(model="UD-Q4_K_XL", vram=32.0, hip=True, vision="yes")+ self.assertEqual(rc, 0, out)+ v = self.cfg["vision"]+ self.assertTrue(v["gpu"])+ self.assertEqual(Path(v["exe"]).name, "strata-vision-hip" + suffix)+ self.assertEqual(Path(v["cpu_exe"]).name, "strata-vision" + suffix)+ self.assertIn("--vision", self.cfg["args"])+ rc, out = self.install(model="UD-Q4_K_XL", vram=32.0, hip=True, vision="cpu")+ v = self.cfg["vision"]+ self.assertFalse(v["gpu"])+ self.assertEqual(Path(v["exe"]).name, "strata-vision" + suffix)+ self.assertNotIn("cpu_exe", v)++ def test_amd_engine_without_the_gpu_reader_reads_pictures_on_the_cpu(self):+ self.hip_reader = False+ with mock.patch.object(setup, "WIN", False), mock.patch.object(setup, "VEXE", "strata-vision"):rc, out = self.install(model="UD-Q4_K_XL", vram=32.0, hip=True, vision="yes")self.assertEqual(rc, 0, out)self.assertFalse(self.cfg["vision"]["gpu"])+ self.assertNotIn("cpu_exe", self.cfg["vision"])self.assertIn("--vision", self.cfg["args"])- with mock.patch.object(setup, "WIN", True):- rc, out = self.install(model="UD-Q4_K_XL", vram=32.0, hip=True, vision="cpu")- self.assertNotIn("vision", self.cfg)- self.assertIn("Linux-only", out)++ def test_amd_pictures_are_on_when_nothing_is_asked(self):+ with mock.patch.object(setup, "WIN", False), mock.patch.object(setup, "VEXE", "strata-vision"), mock.patch.object(setup, "HVEXE", "strata-vision-hip"):+ rc, out = self.install(model="UD-Q4_K_XL", vram=32.0, hip=True)+ self.assertEqual(rc, 0, out)+ self.assertTrue(self.cfg["vision"]["gpu"])def test_parallel_is_opt_in_and_recommended_where_the_experts_fit(self):fam = setup.OWN_PATH_FAMILIES["qwen36"]