Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
setup: BUILD.json archs / vision_archs are read as a list or as the ';'-separated string the packaging writes (setup stopped on the released Windows CUDA package: int(';'), #3810)
Niko12210edd96ccf625
diff --git a/setup.py b/setup.pyindex f7a8c4212..974a7a977 100644--- a/setup.py+++ b/setup.py@@ -1578,6 +1578,30 @@ def check_gpus(sel, found, what="", yes=False, named=False) -> None:fail(f"GPU {i}{'' if g is None else ' (' + g['name'] + ')'} {what}cannot be used: {p}", hint)+def arch_numbers(value) -> list:+ """The CUDA architectures of a BUILD.json `archs` / `vision_archs` value, as numbers. The packaging recipes write the picture+ reader's as ONE string ("75;86;89;120", packaging/package_engine.py stamp_vision_archs, build_cuda_linux.sh), the engine's own as a+ list of numbers ([75, 86, 89, 120]); [60, "sm_61", "70-real", "compute_89"] and "60,61 70" are read too (#3810: a string was+ iterated character by character and setup stopped with int(';'))."""+ if value is None or value == "":+ return []+ parts = list(value) if isinstance(value, (list, tuple, set)) else re.split(r"[;,\s]+", str(value))+ out: list = []+ for part in parts:+ m = re.search(r"\d+", str(part))+ if m and int(m.group()) not in out:+ out.append(int(m.group()))+ return out+++def arch_names(value) -> list:+ """The AMD architectures ("gfx1100") of a BUILD.json `archs` / `vision_archs` value, as strings: a list or one ";" / "," separated string."""+ if value is None or value == "":+ return []+ parts = list(value) if isinstance(value, (list, tuple, set)) else re.split(r"[;,\s]+", str(value))+ return [str(p) for p in parts if str(p)]++def engine_archs(toolkit=13):"""The GPU generations the installed engine has code for: (archs, ptx), or None when there is none."""info = engine_dir(toolkit) / "BUILD.json"@@ -1585,7 +1609,7 @@ def engine_archs(toolkit=13):meta = json.loads(info.read_text(encoding="utf-8"))except (OSError, ValueError):return None- return [int(x) for x in meta.get("archs", [])], bool(meta.get("ptx"))+ return arch_numbers(meta.get("archs")), bool(meta.get("ptx"))def engine_archs_hip():@@ -1594,7 +1618,7 @@ def engine_archs_hip():meta = json.loads((ROOT / "engine" / "BUILD.json").read_text(encoding="utf-8"))except (OSError, ValueError):return None- return [str(x) for x in meta.get("archs", [])] if meta.get("backend") == "hip" else None+ return arch_names(meta.get("archs")) if meta.get("backend") == "hip" else Nonedef engine_runs_on(g, toolkit=13) -> bool:@@ -4169,7 +4193,7 @@ def get_prebuilt_hip(url_base, gpu, updating=False) -> Path | None:meta = {}ver = tuple(int(x) for x in str(meta.get("version", "0")).split(".")[:4] if x.isdigit())if meta.get("backend") == "hip" and meta.get("source") == "prebuilt" and ver >= WIN_HIP_MIN_ENGINE and \- gpu["arch"] in meta.get("archs", []) and not updating:+ gpu["arch"] in arch_numbers(meta.get("archs")) and not updating:ok("ready-made AMD engine already installed")return engif not url_base:@@ -4211,7 +4235,7 @@ def get_prebuilt_hip(url_base, gpu, updating=False) -> Path | None:why = "it is not a HIP engine"elif ver < WIN_HIP_MIN_ENGINE:why = f"it is version {meta.get('version')}; this setup needs {'.'.join(map(str, WIN_HIP_MIN_ENGINE))}"- elif gpu["arch"] not in meta.get("archs", []):+ elif gpu["arch"] not in arch_numbers(meta.get("archs")):why = (f"it is built for {', '.join(meta.get('archs', []))}; your GPU is {gpu['arch']}. A card it has no code ""for runs from a self-build of the engine: see docs/AMD_HIP.md (--build)") # #881if is_strix_halo(gpu): # never a zip without gfx1151 code: say so, and what to do@@ -4357,12 +4381,12 @@ def build_engine_hip(gpu, llama, vision="none") -> Path:meta = json.loads(stamp.read_text(encoding="utf-8")) if stamp.exists() else {}src, vsrc = source_hash(ENGINE_SOURCES), source_hash(VISION_SOURCES)archs = sorted(set(gpu.get("archs") or [gpu["arch"]]))- has_archs = set(archs) <= set(meta.get("archs", []))+ has_archs = set(archs) <= set(arch_names(meta.get("archs")))floor = cpu_floor(cpu_info()[1]) # "" on an AVX2 CPU: the normal engineengine_ok = meta.get("backend") == "hip" and (eng / EXE).exists() and meta.get("src") == src and has_archs and \(meta.get("isa_floor") or "") == floor and not stale_binary(eng / EXE)vision_ok = vision == "none" or ((eng / VEXE).exists() and meta.get("vision_src") == vsrc and- (vision != "gpu" or ((eng / HVEXE).exists() and set(archs) <= set(meta.get("vision_archs", [])))))+ (vision != "gpu" or ((eng / HVEXE).exists() and set(archs) <= set(arch_names(meta.get("vision_archs"))))))if engine_ok and vision_ok:ok("engine already built for this PC")return eng@@ -4422,8 +4446,8 @@ def reader_archs(eng: Path) -> list:except (OSError, ValueError):continuefor key in keys:- if isinstance(data, dict) and isinstance(data.get(key), list) and data[key]:- return [str(x) for x in data[key]]+ if isinstance(data, dict) and arch_names(data.get(key)):+ return arch_names(data.get(key))return []@@ -4936,7 +4960,7 @@ def get_prebuilt(url_base, gpu, vision, updating=False, toolkit=13) -> Path | Nover = tuple(int(x) for x in str(meta.get("version", "0")).split(".")[:4] if x.isdigit())if meta.get("source") == "local": # compiled here: build_engine checks its source and cardsreturn None- have = [int(a) for a in meta.get("archs", [])]+ have = arch_numbers(meta.get("archs"))miss = [int(x) for x in gpu.get("archs", [gpu["arch"]])if have and int(x) not in have and not (meta.get("ptx") and int(x) > max(have))]if miss: # a card it has no code for (#128): compiled here instead@@ -4997,7 +5021,7 @@ def get_prebuilt(url_base, gpu, vision, updating=False, toolkit=13) -> Path | Noshutil.rmtree(tmp, ignore_errors=True)drop_archive(z)return None- archs = [int(a) for a in meta.get("archs", [])]+ archs = arch_numbers(meta.get("archs"))miss = [int(x) for x in gpu.get("archs", [gpu["arch"]])if int(x) not in archs and not (meta.get("ptx") and int(x) > max(archs))]if miss:@@ -5106,11 +5130,11 @@ def update_installed_engine(url_base, toolkit=None) -> None:if meta.get("src") != source_hash(ENGINE_SOURCES) or stale_binary(eng / EXE):try:usable = [x for x in amd_gpus() if amd_problem(x) is None]- g = next((x for x in usable if x["arch"] in meta.get("archs", [])), usable[0] if usable else None)+ g = next((x for x in usable if x["arch"] in arch_names(meta.get("archs"))), usable[0] if usable else None)if g is None:raise RuntimeError("no supported AMD GPU found")# every architecture it was built for (a layer split across two families keeps both)- build_engine_hip({**g, "archs": [x for x in meta.get("archs", []) if x in AMD_ARCHS] or [g["arch"]]},+ build_engine_hip({**g, "archs": [x for x in arch_names(meta.get("archs")) if x in AMD_ARCHS] or [g["arch"]]},get_llama_cpp(), meta.get("vision") or "none")except (Exception, SystemExit) as e:warn(f"could not compile the updated engine{'' if isinstance(e, SystemExit) else f' ({e})'}: "@@ -5138,9 +5162,9 @@ def update_installed_engine(url_base, toolkit=None) -> None:try: # a failed compile must not stop the model from startingif gpu is None:raise RuntimeError("no NVIDIA GPU found")- gpu = {**gpu, "archs": sorted({int(gpu["arch"]), *(int(x) for x in meta.get("archs", []))})}+ gpu = {**gpu, "archs": sorted({int(gpu["arch"]), *arch_numbers(meta.get("archs"))})}if int(toolkit) == 12: # the cards it was compiled for (the main GPU may be newer)- gpu["archs"] = sorted({int(x) for x in meta.get("archs", [])}) or gpu["archs"]+ gpu["archs"] = sorted(arch_numbers(meta.get("archs"))) or gpu["archs"]build_engine(gpu, vision, False, get_llama_cpp(), toolkit=toolkit)except (Exception, SystemExit) as e:warn(f"could not compile the updated engine{'' if isinstance(e, SystemExit) else f' ({e})'}: starting the installed one")@@ -5149,7 +5173,7 @@ def update_installed_engine(url_base, toolkit=None) -> None:if gpu is not None:try:if int(toolkit) == 12: # the cards the CUDA 12 engine serves, not the newest one- gpu = {**gpu, "archs": [x for x in (int(a) for a in meta.get("archs", [])) if x < CUDA13_MIN_ARCH]+ gpu = {**gpu, "archs": [x for x in arch_numbers(meta.get("archs")) if x < CUDA13_MIN_ARCH]or [int(gpu["arch"])]}new = get_prebuilt(url_base, gpu, "gpu", updating=True, toolkit=toolkit)except Exception as e: # a failed download must not stop the model from starting@@ -5398,7 +5422,7 @@ def prebuilt_vision(meta: dict, gpu: dict, vision: str) -> str:of compiling one, which fails on most Windows PCs (no Visual Studio / CUDA toolkit); --build compiles it."""if vision != "gpu":return vision- va = [int(x) for x in meta.get("vision_archs", meta.get("archs", []))]+ va = arch_numbers(meta.get("vision_archs", meta.get("archs")))if va and int(gpu["arch"]) not in va and not (meta.get("ptx") and int(gpu["arch"]) > max(va)):warn(f"the ready-made image encoder has no code for your GPU (sm_{gpu['arch']}): it runs on the CPU instead ""(images take longer; setup --build compiles one for your GPU)")@@ -5423,7 +5447,7 @@ def build_engine(gpu, vision, yes, llama, toolkit=None) -> Path:local = meta.get("source") == "local"src, vsrc = source_hash(ENGINE_SOURCES), source_hash(VISION_SOURCES)archs = sorted({int(x) for x in gpu.get("archs", [gpu["arch"]])}) # every card the model runs on- built = {int(x) for x in meta.get("archs", [])}+ built = set(arch_numbers(meta.get("archs")))# a card the engine has no code for (a GPU added with --gpus, #128) needs a compile even when the source is the# same; the compile keeps the generations it was built fornew_arch = local and not set(archs) <= built@@ -7037,7 +7061,7 @@ def ensure_engine_for(cards, cfg_path: Path, cfg: dict, yes: bool) -> dict:say(" The installed engine has no code for " + ", ".join(f"{g['name']} (sm_{g['arch']})" for g in missing) +": it is compiled for " + ("these cards" if len(cards) > 1 else "it") + " now.")main = gpu_info(cards[0]["index"])- archs = sorted({int(x) for x in meta.get("archs", [])} | {int(g["arch"]) for g in cards})+ archs = sorted(set(arch_numbers(meta.get("archs"))) | {int(g["arch"]) for g in cards})vision = meta.get("vision") or ("gpu" if (eng / VEXE).exists() else "none")build_engine({**main, "archs": archs}, vision, yes, get_llama_cpp(), toolkit=tk)dirs = json.loads(info.read_text(encoding="utf-8")).get("cuda_dirs") or []diff --git a/tools/test_setup_choices.py b/tools/test_setup_choices.pyindex 998bd900e..db995bb1c 100644--- a/tools/test_setup_choices.py+++ b/tools/test_setup_choices.py@@ -56,6 +56,47 @@ class PrebuiltVision(unittest.TestCase):self.assertEqual(quiet(setup.prebuilt_vision, {"archs": [75, 86]}, {"arch": 75}, "gpu")[0], "gpu")+class VisionArchsAsAString(unittest.TestCase):+ """#3810: the packaging recipes write BUILD.json `vision_archs` as ONE string ("75;86;89;120"); 0.2.3 iterated it character by+ character and setup stopped with int(';') on the released Windows CUDA package. A string and a list mean the same."""+ RELEASED = {"version": "0.2.3", "archs": [75, 86, 89, 120], "ptx": True, "cuda": "13.0", "vision": "gpu-cuda13",+ "vision_archs": "75;86;89;120", "portable": True}++ def test_the_released_package_manifest_is_read(self):+ for arch in (75, 86, 89, 120):+ self.assertEqual(quiet(setup.prebuilt_vision, self.RELEASED, {"arch": arch}, "gpu")[0], "gpu")++ def test_a_card_the_string_does_not_list_gets_the_cpu_reader(self):+ meta = {**self.RELEASED, "vision_archs": "86;89;120", "ptx": False}+ got, out = quiet(setup.prebuilt_vision, meta, {"arch": 75}, "gpu")+ self.assertEqual(got, "cpu")+ self.assertIn("runs on the CPU instead", out)+ self.assertEqual(quiet(setup.prebuilt_vision, {**meta, "ptx": True}, {"arch": 120}, "gpu")[0], "gpu")++ def test_a_string_and_a_list_agree(self):+ for value in ("75;86;89;120", [75, 86, 89, 120], ["sm_75", "86-real", "89", "compute_120"], "75,86 89;120"):+ self.assertEqual(setup.arch_numbers(value), [75, 86, 89, 120], value)+ self.assertEqual(setup.arch_numbers(None), [])+ self.assertEqual(setup.arch_numbers(""), [])+ self.assertEqual(setup.arch_names("gfx1100;gfx1201"), ["gfx1100", "gfx1201"])+ self.assertEqual(setup.arch_names(["gfx1100"]), ["gfx1100"])+ self.assertEqual(setup.arch_names(None), [])++ def test_the_other_readers_of_the_manifest_take_a_string_too(self):+ import tempfile+ from pathlib import Path+ from unittest import mock+ with tempfile.TemporaryDirectory() as d:+ (Path(d) / "BUILD.json").write_text('{"archs": "60;61;70", "ptx": true}', encoding="utf-8")+ with mock.patch.object(setup, "engine_dir", return_value=Path(d)):+ self.assertEqual(setup.engine_archs(), ([60, 61, 70], True))+ (Path(d) / "BUILD.json").write_text('{"backend": "hip", "archs": ["gfx1100"], "vision_archs": "gfx1100;gfx1201"}', encoding="utf-8")+ self.assertEqual(setup.reader_archs(Path(d)), ["gfx1100", "gfx1201"])+ (Path(d) / "BUILD.json").write_text('{"backend": "hip", "archs": ["gfx1100", "gfx1201"]}', encoding="utf-8")+ self.assertEqual(setup.reader_archs(Path(d)), ["gfx1100", "gfx1201"])+ self.assertEqual(setup.reader_archs(Path(__file__).parent / "no-such-folder"), [])++class GgufDirShards(unittest.TestCase):"""#305: --gguf-dir reads the shard count from the -of-N part of the name instead of assuming two."""