Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

setup: BUILD.json archs / vision_archs are read as a list or as the ';'-separated string the packaging writes (setup stopped on the released Windows CUDA package: int(';'), #3810)

Niko12210edd96ccf625
diff --git a/setup.py b/setup.py
index f7a8c4212..974a7a977 100644
--- a/setup.py
+++ b/setup.py
@@ -1578,6 +1578,30 @@ def check_gpus(sel, found, what="", yes=False, named=False) -> None:
fail(f"GPU {i}{'' if g is None else ' (' + g['name'] + ')'} {what}cannot be used: {p}", hint)
+def arch_numbers(value) -> list:
+ """The CUDA architectures of a BUILD.json `archs` / `vision_archs` value, as numbers. The packaging recipes write the picture
+ reader's as ONE string ("75;86;89;120", packaging/package_engine.py stamp_vision_archs, build_cuda_linux.sh), the engine's own as a
+ list of numbers ([75, 86, 89, 120]); [60, "sm_61", "70-real", "compute_89"] and "60,61 70" are read too (#3810: a string was
+ iterated character by character and setup stopped with int(';'))."""
+ if value is None or value == "":
+ return []
+ parts = list(value) if isinstance(value, (list, tuple, set)) else re.split(r"[;,\s]+", str(value))
+ out: list = []
+ for part in parts:
+ m = re.search(r"\d+", str(part))
+ if m and int(m.group()) not in out:
+ out.append(int(m.group()))
+ return out
+
+
+def arch_names(value) -> list:
+ """The AMD architectures ("gfx1100") of a BUILD.json `archs` / `vision_archs` value, as strings: a list or one ";" / "," separated string."""
+ if value is None or value == "":
+ return []
+ parts = list(value) if isinstance(value, (list, tuple, set)) else re.split(r"[;,\s]+", str(value))
+ return [str(p) for p in parts if str(p)]
+
+
def engine_archs(toolkit=13):
"""The GPU generations the installed engine has code for: (archs, ptx), or None when there is none."""
info = engine_dir(toolkit) / "BUILD.json"
@@ -1585,7 +1609,7 @@ def engine_archs(toolkit=13):
meta = json.loads(info.read_text(encoding="utf-8"))
except (OSError, ValueError):
return None
- return [int(x) for x in meta.get("archs", [])], bool(meta.get("ptx"))
+ return arch_numbers(meta.get("archs")), bool(meta.get("ptx"))
def engine_archs_hip():
@@ -1594,7 +1618,7 @@ def engine_archs_hip():
meta = json.loads((ROOT / "engine" / "BUILD.json").read_text(encoding="utf-8"))
except (OSError, ValueError):
return None
- return [str(x) for x in meta.get("archs", [])] if meta.get("backend") == "hip" else None
+ return arch_names(meta.get("archs")) if meta.get("backend") == "hip" else None
def engine_runs_on(g, toolkit=13) -> bool:
@@ -4169,7 +4193,7 @@ def get_prebuilt_hip(url_base, gpu, updating=False) -> Path | None:
meta = {}
ver = tuple(int(x) for x in str(meta.get("version", "0")).split(".")[:4] if x.isdigit())
if meta.get("backend") == "hip" and meta.get("source") == "prebuilt" and ver >= WIN_HIP_MIN_ENGINE and \
- gpu["arch"] in meta.get("archs", []) and not updating:
+ gpu["arch"] in arch_numbers(meta.get("archs")) and not updating:
ok("ready-made AMD engine already installed")
return eng
if not url_base:
@@ -4211,7 +4235,7 @@ def get_prebuilt_hip(url_base, gpu, updating=False) -> Path | None:
why = "it is not a HIP engine"
elif ver < WIN_HIP_MIN_ENGINE:
why = f"it is version {meta.get('version')}; this setup needs {'.'.join(map(str, WIN_HIP_MIN_ENGINE))}"
- elif gpu["arch"] not in meta.get("archs", []):
+ elif gpu["arch"] not in arch_numbers(meta.get("archs")):
why = (f"it is built for {', '.join(meta.get('archs', []))}; your GPU is {gpu['arch']}. A card it has no code "
"for runs from a self-build of the engine: see docs/AMD_HIP.md (--build)") # #881
if is_strix_halo(gpu): # never a zip without gfx1151 code: say so, and what to do
@@ -4357,12 +4381,12 @@ def build_engine_hip(gpu, llama, vision="none") -> Path:
meta = json.loads(stamp.read_text(encoding="utf-8")) if stamp.exists() else {}
src, vsrc = source_hash(ENGINE_SOURCES), source_hash(VISION_SOURCES)
archs = sorted(set(gpu.get("archs") or [gpu["arch"]]))
- has_archs = set(archs) <= set(meta.get("archs", []))
+ has_archs = set(archs) <= set(arch_names(meta.get("archs")))
floor = cpu_floor(cpu_info()[1]) # "" on an AVX2 CPU: the normal engine
engine_ok = meta.get("backend") == "hip" and (eng / EXE).exists() and meta.get("src") == src and has_archs and \
(meta.get("isa_floor") or "") == floor and not stale_binary(eng / EXE)
vision_ok = vision == "none" or ((eng / VEXE).exists() and meta.get("vision_src") == vsrc and
- (vision != "gpu" or ((eng / HVEXE).exists() and set(archs) <= set(meta.get("vision_archs", [])))))
+ (vision != "gpu" or ((eng / HVEXE).exists() and set(archs) <= set(arch_names(meta.get("vision_archs"))))))
if engine_ok and vision_ok:
ok("engine already built for this PC")
return eng
@@ -4422,8 +4446,8 @@ def reader_archs(eng: Path) -> list:
except (OSError, ValueError):
continue
for key in keys:
- if isinstance(data, dict) and isinstance(data.get(key), list) and data[key]:
- return [str(x) for x in data[key]]
+ if isinstance(data, dict) and arch_names(data.get(key)):
+ return arch_names(data.get(key))
return []
@@ -4936,7 +4960,7 @@ def get_prebuilt(url_base, gpu, vision, updating=False, toolkit=13) -> Path | No
ver = tuple(int(x) for x in str(meta.get("version", "0")).split(".")[:4] if x.isdigit())
if meta.get("source") == "local": # compiled here: build_engine checks its source and cards
return None
- have = [int(a) for a in meta.get("archs", [])]
+ have = arch_numbers(meta.get("archs"))
miss = [int(x) for x in gpu.get("archs", [gpu["arch"]])
if have and int(x) not in have and not (meta.get("ptx") and int(x) > max(have))]
if miss: # a card it has no code for (#128): compiled here instead
@@ -4997,7 +5021,7 @@ def get_prebuilt(url_base, gpu, vision, updating=False, toolkit=13) -> Path | No
shutil.rmtree(tmp, ignore_errors=True)
drop_archive(z)
return None
- archs = [int(a) for a in meta.get("archs", [])]
+ archs = arch_numbers(meta.get("archs"))
miss = [int(x) for x in gpu.get("archs", [gpu["arch"]])
if int(x) not in archs and not (meta.get("ptx") and int(x) > max(archs))]
if miss:
@@ -5106,11 +5130,11 @@ def update_installed_engine(url_base, toolkit=None) -> None:
if meta.get("src") != source_hash(ENGINE_SOURCES) or stale_binary(eng / EXE):
try:
usable = [x for x in amd_gpus() if amd_problem(x) is None]
- g = next((x for x in usable if x["arch"] in meta.get("archs", [])), usable[0] if usable else None)
+ g = next((x for x in usable if x["arch"] in arch_names(meta.get("archs"))), usable[0] if usable else None)
if g is None:
raise RuntimeError("no supported AMD GPU found")
# every architecture it was built for (a layer split across two families keeps both)
- build_engine_hip({**g, "archs": [x for x in meta.get("archs", []) if x in AMD_ARCHS] or [g["arch"]]},
+ build_engine_hip({**g, "archs": [x for x in arch_names(meta.get("archs")) if x in AMD_ARCHS] or [g["arch"]]},
get_llama_cpp(), meta.get("vision") or "none")
except (Exception, SystemExit) as e:
warn(f"could not compile the updated engine{'' if isinstance(e, SystemExit) else f' ({e})'}: "
@@ -5138,9 +5162,9 @@ def update_installed_engine(url_base, toolkit=None) -> None:
try: # a failed compile must not stop the model from starting
if gpu is None:
raise RuntimeError("no NVIDIA GPU found")
- gpu = {**gpu, "archs": sorted({int(gpu["arch"]), *(int(x) for x in meta.get("archs", []))})}
+ gpu = {**gpu, "archs": sorted({int(gpu["arch"]), *arch_numbers(meta.get("archs"))})}
if int(toolkit) == 12: # the cards it was compiled for (the main GPU may be newer)
- gpu["archs"] = sorted({int(x) for x in meta.get("archs", [])}) or gpu["archs"]
+ gpu["archs"] = sorted(arch_numbers(meta.get("archs"))) or gpu["archs"]
build_engine(gpu, vision, False, get_llama_cpp(), toolkit=toolkit)
except (Exception, SystemExit) as e:
warn(f"could not compile the updated engine{'' if isinstance(e, SystemExit) else f' ({e})'}: starting the installed one")
@@ -5149,7 +5173,7 @@ def update_installed_engine(url_base, toolkit=None) -> None:
if gpu is not None:
try:
if int(toolkit) == 12: # the cards the CUDA 12 engine serves, not the newest one
- gpu = {**gpu, "archs": [x for x in (int(a) for a in meta.get("archs", [])) if x < CUDA13_MIN_ARCH]
+ gpu = {**gpu, "archs": [x for x in arch_numbers(meta.get("archs")) if x < CUDA13_MIN_ARCH]
or [int(gpu["arch"])]}
new = get_prebuilt(url_base, gpu, "gpu", updating=True, toolkit=toolkit)
except Exception as e: # a failed download must not stop the model from starting
@@ -5398,7 +5422,7 @@ def prebuilt_vision(meta: dict, gpu: dict, vision: str) -> str:
of compiling one, which fails on most Windows PCs (no Visual Studio / CUDA toolkit); --build compiles it."""
if vision != "gpu":
return vision
- va = [int(x) for x in meta.get("vision_archs", meta.get("archs", []))]
+ va = arch_numbers(meta.get("vision_archs", meta.get("archs")))
if va and int(gpu["arch"]) not in va and not (meta.get("ptx") and int(gpu["arch"]) > max(va)):
warn(f"the ready-made image encoder has no code for your GPU (sm_{gpu['arch']}): it runs on the CPU instead "
"(images take longer; setup --build compiles one for your GPU)")
@@ -5423,7 +5447,7 @@ def build_engine(gpu, vision, yes, llama, toolkit=None) -> Path:
local = meta.get("source") == "local"
src, vsrc = source_hash(ENGINE_SOURCES), source_hash(VISION_SOURCES)
archs = sorted({int(x) for x in gpu.get("archs", [gpu["arch"]])}) # every card the model runs on
- built = {int(x) for x in meta.get("archs", [])}
+ built = set(arch_numbers(meta.get("archs")))
# a card the engine has no code for (a GPU added with --gpus, #128) needs a compile even when the source is the
# same; the compile keeps the generations it was built for
new_arch = local and not set(archs) <= built
@@ -7037,7 +7061,7 @@ def ensure_engine_for(cards, cfg_path: Path, cfg: dict, yes: bool) -> dict:
say(" The installed engine has no code for " + ", ".join(f"{g['name']} (sm_{g['arch']})" for g in missing) +
": it is compiled for " + ("these cards" if len(cards) > 1 else "it") + " now.")
main = gpu_info(cards[0]["index"])
- archs = sorted({int(x) for x in meta.get("archs", [])} | {int(g["arch"]) for g in cards})
+ archs = sorted(set(arch_numbers(meta.get("archs"))) | {int(g["arch"]) for g in cards})
vision = meta.get("vision") or ("gpu" if (eng / VEXE).exists() else "none")
build_engine({**main, "archs": archs}, vision, yes, get_llama_cpp(), toolkit=tk)
dirs = json.loads(info.read_text(encoding="utf-8")).get("cuda_dirs") or []
diff --git a/tools/test_setup_choices.py b/tools/test_setup_choices.py
index 998bd900e..db995bb1c 100644
--- a/tools/test_setup_choices.py
+++ b/tools/test_setup_choices.py
@@ -56,6 +56,47 @@ class PrebuiltVision(unittest.TestCase):
self.assertEqual(quiet(setup.prebuilt_vision, {"archs": [75, 86]}, {"arch": 75}, "gpu")[0], "gpu")
+class VisionArchsAsAString(unittest.TestCase):
+ """#3810: the packaging recipes write BUILD.json `vision_archs` as ONE string ("75;86;89;120"); 0.2.3 iterated it character by
+ character and setup stopped with int(';') on the released Windows CUDA package. A string and a list mean the same."""
+ RELEASED = {"version": "0.2.3", "archs": [75, 86, 89, 120], "ptx": True, "cuda": "13.0", "vision": "gpu-cuda13",
+ "vision_archs": "75;86;89;120", "portable": True}
+
+ def test_the_released_package_manifest_is_read(self):
+ for arch in (75, 86, 89, 120):
+ self.assertEqual(quiet(setup.prebuilt_vision, self.RELEASED, {"arch": arch}, "gpu")[0], "gpu")
+
+ def test_a_card_the_string_does_not_list_gets_the_cpu_reader(self):
+ meta = {**self.RELEASED, "vision_archs": "86;89;120", "ptx": False}
+ got, out = quiet(setup.prebuilt_vision, meta, {"arch": 75}, "gpu")
+ self.assertEqual(got, "cpu")
+ self.assertIn("runs on the CPU instead", out)
+ self.assertEqual(quiet(setup.prebuilt_vision, {**meta, "ptx": True}, {"arch": 120}, "gpu")[0], "gpu")
+
+ def test_a_string_and_a_list_agree(self):
+ for value in ("75;86;89;120", [75, 86, 89, 120], ["sm_75", "86-real", "89", "compute_120"], "75,86 89;120"):
+ self.assertEqual(setup.arch_numbers(value), [75, 86, 89, 120], value)
+ self.assertEqual(setup.arch_numbers(None), [])
+ self.assertEqual(setup.arch_numbers(""), [])
+ self.assertEqual(setup.arch_names("gfx1100;gfx1201"), ["gfx1100", "gfx1201"])
+ self.assertEqual(setup.arch_names(["gfx1100"]), ["gfx1100"])
+ self.assertEqual(setup.arch_names(None), [])
+
+ def test_the_other_readers_of_the_manifest_take_a_string_too(self):
+ import tempfile
+ from pathlib import Path
+ from unittest import mock
+ with tempfile.TemporaryDirectory() as d:
+ (Path(d) / "BUILD.json").write_text('{"archs": "60;61;70", "ptx": true}', encoding="utf-8")
+ with mock.patch.object(setup, "engine_dir", return_value=Path(d)):
+ self.assertEqual(setup.engine_archs(), ([60, 61, 70], True))
+ (Path(d) / "BUILD.json").write_text('{"backend": "hip", "archs": ["gfx1100"], "vision_archs": "gfx1100;gfx1201"}', encoding="utf-8")
+ self.assertEqual(setup.reader_archs(Path(d)), ["gfx1100", "gfx1201"])
+ (Path(d) / "BUILD.json").write_text('{"backend": "hip", "archs": ["gfx1100", "gfx1201"]}', encoding="utf-8")
+ self.assertEqual(setup.reader_archs(Path(d)), ["gfx1100", "gfx1201"])
+ self.assertEqual(setup.reader_archs(Path(__file__).parent / "no-such-folder"), [])
+
+
class GgufDirShards(unittest.TestCase):
"""#305: --gguf-dir reads the shard count from the -of-N part of the name instead of assuming two."""