Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
setup.py on a DGX Spark: unified-memory GB10, its engine from the signed feed, the measured run-config settings
Niko1221552c56f7a7ae
diff --git a/setup.py b/setup.pyindex 73471f68f..6d0a2d563 100644--- a/setup.py+++ b/setup.py@@ -1348,7 +1348,10 @@ def cpu_info():txt = open("/proc/cpuinfo").read()mflags = re.search(r"^flags\s*:\s*(.*)$", txt, re.M)if mflags is None:- # aarch64 lists Features, not the x86 flags line. The normal engine is the native one.+ # aarch64 lists Features, not the x86 flags line, and no "model name": lscpu names the cores (a DGX Spark:+ # Cortex-X925 + Cortex-A725). No AVX of any kind; the arm64 engine needs none.+ cores = re.findall(r"^\s*Model name:\s*(.+)$", out(["lscpu"]), re.M)+ name = " + ".join(dict.fromkeys(c.strip() for c in cores)) or (platform.processor() or "arm64 CPU")return name, False, Falseflags = set(mflags.group(1).split())avx2 = "avx2" in flags@@ -1386,6 +1389,12 @@ def is_arm64() -> bool:return platform.machine().lower() in ("aarch64", "arm64")+def is_gb10(g) -> bool:+ """A DGX Spark's GPU: the GB10 (compute capability 12.1), on arm64 Linux, with the CPU's memory as its own. The part the Spark settings+ below were measured on (docs/DGX_SPARK_LINUX.md); any other unified-memory NVIDIA GPU gets the ordinary settings."""+ return isinstance(g, dict) and bool(g.get("uma")) and str(g.get("arch")) == "121" and is_arm64() and sys.platform.startswith("linux")++def cpu_floor(avx2: bool) -> str:"""The experimental older-CPU build this PC needs (#394 #595 #623): "" with AVX2 (the normal engine), "avx" (Sandy /Ivy Bridge, AMD Bulldozer), "none" (SSE4.2 + POPCNT: Nehalem, Westmere), or "unsupported". STRATA_ISA_FLOOR=avx|@@ -1484,15 +1493,13 @@ def gpus():except ValueError:if not is_arm64():raise # as before on x86: a card whose memory nvidia-smi does not report is not listed- # GB10 unified memory: memory.total is [N/A]. The pool is the machine RAM.- mem_gb = 0.0- try:- for info in open("/proc/meminfo"):- if info.startswith("MemTotal:"):- mem_gb = int(info.split()[1]) / 1024.0 / 1024.0- break- except OSError:- pass+ # a DGX Spark (GB10): one memory for the CPU and the GPU, and nvidia-smi gives memory.total as [N/A]. Counted as+ # Strix Halo's is (amd_apply_uma): the GPU may use all of the RAM but the OS's share, and none of it is extra room+ # beside the RAM (dedicated_gb 0), so the low-RAM rules do not count it twice+ shared = max(0.0, ram_gb() - UMA_OS_LEFT_GB)+ found.append({"index": int(idx), "name": name, "vram_gb": shared, "arch": cc.replace(".", ""),+ "driver": drv, "uma": True, "dedicated_gb": 0.0, "shared_gb": shared})+ continuefound.append({"index": int(idx), "name": name, "vram_gb": mem_gb, "arch": cc.replace(".", ""),"driver": drv})except ValueError:@@ -5257,6 +5264,13 @@ def get_prebuilt(url_base, gpu, vision, updating=False, toolkit=13) -> Path | Noreturn engsay(f" Updating the ready-made engine ({meta.get('version')} -> {'.'.join(map(str, MIN_ENGINE))} or newer) ...")info.replace(eng / "BUILD.json.prev") # kept for the .previous copy+ if is_arm64() and sys.platform.startswith("linux"):+ # a DGX Spark: the release has no flat engine zip for it (and the CUDA 12 engine is x86). The signed feed has the linux-aarch64+ # package when the release carries one; without it the caller compiles (spark/build_engine.sh does the same by hand)+ got = engine_from_feed(eng, updating) if int(toolkit) != 12 else None+ if got is None:+ warn(f"no ready-made engine for {platform.machine()} in the update feed" + ("" if updating else ": compiling it here, once"))+ return gotif not url_base:return Noneeng.mkdir(exist_ok=True)@@ -5585,8 +5599,9 @@ def install_build_tools(gpu, yes):fail("the CUDA Toolkit can be installed automatically on Ubuntu 22.04 / 24.04 and Debian 12 / 13 only","install it from https://developer.nvidia.com/cuda-downloads and run it again")deb = Path("/tmp/cuda-keyring.deb")- download(f"https://developer.download.nvidia.com/compute/cuda/repos/{repo}/x86_64/cuda-keyring_1.1-1_all.deb",- deb, "CUDA repository key")+ # NVIDIA's arm64 server repository is "sbsa" (a DGX Spark; DGX OS normally has the toolkit already)+ download(f"https://developer.download.nvidia.com/compute/cuda/repos/{repo}/"+ f"{'sbsa' if is_arm64() else 'x86_64'}/cuda-keyring_1.1-1_all.deb", deb, "CUDA repository key")run(["sudo", "dpkg", "-i", str(deb)])run(["sudo", "apt-get", "update"])if repo.startswith("ubuntu"): # as before: 13.0, and a failed install stops setup here@@ -8332,7 +8347,9 @@ def install_dsv41(a, family, gpu, hip, amd, cuda_tk, multi, port, ram, adopted,"and point STRATA_DSV41_EXE at it (docs/DEEPSEEK41.md, \"Building the engine for your card\").")fail("DeepSeek's engine is not available for this computer yet","Supported graphics cards: NVIDIA RTX 30, 40 and 50 series, the Radeon AI PRO R9700, and a Ryzen AI Max (Strix Halo) on "- "Windows with the AMD DeepSeek engine package. " + dsv41_manual_steps(eng)+ "Windows with the AMD DeepSeek engine package. "+ + ("The DGX Spark (GB10) builds its own: spark/build_engine.sh puts strata-dsv41 beside the engine (docs/DGX_SPARK_LINUX.md). "+ if is_arm64() else "") + dsv41_manual_steps(eng)+ " Or update Strata later (a newer version may add your card), pick another model, or build the engine for your card yourself ""(docs/DEEPSEEK41.md, \"Building the engine for your card\"). Nothing has been downloaded.")ok(f"engine: {exe}")@@ -9110,7 +9127,8 @@ def main() -> int:gpu["archs"] = sorted({x["arch"] for x in chosen}) # the engine needs code for every one of themif multi:ok("GPUs: " + " + ".join(gpu_name(x) for x in chosen) + " together (the model's layers are split across them)")- ok(f"GPU: {gpu['name']}, {gpu['vram_gb']:.1f} GB VRAM, compute capability {cc(gpu)}, driver {gpu['driver']}")+ ok(f"GPU: {gpu['name']}, " + (f"unified memory ({gpu['vram_gb']:.0f} GB usable by the GPU)" if gpu.get("uma") else f"{gpu['vram_gb']:.1f} GB VRAM")+ + f", compute capability {cc(gpu)}, driver {gpu['driver']}")cuda_tk, why = cuda_choice(gpu["archs"], a.cuda, driver_major(gpu)) # one engine per model: its oldest card decidesif why:(warn if cuda_tk == 13 or str(a.cuda) == "12" else ok)(f"CUDA {cuda_tk}: {why}")@@ -9144,7 +9162,8 @@ def main() -> int:if pf is not None and pf < 4:warn(page_file_advice(pf, page_file_setting()))ok(f"CPU: {cpu} ({'ARM64' if is_arm64() else 'AVX-512' if avx512 else 'AVX2' if avx2 else 'no AVX2'})")- link = None if hip else pcie_link(int(gpu.get("index", 0)))+ # (a unified-memory GPU - a DGX Spark's GB10 - is on the CPU's own link, not PCIe: nvidia-smi's "1.0 x1" there means nothing)+ link = None if hip or gpu.get("uma") else pcie_link(int(gpu.get("index", 0)))if link is not None:line, problem = pcie_lines(link)ok(line)@@ -9154,11 +9173,11 @@ def main() -> int:if floor == "unsupported":fail("this CPU has neither AVX2 nor SSE4.2; Strata needs at least SSE4.2 (Intel Nehalem, 2008, or newer)")if is_arm64():- # DGX Spark / GB10 (#3622): no ready-made ARM64 engine; setup compiles it here. A warning, not a stop.- warn("ARM64 CPU: setup compiles the engine on this PC (once). The CPU's share of the experts runs on ggml-cpu's "- "NEON kernels. Tested: GGUF packs with i-quant experts (IQ3_S). The Q2_0 and Gyro packs need the x86 "- "kernels and do not run on this CPU. This port is new and has not been measured against an x86 PC")- a.build = True+ # DGX Spark / GB10 (#3622): the engine is the linux-aarch64 package of the signed feed when the release has one, else compiled here+ # (spark/build_engine.sh). A warning, not a stop.+ warn("ARM64 CPU (a DGX Spark): the CPU's share of the experts runs on ggml-cpu's NEON kernels. Tested: Qwen3.8-Flash-Next IQ3_S and "+ "the NVFP4 pack, and DeepSeek-V4.1 2.00 bpw, on one DGX Spark (GB10). The Q2_0 and Gyro packs need the x86 kernels and do not "+ "run on this CPU")elif not avx2:# #394 #595 #623: the ready-made engine is AVX2; an older CPU gets one compiled here, whose CPU experts run on# ggml-cpu's kernels for this CPU. Experimental: measured only on newer CPUs with the older path forced, and by@@ -9752,6 +9771,26 @@ def main() -> int:"experts.bin on the SSD as needed")elif low_ram: # the experts from the pack's experts.bin: the ones the GPU does not hold copied into RAM, or mappedargs += ["--resident-experts" if resident else "--mmap-experts"]+ elif is_gb10(gpu):+ # a DGX Spark: the RAM copy of the experts and the GPU's cache are the same memory. Mapped, every expert fits in the cache (NVFP4+ # on a 128 GB GB10: 24576 of 24576 slots, not 13702; chat decode 44.4 against 40.6 tok/s, 16K prompt 1471 against 1397)+ args += ["--mmap-experts"]+ ok("unified memory: the experts are read from the model files in place (--mmap-experts), so the GPU's cache holds them "+ "instead of a second copy in RAM")+ # the measured settings of this part (docs/DGX_SPARK_LINUX.md): the 32,768-token prompt chunk (auto's 8,192 measured 2-6% slower+ # here); the draft layer as 4-bit copies (+3.5% decode, 3 paired rounds, answers identical: the drafter's proposals are verified+ # by the target); one norm per hyper-connection stream in the draft layer (llama.cpp's graph: acceptance 0.60 -> 0.63, decode +4.3%+ # [1.009, 1.078], 5 of 5 greedy answers identical)+ if "--prefill" in args:+ args[args.index("--prefill") + 1] = "auto:32768"+ if "--mtp" in args:+ args += ["--mtp-q4", "all", "--mtp-hnorm", "stream"]+ ok("prompt chunk 32,768 (--prefill auto:32768), the draft layer as 4-bit copies (--mtp-q4 all) and one norm per "+ "hyper-connection stream in it (--mtp-hnorm stream): measured faster on this part, the answers are unchanged")+ # the language is the user's choice: the default subset keeps Chinese, Japanese and Korean drafts, the English one measured +3.1%+ # decode on English prompts here (16 samples a cell, 90% interval [1.018, 1.049])+ say(" Tip: for English or code only, --draft-vocab en measured +3.1% decode on this part (the default keeps Chinese, Japanese "+ "and Korean drafts).")disk = None if is_wsl() else rotational_disk(ple) # #605 (WSL's virtual disk says rotational)if disk:tensor = None if ready else next((t for t in GGUFFile(ple).tensors if t.name == "per_layer_token_embd.weight"), None)diff --git a/tests/terminal/test_setup_arm64.py b/tests/terminal/test_setup_arm64.pyindex 7f51056a5..f34f68702 100644--- a/tests/terminal/test_setup_arm64.py+++ b/tests/terminal/test_setup_arm64.py@@ -25,14 +25,33 @@ class UnifiedMemoryCard(unittest.TestCase):with mock.patch.object(setup, "out", return_value=GB10), mock.patch.object(setup, "is_arm64", return_value=False):self.assertEqual(setup.gpus(), [])- def test_gb10_takes_the_pool_size_from_the_machine_ram(self):+ def test_gb10_is_a_unified_memory_gpu_with_the_ram_less_the_os_share(self):with mock.patch.object(setup, "out", return_value=GB10), mock.patch.object(setup, "is_arm64", return_value=True), \- mock.patch("builtins.open", mock.mock_open(read_data=MEMINFO)):+ mock.patch.object(setup, "ram_gb", return_value=130000000 / 1024.0 / 1024.0):found = setup.gpus()self.assertEqual(len(found), 1)self.assertEqual(found[0]["name"], "NVIDIA GB10")self.assertEqual(found[0]["arch"], "121")- self.assertAlmostEqual(found[0]["vram_gb"], 130000000 / 1024.0 / 1024.0, places=3)+ usable = 130000000 / 1024.0 / 1024.0 - setup.UMA_OS_LEFT_GB+ self.assertAlmostEqual(found[0]["vram_gb"], usable, places=3)+ self.assertTrue(found[0]["uma"])+ self.assertEqual(found[0]["dedicated_gb"], 0.0) # none of it is room beside the RAM: the low-RAM rules must not count it twice+ self.assertAlmostEqual(found[0]["shared_gb"], usable, places=3)+ self.assertEqual(setup.low_ram_vram(found[0]), 0.0)++ def test_the_spark_settings_are_for_the_gb10_only(self):+ gb10 = {"uma": True, "arch": "121", "name": "NVIDIA GB10"}+ with mock.patch.object(setup, "is_arm64", return_value=True), mock.patch.object(setup.sys, "platform", "linux"):+ self.assertTrue(setup.is_gb10(gb10))+ self.assertFalse(setup.is_gb10({**gb10, "arch": "110"})) # another unified-memory NVIDIA part: not measured+ self.assertFalse(setup.is_gb10({"uma": False, "arch": "121"}))+ self.assertFalse(setup.is_gb10({"arch": "121"}))+ with mock.patch.object(setup, "is_arm64", return_value=False):+ self.assertFalse(setup.is_gb10(gb10))+ self.assertFalse(setup.is_gb10(None))++ def test_a_normal_x86_card_is_never_a_gb10(self):+ self.assertFalse(setup.is_gb10({"arch": "121", "vram_gb": 32.0, "name": "a discrete card"}))def test_a_normal_card_is_unchanged(self):with mock.patch.object(setup, "out", return_value="0, NVIDIA GeForce RTX 5070, 12227, 12.0, 580.1\n"):@@ -47,6 +66,14 @@ class ArmCpu(unittest.TestCase):self.assertFalse(avx2)self.assertFalse(avx512)+ def test_an_arm_cpu_is_named_from_lscpu(self):+ lscpu = "Architecture: aarch64\nModel name: Cortex-X925\nModel name: Cortex-A725\nModel name: Cortex-X925\n"+ with mock.patch.object(setup, "WIN", False), mock.patch("builtins.open", mock.mock_open(read_data=ARM_CPUINFO)), \+ mock.patch.object(setup, "out", return_value=lscpu):+ name, avx2, avx512 = setup.cpu_info()+ self.assertEqual(name, "Cortex-X925 + Cortex-A725")+ self.assertEqual((avx2, avx512), (False, False))+def test_an_x86_cpuinfo_is_read_as_before(self):text = "flags\t\t: fpu avx avx2 avx512f avx512bw avx512vl avx512_vnni avx512vbmi\nmodel name\t: Test CPU\n"with mock.patch.object(setup, "WIN", False), mock.patch("builtins.open", mock.mock_open(read_data=text)):diff --git a/tools/test_setup_pins.py b/tools/test_setup_pins.pyindex f0be55ba0..7315bc29b 100644--- a/tools/test_setup_pins.py+++ b/tools/test_setup_pins.py@@ -215,6 +215,9 @@ class HuggingFacePins(unittest.TestCase):self.assertIn("Downloading from https://hf-mirror.com (HF_ENDPOINT)", text)[email protected](setup.is_arm64() and sys.platform.startswith("linux"),+ "arm64 Linux takes its ready-made engine from the signed feed, not from the flat x86 zips these tests build "+ "(tests/terminal/test_linux_aarch64.py covers that path)")class Engine(unittest.TestCase):def setUp(self):self.tmp = tempfile.TemporaryDirectory()