Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

setup.py on a DGX Spark: unified-memory GB10, its engine from the signed feed, the measured run-config settings

Niko1221552c56f7a7ae
diff --git a/setup.py b/setup.py
index 73471f68f..6d0a2d563 100644
--- a/setup.py
+++ b/setup.py
@@ -1348,7 +1348,10 @@ def cpu_info():
txt = open("/proc/cpuinfo").read()
mflags = re.search(r"^flags\s*:\s*(.*)$", txt, re.M)
if mflags is None:
- # aarch64 lists Features, not the x86 flags line. The normal engine is the native one.
+ # aarch64 lists Features, not the x86 flags line, and no "model name": lscpu names the cores (a DGX Spark:
+ # Cortex-X925 + Cortex-A725). No AVX of any kind; the arm64 engine needs none.
+ cores = re.findall(r"^\s*Model name:\s*(.+)$", out(["lscpu"]), re.M)
+ name = " + ".join(dict.fromkeys(c.strip() for c in cores)) or (platform.processor() or "arm64 CPU")
return name, False, False
flags = set(mflags.group(1).split())
avx2 = "avx2" in flags
@@ -1386,6 +1389,12 @@ def is_arm64() -> bool:
return platform.machine().lower() in ("aarch64", "arm64")
+def is_gb10(g) -> bool:
+ """A DGX Spark's GPU: the GB10 (compute capability 12.1), on arm64 Linux, with the CPU's memory as its own. The part the Spark settings
+ below were measured on (docs/DGX_SPARK_LINUX.md); any other unified-memory NVIDIA GPU gets the ordinary settings."""
+ return isinstance(g, dict) and bool(g.get("uma")) and str(g.get("arch")) == "121" and is_arm64() and sys.platform.startswith("linux")
+
+
def cpu_floor(avx2: bool) -> str:
"""The experimental older-CPU build this PC needs (#394 #595 #623): "" with AVX2 (the normal engine), "avx" (Sandy /
Ivy Bridge, AMD Bulldozer), "none" (SSE4.2 + POPCNT: Nehalem, Westmere), or "unsupported". STRATA_ISA_FLOOR=avx|
@@ -1484,15 +1493,13 @@ def gpus():
except ValueError:
if not is_arm64():
raise # as before on x86: a card whose memory nvidia-smi does not report is not listed
- # GB10 unified memory: memory.total is [N/A]. The pool is the machine RAM.
- mem_gb = 0.0
- try:
- for info in open("/proc/meminfo"):
- if info.startswith("MemTotal:"):
- mem_gb = int(info.split()[1]) / 1024.0 / 1024.0
- break
- except OSError:
- pass
+ # a DGX Spark (GB10): one memory for the CPU and the GPU, and nvidia-smi gives memory.total as [N/A]. Counted as
+ # Strix Halo's is (amd_apply_uma): the GPU may use all of the RAM but the OS's share, and none of it is extra room
+ # beside the RAM (dedicated_gb 0), so the low-RAM rules do not count it twice
+ shared = max(0.0, ram_gb() - UMA_OS_LEFT_GB)
+ found.append({"index": int(idx), "name": name, "vram_gb": shared, "arch": cc.replace(".", ""),
+ "driver": drv, "uma": True, "dedicated_gb": 0.0, "shared_gb": shared})
+ continue
found.append({"index": int(idx), "name": name, "vram_gb": mem_gb, "arch": cc.replace(".", ""),
"driver": drv})
except ValueError:
@@ -5257,6 +5264,13 @@ def get_prebuilt(url_base, gpu, vision, updating=False, toolkit=13) -> Path | No
return eng
say(f" Updating the ready-made engine ({meta.get('version')} -> {'.'.join(map(str, MIN_ENGINE))} or newer) ...")
info.replace(eng / "BUILD.json.prev") # kept for the .previous copy
+ if is_arm64() and sys.platform.startswith("linux"):
+ # a DGX Spark: the release has no flat engine zip for it (and the CUDA 12 engine is x86). The signed feed has the linux-aarch64
+ # package when the release carries one; without it the caller compiles (spark/build_engine.sh does the same by hand)
+ got = engine_from_feed(eng, updating) if int(toolkit) != 12 else None
+ if got is None:
+ warn(f"no ready-made engine for {platform.machine()} in the update feed" + ("" if updating else ": compiling it here, once"))
+ return got
if not url_base:
return None
eng.mkdir(exist_ok=True)
@@ -5585,8 +5599,9 @@ def install_build_tools(gpu, yes):
fail("the CUDA Toolkit can be installed automatically on Ubuntu 22.04 / 24.04 and Debian 12 / 13 only",
"install it from https://developer.nvidia.com/cuda-downloads and run it again")
deb = Path("/tmp/cuda-keyring.deb")
- download(f"https://developer.download.nvidia.com/compute/cuda/repos/{repo}/x86_64/cuda-keyring_1.1-1_all.deb",
- deb, "CUDA repository key")
+ # NVIDIA's arm64 server repository is "sbsa" (a DGX Spark; DGX OS normally has the toolkit already)
+ download(f"https://developer.download.nvidia.com/compute/cuda/repos/{repo}/"
+ f"{'sbsa' if is_arm64() else 'x86_64'}/cuda-keyring_1.1-1_all.deb", deb, "CUDA repository key")
run(["sudo", "dpkg", "-i", str(deb)])
run(["sudo", "apt-get", "update"])
if repo.startswith("ubuntu"): # as before: 13.0, and a failed install stops setup here
@@ -8332,7 +8347,9 @@ def install_dsv41(a, family, gpu, hip, amd, cuda_tk, multi, port, ram, adopted,
"and point STRATA_DSV41_EXE at it (docs/DEEPSEEK41.md, \"Building the engine for your card\").")
fail("DeepSeek's engine is not available for this computer yet",
"Supported graphics cards: NVIDIA RTX 30, 40 and 50 series, the Radeon AI PRO R9700, and a Ryzen AI Max (Strix Halo) on "
- "Windows with the AMD DeepSeek engine package. " + dsv41_manual_steps(eng)
+ "Windows with the AMD DeepSeek engine package. "
+ + ("The DGX Spark (GB10) builds its own: spark/build_engine.sh puts strata-dsv41 beside the engine (docs/DGX_SPARK_LINUX.md). "
+ if is_arm64() else "") + dsv41_manual_steps(eng)
+ " Or update Strata later (a newer version may add your card), pick another model, or build the engine for your card yourself "
"(docs/DEEPSEEK41.md, \"Building the engine for your card\"). Nothing has been downloaded.")
ok(f"engine: {exe}")
@@ -9110,7 +9127,8 @@ def main() -> int:
gpu["archs"] = sorted({x["arch"] for x in chosen}) # the engine needs code for every one of them
if multi:
ok("GPUs: " + " + ".join(gpu_name(x) for x in chosen) + " together (the model's layers are split across them)")
- ok(f"GPU: {gpu['name']}, {gpu['vram_gb']:.1f} GB VRAM, compute capability {cc(gpu)}, driver {gpu['driver']}")
+ ok(f"GPU: {gpu['name']}, " + (f"unified memory ({gpu['vram_gb']:.0f} GB usable by the GPU)" if gpu.get("uma") else f"{gpu['vram_gb']:.1f} GB VRAM")
+ + f", compute capability {cc(gpu)}, driver {gpu['driver']}")
cuda_tk, why = cuda_choice(gpu["archs"], a.cuda, driver_major(gpu)) # one engine per model: its oldest card decides
if why:
(warn if cuda_tk == 13 or str(a.cuda) == "12" else ok)(f"CUDA {cuda_tk}: {why}")
@@ -9144,7 +9162,8 @@ def main() -> int:
if pf is not None and pf < 4:
warn(page_file_advice(pf, page_file_setting()))
ok(f"CPU: {cpu} ({'ARM64' if is_arm64() else 'AVX-512' if avx512 else 'AVX2' if avx2 else 'no AVX2'})")
- link = None if hip else pcie_link(int(gpu.get("index", 0)))
+ # (a unified-memory GPU - a DGX Spark's GB10 - is on the CPU's own link, not PCIe: nvidia-smi's "1.0 x1" there means nothing)
+ link = None if hip or gpu.get("uma") else pcie_link(int(gpu.get("index", 0)))
if link is not None:
line, problem = pcie_lines(link)
ok(line)
@@ -9154,11 +9173,11 @@ def main() -> int:
if floor == "unsupported":
fail("this CPU has neither AVX2 nor SSE4.2; Strata needs at least SSE4.2 (Intel Nehalem, 2008, or newer)")
if is_arm64():
- # DGX Spark / GB10 (#3622): no ready-made ARM64 engine; setup compiles it here. A warning, not a stop.
- warn("ARM64 CPU: setup compiles the engine on this PC (once). The CPU's share of the experts runs on ggml-cpu's "
- "NEON kernels. Tested: GGUF packs with i-quant experts (IQ3_S). The Q2_0 and Gyro packs need the x86 "
- "kernels and do not run on this CPU. This port is new and has not been measured against an x86 PC")
- a.build = True
+ # DGX Spark / GB10 (#3622): the engine is the linux-aarch64 package of the signed feed when the release has one, else compiled here
+ # (spark/build_engine.sh). A warning, not a stop.
+ warn("ARM64 CPU (a DGX Spark): the CPU's share of the experts runs on ggml-cpu's NEON kernels. Tested: Qwen3.8-Flash-Next IQ3_S and "
+ "the NVFP4 pack, and DeepSeek-V4.1 2.00 bpw, on one DGX Spark (GB10). The Q2_0 and Gyro packs need the x86 kernels and do not "
+ "run on this CPU")
elif not avx2:
# #394 #595 #623: the ready-made engine is AVX2; an older CPU gets one compiled here, whose CPU experts run on
# ggml-cpu's kernels for this CPU. Experimental: measured only on newer CPUs with the older path forced, and by
@@ -9752,6 +9771,26 @@ def main() -> int:
"experts.bin on the SSD as needed")
elif low_ram: # the experts from the pack's experts.bin: the ones the GPU does not hold copied into RAM, or mapped
args += ["--resident-experts" if resident else "--mmap-experts"]
+ elif is_gb10(gpu):
+ # a DGX Spark: the RAM copy of the experts and the GPU's cache are the same memory. Mapped, every expert fits in the cache (NVFP4
+ # on a 128 GB GB10: 24576 of 24576 slots, not 13702; chat decode 44.4 against 40.6 tok/s, 16K prompt 1471 against 1397)
+ args += ["--mmap-experts"]
+ ok("unified memory: the experts are read from the model files in place (--mmap-experts), so the GPU's cache holds them "
+ "instead of a second copy in RAM")
+ # the measured settings of this part (docs/DGX_SPARK_LINUX.md): the 32,768-token prompt chunk (auto's 8,192 measured 2-6% slower
+ # here); the draft layer as 4-bit copies (+3.5% decode, 3 paired rounds, answers identical: the drafter's proposals are verified
+ # by the target); one norm per hyper-connection stream in the draft layer (llama.cpp's graph: acceptance 0.60 -> 0.63, decode +4.3%
+ # [1.009, 1.078], 5 of 5 greedy answers identical)
+ if "--prefill" in args:
+ args[args.index("--prefill") + 1] = "auto:32768"
+ if "--mtp" in args:
+ args += ["--mtp-q4", "all", "--mtp-hnorm", "stream"]
+ ok("prompt chunk 32,768 (--prefill auto:32768), the draft layer as 4-bit copies (--mtp-q4 all) and one norm per "
+ "hyper-connection stream in it (--mtp-hnorm stream): measured faster on this part, the answers are unchanged")
+ # the language is the user's choice: the default subset keeps Chinese, Japanese and Korean drafts, the English one measured +3.1%
+ # decode on English prompts here (16 samples a cell, 90% interval [1.018, 1.049])
+ say(" Tip: for English or code only, --draft-vocab en measured +3.1% decode on this part (the default keeps Chinese, Japanese "
+ "and Korean drafts).")
disk = None if is_wsl() else rotational_disk(ple) # #605 (WSL's virtual disk says rotational)
if disk:
tensor = None if ready else next((t for t in GGUFFile(ple).tensors if t.name == "per_layer_token_embd.weight"), None)
diff --git a/tests/terminal/test_setup_arm64.py b/tests/terminal/test_setup_arm64.py
index 7f51056a5..f34f68702 100644
--- a/tests/terminal/test_setup_arm64.py
+++ b/tests/terminal/test_setup_arm64.py
@@ -25,14 +25,33 @@ class UnifiedMemoryCard(unittest.TestCase):
with mock.patch.object(setup, "out", return_value=GB10), mock.patch.object(setup, "is_arm64", return_value=False):
self.assertEqual(setup.gpus(), [])
- def test_gb10_takes_the_pool_size_from_the_machine_ram(self):
+ def test_gb10_is_a_unified_memory_gpu_with_the_ram_less_the_os_share(self):
with mock.patch.object(setup, "out", return_value=GB10), mock.patch.object(setup, "is_arm64", return_value=True), \
- mock.patch("builtins.open", mock.mock_open(read_data=MEMINFO)):
+ mock.patch.object(setup, "ram_gb", return_value=130000000 / 1024.0 / 1024.0):
found = setup.gpus()
self.assertEqual(len(found), 1)
self.assertEqual(found[0]["name"], "NVIDIA GB10")
self.assertEqual(found[0]["arch"], "121")
- self.assertAlmostEqual(found[0]["vram_gb"], 130000000 / 1024.0 / 1024.0, places=3)
+ usable = 130000000 / 1024.0 / 1024.0 - setup.UMA_OS_LEFT_GB
+ self.assertAlmostEqual(found[0]["vram_gb"], usable, places=3)
+ self.assertTrue(found[0]["uma"])
+ self.assertEqual(found[0]["dedicated_gb"], 0.0) # none of it is room beside the RAM: the low-RAM rules must not count it twice
+ self.assertAlmostEqual(found[0]["shared_gb"], usable, places=3)
+ self.assertEqual(setup.low_ram_vram(found[0]), 0.0)
+
+ def test_the_spark_settings_are_for_the_gb10_only(self):
+ gb10 = {"uma": True, "arch": "121", "name": "NVIDIA GB10"}
+ with mock.patch.object(setup, "is_arm64", return_value=True), mock.patch.object(setup.sys, "platform", "linux"):
+ self.assertTrue(setup.is_gb10(gb10))
+ self.assertFalse(setup.is_gb10({**gb10, "arch": "110"})) # another unified-memory NVIDIA part: not measured
+ self.assertFalse(setup.is_gb10({"uma": False, "arch": "121"}))
+ self.assertFalse(setup.is_gb10({"arch": "121"}))
+ with mock.patch.object(setup, "is_arm64", return_value=False):
+ self.assertFalse(setup.is_gb10(gb10))
+ self.assertFalse(setup.is_gb10(None))
+
+ def test_a_normal_x86_card_is_never_a_gb10(self):
+ self.assertFalse(setup.is_gb10({"arch": "121", "vram_gb": 32.0, "name": "a discrete card"}))
def test_a_normal_card_is_unchanged(self):
with mock.patch.object(setup, "out", return_value="0, NVIDIA GeForce RTX 5070, 12227, 12.0, 580.1\n"):
@@ -47,6 +66,14 @@ class ArmCpu(unittest.TestCase):
self.assertFalse(avx2)
self.assertFalse(avx512)
+ def test_an_arm_cpu_is_named_from_lscpu(self):
+ lscpu = "Architecture: aarch64\nModel name: Cortex-X925\nModel name: Cortex-A725\nModel name: Cortex-X925\n"
+ with mock.patch.object(setup, "WIN", False), mock.patch("builtins.open", mock.mock_open(read_data=ARM_CPUINFO)), \
+ mock.patch.object(setup, "out", return_value=lscpu):
+ name, avx2, avx512 = setup.cpu_info()
+ self.assertEqual(name, "Cortex-X925 + Cortex-A725")
+ self.assertEqual((avx2, avx512), (False, False))
+
def test_an_x86_cpuinfo_is_read_as_before(self):
text = "flags\t\t: fpu avx avx2 avx512f avx512bw avx512vl avx512_vnni avx512vbmi\nmodel name\t: Test CPU\n"
with mock.patch.object(setup, "WIN", False), mock.patch("builtins.open", mock.mock_open(read_data=text)):
diff --git a/tools/test_setup_pins.py b/tools/test_setup_pins.py
index f0be55ba0..7315bc29b 100644
--- a/tools/test_setup_pins.py
+++ b/tools/test_setup_pins.py
@@ -215,6 +215,9 @@ class HuggingFacePins(unittest.TestCase):
self.assertIn("Downloading from https://hf-mirror.com (HF_ENDPOINT)", text)
[email protected](setup.is_arm64() and sys.platform.startswith("linux"),
+ "arm64 Linux takes its ready-made engine from the signed feed, not from the flat x86 zips these tests build "
+ "(tests/terminal/test_linux_aarch64.py covers that path)")
class Engine(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()