Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
plan: no GPU below an engine's CUDA floor in the placement (#1906)
JustVugg13d0dcd81548
diff --git a/c/resource_plan.py b/c/resource_plan.pyindex 1f95ed60..71119e5f 100644--- a/c/resource_plan.py+++ b/c/resource_plan.py@@ -2404,18 +2404,32 @@ def _discover_metal_gpus():def _discover_nvidia_gpus():- command = ["nvidia-smi", "--query-gpu=index,name,memory.total,memory.free",- "--format=csv,noheader,nounits"]- try:- result = subprocess.run(command, text=True, capture_output=True, check=True, timeout=5)- except (OSError, subprocess.SubprocessError):+ # compute_cap last: a driver that predates the field refuses the whole query,+ # and then the same without it (setup_hw.py does the same)+ result = None+ for fields_asked in ("index,name,memory.total,memory.free,compute_cap",+ "index,name,memory.total,memory.free"):+ command = ["nvidia-smi", f"--query-gpu={fields_asked}", "--format=csv,noheader,nounits"]+ try:+ result = subprocess.run(command, text=True, capture_output=True, check=True, timeout=5)+ break+ except subprocess.CalledProcessError:+ continue+ except (OSError, subprocess.SubprocessError):+ return []+ if result is None:return []devices = []import csvfor fields in csv.reader(result.stdout.splitlines()):fields = [f.strip() for f in fields]- if len(fields) != 4:+ if len(fields) not in (4, 5):continue+ cap = None+ if len(fields) == 5:+ m = re.fullmatch(r"(\d+)\.(\d+)", fields[4])+ cap = (int(m.group(1)), int(m.group(2))) if m else None+ fields = fields[:4]try:index = int(fields[0])except ValueError:@@ -2435,10 +2449,13 @@ def _discover_nvidia_gpus():total = free = 0name = fields[1]unified = any(token in name.lower() for token in ("gb10", "jetson", "grace blackwell"))- devices.append({"index": index, "name": name,- "total_bytes": total * 1024 * 1024,- "free_bytes": free * 1024 * 1024,- "unified_memory": unified})+ device = {"index": index, "name": name,+ "total_bytes": total * 1024 * 1024,+ "free_bytes": free * 1024 * 1024,+ "unified_memory": unified}+ if cap:+ device["compute_cap"] = cap+ devices.append(device)return devices@@ -2611,6 +2628,13 @@ def _discover_amd_gpus():return devices+#: The oldest compute capability an engine's CUDA tier is built for by default, and+#: what reaches older cards (docs/deepseek-v4.md).+CUDA_TIER_FLOOR = {+ "deepseek_v4": ((8, 0), "a build with CUDA_ARCH=portable-pre-ampere NO_TC=1 runs on Pascal and Turing"),+}++def plans_placement(gpu):"""Whether a discovered device's memory is qualified to drive placement.@@ -3020,6 +3044,12 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,# stops "a GPU exists" from being read as "a GPU should be used" -- see# plans_placement().planning_gpus = [gpu for gpu in gpus if plans_placement(gpu)]+ # A CUDA tier built for newer cards than the one found: that card drives no+ # placement (#1906: a Quadro P2000, sm_61, planned as "GPU compute" for V4)+ floor = CUDA_TIER_FLOOR.get(resolved.descriptor.id)+ below = [gpu for gpu in planning_gpus+ if floor and gpu.get("compute_cap") and tuple(gpu["compute_cap"]) < floor[0]]+ planning_gpus = [gpu for gpu in planning_gpus if gpu not in below]placement_unified = any(gpu.get("unified_memory", False)for gpu in planning_gpus)@@ -3041,7 +3071,7 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,gpu_plan = []safe_vram = 0for gpu in gpus:- usable = max(0, gpu["free_bytes"] - reserve) if plans_placement(gpu) else 0+ usable = max(0, gpu["free_bytes"] - reserve) if plans_placement(gpu) and gpu not in below else 0safe_vram += usablegpu_plan.append(dict(gpu, reserve_bytes=reserve, usable_bytes=usable))requested_vram = int(vram_gb * GB) if vram_gb > 0 else safe_vram@@ -3067,6 +3097,12 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,vram_experts = int(vram_budget // typical) if typical else 0hot_bytes = min(info["expert_bytes"], vram_experts * typical)warnings = []+ for gpu in below:+ cc = gpu["compute_cap"]+ warnings.append(+ f"GPU {gpu['index']} ({gpu['name']}, sm_{cc[0]}{cc[1]}) is below the "+ f"sm_{floor[0][0]}{floor[0][1]} {resolved.descriptor.id}'s CUDA tier is built for by "+ f"default; planned without it ({floor[1]})")if placement_unified:requested_ram = int(ram_gb * GB) if ram_gb > 0 else int(available_memory * 0.88)requested_ram_experts = max(0, requested_ram - info["dense_bytes"] - runtime_bytes)diff --git a/c/tests/test_resource_plan.py b/c/tests/test_resource_plan.pyindex 0dafdc50..49240490 100644--- a/c/tests/test_resource_plan.py+++ b/c/tests/test_resource_plan.py@@ -672,6 +672,22 @@ class ResourcePlanTest(unittest.TestCase):self.assertTrue(any("share one physical memory" in warningfor warning in plan["warnings"]))+ def test_nvidia_compute_capability_is_read_when_the_driver_has_it(self):+ from resource_plan import _discover_nvidia_gpus+ ok = subprocess.CompletedProcess(args=[], returncode=0, stdout="0, Quadro P2000, 5120, 5000, 6.1\n", stderr="")+ with mock.patch("resource_plan.subprocess.run", return_value=ok) as run:+ devices = _discover_nvidia_gpus()+ self.assertEqual(devices[0]["compute_cap"], (6, 1))+ self.assertIn("compute_cap", run.call_args_list[0][0][0][1])+ # a driver that predates the field refuses the query: asked again without it+ old = subprocess.CompletedProcess(args=[], returncode=0, stdout="0, Quadro P2000, 5120, 5000\n", stderr="")+ with mock.patch("resource_plan.subprocess.run",+ side_effect=[subprocess.CalledProcessError(2, "nvidia-smi"), old]) as run:+ devices = _discover_nvidia_gpus()+ self.assertEqual(run.call_count, 2)+ self.assertEqual(devices[0]["free_bytes"], 5000 * 1024 * 1024)+ self.assertNotIn("compute_cap", devices[0])+def test_nvidia_unified_device_is_marked_from_name(self):output = "0, NVIDIA GB10, 130000, 120000\n"with mock.patch("resource_plan.subprocess.run",@@ -1703,6 +1719,24 @@ class VulkanPartialChainTest(unittest.TestCase):write_shard(self.model / "model.safetensors", tensors)return analyze_model(self.model)+ def test_deepseek_v4_plans_without_a_gpu_below_its_cuda_floor(self):+ """#1906: a Quadro P2000 (sm_61) was planned as GPU compute for V4, whose CUDA+ tier is built for sm_80 and newer by default."""+ self.write_v4([0, 4])++ def plan_with(cap):+ gpu = {"index": 0, "name": "Quadro P2000", "total_bytes": 5 * GB, "free_bytes": 5 * GB,+ "unified_memory": False, "compute_cap": cap}+ return build_plan(self.model, ram_gb=16, available_memory=64 * GB, available_disk=1,+ gpus=[gpu], physical_cpus=32, cpu_sockets=1)+ old = plan_with((6, 1))+ self.assertEqual(old["tiers"]["vram"]["budget_bytes"], 0)+ self.assertNotIn("GPU", old["expected_bottleneck"])+ self.assertTrue(any("sm_61" in w and "portable-pre-ampere" in w for w in old["warnings"]))+ new = plan_with((8, 6))+ self.assertGreater(new["tiers"]["vram"]["budget_bytes"], 0)+ self.assertFalse(any("sm_86" in w for w in new["warnings"]))+def test_deepseek_v4_layout_is_the_engines(self):# The numbers the engine printed for tools/make_deepseek_v4_tiny.py's fixture on# Lavapipe ("[VK] deepseek_v4 chain fit: ... the engine's 1891840 B ..., layers