Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

plan: no GPU below an engine's CUDA floor in the placement (#1906)

JustVugg13d0dcd81548
diff --git a/c/resource_plan.py b/c/resource_plan.py
index 1f95ed60..71119e5f 100644
--- a/c/resource_plan.py
+++ b/c/resource_plan.py
@@ -2404,18 +2404,32 @@ def _discover_metal_gpus():
def _discover_nvidia_gpus():
- command = ["nvidia-smi", "--query-gpu=index,name,memory.total,memory.free",
- "--format=csv,noheader,nounits"]
- try:
- result = subprocess.run(command, text=True, capture_output=True, check=True, timeout=5)
- except (OSError, subprocess.SubprocessError):
+ # compute_cap last: a driver that predates the field refuses the whole query,
+ # and then the same without it (setup_hw.py does the same)
+ result = None
+ for fields_asked in ("index,name,memory.total,memory.free,compute_cap",
+ "index,name,memory.total,memory.free"):
+ command = ["nvidia-smi", f"--query-gpu={fields_asked}", "--format=csv,noheader,nounits"]
+ try:
+ result = subprocess.run(command, text=True, capture_output=True, check=True, timeout=5)
+ break
+ except subprocess.CalledProcessError:
+ continue
+ except (OSError, subprocess.SubprocessError):
+ return []
+ if result is None:
return []
devices = []
import csv
for fields in csv.reader(result.stdout.splitlines()):
fields = [f.strip() for f in fields]
- if len(fields) != 4:
+ if len(fields) not in (4, 5):
continue
+ cap = None
+ if len(fields) == 5:
+ m = re.fullmatch(r"(\d+)\.(\d+)", fields[4])
+ cap = (int(m.group(1)), int(m.group(2))) if m else None
+ fields = fields[:4]
try:
index = int(fields[0])
except ValueError:
@@ -2435,10 +2449,13 @@ def _discover_nvidia_gpus():
total = free = 0
name = fields[1]
unified = any(token in name.lower() for token in ("gb10", "jetson", "grace blackwell"))
- devices.append({"index": index, "name": name,
- "total_bytes": total * 1024 * 1024,
- "free_bytes": free * 1024 * 1024,
- "unified_memory": unified})
+ device = {"index": index, "name": name,
+ "total_bytes": total * 1024 * 1024,
+ "free_bytes": free * 1024 * 1024,
+ "unified_memory": unified}
+ if cap:
+ device["compute_cap"] = cap
+ devices.append(device)
return devices
@@ -2611,6 +2628,13 @@ def _discover_amd_gpus():
return devices
+#: The oldest compute capability an engine's CUDA tier is built for by default, and
+#: what reaches older cards (docs/deepseek-v4.md).
+CUDA_TIER_FLOOR = {
+ "deepseek_v4": ((8, 0), "a build with CUDA_ARCH=portable-pre-ampere NO_TC=1 runs on Pascal and Turing"),
+}
+
+
def plans_placement(gpu):
"""Whether a discovered device's memory is qualified to drive placement.
@@ -3020,6 +3044,12 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,
# stops "a GPU exists" from being read as "a GPU should be used" -- see
# plans_placement().
planning_gpus = [gpu for gpu in gpus if plans_placement(gpu)]
+ # A CUDA tier built for newer cards than the one found: that card drives no
+ # placement (#1906: a Quadro P2000, sm_61, planned as "GPU compute" for V4)
+ floor = CUDA_TIER_FLOOR.get(resolved.descriptor.id)
+ below = [gpu for gpu in planning_gpus
+ if floor and gpu.get("compute_cap") and tuple(gpu["compute_cap"]) < floor[0]]
+ planning_gpus = [gpu for gpu in planning_gpus if gpu not in below]
placement_unified = any(gpu.get("unified_memory", False)
for gpu in planning_gpus)
@@ -3041,7 +3071,7 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,
gpu_plan = []
safe_vram = 0
for gpu in gpus:
- usable = max(0, gpu["free_bytes"] - reserve) if plans_placement(gpu) else 0
+ usable = max(0, gpu["free_bytes"] - reserve) if plans_placement(gpu) and gpu not in below else 0
safe_vram += usable
gpu_plan.append(dict(gpu, reserve_bytes=reserve, usable_bytes=usable))
requested_vram = int(vram_gb * GB) if vram_gb > 0 else safe_vram
@@ -3067,6 +3097,12 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,
vram_experts = int(vram_budget // typical) if typical else 0
hot_bytes = min(info["expert_bytes"], vram_experts * typical)
warnings = []
+ for gpu in below:
+ cc = gpu["compute_cap"]
+ warnings.append(
+ f"GPU {gpu['index']} ({gpu['name']}, sm_{cc[0]}{cc[1]}) is below the "
+ f"sm_{floor[0][0]}{floor[0][1]} {resolved.descriptor.id}'s CUDA tier is built for by "
+ f"default; planned without it ({floor[1]})")
if placement_unified:
requested_ram = int(ram_gb * GB) if ram_gb > 0 else int(available_memory * 0.88)
requested_ram_experts = max(0, requested_ram - info["dense_bytes"] - runtime_bytes)
diff --git a/c/tests/test_resource_plan.py b/c/tests/test_resource_plan.py
index 0dafdc50..49240490 100644
--- a/c/tests/test_resource_plan.py
+++ b/c/tests/test_resource_plan.py
@@ -672,6 +672,22 @@ class ResourcePlanTest(unittest.TestCase):
self.assertTrue(any("share one physical memory" in warning
for warning in plan["warnings"]))
+ def test_nvidia_compute_capability_is_read_when_the_driver_has_it(self):
+ from resource_plan import _discover_nvidia_gpus
+ ok = subprocess.CompletedProcess(args=[], returncode=0, stdout="0, Quadro P2000, 5120, 5000, 6.1\n", stderr="")
+ with mock.patch("resource_plan.subprocess.run", return_value=ok) as run:
+ devices = _discover_nvidia_gpus()
+ self.assertEqual(devices[0]["compute_cap"], (6, 1))
+ self.assertIn("compute_cap", run.call_args_list[0][0][0][1])
+ # a driver that predates the field refuses the query: asked again without it
+ old = subprocess.CompletedProcess(args=[], returncode=0, stdout="0, Quadro P2000, 5120, 5000\n", stderr="")
+ with mock.patch("resource_plan.subprocess.run",
+ side_effect=[subprocess.CalledProcessError(2, "nvidia-smi"), old]) as run:
+ devices = _discover_nvidia_gpus()
+ self.assertEqual(run.call_count, 2)
+ self.assertEqual(devices[0]["free_bytes"], 5000 * 1024 * 1024)
+ self.assertNotIn("compute_cap", devices[0])
+
def test_nvidia_unified_device_is_marked_from_name(self):
output = "0, NVIDIA GB10, 130000, 120000\n"
with mock.patch("resource_plan.subprocess.run",
@@ -1703,6 +1719,24 @@ class VulkanPartialChainTest(unittest.TestCase):
write_shard(self.model / "model.safetensors", tensors)
return analyze_model(self.model)
+ def test_deepseek_v4_plans_without_a_gpu_below_its_cuda_floor(self):
+ """#1906: a Quadro P2000 (sm_61) was planned as GPU compute for V4, whose CUDA
+ tier is built for sm_80 and newer by default."""
+ self.write_v4([0, 4])
+
+ def plan_with(cap):
+ gpu = {"index": 0, "name": "Quadro P2000", "total_bytes": 5 * GB, "free_bytes": 5 * GB,
+ "unified_memory": False, "compute_cap": cap}
+ return build_plan(self.model, ram_gb=16, available_memory=64 * GB, available_disk=1,
+ gpus=[gpu], physical_cpus=32, cpu_sockets=1)
+ old = plan_with((6, 1))
+ self.assertEqual(old["tiers"]["vram"]["budget_bytes"], 0)
+ self.assertNotIn("GPU", old["expected_bottleneck"])
+ self.assertTrue(any("sm_61" in w and "portable-pre-ampere" in w for w in old["warnings"]))
+ new = plan_with((8, 6))
+ self.assertGreater(new["tiers"]["vram"]["budget_bytes"], 0)
+ self.assertFalse(any("sm_86" in w for w in new["warnings"]))
+
def test_deepseek_v4_layout_is_the_engines(self):
# The numbers the engine printed for tools/make_deepseek_v4_tiny.py's fixture on
# Lavapipe ("[VK] deepseek_v4 chain fit: ... the engine's 1891840 B ..., layers