Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix: make placement accounting backend neutral

Mario Gutierrez39e5f0c21fb3
diff --git a/c/doctor.py b/c/doctor.py
index 61d0b895..9f3252f3 100644
--- a/c/doctor.py
+++ b/c/doctor.py
@@ -468,20 +468,22 @@ def run_doctor(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, *,
selected_gpus = [gpu for gpu in detected_gpus if gpu["index"] in wanted]
if gpu_indices == []:
- checks.append(_check("accelerator.cuda", "skip", "GPU use was explicitly disabled"))
+ checks.append(_check("accelerator.gpu", "skip", "GPU use was explicitly disabled"))
elif gpu_indices is not None and len(selected_gpus) != len(set(gpu_indices)):
- checks.append(_check("accelerator.cuda", "fail", "one or more requested GPUs were not detected",
+ checks.append(_check("accelerator.gpu", "fail", "one or more requested GPUs were not detected",
requested=gpu_indices, detected=[gpu["index"] for gpu in detected_gpus]))
elif selected_gpus and linkage.get("missing"):
- checks.append(_check("accelerator.cuda", "fail", "CUDA runtime library is missing"))
+ checks.append(_check("accelerator.gpu", "fail", "GPU runtime library is missing"))
elif selected_gpus and linkage.get("linked"):
- checks.append(_check("accelerator.cuda", "pass", "CUDA engine and devices are available",
- devices=[gpu["index"] for gpu in selected_gpus]))
+ checks.append(_check("accelerator.gpu", "pass", "GPU engine and devices are available",
+ devices=[gpu["index"] for gpu in selected_gpus],
+ unified=any(gpu.get("unified_memory", False)
+ for gpu in selected_gpus)))
elif selected_gpus:
- checks.append(_check("accelerator.cuda", "warn", "NVIDIA GPU detected but the engine is CPU-only",
+ checks.append(_check("accelerator.gpu", "warn", "GPU detected but the engine is CPU-only",
devices=[gpu["index"] for gpu in selected_gpus]))
else:
- checks.append(_check("accelerator.cuda", "skip", "no NVIDIA GPU detected; CPU path is available"))
+ checks.append(_check("accelerator.gpu", "skip", "no supported GPU detected; CPU path is available"))
try:
plan = build_plan(model, ram_gb, context, gpu_indices, vram_gb,
diff --git a/c/resource_plan.py b/c/resource_plan.py
index fe785686..7bf18570 100644
--- a/c/resource_plan.py
+++ b/c/resource_plan.py
@@ -73,6 +73,7 @@ def analyze_model(model):
per_layer = {layer: int(statistics.median(sizes)) for layer, sizes in layer_sizes.items()}
per_cap_bytes = sum(per_layer.values())
typical_expert_bytes = int(statistics.median(per_layer.values())) if per_layer else 0
+ max_expert_bytes = max(per_layer.values(), default=0)
model_bytes = sum(shard.stat().st_size for shard in shards)
return {
"path": str(model),
@@ -83,6 +84,8 @@ def analyze_model(model):
"expert_count": len(expert_groups),
"expert_layers": len(per_layer),
"typical_expert_bytes": typical_expert_bytes,
+ "max_expert_bytes": max_expert_bytes,
+ "expert_bytes_by_layer": per_layer,
"per_cap_bytes": per_cap_bytes,
"config": config,
}
@@ -278,9 +281,12 @@ def _discover_nvidia_gpus():
free = memory_available() // (1024 * 1024)
except (OSError, AttributeError):
total = free = 0
- devices.append({"index": index, "name": fields[1],
+ name = fields[1]
+ unified = any(token in name.lower() for token in ("gb10", "jetson", "grace blackwell"))
+ devices.append({"index": index, "name": name,
"total_bytes": total * 1024 * 1024,
- "free_bytes": free * 1024 * 1024})
+ "free_bytes": free * 1024 * 1024,
+ "unified_memory": unified})
return devices
@@ -327,7 +333,8 @@ def _discover_amd_gpus():
free = max(total - used, 0)
name = (row.get(name_col) or "").strip() if name_col else ""
devices.append({"index": index, "name": name or f"AMD GPU {index}",
- "total_bytes": total, "free_bytes": free})
+ "total_bytes": total, "free_bytes": free,
+ "unified_memory": False})
return devices
@@ -377,11 +384,17 @@ def physical_cpu_count():
except (OSError, ValueError, AttributeError) as error:
_physical_cores_warn(f"Windows core probe failed: {error}")
if sys.platform == "darwin":
- # macOS has no lscpu. sysctl reports physical cores directly, and on
- # Apple Silicon hw.physicalcpu counts P+E cores with no SMT sibling to
- # dedupe. Without this branch the lscpu probe below fails and every run
- # prints a spurious over-subscription warning on a machine that cannot
- # over-subscribe.
+ # Apple Silicon's performance cores pace barriered expert matmuls. The
+ # physical count includes efficiency cores, which is not the useful
+ # OpenMP team size for this workload.
+ try:
+ result = subprocess.run(["sysctl", "-n", "hw.perflevel0.logicalcpu"],
+ text=True, capture_output=True, check=True, timeout=5)
+ cores = int(result.stdout.strip())
+ if cores > 0:
+ return cores
+ except (OSError, ValueError, subprocess.SubprocessError):
+ pass
try:
result = subprocess.run(["sysctl", "-n", "hw.physicalcpu"], text=True,
capture_output=True, check=True, timeout=5)
@@ -553,22 +566,17 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,
wanted = set(gpu_indices)
gpus = [gpu for gpu in gpus if gpu["index"] in wanted]
- ram_budget = int(ram_gb * GB) if ram_gb > 0 else int(available_memory * 0.88)
- if ram_budget < 4 * GB:
- ram_budget = 8 * GB
+ unified = any(gpu.get("unified_memory", False) for gpu in gpus)
typical = info["typical_expert_bytes"]
+ max_expert = info["max_expert_bytes"] or typical
layers = int(cfg.get("num_hidden_layers") or 0) + 1
kv_bytes = layers * context * (int(cfg.get("kv_lora_rank") or 0) +
int(cfg.get("qk_rope_head_dim") or 0)) * 4
kv_buffer = context * int(cfg.get("num_attention_heads") or 0) * (
int(cfg.get("qk_nope_head_dim") or 0) + int(cfg.get("v_head_dim") or 0)) * 4
- runtime_bytes = int(1.2 * GB + 2.5 * GB + 64 * typical + kv_bytes + kv_buffer)
- cache_bytes = max(0, ram_budget - info["dense_bytes"] - runtime_bytes)
+ runtime_bytes = int(1.2 * GB + 2.5 * GB + 64 * max_expert + kv_bytes + kv_buffer)
per_cap = info["per_cap_bytes"]
configured_experts = int(cfg.get("n_routed_experts") or 0)
- cap = int(cache_bytes // per_cap) if per_cap else 0
- if configured_experts:
- cap = min(cap, configured_experts)
reserve = 2 * GB
gpu_plan = []
@@ -578,21 +586,48 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,
safe_vram += usable
gpu_plan.append(dict(gpu, reserve_bytes=reserve, usable_bytes=usable))
requested_vram = int(vram_gb * GB) if vram_gb > 0 else safe_vram
- # VRAM-resident experts do not need duplicate RAM backing: the checkpoint is
- # their recovery source. RAM is therefore an independent warm compute tier.
- vram_budget = min(requested_vram, safe_vram, info["expert_bytes"])
+ requested_vram_before_clamp = requested_vram
+ unified_pool = max(0, available_memory - info["dense_bytes"] - runtime_bytes)
+ if unified:
+ # Unified devices expose one physical pool to CUDA and the host. Do not
+ # let an expert tier consume pages that the RAM tier also believes are
+ # available. Dense/runtime reservations are shared exactly once below.
+ requested_vram = min(requested_vram, unified_pool)
+ vram_limit = unified_pool if unified else safe_vram
+ vram_budget = min(requested_vram, vram_limit, info["expert_bytes"])
vram_experts = int(vram_budget // typical) if typical else 0
hot_bytes = min(info["expert_bytes"], vram_experts * typical)
+ warnings = []
+ if unified:
+ requested_ram = int(ram_gb * GB) if ram_gb > 0 else int(available_memory * 0.88)
+ requested_ram_experts = max(0, requested_ram - info["dense_bytes"] - runtime_bytes)
+ ram_expert_bytes = min(requested_ram_experts,
+ max(0, unified_pool - vram_budget))
+ ram_budget = info["dense_bytes"] + runtime_bytes + ram_expert_bytes
+ if requested_ram_experts > ram_expert_bytes:
+ warnings.append(
+ f"RAM budget clamped from {format_bytes(requested_ram)} to "
+ f"{format_bytes(ram_budget)} because the GPU shares physical memory")
+ else:
+ ram_budget = int(ram_gb * GB) if ram_gb > 0 else int(available_memory * 0.88)
+ if ram_budget < 4 * GB:
+ ram_budget = 8 * GB if not unified else max(0, ram_budget)
+ cache_bytes = max(0, ram_budget - info["dense_bytes"] - runtime_bytes)
+ cap = int(cache_bytes // per_cap) if per_cap else 0
+ if configured_experts:
+ cap = min(cap, configured_experts)
warm_bytes = min(max(0, info["expert_bytes"] - hot_bytes), cache_bytes)
cold_bytes = max(0, info["expert_bytes"] - hot_bytes - warm_bytes)
- warnings = []
if cap < 1:
warnings.append("RAM budget cannot hold one expert slot per sparse layer")
if gpu_indices is not None and len(gpus) != len(set(gpu_indices)):
warnings.append("one or more requested GPUs were not detected")
- if gpus and vram_budget < requested_vram:
- warnings.append("VRAM tier was clamped by free VRAM or model expert size")
+ if gpus and vram_budget < requested_vram_before_clamp:
+ warnings.append("VRAM tier was clamped by free VRAM, shared memory, or model expert size")
+ if unified:
+ warnings.append(
+ "GPU and RAM share one physical memory pool; budgets were jointly constrained")
# The plan sizes the hot tier from *free* VRAM, so running it while an engine
# instance already holds the GPUs silently produces a tiny tier and a
# pessimistic hit rate that describe nothing. That is exactly when a user
@@ -640,8 +675,9 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,
"quality_preserving": policy != "experimental-fast"},
"model": {key: value for key, value in info.items() if key != "config"},
"cpu": {"physical_cores": _resolve_physical_cores(physical_cpus),
- "sockets": max(1, int(cpu_sockets)),
- "thread_policy": "physical-cores"},
+ "sockets": max(1, int(cpu_sockets)),
+ "thread_policy": "physical-cores"},
+ "memory": {"unified": unified, "available_bytes": available_memory},
"tiers": {
"disk": {"role": "cold-backing", "model_bytes": info["model_bytes"],
"available_bytes": available_disk, "cold_expert_bytes": cold_bytes},
diff --git a/c/tests/test_doctor.py b/c/tests/test_doctor.py
index 97f933d8..af35f172 100644
--- a/c/tests/test_doctor.py
+++ b/c/tests/test_doctor.py
@@ -86,7 +86,7 @@ class DoctorTest(unittest.TestCase):
self.assertEqual(report["mode"], "standard")
self.assertEqual(report["status"], "ok")
self.assertIsNotNone(report["plan"])
- self.assertEqual(checks["accelerator.cuda"]["status"], "skip")
+ self.assertEqual(checks["accelerator.gpu"]["status"], "skip")
self.assertEqual(checks["memory.ram"]["status"], "pass")
self.assertEqual(checks["model.shards"]["details"]["shards"], 1)
self.assertEqual(exit_code(report), 0)
@@ -122,7 +122,7 @@ class DoctorTest(unittest.TestCase):
def test_requested_missing_gpu_is_a_failure(self):
report = self.report(gpu_indices=[1])
- check = self.checks_by_id(report)["accelerator.cuda"]
+ check = self.checks_by_id(report)["accelerator.gpu"]
self.assertEqual(check["status"], "fail")
self.assertEqual(check["details"], {"requested": [1], "detected": []})
@@ -132,12 +132,20 @@ class DoctorTest(unittest.TestCase):
gpu = {"index": 0, "name": "fixture", "total_bytes": 12 * GB,
"free_bytes": 10 * GB}
report = self.report(gpu_indices=None, gpus=[gpu])
- check = self.checks_by_id(report)["accelerator.cuda"]
+ check = self.checks_by_id(report)["accelerator.gpu"]
self.assertEqual(check["status"], "warn")
self.assertEqual(report["status"], "warning")
self.assertEqual(exit_code(report), 0)
+ def test_gpu_check_uses_backend_neutral_identifier(self):
+ gpu = {"index": 0, "name": "Intel Arc B570", "total_bytes": 10 * GB,
+ "free_bytes": 9 * GB}
+ report = self.report(gpu_indices=None, gpus=[gpu])
+ checks = self.checks_by_id(report)
+ self.assertIn("accelerator.gpu", checks)
+ self.assertNotIn("accelerator.cuda", checks)
+
def test_missing_cuda_runtime_is_a_failure(self):
gpu = {"index": 0, "name": "fixture", "total_bytes": 12 * GB,
"free_bytes": 10 * GB}
@@ -145,8 +153,8 @@ class DoctorTest(unittest.TestCase):
linkage={"linked": False, "missing": True})
self.assertEqual(
- self.checks_by_id(report)["accelerator.cuda"]["summary"],
- "CUDA runtime library is missing",
+ self.checks_by_id(report)["accelerator.gpu"]["summary"],
+ "GPU runtime library is missing",
)
self.assertEqual(report["status"], "error")
diff --git a/c/tests/test_resource_plan.py b/c/tests/test_resource_plan.py
index c5a3fac9..7e27aaf8 100644
--- a/c/tests/test_resource_plan.py
+++ b/c/tests/test_resource_plan.py
@@ -203,6 +203,28 @@ class ResourcePlanTest(unittest.TestCase):
self.assertIn("clamped", plan["warnings"][0])
self.assertIn("0:test-gpu", format_plan(plan))
+ def test_unified_memory_uses_one_shared_pool(self):
+ gpus = [{"index": 0, "name": "NVIDIA GB10", "total_bytes": 130 * GB,
+ "free_bytes": 128 * GB, "unified_memory": True}]
+ plan = build_plan(self.model, ram_gb=100, vram_gb=65,
+ available_memory=121 * GB, available_disk=1,
+ gpus=gpus, physical_cpus=8, cpu_sockets=1)
+ ram = plan["tiers"]["ram"]["budget_bytes"]
+ vram = plan["tiers"]["vram"]["budget_bytes"]
+ self.assertTrue(plan["memory"]["unified"])
+ self.assertLessEqual(ram + vram + plan["model"]["dense_bytes"], 121 * GB)
+ self.assertTrue(any("share one physical memory" in warning
+ for warning in plan["warnings"]))
+
+ def test_nvidia_unified_device_is_marked_from_name(self):
+ output = "0, NVIDIA GB10, 130000, 120000\n"
+ with mock.patch("resource_plan.subprocess.run",
+ return_value=subprocess.CompletedProcess(
+ args=[], returncode=0, stdout=output, stderr="")):
+ from resource_plan import _discover_nvidia_gpus
+ devices = _discover_nvidia_gpus()
+ self.assertTrue(devices[0]["unified_memory"])
+
def test_auto_tier_thread_count_uses_physical_cores(self):
# End-to-end for #325: build_plan + environment_for_plan must export the
# physical (not logical SMT) core count as OMP_NUM_THREADS. The original
@@ -509,6 +531,15 @@ class PhysicalCpuCountTest(unittest.TestCase):
mock.patch("sys.stderr"):
self.assertEqual(physical_cpu_count(), 1)
+ def test_apple_silicon_prefers_performance_cores(self):
+ def sysctl(command, **kwargs):
+ value = "8\n" if command[-1] == "hw.perflevel0.logicalcpu" else "10\n"
+ return subprocess.CompletedProcess(args=command, returncode=0,
+ stdout=value, stderr="")
+ with mock.patch("resource_plan.subprocess.run", side_effect=sysctl), \
+ mock.patch.object(sys, "platform", "darwin"):
+ self.assertEqual(physical_cpu_count(), 8)
+
if __name__ == "__main__":
unittest.main()
diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md
index e1ed32d1..e67256bb 100644
--- a/docs/ENVIRONMENT.md
+++ b/docs/ENVIRONMENT.md
@@ -177,6 +177,7 @@ Per-drive byte counts are reported in a `MIRROR:` stats line. Combine with `DIRE
| Variable | Default | Effect |
|---|---|---|
| `COLI_VULKAN` | off | Enable the Vulkan backend. Requires a `make VK=1` build; fails at startup (no silent fallback) if libvulkan or the compiled shaders are missing. |
+| `COLI_VK_DEV` | unset | Select the primary Vulkan physical-device enumeration index. Without it, the backend prefers a discrete GPU, then integrated/virtual devices. |
| `COLI_VK_SHADERS` | auto | Path to the compiled `qmatmul.spv` **or** the directory holding the `.spv` set; the other shaders are found next to it. Unset: `shaders/` next to the binary, then CWD-relative `shaders/qmatmul.spv`. |
| `COLI_VK_EXPERTS` | `320` | Pinned VRAM expert tier size: top-N experts by `.coli_usage` heat uploaded once at startup and served from VRAM with no RAM slot or disk read. `0` disables the tier (experts stay on the CPU path). ~19 MB VRAM per int4 expert. |
| `COLI_VK_DENSE` | `0` | Run the resident dense matmuls (attention projections, shared expert) on the GPU. |