Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix: make placement accounting backend neutral
Mario Gutierrez39e5f0c21fb3
diff --git a/c/doctor.py b/c/doctor.pyindex 61d0b895..9f3252f3 100644--- a/c/doctor.py+++ b/c/doctor.py@@ -468,20 +468,22 @@ def run_doctor(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, *,selected_gpus = [gpu for gpu in detected_gpus if gpu["index"] in wanted]if gpu_indices == []:- checks.append(_check("accelerator.cuda", "skip", "GPU use was explicitly disabled"))+ checks.append(_check("accelerator.gpu", "skip", "GPU use was explicitly disabled"))elif gpu_indices is not None and len(selected_gpus) != len(set(gpu_indices)):- checks.append(_check("accelerator.cuda", "fail", "one or more requested GPUs were not detected",+ checks.append(_check("accelerator.gpu", "fail", "one or more requested GPUs were not detected",requested=gpu_indices, detected=[gpu["index"] for gpu in detected_gpus]))elif selected_gpus and linkage.get("missing"):- checks.append(_check("accelerator.cuda", "fail", "CUDA runtime library is missing"))+ checks.append(_check("accelerator.gpu", "fail", "GPU runtime library is missing"))elif selected_gpus and linkage.get("linked"):- checks.append(_check("accelerator.cuda", "pass", "CUDA engine and devices are available",- devices=[gpu["index"] for gpu in selected_gpus]))+ checks.append(_check("accelerator.gpu", "pass", "GPU engine and devices are available",+ devices=[gpu["index"] for gpu in selected_gpus],+ unified=any(gpu.get("unified_memory", False)+ for gpu in selected_gpus)))elif selected_gpus:- checks.append(_check("accelerator.cuda", "warn", "NVIDIA GPU detected but the engine is CPU-only",+ checks.append(_check("accelerator.gpu", "warn", "GPU detected but the engine is CPU-only",devices=[gpu["index"] for gpu in selected_gpus]))else:- checks.append(_check("accelerator.cuda", "skip", "no NVIDIA GPU detected; CPU path is available"))+ checks.append(_check("accelerator.gpu", "skip", "no supported GPU detected; CPU path is available"))try:plan = build_plan(model, ram_gb, context, gpu_indices, vram_gb,diff --git a/c/resource_plan.py b/c/resource_plan.pyindex fe785686..7bf18570 100644--- a/c/resource_plan.py+++ b/c/resource_plan.py@@ -73,6 +73,7 @@ def analyze_model(model):per_layer = {layer: int(statistics.median(sizes)) for layer, sizes in layer_sizes.items()}per_cap_bytes = sum(per_layer.values())typical_expert_bytes = int(statistics.median(per_layer.values())) if per_layer else 0+ max_expert_bytes = max(per_layer.values(), default=0)model_bytes = sum(shard.stat().st_size for shard in shards)return {"path": str(model),@@ -83,6 +84,8 @@ def analyze_model(model):"expert_count": len(expert_groups),"expert_layers": len(per_layer),"typical_expert_bytes": typical_expert_bytes,+ "max_expert_bytes": max_expert_bytes,+ "expert_bytes_by_layer": per_layer,"per_cap_bytes": per_cap_bytes,"config": config,}@@ -278,9 +281,12 @@ def _discover_nvidia_gpus():free = memory_available() // (1024 * 1024)except (OSError, AttributeError):total = free = 0- devices.append({"index": index, "name": fields[1],+ name = fields[1]+ unified = any(token in name.lower() for token in ("gb10", "jetson", "grace blackwell"))+ devices.append({"index": index, "name": name,"total_bytes": total * 1024 * 1024,- "free_bytes": free * 1024 * 1024})+ "free_bytes": free * 1024 * 1024,+ "unified_memory": unified})return devices@@ -327,7 +333,8 @@ def _discover_amd_gpus():free = max(total - used, 0)name = (row.get(name_col) or "").strip() if name_col else ""devices.append({"index": index, "name": name or f"AMD GPU {index}",- "total_bytes": total, "free_bytes": free})+ "total_bytes": total, "free_bytes": free,+ "unified_memory": False})return devices@@ -377,11 +384,17 @@ def physical_cpu_count():except (OSError, ValueError, AttributeError) as error:_physical_cores_warn(f"Windows core probe failed: {error}")if sys.platform == "darwin":- # macOS has no lscpu. sysctl reports physical cores directly, and on- # Apple Silicon hw.physicalcpu counts P+E cores with no SMT sibling to- # dedupe. Without this branch the lscpu probe below fails and every run- # prints a spurious over-subscription warning on a machine that cannot- # over-subscribe.+ # Apple Silicon's performance cores pace barriered expert matmuls. The+ # physical count includes efficiency cores, which is not the useful+ # OpenMP team size for this workload.+ try:+ result = subprocess.run(["sysctl", "-n", "hw.perflevel0.logicalcpu"],+ text=True, capture_output=True, check=True, timeout=5)+ cores = int(result.stdout.strip())+ if cores > 0:+ return cores+ except (OSError, ValueError, subprocess.SubprocessError):+ passtry:result = subprocess.run(["sysctl", "-n", "hw.physicalcpu"], text=True,capture_output=True, check=True, timeout=5)@@ -553,22 +566,17 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,wanted = set(gpu_indices)gpus = [gpu for gpu in gpus if gpu["index"] in wanted]- ram_budget = int(ram_gb * GB) if ram_gb > 0 else int(available_memory * 0.88)- if ram_budget < 4 * GB:- ram_budget = 8 * GB+ unified = any(gpu.get("unified_memory", False) for gpu in gpus)typical = info["typical_expert_bytes"]+ max_expert = info["max_expert_bytes"] or typicallayers = int(cfg.get("num_hidden_layers") or 0) + 1kv_bytes = layers * context * (int(cfg.get("kv_lora_rank") or 0) +int(cfg.get("qk_rope_head_dim") or 0)) * 4kv_buffer = context * int(cfg.get("num_attention_heads") or 0) * (int(cfg.get("qk_nope_head_dim") or 0) + int(cfg.get("v_head_dim") or 0)) * 4- runtime_bytes = int(1.2 * GB + 2.5 * GB + 64 * typical + kv_bytes + kv_buffer)- cache_bytes = max(0, ram_budget - info["dense_bytes"] - runtime_bytes)+ runtime_bytes = int(1.2 * GB + 2.5 * GB + 64 * max_expert + kv_bytes + kv_buffer)per_cap = info["per_cap_bytes"]configured_experts = int(cfg.get("n_routed_experts") or 0)- cap = int(cache_bytes // per_cap) if per_cap else 0- if configured_experts:- cap = min(cap, configured_experts)reserve = 2 * GBgpu_plan = []@@ -578,21 +586,48 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,safe_vram += usablegpu_plan.append(dict(gpu, reserve_bytes=reserve, usable_bytes=usable))requested_vram = int(vram_gb * GB) if vram_gb > 0 else safe_vram- # VRAM-resident experts do not need duplicate RAM backing: the checkpoint is- # their recovery source. RAM is therefore an independent warm compute tier.- vram_budget = min(requested_vram, safe_vram, info["expert_bytes"])+ requested_vram_before_clamp = requested_vram+ unified_pool = max(0, available_memory - info["dense_bytes"] - runtime_bytes)+ if unified:+ # Unified devices expose one physical pool to CUDA and the host. Do not+ # let an expert tier consume pages that the RAM tier also believes are+ # available. Dense/runtime reservations are shared exactly once below.+ requested_vram = min(requested_vram, unified_pool)+ vram_limit = unified_pool if unified else safe_vram+ vram_budget = min(requested_vram, vram_limit, info["expert_bytes"])vram_experts = int(vram_budget // typical) if typical else 0hot_bytes = min(info["expert_bytes"], vram_experts * typical)+ warnings = []+ if unified:+ requested_ram = int(ram_gb * GB) if ram_gb > 0 else int(available_memory * 0.88)+ requested_ram_experts = max(0, requested_ram - info["dense_bytes"] - runtime_bytes)+ ram_expert_bytes = min(requested_ram_experts,+ max(0, unified_pool - vram_budget))+ ram_budget = info["dense_bytes"] + runtime_bytes + ram_expert_bytes+ if requested_ram_experts > ram_expert_bytes:+ warnings.append(+ f"RAM budget clamped from {format_bytes(requested_ram)} to "+ f"{format_bytes(ram_budget)} because the GPU shares physical memory")+ else:+ ram_budget = int(ram_gb * GB) if ram_gb > 0 else int(available_memory * 0.88)+ if ram_budget < 4 * GB:+ ram_budget = 8 * GB if not unified else max(0, ram_budget)+ cache_bytes = max(0, ram_budget - info["dense_bytes"] - runtime_bytes)+ cap = int(cache_bytes // per_cap) if per_cap else 0+ if configured_experts:+ cap = min(cap, configured_experts)warm_bytes = min(max(0, info["expert_bytes"] - hot_bytes), cache_bytes)cold_bytes = max(0, info["expert_bytes"] - hot_bytes - warm_bytes)- warnings = []if cap < 1:warnings.append("RAM budget cannot hold one expert slot per sparse layer")if gpu_indices is not None and len(gpus) != len(set(gpu_indices)):warnings.append("one or more requested GPUs were not detected")- if gpus and vram_budget < requested_vram:- warnings.append("VRAM tier was clamped by free VRAM or model expert size")+ if gpus and vram_budget < requested_vram_before_clamp:+ warnings.append("VRAM tier was clamped by free VRAM, shared memory, or model expert size")+ if unified:+ warnings.append(+ "GPU and RAM share one physical memory pool; budgets were jointly constrained")# The plan sizes the hot tier from *free* VRAM, so running it while an engine# instance already holds the GPUs silently produces a tiny tier and a# pessimistic hit rate that describe nothing. That is exactly when a user@@ -640,8 +675,9 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,"quality_preserving": policy != "experimental-fast"},"model": {key: value for key, value in info.items() if key != "config"},"cpu": {"physical_cores": _resolve_physical_cores(physical_cpus),- "sockets": max(1, int(cpu_sockets)),- "thread_policy": "physical-cores"},+ "sockets": max(1, int(cpu_sockets)),+ "thread_policy": "physical-cores"},+ "memory": {"unified": unified, "available_bytes": available_memory},"tiers": {"disk": {"role": "cold-backing", "model_bytes": info["model_bytes"],"available_bytes": available_disk, "cold_expert_bytes": cold_bytes},diff --git a/c/tests/test_doctor.py b/c/tests/test_doctor.pyindex 97f933d8..af35f172 100644--- a/c/tests/test_doctor.py+++ b/c/tests/test_doctor.py@@ -86,7 +86,7 @@ class DoctorTest(unittest.TestCase):self.assertEqual(report["mode"], "standard")self.assertEqual(report["status"], "ok")self.assertIsNotNone(report["plan"])- self.assertEqual(checks["accelerator.cuda"]["status"], "skip")+ self.assertEqual(checks["accelerator.gpu"]["status"], "skip")self.assertEqual(checks["memory.ram"]["status"], "pass")self.assertEqual(checks["model.shards"]["details"]["shards"], 1)self.assertEqual(exit_code(report), 0)@@ -122,7 +122,7 @@ class DoctorTest(unittest.TestCase):def test_requested_missing_gpu_is_a_failure(self):report = self.report(gpu_indices=[1])- check = self.checks_by_id(report)["accelerator.cuda"]+ check = self.checks_by_id(report)["accelerator.gpu"]self.assertEqual(check["status"], "fail")self.assertEqual(check["details"], {"requested": [1], "detected": []})@@ -132,12 +132,20 @@ class DoctorTest(unittest.TestCase):gpu = {"index": 0, "name": "fixture", "total_bytes": 12 * GB,"free_bytes": 10 * GB}report = self.report(gpu_indices=None, gpus=[gpu])- check = self.checks_by_id(report)["accelerator.cuda"]+ check = self.checks_by_id(report)["accelerator.gpu"]self.assertEqual(check["status"], "warn")self.assertEqual(report["status"], "warning")self.assertEqual(exit_code(report), 0)+ def test_gpu_check_uses_backend_neutral_identifier(self):+ gpu = {"index": 0, "name": "Intel Arc B570", "total_bytes": 10 * GB,+ "free_bytes": 9 * GB}+ report = self.report(gpu_indices=None, gpus=[gpu])+ checks = self.checks_by_id(report)+ self.assertIn("accelerator.gpu", checks)+ self.assertNotIn("accelerator.cuda", checks)+def test_missing_cuda_runtime_is_a_failure(self):gpu = {"index": 0, "name": "fixture", "total_bytes": 12 * GB,"free_bytes": 10 * GB}@@ -145,8 +153,8 @@ class DoctorTest(unittest.TestCase):linkage={"linked": False, "missing": True})self.assertEqual(- self.checks_by_id(report)["accelerator.cuda"]["summary"],- "CUDA runtime library is missing",+ self.checks_by_id(report)["accelerator.gpu"]["summary"],+ "GPU runtime library is missing",)self.assertEqual(report["status"], "error")diff --git a/c/tests/test_resource_plan.py b/c/tests/test_resource_plan.pyindex c5a3fac9..7e27aaf8 100644--- a/c/tests/test_resource_plan.py+++ b/c/tests/test_resource_plan.py@@ -203,6 +203,28 @@ class ResourcePlanTest(unittest.TestCase):self.assertIn("clamped", plan["warnings"][0])self.assertIn("0:test-gpu", format_plan(plan))+ def test_unified_memory_uses_one_shared_pool(self):+ gpus = [{"index": 0, "name": "NVIDIA GB10", "total_bytes": 130 * GB,+ "free_bytes": 128 * GB, "unified_memory": True}]+ plan = build_plan(self.model, ram_gb=100, vram_gb=65,+ available_memory=121 * GB, available_disk=1,+ gpus=gpus, physical_cpus=8, cpu_sockets=1)+ ram = plan["tiers"]["ram"]["budget_bytes"]+ vram = plan["tiers"]["vram"]["budget_bytes"]+ self.assertTrue(plan["memory"]["unified"])+ self.assertLessEqual(ram + vram + plan["model"]["dense_bytes"], 121 * GB)+ self.assertTrue(any("share one physical memory" in warning+ for warning in plan["warnings"]))++ def test_nvidia_unified_device_is_marked_from_name(self):+ output = "0, NVIDIA GB10, 130000, 120000\n"+ with mock.patch("resource_plan.subprocess.run",+ return_value=subprocess.CompletedProcess(+ args=[], returncode=0, stdout=output, stderr="")):+ from resource_plan import _discover_nvidia_gpus+ devices = _discover_nvidia_gpus()+ self.assertTrue(devices[0]["unified_memory"])+def test_auto_tier_thread_count_uses_physical_cores(self):# End-to-end for #325: build_plan + environment_for_plan must export the# physical (not logical SMT) core count as OMP_NUM_THREADS. The original@@ -509,6 +531,15 @@ class PhysicalCpuCountTest(unittest.TestCase):mock.patch("sys.stderr"):self.assertEqual(physical_cpu_count(), 1)+ def test_apple_silicon_prefers_performance_cores(self):+ def sysctl(command, **kwargs):+ value = "8\n" if command[-1] == "hw.perflevel0.logicalcpu" else "10\n"+ return subprocess.CompletedProcess(args=command, returncode=0,+ stdout=value, stderr="")+ with mock.patch("resource_plan.subprocess.run", side_effect=sysctl), \+ mock.patch.object(sys, "platform", "darwin"):+ self.assertEqual(physical_cpu_count(), 8)+if __name__ == "__main__":unittest.main()diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.mdindex e1ed32d1..e67256bb 100644--- a/docs/ENVIRONMENT.md+++ b/docs/ENVIRONMENT.md@@ -177,6 +177,7 @@ Per-drive byte counts are reported in a `MIRROR:` stats line. Combine with `DIRE| Variable | Default | Effect ||---|---|---|| `COLI_VULKAN` | off | Enable the Vulkan backend. Requires a `make VK=1` build; fails at startup (no silent fallback) if libvulkan or the compiled shaders are missing. |+| `COLI_VK_DEV` | unset | Select the primary Vulkan physical-device enumeration index. Without it, the backend prefers a discrete GPU, then integrated/virtual devices. || `COLI_VK_SHADERS` | auto | Path to the compiled `qmatmul.spv` **or** the directory holding the `.spv` set; the other shaders are found next to it. Unset: `shaders/` next to the binary, then CWD-relative `shaders/qmatmul.spv`. || `COLI_VK_EXPERTS` | `320` | Pinned VRAM expert tier size: top-N experts by `.coli_usage` heat uploaded once at startup and served from VRAM with no RAM slot or disk read. `0` disables the tier (experts stay on the CPU path). ~19 MB VRAM per int4 expert. || `COLI_VK_DENSE` | `0` | Run the resident dense matmuls (attention projections, shared expert) on the GPU. |