Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
coli doctor: read-only setup/health diagnostics (path, config, shards, disk, RAM budget, placement) (#33)
ZacharyZcR8f5f3e3a2b7b
diff --git a/README.md b/README.mdindex 4af26674..03cbce15 100644--- a/README.md+++ b/README.md@@ -110,6 +110,21 @@ sets the RAM budget and context immediately; the VRAM tier is enabled only whenthe current `glm` binary is linked with CUDA. Explicit flags and environmentvariables keep precedence over automatic values.+Before loading the model, `coli doctor` performs a read-only readiness check and+explains whether the selected Disk/RAM/VRAM placement is runnable:++```bash+COLI_MODEL=/nvme/glm52_i4 ./coli doctor+COLI_MODEL=/nvme/glm52_i4 ./coli doctor --gpu 0 --ram 128 --json+```++Doctor validates the model directory, config, tokenizer, safetensors headers,+engine executable, available RAM, requested NVIDIA devices, CUDA linkage, and the+same placement budget used by `coli plan`. It never starts `glm`, reads tensor+payloads, imports a model framework, or creates a CUDA context. The versioned JSON+report uses stable check IDs for automation. Warnings keep exit status 0; missing+requirements or an unsafe RAM projection return 1, while invalid CLI values return 2.+The engine at runtime is pure C — python is only used by the one-time converter.### Windows 11 (native, no WSL)diff --git a/c/coli b/c/coliindex 32a80dcf..f9e601eb 100755--- a/c/coli+++ b/c/coli@@ -8,6 +8,7 @@ CLI per far girare GLM-5.2 (744B) in locale, su CPU, in ~15-26 GB di RAM.coli run "prompt" generazione singolacoli info stato: modello, RAM, disco, configcoli plan piano risorse Disk / RAM / VRAM+ coli doctor diagnosi installazione e piano di esecuzionecoli bench [task...] benchmark di qualità (MMLU/HellaSwag/...)coli convert converte GLM-5.2-FP8 -> int4 (streaming)coli build compila il motore@@ -334,6 +335,23 @@ def cmd_plan(a):print(textwrap.indent(format_plan(plan)," "))print()+def cmd_doctor(a):+ from doctor import exit_code, format_doctor, run_doctor+ try:+ ram,ctx,devices,vram=resource_request(a,os.environ)+ if ctx<1: raise ValueError("--ctx deve essere positivo")+ if ram<0: raise ValueError("--ram non puo essere negativo")+ if vram<0: raise ValueError("--vram non puo essere negativo")+ except ValueError as error:+ report={"schema_version":1,"status":"error","model":os.path.abspath(a.model),+ "checks":[{"id":"config.arguments","status":"fail","summary":str(error)}],+ "plan":None}+ print(json.dumps(report,indent=2) if a.json else format_doctor(report))+ return 2+ report=run_doctor(a.model,ram,ctx,devices,vram,engine_path=GLM)+ print(json.dumps(report,indent=2) if a.json else format_doctor(report))+ return exit_code(report)+def cmd_run(a):need_model(a.model)prompt=" ".join(a.prompt) if a.prompt else sys.exit('uso: coli run "il tuo prompt"')@@ -507,6 +525,8 @@ def main():sub.add_parser("build", parents=[common]); sub.add_parser("info", parents=[common])pp=sub.add_parser("plan",parents=[common])pp.add_argument("--json",action="store_true")+ pd=sub.add_parser("doctor",parents=[common])+ pd.add_argument("--json",action="store_true",help="emette un report JSON versionato")pr=sub.add_parser("run", parents=[common]); pr.add_argument("prompt", nargs="*")sub.add_parser("chat", parents=[common])ps=sub.add_parser("serve", parents=[common])@@ -523,8 +543,11 @@ def main():pc.add_argument("--ebits",type=int,default=4); pc.add_argument("--io-bits",type=int,default=8); pc.add_argument("--xbits",type=int,default=0)pc.add_argument("--no-mtp",action="store_true",help="salta la testa MTP (niente draft speculativi)")a=ap.parse_args()- {"build":cmd_build,"info":cmd_info,"plan":cmd_plan,"run":cmd_run,"chat":cmd_chat,"serve":cmd_serve,"bench":cmd_bench,- "convert":cmd_convert}.get(a.cmd, lambda _:(banner(),print(__doc__)))(a)+ handler={"build":cmd_build,"info":cmd_info,"plan":cmd_plan,"doctor":cmd_doctor,+ "run":cmd_run,"chat":cmd_chat,"serve":cmd_serve,"bench":cmd_bench,+ "convert":cmd_convert}.get(a.cmd)+ if handler: sys.exit(handler(a) or 0)+ banner(); print(__doc__)if __name__=="__main__":signal.signal(signal.SIGINT, signal.default_int_handler)diff --git a/c/doctor.py b/c/doctor.pynew file mode 100644index 00000000..0fb74af1--- /dev/null+++ b/c/doctor.py@@ -0,0 +1,150 @@+#!/usr/bin/env python3+"""Read-only installation diagnostics for colibri."""++import os+import json+import subprocess+from pathlib import Path++from resource_plan import GB, build_plan, discover_gpus, format_plan, memory_available+++def _check(identifier, status, summary, **details):+ item = {"id": identifier, "status": status, "summary": summary}+ if details:+ item["details"] = details+ return item+++def cuda_linkage(engine_path):+ """Return CUDA linkage state without loading the executable or CUDA runtime."""+ if not Path(engine_path).is_file() or os.name != "posix":+ return {"linked": False, "missing": False}+ try:+ result = subprocess.run(["ldd", str(engine_path)], capture_output=True, text=True,+ timeout=3, check=False)+ except (OSError, subprocess.SubprocessError):+ return {"linked": False, "missing": False}+ lines = [line for line in result.stdout.splitlines() if "libcudart" in line]+ return {"linked": any("not found" not in line for line in lines),+ "missing": any("not found" in line for line in lines)}+++def run_doctor(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, *,+ engine_path, available_memory=None, available_disk=None, gpus=None,+ linkage=None):+ """Collect a complete report. No model payload, engine, or CUDA context is loaded."""+ model = Path(model).expanduser().resolve()+ checks = []+ plan = None++ if model.is_dir() and os.access(model, os.R_OK):+ checks.append(_check("model.path", "pass", "model directory is readable", path=str(model)))+ elif model.is_dir():+ checks.append(_check("model.path", "fail", "model directory is not readable", path=str(model)))+ else:+ checks.append(_check("model.path", "fail", "model directory does not exist", path=str(model)))++ config = model / "config.json"+ try:+ valid_config = isinstance(json.loads(config.read_text()), dict)+ except (OSError, ValueError):+ valid_config = False+ checks.append(_check("model.config", "pass" if valid_config else "fail",+ "config.json is valid" if valid_config else "config.json is missing or invalid"))+ tokenizer = model / "tokenizer.json"+ checks.append(_check("model.tokenizer", "pass" if tokenizer.is_file() else "fail",+ "tokenizer.json found" if tokenizer.is_file() else "tokenizer.json is missing"))+ if model.is_dir() and os.access(model, os.W_OK):+ checks.append(_check("storage.persistence", "pass", "model directory can store usage and KV state"))+ elif model.is_dir():+ checks.append(_check("storage.persistence", "warn", "model directory is read-only; disable persistence or change permissions"))+ else:+ checks.append(_check("storage.persistence", "skip", "persistence requires a model directory"))++ engine = Path(engine_path)+ if engine.is_file() and os.access(engine, os.X_OK):+ checks.append(_check("engine.binary", "pass", "engine executable is ready", path=str(engine)))+ elif engine.is_file():+ checks.append(_check("engine.binary", "fail", "engine exists but is not executable", path=str(engine)))+ else:+ checks.append(_check("engine.binary", "fail", "engine is not built", path=str(engine)))++ available_memory = memory_available() if available_memory is None else available_memory+ detected_gpus = discover_gpus() if gpus is None else list(gpus)+ linkage = cuda_linkage(engine) if linkage is None else linkage+ selected_gpus = detected_gpus+ if gpu_indices is not None:+ wanted = set(gpu_indices)+ selected_gpus = [gpu for gpu in detected_gpus if gpu["index"] in wanted]++ if gpu_indices == []:+ checks.append(_check("accelerator.cuda", "skip", "GPU use was explicitly disabled"))+ elif gpu_indices is not None and len(selected_gpus) != len(set(gpu_indices)):+ checks.append(_check("accelerator.cuda", "fail", "one or more requested GPUs were not detected",+ requested=gpu_indices, detected=[gpu["index"] for gpu in detected_gpus]))+ elif selected_gpus and linkage.get("missing"):+ checks.append(_check("accelerator.cuda", "fail", "CUDA runtime library is missing"))+ elif selected_gpus and linkage.get("linked"):+ checks.append(_check("accelerator.cuda", "pass", "CUDA engine and devices are available",+ devices=[gpu["index"] for gpu in selected_gpus]))+ elif selected_gpus:+ checks.append(_check("accelerator.cuda", "warn", "NVIDIA GPU detected but the engine is CPU-only",+ devices=[gpu["index"] for gpu in selected_gpus]))+ else:+ checks.append(_check("accelerator.cuda", "skip", "no NVIDIA GPU detected; CPU path is available"))++ try:+ plan = build_plan(model, ram_gb, context, gpu_indices, vram_gb,+ available_memory=available_memory, available_disk=available_disk,+ gpus=detected_gpus)+ model_info = plan["model"]+ checks.append(_check("model.shards", "pass", "safetensors headers are valid",+ shards=model_info["shards"], model_bytes=model_info["model_bytes"]))+ disk = plan["tiers"]["disk"]+ disk_status = "warn" if disk["available_bytes"] < GB else "pass"+ disk_summary = ("less than 1 GB is free for runtime state" if disk_status == "warn" else+ "model backing store is available")+ checks.append(_check("storage.disk", disk_status, disk_summary,+ available_bytes=disk["available_bytes"], model_bytes=disk["model_bytes"]))+ ram = plan["tiers"]["ram"]+ if not available_memory:+ ram_status, ram_summary = "warn", "available RAM could not be measured"+ elif ram["budget_bytes"] > available_memory:+ ram_status, ram_summary = "fail", "planned RAM budget exceeds available memory"+ elif ram["cache_slots_per_layer"] < 1:+ ram_status, ram_summary = "fail", "RAM budget cannot hold one expert slot per sparse layer"+ else:+ ram_status, ram_summary = "pass", "RAM budget is viable"+ checks.append(_check("memory.ram", ram_status, ram_summary,+ available_bytes=available_memory, budget_bytes=ram["budget_bytes"],+ cache_slots_per_layer=ram["cache_slots_per_layer"]))+ if plan["warnings"]:+ checks.append(_check("placement.plan", "warn", "; ".join(plan["warnings"])))+ else:+ checks.append(_check("placement.plan", "pass", "tier placement has no warnings"))+ except (OSError, ValueError, KeyError) as error:+ checks.append(_check("model.shards", "fail", str(error)))+ checks.append(_check("storage.disk", "skip", "storage check requires a valid model"))+ checks.append(_check("memory.ram", "skip", "RAM projection requires a valid model"))+ checks.append(_check("placement.plan", "skip", "placement requires a valid model"))++ statuses = {item["status"] for item in checks}+ status = "error" if "fail" in statuses else "warning" if "warn" in statuses else "ok"+ return {"schema_version": 1, "status": status, "model": str(model),+ "checks": checks, "plan": plan}+++def format_doctor(report):+ icons = {"pass": "ok", "warn": "warn", "fail": "fail", "skip": "skip"}+ lines = [f"colibri doctor · {report['model']}"]+ for check in report["checks"]:+ lines.append(f"[{icons[check['status']]:>4}] {check['id']:<18} {check['summary']}")+ if report["plan"]:+ lines.extend(["", format_plan(report["plan"])])+ lines.extend(["", f"result {report['status']}"])+ return "\n".join(lines)+++def exit_code(report):+ return 1 if report["status"] == "error" else 0diff --git a/c/tests/test_doctor.py b/c/tests/test_doctor.pynew file mode 100644index 00000000..c618636d--- /dev/null+++ b/c/tests/test_doctor.py@@ -0,0 +1,167 @@+import json+import struct+import subprocess+import sys+import tempfile+import unittest+from pathlib import Path++from doctor import exit_code, format_doctor, run_doctor+from resource_plan import GB+++def write_shard(path, tensors):+ offset = 0+ header = {}+ payload = b""+ for name, size in tensors:+ header[name] = {"dtype": "U8", "shape": [size],+ "data_offsets": [offset, offset + size]}+ payload += b"\0" * size+ offset += size+ raw = json.dumps(header).encode()+ path.write_bytes(struct.pack("<Q", len(raw)) + raw + payload)+++class DoctorTest(unittest.TestCase):+ def setUp(self):+ self.tmp = tempfile.TemporaryDirectory()+ self.root = Path(self.tmp.name)+ self.model = self.root / "model"+ self.model.mkdir()+ (self.model / "config.json").write_text(json.dumps({+ "num_hidden_layers": 2,+ "n_routed_experts": 2,+ "kv_lora_rank": 4,+ "qk_rope_head_dim": 2,+ "qk_nope_head_dim": 3,+ "v_head_dim": 5,+ "num_attention_heads": 2,+ }))+ (self.model / "tokenizer.json").write_text("{}")+ write_shard(self.model / "model.safetensors", [+ ("model.embed_tokens.weight", 100),+ ("model.layers.0.self_attn.q_a_proj.weight", 200),+ ("model.layers.1.mlp.experts.0.gate_proj.weight", 30),+ ("model.layers.1.mlp.experts.0.up_proj.weight", 30),+ ("model.layers.1.mlp.experts.1.gate_proj.weight", 30),+ ("model.layers.1.mlp.experts.1.up_proj.weight", 30),+ ])+ self.engine = self.root / "glm"+ self.engine.write_text("#!/bin/sh\nexit 0\n")+ self.engine.chmod(0o755)++ def tearDown(self):+ self.tmp.cleanup()++ def report(self, **overrides):+ arguments = {+ "model": self.model,+ "ram_gb": 16,+ "context": 32,+ "gpu_indices": [],+ "vram_gb": 0,+ "engine_path": self.engine,+ "available_memory": 32 * GB,+ "available_disk": 100 * GB,+ "gpus": [],+ "linkage": {"linked": False, "missing": False},+ }+ arguments.update(overrides)+ return run_doctor(**arguments)++ @staticmethod+ def checks_by_id(report):+ return {check["id"]: check for check in report["checks"]}++ def test_healthy_cpu_install_has_versioned_report(self):+ report = self.report()+ checks = self.checks_by_id(report)++ self.assertEqual(report["schema_version"], 1)+ self.assertEqual(report["status"], "ok")+ self.assertIsNotNone(report["plan"])+ self.assertEqual(checks["accelerator.cuda"]["status"], "skip")+ self.assertEqual(checks["memory.ram"]["status"], "pass")+ self.assertEqual(checks["model.shards"]["details"]["shards"], 1)+ self.assertEqual(exit_code(report), 0)++ def test_missing_model_collects_failures_instead_of_stopping_early(self):+ report = self.report(model=self.root / "missing")+ checks = self.checks_by_id(report)++ self.assertEqual(report["status"], "error")+ self.assertEqual(checks["model.path"]["status"], "fail")+ self.assertEqual(checks["model.config"]["status"], "fail")+ self.assertEqual(checks["model.tokenizer"]["status"], "fail")+ self.assertEqual(checks["model.shards"]["status"], "fail")+ self.assertEqual(checks["storage.disk"]["status"], "skip")+ self.assertIsNone(report["plan"])+ self.assertEqual(exit_code(report), 1)++ def test_non_executable_engine_and_excessive_ram_budget_fail(self):+ self.engine.chmod(0o644)+ report = self.report(ram_gb=40)+ checks = self.checks_by_id(report)++ self.assertEqual(checks["engine.binary"]["status"], "fail")+ self.assertEqual(checks["memory.ram"]["status"], "fail")+ self.assertEqual(report["status"], "error")++ def test_requested_missing_gpu_is_a_failure(self):+ report = self.report(gpu_indices=[1])+ check = self.checks_by_id(report)["accelerator.cuda"]++ self.assertEqual(check["status"], "fail")+ self.assertEqual(check["details"], {"requested": [1], "detected": []})+ self.assertEqual(exit_code(report), 1)++ def test_cpu_engine_with_detected_gpu_is_only_a_warning(self):+ gpu = {"index": 0, "name": "fixture", "total_bytes": 12 * GB,+ "free_bytes": 10 * GB}+ report = self.report(gpu_indices=None, gpus=[gpu])+ check = self.checks_by_id(report)["accelerator.cuda"]++ self.assertEqual(check["status"], "warn")+ self.assertEqual(report["status"], "warning")+ self.assertEqual(exit_code(report), 0)++ def test_missing_cuda_runtime_is_a_failure(self):+ gpu = {"index": 0, "name": "fixture", "total_bytes": 12 * GB,+ "free_bytes": 10 * GB}+ report = self.report(gpu_indices=[0], gpus=[gpu],+ linkage={"linked": False, "missing": True})++ self.assertEqual(+ self.checks_by_id(report)["accelerator.cuda"]["summary"],+ "CUDA runtime library is missing",+ )+ self.assertEqual(report["status"], "error")++ def test_text_format_contains_checks_plan_and_result(self):+ output = format_doctor(self.report())++ self.assertIn("model.path", output)+ self.assertIn("disk backing store", output)+ self.assertTrue(output.endswith("result ok"))++ def test_cli_json_is_machine_readable_without_loading_model(self):+ cli = Path(__file__).parents[1] / "coli"+ run = subprocess.run([+ sys.executable, str(cli), "doctor", "--model", str(self.model),+ "--gpu", "none", "--ram", "16", "--ctx", "32", "--json",+ ], text=True, capture_output=True, check=False)++ # The repository engine may be absent; doctor must still return one complete JSON report.+ self.assertIn(run.returncode, (0, 1))+ report = json.loads(run.stdout)+ self.assertEqual(report["schema_version"], 1)+ self.assertEqual(Path(report["model"]), self.model.resolve())+ self.assertIn(report["status"], ("ok", "warning", "error"))+ self.assertNotIn("\033", run.stdout)+ self.assertTrue(run.stdout.lstrip().startswith("{"))+ self.assertTrue(run.stdout.rstrip().endswith("}"))+++if __name__ == "__main__":+ unittest.main()