Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

coli doctor: read-only setup/health diagnostics (path, config, shards, disk, RAM budget, placement) (#33)

ZacharyZcR8f5f3e3a2b7b
diff --git a/README.md b/README.md
index 4af26674..03cbce15 100644
--- a/README.md
+++ b/README.md
@@ -110,6 +110,21 @@ sets the RAM budget and context immediately; the VRAM tier is enabled only when
the current `glm` binary is linked with CUDA. Explicit flags and environment
variables keep precedence over automatic values.
+Before loading the model, `coli doctor` performs a read-only readiness check and
+explains whether the selected Disk/RAM/VRAM placement is runnable:
+
+```bash
+COLI_MODEL=/nvme/glm52_i4 ./coli doctor
+COLI_MODEL=/nvme/glm52_i4 ./coli doctor --gpu 0 --ram 128 --json
+```
+
+Doctor validates the model directory, config, tokenizer, safetensors headers,
+engine executable, available RAM, requested NVIDIA devices, CUDA linkage, and the
+same placement budget used by `coli plan`. It never starts `glm`, reads tensor
+payloads, imports a model framework, or creates a CUDA context. The versioned JSON
+report uses stable check IDs for automation. Warnings keep exit status 0; missing
+requirements or an unsafe RAM projection return 1, while invalid CLI values return 2.
+
The engine at runtime is pure C — python is only used by the one-time converter.
### Windows 11 (native, no WSL)
diff --git a/c/coli b/c/coli
index 32a80dcf..f9e601eb 100755
--- a/c/coli
+++ b/c/coli
@@ -8,6 +8,7 @@ CLI per far girare GLM-5.2 (744B) in locale, su CPU, in ~15-26 GB di RAM.
coli run "prompt" generazione singola
coli info stato: modello, RAM, disco, config
coli plan piano risorse Disk / RAM / VRAM
+ coli doctor diagnosi installazione e piano di esecuzione
coli bench [task...] benchmark di qualità (MMLU/HellaSwag/...)
coli convert converte GLM-5.2-FP8 -> int4 (streaming)
coli build compila il motore
@@ -334,6 +335,23 @@ def cmd_plan(a):
print(textwrap.indent(format_plan(plan)," "))
print()
+def cmd_doctor(a):
+ from doctor import exit_code, format_doctor, run_doctor
+ try:
+ ram,ctx,devices,vram=resource_request(a,os.environ)
+ if ctx<1: raise ValueError("--ctx deve essere positivo")
+ if ram<0: raise ValueError("--ram non puo essere negativo")
+ if vram<0: raise ValueError("--vram non puo essere negativo")
+ except ValueError as error:
+ report={"schema_version":1,"status":"error","model":os.path.abspath(a.model),
+ "checks":[{"id":"config.arguments","status":"fail","summary":str(error)}],
+ "plan":None}
+ print(json.dumps(report,indent=2) if a.json else format_doctor(report))
+ return 2
+ report=run_doctor(a.model,ram,ctx,devices,vram,engine_path=GLM)
+ print(json.dumps(report,indent=2) if a.json else format_doctor(report))
+ return exit_code(report)
+
def cmd_run(a):
need_model(a.model)
prompt=" ".join(a.prompt) if a.prompt else sys.exit('uso: coli run "il tuo prompt"')
@@ -507,6 +525,8 @@ def main():
sub.add_parser("build", parents=[common]); sub.add_parser("info", parents=[common])
pp=sub.add_parser("plan",parents=[common])
pp.add_argument("--json",action="store_true")
+ pd=sub.add_parser("doctor",parents=[common])
+ pd.add_argument("--json",action="store_true",help="emette un report JSON versionato")
pr=sub.add_parser("run", parents=[common]); pr.add_argument("prompt", nargs="*")
sub.add_parser("chat", parents=[common])
ps=sub.add_parser("serve", parents=[common])
@@ -523,8 +543,11 @@ def main():
pc.add_argument("--ebits",type=int,default=4); pc.add_argument("--io-bits",type=int,default=8); pc.add_argument("--xbits",type=int,default=0)
pc.add_argument("--no-mtp",action="store_true",help="salta la testa MTP (niente draft speculativi)")
a=ap.parse_args()
- {"build":cmd_build,"info":cmd_info,"plan":cmd_plan,"run":cmd_run,"chat":cmd_chat,"serve":cmd_serve,"bench":cmd_bench,
- "convert":cmd_convert}.get(a.cmd, lambda _:(banner(),print(__doc__)))(a)
+ handler={"build":cmd_build,"info":cmd_info,"plan":cmd_plan,"doctor":cmd_doctor,
+ "run":cmd_run,"chat":cmd_chat,"serve":cmd_serve,"bench":cmd_bench,
+ "convert":cmd_convert}.get(a.cmd)
+ if handler: sys.exit(handler(a) or 0)
+ banner(); print(__doc__)
if __name__=="__main__":
signal.signal(signal.SIGINT, signal.default_int_handler)
diff --git a/c/doctor.py b/c/doctor.py
new file mode 100644
index 00000000..0fb74af1
--- /dev/null
+++ b/c/doctor.py
@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+"""Read-only installation diagnostics for colibri."""
+
+import os
+import json
+import subprocess
+from pathlib import Path
+
+from resource_plan import GB, build_plan, discover_gpus, format_plan, memory_available
+
+
+def _check(identifier, status, summary, **details):
+ item = {"id": identifier, "status": status, "summary": summary}
+ if details:
+ item["details"] = details
+ return item
+
+
+def cuda_linkage(engine_path):
+ """Return CUDA linkage state without loading the executable or CUDA runtime."""
+ if not Path(engine_path).is_file() or os.name != "posix":
+ return {"linked": False, "missing": False}
+ try:
+ result = subprocess.run(["ldd", str(engine_path)], capture_output=True, text=True,
+ timeout=3, check=False)
+ except (OSError, subprocess.SubprocessError):
+ return {"linked": False, "missing": False}
+ lines = [line for line in result.stdout.splitlines() if "libcudart" in line]
+ return {"linked": any("not found" not in line for line in lines),
+ "missing": any("not found" in line for line in lines)}
+
+
+def run_doctor(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0, *,
+ engine_path, available_memory=None, available_disk=None, gpus=None,
+ linkage=None):
+ """Collect a complete report. No model payload, engine, or CUDA context is loaded."""
+ model = Path(model).expanduser().resolve()
+ checks = []
+ plan = None
+
+ if model.is_dir() and os.access(model, os.R_OK):
+ checks.append(_check("model.path", "pass", "model directory is readable", path=str(model)))
+ elif model.is_dir():
+ checks.append(_check("model.path", "fail", "model directory is not readable", path=str(model)))
+ else:
+ checks.append(_check("model.path", "fail", "model directory does not exist", path=str(model)))
+
+ config = model / "config.json"
+ try:
+ valid_config = isinstance(json.loads(config.read_text()), dict)
+ except (OSError, ValueError):
+ valid_config = False
+ checks.append(_check("model.config", "pass" if valid_config else "fail",
+ "config.json is valid" if valid_config else "config.json is missing or invalid"))
+ tokenizer = model / "tokenizer.json"
+ checks.append(_check("model.tokenizer", "pass" if tokenizer.is_file() else "fail",
+ "tokenizer.json found" if tokenizer.is_file() else "tokenizer.json is missing"))
+ if model.is_dir() and os.access(model, os.W_OK):
+ checks.append(_check("storage.persistence", "pass", "model directory can store usage and KV state"))
+ elif model.is_dir():
+ checks.append(_check("storage.persistence", "warn", "model directory is read-only; disable persistence or change permissions"))
+ else:
+ checks.append(_check("storage.persistence", "skip", "persistence requires a model directory"))
+
+ engine = Path(engine_path)
+ if engine.is_file() and os.access(engine, os.X_OK):
+ checks.append(_check("engine.binary", "pass", "engine executable is ready", path=str(engine)))
+ elif engine.is_file():
+ checks.append(_check("engine.binary", "fail", "engine exists but is not executable", path=str(engine)))
+ else:
+ checks.append(_check("engine.binary", "fail", "engine is not built", path=str(engine)))
+
+ available_memory = memory_available() if available_memory is None else available_memory
+ detected_gpus = discover_gpus() if gpus is None else list(gpus)
+ linkage = cuda_linkage(engine) if linkage is None else linkage
+ selected_gpus = detected_gpus
+ if gpu_indices is not None:
+ wanted = set(gpu_indices)
+ selected_gpus = [gpu for gpu in detected_gpus if gpu["index"] in wanted]
+
+ if gpu_indices == []:
+ checks.append(_check("accelerator.cuda", "skip", "GPU use was explicitly disabled"))
+ elif gpu_indices is not None and len(selected_gpus) != len(set(gpu_indices)):
+ checks.append(_check("accelerator.cuda", "fail", "one or more requested GPUs were not detected",
+ requested=gpu_indices, detected=[gpu["index"] for gpu in detected_gpus]))
+ elif selected_gpus and linkage.get("missing"):
+ checks.append(_check("accelerator.cuda", "fail", "CUDA runtime library is missing"))
+ elif selected_gpus and linkage.get("linked"):
+ checks.append(_check("accelerator.cuda", "pass", "CUDA engine and devices are available",
+ devices=[gpu["index"] for gpu in selected_gpus]))
+ elif selected_gpus:
+ checks.append(_check("accelerator.cuda", "warn", "NVIDIA GPU detected but the engine is CPU-only",
+ devices=[gpu["index"] for gpu in selected_gpus]))
+ else:
+ checks.append(_check("accelerator.cuda", "skip", "no NVIDIA GPU detected; CPU path is available"))
+
+ try:
+ plan = build_plan(model, ram_gb, context, gpu_indices, vram_gb,
+ available_memory=available_memory, available_disk=available_disk,
+ gpus=detected_gpus)
+ model_info = plan["model"]
+ checks.append(_check("model.shards", "pass", "safetensors headers are valid",
+ shards=model_info["shards"], model_bytes=model_info["model_bytes"]))
+ disk = plan["tiers"]["disk"]
+ disk_status = "warn" if disk["available_bytes"] < GB else "pass"
+ disk_summary = ("less than 1 GB is free for runtime state" if disk_status == "warn" else
+ "model backing store is available")
+ checks.append(_check("storage.disk", disk_status, disk_summary,
+ available_bytes=disk["available_bytes"], model_bytes=disk["model_bytes"]))
+ ram = plan["tiers"]["ram"]
+ if not available_memory:
+ ram_status, ram_summary = "warn", "available RAM could not be measured"
+ elif ram["budget_bytes"] > available_memory:
+ ram_status, ram_summary = "fail", "planned RAM budget exceeds available memory"
+ elif ram["cache_slots_per_layer"] < 1:
+ ram_status, ram_summary = "fail", "RAM budget cannot hold one expert slot per sparse layer"
+ else:
+ ram_status, ram_summary = "pass", "RAM budget is viable"
+ checks.append(_check("memory.ram", ram_status, ram_summary,
+ available_bytes=available_memory, budget_bytes=ram["budget_bytes"],
+ cache_slots_per_layer=ram["cache_slots_per_layer"]))
+ if plan["warnings"]:
+ checks.append(_check("placement.plan", "warn", "; ".join(plan["warnings"])))
+ else:
+ checks.append(_check("placement.plan", "pass", "tier placement has no warnings"))
+ except (OSError, ValueError, KeyError) as error:
+ checks.append(_check("model.shards", "fail", str(error)))
+ checks.append(_check("storage.disk", "skip", "storage check requires a valid model"))
+ checks.append(_check("memory.ram", "skip", "RAM projection requires a valid model"))
+ checks.append(_check("placement.plan", "skip", "placement requires a valid model"))
+
+ statuses = {item["status"] for item in checks}
+ status = "error" if "fail" in statuses else "warning" if "warn" in statuses else "ok"
+ return {"schema_version": 1, "status": status, "model": str(model),
+ "checks": checks, "plan": plan}
+
+
+def format_doctor(report):
+ icons = {"pass": "ok", "warn": "warn", "fail": "fail", "skip": "skip"}
+ lines = [f"colibri doctor · {report['model']}"]
+ for check in report["checks"]:
+ lines.append(f"[{icons[check['status']]:>4}] {check['id']:<18} {check['summary']}")
+ if report["plan"]:
+ lines.extend(["", format_plan(report["plan"])])
+ lines.extend(["", f"result {report['status']}"])
+ return "\n".join(lines)
+
+
+def exit_code(report):
+ return 1 if report["status"] == "error" else 0
diff --git a/c/tests/test_doctor.py b/c/tests/test_doctor.py
new file mode 100644
index 00000000..c618636d
--- /dev/null
+++ b/c/tests/test_doctor.py
@@ -0,0 +1,167 @@
+import json
+import struct
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+from doctor import exit_code, format_doctor, run_doctor
+from resource_plan import GB
+
+
+def write_shard(path, tensors):
+ offset = 0
+ header = {}
+ payload = b""
+ for name, size in tensors:
+ header[name] = {"dtype": "U8", "shape": [size],
+ "data_offsets": [offset, offset + size]}
+ payload += b"\0" * size
+ offset += size
+ raw = json.dumps(header).encode()
+ path.write_bytes(struct.pack("<Q", len(raw)) + raw + payload)
+
+
+class DoctorTest(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.TemporaryDirectory()
+ self.root = Path(self.tmp.name)
+ self.model = self.root / "model"
+ self.model.mkdir()
+ (self.model / "config.json").write_text(json.dumps({
+ "num_hidden_layers": 2,
+ "n_routed_experts": 2,
+ "kv_lora_rank": 4,
+ "qk_rope_head_dim": 2,
+ "qk_nope_head_dim": 3,
+ "v_head_dim": 5,
+ "num_attention_heads": 2,
+ }))
+ (self.model / "tokenizer.json").write_text("{}")
+ write_shard(self.model / "model.safetensors", [
+ ("model.embed_tokens.weight", 100),
+ ("model.layers.0.self_attn.q_a_proj.weight", 200),
+ ("model.layers.1.mlp.experts.0.gate_proj.weight", 30),
+ ("model.layers.1.mlp.experts.0.up_proj.weight", 30),
+ ("model.layers.1.mlp.experts.1.gate_proj.weight", 30),
+ ("model.layers.1.mlp.experts.1.up_proj.weight", 30),
+ ])
+ self.engine = self.root / "glm"
+ self.engine.write_text("#!/bin/sh\nexit 0\n")
+ self.engine.chmod(0o755)
+
+ def tearDown(self):
+ self.tmp.cleanup()
+
+ def report(self, **overrides):
+ arguments = {
+ "model": self.model,
+ "ram_gb": 16,
+ "context": 32,
+ "gpu_indices": [],
+ "vram_gb": 0,
+ "engine_path": self.engine,
+ "available_memory": 32 * GB,
+ "available_disk": 100 * GB,
+ "gpus": [],
+ "linkage": {"linked": False, "missing": False},
+ }
+ arguments.update(overrides)
+ return run_doctor(**arguments)
+
+ @staticmethod
+ def checks_by_id(report):
+ return {check["id"]: check for check in report["checks"]}
+
+ def test_healthy_cpu_install_has_versioned_report(self):
+ report = self.report()
+ checks = self.checks_by_id(report)
+
+ self.assertEqual(report["schema_version"], 1)
+ self.assertEqual(report["status"], "ok")
+ self.assertIsNotNone(report["plan"])
+ self.assertEqual(checks["accelerator.cuda"]["status"], "skip")
+ self.assertEqual(checks["memory.ram"]["status"], "pass")
+ self.assertEqual(checks["model.shards"]["details"]["shards"], 1)
+ self.assertEqual(exit_code(report), 0)
+
+ def test_missing_model_collects_failures_instead_of_stopping_early(self):
+ report = self.report(model=self.root / "missing")
+ checks = self.checks_by_id(report)
+
+ self.assertEqual(report["status"], "error")
+ self.assertEqual(checks["model.path"]["status"], "fail")
+ self.assertEqual(checks["model.config"]["status"], "fail")
+ self.assertEqual(checks["model.tokenizer"]["status"], "fail")
+ self.assertEqual(checks["model.shards"]["status"], "fail")
+ self.assertEqual(checks["storage.disk"]["status"], "skip")
+ self.assertIsNone(report["plan"])
+ self.assertEqual(exit_code(report), 1)
+
+ def test_non_executable_engine_and_excessive_ram_budget_fail(self):
+ self.engine.chmod(0o644)
+ report = self.report(ram_gb=40)
+ checks = self.checks_by_id(report)
+
+ self.assertEqual(checks["engine.binary"]["status"], "fail")
+ self.assertEqual(checks["memory.ram"]["status"], "fail")
+ self.assertEqual(report["status"], "error")
+
+ def test_requested_missing_gpu_is_a_failure(self):
+ report = self.report(gpu_indices=[1])
+ check = self.checks_by_id(report)["accelerator.cuda"]
+
+ self.assertEqual(check["status"], "fail")
+ self.assertEqual(check["details"], {"requested": [1], "detected": []})
+ self.assertEqual(exit_code(report), 1)
+
+ def test_cpu_engine_with_detected_gpu_is_only_a_warning(self):
+ gpu = {"index": 0, "name": "fixture", "total_bytes": 12 * GB,
+ "free_bytes": 10 * GB}
+ report = self.report(gpu_indices=None, gpus=[gpu])
+ check = self.checks_by_id(report)["accelerator.cuda"]
+
+ self.assertEqual(check["status"], "warn")
+ self.assertEqual(report["status"], "warning")
+ self.assertEqual(exit_code(report), 0)
+
+ def test_missing_cuda_runtime_is_a_failure(self):
+ gpu = {"index": 0, "name": "fixture", "total_bytes": 12 * GB,
+ "free_bytes": 10 * GB}
+ report = self.report(gpu_indices=[0], gpus=[gpu],
+ linkage={"linked": False, "missing": True})
+
+ self.assertEqual(
+ self.checks_by_id(report)["accelerator.cuda"]["summary"],
+ "CUDA runtime library is missing",
+ )
+ self.assertEqual(report["status"], "error")
+
+ def test_text_format_contains_checks_plan_and_result(self):
+ output = format_doctor(self.report())
+
+ self.assertIn("model.path", output)
+ self.assertIn("disk backing store", output)
+ self.assertTrue(output.endswith("result ok"))
+
+ def test_cli_json_is_machine_readable_without_loading_model(self):
+ cli = Path(__file__).parents[1] / "coli"
+ run = subprocess.run([
+ sys.executable, str(cli), "doctor", "--model", str(self.model),
+ "--gpu", "none", "--ram", "16", "--ctx", "32", "--json",
+ ], text=True, capture_output=True, check=False)
+
+ # The repository engine may be absent; doctor must still return one complete JSON report.
+ self.assertIn(run.returncode, (0, 1))
+ report = json.loads(run.stdout)
+ self.assertEqual(report["schema_version"], 1)
+ self.assertEqual(Path(report["model"]), self.model.resolve())
+ self.assertIn(report["status"], ("ok", "warning", "error"))
+ self.assertNotIn("\033", run.stdout)
+ self.assertTrue(run.stdout.lstrip().startswith("{"))
+ self.assertTrue(run.stdout.rstrip().endswith("}"))
+
+
+if __name__ == "__main__":
+ unittest.main()