Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
coli plan: read-only Disk/RAM/VRAM placement planner (mirrors engine budget math, versioned --json, --auto-tier) (#27)
ZacharyZcR3cbc52ab002d
diff --git a/README.md b/README.mdindex 5d2237c1..b83aed07 100644--- a/README.md+++ b/README.md@@ -91,6 +91,25 @@ cd cCOLI_MODEL=/nvme/glm52_i4 ./coli chat```+Inspect the planned storage hierarchy before loading the model:++```bash+COLI_MODEL=/nvme/glm52_i4 ./coli plan+COLI_MODEL=/nvme/glm52_i4 ./coli plan --gpu 0,1 --ram 128 --vram 48 --json++# apply the bounded plan to the normal runner+COLI_MODEL=/nvme/glm52_i4 ./coli chat --auto-tier+```++`coli plan` reads only safetensors headers and reports the model's exact dense/expert+footprint, runtime RAM reserve, safe expert-cache cap, and bounded VRAM hot tier. Its+versioned JSON output is intended to be shared by the CLI, API server, Web UI, and+desktop shell; it does not allocate model tensors or start inference.+`--auto-tier` applies the same plan to `chat`, `run`, `serve`, and benchmarks. It+sets the RAM budget and context immediately; the VRAM tier is enabled only when+the current `glm` binary is linked with CUDA. Explicit flags and environment+variables keep precedence over automatic values.+The engine at runtime is pure C — python is only used by the one-time converter.### OpenAI-compatible APIdiff --git a/c/coli b/c/coliindex b2f2a031..56f25e6a 100755--- a/c/coli+++ b/c/coli@@ -7,6 +7,7 @@ CLI per far girare GLM-5.2 (744B) in locale, su CPU, in ~15-26 GB di RAM.coli serve API HTTP compatibile OpenAI (motore persistente)coli run "prompt" generazione singolacoli info stato: modello, RAM, disco, config+ coli plan piano risorse Disk / RAM / VRAMcoli bench [task...] benchmark di qualità (MMLU/HellaSwag/...)coli convert converte GLM-5.2-FP8 -> int4 (streaming)coli build compila il motore@@ -98,6 +99,25 @@ def need_model(model):if not os.path.exists(GLM):sys.exit(f"{C.yel}motore non compilato.{C.r} Esegui: coli build")+def cuda_binary():+ if not os.path.exists(GLM) or sys.platform != "linux": return False+ try:+ linked=subprocess.run(["ldd",GLM],capture_output=True,text=True,timeout=3)+ return any("libcudart" in line and "not found" not in line+ for line in linked.stdout.splitlines())+ except (OSError,subprocess.SubprocessError): return False++def resource_request(a, env):+ ctx=a.ctx or int(env.get("CTX",4096))+ ram=a.ram or float(env.get("RAM_GB",0))+ vram=a.vram or float(env.get("CUDA_EXPERT_GB",0))+ gpu=a.gpu+ if gpu is None:+ gpu=env.get("COLI_GPUS",env.get("COLI_GPU","auto"))+ devices=None if gpu=="auto" else ([] if gpu=="none" else+ [int(value) for value in gpu.split(",")])+ return ram,ctx,devices,vram+def env_for(a):e = dict(os.environ, SNAP=a.model)if a.ram: e["RAM_GB"]=str(a.ram)@@ -106,6 +126,28 @@ def env_for(a):if a.topk: e["TOPK"]=str(a.topk)if a.temp is not None: e["TEMP"]=str(a.temp) # 0 = greedy; default motore: 1.0 + nucleus 0.95if a.repin: e["REPIN"]=str(a.repin)+ if a.ctx: e["CTX"]=str(a.ctx)+ if a.auto_tier:+ from resource_plan import build_plan, environment_for_plan, format_bytes+ if a.gpu is not None:+ e.pop("COLI_GPU",None); e.pop("COLI_GPUS",None)+ if a.gpu=="none":+ e["COLI_CUDA"]="0"; e.pop("CUDA_EXPERT_GB",None); e.pop("CUDA_DENSE",None)+ else: e.pop("COLI_CUDA",None)+ elif e.get("COLI_CUDA")=="0":+ e.pop("COLI_GPU",None); e.pop("COLI_GPUS",None)+ e.pop("CUDA_EXPERT_GB",None); e.pop("CUDA_DENSE",None)+ if a.vram and a.gpu!="none": e["CUDA_EXPERT_GB"]=str(a.vram)+ try:+ ram,ctx,devices,vram=resource_request(a,e)+ plan=build_plan(a.model,ram,ctx,devices,vram)+ except (OSError,ValueError,json.JSONDecodeError) as error:+ sys.exit(f"{C.yel}piano risorse non valido:{C.r} {error}")+ has_cuda=cuda_binary()+ e=environment_for_plan(plan,e,has_cuda)+ rt=plan["tiers"]["ram"]; vt=plan["tiers"]["vram"]+ gpu=f" · VRAM {format_bytes(vt['budget_bytes'])}" if has_cuda and vt["devices"] else " · CPU"+ print(f" {C.dim}[PLAN] RAM {format_bytes(rt['budget_bytes'])} · cap {rt['cache_slots_per_layer']}/layer{gpu}{C.r}",file=sys.stderr)return e# ---------- rendering markdown in STREAMING per il terminale ----------@@ -267,6 +309,22 @@ def cmd_info(a):if knobs: row("tuning", " · ".join(knobs))print()+def cmd_plan(a):+ from resource_plan import build_plan, format_plan+ try:+ ram,ctx,devices,vram=resource_request(a,os.environ)+ if ctx<1: raise ValueError("--ctx deve essere positivo")+ if a.vram<0: raise ValueError("--vram non puo essere negativo")+ plan=build_plan(a.model,ram,ctx,devices,vram)+ except (OSError, ValueError, json.JSONDecodeError) as error:+ sys.exit(f"{C.yel}impossibile creare il piano:{C.r} {error}")+ if a.json:+ print(json.dumps(plan,indent=2))+ return+ banner("plan · Disk / RAM / VRAM")+ print(textwrap.indent(format_plan(plan)," "))+ print()+def cmd_run(a):need_model(a.model)prompt=" ".join(a.prompt) if a.prompt else sys.exit('uso: coli run "il tuo prompt"')@@ -400,9 +458,7 @@ def cmd_bench(a):cmd=[py, os.path.join(TOOLS,"eval_glm.py"), "--snap",a.model,"--tasks", tasks, "--limit", str(a.limit), "--data", a.data]if a.ram: cmd+=["--ram",str(a.ram)]- e=dict(os.environ)- if a.topp: e["TOPP"]=str(a.topp)- if a.topk: e["TOPK"]=str(a.topk)+ e=env_for(a)print(f" {C.dim}decode disk-bound: su hardware lento questo richiede ORE. Alza --limit su macchine veloci.{C.r}\n")sys.exit(subprocess.call(cmd, env=e))@@ -428,6 +484,10 @@ def cmd_convert(a):def main():common=argparse.ArgumentParser(add_help=False)common.add_argument("--model", default=DEF_MODEL); common.add_argument("--ram", type=int, default=0) # 0 = auto (il motore usa l'88% della RAM disponibile)+ common.add_argument("--auto-tier",action="store_true",help="applica automaticamente il piano RAM/VRAM")+ common.add_argument("--ctx",type=int,default=0)+ common.add_argument("--gpu",default=None,help="auto, none oppure lista device, es. 0,1")+ common.add_argument("--vram",type=float,default=0,help="budget VRAM totale in GB (0=auto)")common.add_argument("--repin", type=int, default=0, help="adatta gli expert RAM/VRAM ogni N token")common.add_argument("--cap", type=int, default=8); common.add_argument("--ngen", type=int, default=1024) # rete di sicurezza: la fine vera la decidono gli stop tokencommon.add_argument("--topp", type=float, default=0); common.add_argument("--topk", type=int, default=0)@@ -435,6 +495,8 @@ def main():ap=argparse.ArgumentParser(prog="coli", parents=[common], description="colibrì — GLM-5.2 in locale")sub=ap.add_subparsers(dest="cmd")sub.add_parser("build", parents=[common]); sub.add_parser("info", parents=[common])+ pp=sub.add_parser("plan",parents=[common])+ pp.add_argument("--json",action="store_true")pr=sub.add_parser("run", parents=[common]); pr.add_argument("prompt", nargs="*")sub.add_parser("chat", parents=[common])ps=sub.add_parser("serve", parents=[common])@@ -448,7 +510,7 @@ def main():pc.add_argument("--ebits",type=int,default=4); pc.add_argument("--io-bits",type=int,default=8); pc.add_argument("--xbits",type=int,default=0)pc.add_argument("--no-mtp",action="store_true",help="salta la testa MTP (niente draft speculativi)")a=ap.parse_args()- {"build":cmd_build,"info":cmd_info,"run":cmd_run,"chat":cmd_chat,"serve":cmd_serve,"bench":cmd_bench,+ {"build":cmd_build,"info":cmd_info,"plan":cmd_plan,"run":cmd_run,"chat":cmd_chat,"serve":cmd_serve,"bench":cmd_bench,"convert":cmd_convert}.get(a.cmd, lambda _:(banner(),print(__doc__)))(a)if __name__=="__main__":diff --git a/c/resource_plan.py b/c/resource_plan.pynew file mode 100644index 00000000..7949379b--- /dev/null+++ b/c/resource_plan.py@@ -0,0 +1,216 @@+#!/usr/bin/env python3+"""Hardware and model placement planning for colibri's disk/RAM/VRAM tiers."""++import json+import os+import re+import statistics+import subprocess+from pathlib import Path+++GB = 1_000_000_000+EXPERT_RE = re.compile(r"model\.layers\.(\d+)\.mlp\.experts\.(\d+)\.")+++def _tensor_sizes(path):+ file_size = path.stat().st_size+ with path.open("rb") as stream:+ raw = stream.read(8)+ if len(raw) != 8:+ raise ValueError(f"short safetensors header: {path}")+ length = int.from_bytes(raw, "little")+ if length < 2 or length > file_size - 8:+ raise ValueError(f"invalid safetensors header length: {path}")+ header = json.loads(stream.read(length))+ for name, meta in header.items():+ if name == "__metadata__":+ continue+ start, end = meta["data_offsets"]+ if not 0 <= start <= end <= file_size - 8 - length:+ raise ValueError(f"invalid tensor offsets for {name}: {path}")+ yield name, end - start+++def analyze_model(model):+ model = Path(model).resolve()+ config_path = model / "config.json"+ if not config_path.is_file():+ raise ValueError(f"missing config.json: {model}")+ config = json.loads(config_path.read_text())+ shards = sorted(model.glob("*.safetensors"))+ if not shards:+ raise ValueError(f"no safetensors shards: {model}")++ dense_bytes = 0+ expert_groups = {}+ for shard in shards:+ for name, size in _tensor_sizes(shard):+ match = EXPERT_RE.search(name)+ if match:+ key = tuple(map(int, match.groups()))+ expert_groups[key] = expert_groups.get(key, 0) + size+ else:+ dense_bytes += size++ layer_sizes = {}+ for (layer, _), size in expert_groups.items():+ layer_sizes.setdefault(layer, []).append(size)+ per_layer = {layer: int(statistics.median(sizes)) for layer, sizes in layer_sizes.items()}+ per_cap_bytes = sum(per_layer.values())+ typical_expert_bytes = int(statistics.median(per_layer.values())) if per_layer else 0+ model_bytes = sum(shard.stat().st_size for shard in shards)+ return {+ "path": str(model),+ "shards": len(shards),+ "model_bytes": model_bytes,+ "dense_bytes": dense_bytes,+ "expert_bytes": sum(expert_groups.values()),+ "expert_count": len(expert_groups),+ "expert_layers": len(per_layer),+ "typical_expert_bytes": typical_expert_bytes,+ "per_cap_bytes": per_cap_bytes,+ "config": config,+ }+++def memory_available():+ try:+ text = Path("/proc/meminfo").read_text()+ return int(re.search(r"MemAvailable:\s+(\d+)", text).group(1)) * 1024+ except (OSError, AttributeError):+ return 0+++def discover_gpus():+ command = ["nvidia-smi", "--query-gpu=index,name,memory.total,memory.free",+ "--format=csv,noheader,nounits"]+ try:+ result = subprocess.run(command, text=True, capture_output=True, check=True, timeout=5)+ except (OSError, subprocess.SubprocessError):+ return []+ devices = []+ for line in result.stdout.splitlines():+ fields = [field.strip() for field in line.split(",", 3)]+ if len(fields) != 4:+ continue+ try:+ index, total, free = int(fields[0]), int(fields[2]), int(fields[3])+ except ValueError:+ continue+ devices.append({"index": index, "name": fields[1],+ "total_bytes": total * 1024 * 1024,+ "free_bytes": free * 1024 * 1024})+ return devices+++def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,+ available_memory=None, available_disk=None, gpus=None):+ info = analyze_model(model)+ cfg = info["config"]+ available_memory = memory_available() if available_memory is None else available_memory+ if available_disk is None:+ fs = os.statvfs(info["path"])+ available_disk = fs.f_bavail * fs.f_frsize+ gpus = discover_gpus() if gpus is None else gpus+ if gpu_indices is not None:+ wanted = set(gpu_indices)+ gpus = [gpu for gpu in gpus if gpu["index"] in wanted]++ ram_budget = int(ram_gb * GB) if ram_gb > 0 else int(available_memory * 0.88)+ if ram_budget < 4 * GB:+ ram_budget = 8 * GB+ typical = info["typical_expert_bytes"]+ layers = int(cfg.get("num_hidden_layers", 0)) + 1+ kv_bytes = layers * context * (int(cfg.get("kv_lora_rank", 0)) ++ int(cfg.get("qk_rope_head_dim", 0))) * 4+ kv_buffer = context * int(cfg.get("num_attention_heads", 0)) * (+ int(cfg.get("qk_nope_head_dim", 0)) + int(cfg.get("v_head_dim", 0))) * 4+ runtime_bytes = int(1.2 * GB + 2.5 * GB + 64 * typical + kv_bytes + kv_buffer)+ cache_bytes = max(0, ram_budget - info["dense_bytes"] - runtime_bytes)+ per_cap = info["per_cap_bytes"]+ configured_experts = int(cfg.get("n_routed_experts", 0))+ cap = int(cache_bytes // per_cap) if per_cap else 0+ if configured_experts:+ cap = min(cap, configured_experts)++ reserve = 2 * GB+ gpu_plan = []+ safe_vram = 0+ for gpu in gpus:+ usable = max(0, gpu["free_bytes"] - reserve)+ safe_vram += usable+ gpu_plan.append(dict(gpu, reserve_bytes=reserve, usable_bytes=usable))+ requested_vram = int(vram_gb * GB) if vram_gb > 0 else safe_vram+ vram_budget = min(requested_vram, safe_vram, cache_bytes)+ vram_experts = int(vram_budget // typical) if typical else 0++ warnings = []+ if cap < 1:+ warnings.append("RAM budget cannot hold one expert slot per sparse layer")+ if gpu_indices is not None and len(gpus) != len(set(gpu_indices)):+ warnings.append("one or more requested GPUs were not detected")+ if gpus and vram_budget < requested_vram:+ warnings.append("VRAM tier was clamped by free VRAM or its required RAM backing")++ return {+ "version": 1,+ "model": {key: value for key, value in info.items() if key != "config"},+ "tiers": {+ "disk": {"role": "backing", "model_bytes": info["model_bytes"],+ "available_bytes": available_disk},+ "ram": {"role": "resident+cache", "available_bytes": available_memory,+ "budget_bytes": ram_budget, "dense_bytes": info["dense_bytes"],+ "runtime_bytes": runtime_bytes, "expert_cache_bytes": cache_bytes,+ "cache_slots_per_layer": cap},+ "vram": {"role": "hot-experts", "devices": gpu_plan,+ "budget_bytes": vram_budget, "expert_capacity": vram_experts},+ },+ "warnings": warnings,+ }+++def environment_for_plan(plan, env=None, cuda_enabled=True):+ """Apply a plan without overriding explicit user environment settings."""+ result = dict(env or {})+ ram = plan["tiers"]["ram"]+ result.setdefault("RAM_GB", f"{ram['budget_bytes'] / GB:.3f}")++ vram = plan["tiers"]["vram"]+ devices = [device["index"] for device in vram["devices"]]+ if not cuda_enabled or not devices or vram["budget_bytes"] <= 0:+ return result+ if result.get("COLI_CUDA", "1") == "0":+ return result++ result.setdefault("COLI_CUDA", "1")+ if "COLI_GPU" not in result and "COLI_GPUS" not in result:+ key = "COLI_GPU" if len(devices) == 1 else "COLI_GPUS"+ result[key] = ",".join(map(str, devices))+ result.setdefault("CUDA_EXPERT_GB", f"{vram['budget_bytes'] / GB:.3f}")+ if result.get("PIN"):+ result.setdefault("PIN_GB", f"{vram['budget_bytes'] / GB:.3f}")+ return result+++def format_bytes(value):+ return f"{value / GB:.1f} GB"+++def format_plan(plan):+ model, tiers = plan["model"], plan["tiers"]+ lines = [f"model {model['shards']} shards · {format_bytes(model['model_bytes'])}",+ f"disk backing store · {format_bytes(tiers['disk']['available_bytes'])} free",+ f"RAM {format_bytes(tiers['ram']['budget_bytes'])} budget · "+ f"{format_bytes(tiers['ram']['dense_bytes'])} dense · "+ f"{format_bytes(tiers['ram']['runtime_bytes'])} runtime · "+ f"cap {tiers['ram']['cache_slots_per_layer']}/layer"]+ vram = tiers["vram"]+ if vram["devices"]:+ names = ", ".join(f"{gpu['index']}:{gpu['name']}" for gpu in vram["devices"])+ lines.append(f"VRAM {format_bytes(vram['budget_bytes'])} hot tier · "+ f"~{vram['expert_capacity']} experts · {names}")+ else:+ lines.append("VRAM no NVIDIA device detected · CPU path")+ lines.extend(f"warn {warning}" for warning in plan["warnings"])+ return "\n".join(lines)diff --git a/c/tests/test_resource_plan.py b/c/tests/test_resource_plan.pynew file mode 100644index 00000000..3b965fe5--- /dev/null+++ b/c/tests/test_resource_plan.py@@ -0,0 +1,111 @@+import json+import struct+import subprocess+import sys+import tempfile+import unittest+from pathlib import Path++from resource_plan import GB, analyze_model, build_plan, environment_for_plan, format_plan+++def write_shard(path, tensors):+ offset = 0+ header = {}+ payload = b""+ for name, size in tensors:+ header[name] = {"dtype": "U8", "shape": [size], "data_offsets": [offset, offset + size]}+ payload += b"\0" * size+ offset += size+ raw = json.dumps(header).encode()+ path.write_bytes(struct.pack("<Q", len(raw)) + raw + payload)+++class ResourcePlanTest(unittest.TestCase):+ def setUp(self):+ self.tmp = tempfile.TemporaryDirectory()+ self.model = Path(self.tmp.name)+ (self.model / "config.json").write_text(json.dumps({+ "num_hidden_layers": 2,+ "n_routed_experts": 2,+ "kv_lora_rank": 4,+ "qk_rope_head_dim": 2,+ "qk_nope_head_dim": 3,+ "v_head_dim": 5,+ "num_attention_heads": 2,+ }))+ write_shard(self.model / "model.safetensors", [+ ("model.embed_tokens.weight", 100),+ ("model.layers.0.self_attn.q_a_proj.weight", 200),+ ("model.layers.1.mlp.experts.0.gate_proj.weight", 30),+ ("model.layers.1.mlp.experts.0.up_proj.weight", 30),+ ("model.layers.1.mlp.experts.1.gate_proj.weight", 30),+ ("model.layers.1.mlp.experts.1.up_proj.weight", 30),+ ])++ def tearDown(self):+ self.tmp.cleanup()++ def test_analyzes_dense_and_expert_storage(self):+ info = analyze_model(self.model)+ self.assertEqual(info["dense_bytes"], 300)+ self.assertEqual(info["expert_bytes"], 120)+ self.assertEqual(info["expert_count"], 2)+ self.assertEqual(info["per_cap_bytes"], 60)++ def test_builds_bounded_three_tier_plan(self):+ gpus = [{"index": 0, "name": "test-gpu", "total_bytes": 12 * GB,+ "free_bytes": 10 * GB}]+ plan = build_plan(self.model, ram_gb=16, context=32, vram_gb=20,+ available_memory=32 * GB, available_disk=100 * GB, gpus=gpus)+ self.assertEqual(plan["version"], 1)+ self.assertEqual(plan["tiers"]["ram"]["budget_bytes"], 16 * GB)+ self.assertLessEqual(plan["tiers"]["vram"]["budget_bytes"], 8 * GB)+ self.assertIn("required RAM backing", plan["warnings"][0])+ self.assertIn("0:test-gpu", format_plan(plan))++ def test_filters_requested_devices(self):+ gpus = [{"index": 0, "name": "a", "total_bytes": 8 * GB, "free_bytes": 8 * GB}]+ plan = build_plan(self.model, available_memory=16 * GB, available_disk=1,+ gpus=gpus, gpu_indices=[1])+ self.assertEqual(plan["tiers"]["vram"]["devices"], [])+ self.assertIn("not detected", plan["warnings"][0])++ def test_cli_emits_versioned_json(self):+ cli = Path(__file__).parents[1] / "coli"+ run = subprocess.run([+ sys.executable, str(cli), "plan", "--model", str(self.model),+ "--gpu", "none", "--json",+ ], text=True, capture_output=True, check=True)+ plan = json.loads(run.stdout)+ self.assertEqual(plan["version"], 1)+ self.assertEqual(plan["model"]["expert_count"], 2)++ def test_applies_plan_without_overriding_explicit_settings(self):+ gpus = [+ {"index": 0, "name": "a", "total_bytes": 12 * GB, "free_bytes": 10 * GB},+ {"index": 1, "name": "b", "total_bytes": 12 * GB, "free_bytes": 10 * GB},+ ]+ plan = build_plan(self.model, ram_gb=16, available_memory=32 * GB,+ available_disk=1, gpus=gpus)+ env = environment_for_plan(plan, {"RAM_GB": "12", "PIN": "stats.txt",+ "COLI_GPUS": "1"})+ self.assertEqual(env["RAM_GB"], "12")+ self.assertEqual(env["COLI_CUDA"], "1")+ self.assertEqual(env["COLI_GPUS"], "1")+ self.assertEqual(env["PIN_GB"], env["CUDA_EXPERT_GB"])++ def test_cpu_binary_does_not_apply_gpu_tier(self):+ plan = build_plan(self.model, available_memory=16 * GB, available_disk=1,+ gpus=[{"index": 0, "name": "a", "total_bytes": 8 * GB,+ "free_bytes": 8 * GB}])+ env = environment_for_plan(plan, cuda_enabled=False)+ self.assertIn("RAM_GB", env)+ self.assertNotIn("COLI_CUDA", env)+ disabled = environment_for_plan(plan, {"COLI_CUDA": "0"}, cuda_enabled=True)+ self.assertNotIn("COLI_GPU", disabled)+ self.assertNotIn("CUDA_EXPERT_GB", disabled)+++if __name__ == "__main__":+ unittest.main()