Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

Merge dev into fix/macos-metal-gpu-discovery: keep the int8 trunk line in the VRAM branch

JustVugg232b2823c487
diff --cc c/resource_plan.py
index 18087a58,ff81097a..fbd8604d
--- a/c/resource_plan.py
+++ b/c/resource_plan.py
@@@ -1241,14 -1248,20 +1270,22 @@@ def format_plan(plan)
f"cap {tiers['ram']['cache_slots_per_layer']}/layer"]
vram = tiers["vram"]
if vram["devices"]:
- names = ", ".join(
- f"{gpu['index']}:{gpu['name']}"
- + ("" if plans_placement(gpu) else " (identity only)")
- for gpu in vram["devices"])
- trunk = vram.get("trunk_bytes", 0)
- lines.append("VRAM " + (f"{format_bytes(trunk)} int8 trunk + " if trunk else "") +
- f"{format_bytes(vram['budget_bytes'])} hot tier · "
- f"~{vram['expert_capacity']} experts · {names}")
+ metal_identity = [gpu for gpu in vram["devices"]
+ if gpu.get("backend") == "metal"
+ and gpu.get("free_bytes") is None]
+ if len(metal_identity) == len(vram["devices"]):
+ names = ", ".join(f"{gpu['index']}:{gpu['name']}"
+ for gpu in metal_identity)
+ lines.append(f"Metal {names} · unified memory · no independent VRAM budget")
+ else:
+ names = ", ".join(
+ f"{gpu['index']}:{gpu['name']}"
+ + ("" if plans_placement(gpu) else " (identity only)")
+ for gpu in vram["devices"])
- lines.append(f"VRAM {format_bytes(vram['budget_bytes'])} hot tier · "
++ trunk = vram.get("trunk_bytes", 0)
++ lines.append("VRAM " + (f"{format_bytes(trunk)} int8 trunk + " if trunk else "") +
++ f"{format_bytes(vram['budget_bytes'])} hot tier · "
+ f"~{vram['expert_capacity']} experts · {names}")
else:
# Backend-neutral, matching the accelerator wording #903 settled on:
# an AMD or Intel host that finds nothing is not "no NVIDIA device".