Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(planner): export K3_EXPERT_GB and GLM53_EXPERT_GB from the plan
kevin9327b1b9c60e17cb
diff --git a/c/resource_plan.py b/c/resource_plan.pyindex fbd8604d..955f29a7 100644--- a/c/resource_plan.py+++ b/c/resource_plan.py@@ -963,6 +963,26 @@ POLICIES = {}+def _family_expert_cache_knob(family_id, cache_bytes):+ """Env var that actually sizes the expert LRU, when it is not RAM_GB.++ Kimi K3 reads K3_EXPERT_GB (default 8) and treats RAM_GB as a ceiling;+ main() never takes argv as a cap. GLM-5.3 reads GLM53_EXPERT_GB and never+ RAM_GB. Exporting only RAM_GB therefore left both engines on their own+ default cache under --auto-tier.+ """+ if (not isinstance(cache_bytes, int) or isinstance(cache_bytes, bool)+ or cache_bytes <= 0):+ return None+ if family_id == "kimi":+ return ("K3_EXPERT_GB",+ "Kimi K3 sizes the expert LRU from K3_EXPERT_GB; RAM_GB is only a ceiling")+ if family_id == "glm53":+ return ("GLM53_EXPERT_GB",+ "GLM-5.3 sizes the expert LRU from GLM53_EXPERT_GB, not RAM_GB")+ return None++def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,available_memory=None, available_disk=None, gpus=None,policy="quality", physical_cpus=None, cpu_sockets=None,@@ -1129,6 +1149,13 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,tune = _auto_tune(bottleneck_class, projected_hit, planning_gpus, cpu_sockets,plan_has_metal=False,engine_group=resolved.descriptor.engine_group)+ # DRAFT/PIPE/PIN/NUMA stay colibri.c-only (#1585). These two are the+ # opposite leftover: the engines that size the expert LRU from their own+ # *EXPERT_GB variable, which RAM_GB does not set.+ knob = _family_expert_cache_knob(resolved.descriptor.id, cache_bytes)+ if knob:+ name, reason = knob+ tune[name] = {"value": f"{cache_bytes / GB:.3f}", "reason": reason}probe_state, probe_gbs = ssd_probe_state(info["path"])actions = _next_actions(bottleneck_class, projected_hit, probe_state,probe_gbs, planning_gpus)@@ -1213,6 +1240,10 @@ def environment_for_plan(plan, env=None, cuda_enabled=True):result.setdefault("REPIN", "64")ram = plan["tiers"]["ram"]result.setdefault("RAM_GB", f"{ram['budget_bytes'] / GB:.3f}")+ knob = _family_expert_cache_knob(plan.get("model", {}).get("family_id"),+ ram.get("expert_cache_bytes"))+ if knob:+ result.setdefault(knob[0], f"{ram['expert_cache_bytes'] / GB:.3f}")planned_cap = ram.get("cache_slots_per_layer")if (plan.get("model", {}).get("family_id") == "qwen38" and(not isinstance(planned_cap, int) ordiff --git a/c/tests/test_resource_plan.py b/c/tests/test_resource_plan.pyindex 03a7e837..539fb484 100644--- a/c/tests/test_resource_plan.py+++ b/c/tests/test_resource_plan.py@@ -220,6 +220,105 @@ class ResourcePlanTest(unittest.TestCase):for key in ("DRAFT", "PIPE", "COLI_CUDA_PIPE", "COLI_NUMA", "PIN_GB"):self.assertNotIn(key, env)+ def _kimi_model(self):+ other = tempfile.TemporaryDirectory()+ self.addCleanup(other.cleanup)+ model = Path(other.name)+ (model / "config.json").write_text(json.dumps({+ "model_type": "kimi_k3",+ "hidden_size": 64,+ "num_hidden_layers": 2,+ "num_attention_heads": 4,+ "q_lora_rank": 8,+ "kv_lora_rank": 8,+ "qk_nope_head_dim": 8,+ "qk_rope_head_dim": 4,+ "v_head_dim": 8,+ "num_experts": 2,+ "linear_attn_config": {+ "num_heads": 2,+ "head_dim": 8,+ "kda_layers": [1],+ },+ }))+ write_shard(model / "model.safetensors", [+ ("model.embed_tokens.weight", 100),+ ("model.layers.0.block_sparse_moe.experts.0.w1.weight", 80),+ ("model.layers.0.block_sparse_moe.experts.1.w1.weight", 80),+ ("model.layers.1.block_sparse_moe.experts.0.w1.weight", 80),+ ("model.layers.1.block_sparse_moe.experts.1.w1.weight", 80),+ ])+ return model++ def _glm53_model(self):+ other = tempfile.TemporaryDirectory()+ self.addCleanup(other.cleanup)+ model = Path(other.name)+ (model / "config.json").write_text(json.dumps({+ "model_type": "glm5_next",+ "text_config": {+ "num_hidden_layers": 2,+ "n_routed_experts": 2,+ "hidden_size": 32,+ "num_attention_heads": 4,+ "q_lora_rank": 8,+ "kv_lora_rank": 8,+ "qk_nope_head_dim": 8,+ "v_head_dim": 8,+ "index_head_dim": 8,+ "hc_mult": 2,+ "layer_types": ["linear", "full"],+ "linear_attn_config": {+ "num_heads": 2,+ "head_dim": 8,+ "short_conv_kernel_size": 2,+ },+ },+ }))+ write_shard(model / "model.safetensors", [+ ("model.language_model.embed_tokens.weight", 100),+ ("model.language_model.layers.1.mlp.experts.0.gate_proj.weight", 80),+ ("model.language_model.layers.1.mlp.experts.1.gate_proj.weight", 80),+ ])+ return model++ def test_kimi_plan_exports_the_expert_cache_knob_the_engine_reads(self):+ """Kimi K3 sizes the expert LRU from K3_EXPERT_GB (default 8 GB).+ RAM_GB is only a ceiling, and main() never takes argv as a cap, so+ --auto-tier that set RAM_GB and COLI_PLAN_CAP left the 8 GB default+ in place on a machine whose plan had hundreds of GB of warm experts."""+ plan = build_plan(self._kimi_model(), context=32, available_memory=32 * GB,+ available_disk=1, gpus=[], cpu_sockets=1)+ cache = plan["tiers"]["ram"]["expert_cache_bytes"]+ self.assertGreater(cache, 0)+ expected = f"{cache / GB:.3f}"+ self.assertIn("K3_EXPERT_GB", plan["tune"])+ self.assertEqual(plan["tune"]["K3_EXPERT_GB"]["value"], expected)+ self.assertNotIn("GLM53_EXPERT_GB", plan["tune"])+ self.assertIn("K3_EXPERT_GB=", format_plan(plan))+ env = environment_for_plan(plan, {})+ self.assertIn("K3_EXPERT_GB", env)+ self.assertEqual(env["K3_EXPERT_GB"], expected)+ kept = environment_for_plan(plan, {"K3_EXPERT_GB": "3.5"})+ self.assertEqual(kept["K3_EXPERT_GB"], "3.5")++ def test_glm53_plan_exports_the_expert_cache_knob_the_engine_reads(self):+ """glm53.c reads GLM53_EXPERT_GB, not RAM_GB. --auto-tier exported+ RAM_GB (inert) and relied on COLI_PLAN_CAP as argv, which a direct+ engine launch never sees."""+ plan = build_plan(self._glm53_model(), context=32, available_memory=32 * GB,+ available_disk=1, gpus=[], cpu_sockets=1)+ cache = plan["tiers"]["ram"]["expert_cache_bytes"]+ self.assertGreater(cache, 0)+ expected = f"{cache / GB:.3f}"+ self.assertIn("GLM53_EXPERT_GB", plan["tune"])+ self.assertEqual(plan["tune"]["GLM53_EXPERT_GB"]["value"], expected)+ self.assertNotIn("K3_EXPERT_GB", plan["tune"])+ env = environment_for_plan(plan, {})+ self.assertIn("GLM53_EXPERT_GB", env)+ self.assertEqual(env["GLM53_EXPERT_GB"], expected)+ self.assertNotIn("K3_EXPERT_GB", env)+def test_cpu_socket_count_is_positive(self):self.assertGreaterEqual(cpu_socket_count(), 1)