Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
resource_plan: price Qwen3.8's expert cache with the int4-g64 sidecar
JustVugg22bb80bd4f16
diff --git a/c/resource_plan.py b/c/resource_plan.pyindex 95c3fc0d..60ce35b6 100644--- a/c/resource_plan.py+++ b/c/resource_plan.py@@ -295,6 +295,48 @@ def analyze_model(model):return result+QWEN38_INT4_SIDECAR = "experts-int4g64"+++def qwen38_int4_sidecar(info):+ """Price Qwen3.8's routed experts as qwen38 will hold them.++ tools/convert_qwen38_experts_int4.py writes <model>/experts-int4g64/, and+ the engine loads every routed expert from it instead of the FP8 shards+ when its index says it is complete (Q38_EXPERT_INT4=0 keeps FP8). One of+ its records is 56% of an FP8 expert, so pricing the cache with FP8 bytes+ would hand the engine a cap for half the RAM it was given. Only the index+ is read here; the engine validates the files themselves and refuses a+ sidecar that disagrees with it. Returns the analysis unchanged when the+ sidecar does not apply, otherwise a copy priced with its records and+ marked `qwen38_int4_experts`: no FP8 scale bank is built either.+ """+ resolved = info["resolved_family"]+ if resolved.descriptor.id != "qwen38" or os.environ.get("Q38_EXPERT_INT4") == "0":+ return info+ try:+ index = json.loads((Path(info["path"]) / QWEN38_INT4_SIDECAR / "index.json")+ .read_text(encoding="utf-8"))+ except (OSError, ValueError):+ return info+ config = info["config"]+ text = config.get("text_config", config) if isinstance(config, dict) else {}+ want = {"format": "colibri.qwen38.experts-int4g64", "version": 1, "complete": True,+ "layers": text.get("num_hidden_layers"), "experts": text.get("num_experts"),+ "hidden_size": text.get("hidden_size"),+ "moe_intermediate_size": text.get("moe_intermediate_size")}+ record = index.get("record_bytes") if isinstance(index, dict) else None+ if (any(index.get(key) != value for key, value in want.items()) or+ isinstance(record, bool) or not isinstance(record, int) or record < 1):+ return info+ per_layer = {layer: record for layer in info["expert_bytes_by_layer"]}+ return dict(info,+ expert_bytes=record * want["experts"] * len(per_layer),+ typical_expert_bytes=record, max_expert_bytes=record,+ expert_bytes_by_layer=per_layer, per_cap_bytes=record * len(per_layer),+ expert_fixed_bytes=0, qwen38_int4_experts=True)++#: MEMORYSTATUSEX as Windows defines it, in order. Kept as data so a test can#: pin the order without a Windows machine.WINDOWS_MEMORYSTATUSEX_FIELDS = (@@ -1489,10 +1531,18 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,kv_slots=1):if policy not in POLICIES:raise ValueError(f"unknown policy: {policy}")- info = analyze_model(model)+ info = qwen38_int4_sidecar(analyze_model(model))physical_cpus = physical_cpu_count() if physical_cpus is None else physical_cpuscpu_sockets = cpu_socket_count() if cpu_sockets is None else cpu_socketsresolved = info["resolved_family"]+ if info.get("qwen38_int4_experts"):+ # qwen38's CUDA tier streams FP8 experts only and stays off when the+ # int4-g64 sidecar is in use (trunk included): plan the CPU.+ if gpu_indices or vram_gb > 0:+ raise ValueError(+ "Qwen3.8 int4-g64 experts (experts-int4g64/) run on the CPU; "+ "Q38_EXPERT_INT4=0 gives the GPU tier the FP8 experts")+ gpus = []if not resolved.descriptor.supports_accelerator:if gpu_indices:raise ValueError(diff --git a/c/tests/test_resource_plan.py b/c/tests/test_resource_plan.pyindex 7306131d..fc6d9052 100644--- a/c/tests/test_resource_plan.py+++ b/c/tests/test_resource_plan.py@@ -979,7 +979,7 @@ memInfo.free: 23.50 GB (97%)self.assertEqual(plan["tiers"]["vram"]["devices"], [])self.assertIn("not detected", plan["warnings"][0])- def test_qwen38_plan_prices_heterogeneous_cache_exports_cap_and_plans_vram(self):+ def write_qwen38(self):config = {"model_type": "qwen4_exp","text_config": {@@ -1021,6 +1021,10 @@ memInfo.free: 23.50 GB (97%)f"model.layers.0.mlp.experts.1.{projection}.weight", 128, "F32"))write_shard(self.model / "model.safetensors", tensors)++ def test_qwen38_plan_prices_heterogeneous_cache_exports_cap_and_plans_vram(self):+ self.write_qwen38()+ MiB = 1 << 20analysis = analyze_model(self.model)self.assertEqual(analysis["dense_bytes"], 256 + 4 * MiB + 1024 + 4 * MiB)# The stage-1 trunk offload: int8 bytes of the offered matrices only.@@ -1074,6 +1078,46 @@ memInfo.free: 23.50 GB (97%)available_disk=16 * GB, gpus=[gpu])self.assertEqual(tiny["tiers"]["vram"]["trunk_bytes"], 0)+ def test_qwen38_int4_sidecar_prices_the_cache_with_its_records(self):+ # tools/convert_qwen38_experts_int4.py's index for this geometry+ # (2 layers, 2 experts, hidden 8, moe_intermediate 4): codes 16+16+16+ # bytes, scales (4+4+8) floats -> 112 bytes per expert.+ self.write_qwen38()+ sidecar = self.model / "experts-int4g64"+ sidecar.mkdir()+ index = {"format": "colibri.qwen38.experts-int4g64", "version": 1, "complete": True,+ "layers": 2, "experts": 2, "hidden_size": 8, "moe_intermediate_size": 4,+ "record_bytes": 112}+ (sidecar / "index.json").write_text(json.dumps(index))+ gpu = {"index": 0, "name": "unrelated", "total_bytes": 16 * GB,+ "free_bytes": 14 * GB, "unified_memory": True}+ with mock.patch.dict(os.environ, {}, clear=False):+ os.environ.pop("Q38_EXPERT_INT4", None)+ plan = build_plan(self.model, context=64, available_memory=16 * GB,+ available_disk=16 * GB, gpus=[gpu])+ model = plan["model"]+ self.assertTrue(model["qwen38_int4_experts"])+ self.assertEqual((model["per_cap_bytes"], model["expert_bytes"],+ model["expert_fixed_bytes"]), (112, 224, 0))+ # the engine's tier declines int4 experts, so nothing is planned on the GPU+ self.assertEqual(plan["tiers"]["vram"]["devices"], [])+ self.assertNotIn("COLI_CUDA", environment_for_plan(plan))+ with self.assertRaisesRegex(ValueError, "run on the CPU"):+ build_plan(self.model, context=64, gpu_indices=[0], available_memory=16 * GB,+ available_disk=16 * GB, gpus=[gpu])+ # what the engine would not use is not priced: FP8 forced, or a+ # conversion still running+ os.environ["Q38_EXPERT_INT4"] = "0"+ forced = build_plan(self.model, context=64, available_memory=16 * GB,+ available_disk=16 * GB, gpus=[])+ self.assertEqual(forced["model"]["per_cap_bytes"], 384)+ os.environ.pop("Q38_EXPERT_INT4")+ (sidecar / "index.json").write_text(json.dumps(dict(index, complete=False)))+ partial = build_plan(self.model, context=64, available_memory=16 * GB,+ available_disk=16 * GB, gpus=[])+ self.assertNotIn("qwen38_int4_experts", partial["model"])+ self.assertEqual(partial["model"]["per_cap_bytes"], 384)+def test_cli_emits_versioned_json(self):cli = Path(__file__).parents[1] / "coli"run = subprocess.run([