Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat(planner): qwen38's partial-chain layout
JustVugg97a6f760db43
diff --git a/c/qwen38_chain.h b/c/qwen38_chain.hindex 00dfb2d7..e3eb1a74 100644--- a/c/qwen38_chain.h+++ b/c/qwen38_chain.h@@ -260,7 +260,7 @@ static size_t q38c_fit_w(const Q38Weight *w, int payload) {int fmt = q38_vk_fmt(w);return payload ? coli_vk_tensor_payload(fmt, w->cols, w->rows, 0) : vkc_fit_tensor(fmt, w->cols, w->rows, 0);}-static int q38c_kv_rows0(void) { return 3 * (int)vkc_kv_env("COLI_VK_KV_BLOCK", 64); }+static int q38c_kv_rows0(void) { long b = vkc_kv_env("COLI_VK_KV_BLOCK", 64); return 3 * (int)(b < 1 ? 1 : b); }/* layer i's device bytes (as q38c_setup places it, its state at its starting size) and its matrices' payload */static void q38c_fit_layer(Model *m, int i, int rows, size_t *bytes, size_t *payload) {Cfg *c = &m->c; int H = c->hidden, W = c->hc_width;diff --git a/c/resource_plan.py b/c/resource_plan.pyindex ff3e9991..82a5c305 100644--- a/c/resource_plan.py+++ b/c/resource_plan.py@@ -750,6 +750,135 @@ def _olmoe_chain_layout(info, env, vulkan):_VK_CHAIN_LAYOUT["olmoe"] = _olmoe_chain_layout+_Q38_CHAIN_MATRICES = frozenset(_Q38_TRUNK_COMPONENTS) | {+ "linear_attn.in_proj_a", "linear_attn.in_proj_b", "ple.key_proj", "ple.value_proj"}+++def _q38_env_int(env, name, default):+ """C's atol/atoi: the leading integer, 0 when there is none."""+ value = env.get(name)+ if value is None or not value.strip():+ return default+ match = re.match(r"\s*([+-]?\d+)", value)+ return int(match[1]) if match else 0+++def _q38_vk_fmt(tensor, tag, env):+ """q38_vk_fmt: the format a resident matrix goes up in. The trunk's int8 rows (fmt 1)+ when q38_trunk_cpu_int8 quantized it (a trunk component of at least Q38_TRUNK_MIN_KB+ with its float scales, not in Q38_TRUNK_SKIP); else the rows as loaded: bf16 (fmt 11)+ from a BF16 tensor with Q38_NATIVE_BF16, f32 (fmt 10) otherwise."""+ rows, columns = tensor["shape"]+ if tag and env.get("Q38_TRUNK_CPU_INT8") != "0":+ min_kb = _q38_env_int(env, "Q38_TRUNK_MIN_KB", 1024)+ skipped = set((env.get("Q38_TRUNK_SKIP") or "").split(","))+ if min_kb >= 0 and rows * columns + 4 * rows >= min_kb * 1024 and tag not in skipped:+ return 1+ return 11 if tensor["dtype"] == "BF16" and env.get("Q38_NATIVE_BF16") != "0" else 10+++def _q38_chain_layout(info, env, vulkan):+ """qwen38_chain.h: q38c_fit_layer, q38c_fit_fixed, q38c_fit_tail."""+ c = info.get("config") or {}+ c = c.get("text_config") or c+ try:+ L, H, vocab = int(c["num_hidden_layers"]), int(c["hidden_size"]), int(c["vocab_size"])+ kinds = [0 if t == "linear_attention" else 1 for t in c["layer_types"]][:L]+ QH, KVH, D = int(c["num_attention_heads"]), int(c["num_key_value_heads"]), int(c["head_dim"])+ IQ, ID = int(c["indexer_n_heads"]), int(c["indexer_head_dim"])+ budget, ratio = int(c["indexer_budget"]), int(c["indexer_compress_ratio"])+ VH, VD = int(c["linear_num_value_heads"]), int(c["linear_value_head_dim"])+ KH, KD, CK = int(c["linear_num_key_heads"]), int(c["linear_key_head_dim"]), int(c["linear_conv_kernel_dim"])+ E, SI = int(c["num_experts"]), int(c["shared_expert_intermediate_size"])+ except (KeyError, TypeError, ValueError):+ return None+ if L < 1 or len(kinds) != L:+ return None+ C, R = int(c.get("hc_count") or 4), int(c.get("hc_lowrank") or 320)+ W = C * H+ rope = c.get("rope_parameters") or {}+ partial = rope.get("partial_rotary_factor", c.get("partial_rotary_factor", 1.0))+ rotary = int(D * float(partial if partial is not None else 1.0))+ ngram, per_ngram = int(c.get("ngram_size") or 3), int(c.get("heads_per_ngram") or 8)+ ple_dim, ple_k = int(c.get("ple_embed_dim") or H), int(c.get("ple_conv_kernel_size") or 4)+ heads = (ngram - 1) * per_ngram+ Ep = max(heads * (ple_dim // heads if heads > 0 else 0), 1)+ ids = c.get("ple_layer_ids") or []+ ple = int(ids[0]) - 1 if ids else -1+ CD = 2 * KH * KD + VH * VD+ rows = _q38_chain_rows(env)+ rows0 = 3 * max(_q38_env_int(env, "COLI_VK_KV_BLOCK", 64), 1)+ kvo = KVH * D+ # each layer's matrices as they go up, and the shared expert's gate (fmt 10, H x 1)+ layers = [vk_tensor_bytes(10, H, 1) for _ in range(L)]+ for tensor in info.get("dense_tensors", []):+ layer, part, shape = _vk_layer_index(tensor["name"]), _layer_component(tensor["name"]), tensor.get("shape")+ if layer is None or layer >= L or part not in _Q38_CHAIN_MATRICES or not isinstance(shape, list) or len(shape) != 2:+ continue+ if part.startswith("ple.") and layer != ple:+ continue+ fmt = _q38_vk_fmt(tensor, _Q38_TRUNK_COMPONENTS.get(part), env)+ layers[layer] += vk_tensor_bytes(fmt, shape[1], shape[0])+ for i in range(L):+ floats = 2 * W+ if kinds[i]:+ floats += 2 * D + 2 * ID+ layers[i] += (2 * vk_buf_bytes(kvo * rows0 * 4) + vk_buf_bytes(ID * rows0 * 4) ++ vk_buf_bytes(ID * (rows0 // max(ratio, 1) + 1) * 4) + rows * (2 * kvo + ID) * 4)+ else:+ floats += CD * CK + 2 * VH + VD+ layers[i] += 2 * vk_buf_bytes(VH * KD * VD * 4) + 2 * vk_buf_bytes(CD * (CK - 1) * 4)+ if i == ple:+ floats += 3 * W + W * ple_k+ layers[i] += vk_buf_bytes(2 * W * (ple_k - 1) * ngram * 4)+ layers[i] += 4 * floats+ # q38c_bufs' counting pass at `rows` rows, no context yet and no attention layer, the+ # streams' read-back and the final norm+ r, nb, rot = rows, 1, rotary if rotary > 0 else 2+ counts = [r * W, r * W, r * R, r * W, r * H, r * C, r * C, r * H, r * QH * 2 * D, r * kvo, r * kvo,+ r * (IQ + 1) * ID, r * QH * D, r * 2 * nb, r * (budget + ratio), r * CD, r * VH * VD, r * 2 * VH,+ r * CD, r * VH * VD, r * E, r * SI, r * SI, r * SI, r * H, r, r * H, r * W, r * H, r * W, r * W,+ r * H, r * E, r * (2 * kvo + ID), 2 * vocab, r * H, r * Ep, r * rot, nb * rot]+ fixed = sum(4 * (n if n else 1) for n in counts) + (rows + 1) * W * 4+ # the tail: the final mixer and lm_head; under Q38_MTP=1 the MTP head's matrices too+ # (the scan leaves its tensors out: its decoder layer has an attention layer's shapes,+ # its two projections are H x H and its mixer the final mixer's)+ tail, head_dtype = 0, "BF16"+ attn_parts = {}+ last_attn = max([i for i in range(L) if kinds[i]] + [-1])+ for tensor in info.get("dense_tensors", []):+ name, shape = _text_weight_name(tensor["name"]), tensor.get("shape")+ if not isinstance(shape, list) or len(shape) != 2:+ continue+ if name == "lm_head.weight":+ head_dtype = tensor["dtype"]+ tail += vk_tensor_bytes(_q38_vk_fmt(tensor, "lmhead", env), shape[1], shape[0])+ elif name in ("model.hyper_connection_mixer.input_mix_weight_down.weight",+ "model.hyper_connection_mixer.input_mix_weight_up.weight"):+ tail += vk_tensor_bytes(_q38_vk_fmt(tensor, None, env), shape[1], shape[0])+ elif _vk_layer_index(tensor["name"]) == last_attn and _layer_component(tensor["name"]) in _Q38_CHAIN_MATRICES:+ attn_parts[_layer_component(tensor["name"])] = tensor+ if env.get("Q38_MTP") == "1" and int(c.get("mtp_num_hidden_layers") or 0) == 1:+ for part, tensor in attn_parts.items():+ tail += vk_tensor_bytes(_q38_vk_fmt(tensor, _Q38_TRUNK_COMPONENTS.get(part), env),+ tensor["shape"][1], tensor["shape"][0])+ for tag, out, cols in (("mtpfce", H, H), ("mtpfch", H, H), ("mtpmixd", R, W), ("mtpmixu", W, R)):+ tensor = {"shape": [out, cols], "dtype": head_dtype}+ tail += vk_tensor_bytes(_q38_vk_fmt(tensor, tag, env), cols, out)+ return VkChainLayout(layers, fixed, tail)+++def _q38_chain_rows(env):+ """vkc_fit_rows(256): COLI_VK_CHAIN_ROWS when it is a number, else 256."""+ value = (env.get("COLI_VK_CHAIN_ROWS") or "").strip()+ if not value or value == "auto":+ return 256+ return min(max(_q38_env_int(env, "COLI_VK_CHAIN_ROWS", 256), 1), 65535)+++_VK_CHAIN_LAYOUT["qwen38"] = _q38_chain_layout++def _analysis_signature(shards, config_path):parts = [f"v{_ANALYSIS_CACHE_VERSION}"]st = config_path.stat()diff --git a/c/tests/test_resource_plan.py b/c/tests/test_resource_plan.pyindex 0962351e..7ef82c85 100644--- a/c/tests/test_resource_plan.py+++ b/c/tests/test_resource_plan.py@@ -1934,6 +1934,183 @@ class VulkanPartialChainQwenOlmoeTest(unittest.TestCase):self.assertEqual(full["tiers"]["ram"]["dense_on_device_bytes"], released(L))+class Qwen38PartialChainTest(unittest.TestCase):+ """qwen38 on the partial chain: resource_plan's layout (_q38_chain_layout) is the+ engine's fit (q38c_fit_layer, q38c_fit_fixed, q38c_fit_tail in qwen38_chain.h), and+ the plan credits the first N layers' host copies only."""++ # tools/make_qwen38_tiny.py's geometry (the config it writes, the fields the engine reads)+ CONFIG = {+ "architectures": ["Qwen4ExpForCausalLM"], "model_type": "qwen4_exp_text",+ "attention_bias": False, "hidden_act": "silu", "output_gate_type": "sigmoid",+ "tie_word_embeddings": False, "bos_token_id": 1, "eos_token_id": 2, "pad_token_id": 0,+ "hidden_size": 32, "num_hidden_layers": 4, "vocab_size": 64, "max_position_embeddings": 128,+ "num_attention_heads": 4, "num_key_value_heads": 2, "head_dim": 8, "rms_norm_eps": 1e-06,+ "partial_rotary_factor": 0.5,+ "rope_parameters": {"partial_rotary_factor": 0.5, "rope_theta": 10000.0, "rope_type": "default"},+ "layer_types": ["linear_attention", "qwen_sparse_attention", "linear_attention", "qwen_sparse_attention"],+ "linear_conv_kernel_dim": 4, "linear_key_head_dim": 4, "linear_num_key_heads": 2,+ "linear_num_value_heads": 4, "linear_value_head_dim": 4,+ "hc_count": 4, "hc_lowrank": 8, "ngram_size": 3, "heads_per_ngram": 2, "ngram_vocab_size_base": 31,+ "make_ngram_vocab_size_divisible_by": 4, "split_ngram_parts": 2, "ple_layer_ids": [1],+ "ple_embed_dim": 32, "ple_conv_kernel_size": 4, "indexer_n_heads": 2, "indexer_kv_heads": 1,+ "indexer_head_dim": 4, "indexer_budget": 4, "indexer_compress_ratio": 2, "num_experts": 4,+ "num_experts_per_tok": 2, "moe_intermediate_size": 8, "shared_expert_intermediate_size": 8,+ "norm_topk_prob": True,+ }++ def setUp(self):+ self.tmp = tempfile.TemporaryDirectory()+ self.model = Path(self.tmp.name)++ def tearDown(self):+ self.tmp.cleanup()++ def write_q38(self, ple_layer=1):+ """Every tensor of the tiny fixture (BF16), the PLE at ple_layer (one-based)."""+ c = dict(self.CONFIG, ple_layer_ids=[ple_layer])+ (self.model / "config.json").write_text(json.dumps(c))+ H, W, R, V = 32, 128, 8, 64+ tensors = []++ def add(name, *shape):+ count = 1+ for n in shape:+ count *= n+ tensors.append((name, 2 * count, "BF16", list(shape)))+ add("lm_head.weight", V, H)+ add("model.embed_tokens.weight", V, H)+ add("model.hyper_connection_mixer.hc_norm.weight", W)+ add("model.hyper_connection_mixer.input_mix_weight_down.weight", R, W)+ add("model.hyper_connection_mixer.input_mix_weight_up.weight", W, R)+ for i, kind in enumerate(c["layer_types"]):+ p = f"model.layers.{i}."+ for block in ("attn", "mlp"):+ add(p + block + "_hyper_connection.block_inject_weight.weight", 4, W)+ add(p + block + "_hyper_connection.hc_norm.weight", W)+ add(p + block + "_hyper_connection.input_mix_weight_down.weight", R, W)+ add(p + block + "_hyper_connection.input_mix_weight_up.weight", W, R)+ if kind == "linear_attention":+ add(p + "linear_attn.A_log", 4)+ add(p + "linear_attn.conv1d.weight", 32, 1, 4)+ add(p + "linear_attn.dt_bias", 4)+ add(p + "linear_attn.in_proj_a.weight", 4, H)+ add(p + "linear_attn.in_proj_b.weight", 4, H)+ add(p + "linear_attn.in_proj_qkv.weight", 32, H)+ add(p + "linear_attn.in_proj_z.weight", 16, H)+ add(p + "linear_attn.norm.weight", 4)+ add(p + "linear_attn.out_proj.weight", H, 16)+ else:+ add(p + "self_attn.indexer.index_qk_proj.weight", 12, H)+ add(p + "self_attn.indexer.k_layernorm.weight", 4)+ add(p + "self_attn.indexer.q_layernorm.weight", 4)+ add(p + "self_attn.k_norm.weight", 8)+ add(p + "self_attn.k_proj.weight", 16, H)+ add(p + "self_attn.o_proj.weight", H, H)+ add(p + "self_attn.q_norm.weight", 8)+ add(p + "self_attn.q_proj.weight", 64, H)+ add(p + "self_attn.v_proj.weight", 16, H)+ add(p + "mlp.gate.weight", 4, H)+ add(p + "mlp.shared_expert.down_proj.weight", H, 8)+ add(p + "mlp.shared_expert.gate_proj.weight", 8, H)+ add(p + "mlp.shared_expert.up_proj.weight", 8, H)+ add(p + "mlp.shared_expert_gate.weight", 1, H)+ for e in range(4):+ add(p + f"mlp.experts.{e}.down_proj.weight", H, 8)+ add(p + f"mlp.experts.{e}.gate_proj.weight", 8, H)+ add(p + f"mlp.experts.{e}.up_proj.weight", 8, H)+ if i == ple_layer - 1:+ add(p + "ple.conv1d.weight", W, 1, 4)+ add(p + "ple.key_proj.weight", W, 32)+ for norm in ("norm_conv", "norm_key", "norm_query"):+ add(p + f"ple.{norm}.weight", W)+ add(p + "ple.ple_embedding.ngram_embedding.shard_0.weight", 76, 8)+ add(p + "ple.ple_embedding.ngram_embedding.shard_1.weight", 76, 8)+ add(p + "ple.value_proj.weight", H, 32)+ write_shard(self.model / "model.safetensors", tensors)+ return analyze_model(self.model)++ def test_qwen38_layout_is_the_engines(self):+ # The numbers qwen38 printed for these geometries on Lavapipe ("[VK] qwen38 chain+ # fit: ... the engine's 1529872 B ..., tail 8960 B, layers 47664 92768 24112 92768 B"):+ # bf16 by default, the int8 trunk (Q38_TRUNK_MIN_KB=0), f32, the PLE at layer 2, and+ # the MTP head's matrices in the tail.+ from resource_plan import vk_chain_fit, vk_fit_pools+ on = {"COLI_VULKAN": "1", "COLI_VK_CHAIN": "1"}+ device = {"type": "cpu", "budget_bytes": 64 * GB}+ cases = (+ (1, {}, [47664, 92768, 24112, 92768], 8960),+ (1, {"Q38_TRUNK_MIN_KB": "0"}, [40240, 83040, 16688, 83040], 6912),+ (1, {"Q38_TRUNK_CPU_INT8": "0", "Q38_NATIVE_BF16": "0"}, [74544, 113760, 40752, 113760], 17152),+ (3, {}, [24112, 92768, 47664, 92768], 8960),+ )+ for ple, env, layers, tail in cases:+ with self.subTest(ple=ple, env=env):+ info = self.write_q38(ple)+ fit = vk_chain_fit(info, "qwen38", dict(on, **env), device)+ self.assertEqual(fit["layers"], layers)+ self.assertEqual(fit["fixed"] - vk_fit_pools(0), 1529872)+ self.assertEqual(fit["n"], 4)+ self.assertTrue(fit["tail"])+ # the MTP head goes with the tail under Q38_MTP=1 (43008 B in bf16, 27392 B as int8 rows)+ from resource_plan import _q38_chain_layout+ info = self.write_q38(1)+ c = dict(self.CONFIG, mtp_num_hidden_layers=1)+ (self.model / "config.json").write_text(json.dumps(c))+ info = dict(info, config=c)+ self.assertEqual(_q38_chain_layout(info, {"Q38_MTP": "1"}, None).tail, 43008)+ self.assertEqual(_q38_chain_layout(info, {"Q38_MTP": "1", "Q38_TRUNK_MIN_KB": "0"}, None).tail, 27392)+ self.assertEqual(_q38_chain_layout(info, {}, None).tail, 8960)+ # COLI_VK_CHAIN_ROWS lowers the chunk the fit counts: the scratch and the attention+ # layers' read-back rows+ small = vk_chain_fit(info, "qwen38", dict(on, COLI_VK_CHAIN_ROWS="3"), device)+ self.assertLess(small["fixed"], vk_chain_fit(info, "qwen38", on, device)["fixed"])+ self.assertEqual([a < b for a, b in zip(small["layers"], [47664, 92768, 24112, 92768])],+ [False, True, False, True])++ def test_qwen38_device_only_credits_the_layers_on_the_device(self):+ from resource_plan import vk_chain_fit, _vk_released_tensor_bytes+ info = self.write_q38(1)+ kwargs = dict(context=64, ram_gb=4, available_memory=16 * GB, available_disk=16 * GB, gpus=[])+ on = {"COLI_VULKAN": "1", "COLI_VK_CHAIN": "1", "COLI_VK_DENSE_HOST": "0", "COLI_VK_TIER_RESERVE_GB": "0"}+ big = {"type": "discrete", "budget_bytes": 64 * GB}+ fit = vk_chain_fit(info, "qwen38", on, big)++ def credit(k, head):+ total = 0+ for t in info["dense_tensors"]:+ name = t["name"]+ if name.startswith("model.layers."):+ if int(name.split(".")[2]) >= k:+ continue+ elif not head:+ continue+ total += _vk_released_tensor_bytes(t, "qwen38", on)+ return total+ full = build_plan(self.model, env=on, vulkan=big, **kwargs)+ self.assertEqual(full["tiers"]["ram"]["vk_chain_layers"]["on_device"], 4)+ self.assertEqual(full["tiers"]["ram"]["dense_on_device_bytes"], credit(4, True))+ for k in (1, 2, 3):+ # a device holding k layers and half of the next: the plan's N is k+ free = fit["fixed"] + sum(fit["layers"][:k]) + fit["layers"][k] // 2+ with self.subTest(k=k):+ plan = build_plan(self.model, env=on, vulkan={"type": "discrete", "budget_bytes": free}, **kwargs)+ ram = plan["tiers"]["ram"]+ self.assertEqual(ram["vk_chain_layers"]["on_device"], k)+ self.assertEqual(ram["dense_on_device_bytes"], credit(k, False))+ self.assertIn(f"the first {k} of 4 layers", format_plan(plan))+ forced = build_plan(self.model, env=dict(on, COLI_VK_CHAIN_LAYERS=str(k)), vulkan=big, **kwargs)+ self.assertEqual(forced["tiers"]["ram"]["dense_on_device_bytes"], credit(k, False))+ # every layer but no room for the head: the layers' copies only+ free = fit["fixed"] + sum(fit["layers"])+ plan = build_plan(self.model, env=on, vulkan={"type": "discrete", "budget_bytes": free}, **kwargs)+ self.assertEqual(plan["tiers"]["ram"]["vk_chain_layers"]["on_device"], 4)+ self.assertEqual(plan["tiers"]["ram"]["dense_on_device_bytes"], credit(4, False))+ # COLI_VK_CHAIN_LAYERS=0: the chain off, every host copy stays+ off = build_plan(self.model, env=dict(on, COLI_VK_CHAIN_LAYERS="0"), vulkan=big, **kwargs)+ self.assertEqual(off["tiers"]["ram"]["dense_on_device_bytes"], 0)++class PhysicalCpuCountTest(unittest.TestCase):"""Regression for #325: --auto-tier pinned decode to one core becausephysical_cpu_count() silently returned 1.