Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(planner): qwen38's partial-chain layout

JustVugg97a6f760db43
diff --git a/c/qwen38_chain.h b/c/qwen38_chain.h
index 00dfb2d7..e3eb1a74 100644
--- a/c/qwen38_chain.h
+++ b/c/qwen38_chain.h
@@ -260,7 +260,7 @@ static size_t q38c_fit_w(const Q38Weight *w, int payload) {
int fmt = q38_vk_fmt(w);
return payload ? coli_vk_tensor_payload(fmt, w->cols, w->rows, 0) : vkc_fit_tensor(fmt, w->cols, w->rows, 0);
}
-static int q38c_kv_rows0(void) { return 3 * (int)vkc_kv_env("COLI_VK_KV_BLOCK", 64); }
+static int q38c_kv_rows0(void) { long b = vkc_kv_env("COLI_VK_KV_BLOCK", 64); return 3 * (int)(b < 1 ? 1 : b); }
/* layer i's device bytes (as q38c_setup places it, its state at its starting size) and its matrices' payload */
static void q38c_fit_layer(Model *m, int i, int rows, size_t *bytes, size_t *payload) {
Cfg *c = &m->c; int H = c->hidden, W = c->hc_width;
diff --git a/c/resource_plan.py b/c/resource_plan.py
index ff3e9991..82a5c305 100644
--- a/c/resource_plan.py
+++ b/c/resource_plan.py
@@ -750,6 +750,135 @@ def _olmoe_chain_layout(info, env, vulkan):
_VK_CHAIN_LAYOUT["olmoe"] = _olmoe_chain_layout
+_Q38_CHAIN_MATRICES = frozenset(_Q38_TRUNK_COMPONENTS) | {
+ "linear_attn.in_proj_a", "linear_attn.in_proj_b", "ple.key_proj", "ple.value_proj"}
+
+
+def _q38_env_int(env, name, default):
+ """C's atol/atoi: the leading integer, 0 when there is none."""
+ value = env.get(name)
+ if value is None or not value.strip():
+ return default
+ match = re.match(r"\s*([+-]?\d+)", value)
+ return int(match[1]) if match else 0
+
+
+def _q38_vk_fmt(tensor, tag, env):
+ """q38_vk_fmt: the format a resident matrix goes up in. The trunk's int8 rows (fmt 1)
+ when q38_trunk_cpu_int8 quantized it (a trunk component of at least Q38_TRUNK_MIN_KB
+ with its float scales, not in Q38_TRUNK_SKIP); else the rows as loaded: bf16 (fmt 11)
+ from a BF16 tensor with Q38_NATIVE_BF16, f32 (fmt 10) otherwise."""
+ rows, columns = tensor["shape"]
+ if tag and env.get("Q38_TRUNK_CPU_INT8") != "0":
+ min_kb = _q38_env_int(env, "Q38_TRUNK_MIN_KB", 1024)
+ skipped = set((env.get("Q38_TRUNK_SKIP") or "").split(","))
+ if min_kb >= 0 and rows * columns + 4 * rows >= min_kb * 1024 and tag not in skipped:
+ return 1
+ return 11 if tensor["dtype"] == "BF16" and env.get("Q38_NATIVE_BF16") != "0" else 10
+
+
+def _q38_chain_layout(info, env, vulkan):
+ """qwen38_chain.h: q38c_fit_layer, q38c_fit_fixed, q38c_fit_tail."""
+ c = info.get("config") or {}
+ c = c.get("text_config") or c
+ try:
+ L, H, vocab = int(c["num_hidden_layers"]), int(c["hidden_size"]), int(c["vocab_size"])
+ kinds = [0 if t == "linear_attention" else 1 for t in c["layer_types"]][:L]
+ QH, KVH, D = int(c["num_attention_heads"]), int(c["num_key_value_heads"]), int(c["head_dim"])
+ IQ, ID = int(c["indexer_n_heads"]), int(c["indexer_head_dim"])
+ budget, ratio = int(c["indexer_budget"]), int(c["indexer_compress_ratio"])
+ VH, VD = int(c["linear_num_value_heads"]), int(c["linear_value_head_dim"])
+ KH, KD, CK = int(c["linear_num_key_heads"]), int(c["linear_key_head_dim"]), int(c["linear_conv_kernel_dim"])
+ E, SI = int(c["num_experts"]), int(c["shared_expert_intermediate_size"])
+ except (KeyError, TypeError, ValueError):
+ return None
+ if L < 1 or len(kinds) != L:
+ return None
+ C, R = int(c.get("hc_count") or 4), int(c.get("hc_lowrank") or 320)
+ W = C * H
+ rope = c.get("rope_parameters") or {}
+ partial = rope.get("partial_rotary_factor", c.get("partial_rotary_factor", 1.0))
+ rotary = int(D * float(partial if partial is not None else 1.0))
+ ngram, per_ngram = int(c.get("ngram_size") or 3), int(c.get("heads_per_ngram") or 8)
+ ple_dim, ple_k = int(c.get("ple_embed_dim") or H), int(c.get("ple_conv_kernel_size") or 4)
+ heads = (ngram - 1) * per_ngram
+ Ep = max(heads * (ple_dim // heads if heads > 0 else 0), 1)
+ ids = c.get("ple_layer_ids") or []
+ ple = int(ids[0]) - 1 if ids else -1
+ CD = 2 * KH * KD + VH * VD
+ rows = _q38_chain_rows(env)
+ rows0 = 3 * max(_q38_env_int(env, "COLI_VK_KV_BLOCK", 64), 1)
+ kvo = KVH * D
+ # each layer's matrices as they go up, and the shared expert's gate (fmt 10, H x 1)
+ layers = [vk_tensor_bytes(10, H, 1) for _ in range(L)]
+ for tensor in info.get("dense_tensors", []):
+ layer, part, shape = _vk_layer_index(tensor["name"]), _layer_component(tensor["name"]), tensor.get("shape")
+ if layer is None or layer >= L or part not in _Q38_CHAIN_MATRICES or not isinstance(shape, list) or len(shape) != 2:
+ continue
+ if part.startswith("ple.") and layer != ple:
+ continue
+ fmt = _q38_vk_fmt(tensor, _Q38_TRUNK_COMPONENTS.get(part), env)
+ layers[layer] += vk_tensor_bytes(fmt, shape[1], shape[0])
+ for i in range(L):
+ floats = 2 * W
+ if kinds[i]:
+ floats += 2 * D + 2 * ID
+ layers[i] += (2 * vk_buf_bytes(kvo * rows0 * 4) + vk_buf_bytes(ID * rows0 * 4) +
+ vk_buf_bytes(ID * (rows0 // max(ratio, 1) + 1) * 4) + rows * (2 * kvo + ID) * 4)
+ else:
+ floats += CD * CK + 2 * VH + VD
+ layers[i] += 2 * vk_buf_bytes(VH * KD * VD * 4) + 2 * vk_buf_bytes(CD * (CK - 1) * 4)
+ if i == ple:
+ floats += 3 * W + W * ple_k
+ layers[i] += vk_buf_bytes(2 * W * (ple_k - 1) * ngram * 4)
+ layers[i] += 4 * floats
+ # q38c_bufs' counting pass at `rows` rows, no context yet and no attention layer, the
+ # streams' read-back and the final norm
+ r, nb, rot = rows, 1, rotary if rotary > 0 else 2
+ counts = [r * W, r * W, r * R, r * W, r * H, r * C, r * C, r * H, r * QH * 2 * D, r * kvo, r * kvo,
+ r * (IQ + 1) * ID, r * QH * D, r * 2 * nb, r * (budget + ratio), r * CD, r * VH * VD, r * 2 * VH,
+ r * CD, r * VH * VD, r * E, r * SI, r * SI, r * SI, r * H, r, r * H, r * W, r * H, r * W, r * W,
+ r * H, r * E, r * (2 * kvo + ID), 2 * vocab, r * H, r * Ep, r * rot, nb * rot]
+ fixed = sum(4 * (n if n else 1) for n in counts) + (rows + 1) * W * 4
+ # the tail: the final mixer and lm_head; under Q38_MTP=1 the MTP head's matrices too
+ # (the scan leaves its tensors out: its decoder layer has an attention layer's shapes,
+ # its two projections are H x H and its mixer the final mixer's)
+ tail, head_dtype = 0, "BF16"
+ attn_parts = {}
+ last_attn = max([i for i in range(L) if kinds[i]] + [-1])
+ for tensor in info.get("dense_tensors", []):
+ name, shape = _text_weight_name(tensor["name"]), tensor.get("shape")
+ if not isinstance(shape, list) or len(shape) != 2:
+ continue
+ if name == "lm_head.weight":
+ head_dtype = tensor["dtype"]
+ tail += vk_tensor_bytes(_q38_vk_fmt(tensor, "lmhead", env), shape[1], shape[0])
+ elif name in ("model.hyper_connection_mixer.input_mix_weight_down.weight",
+ "model.hyper_connection_mixer.input_mix_weight_up.weight"):
+ tail += vk_tensor_bytes(_q38_vk_fmt(tensor, None, env), shape[1], shape[0])
+ elif _vk_layer_index(tensor["name"]) == last_attn and _layer_component(tensor["name"]) in _Q38_CHAIN_MATRICES:
+ attn_parts[_layer_component(tensor["name"])] = tensor
+ if env.get("Q38_MTP") == "1" and int(c.get("mtp_num_hidden_layers") or 0) == 1:
+ for part, tensor in attn_parts.items():
+ tail += vk_tensor_bytes(_q38_vk_fmt(tensor, _Q38_TRUNK_COMPONENTS.get(part), env),
+ tensor["shape"][1], tensor["shape"][0])
+ for tag, out, cols in (("mtpfce", H, H), ("mtpfch", H, H), ("mtpmixd", R, W), ("mtpmixu", W, R)):
+ tensor = {"shape": [out, cols], "dtype": head_dtype}
+ tail += vk_tensor_bytes(_q38_vk_fmt(tensor, tag, env), cols, out)
+ return VkChainLayout(layers, fixed, tail)
+
+
+def _q38_chain_rows(env):
+ """vkc_fit_rows(256): COLI_VK_CHAIN_ROWS when it is a number, else 256."""
+ value = (env.get("COLI_VK_CHAIN_ROWS") or "").strip()
+ if not value or value == "auto":
+ return 256
+ return min(max(_q38_env_int(env, "COLI_VK_CHAIN_ROWS", 256), 1), 65535)
+
+
+_VK_CHAIN_LAYOUT["qwen38"] = _q38_chain_layout
+
+
def _analysis_signature(shards, config_path):
parts = [f"v{_ANALYSIS_CACHE_VERSION}"]
st = config_path.stat()
diff --git a/c/tests/test_resource_plan.py b/c/tests/test_resource_plan.py
index 0962351e..7ef82c85 100644
--- a/c/tests/test_resource_plan.py
+++ b/c/tests/test_resource_plan.py
@@ -1934,6 +1934,183 @@ class VulkanPartialChainQwenOlmoeTest(unittest.TestCase):
self.assertEqual(full["tiers"]["ram"]["dense_on_device_bytes"], released(L))
+class Qwen38PartialChainTest(unittest.TestCase):
+ """qwen38 on the partial chain: resource_plan's layout (_q38_chain_layout) is the
+ engine's fit (q38c_fit_layer, q38c_fit_fixed, q38c_fit_tail in qwen38_chain.h), and
+ the plan credits the first N layers' host copies only."""
+
+ # tools/make_qwen38_tiny.py's geometry (the config it writes, the fields the engine reads)
+ CONFIG = {
+ "architectures": ["Qwen4ExpForCausalLM"], "model_type": "qwen4_exp_text",
+ "attention_bias": False, "hidden_act": "silu", "output_gate_type": "sigmoid",
+ "tie_word_embeddings": False, "bos_token_id": 1, "eos_token_id": 2, "pad_token_id": 0,
+ "hidden_size": 32, "num_hidden_layers": 4, "vocab_size": 64, "max_position_embeddings": 128,
+ "num_attention_heads": 4, "num_key_value_heads": 2, "head_dim": 8, "rms_norm_eps": 1e-06,
+ "partial_rotary_factor": 0.5,
+ "rope_parameters": {"partial_rotary_factor": 0.5, "rope_theta": 10000.0, "rope_type": "default"},
+ "layer_types": ["linear_attention", "qwen_sparse_attention", "linear_attention", "qwen_sparse_attention"],
+ "linear_conv_kernel_dim": 4, "linear_key_head_dim": 4, "linear_num_key_heads": 2,
+ "linear_num_value_heads": 4, "linear_value_head_dim": 4,
+ "hc_count": 4, "hc_lowrank": 8, "ngram_size": 3, "heads_per_ngram": 2, "ngram_vocab_size_base": 31,
+ "make_ngram_vocab_size_divisible_by": 4, "split_ngram_parts": 2, "ple_layer_ids": [1],
+ "ple_embed_dim": 32, "ple_conv_kernel_size": 4, "indexer_n_heads": 2, "indexer_kv_heads": 1,
+ "indexer_head_dim": 4, "indexer_budget": 4, "indexer_compress_ratio": 2, "num_experts": 4,
+ "num_experts_per_tok": 2, "moe_intermediate_size": 8, "shared_expert_intermediate_size": 8,
+ "norm_topk_prob": True,
+ }
+
+ def setUp(self):
+ self.tmp = tempfile.TemporaryDirectory()
+ self.model = Path(self.tmp.name)
+
+ def tearDown(self):
+ self.tmp.cleanup()
+
+ def write_q38(self, ple_layer=1):
+ """Every tensor of the tiny fixture (BF16), the PLE at ple_layer (one-based)."""
+ c = dict(self.CONFIG, ple_layer_ids=[ple_layer])
+ (self.model / "config.json").write_text(json.dumps(c))
+ H, W, R, V = 32, 128, 8, 64
+ tensors = []
+
+ def add(name, *shape):
+ count = 1
+ for n in shape:
+ count *= n
+ tensors.append((name, 2 * count, "BF16", list(shape)))
+ add("lm_head.weight", V, H)
+ add("model.embed_tokens.weight", V, H)
+ add("model.hyper_connection_mixer.hc_norm.weight", W)
+ add("model.hyper_connection_mixer.input_mix_weight_down.weight", R, W)
+ add("model.hyper_connection_mixer.input_mix_weight_up.weight", W, R)
+ for i, kind in enumerate(c["layer_types"]):
+ p = f"model.layers.{i}."
+ for block in ("attn", "mlp"):
+ add(p + block + "_hyper_connection.block_inject_weight.weight", 4, W)
+ add(p + block + "_hyper_connection.hc_norm.weight", W)
+ add(p + block + "_hyper_connection.input_mix_weight_down.weight", R, W)
+ add(p + block + "_hyper_connection.input_mix_weight_up.weight", W, R)
+ if kind == "linear_attention":
+ add(p + "linear_attn.A_log", 4)
+ add(p + "linear_attn.conv1d.weight", 32, 1, 4)
+ add(p + "linear_attn.dt_bias", 4)
+ add(p + "linear_attn.in_proj_a.weight", 4, H)
+ add(p + "linear_attn.in_proj_b.weight", 4, H)
+ add(p + "linear_attn.in_proj_qkv.weight", 32, H)
+ add(p + "linear_attn.in_proj_z.weight", 16, H)
+ add(p + "linear_attn.norm.weight", 4)
+ add(p + "linear_attn.out_proj.weight", H, 16)
+ else:
+ add(p + "self_attn.indexer.index_qk_proj.weight", 12, H)
+ add(p + "self_attn.indexer.k_layernorm.weight", 4)
+ add(p + "self_attn.indexer.q_layernorm.weight", 4)
+ add(p + "self_attn.k_norm.weight", 8)
+ add(p + "self_attn.k_proj.weight", 16, H)
+ add(p + "self_attn.o_proj.weight", H, H)
+ add(p + "self_attn.q_norm.weight", 8)
+ add(p + "self_attn.q_proj.weight", 64, H)
+ add(p + "self_attn.v_proj.weight", 16, H)
+ add(p + "mlp.gate.weight", 4, H)
+ add(p + "mlp.shared_expert.down_proj.weight", H, 8)
+ add(p + "mlp.shared_expert.gate_proj.weight", 8, H)
+ add(p + "mlp.shared_expert.up_proj.weight", 8, H)
+ add(p + "mlp.shared_expert_gate.weight", 1, H)
+ for e in range(4):
+ add(p + f"mlp.experts.{e}.down_proj.weight", H, 8)
+ add(p + f"mlp.experts.{e}.gate_proj.weight", 8, H)
+ add(p + f"mlp.experts.{e}.up_proj.weight", 8, H)
+ if i == ple_layer - 1:
+ add(p + "ple.conv1d.weight", W, 1, 4)
+ add(p + "ple.key_proj.weight", W, 32)
+ for norm in ("norm_conv", "norm_key", "norm_query"):
+ add(p + f"ple.{norm}.weight", W)
+ add(p + "ple.ple_embedding.ngram_embedding.shard_0.weight", 76, 8)
+ add(p + "ple.ple_embedding.ngram_embedding.shard_1.weight", 76, 8)
+ add(p + "ple.value_proj.weight", H, 32)
+ write_shard(self.model / "model.safetensors", tensors)
+ return analyze_model(self.model)
+
+ def test_qwen38_layout_is_the_engines(self):
+ # The numbers qwen38 printed for these geometries on Lavapipe ("[VK] qwen38 chain
+ # fit: ... the engine's 1529872 B ..., tail 8960 B, layers 47664 92768 24112 92768 B"):
+ # bf16 by default, the int8 trunk (Q38_TRUNK_MIN_KB=0), f32, the PLE at layer 2, and
+ # the MTP head's matrices in the tail.
+ from resource_plan import vk_chain_fit, vk_fit_pools
+ on = {"COLI_VULKAN": "1", "COLI_VK_CHAIN": "1"}
+ device = {"type": "cpu", "budget_bytes": 64 * GB}
+ cases = (
+ (1, {}, [47664, 92768, 24112, 92768], 8960),
+ (1, {"Q38_TRUNK_MIN_KB": "0"}, [40240, 83040, 16688, 83040], 6912),
+ (1, {"Q38_TRUNK_CPU_INT8": "0", "Q38_NATIVE_BF16": "0"}, [74544, 113760, 40752, 113760], 17152),
+ (3, {}, [24112, 92768, 47664, 92768], 8960),
+ )
+ for ple, env, layers, tail in cases:
+ with self.subTest(ple=ple, env=env):
+ info = self.write_q38(ple)
+ fit = vk_chain_fit(info, "qwen38", dict(on, **env), device)
+ self.assertEqual(fit["layers"], layers)
+ self.assertEqual(fit["fixed"] - vk_fit_pools(0), 1529872)
+ self.assertEqual(fit["n"], 4)
+ self.assertTrue(fit["tail"])
+ # the MTP head goes with the tail under Q38_MTP=1 (43008 B in bf16, 27392 B as int8 rows)
+ from resource_plan import _q38_chain_layout
+ info = self.write_q38(1)
+ c = dict(self.CONFIG, mtp_num_hidden_layers=1)
+ (self.model / "config.json").write_text(json.dumps(c))
+ info = dict(info, config=c)
+ self.assertEqual(_q38_chain_layout(info, {"Q38_MTP": "1"}, None).tail, 43008)
+ self.assertEqual(_q38_chain_layout(info, {"Q38_MTP": "1", "Q38_TRUNK_MIN_KB": "0"}, None).tail, 27392)
+ self.assertEqual(_q38_chain_layout(info, {}, None).tail, 8960)
+ # COLI_VK_CHAIN_ROWS lowers the chunk the fit counts: the scratch and the attention
+ # layers' read-back rows
+ small = vk_chain_fit(info, "qwen38", dict(on, COLI_VK_CHAIN_ROWS="3"), device)
+ self.assertLess(small["fixed"], vk_chain_fit(info, "qwen38", on, device)["fixed"])
+ self.assertEqual([a < b for a, b in zip(small["layers"], [47664, 92768, 24112, 92768])],
+ [False, True, False, True])
+
+ def test_qwen38_device_only_credits_the_layers_on_the_device(self):
+ from resource_plan import vk_chain_fit, _vk_released_tensor_bytes
+ info = self.write_q38(1)
+ kwargs = dict(context=64, ram_gb=4, available_memory=16 * GB, available_disk=16 * GB, gpus=[])
+ on = {"COLI_VULKAN": "1", "COLI_VK_CHAIN": "1", "COLI_VK_DENSE_HOST": "0", "COLI_VK_TIER_RESERVE_GB": "0"}
+ big = {"type": "discrete", "budget_bytes": 64 * GB}
+ fit = vk_chain_fit(info, "qwen38", on, big)
+
+ def credit(k, head):
+ total = 0
+ for t in info["dense_tensors"]:
+ name = t["name"]
+ if name.startswith("model.layers."):
+ if int(name.split(".")[2]) >= k:
+ continue
+ elif not head:
+ continue
+ total += _vk_released_tensor_bytes(t, "qwen38", on)
+ return total
+ full = build_plan(self.model, env=on, vulkan=big, **kwargs)
+ self.assertEqual(full["tiers"]["ram"]["vk_chain_layers"]["on_device"], 4)
+ self.assertEqual(full["tiers"]["ram"]["dense_on_device_bytes"], credit(4, True))
+ for k in (1, 2, 3):
+ # a device holding k layers and half of the next: the plan's N is k
+ free = fit["fixed"] + sum(fit["layers"][:k]) + fit["layers"][k] // 2
+ with self.subTest(k=k):
+ plan = build_plan(self.model, env=on, vulkan={"type": "discrete", "budget_bytes": free}, **kwargs)
+ ram = plan["tiers"]["ram"]
+ self.assertEqual(ram["vk_chain_layers"]["on_device"], k)
+ self.assertEqual(ram["dense_on_device_bytes"], credit(k, False))
+ self.assertIn(f"the first {k} of 4 layers", format_plan(plan))
+ forced = build_plan(self.model, env=dict(on, COLI_VK_CHAIN_LAYERS=str(k)), vulkan=big, **kwargs)
+ self.assertEqual(forced["tiers"]["ram"]["dense_on_device_bytes"], credit(k, False))
+ # every layer but no room for the head: the layers' copies only
+ free = fit["fixed"] + sum(fit["layers"])
+ plan = build_plan(self.model, env=on, vulkan={"type": "discrete", "budget_bytes": free}, **kwargs)
+ self.assertEqual(plan["tiers"]["ram"]["vk_chain_layers"]["on_device"], 4)
+ self.assertEqual(plan["tiers"]["ram"]["dense_on_device_bytes"], credit(4, False))
+ # COLI_VK_CHAIN_LAYERS=0: the chain off, every host copy stays
+ off = build_plan(self.model, env=dict(on, COLI_VK_CHAIN_LAYERS="0"), vulkan=big, **kwargs)
+ self.assertEqual(off["tiers"]["ram"]["dense_on_device_bytes"], 0)
+
+
class PhysicalCpuCountTest(unittest.TestCase):
"""Regression for #325: --auto-tier pinned decode to one core because
physical_cpu_count() silently returned 1.