Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat(planner): inkling's and MiMo's partial-chain layouts
JustVugg9e549da65b21
diff --git a/c/resource_plan.py b/c/resource_plan.pyindex 9cfdab60..1f95ed60 100644--- a/c/resource_plan.py+++ b/c/resource_plan.py@@ -1173,6 +1173,249 @@ _VK_CHAIN_LAYOUT["deepseek_v41"] = _v41_chain_layout_VK_CHAIN_LAYOUT["kimi"] = _k3_chain_layout+def _env_number(env, name, default):+ match = re.match(r"\s*([+-]?\d+)", env.get(name) or "")+ return int(match[1]) if match else default+++def _vk_fit_rows(env, default):+ """vkc_fit_rows: COLI_VK_CHAIN_ROWS when it is a number, else the engine's block."""+ value = (env.get("COLI_VK_CHAIN_ROWS") or "").strip()+ if value and value != "auto":+ return min(max(_env_number(env, "COLI_VK_CHAIN_ROWS", 1), 1), 65535)+ return max(default, 1)+++def _inkling_chain_layout(info, env, vulkan):+ """inkling_chain.h inkc_fit_start: each layer's matrices in the form inkling.c holds+ them (f32 fmt 10, bf16 fmt 11, the dense-int4g64 container's int8 fmt 1 and int4-g64+ fmt 4), the router's f32 copy, its share of the parameter arena, its convolution rings+ and its K/V mirror at the window's rows; fixed, inkc_bufs at vkc_fit_rows(256) rows,+ the residual's read-back and the final norm; the tail, lm_head."""+ root = info.get("config") or {}+ c = root.get("text_config") or root+ try:+ D, L = int(c.get("hidden_size", 6144)), int(c.get("num_hidden_layers", 66))+ heads, kvh, hd = int(c.get("num_attention_heads", 64)), int(c.get("num_key_value_heads", 8)), int(c.get("head_dim", 128))+ sheads = int(c.get("swa_num_attention_heads", heads))+ skvh, shd = int(c.get("swa_num_key_value_heads", 16)), int(c.get("swa_head_dim", hd))+ window, dr, ext = int(c.get("sliding_window_size", 512)), int(c.get("d_rel", 16)), int(c.get("rel_extent", 1024))+ taps = int(c.get("sconv_kernel_size", c.get("conv_kernel_size", 4)))+ E, ns = int(c.get("n_routed_experts", 256)), int(c.get("n_shared_experts", 2))+ vocab = int(c.get("vocab_size", 201024))+ unpad = int(c.get("unpadded_vocab_size", vocab))+ if "dense_intermediate_size" in c:+ dense_inter, inter = int(c["dense_intermediate_size"]), int(c.get("intermediate_size", 3072))+ else:+ dense_inter, inter = int(c.get("intermediate_size", 24576)), int(c.get("moe_intermediate_size", 3072))+ except (TypeError, ValueError):+ return None+ if L < 1:+ return None+ types, ids = c.get("layer_types"), c.get("local_layer_ids")+ local = []+ for i in range(L):+ kind = types[i] if isinstance(types, list) and i < len(types) and isinstance(types[i], str) else None+ if kind is not None:+ local.append(kind == "hybrid_sliding")+ elif isinstance(ids, list):+ local.append(i in ids)+ else:+ local.append((i + 1) % 6 != 0)+ mlp, first_dense = c.get("mlp_layer_types"), int(c.get("dense_mlp_idx", 0) or 0)+ sparse = [(mlp[i] == "sparse") if isinstance(mlp, list) and i < len(mlp) and isinstance(mlp[i], str)+ else i >= first_dense for i in range(L)]+ # the dense-int4g64 container beside the snapshot (load_w_quant): a byte tensor (U8 or+ # I8) with its .qs scales replaces the original, int8 rows when it has as many bytes as+ # the original has weights and few scales, int4-g64 when about half as many; any other+ # geometry keeps the original+ container = {}+ sidecar = Path(info.get("path") or ".") / "dense-int4g64"+ for shard in sorted(sidecar.glob("*.safetensors")) if sidecar.is_dir() else []:+ try:+ for name, size, dtype, _ in _tensor_sizes(shard, with_shape=True):+ container[name] = (size, dtype)+ except (OSError, ValueError):+ return None+ main = {}+ for t in info.get("dense_tensors", []):+ numel = 1+ for n in t.get("shape") or [0]:+ numel *= n+ main[t["name"]] = (t["dtype"], numel)++ def form(name):+ dtype, numel = main.get(name, (None, 0))+ q, s = container.get(name), container.get(name + ".qs")+ if q and s and q[1] in ("U8", "I8"):+ scales = s[0] // 4+ if q[0] == numel and scales * 64 < q[0]:+ return 1, 0+ if numel <= 2 * q[0] <= numel + 2 * scales:+ return 4, 64+ return {"F32": (10, 0), "F16": (10, 0), "BF16": (11, 0)}.get(dtype)++ layers = []+ for i in range(L):+ H, KV, d = (sheads, skvh, shd) if local[i] else (heads, kvh, hd)+ kvo, p = KV * d, f"model.layers.{i}."+ mats = [("self_attn.q_proj.weight", D, H * d), ("self_attn.k_proj.weight", D, kvo),+ ("self_attn.v_proj.weight", D, kvo), ("self_attn.r_proj.weight", D, H * dr),+ ("self_attn.o_proj.weight", H * d, D)]+ if not sparse[i]:+ mats += [("mlp.gate_proj.weight", D, dense_inter), ("mlp.up_proj.weight", D, dense_inter),+ ("mlp.down_proj.weight", dense_inter, D)]+ total = 0+ for suffix, columns, rows in mats:+ f = form(p + suffix)+ if f is None:+ return None # a form the shaders do not take: the chain declines, N = L as before+ total += vk_tensor_bytes(f[0], columns, rows, f[1])+ if sparse[i]:+ total += vk_tensor_bytes(10, D, E + ns) # the router's f32 copy+ for suffix, columns, rows in (("mlp.shared_experts.gate_proj", D, inter),+ ("mlp.shared_experts.up_proj", D, inter),+ ("mlp.shared_experts.down_proj", inter, D)):+ f = form(p + suffix)+ if f is None:+ return None+ total += ns * vk_tensor_bytes(f[0], columns, rows, f[1])+ total += 4 * (2 * D + 2 * d + dr * (window if local[i] else ext) + 2 * kvo * taps + 2 * D * taps)+ for bank in range(4):+ cells = (kvo if bank < 2 else D) * (taps - 1)+ total += vk_buf_bytes(4 * max(cells, 1)) + 4 * cells+ total += 2 * vk_buf_bytes(4 * KV * max(window, 1) * d)+ layers.append(total)+ # inkc_bufs at R rows: the scratch's geometry over every layer and the frames (a MoE+ # layer ends one)+ R = _vk_fit_rows(env, 256)+ kvo_max = max((skvh * shd if local[i] else kvh * hd) for i in range(L))+ qo_max = max((sheads * shd if local[i] else heads * hd) for i in range(L))+ ro_max = max((sheads if local[i] else heads) * dr for i in range(L))+ mi_max = max((inter if sparse[i] else dense_inter) for i in range(L))+ nslot, frame = 0, 0+ for i in range(L):+ frame += 1+ nslot = max(nslot, frame)+ if sparse[i]:+ frame = 0+ nsl, ET = max(ns, 1), E + ns+ vs = (R * kvo_max + 63) & ~63+ counts = [R * D, R * D, R * D, R * qo_max, vs + R * kvo_max, R * ro_max, R * qo_max, R * D, R * ET,+ R * mi_max, R * mi_max, nsl * R * D, R * D, D, R * D, R * ET, nslot * 2 * R * kvo_max, 0,+ unpad, R * D, nsl * R, 2 * R]+ fixed = sum(vk_buf_bytes(4 * max(n, 1)) for n in counts) + vk_buf_bytes(4 * R * D) + vk_buf_bytes(4 * D)+ head = form("lm_head.weight")+ if head is None:+ return None+ return VkChainLayout(layers, fixed, vk_tensor_bytes(head[0], D, unpad, head[1]))+++_VK_CHAIN_LAYOUT["inkling"] = _inkling_chain_layout+++# the tower's matrices vk_dense_upload places (mimo_vision.h): the patch embedding (a+# Conv3d read as a Linear over the flattened patch), each block's, the merger's two+_MIMO_TOWER = re.compile(r"visual\.(?:patch_embed\.proj|blocks\.\d+\.(?:attn\.(?:qkv|proj)|mlp\.(?:gate|up|down)_proj)"+ r"|merger\.mlp\.[02])\.weight")+++def _mimo_chain_layout(info, env, vulkan):+ """mimo.c mc_fit_start: each layer's matrices in their MIMO_DENSE_BITS form (0: the+ release's FP8 with 128-column block scales, fmt 12, and BF16, fmt 11; 8: int8 rows,+ fmt 1; 32: f32, fmt 10), its K/V mirror at the size the chain allocates (a sliding+ layer's ring, a full layer's context or COLI_VK_KV_DEVICE_ROWS when that is fewer) and+ its norms and sink logits; fixed, mc_scratch at vkc_fit_rows(MIMO_CHUNK) rows and the+ final norm; the tail, the head (and the vision tower the per-matrix path takes)."""+ c = info.get("config") or {}+ try:+ L, H, V = int(c["num_hidden_layers"]), int(c["hidden_size"]), int(c["vocab_size"])+ dense_inter = int(c.get("intermediate_size") or 0)+ heads = [int(c["num_attention_heads"]), 0]+ kvh = [int(c["num_key_value_heads"]), 0]+ hd = [int(c["head_dim"]), 0]+ vd = [int(c.get("v_head_dim", hd[0])), 0]+ heads[1] = int(c.get("swa_num_attention_heads", heads[0]))+ kvh[1] = int(c.get("swa_num_key_value_heads", kvh[0]))+ hd[1] = int(c.get("swa_head_dim", hd[0]))+ vd[1] = int(c.get("swa_v_head_dim", vd[0]))+ window = int(c.get("sliding_window") or 0)+ prf = float(c.get("partial_rotary_factor", 1.0))+ max_pos = int(c.get("max_position_embeddings", 32768))+ except (KeyError, TypeError, ValueError):+ return None+ if L < 1 or max(hd + vd) > 256:+ return None+ pattern, freq = c.get("hybrid_layer_pattern"), c.get("moe_layer_freq")+ if not isinstance(pattern, list) or not isinstance(freq, list) or len(pattern) < L or len(freq) < L:+ return None+ swa = [pattern[i] == 1 for i in range(L)]+ moe = [freq[i] != 0 for i in range(L)]+ sink = [bool(c.get("add_full_attention_sink_bias")), bool(c.get("add_swa_attention_sink_bias"))]+ bits = _env_number(env, "MIMO_DENSE_BITS", 0)++ def size(fmt_native, columns, rows):+ if bits == 32:+ return vk_tensor_bytes(10, columns, rows)+ if bits == 8:+ return vk_tensor_bytes(1, columns, rows)+ return vk_tensor_bytes(fmt_native, columns, rows, 128 if fmt_native == 12 else 0)++ text = env.get("MIMO_CTX") or env.get("CTX") or "" # mimo.c: MIMO_CTX, else CTX, else 8192+ ctx = _env_number({"v": text}, "v", 0) if text else 8192+ ctx = max(16, min(ctx, max_pos))+ forced = _env_number(env, "COLI_VK_KV_DEVICE_ROWS", 0)+ layers = []+ for i in range(L):+ k = 1 if swa[i] else 0+ kd, vdd, qd = kvh[k] * hd[k], kvh[k] * vd[k], heads[k] * hd[k]+ total = size(12, H, qd + kd + vdd) + size(11, heads[k] * vd[k], H)+ if not moe[i]:+ total += 2 * size(12, H, dense_inter) + size(12, dense_inter, H)+ rows = min(window, ctx) if swa[i] and window > 0 else ctx+ if not swa[i] and 0 < forced < rows:+ rows = forced+ total += vk_buf_bytes(4 * rows * kd) + vk_buf_bytes(4 * rows * vdd)+ total += 4 * (2 * H + (heads[k] if sink[k] else 0))+ layers.append(total)+ # mc_scratch at R rows, one row of logits+ R = _vk_fit_rows(env, _env_number(env, "MIMO_CHUNK", 64) if _env_number(env, "MIMO_CHUNK", 0) > 0 else 64)+ geo = []+ for i in range(L):+ k = 1 if swa[i] else 0+ rows = min(window, ctx) if swa[i] and window > 0 else ctx+ geo.append((swa[i], kvh[k] * hd[k], kvh[k] * vd[k], heads[k] * hd[k], heads[k] * vd[k], rows))+ rw = max(qd + kd + vdd for _, kd, vdd, qd, _, _ in geo)+ kdm = max(g[1] for g in geo)+ vdm = max(g[2] for g in geo)+ ctxw = max(g[4] for g in geo)+ kvd = sum((rows if s and rows < R else R) * (kd + vdd) for s, kd, vdd, _, _, rows in geo)+ rope = [int(hd[0] * prf) // 2 * 2, int(hd[1] * prf) // 2 * 2]+ counts = [R * H, R * H, R * H, R * rw, R * kdm, R * vdm, R * ctxw, H, R * H, kvd, V, R * H,+ R * 2 * (rope[0] // 2 + rope[1] // 2)]+ windowed = [g for g in geo if g[0]]+ if windowed:+ win = max(g[5] for g in windowed) - 1+ counts += [(win + R) * max(g[1] for g in windowed), (win + R) * max(g[2] for g in windowed)]+ if not all(moe):+ counts += [R * dense_inter, R * dense_inter]+ fixed = sum(vk_buf_bytes(4 * max(n, 1)) for n in counts) + vk_buf_bytes(4 * H)+ tail = size(11, H, V)+ _, dense = _vk_chain_dense("mimo", env, vulkan)+ if dense: # the vision tower, on the device through the per-matrix path+ for tensor in info.get("dense_tensors", []):+ shape = tensor.get("shape")+ if _MIMO_TOWER.fullmatch(tensor["name"]) and isinstance(shape, list) and len(shape) >= 2:+ columns = 1+ for n in shape[1:]:+ columns *= n+ tail += vk_tensor_bytes(10 if bits == 32 else 11, columns, shape[0])+ return VkChainLayout(layers, fixed, tail)+++_VK_CHAIN_LAYOUT["mimo"] = _mimo_chain_layout++def _analysis_signature(shards, config_path):parts = [f"v{_ANALYSIS_CACHE_VERSION}"]st = config_path.stat()diff --git a/c/tests/test_resource_plan.py b/c/tests/test_resource_plan.pyindex f1cf5460..0dafdc50 100644--- a/c/tests/test_resource_plan.py+++ b/c/tests/test_resource_plan.py@@ -2485,6 +2485,238 @@ class VulkanPartialChainDskTest(unittest.TestCase):self.assertEqual((none["vk_chain_layers"]["on_device"], none["dense_on_device_bytes"]), (0, 0))+class VulkanPartialChainInklingMimoTest(unittest.TestCase):+ """inkling's and MiMo's partial chain in the planner: their layouts are the bytes the+ engines' fits printed for the tiny fixtures on Lavapipe (tests/vulkan_partial_inkling-mimo.sh+ runs the engines), and the plan credits only the first N layers' host copies."""++ def setUp(self):+ self.tmp = tempfile.TemporaryDirectory()+ self.model = Path(self.tmp.name)++ def tearDown(self):+ self.tmp.cleanup()++ def write_inkling(self, container=False):+ """tools/make_tiny_inkling.py's geometry (8 layers, the global one at 5, layers 0+ and 1 dense), f32; container: the dense-int4g64 converter's output beside it+ (int4-g64 matrices, int8 down projections and lm_head, as the family converts it)."""+ c = {"architectures": ["InklingForCausalLM"], "model_type": "inkling", "hidden_size": 64,+ "num_hidden_layers": 8, "vocab_size": 256, "unpadded_vocab_size": 250,+ "num_attention_heads": 4, "num_key_value_heads": 2, "head_dim": 16,+ "swa_num_attention_heads": 4, "swa_num_key_value_heads": 4, "swa_head_dim": 16,+ "sliding_window_size": 8, "d_rel": 8, "rel_extent": 32, "conv_kernel_size": 4,+ "layer_types": ["hybrid_sliding"] * 5 + ["hybrid"] + ["hybrid_sliding"] * 2,+ "mlp_layer_types": ["dense"] * 2 + ["sparse"] * 6, "intermediate_size": 96,+ "moe_intermediate_size": 32, "n_routed_experts": 8, "n_shared_experts": 2,+ "num_experts_per_tok": 2, "rms_norm_eps": 1e-06, "max_position_embeddings": 4096}+ (self.model / "config.json").write_text(json.dumps(c))+ main, quant = [], []++ def add(name, *shape, q=None):+ count = 1+ for n in shape:+ count *= n+ main.append((name, 4 * count, "F32", list(shape)))+ if q == "int4": # one f32 scale per 64 columns of a row+ quant.append((name, count // 2, "U8", [count // shape[-1], shape[-1] // 2]))+ quant.append((name + ".qs", 4 * (count // 64 or 1), "F32", [count // shape[-1], max(shape[-1] // 64, 1)]))+ elif q == "int8":+ quant.append((name, count, "I8", list(shape)))+ quant.append((name + ".qs", 4 * shape[0], "F32", [shape[0]]))+ add("model.embed_tokens.weight", 256, 64)+ add("model.norm.weight", 64)+ add("lm_head.weight", 256, 64, q="int8")+ for i in range(8):+ p = f"model.layers.{i}."+ kv = 64 if i != 5 else 32+ for name in ("input_layernorm", "post_attention_layernorm"):+ add(p + name + ".weight", 64)+ for name, rows in (("q_proj", 64), ("k_proj", kv), ("v_proj", kv), ("r_proj", 32), ("o_proj", 64)):+ add(p + "self_attn." + name + ".weight", rows, 64, q="int4")+ add(p + "self_attn.q_norm.weight", 16)+ add(p + "self_attn.k_norm.weight", 16)+ add(p + "self_attn.rel_logits_proj.proj", 8, 8)+ for name, width in (("self_attn.k_sconv", kv), ("self_attn.v_sconv", kv), ("attn_sconv", 64), ("mlp_sconv", 64)):+ add(p + name + ".conv1d.weight", width, 1, 4)+ if i < 2:+ add(p + "mlp.gate_proj.weight", 96, 64, q="int4")+ add(p + "mlp.up_proj.weight", 96, 64, q="int4")+ add(p + "mlp.down_proj.weight", 64, 96, q="int8")+ add(p + "mlp.global_scale", 1)+ else:+ add(p + "mlp.gate.weight", 10, 64)+ add(p + "mlp.gate.e_score_correction_bias", 8)+ add(p + "mlp.gate.global_scale", 1)+ add(p + "mlp.shared_experts.gate_proj", 2, 32, 64, q="int4")+ add(p + "mlp.shared_experts.up_proj", 2, 32, 64, q="int4")+ add(p + "mlp.shared_experts.down_proj", 2, 64, 32)+ add(p + "mlp.experts.gate_up_proj", 8, 64, 64)+ add(p + "mlp.experts.down_proj", 8, 64, 32)+ write_shard(self.model / "model.safetensors", main)+ if container:+ (self.model / "dense-int4g64").mkdir()+ write_shard(self.model / "dense-int4g64" / "dense.safetensors", quant)+ return analyze_model(self.model)++ def write_mimo(self):+ """tools/make_mimo_tiny.py's geometry: 6 layers (full attention at 0 and 3, sliding+ windows of 8 elsewhere, the dense layer 0), the release's FP8 and BF16 forms, the+ MXFP4 experts and the vision tower."""+ c = {"architectures": ["MiMoV2ForCausalLM"], "model_type": "mimo_v2", "vocab_size": 320,+ "hidden_size": 256, "intermediate_size": 256, "num_hidden_layers": 6,+ "hybrid_layer_pattern": [0, 1, 1, 0, 1, 1], "moe_layer_freq": [0, 1, 1, 1, 1, 1],+ "num_attention_heads": 4, "num_key_value_heads": 2, "head_dim": 48, "v_head_dim": 32,+ "swa_num_attention_heads": 4, "swa_num_key_value_heads": 4, "swa_head_dim": 48,+ "swa_v_head_dim": 32, "sliding_window": 8, "add_full_attention_sink_bias": False,+ "add_swa_attention_sink_bias": True, "attention_value_scale": 0.707,+ "partial_rotary_factor": 0.334, "rope_theta": 10000000.0, "swa_rope_theta": 10000.0,+ "n_routed_experts": 16, "num_experts_per_tok": 4, "moe_intermediate_size": 64,+ "layernorm_epsilon": 1e-06, "max_position_embeddings": 256, "image_token_id": 300,+ "quantization_config": {"quant_method": "fp8", "store_dtype": "mxfp4", "mxfp4_block_size": 32,+ "weight_block_size": [128, 128]},+ "vision_config": {"depth": 6, "hidden_size": 64, "intermediate_size": 96, "num_heads": 4,+ "num_key_value_heads": 2, "qk_channels": 16, "out_hidden_size": 256,+ "patch_size": 16, "spatial_patch_size": 16, "temporal_patch_size": 2,+ "spatial_merge_size": 2, "in_chans": 3}}+ (self.model / "config.json").write_text(json.dumps(c))+ width = {"BF16": 2, "F32": 4, "F8_E4M3": 1, "U8": 1}+ tensors = []++ def add(name, dtype, *shape):+ count = 1+ for n in shape:+ count *= n+ tensors.append((name, count * width[dtype], dtype, list(shape)))++ def fp8(name, rows, cols):+ add(name + ".weight", "F8_E4M3", rows, cols)+ add(name + ".weight_scale_inv", "F32", -(-rows // 128), -(-cols // 128))+ add("model.embed_tokens.weight", "BF16", 320, 256)+ add("lm_head.weight", "BF16", 320, 256)+ add("model.norm.weight", "BF16", 256)+ for i in range(6):+ p, swa = f"model.layers.{i}.", c["hybrid_layer_pattern"][i]+ add(p + "input_layernorm.weight", "BF16", 256)+ add(p + "post_attention_layernorm.weight", "BF16", 256)+ fp8(p + "self_attn.qkv_proj", 352 if swa else 224, 256)+ add(p + "self_attn.o_proj.weight", "BF16", 256, 128)+ if swa:+ add(p + "self_attn.attention_sink_bias", "BF16", 4)+ if i == 0:+ for name, rows, cols in (("gate_proj", 256, 256), ("up_proj", 256, 256), ("down_proj", 256, 256)):+ fp8(p + "mlp." + name, rows, cols)+ continue+ add(p + "mlp.gate.weight", "BF16", 16, 256)+ add(p + "mlp.gate.e_score_correction_bias", "F32", 16)+ for e in range(16):+ for name, rows, cols in (("gate_proj", 64, 128), ("up_proj", 64, 128), ("down_proj", 256, 32)):+ add(p + f"mlp.experts.{e}.{name}.weight", "U8", rows, cols)+ add(p + f"mlp.experts.{e}.{name}.weight_scale", "U8", rows, cols // 16)+ add("visual.patch_embed.proj.weight", "BF16", 64, 3, 2, 16, 16)+ for b in range(6):+ p = f"visual.blocks.{b}."+ for name, rows, cols in (("attn.qkv", 128, 64), ("attn.proj", 64, 64), ("mlp.gate_proj", 96, 64),+ ("mlp.up_proj", 96, 64), ("mlp.down_proj", 64, 96)):+ add(p + name + ".weight", "BF16", rows, cols)+ add(p + name + ".bias", "BF16", rows)+ add(p + "norm1.weight", "BF16", 64)+ add(p + "norm2.weight", "BF16", 64)+ add("visual.merger.ln_q.weight", "BF16", 64)+ add("visual.merger.mlp.0.weight", "BF16", 256, 256)+ add("visual.merger.mlp.2.weight", "BF16", 256, 256)+ write_shard(self.model / "model.safetensors", tensors)+ return analyze_model(self.model)++ def test_inkling_layout_is_the_engines(self):+ # "[VK] inkling chain fit: ... fixed X B (the engine's 1566464 B, ...), tail 64256 B,+ # layers 164736 164736 143744 ..." on tiny_inkling (f32), and with the dense-int4g64+ # container (tiny_inkling_q): its int4-g64 and int8 forms; lm_head's int8 rows carry+ # one scale a row of the padded vocabulary, a geometry inkling.c does not take, so+ # the head stays f32 there+ from resource_plan import vk_chain_fit, vk_fit_pools+ on = {"COLI_VULKAN": "1", "COLI_VK_CHAIN": "1"}+ lavapipe = {"type": "cpu", "budget_bytes": 64 * GB}+ for container, layers in ((False, [164736, 164736, 143744, 143744, 143744, 123776, 143744, 143744]),+ (True, [39296, 39296, 50560, 50560, 50560, 44928, 50560, 50560])):+ with self.subTest(container=container):+ self.tearDown(); self.setUp()+ info = self.write_inkling(container)+ fit = vk_chain_fit(info, "inkling", on, lavapipe)+ self.assertEqual(fit["layers"], layers)+ self.assertEqual(fit["fixed"] - vk_fit_pools(0), 1566464)+ self.assertEqual((fit["n"], fit["tail"]), (8, True))+ small = vk_chain_fit(info, "inkling", dict(on, COLI_VK_CHAIN_ROWS="3"), lavapipe)+ self.assertLess(small["fixed"], fit["fixed"])+ self.assertEqual(small["layers"], layers)+ # a dtype the shaders do not take: no layout, the plan as before+ self.tearDown(); self.setUp()+ info = self.write_inkling()+ info["dense_tensors"] = [dict(t, dtype="F8_E4M3") if t["name"].endswith("q_proj.weight") else t+ for t in info["dense_tensors"]]+ self.assertIsNone(vk_chain_fit(info, "inkling", on, lavapipe))++ def test_mimo_layout_is_the_engines(self):+ # the "[VK] mimo chain fit:" lines of mimo_tiny on Lavapipe, per MIMO_DENSE_BITS, and+ # with the vision tower on the per-matrix path (COLI_VK_DENSE=1: the tail)+ from resource_plan import vk_chain_fit, vk_fit_pools+ info = self.write_mimo()+ lavapipe = {"type": "cpu", "budget_bytes": 64 * GB}+ cases = (("32", [1445120, 668176, 668176, 657920, 668176, 668176], 327936, None),+ ("0", [527360, 213264, 213264, 324608, 213264, 213264], 164096, 999936),+ ("8", [491008, 179216, 179216, 291328, 179216, 179216], 83200, 919040))+ for bits, layers, tail, tower in cases:+ with self.subTest(bits=bits):+ env = {"COLI_VULKAN": "1", "COLI_VK_CHAIN": "1", "MIMO_DENSE_BITS": bits}+ fit = vk_chain_fit(info, "mimo", env, lavapipe)+ self.assertEqual(fit["layers"], layers)+ self.assertEqual(fit["fixed"] - vk_fit_pools(0), 929792)+ self.assertEqual(fit["n"], 6)+ layout = __import__("resource_plan")._VK_CHAIN_LAYOUT["mimo"](info, env, lavapipe)+ self.assertEqual(layout.tail, tail)+ if tower:+ dense = __import__("resource_plan")._VK_CHAIN_LAYOUT["mimo"](info, dict(env, COLI_VK_DENSE="1"), lavapipe)+ self.assertEqual(dense.tail, tower)+ # COLI_VK_KV_DEVICE_ROWS shrinks the full layers' mirrors (0 and 3), not the rings+ env = {"COLI_VULKAN": "1", "COLI_VK_CHAIN": "1", "MIMO_DENSE_BITS": "32"}+ fewer = vk_chain_fit(info, "mimo", dict(env, COLI_VK_KV_DEVICE_ROWS="16"), lavapipe)["layers"]+ full = vk_chain_fit(info, "mimo", env, lavapipe)["layers"]+ self.assertEqual([a < b for a, b in zip(fewer, full)], [True, False, False, True, False, False])++ def test_mimo_credits_the_layers_on_the_device(self):+ from resource_plan import vk_chain_fit+ info = self.write_mimo()+ kwargs = dict(context=64, ram_gb=4, available_memory=16 * GB, available_disk=16 * GB, gpus=[])+ on = {"COLI_VULKAN": "1", "COLI_VK_CHAIN": "1", "COLI_VK_DENSE_HOST": "0", "COLI_VK_TIER_RESERVE_GB": "0"}+ dgpu = {"type": "discrete", "budget_bytes": 64 * GB}++ def droppable(k, head): # mimo's dho pass: qkv, o_proj, the dense MLP, the head+ total = 0+ for t in info["dense_tensors"]:+ name, shape = t["name"], t["shape"]+ if len(shape) != 2 or name.endswith(("_scale_inv", ".bias")) or ".gate." in name:+ continue+ if name == "lm_head.weight":+ total += min(t["resident"], shape[0] * shape[1]) if head else 0+ elif name.startswith("model.layers.") and int(name.split(".")[2]) < k:+ total += min(t["resident"], shape[0] * shape[1])+ return total+ full = build_plan(self.model, env=on, vulkan=dgpu, **kwargs)["tiers"]["ram"]+ self.assertEqual(full["vk_chain_layers"]["on_device"], 6)+ self.assertEqual(full["dense_on_device_bytes"], droppable(6, True))+ fit = vk_chain_fit(info, "mimo", on, dgpu)+ for k in (1, 3, 5):+ with self.subTest(k=k):+ free = fit["fixed"] + sum(fit["layers"][:k]) + fit["layers"][k] // 2+ ram = build_plan(self.model, env=on, vulkan={"type": "discrete", "budget_bytes": free}, **kwargs)["tiers"]["ram"]+ self.assertEqual(ram["vk_chain_layers"]["on_device"], k)+ self.assertEqual(ram["dense_on_device_bytes"], droppable(k, False))+ forced = build_plan(self.model, env=dict(on, COLI_VK_CHAIN_LAYERS=str(k)), vulkan=dgpu, **kwargs)+ self.assertEqual(forced["tiers"]["ram"]["dense_on_device_bytes"], droppable(k, False))+ none = build_plan(self.model, env=dict(on, COLI_VK_CHAIN_LAYERS="0"), vulkan=dgpu, **kwargs)["tiers"]["ram"]+ self.assertEqual(none["dense_on_device_bytes"], 0)++class PhysicalCpuCountTest(unittest.TestCase):"""Regression for #325: --auto-tier pinned decode to one core becausephysical_cpu_count() silently returned 1.