Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(planner): inkling's and MiMo's partial-chain layouts

JustVugg9e549da65b21
diff --git a/c/resource_plan.py b/c/resource_plan.py
index 9cfdab60..1f95ed60 100644
--- a/c/resource_plan.py
+++ b/c/resource_plan.py
@@ -1173,6 +1173,249 @@ _VK_CHAIN_LAYOUT["deepseek_v41"] = _v41_chain_layout
_VK_CHAIN_LAYOUT["kimi"] = _k3_chain_layout
+def _env_number(env, name, default):
+ match = re.match(r"\s*([+-]?\d+)", env.get(name) or "")
+ return int(match[1]) if match else default
+
+
+def _vk_fit_rows(env, default):
+ """vkc_fit_rows: COLI_VK_CHAIN_ROWS when it is a number, else the engine's block."""
+ value = (env.get("COLI_VK_CHAIN_ROWS") or "").strip()
+ if value and value != "auto":
+ return min(max(_env_number(env, "COLI_VK_CHAIN_ROWS", 1), 1), 65535)
+ return max(default, 1)
+
+
+def _inkling_chain_layout(info, env, vulkan):
+ """inkling_chain.h inkc_fit_start: each layer's matrices in the form inkling.c holds
+ them (f32 fmt 10, bf16 fmt 11, the dense-int4g64 container's int8 fmt 1 and int4-g64
+ fmt 4), the router's f32 copy, its share of the parameter arena, its convolution rings
+ and its K/V mirror at the window's rows; fixed, inkc_bufs at vkc_fit_rows(256) rows,
+ the residual's read-back and the final norm; the tail, lm_head."""
+ root = info.get("config") or {}
+ c = root.get("text_config") or root
+ try:
+ D, L = int(c.get("hidden_size", 6144)), int(c.get("num_hidden_layers", 66))
+ heads, kvh, hd = int(c.get("num_attention_heads", 64)), int(c.get("num_key_value_heads", 8)), int(c.get("head_dim", 128))
+ sheads = int(c.get("swa_num_attention_heads", heads))
+ skvh, shd = int(c.get("swa_num_key_value_heads", 16)), int(c.get("swa_head_dim", hd))
+ window, dr, ext = int(c.get("sliding_window_size", 512)), int(c.get("d_rel", 16)), int(c.get("rel_extent", 1024))
+ taps = int(c.get("sconv_kernel_size", c.get("conv_kernel_size", 4)))
+ E, ns = int(c.get("n_routed_experts", 256)), int(c.get("n_shared_experts", 2))
+ vocab = int(c.get("vocab_size", 201024))
+ unpad = int(c.get("unpadded_vocab_size", vocab))
+ if "dense_intermediate_size" in c:
+ dense_inter, inter = int(c["dense_intermediate_size"]), int(c.get("intermediate_size", 3072))
+ else:
+ dense_inter, inter = int(c.get("intermediate_size", 24576)), int(c.get("moe_intermediate_size", 3072))
+ except (TypeError, ValueError):
+ return None
+ if L < 1:
+ return None
+ types, ids = c.get("layer_types"), c.get("local_layer_ids")
+ local = []
+ for i in range(L):
+ kind = types[i] if isinstance(types, list) and i < len(types) and isinstance(types[i], str) else None
+ if kind is not None:
+ local.append(kind == "hybrid_sliding")
+ elif isinstance(ids, list):
+ local.append(i in ids)
+ else:
+ local.append((i + 1) % 6 != 0)
+ mlp, first_dense = c.get("mlp_layer_types"), int(c.get("dense_mlp_idx", 0) or 0)
+ sparse = [(mlp[i] == "sparse") if isinstance(mlp, list) and i < len(mlp) and isinstance(mlp[i], str)
+ else i >= first_dense for i in range(L)]
+ # the dense-int4g64 container beside the snapshot (load_w_quant): a byte tensor (U8 or
+ # I8) with its .qs scales replaces the original, int8 rows when it has as many bytes as
+ # the original has weights and few scales, int4-g64 when about half as many; any other
+ # geometry keeps the original
+ container = {}
+ sidecar = Path(info.get("path") or ".") / "dense-int4g64"
+ for shard in sorted(sidecar.glob("*.safetensors")) if sidecar.is_dir() else []:
+ try:
+ for name, size, dtype, _ in _tensor_sizes(shard, with_shape=True):
+ container[name] = (size, dtype)
+ except (OSError, ValueError):
+ return None
+ main = {}
+ for t in info.get("dense_tensors", []):
+ numel = 1
+ for n in t.get("shape") or [0]:
+ numel *= n
+ main[t["name"]] = (t["dtype"], numel)
+
+ def form(name):
+ dtype, numel = main.get(name, (None, 0))
+ q, s = container.get(name), container.get(name + ".qs")
+ if q and s and q[1] in ("U8", "I8"):
+ scales = s[0] // 4
+ if q[0] == numel and scales * 64 < q[0]:
+ return 1, 0
+ if numel <= 2 * q[0] <= numel + 2 * scales:
+ return 4, 64
+ return {"F32": (10, 0), "F16": (10, 0), "BF16": (11, 0)}.get(dtype)
+
+ layers = []
+ for i in range(L):
+ H, KV, d = (sheads, skvh, shd) if local[i] else (heads, kvh, hd)
+ kvo, p = KV * d, f"model.layers.{i}."
+ mats = [("self_attn.q_proj.weight", D, H * d), ("self_attn.k_proj.weight", D, kvo),
+ ("self_attn.v_proj.weight", D, kvo), ("self_attn.r_proj.weight", D, H * dr),
+ ("self_attn.o_proj.weight", H * d, D)]
+ if not sparse[i]:
+ mats += [("mlp.gate_proj.weight", D, dense_inter), ("mlp.up_proj.weight", D, dense_inter),
+ ("mlp.down_proj.weight", dense_inter, D)]
+ total = 0
+ for suffix, columns, rows in mats:
+ f = form(p + suffix)
+ if f is None:
+ return None # a form the shaders do not take: the chain declines, N = L as before
+ total += vk_tensor_bytes(f[0], columns, rows, f[1])
+ if sparse[i]:
+ total += vk_tensor_bytes(10, D, E + ns) # the router's f32 copy
+ for suffix, columns, rows in (("mlp.shared_experts.gate_proj", D, inter),
+ ("mlp.shared_experts.up_proj", D, inter),
+ ("mlp.shared_experts.down_proj", inter, D)):
+ f = form(p + suffix)
+ if f is None:
+ return None
+ total += ns * vk_tensor_bytes(f[0], columns, rows, f[1])
+ total += 4 * (2 * D + 2 * d + dr * (window if local[i] else ext) + 2 * kvo * taps + 2 * D * taps)
+ for bank in range(4):
+ cells = (kvo if bank < 2 else D) * (taps - 1)
+ total += vk_buf_bytes(4 * max(cells, 1)) + 4 * cells
+ total += 2 * vk_buf_bytes(4 * KV * max(window, 1) * d)
+ layers.append(total)
+ # inkc_bufs at R rows: the scratch's geometry over every layer and the frames (a MoE
+ # layer ends one)
+ R = _vk_fit_rows(env, 256)
+ kvo_max = max((skvh * shd if local[i] else kvh * hd) for i in range(L))
+ qo_max = max((sheads * shd if local[i] else heads * hd) for i in range(L))
+ ro_max = max((sheads if local[i] else heads) * dr for i in range(L))
+ mi_max = max((inter if sparse[i] else dense_inter) for i in range(L))
+ nslot, frame = 0, 0
+ for i in range(L):
+ frame += 1
+ nslot = max(nslot, frame)
+ if sparse[i]:
+ frame = 0
+ nsl, ET = max(ns, 1), E + ns
+ vs = (R * kvo_max + 63) & ~63
+ counts = [R * D, R * D, R * D, R * qo_max, vs + R * kvo_max, R * ro_max, R * qo_max, R * D, R * ET,
+ R * mi_max, R * mi_max, nsl * R * D, R * D, D, R * D, R * ET, nslot * 2 * R * kvo_max, 0,
+ unpad, R * D, nsl * R, 2 * R]
+ fixed = sum(vk_buf_bytes(4 * max(n, 1)) for n in counts) + vk_buf_bytes(4 * R * D) + vk_buf_bytes(4 * D)
+ head = form("lm_head.weight")
+ if head is None:
+ return None
+ return VkChainLayout(layers, fixed, vk_tensor_bytes(head[0], D, unpad, head[1]))
+
+
+_VK_CHAIN_LAYOUT["inkling"] = _inkling_chain_layout
+
+
+# the tower's matrices vk_dense_upload places (mimo_vision.h): the patch embedding (a
+# Conv3d read as a Linear over the flattened patch), each block's, the merger's two
+_MIMO_TOWER = re.compile(r"visual\.(?:patch_embed\.proj|blocks\.\d+\.(?:attn\.(?:qkv|proj)|mlp\.(?:gate|up|down)_proj)"
+ r"|merger\.mlp\.[02])\.weight")
+
+
+def _mimo_chain_layout(info, env, vulkan):
+ """mimo.c mc_fit_start: each layer's matrices in their MIMO_DENSE_BITS form (0: the
+ release's FP8 with 128-column block scales, fmt 12, and BF16, fmt 11; 8: int8 rows,
+ fmt 1; 32: f32, fmt 10), its K/V mirror at the size the chain allocates (a sliding
+ layer's ring, a full layer's context or COLI_VK_KV_DEVICE_ROWS when that is fewer) and
+ its norms and sink logits; fixed, mc_scratch at vkc_fit_rows(MIMO_CHUNK) rows and the
+ final norm; the tail, the head (and the vision tower the per-matrix path takes)."""
+ c = info.get("config") or {}
+ try:
+ L, H, V = int(c["num_hidden_layers"]), int(c["hidden_size"]), int(c["vocab_size"])
+ dense_inter = int(c.get("intermediate_size") or 0)
+ heads = [int(c["num_attention_heads"]), 0]
+ kvh = [int(c["num_key_value_heads"]), 0]
+ hd = [int(c["head_dim"]), 0]
+ vd = [int(c.get("v_head_dim", hd[0])), 0]
+ heads[1] = int(c.get("swa_num_attention_heads", heads[0]))
+ kvh[1] = int(c.get("swa_num_key_value_heads", kvh[0]))
+ hd[1] = int(c.get("swa_head_dim", hd[0]))
+ vd[1] = int(c.get("swa_v_head_dim", vd[0]))
+ window = int(c.get("sliding_window") or 0)
+ prf = float(c.get("partial_rotary_factor", 1.0))
+ max_pos = int(c.get("max_position_embeddings", 32768))
+ except (KeyError, TypeError, ValueError):
+ return None
+ if L < 1 or max(hd + vd) > 256:
+ return None
+ pattern, freq = c.get("hybrid_layer_pattern"), c.get("moe_layer_freq")
+ if not isinstance(pattern, list) or not isinstance(freq, list) or len(pattern) < L or len(freq) < L:
+ return None
+ swa = [pattern[i] == 1 for i in range(L)]
+ moe = [freq[i] != 0 for i in range(L)]
+ sink = [bool(c.get("add_full_attention_sink_bias")), bool(c.get("add_swa_attention_sink_bias"))]
+ bits = _env_number(env, "MIMO_DENSE_BITS", 0)
+
+ def size(fmt_native, columns, rows):
+ if bits == 32:
+ return vk_tensor_bytes(10, columns, rows)
+ if bits == 8:
+ return vk_tensor_bytes(1, columns, rows)
+ return vk_tensor_bytes(fmt_native, columns, rows, 128 if fmt_native == 12 else 0)
+
+ text = env.get("MIMO_CTX") or env.get("CTX") or "" # mimo.c: MIMO_CTX, else CTX, else 8192
+ ctx = _env_number({"v": text}, "v", 0) if text else 8192
+ ctx = max(16, min(ctx, max_pos))
+ forced = _env_number(env, "COLI_VK_KV_DEVICE_ROWS", 0)
+ layers = []
+ for i in range(L):
+ k = 1 if swa[i] else 0
+ kd, vdd, qd = kvh[k] * hd[k], kvh[k] * vd[k], heads[k] * hd[k]
+ total = size(12, H, qd + kd + vdd) + size(11, heads[k] * vd[k], H)
+ if not moe[i]:
+ total += 2 * size(12, H, dense_inter) + size(12, dense_inter, H)
+ rows = min(window, ctx) if swa[i] and window > 0 else ctx
+ if not swa[i] and 0 < forced < rows:
+ rows = forced
+ total += vk_buf_bytes(4 * rows * kd) + vk_buf_bytes(4 * rows * vdd)
+ total += 4 * (2 * H + (heads[k] if sink[k] else 0))
+ layers.append(total)
+ # mc_scratch at R rows, one row of logits
+ R = _vk_fit_rows(env, _env_number(env, "MIMO_CHUNK", 64) if _env_number(env, "MIMO_CHUNK", 0) > 0 else 64)
+ geo = []
+ for i in range(L):
+ k = 1 if swa[i] else 0
+ rows = min(window, ctx) if swa[i] and window > 0 else ctx
+ geo.append((swa[i], kvh[k] * hd[k], kvh[k] * vd[k], heads[k] * hd[k], heads[k] * vd[k], rows))
+ rw = max(qd + kd + vdd for _, kd, vdd, qd, _, _ in geo)
+ kdm = max(g[1] for g in geo)
+ vdm = max(g[2] for g in geo)
+ ctxw = max(g[4] for g in geo)
+ kvd = sum((rows if s and rows < R else R) * (kd + vdd) for s, kd, vdd, _, _, rows in geo)
+ rope = [int(hd[0] * prf) // 2 * 2, int(hd[1] * prf) // 2 * 2]
+ counts = [R * H, R * H, R * H, R * rw, R * kdm, R * vdm, R * ctxw, H, R * H, kvd, V, R * H,
+ R * 2 * (rope[0] // 2 + rope[1] // 2)]
+ windowed = [g for g in geo if g[0]]
+ if windowed:
+ win = max(g[5] for g in windowed) - 1
+ counts += [(win + R) * max(g[1] for g in windowed), (win + R) * max(g[2] for g in windowed)]
+ if not all(moe):
+ counts += [R * dense_inter, R * dense_inter]
+ fixed = sum(vk_buf_bytes(4 * max(n, 1)) for n in counts) + vk_buf_bytes(4 * H)
+ tail = size(11, H, V)
+ _, dense = _vk_chain_dense("mimo", env, vulkan)
+ if dense: # the vision tower, on the device through the per-matrix path
+ for tensor in info.get("dense_tensors", []):
+ shape = tensor.get("shape")
+ if _MIMO_TOWER.fullmatch(tensor["name"]) and isinstance(shape, list) and len(shape) >= 2:
+ columns = 1
+ for n in shape[1:]:
+ columns *= n
+ tail += vk_tensor_bytes(10 if bits == 32 else 11, columns, shape[0])
+ return VkChainLayout(layers, fixed, tail)
+
+
+_VK_CHAIN_LAYOUT["mimo"] = _mimo_chain_layout
+
+
def _analysis_signature(shards, config_path):
parts = [f"v{_ANALYSIS_CACHE_VERSION}"]
st = config_path.stat()
diff --git a/c/tests/test_resource_plan.py b/c/tests/test_resource_plan.py
index f1cf5460..0dafdc50 100644
--- a/c/tests/test_resource_plan.py
+++ b/c/tests/test_resource_plan.py
@@ -2485,6 +2485,238 @@ class VulkanPartialChainDskTest(unittest.TestCase):
self.assertEqual((none["vk_chain_layers"]["on_device"], none["dense_on_device_bytes"]), (0, 0))
+class VulkanPartialChainInklingMimoTest(unittest.TestCase):
+ """inkling's and MiMo's partial chain in the planner: their layouts are the bytes the
+ engines' fits printed for the tiny fixtures on Lavapipe (tests/vulkan_partial_inkling-mimo.sh
+ runs the engines), and the plan credits only the first N layers' host copies."""
+
+ def setUp(self):
+ self.tmp = tempfile.TemporaryDirectory()
+ self.model = Path(self.tmp.name)
+
+ def tearDown(self):
+ self.tmp.cleanup()
+
+ def write_inkling(self, container=False):
+ """tools/make_tiny_inkling.py's geometry (8 layers, the global one at 5, layers 0
+ and 1 dense), f32; container: the dense-int4g64 converter's output beside it
+ (int4-g64 matrices, int8 down projections and lm_head, as the family converts it)."""
+ c = {"architectures": ["InklingForCausalLM"], "model_type": "inkling", "hidden_size": 64,
+ "num_hidden_layers": 8, "vocab_size": 256, "unpadded_vocab_size": 250,
+ "num_attention_heads": 4, "num_key_value_heads": 2, "head_dim": 16,
+ "swa_num_attention_heads": 4, "swa_num_key_value_heads": 4, "swa_head_dim": 16,
+ "sliding_window_size": 8, "d_rel": 8, "rel_extent": 32, "conv_kernel_size": 4,
+ "layer_types": ["hybrid_sliding"] * 5 + ["hybrid"] + ["hybrid_sliding"] * 2,
+ "mlp_layer_types": ["dense"] * 2 + ["sparse"] * 6, "intermediate_size": 96,
+ "moe_intermediate_size": 32, "n_routed_experts": 8, "n_shared_experts": 2,
+ "num_experts_per_tok": 2, "rms_norm_eps": 1e-06, "max_position_embeddings": 4096}
+ (self.model / "config.json").write_text(json.dumps(c))
+ main, quant = [], []
+
+ def add(name, *shape, q=None):
+ count = 1
+ for n in shape:
+ count *= n
+ main.append((name, 4 * count, "F32", list(shape)))
+ if q == "int4": # one f32 scale per 64 columns of a row
+ quant.append((name, count // 2, "U8", [count // shape[-1], shape[-1] // 2]))
+ quant.append((name + ".qs", 4 * (count // 64 or 1), "F32", [count // shape[-1], max(shape[-1] // 64, 1)]))
+ elif q == "int8":
+ quant.append((name, count, "I8", list(shape)))
+ quant.append((name + ".qs", 4 * shape[0], "F32", [shape[0]]))
+ add("model.embed_tokens.weight", 256, 64)
+ add("model.norm.weight", 64)
+ add("lm_head.weight", 256, 64, q="int8")
+ for i in range(8):
+ p = f"model.layers.{i}."
+ kv = 64 if i != 5 else 32
+ for name in ("input_layernorm", "post_attention_layernorm"):
+ add(p + name + ".weight", 64)
+ for name, rows in (("q_proj", 64), ("k_proj", kv), ("v_proj", kv), ("r_proj", 32), ("o_proj", 64)):
+ add(p + "self_attn." + name + ".weight", rows, 64, q="int4")
+ add(p + "self_attn.q_norm.weight", 16)
+ add(p + "self_attn.k_norm.weight", 16)
+ add(p + "self_attn.rel_logits_proj.proj", 8, 8)
+ for name, width in (("self_attn.k_sconv", kv), ("self_attn.v_sconv", kv), ("attn_sconv", 64), ("mlp_sconv", 64)):
+ add(p + name + ".conv1d.weight", width, 1, 4)
+ if i < 2:
+ add(p + "mlp.gate_proj.weight", 96, 64, q="int4")
+ add(p + "mlp.up_proj.weight", 96, 64, q="int4")
+ add(p + "mlp.down_proj.weight", 64, 96, q="int8")
+ add(p + "mlp.global_scale", 1)
+ else:
+ add(p + "mlp.gate.weight", 10, 64)
+ add(p + "mlp.gate.e_score_correction_bias", 8)
+ add(p + "mlp.gate.global_scale", 1)
+ add(p + "mlp.shared_experts.gate_proj", 2, 32, 64, q="int4")
+ add(p + "mlp.shared_experts.up_proj", 2, 32, 64, q="int4")
+ add(p + "mlp.shared_experts.down_proj", 2, 64, 32)
+ add(p + "mlp.experts.gate_up_proj", 8, 64, 64)
+ add(p + "mlp.experts.down_proj", 8, 64, 32)
+ write_shard(self.model / "model.safetensors", main)
+ if container:
+ (self.model / "dense-int4g64").mkdir()
+ write_shard(self.model / "dense-int4g64" / "dense.safetensors", quant)
+ return analyze_model(self.model)
+
+ def write_mimo(self):
+ """tools/make_mimo_tiny.py's geometry: 6 layers (full attention at 0 and 3, sliding
+ windows of 8 elsewhere, the dense layer 0), the release's FP8 and BF16 forms, the
+ MXFP4 experts and the vision tower."""
+ c = {"architectures": ["MiMoV2ForCausalLM"], "model_type": "mimo_v2", "vocab_size": 320,
+ "hidden_size": 256, "intermediate_size": 256, "num_hidden_layers": 6,
+ "hybrid_layer_pattern": [0, 1, 1, 0, 1, 1], "moe_layer_freq": [0, 1, 1, 1, 1, 1],
+ "num_attention_heads": 4, "num_key_value_heads": 2, "head_dim": 48, "v_head_dim": 32,
+ "swa_num_attention_heads": 4, "swa_num_key_value_heads": 4, "swa_head_dim": 48,
+ "swa_v_head_dim": 32, "sliding_window": 8, "add_full_attention_sink_bias": False,
+ "add_swa_attention_sink_bias": True, "attention_value_scale": 0.707,
+ "partial_rotary_factor": 0.334, "rope_theta": 10000000.0, "swa_rope_theta": 10000.0,
+ "n_routed_experts": 16, "num_experts_per_tok": 4, "moe_intermediate_size": 64,
+ "layernorm_epsilon": 1e-06, "max_position_embeddings": 256, "image_token_id": 300,
+ "quantization_config": {"quant_method": "fp8", "store_dtype": "mxfp4", "mxfp4_block_size": 32,
+ "weight_block_size": [128, 128]},
+ "vision_config": {"depth": 6, "hidden_size": 64, "intermediate_size": 96, "num_heads": 4,
+ "num_key_value_heads": 2, "qk_channels": 16, "out_hidden_size": 256,
+ "patch_size": 16, "spatial_patch_size": 16, "temporal_patch_size": 2,
+ "spatial_merge_size": 2, "in_chans": 3}}
+ (self.model / "config.json").write_text(json.dumps(c))
+ width = {"BF16": 2, "F32": 4, "F8_E4M3": 1, "U8": 1}
+ tensors = []
+
+ def add(name, dtype, *shape):
+ count = 1
+ for n in shape:
+ count *= n
+ tensors.append((name, count * width[dtype], dtype, list(shape)))
+
+ def fp8(name, rows, cols):
+ add(name + ".weight", "F8_E4M3", rows, cols)
+ add(name + ".weight_scale_inv", "F32", -(-rows // 128), -(-cols // 128))
+ add("model.embed_tokens.weight", "BF16", 320, 256)
+ add("lm_head.weight", "BF16", 320, 256)
+ add("model.norm.weight", "BF16", 256)
+ for i in range(6):
+ p, swa = f"model.layers.{i}.", c["hybrid_layer_pattern"][i]
+ add(p + "input_layernorm.weight", "BF16", 256)
+ add(p + "post_attention_layernorm.weight", "BF16", 256)
+ fp8(p + "self_attn.qkv_proj", 352 if swa else 224, 256)
+ add(p + "self_attn.o_proj.weight", "BF16", 256, 128)
+ if swa:
+ add(p + "self_attn.attention_sink_bias", "BF16", 4)
+ if i == 0:
+ for name, rows, cols in (("gate_proj", 256, 256), ("up_proj", 256, 256), ("down_proj", 256, 256)):
+ fp8(p + "mlp." + name, rows, cols)
+ continue
+ add(p + "mlp.gate.weight", "BF16", 16, 256)
+ add(p + "mlp.gate.e_score_correction_bias", "F32", 16)
+ for e in range(16):
+ for name, rows, cols in (("gate_proj", 64, 128), ("up_proj", 64, 128), ("down_proj", 256, 32)):
+ add(p + f"mlp.experts.{e}.{name}.weight", "U8", rows, cols)
+ add(p + f"mlp.experts.{e}.{name}.weight_scale", "U8", rows, cols // 16)
+ add("visual.patch_embed.proj.weight", "BF16", 64, 3, 2, 16, 16)
+ for b in range(6):
+ p = f"visual.blocks.{b}."
+ for name, rows, cols in (("attn.qkv", 128, 64), ("attn.proj", 64, 64), ("mlp.gate_proj", 96, 64),
+ ("mlp.up_proj", 96, 64), ("mlp.down_proj", 64, 96)):
+ add(p + name + ".weight", "BF16", rows, cols)
+ add(p + name + ".bias", "BF16", rows)
+ add(p + "norm1.weight", "BF16", 64)
+ add(p + "norm2.weight", "BF16", 64)
+ add("visual.merger.ln_q.weight", "BF16", 64)
+ add("visual.merger.mlp.0.weight", "BF16", 256, 256)
+ add("visual.merger.mlp.2.weight", "BF16", 256, 256)
+ write_shard(self.model / "model.safetensors", tensors)
+ return analyze_model(self.model)
+
+ def test_inkling_layout_is_the_engines(self):
+ # "[VK] inkling chain fit: ... fixed X B (the engine's 1566464 B, ...), tail 64256 B,
+ # layers 164736 164736 143744 ..." on tiny_inkling (f32), and with the dense-int4g64
+ # container (tiny_inkling_q): its int4-g64 and int8 forms; lm_head's int8 rows carry
+ # one scale a row of the padded vocabulary, a geometry inkling.c does not take, so
+ # the head stays f32 there
+ from resource_plan import vk_chain_fit, vk_fit_pools
+ on = {"COLI_VULKAN": "1", "COLI_VK_CHAIN": "1"}
+ lavapipe = {"type": "cpu", "budget_bytes": 64 * GB}
+ for container, layers in ((False, [164736, 164736, 143744, 143744, 143744, 123776, 143744, 143744]),
+ (True, [39296, 39296, 50560, 50560, 50560, 44928, 50560, 50560])):
+ with self.subTest(container=container):
+ self.tearDown(); self.setUp()
+ info = self.write_inkling(container)
+ fit = vk_chain_fit(info, "inkling", on, lavapipe)
+ self.assertEqual(fit["layers"], layers)
+ self.assertEqual(fit["fixed"] - vk_fit_pools(0), 1566464)
+ self.assertEqual((fit["n"], fit["tail"]), (8, True))
+ small = vk_chain_fit(info, "inkling", dict(on, COLI_VK_CHAIN_ROWS="3"), lavapipe)
+ self.assertLess(small["fixed"], fit["fixed"])
+ self.assertEqual(small["layers"], layers)
+ # a dtype the shaders do not take: no layout, the plan as before
+ self.tearDown(); self.setUp()
+ info = self.write_inkling()
+ info["dense_tensors"] = [dict(t, dtype="F8_E4M3") if t["name"].endswith("q_proj.weight") else t
+ for t in info["dense_tensors"]]
+ self.assertIsNone(vk_chain_fit(info, "inkling", on, lavapipe))
+
+ def test_mimo_layout_is_the_engines(self):
+ # the "[VK] mimo chain fit:" lines of mimo_tiny on Lavapipe, per MIMO_DENSE_BITS, and
+ # with the vision tower on the per-matrix path (COLI_VK_DENSE=1: the tail)
+ from resource_plan import vk_chain_fit, vk_fit_pools
+ info = self.write_mimo()
+ lavapipe = {"type": "cpu", "budget_bytes": 64 * GB}
+ cases = (("32", [1445120, 668176, 668176, 657920, 668176, 668176], 327936, None),
+ ("0", [527360, 213264, 213264, 324608, 213264, 213264], 164096, 999936),
+ ("8", [491008, 179216, 179216, 291328, 179216, 179216], 83200, 919040))
+ for bits, layers, tail, tower in cases:
+ with self.subTest(bits=bits):
+ env = {"COLI_VULKAN": "1", "COLI_VK_CHAIN": "1", "MIMO_DENSE_BITS": bits}
+ fit = vk_chain_fit(info, "mimo", env, lavapipe)
+ self.assertEqual(fit["layers"], layers)
+ self.assertEqual(fit["fixed"] - vk_fit_pools(0), 929792)
+ self.assertEqual(fit["n"], 6)
+ layout = __import__("resource_plan")._VK_CHAIN_LAYOUT["mimo"](info, env, lavapipe)
+ self.assertEqual(layout.tail, tail)
+ if tower:
+ dense = __import__("resource_plan")._VK_CHAIN_LAYOUT["mimo"](info, dict(env, COLI_VK_DENSE="1"), lavapipe)
+ self.assertEqual(dense.tail, tower)
+ # COLI_VK_KV_DEVICE_ROWS shrinks the full layers' mirrors (0 and 3), not the rings
+ env = {"COLI_VULKAN": "1", "COLI_VK_CHAIN": "1", "MIMO_DENSE_BITS": "32"}
+ fewer = vk_chain_fit(info, "mimo", dict(env, COLI_VK_KV_DEVICE_ROWS="16"), lavapipe)["layers"]
+ full = vk_chain_fit(info, "mimo", env, lavapipe)["layers"]
+ self.assertEqual([a < b for a, b in zip(fewer, full)], [True, False, False, True, False, False])
+
+ def test_mimo_credits_the_layers_on_the_device(self):
+ from resource_plan import vk_chain_fit
+ info = self.write_mimo()
+ kwargs = dict(context=64, ram_gb=4, available_memory=16 * GB, available_disk=16 * GB, gpus=[])
+ on = {"COLI_VULKAN": "1", "COLI_VK_CHAIN": "1", "COLI_VK_DENSE_HOST": "0", "COLI_VK_TIER_RESERVE_GB": "0"}
+ dgpu = {"type": "discrete", "budget_bytes": 64 * GB}
+
+ def droppable(k, head): # mimo's dho pass: qkv, o_proj, the dense MLP, the head
+ total = 0
+ for t in info["dense_tensors"]:
+ name, shape = t["name"], t["shape"]
+ if len(shape) != 2 or name.endswith(("_scale_inv", ".bias")) or ".gate." in name:
+ continue
+ if name == "lm_head.weight":
+ total += min(t["resident"], shape[0] * shape[1]) if head else 0
+ elif name.startswith("model.layers.") and int(name.split(".")[2]) < k:
+ total += min(t["resident"], shape[0] * shape[1])
+ return total
+ full = build_plan(self.model, env=on, vulkan=dgpu, **kwargs)["tiers"]["ram"]
+ self.assertEqual(full["vk_chain_layers"]["on_device"], 6)
+ self.assertEqual(full["dense_on_device_bytes"], droppable(6, True))
+ fit = vk_chain_fit(info, "mimo", on, dgpu)
+ for k in (1, 3, 5):
+ with self.subTest(k=k):
+ free = fit["fixed"] + sum(fit["layers"][:k]) + fit["layers"][k] // 2
+ ram = build_plan(self.model, env=on, vulkan={"type": "discrete", "budget_bytes": free}, **kwargs)["tiers"]["ram"]
+ self.assertEqual(ram["vk_chain_layers"]["on_device"], k)
+ self.assertEqual(ram["dense_on_device_bytes"], droppable(k, False))
+ forced = build_plan(self.model, env=dict(on, COLI_VK_CHAIN_LAYERS=str(k)), vulkan=dgpu, **kwargs)
+ self.assertEqual(forced["tiers"]["ram"]["dense_on_device_bytes"], droppable(k, False))
+ none = build_plan(self.model, env=dict(on, COLI_VK_CHAIN_LAYERS="0"), vulkan=dgpu, **kwargs)["tiers"]["ram"]
+ self.assertEqual(none["dense_on_device_bytes"], 0)
+
+
class PhysicalCpuCountTest(unittest.TestCase):
"""Regression for #325: --auto-tier pinned decode to one core because
physical_cpu_count() silently returned 1.