Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
#!/usr/bin/env python3
"""Generate the tiny MiMo-V2.6 checkpoint used by tests/mimo_tiny_harness.py.
numpy-only and deterministic: CI regenerates the shards from SEED and
byte-identical weights come out on every platform. The reference tokens in
ref.json are NOT produced here: they come from Xiaomi's own modeling code
(tools/make_mimo_ref.py, run offline once, vendor files pinned by SHA-256),
so the engine and its oracle never share an author.
The fixture stores every tensor the way the released checkpoint does, because
the storage is where a reading of MiMo goes wrong while a float reference
still agrees:
- routed experts are MXFP4 g32 (e2m1 nibble pairs, low nibble = even column,
one e8m0 byte per 32 columns), the six tensors of an expert back to back
in the file order of the real shards: down, down scale, gate, gate scale,
up, up scale;
- qkv_proj and the layer-0 MLP are FP8 e4m3 with a F32 weight_scale_inv per
128x128 block;
- the fused qkv_proj is PRE-SHARDED in num_key_value_heads chunks
[Q_0|K_0|V_0|Q_1|K_1|V_1] with the scale grid tiled per chunk. The full
layers' chunk (176 rows) is not a whole number of blocks, so the per-chunk
grid has 4 rows where a contiguous one would have 3: a reader that assumes
[Q|K|V] with one continuous grid fails here, as it would on the real
checkpoint (108 scale rows, not 106, on a Flash full-attention layer);
- o_proj, router, norms, sinks, embed and lm_head are BF16.
The geometry crosses every path of mimo.c: full-attention and sliding-window
layers with different KV head counts (2 and 4 heads over 2 chunks), a window
of 8 that every case outgrows, the SWA sink logit, partial RoPE with the two
thetas, the value scale, the dense layer 0 and MoE layers.
No tokenizer.json on purpose: without one the engine prints raw generated
token ids on stdout, which is what the token-exact test parses. Greedy
determinism is engineered, then verified by make_mimo_ref.py: the head is
random at a large scale (wide top-2 gaps), the router's top-k boundary is
checked for near-ties at every token and layer, and the layers are checked to
matter (the argmax must differ from the embedding-only one at most positions).
"""
import argparse
import json
import struct
import sys
from pathlib import Path
import numpy as np
SEED = 20260930
VOCAB = 320
HIDDEN = 256
LAYERS = 6
PATTERN = [0, 1, 1, 0, 1, 1] # 1 = sliding window, 0 = full attention
MOE_FREQ = [0, 1, 1, 1, 1, 1] # layer 0 dense, like the release
DENSE_INTER = 256
HEADS = 4 # full attention
KV_HEADS = 2 # also the qkv pre-shard count (ckpt_tp)
SWA_HEADS = 4
SWA_KV_HEADS = 4 # two per chunk, as on the release
HEAD_DIM = 48 # rope_dim = int(48 * 0.334) = 16
V_HEAD_DIM = 32
EXPERTS = 16
TOPK = 4
MOE_INTER = 64 # MXFP4 needs multiples of 32
WINDOW = 8
EOS_ID = 1
BLOCK = 128
EMBED_S = 0.05 # embedding scale
LM_S = 4.0 # head scale
EXP_LO, EXP_HI = 121, 126 # MXFP4 e8m0 exponent range
CONFIG = {
"architectures": ["MiMoV2ForCausalLM"],
"model_type": "mimo_v2",
"vocab_size": VOCAB,
"hidden_size": HIDDEN,
"intermediate_size": DENSE_INTER,
"num_hidden_layers": LAYERS,
"hybrid_layer_pattern": PATTERN,
"moe_layer_freq": MOE_FREQ,
"num_attention_heads": HEADS,
"num_key_value_heads": KV_HEADS,
"head_dim": HEAD_DIM,
"v_head_dim": V_HEAD_DIM,
"swa_num_attention_heads": SWA_HEADS,
"swa_num_key_value_heads": SWA_KV_HEADS,
"swa_head_dim": HEAD_DIM,
"swa_v_head_dim": V_HEAD_DIM,
"sliding_window": WINDOW,
"sliding_window_size": WINDOW,
"attention_chunk_size": WINDOW,
"add_full_attention_sink_bias": False,
"add_swa_attention_sink_bias": True,
"attention_value_scale": 0.707,
"attention_projection_layout": "fused_qkv",
"attention_bias": False,
"partial_rotary_factor": 0.334,
"rope_theta": 10000000.0,
"swa_rope_theta": 10000.0,
"rope_parameters": {"partial_rotary_factor": 0.334, "rope_theta": 10000000.0,
"rope_type": "default", "type": "default"},
"n_routed_experts": EXPERTS,
"num_experts_per_tok": TOPK,
"moe_intermediate_size": MOE_INTER,
"n_shared_experts": None,
"n_group": 1,
"topk_group": 1,
"topk_method": "noaux_tc",
"scoring_func": "sigmoid",
"norm_topk_prob": True,
"routed_scaling_factor": None,
"moe_router_dtype": "bfloat16",
"hidden_act": "silu",
"layernorm_epsilon": 1e-6,
"max_position_embeddings": 256,
"tie_word_embeddings": False,
"bos_token_id": None,
"eos_token_id": EOS_ID,
"pad_token_id": 0,
"dtype": "bfloat16",
"quantization_config": {
"activation_scheme": "dynamic",
"fmt": "e4m3",
"quant_method": "fp8",
"store_dtype": "mxfp4",
"mxfp4_block_size": 32,
"weight_block_size": [BLOCK, BLOCK],
"ignored_layers": [f"model.layers.{i}.self_attn.o_proj"
for i in range(LAYERS)],
},
}
# --vision: a tiny MiMo ViT on the same language model. Six blocks cross every
# path of the tower: full attention at both ends (no sink), row-order and
# column-order windows, and the permutations entering and leaving a column run
# (types -1, 0, 1, 1, 0, -1; the release's pattern is the same alphabet).
VISION = {
"depth": 6,
"hidden_size": 64,
"intermediate_size": 96,
"num_heads": 4,
"num_key_value_heads": 2,
"qk_channels": 16,
"out_hidden_size": HIDDEN,
"patch_size": 16,
"spatial_patch_size": 16,
"temporal_patch_size": 2,
"spatial_merge_size": 2,
"in_chans": 3,
"hidden_act": "silu",
"fullatt_block_indexes": [0, 5],
"vit_window_attn_types": [-1, 0, 1, 1, 0, -1],
"visual_token_window_size": 5,
"use_sink": True,
"window_size": 128,
"tokens_per_second": 2,
}
IMAGE_TOKEN, VISION_START, VISION_END = 300, 301, 302
IMAGE_W, IMAGE_H = 192, 128 # already a multiple of 32: no resize
PREPROCESSOR = {
"min_pixels": 3136,
"max_pixels": 12845056,
"patch_size": 16,
"temporal_patch_size": 2,
"merge_size": 2,
"image_mean": [0.48145466, 0.4578275, 0.40821073],
"image_std": [0.26862954, 0.26130258, 0.27577711],
"image_processor_type": "Qwen2VLImageProcessor",
"processor_class": "Qwen2_5_VLProcessor",
}
def bf16_bits(x):
"""f32 -> bf16 bit patterns, round to nearest even."""
bits = np.ascontiguousarray(x, dtype=np.float32).view(np.uint32)
rounding = ((bits >> 16) & 1) + np.uint32(0x7FFF)
return ((bits + rounding) >> 16).astype(np.uint16)
class BF16(np.ndarray):
"""uint16 bit patterns tagged for the writer as BF16."""
class F8(np.ndarray):
"""uint8 e4m3fn codes tagged for the writer as F8_E4M3."""
def tag(arr, cls):
return np.ascontiguousarray(arr).view(cls)
def st_dtype(arr):
if isinstance(arr, BF16):
return "BF16"
if isinstance(arr, F8):
return "F8_E4M3"
return {np.dtype("float32"): "F32", np.dtype("uint8"): "U8"}[arr.dtype]
def build_tensors(rng):
"""Every weight of the tiny model, in emission order, as {name: array}."""
tensors = {}
def put(name, arr):
assert name not in tensors, name
tensors[name] = arr
def normal(*shape, s=0.02):
return (rng.standard_normal(shape) * s).astype(np.float32)
def bf16(x):
return tag(bf16_bits(x), BF16)
def norm_weight(n):
return bf16(1.0 + normal(n, s=0.1))
def fp8_codes(rows, cols):
"""Random finite e4m3fn codes with exponent fields 4..9 (|w| roughly
1/8 .. 7.5 before the block scale); never the 0x7F/0xFF NaN codes."""
sign = rng.integers(0, 2, size=(rows, cols), dtype=np.uint8) << 7
exp = rng.integers(4, 10, size=(rows, cols), dtype=np.uint8) << 3
man = rng.integers(0, 8, size=(rows, cols), dtype=np.uint8)
return sign | exp | man
def fp8_scales(rows, cols):
return (rng.uniform(0.004, 0.012, size=(rows, cols))).astype(np.float32)
def fp8_matrix(name, out, inp):
put(name + ".weight", tag(fp8_codes(out, inp), F8))
put(name + ".weight_scale_inv",
fp8_scales(-(-out // BLOCK), -(-inp // BLOCK)))
def fp8_qkv(name, heads, kv_heads):
"""Pre-sharded fused qkv: KV_HEADS chunks of [Q_c|K_c|V_c] rows, the
scale grid tiled per chunk (ceil(rows_per_chunk / 128) rows each)."""
chunks = KV_HEADS
rows_per_chunk = ((heads // chunks) * HEAD_DIM
+ (kv_heads // chunks) * HEAD_DIM
+ (kv_heads // chunks) * V_HEAD_DIM)
put(name + ".weight",
tag(fp8_codes(chunks * rows_per_chunk, HIDDEN), F8))
put(name + ".weight_scale_inv",
fp8_scales(chunks * -(-rows_per_chunk // BLOCK),
-(-HIDDEN // BLOCK)))
def mxfp4(rows, cols):
"""One packed MXFP4 matrix. All 16 e2m1 codes are finite; exponents
stay near 127 so dequantized magnitudes stay tame, never 0xFF."""
packed = rng.integers(0, 256, size=(rows, cols // 2), dtype=np.uint8)
scale = rng.integers(EXP_LO, EXP_HI, size=(rows, cols // 32), dtype=np.uint8)
return packed, scale
embed = normal(VOCAB, HIDDEN, s=EMBED_S)
put("model.embed_tokens.weight", bf16(embed))
# A random head at a large scale: logits spread over ~LM_S * 16, so the
# top-2 gap clears float noise by orders of magnitude, while the choice
# still depends on the whole residual stream (make_mimo_ref.py checks that
# the layers change the argmax, not just the embedding). EOS is a zero
# row: never the argmax, so a fixed-length run detects early truncation.
lm_head = normal(VOCAB, HIDDEN, s=LM_S)
lm_head[EOS_ID] = 0.0
put("model.norm.weight", norm_weight(HIDDEN))
put("lm_head.weight", bf16(lm_head))
for i in range(LAYERS):
p = f"model.layers.{i}."
swa = PATTERN[i] == 1
heads = SWA_HEADS if swa else HEADS
kv_heads = SWA_KV_HEADS if swa else KV_HEADS
put(p + "input_layernorm.weight", norm_weight(HIDDEN))
put(p + "post_attention_layernorm.weight", norm_weight(HIDDEN))
fp8_qkv(p + "self_attn.qkv_proj", heads, kv_heads)
put(p + "self_attn.o_proj.weight",
bf16(normal(HIDDEN, heads * V_HEAD_DIM, s=0.05)))
if swa:
put(p + "self_attn.attention_sink_bias", bf16(normal(heads, s=1.0)))
if not MOE_FREQ[i]:
fp8_matrix(p + "mlp.gate_proj", DENSE_INTER, HIDDEN)
fp8_matrix(p + "mlp.up_proj", DENSE_INTER, HIDDEN)
fp8_matrix(p + "mlp.down_proj", HIDDEN, DENSE_INTER)
continue
# closed-form router: wide, stable expert preferences per hidden
# direction, so top-k selection cannot ride a near-tie
gate = 0.08 * np.sin(np.arange(EXPERTS * HIDDEN, dtype=np.float32)
* 0.013 + 0.7 * i).reshape(EXPERTS, HIDDEN)
put(p + "mlp.gate.weight", bf16(gate))
put(p + "mlp.gate.e_score_correction_bias",
np.linspace(-0.02, 0.02, EXPERTS, dtype=np.float32))
for e in range(EXPERTS):
q = p + f"mlp.experts.{e}."
down, down_s = mxfp4(HIDDEN, MOE_INTER)
gate_w, gate_s = mxfp4(MOE_INTER, HIDDEN)
up, up_s = mxfp4(MOE_INTER, HIDDEN)
# the file order of the release, back to back
put(q + "down_proj.weight", down)
put(q + "down_proj.weight_scale", down_s)
put(q + "gate_proj.weight", gate_w)
put(q + "gate_proj.weight_scale", gate_s)
put(q + "up_proj.weight", up)
put(q + "up_proj.weight_scale", up_s)
return tensors
def build_vision(rng):
"""The tower's tensors, named and shaped as in the release (BF16; the merger
carries no biases and its LayerNorm no bias, as there)."""
v = VISION
d, f, heads, kvh = v["hidden_size"], v["intermediate_size"], v["num_heads"], v["num_key_value_heads"]
hd = v["qk_channels"]
merged = d * v["spatial_merge_size"] ** 2
tensors = {}
def bf16(x):
return tag(bf16_bits(x), BF16)
def normal(*shape, s=0.02):
return (rng.standard_normal(shape) * s).astype(np.float32)
tensors["visual.patch_embed.proj.weight"] = bf16(normal(d, 3, 2, 16, 16, s=0.03))
for i in range(v["depth"]):
p = f"visual.blocks.{i}."
tensors[p + "norm1.weight"] = bf16(1.0 + normal(d, s=0.1))
tensors[p + "norm2.weight"] = bf16(1.0 + normal(d, s=0.1))
tensors[p + "attn.qkv.weight"] = bf16(normal((heads + 2 * kvh) * hd, d, s=0.15))
tensors[p + "attn.qkv.bias"] = bf16(normal((heads + 2 * kvh) * hd, s=0.1))
tensors[p + "attn.proj.weight"] = bf16(normal(d, heads * hd, s=0.1))
tensors[p + "attn.proj.bias"] = bf16(normal(d, s=0.05))
if i not in v["fullatt_block_indexes"]:
tensors[p + "attn.sinks"] = bf16(normal(heads, s=1.0))
for n in ("gate_proj", "up_proj"):
tensors[p + f"mlp.{n}.weight"] = bf16(normal(f, d, s=0.1))
tensors[p + f"mlp.{n}.bias"] = bf16(normal(f, s=0.05))
tensors[p + "mlp.down_proj.weight"] = bf16(normal(d, f, s=0.1))
tensors[p + "mlp.down_proj.bias"] = bf16(normal(d, s=0.05))
tensors["visual.merger.ln_q.weight"] = bf16(1.0 + normal(d, s=0.1))
tensors["visual.merger.mlp.0.weight"] = bf16(normal(merged, merged, s=0.05))
tensors["visual.merger.mlp.2.weight"] = bf16(normal(HIDDEN, merged, s=0.2))
return tensors
def test_image(rng):
"""A deterministic picture with structure at several scales, so the
windows and the merge see different content everywhere."""
y, x = np.mgrid[0:IMAGE_H, 0:IMAGE_W].astype(np.float32)
r = 127 + 120 * np.sin(x / 9.0) * np.cos(y / 13.0)
g = 127 + 120 * np.sin((x + y) / 17.0)
b = (x * 3 + y * 5) % 256
img = np.stack([r, g, b], axis=-1) + rng.integers(0, 24, size=(IMAGE_H, IMAGE_W, 3))
return np.clip(img, 0, 255).astype(np.uint8)
def write_safetensors(path, tensors):
"""Hand-rolled writer: keeps insertion order, which the expert contiguity
the engine exploits depends on (the safetensors library reorders)."""
header, offset = {}, 0
for name, arr in tensors.items():
header[name] = {"dtype": st_dtype(arr), "shape": list(arr.shape),
"data_offsets": [offset, offset + arr.nbytes]}
offset += arr.nbytes
blob = json.dumps(header, separators=(",", ":")).encode()
blob += b" " * ((-len(blob)) % 8)
with open(path, "wb") as fh:
fh.write(struct.pack("<Q", len(blob)))
fh.write(blob)
for arr in tensors.values():
fh.write(np.ascontiguousarray(arr).view(np.uint8).tobytes())
def main():
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--output", default="./mimo_tiny")
parser.add_argument("--force", action="store_true",
help="overwrite existing shards")
parser.add_argument("--vision", action="store_true",
help="add a tiny ViT, the image tokens, a test picture "
"and its patches (the vision fixture)")
args = parser.parse_args()
out = Path(args.output)
out.mkdir(parents=True, exist_ok=True)
shard_names = ["model-00001-of-00002.safetensors",
"model-00002-of-00002.safetensors"]
if any((out / s).exists() for s in shard_names) and not args.force:
print(f"{out} already has shards; use --force to regenerate",
file=sys.stderr)
return 1
rng = np.random.default_rng(SEED)
tensors = build_tensors(rng)
config = dict(CONFIG)
if args.vision:
tensors.update(build_vision(np.random.default_rng(SEED + 1)))
config.update({"vision_config": VISION, "image_token_id": IMAGE_TOKEN,
"vision_start_token_id": VISION_START,
"vision_end_token_id": VISION_END,
"vision_model_type": "mimovl"})
(out / "preprocessor_config.json").write_text(
json.dumps(PREPROCESSOR, indent=2) + "\n")
from PIL import Image
Image.fromarray(test_image(np.random.default_rng(SEED + 2))).save(
out / "image.png")
# the patches exactly as the gateway builds them (tools/qwen38_image.py,
# the Qwen2-VL processor); make_mimo_ref.py checks them bit for bit
# against the official processor
sys.path.insert(0, str(Path(__file__).resolve().parent))
import qwen38_image
patches, grid_h, grid_w = qwen38_image.preprocess(
(out / "image.png").read_bytes(), model_dir=str(out))
patches.tofile(out / "patches.f32")
(out / "image.json").write_text(json.dumps(
{"grid_h": grid_h, "grid_w": grid_w,
"tokens": grid_h * grid_w // 4}) + "\n")
# two shards like the release: the experts of layers 3.. in the second,
# so the loader has to follow the index across files
second = {n: a for n, a in tensors.items()
if ".mlp.experts." in n and int(n.split(".")[2]) >= 3}
first = {n: a for n, a in tensors.items() if n not in second}
weight_map = {}
for shard, part in zip(shard_names, (first, second)):
write_safetensors(out / shard, part)
weight_map.update({n: shard for n in part})
total = sum(a.nbytes for a in tensors.values())
(out / "model.safetensors.index.json").write_text(json.dumps(
{"metadata": {"total_size": total}, "weight_map": weight_map},
indent=1) + "\n")
(out / "config.json").write_text(json.dumps(config, indent=2) + "\n")
print(f"{out}: {len(tensors)} tensors, {total / 1e6:.2f} MB in 2 shards, "
f"{LAYERS} layers (pattern {PATTERN}), {EXPERTS} experts top-{TOPK}, "
f"seed {SEED}")
return 0
if __name__ == "__main__":
sys.exit(main())