Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
21.6 KB
Raw Blame
#!/usr/bin/env python3
"""Produce ref.json for the tiny MiMo-V2.6 fixture from Xiaomi's own code.

Maintainer tool, run offline once per fixture change; CI never executes it
(tests/mimo_tiny_harness.py only reads the committed ref.json).  The reference
tokens come from the vendor's published implementation (modeling_mimo_v2.py on
the MiMo-V2.6-Flash-MOPD Hugging Face repo, pinned below by SHA-256), so the
engine and its oracle never share an author.

Parity is by construction:
- the engine is tested with its dense weights at f32 (ref.json engine_env) and
  float expert matmuls, and this script dequantizes THE SAME BYTES into the
  vendor model's f32 Linear weights: BF16 by widening, FP8 e4m3 through torch's
  own float8_e4m3fn type times the 128x128 block scale, MXFP4 as
  e2m1 * 2^(scale-127) with the low nibble on the even column;
- the fused qkv_proj is stored pre-sharded ([Q_0|K_0|V_0|Q_1|...], scale grid
  tiled per chunk) as on the release, while the vendor module wants [Q|K|V]:
  this script de-interleaves it with the layout vLLM documents for the release
  (vllm/model_executor/models/mimo_v2.py, _shard_fp8_qkv_proj), so a reader
  that gets the layout wrong disagrees with the oracle;
- token-level argmax must then agree wherever the logit margin exceeds float
  noise; the generator engineers wide margins and this script VERIFIES them per
  emitted position (--min-margin, refuse on violation).

Requires: torch (CPU is fine), transformers >= 5.3, numpy.  Network only if the
vendor files are not already in --vendor-dir.
"""

import argparse
import hashlib
import importlib.machinery
import importlib.util
import json
import struct
import sys
import urllib.request
from pathlib import Path

import numpy as np

VENDOR_REPO = "XiaomiMiMo/MiMo-V2.6-Flash-MOPD"
VENDOR_REVISION = "2479e2d0029eca9a34cc7e7f55a121925f81908e"
VENDOR_FILES = {
    "configuration_mimo_v2.py":
        "773062ac9850b908eb54751b3e4dbe00e653c0e80595599409e97d0c1af2ce3e",
    "modeling_mimo_v2.py":
        "a8c3cb3aae473bcc15f023010547c919f15eba6546e6ed7efb61a8937b12f3ad",
}
BLOCK = 128

MX4_LUT = np.array([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0,
                    -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0],
                   dtype=np.float32)


def read_fixture(fixture):
    """{name: (dtype string, raw numpy array)} across the indexed shards."""
    index = json.loads((fixture / "model.safetensors.index.json").read_text())
    raw_dtypes = {"F32": np.float32, "U8": np.uint8, "BF16": np.uint16,
                  "F8_E4M3": np.uint8}
    out = {}
    for shard in sorted(set(index["weight_map"].values())):
        with open(fixture / shard, "rb") as fh:
            header_len = struct.unpack("<Q", fh.read(8))[0]
            header = json.loads(fh.read(header_len))
            raw = fh.read()
        for name, meta in header.items():
            if name == "__metadata__":
                continue
            lo, hi = meta["data_offsets"]
            dtype = raw_dtypes[meta["dtype"]]
            arr = np.frombuffer(raw, dtype=dtype,
                                count=(hi - lo) // np.dtype(dtype).itemsize,
                                offset=lo).reshape(meta["shape"]).copy()
            out[name] = (meta["dtype"], arr)
    return out


def bf16_to_f32(bits):
    return (bits.astype(np.uint32) << 16).view(np.float32)


def fp8_to_f32(codes):
    """e4m3fn decode by torch's own float8 type (not colibri's decoder)."""
    import torch
    return torch.from_numpy(codes).view(torch.float8_e4m3fn).float().numpy()


def mxfp4_dequant(packed, scale):
    rows, half = packed.shape
    cols = half * 2
    codes = np.empty((rows, cols), dtype=np.uint8)
    codes[:, 0::2] = packed & 0x0F
    codes[:, 1::2] = packed >> 4
    values = MX4_LUT[codes]
    exponents = np.ldexp(np.float32(1.0), scale.astype(np.int32) - 127)
    return (values.reshape(rows, cols // 32, 32) *
            exponents[:, :, None]).reshape(rows, cols).astype(np.float32)


def fp8_block_dequant(codes, scale_inv, row_scale_index=None):
    """w[r, c] = fp8(codes[r, c]) * scale_inv[row_block(r), c // 128]."""
    w = fp8_to_f32(codes)
    rows, cols = w.shape
    if row_scale_index is None:
        row_scale_index = np.arange(rows) // BLOCK
    col_block = np.arange(cols) // BLOCK
    return (w * scale_inv[row_scale_index][:, col_block]).astype(np.float32)


def qkv_contiguous(codes, scale_inv, heads, kv_heads, head_dim, v_head_dim,
                   chunks):
    """Pre-sharded fused qkv (vLLM's documented layout) -> [Q|K|V] f32."""
    rows = codes.shape[0]
    per_chunk = rows // chunks
    q_rows = (heads // chunks) * head_dim
    k_rows = (kv_heads // chunks) * head_dim
    v_rows = (kv_heads // chunks) * v_head_dim
    assert q_rows + k_rows + v_rows == per_chunk, (rows, chunks)
    chunk_scale_rows = -(-per_chunk // BLOCK)
    r = np.arange(rows)
    if scale_inv.shape[0] == chunks * chunk_scale_rows:
        index = (r // per_chunk) * chunk_scale_rows + (r % per_chunk) // BLOCK
    elif scale_inv.shape[0] == -(-rows // BLOCK):
        index = r // BLOCK
    else:
        raise SystemExit(f"qkv scale grid {scale_inv.shape} fits neither layout")
    w = fp8_block_dequant(codes, scale_inv, index)
    parts = [w[c * per_chunk:(c + 1) * per_chunk] for c in range(chunks)]
    q = np.concatenate([p[:q_rows] for p in parts])
    k = np.concatenate([p[q_rows:q_rows + k_rows] for p in parts])
    v = np.concatenate([p[q_rows + k_rows:] for p in parts])
    return np.concatenate([q, k, v]).astype(np.float32)


def vision_state(tensors, model):
    """visual.* widened to f32, plus the zeros the release does not store: the
    merger's two Linear biases and its LayerNorm bias (vLLM builds that merger
    with bias=False; the HF module declares them and would otherwise keep its
    random init)."""
    import torch
    state = {}
    for name, (dtype, arr) in tensors.items():
        if name.startswith("visual."):
            state[name] = torch.from_numpy(bf16_to_f32(arr) if dtype == "BF16"
                                           else arr.astype(np.float32))
    for name, param in model.named_parameters():
        if name.startswith("visual.merger.") and name.endswith(".bias") \
                and name not in state:
            state[name] = torch.zeros_like(param)
    return state


def to_state_dict(tensors, config):
    import torch
    chunks = config["num_key_value_heads"]
    state = {}
    consumed = set()
    for name, (dtype, arr) in tensors.items():
        if name in consumed:
            continue
        if name.endswith(".weight_scale_inv") or name.endswith(".weight_scale"):
            continue
        if name.startswith("visual."):
            continue
        if dtype == "BF16":
            value = bf16_to_f32(arr)
        elif dtype == "F32":
            value = arr.astype(np.float32)
        elif name.endswith("qkv_proj.weight"):
            layer = int(name.split(".")[2])
            swa = config["hybrid_layer_pattern"][layer] == 1
            heads = config["swa_num_attention_heads" if swa else "num_attention_heads"]
            kv = config["swa_num_key_value_heads" if swa else "num_key_value_heads"]
            value = qkv_contiguous(arr, tensors[name + "_scale_inv"][1], heads,
                                   kv, config["head_dim"], config["v_head_dim"],
                                   chunks)
        elif dtype == "F8_E4M3":
            value = fp8_block_dequant(arr, tensors[name + "_scale_inv"][1])
        elif dtype == "U8" and ".experts." in name:
            value = mxfp4_dequant(arr, tensors[name + "_scale"][1])
        else:
            raise SystemExit(f"{name}: unexpected {dtype}")
        state[name] = torch.from_numpy(np.ascontiguousarray(value))
    return state


def fetch_vendor(vendor_dir):
    vendor_dir.mkdir(parents=True, exist_ok=True)
    for name, expected in VENDOR_FILES.items():
        path = vendor_dir / name
        if not path.exists():
            url = (f"https://huggingface.co/{VENDOR_REPO}/raw/"
                   f"{VENDOR_REVISION}/{name}")
            print(f"[ref] downloading {url}", file=sys.stderr)
            with urllib.request.urlopen(url, timeout=60) as resp:
                path.write_bytes(resp.read())
        digest = hashlib.sha256(path.read_bytes()).hexdigest()
        if digest != expected:
            raise SystemExit(
                f"{name}: sha256 {digest} does not match the pinned {expected}; "
                f"the vendor file changed. Review it, then update VENDOR_FILES.")


def load_vendor(vendor_dir):
    """The vendor files as a synthetic package, so the relative import of the
    configuration module works whatever the directory is called."""
    pkg = "mimo_v2_vendor"
    spec = importlib.machinery.ModuleSpec(pkg, None, is_package=True)
    package = importlib.util.module_from_spec(spec)
    package.__path__ = [str(vendor_dir)]
    sys.modules[pkg] = package

    def load(stem):
        file_spec = importlib.util.spec_from_file_location(
            f"{pkg}.{stem}", vendor_dir / f"{stem}.py")
        module = importlib.util.module_from_spec(file_spec)
        sys.modules[f"{pkg}.{stem}"] = module
        file_spec.loader.exec_module(module)
        return module

    config_module = load("configuration_mimo_v2")
    modeling = load("modeling_mimo_v2")
    shim_mask_builders(modeling)
    return config_module, modeling


def shim_mask_builders(modeling):
    """The vendor file targets transformers 5.3, whose mask builders take
    `input_embeds` and `cache_position`; later releases renamed the first
    (`inputs_embeds`, 5.6 dropped the old name) and derive the second
    themselves. Rename, drop what the installed builder no longer takes, and
    nothing else: the masks, and every number, stay the library's own. The
    reference runs without a cache from position 0, where the derived positions
    are the passed ones; the committed ref.json is identical under 5.3 and 5.12."""
    import inspect
    for name in ("create_causal_mask", "create_sliding_window_causal_mask"):
        builder = getattr(modeling, name)
        params = inspect.signature(builder).parameters
        if "input_embeds" in params:
            continue

        def adapted(*args, _builder=builder, _params=params, **kwargs):
            if "input_embeds" in kwargs:
                kwargs["inputs_embeds"] = kwargs.pop("input_embeds")
            if not any(p.kind == p.VAR_KEYWORD for p in _params.values()):
                kwargs = {k: v for k, v in kwargs.items() if k in _params}
            return _builder(*args, **kwargs)
        setattr(modeling, name, adapted)


def build_model(config_module, modeling, config):
    import torch
    cfg = config_module.MiMoV2Config(**dict(config, use_cache=False))
    cfg._attn_implementation = "eager"   # the sink path is eager-only
    torch.manual_seed(0)                 # every weight is overwritten below
    model = modeling.MiMoV2ForCausalLM(cfg).float().eval()
    return model


def check_rope(model, config):
    """The two rotary tables must be what the config says (theta 1e7 on full
    layers, 1e4 on SWA layers, over int(head_dim * 0.334) dims): refuse a
    transformers version that silently standardizes them away."""
    dim = int(config["head_dim"] * config["partial_rotary_factor"])
    for module, theta in ((model.model.rotary_emb, config["rope_theta"]),
                          (model.model.swa_rotary_emb, config["swa_rope_theta"])):
        want = 1.0 / (theta ** (np.arange(0, dim, 2, dtype=np.float64) / dim))
        got = module.inv_freq.double().numpy()
        if got.shape != want.shape or not np.allclose(got, want, rtol=1e-6):
            raise SystemExit(f"rotary table {got} is not theta={theta} over "
                             f"{dim} dims: this transformers version changes "
                             f"the vendor's RoPE")


class RouterWatch:
    """Records, for every token at every MoE layer, the gap between the k-th
    and (k+1)-th biased router score: a near-tie there lets float noise swap an
    expert, which no logit margin downstream can protect against."""

    def __init__(self, model):
        self.gaps = []
        for layer in model.model.layers:
            gate = getattr(layer.mlp, "gate", None)
            if gate is not None:
                gate.register_forward_hook(self.hook)

    def hook(self, gate, inputs, _output):
        import torch
        x = inputs[0].reshape(-1, inputs[0].shape[-1]).float()
        scores = torch.sigmoid(x @ gate.weight.float().T)
        biased = scores + gate.e_score_correction_bias.float()
        top = torch.topk(biased, gate.top_k + 1, dim=-1).values
        self.gaps.extend((top[:, -2] - top[:, -1]).tolist())


def logits_for(model, ids):
    import torch
    with torch.no_grad():
        return model(input_ids=torch.tensor([ids], dtype=torch.long),
                     use_cache=False).logits[0]


def case_reference(model, prompt, max_new, min_margin):
    import torch
    sequence = list(prompt)
    margins = []
    for _ in range(max_new):
        logits = logits_for(model, sequence)[-1]
        top2 = torch.topk(logits, 2).values
        margins.append(float(top2[0] - top2[1]))
        sequence.append(int(logits.argmax()))
    full = logits_for(model, sequence)
    teacher = full.argmax(-1).tolist()
    # context-free control: the head applied to the embedding alone
    with torch.no_grad():
        emb = model.model.embed_tokens(torch.tensor(sequence))
        normed = model.model.norm(emb)
        alone = model.lm_head(normed).argmax(-1).tolist()
    context_free = sum(a == b for a, b in zip(alone, teacher)) / len(teacher)
    tf_top2 = torch.topk(full, 2, dim=-1).values
    tf_margins = (tf_top2[:, 0] - tf_top2[:, 1]).tolist()
    worst = min(margins + tf_margins)
    if worst < min_margin:
        raise SystemExit(
            f"logit margin {worst:.4f} below --min-margin {min_margin}: the "
            f"fixture rides a near-tie; re-tune the generator instead of "
            f"lowering the margin.")
    return {
        "prompt_ids": list(prompt),
        "teacher_forcing_ids": teacher,
        "greedy_full_ids": sequence,
        "greedy_new_ids": sequence[len(prompt):],
        "max_new_tokens": max_new,
        "min_logit_margin": round(worst, 4),
        "context_free_agreement": round(context_free, 3),
    }


def vision_reference(model, fixture, config, max_new, min_margin):
    """The official Qwen2-VL processor on the fixture's picture must give the
    fixture's patches bit for bit (the gateway's preprocessing); then Xiaomi's
    ViT and language model answer a prompt that carries the picture."""
    import torch
    from PIL import Image
    from transformers import Qwen2VLImageProcessor
    processor = Qwen2VLImageProcessor.from_pretrained(str(fixture))
    image = Image.open(fixture / "image.png").convert("RGB")
    batch = processor(images=[image], return_tensors="pt")
    official = batch["pixel_values"].float().numpy()
    thw = batch["image_grid_thw"]
    grid = json.loads((fixture / "image.json").read_text())
    ours = np.fromfile(fixture / "patches.f32", dtype=np.float32).reshape(official.shape)
    if thw.tolist() != [[1, grid["grid_h"], grid["grid_w"]]]:
        raise SystemExit(f"grid {thw.tolist()} != fixture {grid}")
    diff = float(np.abs(official - ours).max())
    if diff > 1e-6:
        raise SystemExit(f"gateway patches differ from the official processor by {diff}")
    n_img = grid["tokens"]
    prompt = ([5, 7, config["vision_start_token_id"]] + [config["image_token_id"]] * n_img
              + [config["vision_end_token_id"], 11, 13, 17, 19])
    pixels = torch.from_numpy(official)

    def logits_img(ids, pv):
        with torch.no_grad():
            return model(input_ids=torch.tensor([ids]), pixel_values=pv,
                         image_grid_thw=thw, use_cache=False).logits[0]

    def greedy(pv):
        seq, margins = list(prompt), []
        for _ in range(max_new):
            lg = logits_img(seq, pv)[-1]
            top2 = torch.topk(lg, 2).values
            margins.append(float(top2[0] - top2[1]))
            seq.append(int(lg.argmax()))
        return seq, margins

    seq, margins = greedy(pixels)
    full = logits_img(seq, pixels)
    tf_top2 = torch.topk(full, 2, dim=-1).values
    worst = min(margins + (tf_top2[:, 0] - tf_top2[:, 1]).tolist())
    if worst < min_margin:
        raise SystemExit(f"vision case: logit margin {worst:.4f} below {min_margin}")
    other, _ = greedy(-pixels)
    if other[len(prompt):] == seq[len(prompt):]:
        raise SystemExit("the answer does not depend on the picture: re-tune the tower")
    with torch.no_grad():
        rows = model.visual(pixel_values=pixels, grid_thw=thw).float().numpy()
    return {
        "prompt_ids": prompt,
        "grid_h": grid["grid_h"], "grid_w": grid["grid_w"],
        "patches_sha256": hashlib.sha256(ours.tobytes()).hexdigest(),
        "patch_max_diff_vs_official_processor": diff,
        "vision_rows_sum": float(rows.sum()),
        "vision_rows_abs_sum": float(np.abs(rows).sum()),
        "teacher_forcing_ids": full.argmax(-1).tolist(),
        "greedy_full_ids": seq,
        "greedy_new_ids": seq[len(prompt):],
        "max_new_tokens": max_new,
        "min_logit_margin": round(worst, 4),
        "negated_image_new_ids": other[len(prompt):],
    }


def main():
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument("--fixture", default="./mimo_tiny")
    parser.add_argument("--vendor-dir", default=None,
                        help="default: <fixture>/.vendor_mimo_v2")
    parser.add_argument("--output", default=None,
                        help="default: <fixture>/ref.json")
    parser.add_argument("--min-margin", type=float, default=0.05)
    parser.add_argument(
        "--min-router-gap", type=float, default=1e-5,
        help="router scores are f32 dot products over the hidden size; engine "
             "and torch sum them in different orders, about 1e-6 apart")
    parser.add_argument("--max-context-free", type=float, default=0.5)
    args = parser.parse_args()

    fixture = Path(args.fixture)
    vendor_dir = (Path(args.vendor_dir) if args.vendor_dir
                  else fixture / ".vendor_mimo_v2")
    output = Path(args.output) if args.output else fixture / "ref.json"
    config = json.loads((fixture / "config.json").read_text())

    fetch_vendor(vendor_dir)
    config_module, modeling = load_vendor(vendor_dir)
    model = build_model(config_module, modeling, config)
    check_rope(model, config)
    state = to_state_dict(read_fixture(fixture), config)
    if "vision_config" in config:
        state.update(vision_state(read_fixture(fixture), model))
    missing, unexpected = model.load_state_dict(state, strict=False)
    if missing or unexpected:
        raise SystemExit(f"state dict mismatch:\n  missing: {missing}\n"
                         f"  unexpected: {unexpected}")

    eos = config["eos_token_id"]
    vocab = config["vocab_size"]
    cases = {
        # every case outgrows the 8-token window; "long" crosses a 64-token
        # prefill chunk too
        "short": ([5, 7, 9, 11, 13, 17, 19, 23], 12),
        "window": ([(5 + i * 7) % vocab for i in range(40)], 6),
        "long": ([(5 + i * 11) % vocab for i in range(72)], 6),
    }
    watch = RouterWatch(model)
    reference = {}
    for name, (prompt, max_new) in cases.items():
        prompt = [t if t != eos else t + 1 for t in prompt]
        reference[name] = case_reference(model, prompt, max_new,
                                         args.min_margin)
        if eos in reference[name]["greedy_new_ids"]:
            raise SystemExit(f"case {name}: greedy emitted EOS")
        print(f"[ref] {name}: prompt {len(prompt)} -> +{max_new}, "
              f"min margin {reference[name]['min_logit_margin']}, "
              f"context-free agreement "
              f"{reference[name]['context_free_agreement']}", file=sys.stderr)
        if reference[name]["context_free_agreement"] > args.max_context_free:
            raise SystemExit(
                f"case {name}: the embedding alone predicts "
                f"{reference[name]['context_free_agreement']:.0%} of the "
                f"teacher-forced argmax; the layers barely matter and an engine "
                f"that skipped them would pass. Re-tune the generator.")
    router_gap = min(watch.gaps)
    print(f"[ref] router: {len(watch.gaps)} top-k boundaries, min gap "
          f"{router_gap:.3g}", file=sys.stderr)
    if router_gap < args.min_router_gap:
        raise SystemExit(f"router top-k boundary gap {router_gap:.3g} below "
                         f"--min-router-gap {args.min_router_gap}: float noise "
                         f"could swap an expert. Re-tune the generator.")

    image = None
    if "vision_config" in config:
        image = vision_reference(model, fixture, config, 8, args.min_margin)
        print(f"[ref] image: {image['grid_h']}x{image['grid_w']} patches, margin "
              f"{image['min_logit_margin']}, patches vs official processor "
              f"{image['patch_max_diff_vs_official_processor']}", file=sys.stderr)

    import torch
    import transformers
    output.write_text(json.dumps({
        "schema_version": 1,
        "source": "xiaomi-vendor",
        "vendor_repo": VENDOR_REPO,
        "vendor_revision": VENDOR_REVISION,
        "vendor_files": VENDOR_FILES,
        "torch_version": torch.__version__,
        "transformers_version": transformers.__version__,
        "engine_env": {
            # exact-math configuration: dense weights widened to f32 instead of
            # the default int8 at load, float expert matmuls instead of int8
            # activations (a speed approximation, not an oracle target)
            "MIMO_DENSE_BITS": "32", "MIMO_IDOT": "0", "COLI_TEMP": "0"},
        "cases": reference,
        **({"image": image} if image else {}),
    }, indent=1) + "\n")
    print(f"[ref] wrote {output}", file=sys.stderr)
    return 0


if __name__ == "__main__":
    sys.exit(main())