Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
15.4 KB
Raw Blame
#!/usr/bin/env python3
"""Token-exact checks for the MiMo-V2.6 engine against an independent oracle.

Dependency-free (stdlib only).  The fixture is generated by
tools/make_mimo_tiny.py (numpy, deterministic); the expected tokens in
ref.json come from Xiaomi's own modeling code via tools/make_mimo_ref.py (run
offline by a maintainer, vendor files pinned by SHA-256), so the engine and its
oracle never share an author.

What is checked, all token-exact (argmax, no tolerances):
  - greedy generation for three cases, each longer than the 8-token window
  - teacher forcing at every position via the MIMO_LOGITS dump
  - the same tokens from every exact configuration: native FP8/BF16 dense
    weights, an expert cache that evicts at every layer, prefill in blocks of 3
    and of 1 (the sliding-window ring across blocks), buffered expert reads
  - determinism (two identical runs, identical tokens)
  - block invariance, BIT for bit: the logits of every position are the same
    bytes whatever the prefill blocks (64, 3, 1) and the expert cache (one that
    evicts at every layer), text and picture. A prompt resumed from a prefix or
    a Brio photo is computed in other blocks than the same prompt cold, so this
    is what lets the two agree to the last bit rather than to the argmax
  - with the vision fixture (make_mimo_tiny.py --vision): the picture through
    the tower, greedy and teacher-forced, f32 and native BF16, and the negated
    picture giving the reference's own answer for it
  - the oracle BITES: one corrupted expert scale byte per layer-2 expert, and
    separately the qkv scale rows read as one continuous grid instead of per
    chunk, must each change the output (guards against a comparison that
    silently accepts anything, and pins the pre-sharded qkv layout)

The engine runs in its exact-math configuration (ref.json's engine_env): dense
weights widened to f32 and float expert matmuls.  The fixture ships no
tokenizer.json on purpose: the engine then prints raw token ids on stdout.
"""

import argparse
import array
import hashlib
import json
import os
import shutil
import struct
import subprocess
import sys
import tempfile
from pathlib import Path


def run_engine(binary, fixture, prompt_ids, max_new, env, extra_env=None,
               image=None):
    run_env = dict(os.environ, **env)
    if extra_env:
        run_env.update(extra_env)
    command = [binary, str(fixture), "--ids", " ".join(map(str, prompt_ids)),
               "--ngen", str(max_new)]
    if image:
        path, grid_h, grid_w = image
        command += ["--image", str(path), "--grid", str(grid_h), str(grid_w)]
    result = subprocess.run(command, env=run_env, capture_output=True,
                            text=True, timeout=600)
    if result.returncode != 0:
        raise AssertionError(
            f"engine exited {result.returncode}:\n{result.stderr[-2000:]}")
    tokens = []
    for word in result.stdout.split():
        if not word.lstrip("-").isdigit():
            break
        tokens.append(int(word))
    return tokens


def check_greedy(binary, fixture, reference):
    for name, case in reference["cases"].items():
        got = run_engine(binary, fixture, case["prompt_ids"],
                         case["max_new_tokens"], reference["engine_env"])
        want = case["greedy_new_ids"]
        if got != want:
            raise AssertionError(f"greedy mismatch in case '{name}': engine "
                                 f"{got} vs reference {want}")
        print(f"ok greedy {name}: {len(want)}/{len(want)} tokens")


def check_exact_variants(binary, fixture, reference):
    """Every configuration that is exact by construction must give the same
    tokens: the native FP8/BF16 dense path, an expert cache no larger than one
    routing step (every layer evicts), and prefill in odd-sized blocks and one
    token at a time (the window ring is filled and read across blocks)."""
    variants = {
        "native dense, cache 4, prefill blocks of 3":
            {"MIMO_DENSE_BITS": "0", "MIMO_CAP": "4", "MIMO_CHUNK": "3"},
        "prefill one token at a time": {"MIMO_CHUNK": "1"},
        "buffered expert reads": {"MIMO_DIRECT": "0"},
    }
    for label, extra in variants.items():
        env = dict(reference["engine_env"], **extra)
        for name, case in reference["cases"].items():
            got = run_engine(binary, fixture, case["prompt_ids"],
                             case["max_new_tokens"], env)
            if got != case["greedy_new_ids"]:
                raise AssertionError(f"{label}: greedy mismatch in case '{name}': "
                                     f"engine {got} vs reference "
                                     f"{case['greedy_new_ids']}")
        print(f"ok {label}: identical in {len(reference['cases'])} cases")


def check_determinism(binary, fixture, reference):
    case = reference["cases"]["short"]
    first = run_engine(binary, fixture, case["prompt_ids"],
                       case["max_new_tokens"], reference["engine_env"])
    second = run_engine(binary, fixture, case["prompt_ids"],
                        case["max_new_tokens"], reference["engine_env"])
    if first != second:
        raise AssertionError(f"non-deterministic greedy: {first} vs {second}")
    print("ok determinism: two identical runs, identical tokens")


def teacher_forced(binary, fixture, ids, env, vocab, image=None):
    with tempfile.NamedTemporaryFile(suffix=".f32", delete=False) as tmp:
        path = tmp.name
    try:
        run_engine(binary, fixture, ids, 0, env, extra_env={"MIMO_LOGITS": path},
                   image=image)
        raw = array.array("f")
        with open(path, "rb") as fh:
            raw.frombytes(fh.read())
    finally:
        os.unlink(path)
    if len(raw) != len(ids) * vocab:
        raise AssertionError(f"MIMO_LOGITS dumped {len(raw)} floats, expected "
                             f"{len(ids) * vocab}")
    return [max(range(vocab), key=raw[p * vocab:(p + 1) * vocab].__getitem__)
            for p in range(len(ids))]


def logits_bytes(binary, fixture, ids, env, image=None):
    with tempfile.NamedTemporaryFile(suffix=".f32", delete=False) as tmp:
        path = tmp.name
    try:
        run_engine(binary, fixture, ids, 0, env, extra_env={"MIMO_LOGITS": path},
                   image=image)
        with open(path, "rb") as fh:
            return fh.read()
    finally:
        os.unlink(path)


def check_block_invariance(binary, fixture, reference):
    """A row is a function of its own inputs, never of the rows that share its
    prefill block: the logits must be the same BYTES for every block size.

    A CPU property. With COLI_VULKAN=1 the device takes a block of two or more
    rows through the tiled GEMM and a single row through the GEMV (#1834), and
    the two sum in different orders, so the bytes depend on the block size by
    design; tests/vulkan_engines.sh checks the device against the CPU's tokens
    instead."""
    vk = os.environ.get("COLI_VULKAN", "")
    if vk and vk != "0":
        print("skip block invariance: COLI_VULKAN=1 (GEMM for blocks, GEMV for one row)")
        return
    variants = {"blocks of 3": {"MIMO_CHUNK": "3"},
                "one token at a time": {"MIMO_CHUNK": "1"},
                "cache 4 (evicts at every layer)": {"MIMO_CAP": "4"}}
    inputs = [(name, case["greedy_full_ids"], None)
              for name, case in reference["cases"].items()]
    if "image" in reference:
        image = reference["image"]
        inputs.append(("image", image["greedy_full_ids"],
                       (fixture / "patches.f32", image["grid_h"], image["grid_w"])))
    for name, ids, image in inputs:
        base = logits_bytes(binary, fixture, ids, reference["engine_env"], image)
        for label, extra in variants.items():
            env = dict(reference["engine_env"], **extra)
            if logits_bytes(binary, fixture, ids, env, image) != base:
                raise AssertionError(f"block invariance: case '{name}', {label}: the "
                                     f"logits are not the bytes of the default blocks")
    print(f"ok block invariance: {len(inputs)} cases x {len(variants)} variants, "
          f"bit-identical logits")


def check_teacher_forcing(binary, fixture, reference, vocab):
    for name, case in reference["cases"].items():
        got = teacher_forced(binary, fixture, case["greedy_full_ids"],
                             reference["engine_env"], vocab)
        want = case["teacher_forcing_ids"]
        if got != want:
            bad = next(i for i, (a, b) in enumerate(zip(got, want)) if a != b)
            raise AssertionError(
                f"teacher forcing mismatch in case '{name}' at position {bad}: "
                f"engine {got[bad]} vs reference {want[bad]}")
        print(f"ok teacher forcing {name}: {len(want)}/{len(want)} positions")


def patch_fixture(fixture, edit):
    """A copy of the fixture with edit(raw_bytes_of_shard1, header, base)
    applied to the first shard; returns the copy's path."""
    copy = Path(tempfile.mkdtemp(prefix="mimo-tiny-bite-")) / "fixture"
    shutil.copytree(fixture, copy)
    shard = copy / "model-00001-of-00002.safetensors"
    raw = bytearray(shard.read_bytes())
    header_len = struct.unpack("<Q", bytes(raw[:8]))[0]
    header = json.loads(bytes(raw[8:8 + header_len]))
    edit(raw, header, 8 + header_len)
    shard.write_bytes(bytes(raw))
    return copy


def diverged_cases(binary, fixture, reference):
    count = 0
    for case in reference["cases"].values():
        got = run_engine(binary, fixture, case["prompt_ids"],
                         case["max_new_tokens"], reference["engine_env"])
        if got != case["greedy_new_ids"]:
            count += 1
    return count


def check_oracle_bites(binary, fixture, reference, experts):
    def expert_scales(raw, header, base):
        for expert in range(experts):
            lo, _ = header[f"model.layers.2.mlp.experts.{expert}"
                           f".gate_proj.weight_scale"]["data_offsets"]
            raw[base + lo] = (raw[base + lo] + 3) % 256     # x8 on one group
    copy = patch_fixture(fixture, expert_scales)
    try:
        count = diverged_cases(binary, copy, reference)
    finally:
        shutil.rmtree(copy.parent)
    if not count:
        raise AssertionError("corrupted expert scales produced identical tokens "
                             "in every case: the oracle does not bite")
    print(f"ok oracle bites: corrupted expert scales diverge in "
          f"{count}/{len(reference['cases'])} cases")

    # The full layers' qkv scale grid is tiled per chunk (4 rows where one
    # continuous grid would have 3).  Swapping two of its rows is invisible to
    # a reader that ignores the chunk layout only if it never uses them; the
    # engine must change its output.
    def swap_qkv_scale_rows(raw, header, base):
        meta = header["model.layers.0.self_attn.qkv_proj.weight_scale_inv"]
        lo, hi = meta["data_offsets"]
        cols = meta["shape"][1]
        row = cols * 4
        grid = raw[base + lo:base + hi]
        first, second = grid[row:2 * row], grid[2 * row:3 * row]
        grid[row:2 * row], grid[2 * row:3 * row] = second, first
        raw[base + lo:base + hi] = grid
    copy = patch_fixture(fixture, swap_qkv_scale_rows)
    try:
        count = diverged_cases(binary, copy, reference)
    finally:
        shutil.rmtree(copy.parent)
    if not count:
        raise AssertionError("swapping qkv scale rows changed nothing: the "
                             "per-chunk scale grid is not being read")
    print(f"ok qkv scale grid bites: {count}/{len(reference['cases'])} cases")


def check_image(binary, fixture, reference, vocab):
    """The fixture's picture, preprocessed the gateway's way (patches.f32,
    pinned by hash to what the official Qwen2-VL processor produced), through
    the engine's tower: the same tokens as Xiaomi's ViT and language model,
    greedy and teacher-forced, on the f32 and the native BF16 tower. Negating
    the patches must give exactly the tokens the reference gave for the
    negated picture -- the image is read, not just tolerated."""
    image = reference["image"]
    patches = fixture / "patches.f32"
    digest = hashlib.sha256(patches.read_bytes()).hexdigest()
    if digest != image["patches_sha256"]:
        raise AssertionError("patches.f32 differs from the patches the reference "
                             "checked against the official processor")
    spec = (patches, image["grid_h"], image["grid_w"])
    for label, extra in (("f32", {}), ("native BF16 tower", {"MIMO_DENSE_BITS": "0"})):
        env = dict(reference["engine_env"], **extra)
        got = run_engine(binary, fixture, image["prompt_ids"],
                         image["max_new_tokens"], env, image=spec)
        if got != image["greedy_new_ids"]:
            raise AssertionError(f"image greedy mismatch ({label}): engine {got} vs "
                                 f"reference {image['greedy_new_ids']}")
        print(f"ok image greedy ({label}): {len(got)}/{len(got)} tokens")
    got = teacher_forced(binary, fixture, image["greedy_full_ids"],
                         reference["engine_env"], vocab, image=spec)
    if got != image["teacher_forcing_ids"]:
        bad = next(i for i, (a, b) in enumerate(zip(got, image["teacher_forcing_ids"]))
                   if a != b)
        raise AssertionError(f"image teacher forcing mismatch at position {bad}")
    print(f"ok image teacher forcing: {len(got)}/{len(got)} positions")
    negated = Path(tempfile.mkdtemp(prefix="mimo-neg-")) / "patches.f32"
    try:
        values = array.array("f")
        values.frombytes(patches.read_bytes())
        negated.write_bytes(array.array("f", (-v for v in values)).tobytes())
        got = run_engine(binary, fixture, image["prompt_ids"],
                         image["max_new_tokens"], reference["engine_env"],
                         image=(negated, image["grid_h"], image["grid_w"]))
    finally:
        shutil.rmtree(negated.parent)
    if got != image["negated_image_new_ids"]:
        raise AssertionError(f"negated image: engine {got} vs reference "
                             f"{image['negated_image_new_ids']}")
    print("ok image bites: the negated picture gives the reference's other answer")


def main():
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument("--binary", required=True)
    parser.add_argument("--fixture", required=True)
    args = parser.parse_args()

    fixture = Path(args.fixture)
    reference = json.loads((fixture / "ref.json").read_text())
    if reference.get("source") != "xiaomi-vendor":
        raise AssertionError("ref.json must come from the vendor implementation "
                             f"(source={reference.get('source')!r})")
    config = json.loads((fixture / "config.json").read_text())
    if (fixture / "tokenizer.json").exists():
        raise AssertionError("fixture must not ship tokenizer.json: raw-id "
                             "stdout is the comparison channel")

    check_greedy(args.binary, fixture, reference)
    check_exact_variants(args.binary, fixture, reference)
    check_determinism(args.binary, fixture, reference)
    check_block_invariance(args.binary, fixture, reference)
    check_teacher_forcing(args.binary, fixture, reference, config["vocab_size"])
    check_oracle_bites(args.binary, fixture, reference,
                       config["n_routed_experts"])
    if "image" in reference:
        check_image(args.binary, fixture, reference, config["vocab_size"])
    print("mimo-tiny-check: all checks passed")
    return 0


if __name__ == "__main__":
    sys.exit(main())