Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

tests: MiMo's dense chain on Lavapipe, and under ASan and UBSan

JustVuggda40e539e9cb
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 005cf6aa..e580f786 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -194,6 +194,11 @@ jobs:
req: c/tools/requirements-kimi-k3-tiny.txt
- family: kimi-mimo-sanitize
req: c/tools/requirements-mimo-tiny.txt
+ # MiMo-V2.6's dense chain (COLI_VK_CHAIN=1), and the same under ASan and UBSan
+ - family: mimo-chain
+ req: c/tools/requirements-mimo-tiny.txt
+ - family: mimo-chain-sanitize
+ req: c/tools/requirements-mimo-tiny.txt
# GLM-5.2 (colibri) and GLM-5.3 Flash (glm53) on the routed-expert tier, and
# the same under ASan and UBSan
- family: glm
diff --git a/c/tests/vulkan_chain_serve.py b/c/tests/vulkan_chain_serve.py
index e760e72d..701e15f9 100644
--- a/c/tests/vulkan_chain_serve.py
+++ b/c/tests/vulkan_chain_serve.py
@@ -7,7 +7,9 @@ The turns exercise what moves the chain's device state around: a pinned snapshot
a prompt that diverges and one that starts over, a cache that grows between turns,
and the prefill read-out (logprobs=k, ECHO frames). Token ids and texts must be the
CPU's exactly; the logprobs printed with them may differ in their last digits (the
-device sums in another order), so numbers compare within 1e-4.
+device sums in another order), so numbers compare within 1e-4 (CHAIN_SERVE_TOL in the
+environment sets another bound: a fixture whose logits run to the hundreds, as MiMo's
+does, prints logprobs whose rounding is larger in absolute terms).
usage: vulkan_chain_serve.py <engine> <snapshot> [KEY=VALUE ...] (extra environment)
COLI_VK_CHAIN in the caller's environment picks the chain's mode (default 1; 2 runs
@@ -68,6 +70,10 @@ def session(engine, snap, extra, chain):
return out, b"".join(err).decode(errors="replace")
+TOL = float(os.environ.get("CHAIN_SERVE_TOL", "1e-4"))
+WORST = [0.0]
+
+
def close(a, b):
if a == b:
return True
@@ -78,8 +84,9 @@ def close(a, b):
if x == y:
continue
try:
- if b"." not in x or abs(float(x) - float(y)) > 1e-4:
+ if b"." not in x or abs(float(x) - float(y)) > TOL:
return False
+ WORST[0] = max(WORST[0], abs(float(x) - float(y)))
except ValueError:
return False
return True
@@ -104,8 +111,9 @@ def main():
print(f" cpu: {a[:200]!r}\n vk : {b[:200]!r}")
break
sys.exit(f"FAIL: the chain's frames differ from the CPU's ({len(cpu)} vs {len(dev)})")
- print(f"OK serve {os.path.basename(snap)} {' '.join(sys.argv[3:])}: {len(cpu)} frames = CPU, "
- f"{pins} pin lines, {len(reuse)} turns reusing state, {forwards[-1].split('] ', 1)[1][:60]}")
+ print(f"OK serve {os.path.basename(snap)} {' '.join(sys.argv[3:])}: {len(cpu)} frames = CPU "
+ f"(numbers within {WORST[0]:.1e}), {pins} pin lines, {len(reuse)} turns reusing state, "
+ f"{forwards[-1].split('] ', 1)[1][:60]}")
if __name__ == "__main__":
diff --git a/c/tests/vulkan_engines.sh b/c/tests/vulkan_engines.sh
index ab4f9ff7..0c630ef2 100755
--- a/c/tests/vulkan_engines.sh
+++ b/c/tests/vulkan_engines.sh
@@ -7,6 +7,7 @@
# bash tests/vulkan_engines.sh glm | glm-sanitize # GLM-5.2 (colibri) and GLM-5.3 Flash (glm53)
# bash tests/vulkan_engines.sh shader # the qmatmul formats, the expert batch and the tier, no engine
# bash tests/vulkan_engines.sh qwen-chain | qwen-chain-sanitize # the dense chain (COLI_VK_CHAIN=1)
+# bash tests/vulkan_engines.sh mimo-chain | mimo-chain-sanitize # MiMo-V2.6's dense chain
#
# Needs libvulkan-dev, glslc and mesa-vulkan-drivers, plus the Python packages of
# the family's tiny fixtures (see the vulkan-engines job in .github/workflows/ci.yml).
@@ -963,6 +964,162 @@ family_kimi_mimo_sanitize() {
make clean >/dev/null 2>&1 || true
}
+# MiMo-V2.6's dense chain (mimo_chain.h, COLI_VK_CHAIN=1): every layer's attention with
+# its sliding window (a ring of 8 rows on the fixture, which every case outgrows), the
+# sink logits, partial RoPE with two thetas and the value scale on the device, the
+# routed experts on the host. Gates, per configuration (mimo_chain_gate): the CPU run's
+# generated tokens; the teacher-forced logits of every position (MIMO_LOGITS) within
+# 1e-4 of the largest one; a "[VK] mimo chain: N forwards" line with N > 0.
+mimo_ids() { # <case|image> <field>: a field of ref.json's case
+ $PY -c "import json,sys;r=json.load(open('mimo_tiny/ref.json'));c=r['image'] if sys.argv[1]=='image' else r['cases'][sys.argv[1]];print(' '.join(map(str,c[sys.argv[2]])))" "$1" "$2"
+}
+mimo_chain_gate() { # <tag> <case> <env...> (CHAINMODE: COLI_VK_CHAIN's value, default 1)
+ local tag=$1 c=$2 extra=() fw lg; shift 2
+ [ "$c" = image ] && extra=(--image mimo_tiny/patches.f32 --grid $MGRID)
+ env "$@" COLI_TEMP=0 ./mimo mimo_tiny --ids "$(mimo_ids $c prompt_ids)" --ngen 6 "${extra[@]}" > mimo-cpu.txt 2>/dev/null
+ env "$@" COLI_TEMP=0 COLI_VULKAN=1 COLI_VK_CHAIN=${CHAINMODE:-1} COLI_VK_TIER_SYNC=1 \
+ ./mimo mimo_tiny --ids "$(mimo_ids $c prompt_ids)" --ngen 6 "${extra[@]}" > mimo-vk.txt 2> mimo-vk.err
+ [ -s mimo-cpu.txt ] && cmp -s mimo-cpu.txt mimo-vk.txt || { cat mimo-cpu.txt mimo-vk.txt mimo-vk.err; fail "$tag $c: the chain's tokens differ from the CPU"; }
+ fw=$(chain_count mimo mimo-vk.err)
+ [ "$fw" -gt 0 ] || { cat mimo-vk.err; fail "$tag $c: the chain never ran"; }
+ rm -f cpu.f32 vk.f32
+ env "$@" MIMO_LOGITS=cpu.f32 ./mimo mimo_tiny --ids "$(mimo_ids $c greedy_full_ids)" --ngen 0 "${extra[@]}" > /dev/null 2>&1
+ env "$@" MIMO_LOGITS=vk.f32 COLI_VULKAN=1 COLI_VK_CHAIN=${CHAINMODE:-1} COLI_VK_TIER_SYNC=1 \
+ ./mimo mimo_tiny --ids "$(mimo_ids $c greedy_full_ids)" --ngen 0 "${extra[@]}" > /dev/null 2> mimo-vk.err
+ lg=$(logits_close cpu.f32 vk.f32) || { echo "$lg"; fail "$tag $c: logits"; }
+ echo "OK $tag $c: tokens = CPU, $lg, $fw chain forwards"
+}
+# mimo_lost_gate <tag> <case> <back> <env...>: the device "lost" (COLI_VK_CHAIN_FAULT) at
+# the chain frame <back> frames before the end of the same run without a fault (the
+# frames a device spends setting up differ: on Lavapipe its memory needs no zeroing
+# frames). The host's caches hold whole steps only, so the CPU runs the rest of the
+# step from where they end, with nothing to rebuild (a picture included): the CPU
+# run's tokens all the same.
+mimo_lost_gate() {
+ local tag=$1 c=$2 back=$3 extra=() frames; shift 3
+ [ "$c" = image ] && extra=(--image mimo_tiny/patches.f32 --grid $MGRID)
+ env "$@" COLI_TEMP=0 ./mimo mimo_tiny --ids "$(mimo_ids $c prompt_ids)" --ngen 6 "${extra[@]}" > mimo-cpu.txt 2>/dev/null
+ env "$@" COLI_TEMP=0 COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 \
+ ./mimo mimo_tiny --ids "$(mimo_ids $c prompt_ids)" --ngen 6 "${extra[@]}" > /dev/null 2> mimo-vk.err
+ frames=$(sed -n 's/^\[VK\] mimo chain: [0-9]* forwards, \([0-9]*\) frames.*/\1/p' mimo-vk.err | tail -1)
+ [ "${frames:-0}" -gt "$back" ] || { cat mimo-vk.err; fail "$tag: the run had no frame $back before its end"; }
+ env "$@" COLI_TEMP=0 COLI_VK_CHAIN_FAULT=$((frames - back)) COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 \
+ ./mimo mimo_tiny --ids "$(mimo_ids $c prompt_ids)" --ngen 6 "${extra[@]}" > mimo-vk.txt 2> mimo-vk.err
+ [ -s mimo-cpu.txt ] && cmp -s mimo-cpu.txt mimo-vk.txt || { cat mimo-cpu.txt mimo-vk.txt mimo-vk.err; fail "$tag: tokens differ from the CPU"; }
+ grep -q "the device was lost at position [0-9]" mimo-vk.err || { cat mimo-vk.err; fail "$tag: the device was never lost"; }
+ echo "OK $tag: tokens = CPU, $(grep -o 'the device was lost at position [0-9]*' mimo-vk.err)"
+}
+mimo_served_fixture() { # the fixture plus a byte tokenizer, for the serve tests
+ rm -rf mimo_tiny_served
+ $PY -c "import sys, shutil; sys.path.insert(0, 'tests'); import mimo_serve_fixture as f; shutil.copytree(f.served_fixture(), 'mimo_tiny_served')"
+}
+
+family_mimo_chain() {
+ make mimo tests/test_vk_chain VK=1
+ ./tests/test_vk_chain shaders/qmatmul.spv | tee vk_chain.log
+ tail -1 vk_chain.log | grep -qx PASS || fail "the chain's ops"
+ $PY tools/make_mimo_tiny.py --output ./mimo_tiny --force --vision
+ MGRID=$($PY -c "import json;i=json.load(open('mimo_tiny/ref.json'))['image'];print(i['grid_h'],i['grid_w'])")
+ # Xiaomi's vendor oracle with the chain on: greedy and teacher-forced in every case,
+ # the native FP8/BF16 trunk, prefill in blocks of 3 and of 1, the picture
+ COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 $PY tests/mimo_tiny_harness.py --binary ./mimo --fixture ./mimo_tiny
+ # every dense form (MIMO_DENSE_BITS 32: f32, fmt 10; 0: the release's FP8 and BF16, fmt
+ # 12 and 11; 8: int8 rows, fmt 1), the prompt in one block (the tiled GEMM, the window
+ # gathered beside the block) and one token at a time (the ring read in place), text
+ # and picture, the routed experts on the tier
+ local bits ch c k
+ for bits in 32 0 8; do for ch in 64 1; do for c in short window long image; do
+ mimo_chain_gate "chain mimo bits=$bits MIMO_CHUNK=$ch" $c MIMO_DENSE_BITS=$bits MIMO_CHUNK=$ch
+ done; done; done
+ for c in short long; do
+ mimo_chain_gate "chain mimo blocks of 3" $c MIMO_DENSE_BITS=32 MIMO_CHUNK=3
+ mimo_chain_gate "chain mimo tier off" $c MIMO_DENSE_BITS=0 COLI_VK_TIER=0
+ mimo_chain_gate "chain mimo experts on the CPU (MIMO_VK_EXPERTS=0)" $c MIMO_DENSE_BITS=32 MIMO_VK_EXPERTS=0
+ mimo_chain_gate "chain mimo a block in chunks of 3 rows" $c MIMO_DENSE_BITS=32 COLI_VK_CHAIN_ROWS=3
+ mimo_chain_gate "chain mimo tiled GEMM from S=2" $c MIMO_DENSE_BITS=0 COLI_VK_GEMM_MIN_S=2
+ mimo_chain_gate "chain mimo the per-row GEMV" $c MIMO_DENSE_BITS=32 COLI_VK_CHAIN_GEMV=0
+ mimo_chain_gate "chain mimo eviction (MIMO_VK_EXPERTS=2)" $c MIMO_DENSE_BITS=32 MIMO_VK_EXPERTS=2
+ done
+ # the picture's tower on the device too (COLI_VK_DENSE=1: the per-matrix path for the
+ # tower, the chain for the layers)
+ mimo_chain_gate "chain mimo tower on the device" image MIMO_DENSE_BITS=0 COLI_VK_DENSE=1
+ # The window boundary, bit for bit: with every matrix on the per-row GEMV and the tier
+ # off the chain's logits are the same bytes whatever the blocks -- one row reading the
+ # ring in place, blocks of 3, of the window's 8 and of 9 rows gathering it, the whole
+ # prompt -- so a row's window is the same rows in the same order on every path.
+ for c in window long image; do
+ local x=(); [ $c = image ] && x=(--image mimo_tiny/patches.f32 --grid $MGRID)
+ for ch in 64 1 3 8 9; do
+ COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER=0 COLI_VK_GEMM_MIN_S=0 MIMO_DENSE_BITS=32 MIMO_CHUNK=$ch MIMO_LOGITS=blk$ch.f32 \
+ ./mimo mimo_tiny --ids "$(mimo_ids $c greedy_full_ids)" --ngen 0 "${x[@]}" > /dev/null 2> mimo-vk.err
+ [ "$(chain_count mimo mimo-vk.err)" -gt 0 ] || { cat mimo-vk.err; fail "chain mimo blocks $ch $c: the chain never ran"; }
+ [ $ch = 64 ] || cmp -s blk64.f32 blk$ch.f32 || fail "chain mimo $c: blocks of $ch are not the bytes of one block"
+ done
+ echo "OK chain mimo window boundary $c: blocks of 64, 1, 3, 8, 9 give the same logits bytes"
+ done
+ # COLI_VK_CHAIN=2: prompts on the device, decode on the CPU, the caches moving between them
+ for c in short long image; do CHAINMODE=2 mimo_chain_gate "chain mimo prompts only" $c MIMO_DENSE_BITS=32; done
+ # the device lost (frames counted back from the end of the run: 5 decode steps of 6
+ # frames follow the prompt): inside the last decode step; inside the first block of
+ # a prompt split in chunks of 3 rows (the CPU takes the step's remaining rows); inside
+ # a picture's prompt; inside a prompt fed one token at a time
+ mimo_lost_gate "chain mimo device lost mid-decode" long 3 MIMO_DENSE_BITS=32
+ mimo_lost_gate "chain mimo device lost inside a chunked prompt" long 100 MIMO_DENSE_BITS=32 COLI_VK_CHAIN_ROWS=3
+ mimo_lost_gate "chain mimo device lost inside a picture's prompt" image 33 MIMO_DENSE_BITS=0
+ mimo_lost_gate "chain mimo device lost, one token at a time" short 50 MIMO_DENSE_BITS=32 MIMO_CHUNK=1
+ # the prefix-reuse contract and Brio photos (the rings restored into the device's
+ # mirrors), compared bit for bit with a cold engine: the per-row GEMV and no tier, so
+ # a resumed prompt's rows get the bits of the same rows computed cold
+ COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER=0 COLI_VK_GEMM_MIN_S=0 $PY -m unittest tests.test_mimo_prefix_serve
+ # a serve session against the CPU's, frame for frame, in both modes and with the
+ # device lost mid-session. The fixture's logits run to 300 (its head is scaled for wide
+ # greedy margins), so its logprobs compare within 1e-3: about 3e-6 of the largest logit
+ mimo_served_fixture
+ for k in 1 2; do CHAIN_SERVE_TOL=1e-3 COLI_VK_CHAIN=$k $PY tests/vulkan_chain_serve.py ./mimo mimo_tiny_served OMP_NUM_THREADS=2; done
+ CHAIN_SERVE_TOL=1e-3 $PY tests/vulkan_chain_serve.py ./mimo mimo_tiny_served OMP_NUM_THREADS=2 COLI_VK_CHAIN_FAULT=60
+}
+
+# MiMo's chain under ASan and UBSan: memory safety is the gate (a sanitized build
+# vectorizes differently, so tokens are not compared); each run must have run the chain.
+family_mimo_chain_sanitize() {
+ make clean >/dev/null 2>&1 || true
+ make mimo tests/test_vk_chain VK=1 EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
+ export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1
+ ./tests/test_vk_chain shaders/qmatmul.spv > san.log 2>&1 || true
+ if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log || ! tail -1 san.log | grep -qx PASS; then cat san.log; fail "asan: the chain's ops"; fi
+ echo "OK asan: the chain's ops"
+ $PY tools/make_mimo_tiny.py --output ./mimo_tiny --force --vision
+ MGRID=$($PY -c "import json;i=json.load(open('mimo_tiny/ref.json'))['image'];print(i['grid_h'],i['grid_w'])")
+ msan() { # <tag> <case> <env...>
+ local tag=$1 c=$2 x=(); shift 2
+ [ "$c" = image ] && x=(--image mimo_tiny/patches.f32 --grid $MGRID)
+ env OMP_NUM_THREADS=2 COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_TEMP=0 "$@" \
+ ./mimo mimo_tiny --ids "$(mimo_ids $c prompt_ids)" --ngen 6 "${x[@]}" > san.log 2>&1 || true
+ if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "$tag: sanitizer diagnostic"; fi
+ [ "$(chain_count mimo san.log)" -gt 0 ] || { cat san.log; fail "$tag: the chain never ran"; }
+ echo "OK $tag: sanitizers clean, $(chain_count mimo san.log) chain forwards"
+ }
+ msan "asan chain mimo f32, the tier" long MIMO_DENSE_BITS=32
+ msan "asan chain mimo native FP8/BF16, picture, tower on the device" image MIMO_DENSE_BITS=0 COLI_VK_DENSE=1
+ msan "asan chain mimo blocks of 3 in chunks of 2 rows" long MIMO_DENSE_BITS=32 MIMO_CHUNK=3 COLI_VK_CHAIN_ROWS=2
+ msan "asan chain mimo one token at a time, int8" window MIMO_DENSE_BITS=8 MIMO_CHUNK=1
+ msan "asan chain mimo experts on the CPU" short MIMO_DENSE_BITS=32 MIMO_VK_EXPERTS=0
+ # the device lost inside a prompt split in chunks of 3 rows: the frame 100 before the
+ # end of the same run without a fault (a device's setup frames differ)
+ msan "asan chain mimo frames to the end" long MIMO_DENSE_BITS=32 COLI_VK_CHAIN_ROWS=3
+ local frames
+ frames=$(sed -n 's/^\[VK\] mimo chain: [0-9]* forwards, \([0-9]*\) frames.*/\1/p' san.log | tail -1)
+ env OMP_NUM_THREADS=2 COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_TEMP=0 MIMO_DENSE_BITS=32 COLI_VK_CHAIN_ROWS=3 \
+ COLI_VK_CHAIN_FAULT=$((frames - 100)) ./mimo mimo_tiny --ids "$(mimo_ids long prompt_ids)" --ngen 6 > san.log 2>&1 || true
+ if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "asan chain mimo device lost: sanitizer diagnostic"; fi
+ grep -q "the device was lost at position [1-9]" san.log || { cat san.log; fail "asan chain mimo device lost: not lost inside the prompt"; }
+ echo "OK asan chain mimo device lost: sanitizers clean, $(grep -o 'the device was lost at position [0-9]*' san.log)"
+ mimo_served_fixture
+ CHAIN_SERVE_TOL=1e-3 $PY tests/vulkan_chain_serve.py ./mimo mimo_tiny_served OMP_NUM_THREADS=2 > san.log 2>&1 || { cat san.log; fail "asan chain mimo serve"; }
+ echo "OK asan chain mimo serve: $(tail -1 san.log)"
+ make clean >/dev/null 2>&1 || true
+}
+
# GLM-5.2 (colibri) and GLM-5.3 Flash (glm53) on the routed-expert tier: the fixtures.
# colibri: the bf16 oracle, whose experts the loader quantizes to the bits asked for
# (16: f32, fmt 10 on the device; 8: int8, fmt 1; 4: int4 per row, fmt 2; 3: int3-g64,
@@ -1351,9 +1508,11 @@ case "${1:-}" in
deepseek-sanitize) family_deepseek_sanitize ;;
kimi) family_kimi ;;
kimi-mimo-sanitize) family_kimi_mimo_sanitize ;;
+ mimo-chain) family_mimo_chain ;;
+ mimo-chain-sanitize) family_mimo_chain_sanitize ;;
glm) family_glm ;;
glm-sanitize) family_glm_sanitize ;;
qwen-chain) family_qwen_chain ;;
qwen-chain-sanitize) family_qwen_chain_sanitize ;;
- *) echo "usage: $0 shader|qwen|qwen-sanitize|inkling-olmoe|inkling-olmoe-sanitize|mimo-qwenimage|deepseek|deepseek-sanitize|kimi|kimi-mimo-sanitize|glm|glm-sanitize|qwen-chain|qwen-chain-sanitize" >&2; exit 2 ;;
+ *) echo "usage: $0 shader|qwen|qwen-sanitize|inkling-olmoe|inkling-olmoe-sanitize|mimo-qwenimage|deepseek|deepseek-sanitize|kimi|kimi-mimo-sanitize|glm|glm-sanitize|qwen-chain|qwen-chain-sanitize|mimo-chain|mimo-chain-sanitize" >&2; exit 2 ;;
esac