Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
tests: MiMo's dense chain on Lavapipe, and under ASan and UBSan
JustVuggda40e539e9cb
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 005cf6aa..e580f786 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -194,6 +194,11 @@ jobs:req: c/tools/requirements-kimi-k3-tiny.txt- family: kimi-mimo-sanitizereq: c/tools/requirements-mimo-tiny.txt+ # MiMo-V2.6's dense chain (COLI_VK_CHAIN=1), and the same under ASan and UBSan+ - family: mimo-chain+ req: c/tools/requirements-mimo-tiny.txt+ - family: mimo-chain-sanitize+ req: c/tools/requirements-mimo-tiny.txt# GLM-5.2 (colibri) and GLM-5.3 Flash (glm53) on the routed-expert tier, and# the same under ASan and UBSan- family: glmdiff --git a/c/tests/vulkan_chain_serve.py b/c/tests/vulkan_chain_serve.pyindex e760e72d..701e15f9 100644--- a/c/tests/vulkan_chain_serve.py+++ b/c/tests/vulkan_chain_serve.py@@ -7,7 +7,9 @@ The turns exercise what moves the chain's device state around: a pinned snapshota prompt that diverges and one that starts over, a cache that grows between turns,and the prefill read-out (logprobs=k, ECHO frames). Token ids and texts must be theCPU's exactly; the logprobs printed with them may differ in their last digits (the-device sums in another order), so numbers compare within 1e-4.+device sums in another order), so numbers compare within 1e-4 (CHAIN_SERVE_TOL in the+environment sets another bound: a fixture whose logits run to the hundreds, as MiMo's+does, prints logprobs whose rounding is larger in absolute terms).usage: vulkan_chain_serve.py <engine> <snapshot> [KEY=VALUE ...] (extra environment)COLI_VK_CHAIN in the caller's environment picks the chain's mode (default 1; 2 runs@@ -68,6 +70,10 @@ def session(engine, snap, extra, chain):return out, b"".join(err).decode(errors="replace")+TOL = float(os.environ.get("CHAIN_SERVE_TOL", "1e-4"))+WORST = [0.0]++def close(a, b):if a == b:return True@@ -78,8 +84,9 @@ def close(a, b):if x == y:continuetry:- if b"." not in x or abs(float(x) - float(y)) > 1e-4:+ if b"." not in x or abs(float(x) - float(y)) > TOL:return False+ WORST[0] = max(WORST[0], abs(float(x) - float(y)))except ValueError:return Falsereturn True@@ -104,8 +111,9 @@ def main():print(f" cpu: {a[:200]!r}\n vk : {b[:200]!r}")breaksys.exit(f"FAIL: the chain's frames differ from the CPU's ({len(cpu)} vs {len(dev)})")- print(f"OK serve {os.path.basename(snap)} {' '.join(sys.argv[3:])}: {len(cpu)} frames = CPU, "- f"{pins} pin lines, {len(reuse)} turns reusing state, {forwards[-1].split('] ', 1)[1][:60]}")+ print(f"OK serve {os.path.basename(snap)} {' '.join(sys.argv[3:])}: {len(cpu)} frames = CPU "+ f"(numbers within {WORST[0]:.1e}), {pins} pin lines, {len(reuse)} turns reusing state, "+ f"{forwards[-1].split('] ', 1)[1][:60]}")if __name__ == "__main__":diff --git a/c/tests/vulkan_engines.sh b/c/tests/vulkan_engines.shindex ab4f9ff7..0c630ef2 100755--- a/c/tests/vulkan_engines.sh+++ b/c/tests/vulkan_engines.sh@@ -7,6 +7,7 @@# bash tests/vulkan_engines.sh glm | glm-sanitize # GLM-5.2 (colibri) and GLM-5.3 Flash (glm53)# bash tests/vulkan_engines.sh shader # the qmatmul formats, the expert batch and the tier, no engine# bash tests/vulkan_engines.sh qwen-chain | qwen-chain-sanitize # the dense chain (COLI_VK_CHAIN=1)+# bash tests/vulkan_engines.sh mimo-chain | mimo-chain-sanitize # MiMo-V2.6's dense chain## Needs libvulkan-dev, glslc and mesa-vulkan-drivers, plus the Python packages of# the family's tiny fixtures (see the vulkan-engines job in .github/workflows/ci.yml).@@ -963,6 +964,162 @@ family_kimi_mimo_sanitize() {make clean >/dev/null 2>&1 || true}+# MiMo-V2.6's dense chain (mimo_chain.h, COLI_VK_CHAIN=1): every layer's attention with+# its sliding window (a ring of 8 rows on the fixture, which every case outgrows), the+# sink logits, partial RoPE with two thetas and the value scale on the device, the+# routed experts on the host. Gates, per configuration (mimo_chain_gate): the CPU run's+# generated tokens; the teacher-forced logits of every position (MIMO_LOGITS) within+# 1e-4 of the largest one; a "[VK] mimo chain: N forwards" line with N > 0.+mimo_ids() { # <case|image> <field>: a field of ref.json's case+ $PY -c "import json,sys;r=json.load(open('mimo_tiny/ref.json'));c=r['image'] if sys.argv[1]=='image' else r['cases'][sys.argv[1]];print(' '.join(map(str,c[sys.argv[2]])))" "$1" "$2"+}+mimo_chain_gate() { # <tag> <case> <env...> (CHAINMODE: COLI_VK_CHAIN's value, default 1)+ local tag=$1 c=$2 extra=() fw lg; shift 2+ [ "$c" = image ] && extra=(--image mimo_tiny/patches.f32 --grid $MGRID)+ env "$@" COLI_TEMP=0 ./mimo mimo_tiny --ids "$(mimo_ids $c prompt_ids)" --ngen 6 "${extra[@]}" > mimo-cpu.txt 2>/dev/null+ env "$@" COLI_TEMP=0 COLI_VULKAN=1 COLI_VK_CHAIN=${CHAINMODE:-1} COLI_VK_TIER_SYNC=1 \+ ./mimo mimo_tiny --ids "$(mimo_ids $c prompt_ids)" --ngen 6 "${extra[@]}" > mimo-vk.txt 2> mimo-vk.err+ [ -s mimo-cpu.txt ] && cmp -s mimo-cpu.txt mimo-vk.txt || { cat mimo-cpu.txt mimo-vk.txt mimo-vk.err; fail "$tag $c: the chain's tokens differ from the CPU"; }+ fw=$(chain_count mimo mimo-vk.err)+ [ "$fw" -gt 0 ] || { cat mimo-vk.err; fail "$tag $c: the chain never ran"; }+ rm -f cpu.f32 vk.f32+ env "$@" MIMO_LOGITS=cpu.f32 ./mimo mimo_tiny --ids "$(mimo_ids $c greedy_full_ids)" --ngen 0 "${extra[@]}" > /dev/null 2>&1+ env "$@" MIMO_LOGITS=vk.f32 COLI_VULKAN=1 COLI_VK_CHAIN=${CHAINMODE:-1} COLI_VK_TIER_SYNC=1 \+ ./mimo mimo_tiny --ids "$(mimo_ids $c greedy_full_ids)" --ngen 0 "${extra[@]}" > /dev/null 2> mimo-vk.err+ lg=$(logits_close cpu.f32 vk.f32) || { echo "$lg"; fail "$tag $c: logits"; }+ echo "OK $tag $c: tokens = CPU, $lg, $fw chain forwards"+}+# mimo_lost_gate <tag> <case> <back> <env...>: the device "lost" (COLI_VK_CHAIN_FAULT) at+# the chain frame <back> frames before the end of the same run without a fault (the+# frames a device spends setting up differ: on Lavapipe its memory needs no zeroing+# frames). The host's caches hold whole steps only, so the CPU runs the rest of the+# step from where they end, with nothing to rebuild (a picture included): the CPU+# run's tokens all the same.+mimo_lost_gate() {+ local tag=$1 c=$2 back=$3 extra=() frames; shift 3+ [ "$c" = image ] && extra=(--image mimo_tiny/patches.f32 --grid $MGRID)+ env "$@" COLI_TEMP=0 ./mimo mimo_tiny --ids "$(mimo_ids $c prompt_ids)" --ngen 6 "${extra[@]}" > mimo-cpu.txt 2>/dev/null+ env "$@" COLI_TEMP=0 COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 \+ ./mimo mimo_tiny --ids "$(mimo_ids $c prompt_ids)" --ngen 6 "${extra[@]}" > /dev/null 2> mimo-vk.err+ frames=$(sed -n 's/^\[VK\] mimo chain: [0-9]* forwards, \([0-9]*\) frames.*/\1/p' mimo-vk.err | tail -1)+ [ "${frames:-0}" -gt "$back" ] || { cat mimo-vk.err; fail "$tag: the run had no frame $back before its end"; }+ env "$@" COLI_TEMP=0 COLI_VK_CHAIN_FAULT=$((frames - back)) COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 \+ ./mimo mimo_tiny --ids "$(mimo_ids $c prompt_ids)" --ngen 6 "${extra[@]}" > mimo-vk.txt 2> mimo-vk.err+ [ -s mimo-cpu.txt ] && cmp -s mimo-cpu.txt mimo-vk.txt || { cat mimo-cpu.txt mimo-vk.txt mimo-vk.err; fail "$tag: tokens differ from the CPU"; }+ grep -q "the device was lost at position [0-9]" mimo-vk.err || { cat mimo-vk.err; fail "$tag: the device was never lost"; }+ echo "OK $tag: tokens = CPU, $(grep -o 'the device was lost at position [0-9]*' mimo-vk.err)"+}+mimo_served_fixture() { # the fixture plus a byte tokenizer, for the serve tests+ rm -rf mimo_tiny_served+ $PY -c "import sys, shutil; sys.path.insert(0, 'tests'); import mimo_serve_fixture as f; shutil.copytree(f.served_fixture(), 'mimo_tiny_served')"+}++family_mimo_chain() {+ make mimo tests/test_vk_chain VK=1+ ./tests/test_vk_chain shaders/qmatmul.spv | tee vk_chain.log+ tail -1 vk_chain.log | grep -qx PASS || fail "the chain's ops"+ $PY tools/make_mimo_tiny.py --output ./mimo_tiny --force --vision+ MGRID=$($PY -c "import json;i=json.load(open('mimo_tiny/ref.json'))['image'];print(i['grid_h'],i['grid_w'])")+ # Xiaomi's vendor oracle with the chain on: greedy and teacher-forced in every case,+ # the native FP8/BF16 trunk, prefill in blocks of 3 and of 1, the picture+ COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 $PY tests/mimo_tiny_harness.py --binary ./mimo --fixture ./mimo_tiny+ # every dense form (MIMO_DENSE_BITS 32: f32, fmt 10; 0: the release's FP8 and BF16, fmt+ # 12 and 11; 8: int8 rows, fmt 1), the prompt in one block (the tiled GEMM, the window+ # gathered beside the block) and one token at a time (the ring read in place), text+ # and picture, the routed experts on the tier+ local bits ch c k+ for bits in 32 0 8; do for ch in 64 1; do for c in short window long image; do+ mimo_chain_gate "chain mimo bits=$bits MIMO_CHUNK=$ch" $c MIMO_DENSE_BITS=$bits MIMO_CHUNK=$ch+ done; done; done+ for c in short long; do+ mimo_chain_gate "chain mimo blocks of 3" $c MIMO_DENSE_BITS=32 MIMO_CHUNK=3+ mimo_chain_gate "chain mimo tier off" $c MIMO_DENSE_BITS=0 COLI_VK_TIER=0+ mimo_chain_gate "chain mimo experts on the CPU (MIMO_VK_EXPERTS=0)" $c MIMO_DENSE_BITS=32 MIMO_VK_EXPERTS=0+ mimo_chain_gate "chain mimo a block in chunks of 3 rows" $c MIMO_DENSE_BITS=32 COLI_VK_CHAIN_ROWS=3+ mimo_chain_gate "chain mimo tiled GEMM from S=2" $c MIMO_DENSE_BITS=0 COLI_VK_GEMM_MIN_S=2+ mimo_chain_gate "chain mimo the per-row GEMV" $c MIMO_DENSE_BITS=32 COLI_VK_CHAIN_GEMV=0+ mimo_chain_gate "chain mimo eviction (MIMO_VK_EXPERTS=2)" $c MIMO_DENSE_BITS=32 MIMO_VK_EXPERTS=2+ done+ # the picture's tower on the device too (COLI_VK_DENSE=1: the per-matrix path for the+ # tower, the chain for the layers)+ mimo_chain_gate "chain mimo tower on the device" image MIMO_DENSE_BITS=0 COLI_VK_DENSE=1+ # The window boundary, bit for bit: with every matrix on the per-row GEMV and the tier+ # off the chain's logits are the same bytes whatever the blocks -- one row reading the+ # ring in place, blocks of 3, of the window's 8 and of 9 rows gathering it, the whole+ # prompt -- so a row's window is the same rows in the same order on every path.+ for c in window long image; do+ local x=(); [ $c = image ] && x=(--image mimo_tiny/patches.f32 --grid $MGRID)+ for ch in 64 1 3 8 9; do+ COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER=0 COLI_VK_GEMM_MIN_S=0 MIMO_DENSE_BITS=32 MIMO_CHUNK=$ch MIMO_LOGITS=blk$ch.f32 \+ ./mimo mimo_tiny --ids "$(mimo_ids $c greedy_full_ids)" --ngen 0 "${x[@]}" > /dev/null 2> mimo-vk.err+ [ "$(chain_count mimo mimo-vk.err)" -gt 0 ] || { cat mimo-vk.err; fail "chain mimo blocks $ch $c: the chain never ran"; }+ [ $ch = 64 ] || cmp -s blk64.f32 blk$ch.f32 || fail "chain mimo $c: blocks of $ch are not the bytes of one block"+ done+ echo "OK chain mimo window boundary $c: blocks of 64, 1, 3, 8, 9 give the same logits bytes"+ done+ # COLI_VK_CHAIN=2: prompts on the device, decode on the CPU, the caches moving between them+ for c in short long image; do CHAINMODE=2 mimo_chain_gate "chain mimo prompts only" $c MIMO_DENSE_BITS=32; done+ # the device lost (frames counted back from the end of the run: 5 decode steps of 6+ # frames follow the prompt): inside the last decode step; inside the first block of+ # a prompt split in chunks of 3 rows (the CPU takes the step's remaining rows); inside+ # a picture's prompt; inside a prompt fed one token at a time+ mimo_lost_gate "chain mimo device lost mid-decode" long 3 MIMO_DENSE_BITS=32+ mimo_lost_gate "chain mimo device lost inside a chunked prompt" long 100 MIMO_DENSE_BITS=32 COLI_VK_CHAIN_ROWS=3+ mimo_lost_gate "chain mimo device lost inside a picture's prompt" image 33 MIMO_DENSE_BITS=0+ mimo_lost_gate "chain mimo device lost, one token at a time" short 50 MIMO_DENSE_BITS=32 MIMO_CHUNK=1+ # the prefix-reuse contract and Brio photos (the rings restored into the device's+ # mirrors), compared bit for bit with a cold engine: the per-row GEMV and no tier, so+ # a resumed prompt's rows get the bits of the same rows computed cold+ COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER=0 COLI_VK_GEMM_MIN_S=0 $PY -m unittest tests.test_mimo_prefix_serve+ # a serve session against the CPU's, frame for frame, in both modes and with the+ # device lost mid-session. The fixture's logits run to 300 (its head is scaled for wide+ # greedy margins), so its logprobs compare within 1e-3: about 3e-6 of the largest logit+ mimo_served_fixture+ for k in 1 2; do CHAIN_SERVE_TOL=1e-3 COLI_VK_CHAIN=$k $PY tests/vulkan_chain_serve.py ./mimo mimo_tiny_served OMP_NUM_THREADS=2; done+ CHAIN_SERVE_TOL=1e-3 $PY tests/vulkan_chain_serve.py ./mimo mimo_tiny_served OMP_NUM_THREADS=2 COLI_VK_CHAIN_FAULT=60+}++# MiMo's chain under ASan and UBSan: memory safety is the gate (a sanitized build+# vectorizes differently, so tokens are not compared); each run must have run the chain.+family_mimo_chain_sanitize() {+ make clean >/dev/null 2>&1 || true+ make mimo tests/test_vk_chain VK=1 EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"+ export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1+ ./tests/test_vk_chain shaders/qmatmul.spv > san.log 2>&1 || true+ if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log || ! tail -1 san.log | grep -qx PASS; then cat san.log; fail "asan: the chain's ops"; fi+ echo "OK asan: the chain's ops"+ $PY tools/make_mimo_tiny.py --output ./mimo_tiny --force --vision+ MGRID=$($PY -c "import json;i=json.load(open('mimo_tiny/ref.json'))['image'];print(i['grid_h'],i['grid_w'])")+ msan() { # <tag> <case> <env...>+ local tag=$1 c=$2 x=(); shift 2+ [ "$c" = image ] && x=(--image mimo_tiny/patches.f32 --grid $MGRID)+ env OMP_NUM_THREADS=2 COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_TEMP=0 "$@" \+ ./mimo mimo_tiny --ids "$(mimo_ids $c prompt_ids)" --ngen 6 "${x[@]}" > san.log 2>&1 || true+ if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "$tag: sanitizer diagnostic"; fi+ [ "$(chain_count mimo san.log)" -gt 0 ] || { cat san.log; fail "$tag: the chain never ran"; }+ echo "OK $tag: sanitizers clean, $(chain_count mimo san.log) chain forwards"+ }+ msan "asan chain mimo f32, the tier" long MIMO_DENSE_BITS=32+ msan "asan chain mimo native FP8/BF16, picture, tower on the device" image MIMO_DENSE_BITS=0 COLI_VK_DENSE=1+ msan "asan chain mimo blocks of 3 in chunks of 2 rows" long MIMO_DENSE_BITS=32 MIMO_CHUNK=3 COLI_VK_CHAIN_ROWS=2+ msan "asan chain mimo one token at a time, int8" window MIMO_DENSE_BITS=8 MIMO_CHUNK=1+ msan "asan chain mimo experts on the CPU" short MIMO_DENSE_BITS=32 MIMO_VK_EXPERTS=0+ # the device lost inside a prompt split in chunks of 3 rows: the frame 100 before the+ # end of the same run without a fault (a device's setup frames differ)+ msan "asan chain mimo frames to the end" long MIMO_DENSE_BITS=32 COLI_VK_CHAIN_ROWS=3+ local frames+ frames=$(sed -n 's/^\[VK\] mimo chain: [0-9]* forwards, \([0-9]*\) frames.*/\1/p' san.log | tail -1)+ env OMP_NUM_THREADS=2 COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_TEMP=0 MIMO_DENSE_BITS=32 COLI_VK_CHAIN_ROWS=3 \+ COLI_VK_CHAIN_FAULT=$((frames - 100)) ./mimo mimo_tiny --ids "$(mimo_ids long prompt_ids)" --ngen 6 > san.log 2>&1 || true+ if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "asan chain mimo device lost: sanitizer diagnostic"; fi+ grep -q "the device was lost at position [1-9]" san.log || { cat san.log; fail "asan chain mimo device lost: not lost inside the prompt"; }+ echo "OK asan chain mimo device lost: sanitizers clean, $(grep -o 'the device was lost at position [0-9]*' san.log)"+ mimo_served_fixture+ CHAIN_SERVE_TOL=1e-3 $PY tests/vulkan_chain_serve.py ./mimo mimo_tiny_served OMP_NUM_THREADS=2 > san.log 2>&1 || { cat san.log; fail "asan chain mimo serve"; }+ echo "OK asan chain mimo serve: $(tail -1 san.log)"+ make clean >/dev/null 2>&1 || true+}+# GLM-5.2 (colibri) and GLM-5.3 Flash (glm53) on the routed-expert tier: the fixtures.# colibri: the bf16 oracle, whose experts the loader quantizes to the bits asked for# (16: f32, fmt 10 on the device; 8: int8, fmt 1; 4: int4 per row, fmt 2; 3: int3-g64,@@ -1351,9 +1508,11 @@ case "${1:-}" indeepseek-sanitize) family_deepseek_sanitize ;;kimi) family_kimi ;;kimi-mimo-sanitize) family_kimi_mimo_sanitize ;;+ mimo-chain) family_mimo_chain ;;+ mimo-chain-sanitize) family_mimo_chain_sanitize ;;glm) family_glm ;;glm-sanitize) family_glm_sanitize ;;qwen-chain) family_qwen_chain ;;qwen-chain-sanitize) family_qwen_chain_sanitize ;;- *) echo "usage: $0 shader|qwen|qwen-sanitize|inkling-olmoe|inkling-olmoe-sanitize|mimo-qwenimage|deepseek|deepseek-sanitize|kimi|kimi-mimo-sanitize|glm|glm-sanitize|qwen-chain|qwen-chain-sanitize" >&2; exit 2 ;;+ *) echo "usage: $0 shader|qwen|qwen-sanitize|inkling-olmoe|inkling-olmoe-sanitize|mimo-qwenimage|deepseek|deepseek-sanitize|kimi|kimi-mimo-sanitize|glm|glm-sanitize|qwen-chain|qwen-chain-sanitize|mimo-chain|mimo-chain-sanitize" >&2; exit 2 ;;esac