Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
tests: the qwen family on Lavapipe with the routed-expert tier on
JustVugg502c92e360ed
diff --git a/c/tests/vulkan_engines.sh b/c/tests/vulkan_engines.shindex b2014b83..3b557427 100755--- a/c/tests/vulkan_engines.sh+++ b/c/tests/vulkan_engines.sh@@ -3,7 +3,7 @@# Vulkan), one family per call so CI can run them side by side:## bash tests/vulkan_engines.sh qwen | inkling-olmoe | mimo-qwenimage | deepseek-# bash tests/vulkan_engines.sh shader # the qmatmul formats alone, no engine+# bash tests/vulkan_engines.sh shader # the qmatmul formats, the expert batch and the tier, no engine## Needs libvulkan-dev, glslc and mesa-vulkan-drivers, plus the Python packages of# the family's tiny fixtures (see the vulkan-engines job in .github/workflows/ci.yml).@@ -38,11 +38,48 @@ same_tokens() { # <cpu log> <vk log> <tag>: the engines' "C engine" token lines{ [ -s cpu.tok ] && cmp -s cpu.tok vk.tok; } || { cat cpu.tok vk.tok; fail "$3: Vulkan tokens differ from the CPU"; }}-# The shader itself: every weight format against a CPU reference, before any engine.+# The shader itself: every weight format against a CPU reference, before any engine;+# then the expert batch and the weight pool in the same harness, and the routed-expert+# tier (vk_tier.c) on a synthetic model in every source format.shader_formats() {- cc -O2 -DVK_TEST backend_vulkan.c -o vk_test -lvulkan -lm+ cc -O2 -pthread -DVK_TEST backend_vulkan.c -o vk_test -lvulkan -lmCOLI_VK_TEST_MATMUL_ONLY=1 ./vk_test shaders/qmatmul.spv | tee vk_test.logtail -1 vk_test.log | grep -qx PASS || fail "qmatmul format cases"+ make tests/test_vk_tier VK=1+ ./tests/test_vk_tier shaders/qmatmul.spv | tee vk_tier.log+ tail -1 vk_tier.log | grep -qx PASS || fail "routed-expert tier"+}++# tier_count <engine> <log>: N from the last "[VK] tier <engine> run: device N of M" line;+# tier_evictions <engine> <log>: the evictions of that line+tier_count() {+ local n+ n=$(sed -n "s/^\[VK\] tier $1 run: device \([0-9][0-9]*\) of .*/\1/p" "$2" | tail -1)+ echo "${n:-0}"+}+tier_evictions() {+ local n+ n=$(sed -n "s/^\[VK\] tier $1 run: .* evictions \([0-9][0-9]*\),.*/\1/p" "$2" | tail -1)+ echo "${n:-0}"+}++# tier_gate <engine> <tag> <env...> -- <argv...>+# The routed-expert tier (vk_tier.c) against the CPU: the same tokens as the CPU run+# with the same settings, and the device served some of the routed experts. With+# EVICT=1 the run must also have evicted (its budget is set below the hot set).+# COLI_USAGE points at a fresh file: no warm start from an earlier run's history.+tier_gate() {+ local eng=$1 tag=$2; shift 2+ local envs=(); while [ "$1" != "--" ]; do envs+=("$1"); shift; done; shift+ rm -f tier.usage+ env "${envs[@]}" ./"$eng" "$@" > cpu.log 2>&1 || true+ env "${envs[@]}" COLI_USAGE=tier.usage COLI_VULKAN=1 ./"$eng" "$@" > vk.log 2>&1 || true+ same_tokens cpu.log vk.log "$tag"+ [ "$(tier_count "$eng" vk.log)" -gt 0 ] || { cat vk.log; fail "$tag: no routed expert ran on the device"; }+ if [ "${EVICT:-0}" = 1 ]; then+ [ "$(tier_evictions "$eng" vk.log)" -gt 0 ] || { grep '\[VK\] tier' vk.log; fail "$tag: the budget forced no eviction"; }+ fi+ echo "OK $tag: tokens = CPU, $(grep -a -o 'device [0-9]* of [0-9]* routed experts' vk.log | tail -1), $(grep -a -o 'evictions [0-9]*' vk.log | tail -1)"}# vk_gate <engine> <placed-regex> <tag> <env...> -- <argv...>@@ -88,6 +125,33 @@ family_qwen() {echo "OK qwen36 $fx int8: tokens = CPU, $(grep -o '[0-9]* matmuls on the GPU.*' vk.log | tail -1)"done+ # The routed-expert tier on every expert container qwen36 reads (COLI_VULKAN=1 turns+ # it on; the runs above already had it): int8 per row (fmt 1), the shared kernel's+ # planar int4-g64 (fmt 4), int4 per row from the unpacked slots (fmt 2), int8 gs64+ # (fmt 13) and the mixed int4 gate/up + int8 down container; the tier alone with the+ # dense trunk on the CPU (COLI_VK_DENSE=0); and a budget of two experts, which must+ # evict as the routing moves.+ $PY tools/make_qwen36_tiny.py --out qwen36_tiny64 --ref-mode full --inter 64 --emit-ref qwen36_tiny64/ref_full.json+ $PY tools/convert_qwen36.py --model qwen36_tiny64 --out qwen36_tiny64_c --ebits 4 --gs 64+ $PY tools/convert_qwen36.py --model qwen36_tiny64 --out qwen36_tiny64_g8 --ebits 8 --gs 64+ $PY tools/convert_qwen36.py --model qwen36_tiny64 --out qwen36_tiny64_d8 --ebits 4 --gs 64 --down-bits 8+ $PY tools/convert_qwen36.py --model qwen36_tiny --out qwen36_tiny_i4r --ebits 4+ for cap in 1 8; do+ tier_gate qwen36 "qwen36 tier int8 cap=$cap" COLI_DENSE_I8=0 SNAP=qwen36_tiny_c -- $cap 8 qwen36_tiny/ref_full.json+ tier_gate qwen36 "qwen36 tier int4-g64 planar cap=$cap" COLI_DENSE_I8=0 SNAP=qwen36_tiny64_c -- $cap 4 qwen36_tiny64/ref_full.json+ tier_gate qwen36 "qwen36 tier int4 per row cap=$cap" COLI_DENSE_I8=0 SNAP=qwen36_tiny_i4r -- $cap 4 qwen36_tiny/ref_full.json+ tier_gate qwen36 "qwen36 tier int8 gs64 cap=$cap" COLI_DENSE_I8=0 SNAP=qwen36_tiny64_g8 -- $cap 8 qwen36_tiny64/ref_full.json+ tier_gate qwen36 "qwen36 tier mixed int4/int8 cap=$cap" COLI_DENSE_I8=0 SNAP=qwen36_tiny64_d8 -- $cap 4 qwen36_tiny64/ref_full.json+ done+ tier_gate qwen36 "qwen36 tier alone (COLI_VK_DENSE=0)" COLI_VK_DENSE=0 COLI_DENSE_I8=0 SNAP=qwen36_tiny64_c -- 8 4 qwen36_tiny64/ref_full.json+ EVICT=1 tier_gate qwen36 "qwen36 tier, a budget of two experts" COLI_VK_TIER_GB=0.00002 COLI_DENSE_I8=0 SNAP=qwen36_tiny_c -- 8 8 qwen36_tiny/ref_full.json+ # COLI_VK_TIER=0: the dense trunk alone, as before the tier+ COLI_VK_TIER=0 COLI_DENSE_I8=0 SNAP=qwen36_tiny_c ./qwen36 8 8 qwen36_tiny/ref_full.json > cpu.log 2>&1 || true+ COLI_VK_TIER=0 COLI_VULKAN=1 COLI_DENSE_I8=0 SNAP=qwen36_tiny_c ./qwen36 8 8 qwen36_tiny/ref_full.json > vk.log 2>&1 || true+ same_tokens cpu.log vk.log "qwen36 COLI_VK_TIER=0"+ ! grep -q '^\[VK\] tier' vk.log || { cat vk.log; fail "qwen36 COLI_VK_TIER=0: the tier started"; }+ echo "OK qwen36 COLI_VK_TIER=0: tokens = CPU, no tier"+# qwen38: one fixture, every resident format, with and without prefill batching$PY tools/make_qwen38_tiny.py --out qwen38_tinylocal batch O@@ -100,6 +164,30 @@ family_qwen() {# Q38_NATIVE_BF16=0 expands the rows to f32 at load (fmt 10)vk_gate qwen38 'f32 [1-9]' "qwen38 f32 batch=$batch" $O Q38_TRUNK_CPU_INT8=0 Q38_NATIVE_BF16=0 -- 1 8 qwen38_tiny/ref.jsondone++ # The routed-expert tier on qwen38's three expert forms: BF16 (fmt 11), the release's+ # FP8 with 128x128 block scales (fmt 12, gs 128), the experts-int4g64 sidecar's planar+ # int4 (fmt 4); decode one row at a time and prefill batched; with the MTP head+ # drafting (its verify rows take the device's per-row route); the tier alone; and a+ # budget of two experts, which must evict.+ $PY tools/make_qwen38_tiny.py --out qwen38_tiny_fp8 --fp8-experts+ $PY tools/make_qwen38_tiny.py --out qwen38_tiny_int4 --fp8-experts --int4-experts --expert-gain 3+ local fx ref+ for fx in qwen38_tiny qwen38_tiny_fp8 qwen38_tiny_int4; do+ ref=$fx/ref.json; [ $fx = qwen38_tiny_int4 ] && ref=$fx/ref_int4.json+ for batch in 0 1; do for cap in 1 4; do+ tier_gate qwen38 "qwen38 tier $fx batch=$batch cap=$cap" OMP_NUM_THREADS=2 Q38_PREFILL_BATCH=$batch SNAP=$fx -- $cap 8 $ref+ done; done+ done+ tier_gate qwen38 "qwen38 tier alone (COLI_VK_DENSE=0)" OMP_NUM_THREADS=2 COLI_VK_DENSE=0 SNAP=qwen38_tiny_int4 -- 4 8 qwen38_tiny_int4/ref_int4.json+ EVICT=1 tier_gate qwen38 "qwen38 tier, a budget of two experts" OMP_NUM_THREADS=2 Q38_PREFILL_BATCH=0 COLI_VK_TIER_GB=0.0000065 SNAP=qwen38_tiny_fp8 -- 1 8 qwen38_tiny_fp8/ref.json+ $PY tools/make_qwen38_tiny.py --out qwen38_tiny_fp8_mtp --fp8-experts --mtp+ $PY tools/make_qwen38_tiny.py --out qwen38_tiny_int4_mtp --fp8-experts --int4-experts --expert-gain 3 --mtp+ for fx in qwen38_tiny_fp8_mtp qwen38_tiny_int4_mtp; do+ for batch in 0 1; do+ tier_gate qwen38 "qwen38 tier MTP $fx batch=$batch" OMP_NUM_THREADS=2 Q38_MTP=1 Q38_PREFILL_BATCH=$batch SNAP=$fx -- 2 8 $fx/ref.json+ done+ done}family_inkling_olmoe() {