Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

tests: vulkan_engines.sh, every engine's Vulkan path against its CPU run

JustVuggba76bf10038d
diff --git a/c/tests/vulkan_engines.sh b/c/tests/vulkan_engines.sh
new file mode 100755
index 00000000..b2359d85
--- /dev/null
+++ b/c/tests/vulkan_engines.sh
@@ -0,0 +1,254 @@
+#!/usr/bin/env bash
+# Every engine's Vulkan path against its own CPU run, on Lavapipe (Mesa's software
+# Vulkan), one family per call so CI can run them side by side:
+#
+# bash tests/vulkan_engines.sh qwen | inkling-olmoe | mimo-qwenimage | deepseek
+# bash tests/vulkan_engines.sh shader # the qmatmul formats alone, no engine
+#
+# Needs libvulkan-dev, glslc and mesa-vulkan-drivers, plus the Python packages of
+# the family's tiny fixtures (see the vulkan-engines job in .github/workflows/ci.yml).
+#
+# Lavapipe is a CPU rasteriser: nothing here says anything about speed. What it does
+# prove is that every resident format an engine uploads reaches the shader and comes
+# back as the CPU computes it. Each configuration is gated on two things:
+# - the Vulkan run gives the tokens of the CPU run with the same snapshot and
+# settings (and, where the engine has one, passes its own oracle);
+# - its "[VK] <engine>: N matmuls on the GPU" line has N > 0, because a hook that
+# declines every matrix would otherwise pass the first gate trivially.
+set -euo pipefail
+cd "$(dirname "$0")/.."
+export VK_ICD_FILENAMES=${VK_ICD_FILENAMES:-/usr/share/vulkan/icd.d/lvp_icd.json}
+export COLI_NO_OMP_TUNE=1
+PY=${PY:-python3}
+
+fail() { echo "FAIL: $*"; exit 1; }
+
+# vk_count <engine> <log>: N from the last "[VK] <engine>: N matmuls on the GPU" line
+vk_count() {
+ local n
+ n=$(sed -n "s/^\[VK\] $1: \([0-9][0-9]*\) matmuls on the GPU.*/\1/p" "$2" | tail -1)
+ echo "${n:-0}"
+}
+need_gpu() { # <engine> <log> <tag>
+ [ "$(vk_count "$1" "$2")" -gt 0 ] || { cat "$2"; fail "$3: no matmul ran on the device"; }
+}
+same_tokens() { # <cpu log> <vk log> <tag>: the engines' "C engine" token lines
+ grep -a '^C engine' "$1" > cpu.tok || true
+ grep -a '^C engine' "$2" > vk.tok || true
+ { [ -s cpu.tok ] && cmp -s cpu.tok vk.tok; } || { cat cpu.tok vk.tok; fail "$3: Vulkan tokens differ from the CPU"; }
+}
+
+# The shader itself: every weight format against a CPU reference, before any engine.
+shader_formats() {
+ cc -O2 -DVK_TEST backend_vulkan.c -o vk_test -lvulkan -lm
+ COLI_VK_TEST_MATMUL_ONLY=1 ./vk_test shaders/qmatmul.spv | tee vk_test.log
+ tail -1 vk_test.log | grep -qx PASS || fail "qmatmul format cases"
+}
+
+# vk_gate <engine> <placed-regex> <tag> <env...> -- <argv...>
+# CPU arm and Vulkan arm of one configuration; both must exit 0 (the engine's own
+# oracle), give the same tokens, run matmuls on the device, and report the expected
+# placement ("placed int8 a, int4 b, f32 c" / "int8 a, bf16 b, f32 c").
+vk_gate() {
+ local eng=$1 placed=$2 tag=$3; shift 3
+ local envs=(); while [ "$1" != "--" ]; do envs+=("$1"); shift; done; shift
+ env "${envs[@]}" ./"$eng" "$@" > cpu.log 2>&1 || { cat cpu.log; fail "$tag: CPU run"; }
+ env "${envs[@]}" COLI_VULKAN=1 ./"$eng" "$@" > vk.log 2>&1 || { cat vk.log; fail "$tag: Vulkan run misses the oracle"; }
+ same_tokens cpu.log vk.log "$tag"
+ need_gpu "$eng" vk.log "$tag"
+ grep -qE "\[VK\] $eng: .*placed .*$placed" vk.log || { grep '\[VK\]' vk.log; fail "$tag: expected placement '$placed'"; }
+ echo "OK $tag: $(grep -a -o 'Matching tokens: [0-9/]*' vk.log), tokens = CPU, $(grep -o '[0-9]* matmuls on the GPU.*' vk.log | tail -1)"
+}
+
+family_qwen() {
+ make qwen36 qwen38 VK=1
+ # qwen36: the hybrid, Qwen3-Coder (qwen3_moe), the 27B dense and the 2.4T geometry
+ $PY tools/make_qwen36_tiny.py --out qwen36_tiny --ref-mode full --emit-ref qwen36_tiny/ref_full.json
+ $PY tools/make_qwen36_tiny.py --geometry qwen3-coder-30b --out qwen3_coder_tiny --ref-mode full --emit-ref qwen3_coder_tiny/ref_full.json
+ $PY tools/make_qwen36_tiny.py --geometry qwen38-27b-dense --out qwen38_27b_tiny --ref-mode full --emit-ref qwen38_27b_tiny/ref_full.json
+ $PY tools/make_qwen36_tiny.py --geometry qwen38-2p4t --seed 3 --out qwen38_2p4t_tiny --ref-mode full --emit-ref qwen38_2p4t_tiny/ref_full.json
+ local fx cap caps
+ for fx in qwen36_tiny qwen3_coder_tiny qwen38_27b_tiny qwen38_2p4t_tiny; do
+ $PY tools/convert_qwen36.py --model $fx --out ${fx}_c --ebits 8
+ # cap=1 evicts on every routed expert; on the 92-layer 2.4T geometry that costs
+ # ~90 s on Lavapipe for nothing the device path adds, so that one runs at cap=8.
+ caps="1 8"; [ $fx = qwen38_2p4t_tiny ] && caps=8
+ for cap in $caps; do
+ # the CPU job's own configuration: f32 dense weights (fmt 10), token-exact
+ # against transformers and equal to the CPU run
+ vk_gate qwen36 'f32 [1-9]' "qwen36 $fx f32 cap=$cap" COLI_DENSE_I8=0 SNAP=${fx}_c -- $cap 8 $fx/ref_full.json
+ done
+ # int8 dense rows (fmt 1). int8 weights alone miss the torch oracle on some
+ # fixtures, CPU or GPU alike, so this arm gates on the CPU's tokens only, with
+ # COLI_DENSE_IDOT=0 giving the CPU the shader's f32 activations.
+ COLI_DENSE_IDOT=0 SNAP=${fx}_c ./qwen36 8 8 $fx/ref_full.json > cpu.log 2>&1 || true
+ COLI_DENSE_IDOT=0 COLI_VULKAN=1 SNAP=${fx}_c ./qwen36 8 8 $fx/ref_full.json > vk.log 2>&1 || true
+ same_tokens cpu.log vk.log "qwen36 $fx int8"
+ grep -qE '\[VK\] qwen36: [1-9][0-9]* matmuls on the GPU.*placed int8 [1-9]' vk.log || { cat vk.log; fail "qwen36 $fx int8: nothing placed"; }
+ echo "OK qwen36 $fx int8: tokens = CPU, $(grep -o '[0-9]* matmuls on the GPU.*' vk.log | tail -1)"
+ done
+
+ # qwen38: one fixture, every resident format, with and without prefill batching
+ $PY tools/make_qwen38_tiny.py --out qwen38_tiny
+ local batch O
+ for batch in 0 1; do
+ O="OMP_NUM_THREADS=2 SNAP=qwen38_tiny Q38_PREFILL_BATCH=$batch"
+ # the default: every fixture matrix is under 1 MiB, so the trunk stays BF16 (fmt 11)
+ vk_gate qwen38 'bf16 [1-9]' "qwen38 bf16 batch=$batch" $O -- 1 8 qwen38_tiny/ref.json
+ # Q38_TRUNK_MIN_KB=0: the int8 trunk (fmt 1); what stays outside it is BF16
+ vk_gate qwen38 'int8 [1-9].*bf16 [1-9]' "qwen38 int8 batch=$batch" $O Q38_TRUNK_MIN_KB=0 -- 1 8 qwen38_tiny/ref.json
+ # Q38_NATIVE_BF16=0 expands the rows to f32 at load (fmt 10)
+ vk_gate qwen38 'f32 [1-9]' "qwen38 f32 batch=$batch" $O Q38_TRUNK_CPU_INT8=0 Q38_NATIVE_BF16=0 -- 1 8 qwen38_tiny/ref.json
+ done
+}
+
+family_inkling_olmoe() {
+ make inkling olmoe VK=1
+ local cap
+ # inkling, f32 fixture (fmt 10): the oracle and the CPU's ids, at three caps
+ $PY tools/make_tiny_inkling.py tiny_inkling
+ for cap in 1 2 8; do
+ SNAP=tiny_inkling ./inkling $cap 0 tiny_inkling/ref_inkling.json > cpu.log 2>&1
+ COLI_VULKAN=1 SNAP=tiny_inkling ./inkling $cap 0 tiny_inkling/ref_inkling.json > vk.log 2>&1
+ grep -qE 'Matching tokens: ([0-9]+)/\1$' vk.log || { cat vk.log; fail "inkling f32 cap=$cap: oracle"; }
+ same_tokens cpu.log vk.log "inkling f32 cap=$cap"
+ need_gpu inkling vk.log "inkling f32 cap=$cap"
+ echo "OK inkling f32 cap=$cap: $(grep -o 'Matching tokens: [0-9/]*' vk.log), $(vk_count inkling vk.log) matmuls on the GPU"
+ done
+
+ # the same weights stored as bf16 (fmt 11). A build with the AVX512-BF16 dot rounds
+ # activations to bf16 on the CPU and keeps those matrices there ("0 bf16" in the
+ # placement line), so the count is only required when the line places any.
+ mkdir -p tiny_inkling_bf16
+ cp tiny_inkling/config.json tiny_inkling/generation_config.json tiny_inkling_bf16/
+ $PY - <<'EOF'
+import json, struct, numpy as np
+src, dst = "tiny_inkling/model.safetensors", "tiny_inkling_bf16/model.safetensors"
+raw = open(src, "rb").read(); n = struct.unpack("<Q", raw[:8])[0]; hdr = json.loads(raw[8:8 + n])
+out, blobs, off = {}, [], 0
+for k, v in hdr.items():
+ if k == "__metadata__": out[k] = v; continue
+ a = np.frombuffer(raw[8 + n + v["data_offsets"][0]: 8 + n + v["data_offsets"][1]], np.float32)
+ u = a.view(np.uint32).astype(np.uint64)
+ b = ((u + 0x7FFF + ((u >> 16) & 1)) >> 16).astype(np.uint16).tobytes()
+ out[k] = {"dtype": "BF16", "shape": v["shape"], "data_offsets": [off, off + len(b)]}
+ blobs.append(b); off += len(b)
+h = json.dumps(out).encode(); h += b" " * (-len(h) % 8)
+with open(dst, "wb") as f:
+ f.write(struct.pack("<Q", len(h))); f.write(h); [f.write(b) for b in blobs]
+EOF
+ SNAP=tiny_inkling_bf16 ./inkling 8 0 tiny_inkling/ref_inkling.json > cpu.log 2>&1 || true
+ COLI_VULKAN=1 SNAP=tiny_inkling_bf16 ./inkling 8 0 tiny_inkling/ref_inkling.json > vk.log 2>&1 || true
+ same_tokens cpu.log vk.log "inkling bf16"
+ grep -q ', 0 bf16)' vk.log || need_gpu inkling vk.log "inkling bf16"
+ echo "OK inkling bf16: tokens = CPU, $(vk_count inkling vk.log) matmuls on the GPU"
+
+ # the dense-int4g64 container (int8 fmt 1, int4-g64 fmt 4, f32 fmt 10 side by side)
+ rm -rf tiny_inkling_q && cp -r tiny_inkling tiny_inkling_q
+ $PY - tools/convert_inkling_dense_int4.py tiny_inkling_q <<'EOF'
+import importlib.util, sys
+spec = importlib.util.spec_from_file_location("conv", sys.argv[1])
+conv = importlib.util.module_from_spec(spec); spec.loader.exec_module(conv)
+conv.MIN_ELEMS = 0
+conv.ATTN_BITS = 4
+base = conv.classify
+conv.classify = lambda n, s, d: "int8" if n.endswith(".mlp.down_proj.weight") else base(n, s, d)
+sys.argv = ["convert_inkling_dense_int4.py", "--dir", sys.argv[2]]
+conv.main()
+EOF
+ mkdir -p tiny_inkling_q/dense-int4g64
+ mv tiny_inkling_q/dense-int4g64.safetensors tiny_inkling_q/dense-int4g64/dense.safetensors
+ SNAP=tiny_inkling_q ./inkling 8 0 tiny_inkling/ref_inkling.json > cpu.log 2>&1 || true
+ COLI_VULKAN=1 SNAP=tiny_inkling_q ./inkling 8 0 tiny_inkling/ref_inkling.json > vk.log 2>&1 || true
+ same_tokens cpu.log vk.log "inkling int4-g64 container"
+ need_gpu inkling vk.log "inkling int4-g64 container"
+ echo "OK inkling int4-g64 container: tokens = CPU, $(vk_count inkling vk.log) matmuls on the GPU"
+
+ # olmoe: f32 residents (fmt 10), the oracle and the CPU's ids
+ $PY tools/make_olmoe_tiny.py --output olmoe_tiny
+ $PY tools/convert_olmoe_merged.py --model olmoe_tiny --out olmoe_tiny_c
+ SNAP=olmoe_tiny_c ./olmoe 8 8 olmoe_tiny/ref_olmoe.json > cpu.log 2>&1
+ COLI_VULKAN=1 SNAP=olmoe_tiny_c ./olmoe 8 8 olmoe_tiny/ref_olmoe.json > vk.log 2>&1
+ grep -qE 'Matching tokens: ([0-9]+)/\1$' vk.log || { cat vk.log; fail "olmoe: oracle"; }
+ same_tokens cpu.log vk.log "olmoe"
+ need_gpu olmoe vk.log "olmoe"
+ echo "OK olmoe: $(grep -o 'Matching tokens: [0-9/]*' vk.log), $(vk_count olmoe vk.log) matmuls on the GPU"
+}
+
+family_mimo_qwenimage() {
+ make mimo qwenimage VK=1
+ # mimo: Xiaomi's vendor oracle on the GPU (its engine_env is the f32 dense
+ # configuration; its variants include the native FP8/BF16 one and the BF16 vision
+ # tower), once with the experts on the CPU, once all on the GPU (MXFP4, fmt 7).
+ $PY tools/make_mimo_tiny.py --output ./mimo_tiny --force --vision
+ COLI_VULKAN=1 $PY tests/mimo_tiny_harness.py --binary ./mimo --fixture ./mimo_tiny
+ COLI_VULKAN=1 MIMO_VK_EXPERTS=4096 $PY tests/mimo_tiny_harness.py --binary ./mimo --fixture ./mimo_tiny
+ # MIMO_DENSE_BITS 0 (native fp8/bf16: fmt 12, 11), 8 (fmt 1) and 32 (fmt 10):
+ # the CPU's tokens for every case of ref.json and for the picture.
+ ids() { $PY -c "import json,sys;r=json.load(open('mimo_tiny/ref.json'));c=r['image'] if sys.argv[1]=='image' else r['cases'][sys.argv[1]];print(' '.join(map(str,c['prompt_ids'])))" "$1"; }
+ local grid bits c x extra
+ grid=$($PY -c "import json;i=json.load(open('mimo_tiny/ref.json'))['image'];print(i['grid_h'],i['grid_w'])")
+ for bits in 0 8 32; do
+ for c in short window long image; do
+ extra=(); [ "$c" = image ] && extra=(--image mimo_tiny/patches.f32 --grid $grid)
+ MIMO_DENSE_BITS=$bits COLI_TEMP=0 ./mimo mimo_tiny --ids "$(ids $c)" --ngen 6 "${extra[@]}" > mimo-cpu.txt 2>/dev/null
+ for x in 0 4096; do
+ COLI_VULKAN=1 MIMO_VK_EXPERTS=$x MIMO_DENSE_BITS=$bits COLI_TEMP=0 \
+ ./mimo mimo_tiny --ids "$(ids $c)" --ngen 6 "${extra[@]}" > mimo-vk.txt 2> mimo-vk.err
+ cmp -s mimo-cpu.txt mimo-vk.txt || { cat mimo-vk.err; fail "mimo bits=$bits $c MIMO_VK_EXPERTS=$x differs from the CPU"; }
+ need_gpu mimo mimo-vk.err "mimo bits=$bits $c MIMO_VK_EXPERTS=$x"
+ done
+ done
+ echo "OK mimo MIMO_DENSE_BITS=$bits: the CPU's tokens, text and image, experts on the CPU and on the GPU"
+ done
+
+ # qwenimage: at 8, 16 and 32 bits (fmt 1, 11, 10) every oracle stage of the
+ # Lavapipe run against the CPU run with the same bits and f32 activations. int8 is
+ # outside the oracle's own tolerance on both sides, so the reports are compared
+ # with each other, not with the reference; at 16 bits the Vulkan run must also pass.
+ $PY tools/make_qwenimage_tiny.py qwenimage_tiny
+ for bits in 8 16 32; do
+ COLI_IMG_BITS=$bits COLI_IMG_ACT8=0 ./qwenimage --model qwenimage_tiny --ref qwenimage_tiny/ref > qi-cpu.log 2>&1 || true
+ local rc=0
+ COLI_VULKAN=1 COLI_IMG_BITS=$bits COLI_IMG_ACT8=0 ./qwenimage --model qwenimage_tiny --ref qwenimage_tiny/ref > qi-vk.log 2>&1 || rc=$?
+ [ $bits != 16 ] || [ $rc = 0 ] || { cat qi-vk.log; fail "qwenimage bf16 oracle on Vulkan"; }
+ need_gpu qwenimage qi-vk.log "qwenimage oracle bits=$bits"
+ BITS=$bits $PY - <<'PY'
+import os, re
+def stages(p):
+ return {k.strip(): float(r) for k, r in
+ re.findall(r"\[oracle\] (.+?)\s+n=\d+\s+max\|err\| \S+\s+rel (\S+)", open(p).read())}
+c, v = stages("qi-cpu.log"), stages("qi-vk.log")
+assert len(c) >= 10 and c.keys() == v.keys(), (c, v)
+for k in c: # relative errors against the reference, CPU vs GPU: equal up to summation order
+ assert abs(c[k] - v[k]) <= 0.25 * c[k] + 2e-7, (k, c[k], v[k])
+print(f"OK qwenimage oracle: {len(c)} stages of the Lavapipe run match the CPU run at {os.environ['BITS']} bits")
+PY
+ done
+ # a generated picture at the default int8 weights, Lavapipe against the CPU
+ COLI_IMG_BITS=8 COLI_IMG_ACT8=0 ./qwenimage --model qwenimage_tiny --prompt "a red fox in the snow" \
+ --width 256 --height 256 --steps 2 --seed 1 --out qi-cpu.png
+ COLI_VULKAN=1 COLI_IMG_BITS=8 COLI_IMG_ACT8=0 ./qwenimage --model qwenimage_tiny --prompt "a red fox in the snow" \
+ --width 256 --height 256 --steps 2 --seed 1 --out qi-vk.png 2> qi-vk-gen.err
+ need_gpu qwenimage qi-vk-gen.err "qwenimage picture"
+ $PY - <<'PY'
+import sys; sys.path.insert(0, ".")
+import image_engine as e
+_, _, _, a = e.decode_png(open("qi-cpu.png", "rb").read())
+_, _, _, b = e.decode_png(open("qi-vk.png", "rb").read())
+d = [abs(x - y) for x, y in zip(a, b)]
+assert len(a) == len(b) and max(d) <= 2 and sum(t > 0 for t in d) <= len(d) // 1000, (max(d), sum(t > 0 for t in d))
+print(f"OK qwenimage picture: {sum(t > 0 for t in d)} of {len(d)} bytes differ from the CPU's, max {max(d)}")
+PY
+ # the serve protocol with the device on
+ COLI_VULKAN=1 QWENIMAGE_TINY=qwenimage_tiny $PY -m unittest tests.test_qwenimage_engine_serve
+}
+
+case "${1:-}" in
+ shader) shader_formats ;;
+ qwen) family_qwen ;;
+ inkling-olmoe) family_inkling_olmoe ;;
+ mimo-qwenimage) family_mimo_qwenimage ;;
+ deepseek) family_deepseek ;;
+ *) echo "usage: $0 shader|qwen|inkling-olmoe|mimo-qwenimage|deepseek" >&2; exit 2 ;;
+esac