Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
tests: Kimi K3's dense chain on Lavapipe, and under ASan and UBSan
JustVugg1b712da4758f
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 8cced225..00dcdb31 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -206,6 +206,12 @@ jobs:req: c/tools/requirements-glm53-tiny.txt- family: glm-chain-sanitizereq: c/tools/requirements-glm53-tiny.txt+ # the dense chain (COLI_VK_CHAIN=1) of Kimi K3: its attention residuals, KDA+ # and gated MLA layers on the device, and the same under ASan and UBSan+ - family: kimi-chain+ req: c/tools/requirements-kimi-k3-tiny.txt+ - family: kimi-chain-sanitize+ req: c/tools/requirements-kimi-k3-tiny.txtsteps:- uses: actions/checkout@v4- uses: actions/setup-python@v5diff --git a/c/tests/vulkan_chain_serve.py b/c/tests/vulkan_chain_serve.pyindex 48ff5748..706448a5 100644--- a/c/tests/vulkan_chain_serve.py+++ b/c/tests/vulkan_chain_serve.py@@ -7,7 +7,10 @@ The turns exercise what moves the chain's device state around: a pinned snapshota prompt that diverges and one that starts over, a cache that grows between turns,and the prefill read-out (logprobs=k, ECHO frames). Token ids and texts must be theCPU's exactly; the logprobs printed with them may differ in their last digits (the-device sums in another order), so numbers compare within 1e-4.+device sums in another order), so numbers compare within 1e-4 (CHAIN_SERVE_TOL sets+another bound, for an engine whose fixture amplifies rounding more; the OK line gives+the largest difference seen). CHAIN_SERVE_EXPECT, a regular expression the chain+session's stderr must match (a lost device's rebuild, say, with COLI_VK_CHAIN_FAULT).usage: vulkan_chain_serve.py <engine> <snapshot> [KEY=VALUE ...] (extra environment)COLI_VK_CHAIN in the caller's environment picks the chain's mode (default 1; 2 runs@@ -20,6 +23,7 @@ i % n (the engine needs KV_SLOTS=n): the turns move between sessions, and with twhatever state the chain keeps on the device."""import os+import reimport subprocessimport sysimport threading@@ -78,6 +82,10 @@ def session(engine, snap, extra, chain):return out, b"".join(err).decode(errors="replace")+TOL = float(os.environ.get("CHAIN_SERVE_TOL", "1e-4"))+WORST = [0.0]++def close(a, b):if a == b:return True@@ -88,8 +96,9 @@ def close(a, b):if x == y:continuetry:- if b"." not in x or abs(float(x) - float(y)) > 1e-4:+ if b"." not in x or abs(float(x) - float(y)) > TOL:return False+ WORST[0] = max(WORST[0], abs(float(x) - float(y)))except ValueError:return Falsereturn True@@ -108,6 +117,9 @@ def main():pins = err.count("[PIN]")if not forwards:sys.exit("FAIL: the chain never ran\n" + err[-3000:])+ expect = os.environ.get("CHAIN_SERVE_EXPECT")+ if expect and not re.search(expect, err):+ sys.exit(f"FAIL: the chain session's stderr has no {expect!r}\n" + err[-3000:])if len(cpu) != len(dev) or not all(close(a, b) for a, b in zip(cpu, dev)):for a, b in zip(cpu, dev):if not close(a, b):@@ -115,7 +127,8 @@ def main():breaksys.exit(f"FAIL: the chain's frames differ from the CPU's ({len(cpu)} vs {len(dev)})")print(f"OK serve {os.path.basename(snap)} {' '.join(sys.argv[3:])}: {len(cpu)} frames = CPU, "- f"{pins} pin lines, {len(reuse)} turns reusing state, {forwards[-1].split('] ', 1)[1][:60]}")+ f"{pins} pin lines, {len(reuse)} turns reusing state, numbers within {WORST[0]:.1e}, "+ f"{forwards[-1].split('] ', 1)[1][:60]}")if __name__ == "__main__":diff --git a/c/tests/vulkan_engines.sh b/c/tests/vulkan_engines.shindex 850c5afd..f4c8b46b 100755--- a/c/tests/vulkan_engines.sh+++ b/c/tests/vulkan_engines.sh@@ -8,6 +8,7 @@# bash tests/vulkan_engines.sh shader # the qmatmul formats, the expert batch and the tier, no engine# bash tests/vulkan_engines.sh qwen-chain | qwen-chain-sanitize # the dense chain (COLI_VK_CHAIN=1)# bash tests/vulkan_engines.sh glm-chain | glm-chain-sanitize # the same for colibri and glm53+# bash tests/vulkan_engines.sh kimi-chain | kimi-chain-sanitize # the same for Kimi K3## Needs libvulkan-dev, glslc and mesa-vulkan-drivers, plus the Python packages of# the family's tiny fixtures (see the vulkan-engines job in .github/workflows/ci.yml).@@ -1529,6 +1530,190 @@ family_glm_chain_sanitize() {make clean >/dev/null 2>&1 || true}+# Kimi K3's dense chain (kimi_k3_chain.h). The tolerance: the tiny fixture amplifies+# rounding at a few positions. The CPU against itself, with only its RMSNorm's sum taken+# in float instead of double, moves the logits by up to 1.4e-4 of the largest one on the+# f32 trunk and 1.0e-3 on the 8-bit one, and the served logprobs by up to 4.1e-3 (int4+# trunk), at the positions where the chain moves them most (measured on Lavapipe: 1.8e-4+# and 4.5e-4; logprobs 6.1e-3): every logits row is held within 2e-3 of the largest+# |logit|, the logprobs of the serve sessions within 2e-2, the tokens exactly. With the CPU's int8 expert activations+# (K3_IDOT=1) a rounding that flips an int8 step moves the logits further (8e-3, the+# tokens unchanged): that configuration gates on its tokens (TOKENS=1).+k3c_ids() { $PY -c "import json,sys;print(' '.join(map(str,json.load(open('kimi_k3_tiny/ref.json'))['cases'][sys.argv[1]]['prompt_ids'])))" "$1"; }+k3c_close() { # <cpu.f32> <vk.f32>: max |diff| within 2e-3 of the largest |logit|+ $PY - "$1" "$2" <<'PY'+import array, sys+a = array.array("f", open(sys.argv[1], "rb").read()); b = array.array("f", open(sys.argv[2], "rb").read())+d = max(abs(x - y) for x, y in zip(a, b)) if a and len(a) == len(b) else float("inf")+m = max(abs(x) for x in a) if a else 0.0+print(f"max |logit diff| {d / m if m else d:.1e} of the largest")+sys.exit(0 if d <= 2e-3 * m else 1)+PY+}+# k3c_gate <tag> <env...>: for each of the oracle's prompts (K3C_CASES, default all+# three), the CPU run's tokens, every logits row (K3_VAL_LOGITS: every prefill row and+# every decode step) within k3c_close (TOKENS=1: reported, not gated), and the chain+# ran. EVICT=1: the tier evicted.+# FAULT_BACK=k: the device is lost k frames before the end of the same run without a+# fault (counted from that run's frames, so on any device the loss lands in the same+# forward), and the run must say so; REBUILD=1: and rebuild the KDA state of some+# positions on the CPU.+k3c_gate() {+ local tag=$1 c frames fault; shift+ for c in ${K3C_CASES:-short chunk long}; do+ rm -f k3c.usage cpu.f32 vk.f32+ env "$@" COLI_USAGE=$PWD/k3c.usage USAGE_SAVE=0 K3_VAL_LOGITS=cpu.f32 ./kimi_k3 kimi_k3_tiny --ids "$(k3c_ids $c)" --ngen 8 \+ 2> cpu.log | sed 's/ *TUNE.*//' > cpu.tok+ fault=""+ if [ -n "${FAULT_BACK:-}" ]; then+ env "$@" COLI_USAGE=$PWD/k3c.usage USAGE_SAVE=0 COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 \+ ./kimi_k3 kimi_k3_tiny --ids "$(k3c_ids $c)" --ngen 8 2> vk.log > /dev/null+ frames=$(sed -n 's/^\[VK\] kimi_k3 chain: [0-9]* forwards, \([0-9]*\) frames.*/\1/p' vk.log | tail -1)+ [ -n "$frames" ] && [ "$frames" -gt "$FAULT_BACK" ] || { cat vk.log; fail "$tag $c: no fault-free run to count frames from"; }+ fault="COLI_VK_CHAIN_FAULT=$((frames - FAULT_BACK + 1))"+ fi+ env "$@" $fault COLI_USAGE=$PWD/k3c.usage USAGE_SAVE=0 K3_VAL_LOGITS=vk.f32 COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 \+ COLI_VK_CHAIN=${CHAINMODE:-1} ./kimi_k3 kimi_k3_tiny --ids "$(k3c_ids $c)" --ngen 8 2> vk.log | sed 's/ *TUNE.*//' > vk.tok+ { [ -s cpu.tok ] && cmp -s cpu.tok vk.tok; } || { cat cpu.tok vk.tok; tail -20 vk.log; fail "$tag $c: the chain's tokens differ from the CPU"; }+ if [ -n "$fault" ]; then+ grep -q "kimi_k3 chain: the device was lost" vk.log || { cat vk.log; fail "$tag $c: no loss was handled"; }+ if [ "${REBUILD:-0}" = 1 ]; then grep -q "rebuilding the state of [1-9]" vk.log || { cat vk.log; fail "$tag $c: no state was rebuilt"; }; fi+ else+ [ "$(chain_count kimi_k3 vk.log)" -gt 0 ] || { cat vk.log; fail "$tag $c: the chain never ran"; }+ fi+ if [ "${EVICT:-0}" = 1 ]; then+ [ "$(tier_evictions kimi_k3 vk.log)" -gt 0 ] || { grep '\[VK\] tier' vk.log; fail "$tag $c: the budget forced no eviction"; }+ grep -q ' failed 0 ' vk.log || { grep '\[VK\] tier' vk.log; fail "$tag $c: an upload failed"; }+ fi+ local lg; lg=$(k3c_close cpu.f32 vk.f32) || { echo "$lg"; [ "${TOKENS:-0}" = 1 ] || fail "$tag $c: logits"; }+ echo "OK $tag $c: tokens = CPU, $lg, $(chain_count kimi_k3 vk.log) chain forwards$(grep -o 'rebuilding the state of [0-9]* positions\|the host.s state is current' vk.log | sed 's/^/, /')"+ done+}+k3c_serve_fixture() { # the tiny fixture with the tokenizer the serve tests speak through+ rm -rf kimi_k3_serve && mkdir kimi_k3_serve+ cp kimi_k3_tiny/config.json kimi_k3_tiny/model.safetensors kimi_k3_serve/+ cp tests/tok_kimi_tiny.json kimi_k3_serve/tokenizer.json+}++# Every configuration the chain takes: Moonshot's oracle with the chain on (the tier on,+# off, the shared experts on the device), every dense format (f32, int8 rows, int4-g64,+# mixed), the CPU's int8 expert activations with the tier off, prefill a token at a+# time and in chain chunks of 3, the tiled GEMM and the per-row GEMV, K3_TOPP, a tier+# budget that evicts, prompts only (the KDA state crossing between the device and the+# CPU at every decode), a device lost mid-decode (the KDA state rebuilt on the CPU), in+# the first prompt chunk (the host's state current) and in a later one (rebuilt); serve+# sessions frame for frame (pins, the prompt cache, prefix reuse, a prompt that diverges+# and one that starts over, the prefill read-out), with recurrent-state checkpoints+# (COLI_K3_CKPT: photos taken from the device's state and restored to it), in chain+# chunks, on prompts only, with a device lost mid-session; the checkpoint and dashboard+# harnesses with the chain on.+family_kimi_chain() {+ make kimi_k3 tests/test_vk_chain VK=1+ ./tests/test_vk_chain shaders/qmatmul.spv | tee vk_chain.log+ tail -1 vk_chain.log | grep -qx PASS || fail "the chain's ops"+ $PY tools/make_kimi_k3_tiny.py --output ./kimi_k3_tiny --force+ k3c_serve_fixture+ export OMP_NUM_THREADS=2+ local e b out frames+ for e in COLI_VK_TIER=1 COLI_VK_TIER=0 COLI_VK_DENSE=1; do+ env $e COLI_USAGE=$PWD/k3c.usage USAGE_SAVE=0 COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 \+ $PY tests/test_kimi_k3_tiny.py --binary ./kimi_k3 --fixture ./kimi_k3_tiny+ echo "OK chain kimi_k3 vendor oracle ($e)"+ done+ local O="K3_BITS=32 K3_MLA_BITS=32 K3_HEAD_BITS=32 K3_IDOT=0 COLI_TEMP=0"+ k3c_gate "chain kimi_k3 f32" $O+ k3c_gate "chain kimi_k3 f32, tier off" $O COLI_VK_TIER=0+ k3c_gate "chain kimi_k3 f32, shared experts by COLI_VK_DENSE=1" $O COLI_VK_DENSE=1+ for b in 8 4; do # int8 rows (fmt 1); int4-g64 (fmt 4) where a row is whole groups, int8 elsewhere+ k3c_gate "chain kimi_k3 ${b}-bit trunk" K3_BITS=$b K3_MLA_BITS=$b K3_HEAD_BITS=$b K3_IDOT=0 COLI_TEMP=0+ done+ k3c_gate "chain kimi_k3 int4 KDA and MoE, int8 MLA and head" K3_BITS=4 K3_IDOT=0 COLI_TEMP=0+ TOKENS=1 k3c_gate "chain kimi_k3 int8 expert activations, tier off" K3_BITS=32 K3_MLA_BITS=32 K3_HEAD_BITS=32 COLI_TEMP=0 COLI_VK_TIER=0+ k3c_gate "chain kimi_k3 prefill one token at a time" $O K3_CHUNK=1+ k3c_gate "chain kimi_k3 chain chunks of 3" $O COLI_VK_CHAIN_ROWS=3+ k3c_gate "chain kimi_k3 tiled GEMM from 2 rows" $O COLI_VK_GEMM_MIN_S=2+ k3c_gate "chain kimi_k3 the per-row GEMV" $O COLI_VK_CHAIN_GEMV=0+ k3c_gate "chain kimi_k3 K3_TOPP=0.6" $O K3_TOPP=0.6+ EVICT=1 k3c_gate "chain kimi_k3 a tier budget of two experts" $O COLI_VK_TIER_GB=0.000005+ CHAINMODE=2 k3c_gate "chain kimi_k3 prompts only" $O+ # the device lost: 9 frames a forward on this fixture (two per sparse layer and the+ # head); the long prompt is three forwards (32, 32 and 8 rows), then 7 decode steps+ FAULT_BACK=3 REBUILD=1 k3c_gate "chain kimi_k3 device lost mid-decode" $O+ K3C_CASES=long FAULT_BACK=86 k3c_gate "chain kimi_k3 device lost in the first prompt chunk" $O+ K3C_CASES=long FAULT_BACK=77 REBUILD=1 k3c_gate "chain kimi_k3 device lost in a later prompt chunk" $O+ # in chain chunks of 5 rows (seven chunks a 32-row forward), the loss in the third+ # chunk of the second forward: the state its first chunks advanced is not used+ K3C_CASES=long FAULT_BACK=124 REBUILD=1 k3c_gate "chain kimi_k3 device lost inside a chunked forward" $O COLI_VK_CHAIN_ROWS=5+ # serve sessions frame for frame, the KDA state across turns+ local S="K3_BITS=32 K3_MLA_BITS=32 K3_HEAD_BITS=32 K3_IDOT=0 K3_PREFIX_LOG=1 USAGE_SAVE=0"+ export CHAIN_SERVE_TOL=2e-2+ rm -rf k3c_photos && mkdir k3c_photos+ # shellcheck disable=SC2086+ {+ $PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve $S+ out=$($PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve $S COLI_K3_CKPT=4); echo "$out"+ $PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve $S COLI_K3_CKPT=2 COLI_K3_CKPT_DIR=$PWD/k3c_photos+ $PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve K3_BITS=4 K3_IDOT=0 K3_PREFIX_LOG=1 USAGE_SAVE=0 COLI_VK_CHAIN_ROWS=3+ COLI_VK_CHAIN=2 $PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve $S COLI_K3_CKPT=4+ # a device lost mid-session, 134 frames before the end of the same session without a+ # fault (on Lavapipe frame 200 of 334: after a photo brought the state to the host,+ # the state of the 3 positions since then is rebuilt from it)+ frames=$(echo "$out" | sed -n 's/.*kimi_k3 chain: [0-9]* forwards, \([0-9]*\) frames.*/\1/p')+ [ -n "$frames" ] && [ "$frames" -gt 134 ] || fail "chain kimi_k3 serve: no fault-free session to count frames from"+ CHAIN_SERVE_EXPECT='kimi_k3 chain: the device was lost' \+ $PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve $S COLI_K3_CKPT=4 COLI_VK_CHAIN_FAULT=$((frames - 134))+ }+ unset CHAIN_SERVE_TOL+ # recurrent-state checkpoints and the dashboard's lines with the chain on+ COLI_USAGE=$PWD/k3c.usage USAGE_SAVE=0 COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 \+ $PY tests/test_kimi_k3_ckpt.py --binary ./kimi_k3 --fixture ./kimi_k3_tiny+ COLI_USAGE=$PWD/k3c.usage USAGE_SAVE=0 COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 \+ $PY tests/test_kimi_k3_dashboard.py --binary ./kimi_k3 --fixture ./kimi_k3_tiny+ unset OMP_NUM_THREADS+}++# The same chain under ASan and UBSan: memory safety is the gate; each run must have+# run the chain (or handled the loss).+family_kimi_chain_sanitize() {+ make clean >/dev/null 2>&1 || true+ make kimi_k3 tests/test_vk_chain VK=1 EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"+ export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1+ ./tests/test_vk_chain shaders/qmatmul.spv > san.log 2>&1 || true+ if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log || ! tail -1 san.log | grep -qx PASS; then cat san.log; fail "asan: the chain's ops"; fi+ echo "OK asan: the chain's ops"+ $PY tools/make_kimi_k3_tiny.py --output ./kimi_k3_tiny --force+ k3c_serve_fixture+ export OMP_NUM_THREADS=2+ k3san() { # <tag> <env and argv...>+ local tag=$1; shift+ rm -f k3c.usage+ env COLI_USAGE=$PWD/k3c.usage USAGE_SAVE=0 COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 "$@" > san.log 2>&1 || true+ if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "$tag: sanitizer diagnostic"; fi+ grep -q "kimi_k3 chain: \([1-9][0-9]* forwards\|the device was lost\)" san.log || { cat san.log; fail "$tag: the chain never ran"; }+ echo "OK $tag: sanitizers clean, $(chain_count kimi_k3 san.log) chain forwards"+ }+ local O="K3_BITS=32 K3_MLA_BITS=32 K3_HEAD_BITS=32 K3_IDOT=0 COLI_TEMP=0" ids frames+ ids=$(k3c_ids long)+ # shellcheck disable=SC2086+ {+ k3san "asan chain kimi_k3 f32" env $O ./kimi_k3 kimi_k3_tiny --ids "$ids" --ngen 8+ frames=$(sed -n 's/^\[VK\] kimi_k3 chain: [0-9]* forwards, \([0-9]*\) frames.*/\1/p' san.log | tail -1)+ k3san "asan chain kimi_k3 int4 trunk, chain chunks of 3" env K3_BITS=4 K3_IDOT=0 COLI_VK_CHAIN_ROWS=3 ./kimi_k3 kimi_k3_tiny --ids "$ids" --ngen 8+ k3san "asan chain kimi_k3 tier off, every logits row" env $O COLI_VK_TIER=0 K3_VAL_LOGITS=san.f32 ./kimi_k3 kimi_k3_tiny --ids "$ids" --ngen 8+ k3san "asan chain kimi_k3 prompts only" env $O COLI_VK_CHAIN=2 ./kimi_k3 kimi_k3_tiny --ids "$ids" --ngen 8+ # the device lost mid-decode: 3 frames before the end of the run above (a fixed frame+ # number can land in the buffers' setup on a device whose memory is not mapped)+ k3san "asan chain kimi_k3 device lost, rebuilt" env $O COLI_VK_CHAIN_FAULT=$((frames - 2)) ./kimi_k3 kimi_k3_tiny --ids "$ids" --ngen 8+ grep -q "rebuilding the state of [1-9]" san.log || { cat san.log; fail "asan chain kimi_k3 device lost: no state was rebuilt"; }+ }+ CHAIN_SERVE_TOL=2e-2 $PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve K3_BITS=32 K3_MLA_BITS=32 K3_HEAD_BITS=32 K3_IDOT=0 \+ K3_PREFIX_LOG=1 USAGE_SAVE=0 COLI_K3_CKPT=4 > san.log 2>&1 || { cat san.log; fail "asan chain kimi_k3 serve"; }+ if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "asan chain kimi_k3 serve: sanitizer diagnostic"; fi+ echo "OK asan chain kimi_k3 serve: $(tail -1 san.log)"+ unset OMP_NUM_THREADS+ make clean >/dev/null 2>&1 || true+}+case "${1:-}" inshader) shader_formats ;;qwen) family_qwen ;;@@ -1546,5 +1731,7 @@ case "${1:-}" inqwen-chain-sanitize) family_qwen_chain_sanitize ;;glm-chain) family_glm_chain ;;glm-chain-sanitize) family_glm_chain_sanitize ;;- *) echo "usage: $0 shader|qwen|qwen-sanitize|inkling-olmoe|inkling-olmoe-sanitize|mimo-qwenimage|deepseek|deepseek-sanitize|kimi|kimi-mimo-sanitize|glm|glm-sanitize|qwen-chain|qwen-chain-sanitize|glm-chain|glm-chain-sanitize" >&2; exit 2 ;;+ kimi-chain) family_kimi_chain ;;+ kimi-chain-sanitize) family_kimi_chain_sanitize ;;+ *) echo "usage: $0 shader|qwen|qwen-sanitize|inkling-olmoe|inkling-olmoe-sanitize|mimo-qwenimage|deepseek|deepseek-sanitize|kimi|kimi-mimo-sanitize|glm|glm-sanitize|qwen-chain|qwen-chain-sanitize|glm-chain|glm-chain-sanitize|kimi-chain|kimi-chain-sanitize" >&2; exit 2 ;;esac