Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
#!/usr/bin/env bash
# Every engine's Vulkan path against its own CPU run, on Lavapipe (Mesa's software
# Vulkan), one family per call so CI can run them side by side:
#
# bash tests/vulkan_engines.sh qwen | qwen-sanitize | inkling-olmoe | inkling-olmoe-sanitize
# bash tests/vulkan_engines.sh mimo-qwenimage | kimi | kimi-mimo-sanitize | deepseek | deepseek-sanitize
# bash tests/vulkan_engines.sh glm | glm-sanitize # GLM-5.2 (colibri) and GLM-5.3 Flash (glm53)
# bash tests/vulkan_engines.sh shader # the qmatmul formats, the expert batch and the tier, no engine
# bash tests/vulkan_engines.sh qwen-chain | qwen-chain-sanitize # the dense chain (COLI_VK_CHAIN=1)
# bash tests/vulkan_engines.sh qwen-spec | qwen-spec-sanitize # MTP and prompt-lookup verifies (docs/speculative.md)
# bash tests/vulkan_engines.sh mimo-chain | mimo-chain-sanitize # MiMo-V2.6's dense chain
# bash tests/vulkan_engines.sh inkling-olmoe-chain | inkling-olmoe-chain-sanitize
# bash tests/vulkan_engines.sh glm-chain | glm-chain-sanitize # the same for colibri and glm53
# bash tests/vulkan_engines.sh kimi-chain | kimi-chain-sanitize # the same for Kimi K3
# bash tests/vulkan_engines.sh deepseek-chain | deepseek-chain-sanitize # deepseek_v41 and deepseek_v4
# bash tests/vulkan_engines.sh prefill-qwen | prefill-qwen-sanitize | prefill-inkling-olmoe | prefill-mimo-kimi
# bash tests/vulkan_engines.sh prefill-glm | prefill-deepseek # big prompt chunks and expert streaming
# bash tests/vulkan_engines.sh kv-split | kv-split-sanitize # the KV cache split between the device and
# # the host past the device's budget, every chain engine
# bash tests/vulkan_engines.sh kv-split-deepseek | kv-split-deepseek-sanitize # the same for DeepSeek's
# bash tests/vulkan_engines.sh staged # staged uploads: the same bits as mapped memory, the decision
# bash tests/vulkan_engines.sh <family>-staged # a family with COLI_VK_STAGED=1 (qwen-staged: and
# # an engine under an emulated small window)
# bash tests/vulkan_engines.sh staged-faults | staged-faults-sanitize # staged uploads failing
# bash tests/vulkan_engines.sh dense-only-<group> | dense-only-<group>-sanitize # the dense weights
# on the device only (COLI_VK_DENSE_HOST=0), tests/vulkan_dense_only_<group>.sh
# bash tests/vulkan_engines.sh partial-<group> | partial-<group>-sanitize # the partial chain:
# the first N layers on the device (COLI_VK_CHAIN_LAYERS, COLI_VK_DEVICE_CAP_MB),
# tests/vulkan_partial_<group>.sh
# bash tests/vulkan_engines.sh decide | decide-sanitize # Laya, GLiNER2.5-Decide and Clef's DECIDE
# bash tests/vulkan_engines.sh mux | mux-sanitize | mux-deepseek | mux-deepseek-sanitize # several
# conversations at once (KV_SLOTS), every engine but colibri and glm53 (glm-chain has theirs)
# bash tests/vulkan_engines.sh dev2 | dev2-deepseek-kimi-mimo | dev2-sanitize # the expert tier on
# two devices (COLI_VK_DEV2), every MoE engine, tests/vulkan_dev2.sh
#
# Needs libvulkan-dev, glslc and mesa-vulkan-drivers, plus the Python packages of
# the family's tiny fixtures (see the vulkan-engines job in .github/workflows/ci.yml).
#
# Lavapipe is a CPU rasteriser: nothing here says anything about speed. What it does
# prove is that every resident format an engine uploads reaches the shader and comes
# back as the CPU computes it. Each configuration is gated on two things:
# - the Vulkan run gives the tokens of the CPU run with the same snapshot and
# settings (and, where the engine has one, passes its own oracle);
# - its per-matrix and dense-chain matmul counts sum to N > 0, because a hook that
# declines every matrix would otherwise pass the first gate trivially. A
# configuration of the routed-expert tier with the dense trunk on the CPU (the
# default on Lavapipe while the tier is on, see dense_where) gates on its count of
# routed experts the device served instead.
set -euo pipefail
cd "$(dirname "$0")/.."
export VK_ICD_FILENAMES=${VK_ICD_FILENAMES:-/usr/share/vulkan/icd.d/lvp_icd.json}
export COLI_NO_OMP_TUNE=1
PY=${PY:-python3}
# Speculative decoding is on by default (MTP and prompt lookup, docs/speculative.md): its
# verifies change how many rows a forward dumps and when (the gate decides from measured
# times), which the families' row-by-row logits comparisons do not expect. They run with it
# off; the cases that test it turn it on themselves (Q38_MTP=1, COLI_LOOKUP=1, DRAFT=n).
export COLI_LOOKUP=${COLI_LOOKUP:-0} Q38_MTP=${Q38_MTP:-0}
fail() { echo "FAIL: $*"; exit 1; }
# Every sanitized run below sets ASAN_OPTIONS with detect_stack_use_after_return=0:
# ASan's fake stack frames are only 32-byte aligned, and on an AVX-512 runner
# (-march=native) GCC keeps 64-byte aligned locals there and stores them with
# vmovdqa64, which faults (c/Makefile, ASAN_ENV, has the details).
# vk_count <engine> <log>: the latest per-matrix and dense-chain matmul counts.
# The counters are independent and cumulative; sum their last reports, not turns.
vk_count() {
local n chain
n=$(sed -n "s/^\[VK\] $1: \([0-9][0-9]*\) matmuls on the GPU.*/\1/p" "$2" | tail -1)
chain=$(sed -n "s/^\[VK\] $1 chain: .* ops, \([0-9][0-9]*\) matmuls,.*/\1/p" "$2" | tail -1)
echo "$(( ${n:-0} + ${chain:-0} ))"
}
need_gpu() { # <engine> <log> <tag>
[ "$(vk_count "$1" "$2")" -gt 0 ] || { cat "$2"; fail "$3: no matmul ran on the device"; }
}
same_tokens() { # <cpu log> <vk log> <tag>: the engines' "C engine" token lines
grep -a '^C engine' "$1" > cpu.tok || true
grep -a '^C engine' "$2" > vk.tok || true
{ [ -s cpu.tok ] && cmp -s cpu.tok vk.tok; } || { cat cpu.tok vk.tok; fail "$3: Vulkan tokens differ from the CPU"; }
}
# The shader itself: every weight format against a CPU reference, before any engine;
# then the expert batch and the weight pool in the same harness, and the routed-expert
# tier (vk_tier.c) on a synthetic model in every source format.
shader_formats() {
make tests/test_vk_tier tests/test_glm53_vk_f32 VK=1 # every shader too: the harness's expert batch needs them
cc -O2 -pthread -DVK_TEST backend_vulkan.c -o vk_test -ldl -lm
COLI_VK_TEST_MATMUL_ONLY=1 ./vk_test shaders/qmatmul.spv | tee vk_test.log
tail -1 vk_test.log | grep -qx PASS || fail "qmatmul format cases"
./tests/test_vk_tier shaders/qmatmul.spv | tee vk_tier.log
tail -1 vk_tier.log | grep -qx PASS || fail "routed-expert tier"
./tests/test_glm53_vk_f32
COLI_VK_GEMM_MIN_S=0 ./tests/test_glm53_vk_f32
}
# Staged uploads (docs/vulkan.md, "Memory placement without Resizable BAR"): resident
# data copied from a host staging buffer into device-local memory the host does not map,
# as a discrete card without Resizable BAR needs, against the mapped path on this device:
# - the harness's results (every format, the tiled GEMMs, the expert batch) bit for bit
# the same mapped (COLI_VK_STAGED=0), staged (=1, twice: no run-to-run difference) and
# under an emulated 246 MB host-visible window (COLI_VK_HOST_VISIBLE_CAP_MB=246 and
# nothing else), which must choose staging on its own;
# - the routed-expert tier (test_vk_tier) and the chain's ops (test_vk_chain) staged;
# - each staged run ends with no resident data in host memory ("[VK] memory at exit").
staged_check() { # <err log> <tag>: staged, and nothing resident left in host memory
grep -q '^\[VK\] memory: staged uploads' "$1" || { cat "$1"; fail "$2: not staged"; }
grep -q 'resident data in host memory: 0.0 MiB' "$1" || { grep '\[VK\] memory' "$1"; fail "$2: resident data in host memory"; }
}
family_staged() {
make tests/test_vk_tier tests/test_vk_chain VK=1 # the shaders too
cc -O2 -pthread -DVK_TEST backend_vulkan.c -o vk_test -ldl -lm
local m e d d0=""
for m in mapped staged staged-again staged-noimport window; do
case $m in
mapped) e=COLI_VK_STAGED=0 ;;
staged|staged-again) e=COLI_VK_STAGED=1 ;;
staged-noimport) e="COLI_VK_STAGED=1 COLI_VK_UP_IMPORT=0" ;; # every copy through the staging buffer
window) e=COLI_VK_HOST_VISIBLE_CAP_MB=246 ;;
esac
env -u COLI_VK_STAGED -u COLI_VK_HOST_VISIBLE_CAP_MB $e COLI_VK_TEST_MATMUL_ONLY=1 ./vk_test shaders/qmatmul.spv > vk_test.log 2> vk_test.err
tail -1 vk_test.log | grep -qx PASS || { cat vk_test.log vk_test.err; fail "staged harness, $m"; }
if [ $m = mapped ]; then grep -qx 'memory: mapped' vk_test.log || { cat vk_test.err; fail "staged harness: COLI_VK_STAGED=0 staged"; }
else staged_check vk_test.err "staged harness, $m"; fi
d=$(sed -n 's/^outputs digest //p' vk_test.log)
[ -n "$d" ] || fail "staged harness, $m: no digest"
[ -n "$d0" ] || d0=$d
[ "$d" = "$d0" ] || fail "staged harness, $m: the results differ from the mapped run's ($d against $d0)"
echo "OK staged harness $m: $(grep '^memory:' vk_test.log), results digest $d"
done
COLI_VK_STAGED=1 ./tests/test_vk_tier shaders/qmatmul.spv > vk_tier.log 2> vk_tier.err
tail -1 vk_tier.log | grep -qx PASS || { cat vk_tier.log vk_tier.err; fail "staged routed-expert tier"; }
staged_check vk_tier.err "staged routed-expert tier"
echo "OK staged routed-expert tier: $(grep -o 'resident data in host memory: .*' vk_tier.err)"
COLI_VK_STAGED=1 COLI_VK_UP_IMPORT=0 ./tests/test_vk_tier shaders/qmatmul.spv > vk_tier.log 2> vk_tier.err
tail -1 vk_tier.log | grep -qx PASS || { cat vk_tier.log vk_tier.err; fail "staged routed-expert tier, no imports"; }
echo "OK staged routed-expert tier, every copy staged (COLI_VK_UP_IMPORT=0)"
COLI_VK_STAGED=1 ./tests/test_vk_chain shaders/qmatmul.spv > vk_chain.log 2> vk_chain.err
tail -1 vk_chain.log | grep -qx PASS || { cat vk_chain.log vk_chain.err; fail "staged chain ops"; }
staged_check vk_chain.err "staged chain ops"
echo "OK staged chain ops: $(grep -o 'resident data in host memory: .*' vk_chain.err)"
}
# imported_copies <log>: the copies the exit report says came straight from host memory
imported_copies() { sed -n 's/.* MiB of them straight from host memory in \([0-9]*\) copies.*/\1/p' "$1" | tail -1; }
# Staged uploads straight from host memory (up_import, VK_EXT_external_memory_host): the
# tier's experts from the host image they are converted in, the trunk's rows from the
# weights. The same tokens and logits as every copy through the staging buffer
# (COLI_VK_UP_IMPORT=0); the exit report counts the imported copies; an import the driver
# refuses (COLI_VK_STAGED_FAULT=import) is staged instead.
staged_import() {
make qwen36 VK=1
[ -f qwen36_tiny/ref_full.json ] && [ -d qwen36_tiny_c ] || {
$PY tools/make_qwen36_tiny.py --out qwen36_tiny --ref-mode full --emit-ref qwen36_tiny/ref_full.json
$PY tools/convert_qwen36.py --model qwen36_tiny --out qwen36_tiny_c --ebits 8; }
local i n f
SNAP=qwen36_tiny_c COLI_DENSE_I8=0 ./qwen36 8 8 qwen36_tiny/ref_full.json > cpu.log 2>&1 || true
COLI_VULKAN=1 COLI_VK_STAGED=1 SNAP=qwen36_tiny_c ./qwen36 1 8 qwen36_tiny/ref_full.json > imp.log 2>&1 || true
grep -q '^\[VK\] ready' imp.log || { cat imp.log; fail "staged imports: the probe did not open the device"; }
if ! grep -q '^\[VK\] staged uploads: copied straight from host memory' imp.log; then
echo "OK staged imports: this driver has no VK_EXT_external_memory_host, every copy is staged"; return 0
fi
for i in 1 0; do
rm -f imp.usage
env COLI_USAGE=imp.usage COLI_VULKAN=1 COLI_VK_STAGED=1 COLI_VK_UP_IMPORT=$i COLI_VK_DENSE=1 COLI_VK_TIER_SYNC=1 \
COLI_DENSE_I8=0 DUMP=imp$i.f32 SNAP=qwen36_tiny_c ./qwen36 8 8 qwen36_tiny/ref_full.json > imp$i.log 2>&1 || true
same_tokens cpu.log imp$i.log "staged imports COLI_VK_UP_IMPORT=$i"
staged_check imp$i.log "staged imports COLI_VK_UP_IMPORT=$i"
done
cmp -s imp1.f32 imp0.f32 || fail "staged imports: the logits differ from every copy staged"
n=$(imported_copies imp1.log); [ "${n:-0}" -gt 0 ] || { grep '\[VK\] memory' imp1.log; fail "staged imports: nothing imported"; }
[ "$(imported_copies imp0.log)" = 0 ] || { grep '\[VK\] memory' imp0.log; fail "staged imports: COLI_VK_UP_IMPORT=0 imported"; }
echo "OK staged imports: tokens = CPU, logits = every copy staged, $n copies straight from host memory"
rm -f imp.usage
env COLI_USAGE=imp.usage COLI_VULKAN=1 COLI_VK_STAGED=1 COLI_VK_STAGED_FAULT=import:1 COLI_VK_DENSE=1 COLI_VK_TIER_SYNC=1 \
COLI_DENSE_I8=0 SNAP=qwen36_tiny_c ./qwen36 8 8 qwen36_tiny/ref_full.json > impf.log 2>&1 || true
grep -q "COLI_VK_STAGED_FAULT: import #1 fails" impf.log || { cat impf.log; fail "staged imports: the import fault never fired"; }
same_tokens cpu.log impf.log "staged imports, a refused import"
f=$(sed -n 's/.* copies (\([0-9]*\) not imported).*/\1/p' impf.log | tail -1)
[ "${f:-0}" -ge 1 ] || { grep '\[VK\] memory' impf.log; fail "staged imports: the refused import is not counted"; }
echo "OK staged imports, a refused import: staged instead, tokens = CPU"
rm -f imp.usage imp0.f32 imp1.f32
}
# An engine under the emulated window, COLI_VK_STAGED unset: it stages on its own, the
# tier with the trunk on the device and the chain give the CPU's tokens (and logits),
# and nothing resident ends in host memory.
staged_window() {
make qwen36 VK=1
$PY tools/make_qwen36_tiny.py --out qwen36_tiny --ref-mode full --emit-ref qwen36_tiny/ref_full.json
$PY tools/convert_qwen36.py --model qwen36_tiny --out qwen36_tiny_c --ebits 8
tier_gate qwen36 "window qwen36 tier and trunk" COLI_VK_HOST_VISIBLE_CAP_MB=246 COLI_VK_DENSE=1 COLI_DENSE_I8=0 SNAP=qwen36_tiny_c -- 8 8 qwen36_tiny/ref_full.json
staged_check vk.log "window qwen36 tier and trunk"
chain_gate qwen36 "window qwen36 chain" 1 COLI_VK_HOST_VISIBLE_CAP_MB=246 COLI_DENSE_I8=0 SNAP=qwen36_tiny_c -- 8 8 qwen36_tiny/ref_full.json
staged_check vk.log "window qwen36 chain"
echo "OK window qwen36: staged on its own, $(grep -o 'resident data in host memory: .*' vk.log)"
staged_import # here: the engine's fixture needs the fixture dependencies, which staged's job has not
}
# Staged uploads failing (COLI_VK_STAGED_FAULT=<point>[:n], the n-th time) at every point
# they can: the uploader's staging buffer (stage), the KV mirror's (pwstage), a weight
# pool's device-local block (block), a KV mirror or norm-weight buffer (kvbuf), a command
# buffer's begin or end (record), a submit, a fence wait (the device is then lost), a tier
# expert's commit after its first matrix (commit). qwen36 with its trunk's matrices on the
# device one by one, its tier awaited (COLI_VK_TIER_SYNC=1) and with the uploader thread
# free after a warm start, and its chain; colibri's attention core for the KV mirror. Every
# run: no crash and no sanitizer report, the fault fired, the CPU's tokens. A matrix that
# failed stays on the CPU (one fewer resident); an expert whose commit failed stays on the
# CPU and the tier's budget stands; a lost device takes everything to the CPU, the chain
# rebuilding its state there. SAN=1 (staged-faults-sanitize): a sanitized build.
fault_run() { # <tag> <fault> <cpu tokens> <env...> -- <command...>
local tag=$1 fault=$2 ref=$3 rc=0; shift 3
local envs=(); while [ "$1" != "--" ]; do envs+=("$1"); shift; done; shift
if [ -f hist.usage ]; then cp hist.usage run.usage; else rm -f run.usage; fi
env "${envs[@]}" COLI_USAGE=run.usage COLI_VULKAN=1 COLI_VK_STAGED=1 ${fault:+COLI_VK_STAGED_FAULT=$fault} "$@" > vk.log 2>&1 || rc=$?
if [ $rc -ge 128 ] || grep -qE "ERROR: AddressSanitizer|runtime error:" vk.log; then cat vk.log; fail "$tag: crashed (exit $rc)"; fi
[ -z "$fault" ] || grep -q "COLI_VK_STAGED_FAULT: ${fault%%:*} " vk.log || { cat vk.log; fail "$tag: the fault never fired"; }
grep -a 'C engine' vk.log > vk.tok || true
{ [ -s "$ref" ] && cmp -s "$ref" vk.tok; } || { cat "$ref" vk.tok; fail "$tag: tokens differ from the CPU's"; }
}
fault_num() { # <sed expression> <log>: the first number it extracts, 0 without one
local n; n=$(sed -n "$1" "$2" | tail -1); echo "${n:-0}"
}
family_staged_faults() {
if [ "${SAN:-0}" = 1 ]; then
make clean >/dev/null 2>&1 || true
make qwen36 colibri VK=1 EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1
else
make qwen36 colibri VK=1
fi
$PY tools/make_qwen36_tiny.py --out qwen36_tiny --ref-mode full --emit-ref qwen36_tiny/ref_full.json
$PY tools/convert_qwen36.py --model qwen36_tiny --out qwen36_tiny_c --ebits 8
mkdir -p glm_fp8 && (cd glm_fp8 && $PY ../tools/make_glm_oracle.py --fp8 > /dev/null)
$PY tools/convert_fp8_to_int4.py --indir glm_fp8/glm_tiny --outdir glm_tiny_i4 --ebits 4 --io-bits 4 \
--n-layers 5 --min-free-gb 0 > /dev/null
cp glm_fp8/ref_glm.json glm_tiny_i4/
local q=(COLI_DENSE_I8=0 SNAP=qwen36_tiny_c ./qwen36 8 8 qwen36_tiny/ref_full.json)
local g=(IDOT=0 SNAP=glm_tiny_i4 REF=glm_tiny_i4/ref_glm.json ./colibri 2 4 4)
env "${q[@]}" > cpu.log 2>&1 || true; grep -a 'C engine' cpu.log > q.tok || true
env "${g[@]}" > cpu.log 2>&1 || true; grep -a 'C engine' cpu.log > g.tok || true
local f n S
rm -f hist.usage
# the trunk's matrices one by one (no tier, no chain)
local D=(COLI_VK_TIER=0 COLI_VK_CHAIN=0 COLI_VK_DENSE=1)
fault_run "faults: qwen36 matrices, none" "" q.tok "${D[@]}" -- "${q[@]}"
n=$(fault_num 's/.*matmuls on the GPU (\([0-9]*\) matrices.*/\1/p' vk.log)
for f in stage:1 block:1 record:4 submit:4 wait:1 wait:4; do
fault_run "faults: qwen36 matrices, $f" $f q.tok "${D[@]}" -- "${q[@]}"
case $f in block:1|record:4|submit:4)
[ "$(fault_num 's/.*matmuls on the GPU (\([0-9]*\) matrices.*/\1/p' vk.log)" = $((n - 1)) ] || {
grep '\[VK\]' vk.log; fail "faults: qwen36 matrices, $f: not exactly one matrix left on the CPU"; } ;;
wait:*) grep -q 'the device is lost' vk.log || { cat vk.log; fail "faults: qwen36 matrices, $f: the device was not lost"; } ;;
esac
echo "OK faults: qwen36 matrices, $f: tokens = CPU, $(grep -ao 'staged upload failed[^:]*: [^)]*): .*' vk.log | head -1 | sed 's/.*): //')"
done
# the tier awaited: the trunk on the CPU, every upload at the next step
local T=(COLI_VK_CHAIN=0 COLI_VK_DENSE=0 COLI_VK_TIER_SYNC=1 COLI_VK_TIER_BALANCE=0)
fault_run "faults: qwen36 tier, none" "" q.tok "${T[@]}" -- "${q[@]}"
local budget; budget=$(fault_num 's/.*resident [0-9]* (budget \([0-9]*\),.*/\1/p' vk.log)
cp run.usage hist.usage.next
for f in stage:1 block:1 record:3 record:4 submit:2 wait:2 commit:1 commit:3; do
fault_run "faults: qwen36 tier, $f" $f q.tok "${T[@]}" -- "${q[@]}"
case $f in commit:*|record:*|submit:*)
[ "$(fault_num 's/.*resident [0-9]* (budget \([0-9]*\),.*/\1/p' vk.log)" = "$budget" ] || {
grep '\[VK\] tier' vk.log; fail "faults: qwen36 tier, $f: the budget shrank"; } ;;
esac
echo "OK faults: qwen36 tier, $f: tokens = CPU, $(grep -ao 'resident [0-9]* (budget [0-9]*' vk.log | tail -1), $(grep -ao 'failed [0-9]* |' vk.log | tail -1 | tr -d '|')"
done
# the tier's uploader thread free: a warm start from a history (vkt_put, any thread),
# then promotions as experts pass
mv hist.usage.next hist.usage
T=(COLI_VK_CHAIN=0 COLI_VK_DENSE=0)
for f in stage:1 block:1 record:3 submit:2 wait:3 commit:1 commit:5; do
fault_run "faults: qwen36 tier, uploader thread, $f" $f q.tok "${T[@]}" -- "${q[@]}"
echo "OK faults: qwen36 tier, uploader thread, $f: tokens = CPU, $(grep -ao 'resident [0-9]* (budget [0-9]*' vk.log | tail -1)"
done
rm -f hist.usage
# the chain: a failure before it starts declines it (or leaves one matrix behind), a
# lost device in the middle has the CPU rebuild the state
local C=(COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1)
fault_run "faults: qwen36 chain, none" "" q.tok "${C[@]}" -- "${q[@]}"
S=$(fault_num 's/.*copies (\([0-9]*\) submits).*/\1/p' vk.log)
for f in stage:1 block:1 record:4 submit:4 commit:1 wait:4 wait:$((S - 2)); do
fault_run "faults: qwen36 chain, $f" $f q.tok "${C[@]}" -- "${q[@]}"
[ $f != wait:$((S - 2)) ] || grep -q 'rebuilding the state of [1-9]' vk.log || { cat vk.log; fail "faults: qwen36 chain, $f: no state rebuilt on the CPU"; }
echo "OK faults: qwen36 chain, $f: tokens = CPU, $(grep -ao 'qwen36 chain: [0-9]* forwards\|rebuilding the state of [0-9]* positions' vk.log | tr '\n' ' ')"
done
# colibri's attention core: the KV mirror's buffers and its staging
local A=(COLI_VK_DENSE=1 COLI_VK_ATTN=1 COLI_VK_TIER_SYNC=1)
for f in pwstage:1 kvbuf:1 kvbuf:3 wait:3; do
fault_run "faults: colibri attention, $f" $f g.tok "${A[@]}" -- "${g[@]}"
echo "OK faults: colibri attention, $f: tokens = CPU"
done
if [ "${SAN:-0}" = 1 ]; then make clean >/dev/null 2>&1 || true; fi
}
# tier_count <engine> <log>: N from the last "[VK] tier <engine> run: device N of M" line;
# tier_evictions <engine> <log>: the evictions of that line; tier_failed: its failed
# uploads (an eviction case must have none: a promotion that displaces a resident
# while a batch is in flight waits for the join to free it, even with
# COLI_VK_TIER_SYNC=1, instead of failing and shrinking the budget)
tier_count() {
local n
n=$(sed -n "s/^\[VK\] tier $1 run: device \([0-9][0-9]*\) of .*/\1/p" "$2" | tail -1)
echo "${n:-0}"
}
tier_evictions() {
local n
n=$(sed -n "s/^\[VK\] tier $1 run: .* evictions \([0-9][0-9]*\),.*/\1/p" "$2" | tail -1)
echo "${n:-0}"
}
tier_failed() {
local n
n=$(sed -n "s/^\[VK\] tier $1 run: .* failed \([0-9][0-9]*\) |.*/\1/p" "$2" | tail -1)
echo "${n:-0}"
}
# dense_where <engine> <log> <tag> <env...>: where the run put the dense trunk, checked
# against what it asked for. "[VK] <engine>: device ready, dense matrices on the
# device|CPU" must agree with the matmul count; COLI_VK_DENSE=1 must put the trunk on
# the device; with COLI_VK_DENSE unset and the tier on, Lavapipe (a CPU device, its
# memory the CPU's RAM) must keep it on the CPU, as an integrated GPU does.
dense_where() {
local eng=$1 log=$2 tag=$3 where; shift 3
if grep -qa "^\[VK\] $eng: device ready, dense matrices on the device" "$log"; then
# inkling places none when the host keeps them by design (bf16 weights under the
# AVX512-BF16 dot, which rounds the activations as the shader does not): its
# placement line then says "0 resident matrices go to the GPU", and the count is
# not required, as in family_inkling_olmoe's own bf16 check.
grep -qa "^\[VK\] $eng: 0 resident matrices go to the GPU" "$log" || need_gpu "$eng" "$log" "$tag"
where=device
else
[ "$(vk_count "$eng" "$log")" = 0 ] || { cat "$log"; fail "$tag: dense matmuls ran on the device with the trunk on the CPU"; }
where=CPU
fi
case " $* " in
*" COLI_VK_DENSE=1 "*) [ $where = device ] || { cat "$log"; fail "$tag: COLI_VK_DENSE=1 left the trunk on the CPU"; } ;;
*" COLI_VK_DENSE="*) ;;
*) if grep -qa '^\[VK\] ready: llvmpipe' "$log" && grep -qa "^\[VK\] tier $eng: on" "$log"; then
[ $where = CPU ] || { cat "$log"; fail "$tag: Lavapipe with the tier on kept the trunk on the device"; }
fi ;;
esac
echo $where
}
# tier_gate <engine> <tag> <env...> -- <argv...>
# The routed-expert tier (vk_tier.c) against the CPU: the same tokens as the CPU run
# with the same settings, the device served some of the routed experts, and the dense
# trunk ran where it was asked to (dense_where). With EVICT=1 the run must also have
# evicted (its budget is set below the hot set). COLI_USAGE points at a fresh file: no
# warm start from an earlier run's history. COLI_VK_TIER_SYNC=1: a fixture's whole run
# can end before the uploader thread is first scheduled (it did in 5 of 40 runs with
# the trunk on the CPU), so the gate waits for each staged upload at the next step.
tier_gate() {
local eng=$1 tag=$2; shift 2
local envs=(); while [ "$1" != "--" ]; do envs+=("$1"); shift; done; shift
rm -f tier.usage
env "${envs[@]}" ./"$eng" "$@" > cpu.log 2>&1 || true
env "${envs[@]}" COLI_USAGE=tier.usage COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 ./"$eng" "$@" > vk.log 2>&1 || true
same_tokens cpu.log vk.log "$tag"
[ "$(tier_count "$eng" vk.log)" -gt 0 ] || { cat vk.log; fail "$tag: no routed expert ran on the device"; }
if [ "${EVICT:-0}" = 1 ]; then
[ "$(tier_evictions "$eng" vk.log)" -gt 0 ] || { grep '\[VK\] tier' vk.log; fail "$tag: the budget forced no eviction"; }
[ "$(tier_failed "$eng" vk.log)" = 0 ] || { grep '\[VK\] tier' vk.log; fail "$tag: an upload failed (the budget shrank)"; }
fi
local where; where=$(dense_where "$eng" vk.log "$tag" "${envs[@]}") || { echo "$where"; exit 1; }
echo "OK $tag: tokens = CPU, $(grep -a -o 'device [0-9]* of [0-9]* routed experts' vk.log | tail -1), $(grep -a -o 'evictions [0-9]*' vk.log | tail -1), trunk on the $where"
}
# vk_gate <engine> <placed-regex> <tag> <env...> -- <argv...>
# CPU arm and Vulkan arm of one configuration; both must exit 0 (the engine's own
# oracle), give the same tokens, run matmuls on the device, and report the expected
# placement ("placed int8 a, int4 b, f32 c" / "int8 a, bf16 b, f32 c").
vk_gate() {
local eng=$1 placed=$2 tag=$3; shift 3
local envs=(); while [ "$1" != "--" ]; do envs+=("$1"); shift; done; shift
env "${envs[@]}" ./"$eng" "$@" > cpu.log 2>&1 || { cat cpu.log; fail "$tag: CPU run"; }
env "${envs[@]}" COLI_VULKAN=1 ./"$eng" "$@" > vk.log 2>&1 || { cat vk.log; fail "$tag: Vulkan run misses the oracle"; }
same_tokens cpu.log vk.log "$tag"
need_gpu "$eng" vk.log "$tag"
grep -qE "\[VK\] $eng: .*placed .*$placed" vk.log || { grep '\[VK\]' vk.log; fail "$tag: expected placement '$placed'"; }
echo "OK $tag: $(grep -a -o 'Matching tokens: [0-9/]*' vk.log), tokens = CPU, $(grep -o '[0-9]* matmuls on the GPU.*' vk.log | tail -1)"
}
family_qwen() {
make qwen36 qwen38 VK=1
# qwen36: the hybrid, Qwen3-Coder (qwen3_moe), the 27B dense and the 2.4T geometry
$PY tools/make_qwen36_tiny.py --out qwen36_tiny --ref-mode full --emit-ref qwen36_tiny/ref_full.json
$PY tools/make_qwen36_tiny.py --geometry qwen3-coder-30b --out qwen3_coder_tiny --ref-mode full --emit-ref qwen3_coder_tiny/ref_full.json
$PY tools/make_qwen36_tiny.py --geometry qwen38-27b-dense --out qwen38_27b_tiny --ref-mode full --emit-ref qwen38_27b_tiny/ref_full.json
$PY tools/make_qwen36_tiny.py --geometry qwen38-2p4t --seed 3 --out qwen38_2p4t_tiny --ref-mode full --emit-ref qwen38_2p4t_tiny/ref_full.json
local fx cap caps
# These arms test the dense trunk's formats: COLI_VK_DENSE=1, because on Lavapipe (a
# CPU device sharing the CPU's RAM) the trunk otherwise stays on the CPU while the
# expert tier is on; the tier runs beside it.
for fx in qwen36_tiny qwen3_coder_tiny qwen38_27b_tiny qwen38_2p4t_tiny; do
$PY tools/convert_qwen36.py --model $fx --out ${fx}_c --ebits 8
# cap=1 evicts on every routed expert; on the 92-layer 2.4T geometry that costs
# ~90 s on Lavapipe for nothing the device path adds, so that one runs at cap=8.
caps="1 8"; [ $fx = qwen38_2p4t_tiny ] && caps=8
for cap in $caps; do
# the CPU job's own configuration: f32 dense weights (fmt 10), token-exact
# against transformers and equal to the CPU run
vk_gate qwen36 'f32 [1-9]' "qwen36 $fx f32 cap=$cap" COLI_VK_DENSE=1 COLI_DENSE_I8=0 SNAP=${fx}_c -- $cap 8 $fx/ref_full.json
done
# int8 dense rows (fmt 1). int8 weights alone miss the torch oracle on some
# fixtures, CPU or GPU alike, so this arm gates on the CPU's tokens only, with
# COLI_DENSE_IDOT=0 giving the CPU the shader's f32 activations.
COLI_DENSE_IDOT=0 SNAP=${fx}_c ./qwen36 8 8 $fx/ref_full.json > cpu.log 2>&1 || true
COLI_DENSE_IDOT=0 COLI_VK_DENSE=1 COLI_VULKAN=1 SNAP=${fx}_c ./qwen36 8 8 $fx/ref_full.json > vk.log 2>&1 || true
same_tokens cpu.log vk.log "qwen36 $fx int8"
grep -qE '\[VK\] qwen36: [1-9][0-9]* matmuls on the GPU.*placed int8 [1-9]' vk.log || { cat vk.log; fail "qwen36 $fx int8: nothing placed"; }
echo "OK qwen36 $fx int8: tokens = CPU, $(grep -o '[0-9]* matmuls on the GPU.*' vk.log | tail -1)"
done
# The routed-expert tier on every expert container qwen36 reads (COLI_VULKAN=1 turns
# it on; the runs above already had it): int8 per row (fmt 1), the shared kernel's
# planar int4-g64 (fmt 4), int4 per row from the unpacked slots (fmt 2), int8 gs64
# (fmt 13) and the mixed int4 gate/up + int8 down container, at cap=1 with the trunk
# where the default puts it (on the CPU here) and at cap=8 with the trunk on the
# device (COLI_VK_DENSE=1); the tier alone (COLI_VK_DENSE=0); and a budget of two
# experts, which must evict as the routing moves.
$PY tools/make_qwen36_tiny.py --out qwen36_tiny64 --ref-mode full --inter 64 --emit-ref qwen36_tiny64/ref_full.json
$PY tools/convert_qwen36.py --model qwen36_tiny64 --out qwen36_tiny64_c --ebits 4 --gs 64
$PY tools/convert_qwen36.py --model qwen36_tiny64 --out qwen36_tiny64_g8 --ebits 8 --gs 64
$PY tools/convert_qwen36.py --model qwen36_tiny64 --out qwen36_tiny64_d8 --ebits 4 --gs 64 --down-bits 8
$PY tools/convert_qwen36.py --model qwen36_tiny --out qwen36_tiny_i4r --ebits 4
local D
for cap in 1 8; do
D=; [ $cap = 8 ] && D=COLI_VK_DENSE=1
tier_gate qwen36 "qwen36 tier int8 cap=$cap" $D COLI_DENSE_I8=0 SNAP=qwen36_tiny_c -- $cap 8 qwen36_tiny/ref_full.json
tier_gate qwen36 "qwen36 tier int4-g64 planar cap=$cap" $D COLI_DENSE_I8=0 SNAP=qwen36_tiny64_c -- $cap 4 qwen36_tiny64/ref_full.json
tier_gate qwen36 "qwen36 tier int4 per row cap=$cap" $D COLI_DENSE_I8=0 SNAP=qwen36_tiny_i4r -- $cap 4 qwen36_tiny/ref_full.json
tier_gate qwen36 "qwen36 tier int8 gs64 cap=$cap" $D COLI_DENSE_I8=0 SNAP=qwen36_tiny64_g8 -- $cap 8 qwen36_tiny64/ref_full.json
tier_gate qwen36 "qwen36 tier mixed int4/int8 cap=$cap" $D COLI_DENSE_I8=0 SNAP=qwen36_tiny64_d8 -- $cap 4 qwen36_tiny64/ref_full.json
done
tier_gate qwen36 "qwen36 tier alone (COLI_VK_DENSE=0)" COLI_VK_DENSE=0 COLI_DENSE_I8=0 SNAP=qwen36_tiny64_c -- 8 4 qwen36_tiny64/ref_full.json
grep -q 'dense chain off (COLI_VK_DENSE=0' vk.log || { cat vk.log; fail "qwen36 tier alone: COLI_VK_DENSE=0 did not keep the chain off"; }
EVICT=1 tier_gate qwen36 "qwen36 tier, a budget of two experts" COLI_VK_TIER_GB=0.00002 COLI_DENSE_I8=0 SNAP=qwen36_tiny_c -- 8 8 qwen36_tiny/ref_full.json
# COLI_VK_TIER=0: the dense trunk alone, as before the tier; with no tier the default
# puts the trunk on the device, Lavapipe included
COLI_VK_TIER=0 COLI_DENSE_I8=0 SNAP=qwen36_tiny_c ./qwen36 8 8 qwen36_tiny/ref_full.json > cpu.log 2>&1 || true
COLI_VK_TIER=0 COLI_VULKAN=1 COLI_DENSE_I8=0 SNAP=qwen36_tiny_c ./qwen36 8 8 qwen36_tiny/ref_full.json > vk.log 2>&1 || true
same_tokens cpu.log vk.log "qwen36 COLI_VK_TIER=0"
! grep -q '^\[VK\] tier' vk.log || { cat vk.log; fail "qwen36 COLI_VK_TIER=0: the tier started"; }
need_gpu qwen36 vk.log "qwen36 COLI_VK_TIER=0"
echo "OK qwen36 COLI_VK_TIER=0: tokens = CPU, no tier, $(grep -o '[0-9]* matmuls on the GPU' vk.log | tail -1)"
# qwen38: one fixture, every resident format, with and without prefill batching
$PY tools/make_qwen38_tiny.py --out qwen38_tiny
local batch O
for batch in 0 1; do
O="OMP_NUM_THREADS=2 SNAP=qwen38_tiny Q38_PREFILL_BATCH=$batch COLI_VK_DENSE=1" # the trunk's formats, see qwen36 above
# the default: every fixture matrix is under 1 MiB, so the trunk stays BF16 (fmt 11)
vk_gate qwen38 'bf16 [1-9]' "qwen38 bf16 batch=$batch" $O -- 1 8 qwen38_tiny/ref.json
# Q38_TRUNK_MIN_KB=0: the int8 trunk (fmt 1); what stays outside it is BF16
vk_gate qwen38 'int8 [1-9].*bf16 [1-9]' "qwen38 int8 batch=$batch" $O Q38_TRUNK_MIN_KB=0 -- 1 8 qwen38_tiny/ref.json
# Q38_NATIVE_BF16=0 expands the rows to f32 at load (fmt 10)
vk_gate qwen38 'f32 [1-9]' "qwen38 f32 batch=$batch" $O Q38_TRUNK_CPU_INT8=0 Q38_NATIVE_BF16=0 -- 1 8 qwen38_tiny/ref.json
done
# The routed-expert tier on qwen38's three expert forms: BF16 (fmt 11), the release's
# FP8 with 128x128 block scales (fmt 12, gs 128), the experts-int4g64 sidecar's planar
# int4 (fmt 4); decode one row at a time and prefill batched; at cap=1 with the trunk
# where the default puts it (the CPU here), at cap=4 with the trunk on the device;
# with the MTP head drafting (its verify rows take the device's per-row route; the
# batched one with the trunk on the device); the tier alone; and a budget of two
# experts, which must evict.
$PY tools/make_qwen38_tiny.py --out qwen38_tiny_fp8 --fp8-experts
$PY tools/make_qwen38_tiny.py --out qwen38_tiny_int4 --fp8-experts --int4-experts --expert-gain 3
local fx ref
for fx in qwen38_tiny qwen38_tiny_fp8 qwen38_tiny_int4; do
ref=$fx/ref.json; [ $fx = qwen38_tiny_int4 ] && ref=$fx/ref_int4.json
for batch in 0 1; do for cap in 1 4; do
D=; [ $cap = 4 ] && D=COLI_VK_DENSE=1
tier_gate qwen38 "qwen38 tier $fx batch=$batch cap=$cap" OMP_NUM_THREADS=2 $D Q38_PREFILL_BATCH=$batch SNAP=$fx -- $cap 8 $ref
done; done
done
tier_gate qwen38 "qwen38 tier alone (COLI_VK_DENSE=0)" OMP_NUM_THREADS=2 COLI_VK_DENSE=0 SNAP=qwen38_tiny_int4 -- 4 8 qwen38_tiny_int4/ref_int4.json
EVICT=1 tier_gate qwen38 "qwen38 tier, a budget of two experts" OMP_NUM_THREADS=2 Q38_PREFILL_BATCH=0 COLI_VK_TIER_GB=0.0000065 SNAP=qwen38_tiny_fp8 -- 1 8 qwen38_tiny_fp8/ref.json
$PY tools/make_qwen38_tiny.py --out qwen38_tiny_fp8_mtp --fp8-experts --mtp
$PY tools/make_qwen38_tiny.py --out qwen38_tiny_int4_mtp --fp8-experts --int4-experts --expert-gain 3 --mtp
# the MTP head's layer as the tier's extra layer (COLI_VK_TIER_MTP=1; the default on a
# discrete GPU): its experts (FP8, beside the int4 sidecar on the second fixture) run
# on the device too. Lavapipe shares the RAM, where the default keeps them on the CPU;
# the tokens are the same either way.
for fx in qwen38_tiny_fp8_mtp qwen38_tiny_int4_mtp; do
for batch in 0 1; do
D=; [ $batch = 1 ] && D=COLI_VK_DENSE=1
tier_gate qwen38 "qwen38 tier MTP $fx batch=$batch" OMP_NUM_THREADS=2 $D Q38_MTP=1 COLI_VK_TIER_MTP=1 Q38_PREFILL_BATCH=$batch SNAP=$fx -- 2 8 $fx/ref.json
mtp_on_device "qwen38 tier MTP $fx batch=$batch"
done
done
tier_gate qwen38 "qwen38 tier MTP head on the CPU (shared RAM default)" OMP_NUM_THREADS=2 Q38_MTP=1 Q38_PREFILL_BATCH=0 SNAP=qwen38_tiny_int4_mtp -- 2 8 qwen38_tiny_int4_mtp/ref.json
! grep -aq 'extra layers' vk.log || { grep -a '\[VK\] tier' vk.log; fail "shared RAM: the MTP head's layer went on the tier by default"; }
echo "OK shared RAM: the MTP head's experts on the CPU by default"
}
mtp_on_device() { # <tag> [log, vk.log]: the tier served experts of the MTP head's layers (its extra layers)
local log=${2:-vk.log} n; n=$(grep -a -o 'extra layers ([0-9]*): [0-9]*' "$log" | tail -1 | grep -o '[0-9]*$')
[ "${n:-0}" -gt 0 ] || { grep -a '\[VK\] tier' "$log"; fail "$1: no expert of the MTP head's layers ran on the device"; }
echo "OK $1: $(grep -a -o 'extra layers ([0-9]*): [0-9]* of [0-9]* routed experts on the device' "$log" | tail -1)"
}
# The routed-expert tier under ASan and UBSan: a sanitized VK=1 build of both qwen
# engines, the tier's configurations on Lavapipe (formats, eviction, PILOT's worker
# against the tier, MTP, the tier alone, the trunk on the device or on the CPU). Memory safety is the gate, not the tokens
# (a sanitized build vectorizes differently); each run must still put experts on the
# device, or the tier was never exercised.
family_qwen_sanitize() {
make clean >/dev/null 2>&1 || true
make qwen36 qwen38 VK=1 EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
$PY tools/make_qwen36_tiny.py --out qwen36_tiny --ref-mode full --emit-ref qwen36_tiny/ref_full.json
$PY tools/convert_qwen36.py --model qwen36_tiny --out qwen36_tiny_c --ebits 8
$PY tools/make_qwen36_tiny.py --out qwen36_tiny64 --ref-mode full --inter 64 --emit-ref qwen36_tiny64/ref_full.json
$PY tools/convert_qwen36.py --model qwen36_tiny64 --out qwen36_tiny64_c --ebits 4 --gs 64
$PY tools/convert_qwen36.py --model qwen36_tiny64 --out qwen36_tiny64_g8 --ebits 8 --gs 64
$PY tools/make_qwen38_tiny.py --out qwen38_tiny_fp8 --fp8-experts
$PY tools/make_qwen38_tiny.py --out qwen38_tiny_int4 --fp8-experts --int4-experts --expert-gain 3
$PY tools/make_qwen38_tiny.py --out qwen38_tiny_int4_mtp --fp8-experts --int4-experts --expert-gain 3 --mtp
san() { # <engine> <tag> <env and argv...>
local eng=$1 tag=$2; shift 2
rm -f tier.usage
env ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1 OMP_NUM_THREADS=2 \
COLI_USAGE=tier.usage COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 "$@" > san.log 2>&1 || true
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "$tag: sanitizer diagnostic"; fi
[ "$(tier_count "$eng" san.log)" -gt 0 ] || { cat san.log; fail "$tag: no routed expert ran on the device"; }
echo "OK $tag: sanitizers clean, $(grep -a -o 'device [0-9]* of [0-9]* routed experts' san.log | tail -1), $(grep -a -o 'dense matrices on the [a-zA-Z]*' san.log | head -1)"
}
local cap D # cap=8 and batch=1 with the trunk on the device, the others where the default puts it
for cap in 1 8; do
D=; [ $cap = 8 ] && D=COLI_VK_DENSE=1
san qwen36 "asan qwen36 int8 PILOT cap=$cap" $D COLI_DENSE_I8=0 PILOT=1 WIDE=2 SNAP=qwen36_tiny_c ./qwen36 $cap 8 qwen36_tiny/ref_full.json
san qwen36 "asan qwen36 int4-g64 PILOT cap=$cap" $D COLI_DENSE_I8=0 PILOT=1 WIDE=2 SNAP=qwen36_tiny64_c ./qwen36 $cap 4 qwen36_tiny64/ref_full.json
san qwen36 "asan qwen36 int8 gs64 cap=$cap" $D COLI_DENSE_I8=0 SNAP=qwen36_tiny64_g8 ./qwen36 $cap 8 qwen36_tiny64/ref_full.json
done
san qwen36 "asan qwen36 eviction" COLI_VK_TIER_GB=0.00002 COLI_DENSE_I8=0 PILOT=1 SNAP=qwen36_tiny_c ./qwen36 8 8 qwen36_tiny/ref_full.json
san qwen36 "asan qwen36 tier alone" COLI_VK_DENSE=0 SNAP=qwen36_tiny64_c ./qwen36 8 4 qwen36_tiny64/ref_full.json
local b
for b in 0 1; do
D=; [ $b = 1 ] && D=COLI_VK_DENSE=1
san qwen38 "asan qwen38 fp8 batch=$b" $D Q38_PREFILL_BATCH=$b SNAP=qwen38_tiny_fp8 ./qwen38 1 8 qwen38_tiny_fp8/ref.json
san qwen38 "asan qwen38 int4 batch=$b" $D Q38_PREFILL_BATCH=$b SNAP=qwen38_tiny_int4 ./qwen38 1 8 qwen38_tiny_int4/ref_int4.json
done
san qwen38 "asan qwen38 eviction" Q38_PREFILL_BATCH=0 COLI_VK_TIER_GB=0.0000065 SNAP=qwen38_tiny_fp8 ./qwen38 1 8 qwen38_tiny_fp8/ref.json
san qwen38 "asan qwen38 MTP" COLI_VK_DENSE=1 Q38_MTP=1 COLI_VK_TIER_MTP=1 Q38_PREFILL_BATCH=0 SNAP=qwen38_tiny_int4_mtp ./qwen38 2 8 qwen38_tiny_int4_mtp/ref.json
mtp_on_device "asan qwen38 MTP" san.log
san qwen38 "asan qwen38 tier alone" COLI_VK_DENSE=0 SNAP=qwen38_tiny_int4 ./qwen38 4 8 qwen38_tiny_int4/ref_int4.json
make clean >/dev/null 2>&1 || true
}
family_inkling_olmoe() {
make inkling olmoe VK=1
local cap
# inkling, f32 fixture (fmt 10): the oracle and the CPU's ids, at three caps. These
# arms and the next two test the dense matrices' formats: COLI_VK_DENSE=1, because on
# Lavapipe (a CPU device sharing the CPU's RAM) the trunk otherwise stays on the CPU
# while the expert tier is on; the tier runs beside it.
$PY tools/make_tiny_inkling.py tiny_inkling
for cap in 1 2 8; do
SNAP=tiny_inkling ./inkling $cap 0 tiny_inkling/ref_inkling.json > cpu.log 2>&1
COLI_VK_DENSE=1 COLI_VULKAN=1 SNAP=tiny_inkling ./inkling $cap 0 tiny_inkling/ref_inkling.json > vk.log 2>&1
grep -qE 'Matching tokens: ([0-9]+)/\1$' vk.log || { cat vk.log; fail "inkling f32 cap=$cap: oracle"; }
same_tokens cpu.log vk.log "inkling f32 cap=$cap"
need_gpu inkling vk.log "inkling f32 cap=$cap"
echo "OK inkling f32 cap=$cap: $(grep -o 'Matching tokens: [0-9/]*' vk.log), $(vk_count inkling vk.log) matmuls on the GPU"
done
# the same weights stored as bf16 (fmt 11). A build with the AVX512-BF16 dot rounds
# activations to bf16 on the CPU and keeps those matrices there ("0 bf16" in the
# placement line), so the count is only required when the line places any.
mkdir -p tiny_inkling_bf16
cp tiny_inkling/config.json tiny_inkling/generation_config.json tiny_inkling_bf16/
$PY - <<'EOF'
import json, struct, numpy as np
src, dst = "tiny_inkling/model.safetensors", "tiny_inkling_bf16/model.safetensors"
raw = open(src, "rb").read(); n = struct.unpack("<Q", raw[:8])[0]; hdr = json.loads(raw[8:8 + n])
out, blobs, off = {}, [], 0
for k, v in hdr.items():
if k == "__metadata__": out[k] = v; continue
a = np.frombuffer(raw[8 + n + v["data_offsets"][0]: 8 + n + v["data_offsets"][1]], np.float32)
u = a.view(np.uint32).astype(np.uint64)
b = ((u + 0x7FFF + ((u >> 16) & 1)) >> 16).astype(np.uint16).tobytes()
out[k] = {"dtype": "BF16", "shape": v["shape"], "data_offsets": [off, off + len(b)]}
blobs.append(b); off += len(b)
h = json.dumps(out).encode(); h += b" " * (-len(h) % 8)
with open(dst, "wb") as f:
f.write(struct.pack("<Q", len(h))); f.write(h); [f.write(b) for b in blobs]
EOF
SNAP=tiny_inkling_bf16 ./inkling 8 0 tiny_inkling/ref_inkling.json > cpu.log 2>&1 || true
COLI_VK_DENSE=1 COLI_VULKAN=1 SNAP=tiny_inkling_bf16 ./inkling 8 0 tiny_inkling/ref_inkling.json > vk.log 2>&1 || true
same_tokens cpu.log vk.log "inkling bf16"
grep -q ', 0 bf16)' vk.log || need_gpu inkling vk.log "inkling bf16"
echo "OK inkling bf16: tokens = CPU, $(vk_count inkling vk.log) matmuls on the GPU"
# the dense-int4g64 container (int8 fmt 1, int4-g64 fmt 4, f32 fmt 10 side by side)
rm -rf tiny_inkling_q && cp -r tiny_inkling tiny_inkling_q
$PY - tools/convert_inkling_dense_int4.py tiny_inkling_q <<'EOF'
import importlib.util, sys
spec = importlib.util.spec_from_file_location("conv", sys.argv[1])
conv = importlib.util.module_from_spec(spec); spec.loader.exec_module(conv)
conv.MIN_ELEMS = 0
conv.ATTN_BITS = 4
base = conv.classify
conv.classify = lambda n, s, d: "int8" if n.endswith(".mlp.down_proj.weight") else base(n, s, d)
sys.argv = ["convert_inkling_dense_int4.py", "--dir", sys.argv[2]]
conv.main()
EOF
mkdir -p tiny_inkling_q/dense-int4g64
mv tiny_inkling_q/dense-int4g64.safetensors tiny_inkling_q/dense-int4g64/dense.safetensors
SNAP=tiny_inkling_q ./inkling 8 0 tiny_inkling/ref_inkling.json > cpu.log 2>&1 || true
COLI_VK_DENSE=1 COLI_VULKAN=1 SNAP=tiny_inkling_q ./inkling 8 0 tiny_inkling/ref_inkling.json > vk.log 2>&1 || true
same_tokens cpu.log vk.log "inkling int4-g64 container"
need_gpu inkling vk.log "inkling int4-g64 container"
echo "OK inkling int4-g64 container: tokens = CPU, $(vk_count inkling vk.log) matmuls on the GPU"
# olmoe: f32 residents (fmt 10), the oracle and the CPU's ids (the trunk on the
# device, as above)
$PY tools/make_olmoe_tiny.py --output olmoe_tiny
$PY tools/convert_olmoe_merged.py --model olmoe_tiny --out olmoe_tiny_c
SNAP=olmoe_tiny_c ./olmoe 8 8 olmoe_tiny/ref_olmoe.json > cpu.log 2>&1
COLI_VK_DENSE=1 COLI_VULKAN=1 SNAP=olmoe_tiny_c ./olmoe 8 8 olmoe_tiny/ref_olmoe.json > vk.log 2>&1
grep -qE 'Matching tokens: ([0-9]+)/\1$' vk.log || { cat vk.log; fail "olmoe: oracle"; }
same_tokens cpu.log vk.log "olmoe"
need_gpu olmoe vk.log "olmoe"
echo "OK olmoe: $(grep -o 'Matching tokens: [0-9/]*' vk.log), $(vk_count olmoe vk.log) matmuls on the GPU"
inkling_olmoe_tier_fixtures
# The routed-expert tier (COLI_VULKAN=1 turns it on; the runs above already had it).
# inkling: experts in f32 (fmt 10) under the f32, bf16 and dense-int4g64 snapshots,
# the int4 and int8 expert containers (fmt 2 and fmt 1: gate and up in one fused
# tensor, up I rows in), the runtime int4 and int8 quantization (fmt 2 from int8 rows,
# fmt 1), at cap=1 with the trunk where the default puts it (on the CPU here) and at
# cap=8 with the trunk on the device (COLI_VK_DENSE=1); TOPP's trimmed ranks; the tier
# alone (COLI_VK_DENSE=0); and a budget of two experts, which must evict.
local fx D bits R=tiny_inkling/ref_inkling.json OR=olmoe_tiny/ref_olmoe.json
for fx in tiny_inkling tiny_inkling_bf16 tiny_inkling_q tiny_inkling_x-i4 tiny_inkling_x-i8; do
for cap in 1 8; do
D=; [ $cap = 8 ] && D=COLI_VK_DENSE=1
tier_gate inkling "inkling tier $fx cap=$cap" $D SNAP=$fx -- $cap 0 $R
done
done
for bits in 4 8; do tier_gate inkling "inkling tier runtime int$bits" SNAP=tiny_inkling -- 2 $bits $R; done
tier_gate inkling "inkling tier TOPP" TOPP=0.3 SNAP=tiny_inkling_x-i4 -- 2 0 $R
tier_gate inkling "inkling tier alone (COLI_VK_DENSE=0)" COLI_VK_DENSE=0 SNAP=tiny_inkling_x-i4 -- 8 0 $R
EVICT=1 tier_gate inkling "inkling tier, a budget of two experts" COLI_VK_TIER_GB=0.00006 SNAP=tiny_inkling -- 8 0 $R
# olmoe: int8 rows (fmt 1), the same placements, PILOT's prefetch worker beside the
# tier, the tier alone and a budget of three experts, which must evict.
for cap in 1 8; do
D=; [ $cap = 8 ] && D=COLI_VK_DENSE=1
tier_gate olmoe "olmoe tier cap=$cap" $D SNAP=olmoe_tiny_c -- $cap 8 $OR
done
tier_gate olmoe "olmoe tier PILOT" PILOT=1 WIDE=2 SNAP=olmoe_tiny_c -- 2 8 $OR
tier_gate olmoe "olmoe tier alone (COLI_VK_DENSE=0)" COLI_VK_DENSE=0 SNAP=olmoe_tiny_c -- 8 8 $OR
EVICT=1 tier_gate olmoe "olmoe tier, a budget of three experts" COLI_VK_TIER_GB=0.000025 SNAP=olmoe_tiny_c -- 8 8 $OR
# The warm start: a CPU run writes the history (inkling's PIN=<file>, olmoe's
# COLI_USAGE), and the tier's run fills the device from it before the first token.
rm -f tier.hist
# Streaming can report its first device batch after the prompt has been printed.
# Keep diagnostics separate so the exact text comparison measures stdout alone.
PIN=tier.hist SNAP=tiny_inkling ./inkling 8 -p "The capital of France is" -n 8 > cpu.log 2> cpu.err
PIN=tier.hist COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 SNAP=tiny_inkling ./inkling 8 -p "The capital of France is" -n 8 > vk.log 2> vk.err
$PY - cpu.log vk.log <<'PY' || { cat cpu.log vk.log vk.err; fail "inkling tier warm start: the text differs from the CPU's"; }
import re, sys
def text(p):
m = re.search(rb"\[\d+ prompt tokens\](.*?)\n\[prefill", open(p, "rb").read(), re.S)
return m.group(1) if m else None
a, b = text(sys.argv[1]), text(sys.argv[2])
sys.exit(0 if a is not None and a == b else 1)
PY
grep -qa '^\[VK\] tier inkling: warm start, [1-9]' vk.err || { cat vk.err; fail "inkling tier: no warm start"; }
[ "$(tier_count inkling vk.err)" -gt 0 ] || { cat vk.err; fail "inkling tier warm start: no routed expert ran on the device"; }
echo "OK inkling tier warm start: text = CPU, $(grep -a -o 'warm start, [0-9]* experts' vk.err), $(grep -a -o 'device [0-9]* of [0-9]* routed experts' vk.err | tail -1)"
rm -f tier.hist
COLI_USAGE=tier.hist SNAP=olmoe_tiny_c ./olmoe 8 8 $OR > cpu.log 2>&1
COLI_USAGE=tier.hist COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 SNAP=olmoe_tiny_c ./olmoe 8 8 $OR > vk.log 2>&1
same_tokens cpu.log vk.log "olmoe tier warm start"
grep -qa '^\[VK\] tier olmoe: warm start, [1-9]' vk.log || { cat vk.log; fail "olmoe tier: no warm start"; }
[ "$(tier_count olmoe vk.log)" -gt 0 ] || { cat vk.log; fail "olmoe tier warm start: no routed expert ran on the device"; }
echo "OK olmoe tier warm start: tokens = CPU, $(grep -a -o 'warm start, [0-9]* experts' vk.log), $(grep -a -o 'device [0-9]* of [0-9]* routed experts' vk.log | tail -1)"
# The serve protocol with the tier on: KV prefix reuse token-identical to a cold
# engine, HITS and EMAP (tier 2 on the device) after every turn, Brio's snapshot
# scoring; every engine after the first warm-starts from the history the one before
# it saved.
COLI_VULKAN=1 COLI_VK_TIER_SYNC=1 INKLING_TINY=tiny_inkling \
$PY -m unittest tests.test_inkling_prefix_serve tests.test_inkling_dashboard_hits
COLI_VULKAN=1 COLI_VK_TIER_SYNC=1 OLMOE_TINY=olmoe_tiny_c \
$PY -m unittest tests.test_olmoe_prefix_serve tests.test_olmoe_dashboard_hits tests.test_brio_serve
}
# The fixtures the expert tier's arms add to the family's own: inkling's int4 and int8
# expert containers through tools/convert_inkling_int4.py's round trip (its fake
# vendor checkpoint knows the embedding norm by the name transformers used before
# 5.18), a tokenizer for inkling's -p mode and serve, and olmoe's for serve.
inkling_olmoe_tier_fixtures() {
$PY - <<'PY'
import importlib.util, os, shutil
from safetensors.torch import load_file, save_file
spec = importlib.util.spec_from_file_location("conv", "tools/convert_inkling_int4.py")
conv = importlib.util.module_from_spec(spec); spec.loader.exec_module(conv)
t = load_file("tiny_inkling/model.safetensors")
if "model.embed_tokens.embed_norm.weight" in t:
t["model.embed_norm.weight"] = t.pop("model.embed_tokens.embed_norm.weight")
for d in ("tiny_inkling_hf", "tiny_inkling_x-tml", "tiny_inkling_x-pass", "tiny_inkling_x-i4", "tiny_inkling_x-i8"):
shutil.rmtree(d, ignore_errors=True)
os.makedirs("tiny_inkling_hf")
save_file(t, "tiny_inkling_hf/model.safetensors")
shutil.copy("tiny_inkling/config.json", "tiny_inkling_hf/")
conv.selftest_e2e("tiny_inkling_hf", "tiny_inkling_x") # tiny_inkling_x-i4: int4 experts
conv.convert_dir("tiny_inkling_x-tml", "tiny_inkling_x-i8", 8) # int8 experts
PY
$PY -c 'import sys; from pathlib import Path; sys.path.insert(0, "."); from tests.test_inkling_prefix_serve import ensure_tokenizer; ensure_tokenizer(Path("tiny_inkling"))'
$PY tools/make_edge_tiny_tokenizer.py --vocab-size 128 olmoe_tiny_c
}
# inkling's and olmoe's expert tier under ASan and UBSan: a sanitized VK=1 build, the
# tier's configurations on Lavapipe (the expert formats, eviction, TOPP, PILOT's worker
# beside the tier, the tier alone, the trunk on the device or on the CPU, the warm
# start) and a serve session of each engine, twice, the second warm-started. Memory
# safety is the gate, not the tokens (a sanitized build vectorizes differently); each
# run must still put experts on the device, or the tier was never exercised.
family_inkling_olmoe_sanitize() {
make clean >/dev/null 2>&1 || true
make inkling olmoe VK=1 EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
$PY tools/make_tiny_inkling.py tiny_inkling
$PY tools/make_olmoe_tiny.py --output olmoe_tiny --force
$PY tools/convert_olmoe_merged.py --model olmoe_tiny --out olmoe_tiny_c
inkling_olmoe_tier_fixtures
san_io() { # <engine> <tag> <env and argv...>; KEEP=1 keeps the history of the run before
local eng=$1 tag=$2; shift 2
[ "${KEEP:-0}" = 1 ] || rm -f tier.usage
env ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1 OMP_NUM_THREADS=2 \
COLI_USAGE=tier.usage COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 "$@" > san.log 2>&1 || true
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "$tag: sanitizer diagnostic"; fi
[ "$(tier_count "$eng" san.log)" -gt 0 ] || { cat san.log; fail "$tag: no routed expert ran on the device"; }
echo "OK $tag: sanitizers clean, $(grep -a -o 'device [0-9]* of [0-9]* routed experts' san.log | tail -1), $(grep -a -o 'evictions [0-9]*' san.log | tail -1)$(grep -a -o 'warm start, [0-9]* experts' san.log | sed 's/^/, /')"
}
local fx cap D R=tiny_inkling/ref_inkling.json OR=olmoe_tiny/ref_olmoe.json
for fx in tiny_inkling tiny_inkling_x-i4 tiny_inkling_x-i8; do
for cap in 1 8; do
D=; [ $cap = 8 ] && D=COLI_VK_DENSE=1
san_io inkling "asan inkling $fx cap=$cap" $D SNAP=$fx ./inkling $cap 0 $R
done
done
san_io inkling "asan inkling runtime int4" SNAP=tiny_inkling ./inkling 2 4 $R
san_io inkling "asan inkling TOPP" TOPP=0.3 SNAP=tiny_inkling_x-i4 ./inkling 2 0 $R
san_io inkling "asan inkling eviction" COLI_VK_TIER_GB=0.00006 SNAP=tiny_inkling ./inkling 8 0 $R
san_io inkling "asan inkling tier alone" COLI_VK_DENSE=0 SNAP=tiny_inkling_x-i4 ./inkling 8 0 $R
san_io inkling "asan inkling -p, history written" PIN=tier.usage SNAP=tiny_inkling ./inkling 8 -p "The capital of France is" -n 8
KEEP=1 san_io inkling "asan inkling -p, warm start" PIN=tier.usage SNAP=tiny_inkling ./inkling 8 -p "The capital of France is" -n 8
grep -qa 'tier inkling: warm start, [1-9]' san.log || { cat san.log; fail "asan inkling: no warm start"; }
for cap in 1 8; do
D=; [ $cap = 8 ] && D=COLI_VK_DENSE=1
san_io olmoe "asan olmoe PILOT cap=$cap" $D PILOT=1 WIDE=2 SNAP=olmoe_tiny_c ./olmoe $cap 8 $OR
done
san_io olmoe "asan olmoe eviction" COLI_VK_TIER_GB=0.000025 PILOT=1 SNAP=olmoe_tiny_c ./olmoe 8 8 $OR
san_io olmoe "asan olmoe tier alone" COLI_VK_DENSE=0 SNAP=olmoe_tiny_c ./olmoe 8 8 $OR
KEEP=1 san_io olmoe "asan olmoe warm start" PILOT=1 SNAP=olmoe_tiny_c ./olmoe 2 8 $OR
grep -qa 'tier olmoe: warm start, [1-9]' san.log || { cat san.log; fail "asan olmoe: no warm start"; }
# serve: two turns, then a second engine warm-started from the history the first saved
local eng
for eng in inkling olmoe; do
rm -f tier.usage
$PY - $eng <<'PY' || fail "asan $eng serve"
import os, subprocess, sys, threading
eng = sys.argv[1]
snap, argv = ("tiny_inkling", ["8"]) if eng == "inkling" else ("olmoe_tiny_c", ["4", "8"])
env = dict(os.environ, SNAP=snap, SERVE="1", PIN="tier.usage", COLI_USAGE="tier.usage", COLI_VULKAN="1",
COLI_VK_TIER_SYNC="1", OMP_NUM_THREADS="2", ASAN_OPTIONS="detect_leaks=0:detect_stack_use_after_return=0",
UBSAN_OPTIONS="print_stacktrace=1")
for run in range(2):
p = subprocess.Popen(["./" + eng] + argv, env=env, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, bufsize=0)
err = []
t = threading.Thread(target=lambda: err.extend(iter(p.stderr.readline, b"")), daemon=True)
t.start()
while b"READY" not in p.stdout.readline():
pass
for rid, prompt in (("1", b"The capital of France is"), ("2", b"The capital of France is, and Spain")):
p.stdin.write(f"SUBMIT {rid} 0 {len(prompt)} 6 0 1\n".encode() + prompt + b"\n")
p.stdin.flush()
while True:
line = p.stdout.readline()
if not line or line.split(b" ", 1)[0] in (b"DONE", b"ERROR"):
break
if line.startswith(b"DATA "):
p.stdout.read(int(line.split()[2])); p.stdout.readline()
p.stdin.close(); p.wait(timeout=300); t.join(10)
log = b"".join(err).decode(errors="replace")
if "ERROR: AddressSanitizer" in log or "runtime error:" in log:
print(log); sys.exit(1)
turns = [l for l in log.splitlines() if l.startswith(f"[VK] tier {eng} turn: device ")]
if len(turns) != 2 or turns[-1].split()[5] == "0" or (run == 1 and "warm start" not in log):
print(log); sys.exit(1)
print(f"OK asan {eng} serve, engine {run + 1}: sanitizers clean, " + " ".join(turns[-1].split()[4:9]) +
(", warm-started" if run else ""))
PY
done
make clean >/dev/null 2>&1 || true
}
family_mimo_qwenimage() {
make mimo qwenimage VK=1
# mimo: Xiaomi's vendor oracle on the GPU (its engine_env is the f32 dense
# configuration; its variants include the native FP8/BF16 one and the BF16 vision
# tower): the trunk on the device with the experts on the CPU (COLI_VK_TIER=0), the
# routed experts on the shared tier (MXFP4, fmt 7) with the trunk where the default
# puts it (the CPU on Lavapipe), and both on the device.
$PY tools/make_mimo_tiny.py --output ./mimo_tiny --force --vision
COLI_VULKAN=1 COLI_VK_TIER=0 $PY tests/mimo_tiny_harness.py --binary ./mimo --fixture ./mimo_tiny
COLI_VULKAN=1 COLI_VK_TIER_SYNC=1 $PY tests/mimo_tiny_harness.py --binary ./mimo --fixture ./mimo_tiny
COLI_VULKAN=1 COLI_VK_TIER_SYNC=1 COLI_VK_DENSE=1 $PY tests/mimo_tiny_harness.py --binary ./mimo --fixture ./mimo_tiny
# MIMO_DENSE_BITS 0 (native fp8/bf16: fmt 12, 11), 8 (fmt 1) and 32 (fmt 10):
# the CPU's tokens for every case of ref.json and for the picture, with the experts
# on the CPU (MIMO_VK_EXPERTS=0, the trunk on the device), on the tier with the trunk
# on the CPU (the tier must have served some), and on the tier with the trunk on the
# device (both).
ids() { $PY -c "import json,sys;r=json.load(open('mimo_tiny/ref.json'));c=r['image'] if sys.argv[1]=='image' else r['cases'][sys.argv[1]];print(' '.join(map(str,c['prompt_ids'])))" "$1"; }
local grid bits c x extra
grid=$($PY -c "import json;i=json.load(open('mimo_tiny/ref.json'))['image'];print(i['grid_h'],i['grid_w'])")
for bits in 0 8 32; do
for c in short window long image; do
extra=(); [ "$c" = image ] && extra=(--image mimo_tiny/patches.f32 --grid $grid)
MIMO_DENSE_BITS=$bits COLI_TEMP=0 ./mimo mimo_tiny --ids "$(ids $c)" --ngen 6 "${extra[@]}" > mimo-cpu.txt 2>/dev/null
for x in MIMO_VK_EXPERTS=0 COLI_VK_DENSE=0 COLI_VK_DENSE=1; do
env COLI_VULKAN=1 COLI_VK_TIER_SYNC=1 $x MIMO_DENSE_BITS=$bits COLI_TEMP=0 \
./mimo mimo_tiny --ids "$(ids $c)" --ngen 6 "${extra[@]}" > mimo-vk.txt 2> mimo-vk.err
cmp -s mimo-cpu.txt mimo-vk.txt || { cat mimo-vk.err; fail "mimo bits=$bits $c $x differs from the CPU"; }
[ $x = COLI_VK_DENSE=0 ] || need_gpu mimo mimo-vk.err "mimo bits=$bits $c $x"
if [ $x = MIMO_VK_EXPERTS=0 ]; then
! grep -q '^\[VK\] tier' mimo-vk.err || { cat mimo-vk.err; fail "mimo bits=$bits $c: MIMO_VK_EXPERTS=0 started the tier"; }
else
[ "$(tier_count mimo mimo-vk.err)" -gt 0 ] || { cat mimo-vk.err; fail "mimo bits=$bits $c $x: no routed expert ran on the device"; }
fi
done
done
echo "OK mimo MIMO_DENSE_BITS=$bits: the CPU's tokens, text and image, experts on the CPU and on the tier, trunk on the device and on the CPU"
done
# MIMO_VK_EXPERTS=N sizes the tier at N experts: at 2 it must evict as the routing
# moves, and still give the CPU's tokens
MIMO_DENSE_BITS=32 COLI_TEMP=0 ./mimo mimo_tiny --ids "$(ids long)" --ngen 6 > mimo-cpu.txt 2>/dev/null
COLI_VULKAN=1 COLI_VK_TIER_SYNC=1 MIMO_VK_EXPERTS=2 MIMO_DENSE_BITS=32 COLI_TEMP=0 \
./mimo mimo_tiny --ids "$(ids long)" --ngen 6 > mimo-vk.txt 2> mimo-vk.err
cmp -s mimo-cpu.txt mimo-vk.txt || { cat mimo-vk.err; fail "mimo MIMO_VK_EXPERTS=2 differs from the CPU"; }
grep -q 'budget [0-9.]* KiB = 2 experts' mimo-vk.err || { grep '\[VK\]' mimo-vk.err; fail "mimo MIMO_VK_EXPERTS=2: not a budget of two experts"; }
[ "$(tier_count mimo mimo-vk.err)" -gt 0 ] && [ "$(tier_evictions mimo mimo-vk.err)" -gt 0 ] || { grep '\[VK\] tier' mimo-vk.err; fail "mimo MIMO_VK_EXPERTS=2: the tier served nothing or never evicted"; }
grep -q ' failed 0 ' mimo-vk.err || { grep '\[VK\] tier' mimo-vk.err; fail "mimo MIMO_VK_EXPERTS=2: an upload failed"; }
echo "OK mimo, a budget of two experts: tokens = CPU, $(grep -a -o 'device [0-9]* of [0-9]* routed experts' mimo-vk.err | tail -1), $(grep -a -o 'evictions [0-9]*' mimo-vk.err | tail -1)"
# qwenimage: at 8, 16 and 32 bits (fmt 1, 11, 10) every oracle stage of the
# Lavapipe run against the CPU run with the same bits and f32 activations. int8 is
# outside the oracle's own tolerance on both sides, so the reports are compared
# with each other, not with the reference; at 16 bits the Vulkan run must also pass.
$PY tools/make_qwenimage_tiny.py qwenimage_tiny
for bits in 8 16 32; do
COLI_IMG_BITS=$bits COLI_IMG_ACT8=0 ./qwenimage --model qwenimage_tiny --ref qwenimage_tiny/ref > qi-cpu.log 2>&1 || true
local rc=0
COLI_VULKAN=1 COLI_IMG_BITS=$bits COLI_IMG_ACT8=0 ./qwenimage --model qwenimage_tiny --ref qwenimage_tiny/ref > qi-vk.log 2>&1 || rc=$?
[ $bits != 16 ] || [ $rc = 0 ] || { cat qi-vk.log; fail "qwenimage bf16 oracle on Vulkan"; }
need_gpu qwenimage qi-vk.log "qwenimage oracle bits=$bits"
BITS=$bits $PY - <<'PY'
import os, re
def stages(p):
return {k.strip(): float(r) for k, r in
re.findall(r"\[oracle\] (.+?)\s+n=\d+\s+max\|err\| \S+\s+rel (\S+)", open(p).read())}
c, v = stages("qi-cpu.log"), stages("qi-vk.log")
assert len(c) >= 10 and c.keys() == v.keys(), (c, v)
for k in c: # relative errors against the reference, CPU vs GPU: equal up to summation order
assert abs(c[k] - v[k]) <= 0.25 * c[k] + 2e-7, (k, c[k], v[k])
print(f"OK qwenimage oracle: {len(c)} stages of the Lavapipe run match the CPU run at {os.environ['BITS']} bits")
PY
done
# a generated picture at the default int8 weights, Lavapipe against the CPU
COLI_IMG_BITS=8 COLI_IMG_ACT8=0 ./qwenimage --model qwenimage_tiny --prompt "a red fox in the snow" \
--width 256 --height 256 --steps 2 --seed 1 --out qi-cpu.png
COLI_VULKAN=1 COLI_IMG_BITS=8 COLI_IMG_ACT8=0 ./qwenimage --model qwenimage_tiny --prompt "a red fox in the snow" \
--width 256 --height 256 --steps 2 --seed 1 --out qi-vk.png 2> qi-vk-gen.err
need_gpu qwenimage qi-vk-gen.err "qwenimage picture"
$PY - <<'PY'
import sys; sys.path.insert(0, ".")
import image_engine as e
_, _, _, a = e.decode_png(open("qi-cpu.png", "rb").read())
_, _, _, b = e.decode_png(open("qi-vk.png", "rb").read())
d = [abs(x - y) for x, y in zip(a, b)]
assert len(a) == len(b) and max(d) <= 2 and sum(t > 0 for t in d) <= len(d) // 1000, (max(d), sum(t > 0 for t in d))
print(f"OK qwenimage picture: {sum(t > 0 for t in d)} of {len(d)} bytes differ from the CPU's, max {max(d)}")
PY
# the serve protocol with the device on
COLI_VULKAN=1 QWENIMAGE_TINY=qwenimage_tiny $PY -m unittest tests.test_qwenimage_engine_serve
# the DiT on the chain (qwenimage_chain.h; off by default on Lavapipe, a CPU device):
# every oracle stage against the CPU's at 8, 16 and 32 bits, the attention in slices of
# rows, the device lost in a step's frames (the CPU takes the step again), a picture
qi_chain() { # <tag> <bits> <env...>
local tag=$1 bits=$2; shift 2
COLI_IMG_BITS=$bits COLI_IMG_ACT8=0 ./qwenimage --model qwenimage_tiny --ref qwenimage_tiny/ref > qi-cpu.log 2>&1 || true
env "$@" COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_IMG_BITS=$bits COLI_IMG_ACT8=0 ./qwenimage --model qwenimage_tiny --ref qwenimage_tiny/ref \
> qi-vk.log 2>&1 || true
grep -qa 'qwenimage chain: [0-9]* blocks on the device' qi-vk.log || { cat qi-vk.log; fail "$tag: the chain never ran"; }
TAG="$tag" $PY - <<'QIPY'
import os, re
def stages(p):
return {k.strip(): float(r) for k, r in
re.findall(r"\[oracle\] (.+?)\s+n=\d+\s+max\|err\| \S+\s+rel (\S+)", open(p).read())}
c, v = stages("qi-cpu.log"), stages("qi-vk.log")
assert len(c) >= 10 and c.keys() == v.keys(), (c, v)
for k in c:
assert abs(c[k] - v[k]) <= 0.25 * c[k] + 2e-7, (k, c[k], v[k])
print(f"OK {os.environ['TAG']}: {len(c)} stages match the CPU run")
QIPY
}
for bits in 8 16 32; do
qi_chain "qwenimage chain bits=$bits" $bits
grep -qa 'qwenimage vae: [0-9]* decodes on the device' qi-vk.log || { cat qi-vk.log; fail "qwenimage chain bits=$bits: the VAE did not decode on the device"; }
done
qi_chain "qwenimage chain, the attention in slices" 32 COLI_VK_ATTN_SLICE=60000
grep -qa 'qwenimage chain: [0-9]* steps' qi-vk.log || { cat qi-vk.log; fail "qwenimage chain, slices: no step on the device"; }
# a device that holds one block of the two, or none: the others go up each step
for r in 1 0; do
qi_chain "qwenimage chain, $r of 2 blocks resident" 8 COLI_VK_QI_RESIDENT=$r
grep -qa "qwenimage chain: $((2 - r)) blocks streamed" qi-vk.log || { cat qi-vk.log; fail "qwenimage chain, $r resident: no block streamed"; }
done
qi_chain "qwenimage chain, the device lost" 32 COLI_VK_CHAIN_FAULT=9
grep -qa 'qwenimage chain: a frame failed' qi-vk.log || { cat qi-vk.log; fail "qwenimage chain, device lost: no loss handled"; }
COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_IMG_BITS=8 COLI_IMG_ACT8=0 ./qwenimage --model qwenimage_tiny --prompt "a red fox in the snow" \
--width 256 --height 256 --steps 2 --seed 1 --out qi-vk.png 2> qi-vk-gen.err
grep -qa 'qwenimage chain: [0-9]* steps' qi-vk-gen.err || { cat qi-vk-gen.err; fail "qwenimage chain picture: the chain never ran"; }
grep -qa 'qwenimage vae: 1 decodes on the device' qi-vk-gen.err || { cat qi-vk-gen.err; fail "qwenimage chain picture: the VAE did not decode on the device"; }
$PY - <<'QIPY'
import sys; sys.path.insert(0, ".")
import image_engine as e
_, _, _, a = e.decode_png(open("qi-cpu.png", "rb").read())
_, _, _, b = e.decode_png(open("qi-vk.png", "rb").read())
d = [abs(x - y) for x, y in zip(a, b)]
assert len(a) == len(b) and max(d) <= 2 and sum(t > 0 for t in d) <= len(d) // 1000, (max(d), sum(t > 0 for t in d))
print(f"OK qwenimage chain picture: {sum(t > 0 for t in d)} of {len(d)} bytes differ from the CPU's, max {max(d)}")
QIPY
}
family_deepseek() {
make deepseek_v41 VK=1
# deepseek_v41: fp8 dense in 32x32 ue8m0 tiles (fmt 12, gs 32, the tile scale
# repeated over its rows) and bf16 (fmt 11). The engine exits non-zero on any
# token mismatch with the reference; the CPU run must print the same stream.
# These arms test the dense trunk: COLI_VK_DENSE=1, because on Lavapipe the trunk
# otherwise stays on the CPU while the expert tier is on; the tier runs beside it.
$PY tools/make_dsv41_tiny.py --out dsv41_tiny --emit-ref dsv41_tiny/ref.json
$PY tools/make_dsv41_tiny.py --out dsv41_long --emit-ref dsv41_long/ref.json --prompt-len 40 --max-new 6
v41() { # <tag> <env and argv...>
local tag=$1; shift
env "$@" > v41-cpu.txt 2> v41-cpu.err || { cat v41-cpu.err; fail "deepseek_v41 $tag: CPU run"; }
env COLI_VULKAN=1 "$@" > v41-vk.txt 2> v41-vk.err || { cat v41-vk.err; fail "deepseek_v41 $tag: Vulkan run misses the oracle"; }
cmp -s v41-cpu.txt v41-vk.txt || { diff v41-cpu.txt v41-vk.txt | head; fail "deepseek_v41 $tag: Vulkan output differs from the CPU"; }
need_gpu deepseek_v41 v41-vk.err "deepseek_v41 $tag"
echo "OK deepseek_v41 $tag: output = CPU, $(vk_count deepseek_v41 v41-vk.err) matmuls on the GPU, $(grep -a -o 'device [0-9]* of [0-9]* routed experts' v41-vk.err | tail -1)"
}
local cap force
for cap in 1 2 8; do v41 "cap=$cap" COLI_VK_DENSE=1 SNAP=dsv41_tiny ./deepseek_v41 $cap dsv41_tiny/ref.json; done
for cap in 2 8; do v41 "40-token prompt cap=$cap" COLI_VK_DENSE=1 SNAP=dsv41_long ./deepseek_v41 $cap dsv41_long/ref.json; done
for force in 1 2 3 4 5; do
v41 "DSpark spec=$force" COLI_VK_DENSE=1 SNAP=dsv41_tiny V41_DSPARK=1 V41_SPEC_FORCE=$force ./deepseek_v41 8 dsv41_tiny/ref.json
done
# The routed-expert tier (vk_tier.c) on deepseek_v41's experts, fp4 with a ue8m0
# scale per 32 (fmt 7), the clamped SwiGLU; the trunk where the default puts it (the
# CPU here). Each run must pass the oracle, print the CPU run's stream and have
# served routed experts from the device; with EVICT=1 the budget (below the hot set)
# must also have evicted. COLI_USAGE points at a fresh history, COLI_VK_TIER_SYNC=1
# awaits each staged upload at the next step (as in tier_gate).
v41_tier() { # <tag> <env and argv...>
local tag=$1; shift
rm -f tier.usage
env "$@" > v41-cpu.txt 2> v41-cpu.err || { cat v41-cpu.err; fail "deepseek_v41 tier $tag: CPU run"; }
env COLI_USAGE=tier.usage COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 "$@" > v41-vk.txt 2> v41-vk.err ||
{ cat v41-vk.err; fail "deepseek_v41 tier $tag: Vulkan run misses the oracle"; }
cmp -s v41-cpu.txt v41-vk.txt || { diff v41-cpu.txt v41-vk.txt | head; fail "deepseek_v41 tier $tag: Vulkan output differs from the CPU"; }
[ "$(tier_count deepseek_v41 v41-vk.err)" -gt 0 ] || { cat v41-vk.err; fail "deepseek_v41 tier $tag: no routed expert ran on the device"; }
if [ "${EVICT:-0}" = 1 ]; then
[ "$(tier_evictions deepseek_v41 v41-vk.err)" -gt 0 ] || { grep '\[VK\] tier' v41-vk.err; fail "deepseek_v41 tier $tag: the budget forced no eviction"; }
[ "$(tier_failed deepseek_v41 v41-vk.err)" = 0 ] || { grep '\[VK\] tier' v41-vk.err; fail "deepseek_v41 tier $tag: an upload failed (the budget shrank)"; }
fi
echo "OK deepseek_v41 tier $tag: output = CPU, $(grep -a -o 'device [0-9]* of [0-9]* routed experts' v41-vk.err | tail -1), $(grep -a -o 'evictions [0-9]*' v41-vk.err | tail -1)"
}
for cap in 1 2 8; do v41_tier "cap=$cap" SNAP=dsv41_tiny ./deepseek_v41 $cap dsv41_tiny/ref.json; done
for cap in 2 8; do v41_tier "40-token prompt cap=$cap" SNAP=dsv41_long ./deepseek_v41 $cap dsv41_long/ref.json; done
# DSpark: the verify rows of the backbone take the device, and the drafts stay on the
# CPU (the default on Lavapipe, which shares the RAM); with COLI_VK_TIER_MTP=1 (the
# default on a discrete GPU) the stages are the tier's extra layers, their experts on
# the device too
for force in 1 2 3 4 5; do
v41_tier "DSpark spec=$force" SNAP=dsv41_tiny V41_DSPARK=1 V41_SPEC_FORCE=$force ./deepseek_v41 1 dsv41_tiny/ref.json
! grep -aq 'extra layers' v41-vk.err || { grep -a '\[VK\] tier' v41-vk.err; fail "deepseek_v41 DSpark spec=$force: the stages went on the tier by default on shared RAM"; }
done
for force in 1 3 5; do
v41_tier "DSpark stages on the tier spec=$force" SNAP=dsv41_tiny V41_DSPARK=1 V41_SPEC_FORCE=$force COLI_VK_TIER_MTP=1 ./deepseek_v41 1 dsv41_tiny/ref.json
mtp_on_device "deepseek_v41 DSpark stages on the tier spec=$force" v41-vk.err
done
EVICT=1 v41_tier "a budget of three experts" COLI_VK_TIER_GB=0.00005 SNAP=dsv41_long ./deepseek_v41 8 dsv41_long/ref.json
v41_tier "trunk on the device too" COLI_VK_DENSE=1 SNAP=dsv41_long ./deepseek_v41 2 dsv41_long/ref.json
# deepseek_v4: fp8 128x128 blocks (fmt 12, gs 128) and the bf16 router, compressors
# and head (fmt 11). The GPU gets the activations after the CPU's own E4M3 rounding,
# so the two runs do the same arithmetic. The tiny check builds the VK=1 binary and
# keeps passing with the device open (the expert tier on, the trunk on the CPU);
# its --oracle path reloads the dense weights every forward and so stays on the CPU,
# which is why the device is checked on the session path below: ids and
# teacher-forced predictions equal to the CPU's and to the reference's greedy stream.
COLI_VULKAN=1 make deepseek-v4-tiny-check VK=1
local prompt
prompt=$($PY -c 'import json; c=json.load(open("deepseek_v4_tiny/ref.json"))["cases"]["long"]; print("".join("<t%03d>" % t for t in c["prompt_ids"]))')
./deepseek_v4 ./deepseek_v4_tiny "$prompt" --raw-prompt --max-tokens 4 --record-oracle v4-cpu.json > /dev/null
COLI_VK_DENSE=1 COLI_VULKAN=1 ./deepseek_v4 ./deepseek_v4_tiny "$prompt" --raw-prompt --max-tokens 4 --record-oracle v4-vk.json > /dev/null 2> v4-vk.err
$PY - <<'PY' || fail "deepseek_v4: the Vulkan session differs from the CPU's"
import json, sys
a, b = json.load(open("v4-cpu.json")), json.load(open("v4-vk.json"))
ref = json.load(open("deepseek_v4_tiny/ref.json"))["cases"]["long"]["greedy_full_ids"]
ok = a["full_ids"] == b["full_ids"] == ref and a["tf_pred"] == b["tf_pred"]
print("OK deepseek_v4 session: ids = CPU = reference" if ok else ("CPU", a, "VK", b, "ref", ref))
sys.exit(0 if ok else 1)
PY
need_gpu deepseek_v4 v4-vk.err "deepseek_v4 session"
echo "OK deepseek_v4: $(vk_count deepseek_v4 v4-vk.err) matmuls on the GPU, $(grep -a -o 'device [0-9]* of [0-9]* routed experts' v4-vk.err | tail -1)"
# The routed-expert tier on deepseek_v4's fp4 experts (fmt 7) with its own
# activation (VKT_ACT_SWIGLU_V4: the CPU kernel's bf16 and E4M3 roundings and the
# route weight before down, on the device). Every oracle case against the CPU run:
# ids equal to the CPU's and the reference's, teacher-forced predictions equal,
# routed experts served by the device. The history is the store's own .coli_usage
# in the fixture, removed before each run unless WARM=1 (the tier then warm-starts
# from the CPU run's): the runs use a copy of the fixture, whose committed history
# stays as it is. An 8-expert variant of the fixture has room for pinned hot
# experts, which the store keeps in its rows16 layout: the tier gets them unpacked.
rm -rf deepseek_v4_tiny_t && cp -r deepseek_v4_tiny deepseek_v4_tiny_t
$PY - tools/make_deepseek_v4_tiny.py deepseek_v4_tiny_e8 <<'PY'
import importlib.util, sys
spec = importlib.util.spec_from_file_location("gen", sys.argv[1])
gen = importlib.util.module_from_spec(spec); spec.loader.exec_module(gen)
gen.EXPERTS = 8 # 8 routed experts a layer: two cache slots a layer for pins
sys.argv = ["make_deepseek_v4_tiny.py", "--output", sys.argv[2], "--force"]
gen.main()
PY
v4_tier() { # <tag> <fixture> <case> <env...>
local tag=$1 fx=$2 c=$3; shift 3
local p mt
p=$($PY -c 'import json,sys; c=json.load(open(sys.argv[1]+"/ref.json"))["cases"][sys.argv[2]]; print("".join("<t%03d>" % t for t in c["prompt_ids"]))' $fx $c)
mt=$($PY -c 'import json,sys; print(json.load(open(sys.argv[1]+"/ref.json"))["cases"][sys.argv[2]]["max_new_tokens"])' $fx $c)
rm -f $fx/.coli_usage
env "$@" ./deepseek_v4 ./$fx "$p" --raw-prompt --max-tokens $mt --record-oracle v4-cpu.json > /dev/null 2> v4-cpu.err ||
{ cat v4-cpu.err; fail "deepseek_v4 tier $tag: CPU run"; }
[ "${WARM:-0}" = 1 ] || rm -f $fx/.coli_usage
env "$@" COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 ./deepseek_v4 ./$fx "$p" --raw-prompt --max-tokens $mt --record-oracle v4-vk.json > /dev/null 2> v4-vk.err ||
{ cat v4-vk.err; fail "deepseek_v4 tier $tag: Vulkan run"; }
rm -f $fx/.coli_usage
$PY - $fx $c <<'PY' || fail "deepseek_v4 tier $tag: the Vulkan session differs from the CPU's"
import json, sys
a, b = json.load(open("v4-cpu.json")), json.load(open("v4-vk.json"))
ref = json.load(open(sys.argv[1] + "/ref.json"))["cases"][sys.argv[2]]["greedy_full_ids"]
ok = a["full_ids"] == b["full_ids"] == ref and a["tf_pred"] == b["tf_pred"]
if not ok: print("CPU", a, "VK", b, "ref", ref)
sys.exit(0 if ok else 1)
PY
[ "$(tier_count deepseek_v4 v4-vk.err)" -gt 0 ] || { cat v4-vk.err; fail "deepseek_v4 tier $tag: no routed expert ran on the device"; }
if [ "${EVICT:-0}" = 1 ]; then
[ "$(tier_evictions deepseek_v4 v4-vk.err)" -gt 0 ] || { grep '\[VK\] tier' v4-vk.err; fail "deepseek_v4 tier $tag: the budget forced no eviction"; }
[ "$(tier_failed deepseek_v4 v4-vk.err)" = 0 ] || { grep '\[VK\] tier' v4-vk.err; fail "deepseek_v4 tier $tag: an upload failed (the budget shrank)"; }
fi
if [ "${WARM:-0}" = 1 ]; then
grep -q '^\[VK\] tier deepseek_v4: warm start' v4-vk.err || { grep '\[VK\] tier' v4-vk.err; fail "deepseek_v4 tier $tag: no warm start"; }
fi
echo "OK deepseek_v4 tier $tag ($c): ids = CPU = reference, $(grep -a -o 'device [0-9]* of [0-9]* routed experts' v4-vk.err | tail -1), $(grep -a -o 'evictions [0-9]*' v4-vk.err | tail -1), $(grep -a -o 'packed_slots=[0-9]*' v4-vk.err) rows16"
}
for c in short compressed long; do
v4_tier "4 experts" deepseek_v4_tiny_t $c
v4_tier "8 experts, pinned rows16" deepseek_v4_tiny_e8 $c
done
EVICT=1 v4_tier "a budget of three experts" deepseek_v4_tiny_e8 long COLI_VK_TIER_GB=0.00009
v4_tier "trunk on the device too" deepseek_v4_tiny_t long COLI_VK_DENSE=1
v4_tier "the GEMM route from 2 rows" deepseek_v4_tiny_t long COLI_VK_TIER_GEMM_ROWS=2
WARM=1 v4_tier "warm start from the CPU run's history" deepseek_v4_tiny_e8 long
# A served prompt with every routed expert on the device (warm start from the CPU
# run's history): its per-position logprob echoes against the CPU's. Measured on
# Lavapipe they are the same bytes; the gate allows 1e-3, for an exp() that rounds
# one hidden value to the other bf16 neighbour on another driver.
$PY - deepseek_v4_tiny_e8 <<'PY' || fail "deepseek_v4 tier: served logprobs differ from the CPU's"
import json, os, sys
from pathlib import Path
sys.path.insert(0, "tests")
import test_deepseek_v4_brio as b
fx = Path(sys.argv[1])
case = json.load(open(fx / "ref.json"))["cases"]["long"]
def echoes(vk):
os.environ.pop("COLI_VULKAN", None)
if vk: os.environ.update(COLI_VULKAN="1", COLI_VK_TIER_SYNC="1")
else:
try: os.remove(fx / ".coli_usage")
except FileNotFoundError: pass
s = b.Serve(Path("deepseek_v4").resolve(), fx)
try: return s.submit(b.token_prompt(case["prompt_ids"]), 0, logprobs=5).echoes
finally:
s.close()
tier = [l for l in s.process.stderr.read().decode(errors="replace").splitlines() if "tier deepseek_v4 turn:" in l]
if vk: print(" ", tier[-1][:90] if tier else "no tier line")
cpu, dev = echoes(False), echoes(True)
os.environ.pop("COLI_VULKAN", None)
same = sum(cpu[p] == dev.get(p) for p in cpu)
worst = max(abs(cpu[p]["lp"] - dev[p]["lp"]) for p in cpu) if len(cpu) == len(dev) else 1.0
print(f"OK deepseek_v4 tier served: {same} of {len(cpu)} positions' logprob echoes identical to the CPU's, worst |delta| {worst:.2e}")
sys.exit(0 if len(cpu) == len(dev) and worst <= 1e-3 else 1)
PY
rm -rf deepseek_v4_tiny_t deepseek_v4_tiny_e8
}
# The routed-expert tier of both deepseek engines under ASan and UBSan: sanitized VK=1
# builds, the tier's configurations on Lavapipe (decode and prefill, eviction, DSpark,
# pinned rows16 experts, the warm start, the trunk on the device). Memory safety is
# the gate; each run must still put experts on the device.
family_deepseek_sanitize() {
local SAN="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
make clean >/dev/null 2>&1 || true
make deepseek_v41 VK=1 EXTRA_CFLAGS="$SAN"
make deepseek-v4 VK=1 LTO=0 EXTRA_CFLAGS="$SAN" EXTRA_LDFLAGS="$SAN"
$PY tools/make_dsv41_tiny.py --out dsv41_tiny --emit-ref dsv41_tiny/ref.json
$PY tools/make_dsv41_tiny.py --out dsv41_long --emit-ref dsv41_long/ref.json --prompt-len 40 --max-new 6
$PY tools/make_deepseek_v4_tiny.py --output deepseek_v4_tiny_t --force
$PY - tools/make_deepseek_v4_tiny.py deepseek_v4_tiny_e8 <<'PY'
import importlib.util, sys
spec = importlib.util.spec_from_file_location("gen", sys.argv[1])
gen = importlib.util.module_from_spec(spec); spec.loader.exec_module(gen)
gen.EXPERTS = 8
sys.argv = ["make_deepseek_v4_tiny.py", "--output", sys.argv[2], "--force"]
gen.main()
PY
san() { # <engine> <tag> <env and argv...>
local eng=$1 tag=$2; shift 2
rm -f tier.usage deepseek_v4_tiny_t/.coli_usage deepseek_v4_tiny_e8/.coli_usage
env ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1 OMP_NUM_THREADS=2 \
COLI_USAGE=tier.usage COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 "$@" > san.log 2>&1 || true
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "$tag: sanitizer diagnostic"; fi
[ "$(tier_count "$eng" san.log)" -gt 0 ] || { cat san.log; fail "$tag: no routed expert ran on the device"; }
echo "OK $tag: sanitizers clean, $(grep -a -o 'device [0-9]* of [0-9]* routed experts' san.log | tail -1), $(grep -a -o 'evictions [0-9]*' san.log | tail -1)"
}
local cap force p
for cap in 1 8; do san deepseek_v41 "asan deepseek_v41 cap=$cap" SNAP=dsv41_long ./deepseek_v41 $cap dsv41_long/ref.json; done
for force in 2 4; do san deepseek_v41 "asan deepseek_v41 DSpark spec=$force" SNAP=dsv41_tiny V41_DSPARK=1 V41_SPEC_FORCE=$force ./deepseek_v41 1 dsv41_tiny/ref.json; done
san deepseek_v41 "asan deepseek_v41 DSpark stages on the tier" SNAP=dsv41_tiny V41_DSPARK=1 V41_SPEC_FORCE=3 COLI_VK_TIER_MTP=1 ./deepseek_v41 1 dsv41_tiny/ref.json
mtp_on_device "asan deepseek_v41 DSpark stages on the tier" san.log
san deepseek_v41 "asan deepseek_v41 eviction" COLI_VK_TIER_GB=0.00005 SNAP=dsv41_long ./deepseek_v41 8 dsv41_long/ref.json
san deepseek_v41 "asan deepseek_v41 trunk on the device" COLI_VK_DENSE=1 SNAP=dsv41_tiny ./deepseek_v41 2 dsv41_tiny/ref.json
p=$($PY -c 'import json; c=json.load(open("deepseek_v4_tiny_t/ref.json"))["cases"]["long"]; print("".join("<t%03d>" % t for t in c["prompt_ids"]))')
san deepseek_v4 "asan deepseek_v4" ./deepseek_v4 ./deepseek_v4_tiny_t "$p" --raw-prompt --max-tokens 4 --record-oracle san.json
san deepseek_v4 "asan deepseek_v4 pinned rows16" ./deepseek_v4 ./deepseek_v4_tiny_e8 "$p" --raw-prompt --max-tokens 4 --record-oracle san.json
san deepseek_v4 "asan deepseek_v4 eviction" COLI_VK_TIER_GB=0.00009 ./deepseek_v4 ./deepseek_v4_tiny_e8 "$p" --raw-prompt --max-tokens 4 --record-oracle san.json
san deepseek_v4 "asan deepseek_v4 trunk on the device" COLI_VK_DENSE=1 ./deepseek_v4 ./deepseek_v4_tiny_t "$p" --raw-prompt --max-tokens 4
# the warm start: a run that leaves its history, then one that starts from it
./deepseek_v4 ./deepseek_v4_tiny_e8 "$p" --raw-prompt --max-tokens 4 > /dev/null 2>&1 || true
env ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1 OMP_NUM_THREADS=2 COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 \
./deepseek_v4 ./deepseek_v4_tiny_e8 "$p" --raw-prompt --max-tokens 4 > san.log 2>&1 || true
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "asan deepseek_v4 warm start: sanitizer diagnostic"; fi
grep -q '^\[VK\] tier deepseek_v4: warm start' san.log || { cat san.log; fail "asan deepseek_v4 warm start: no warm start"; }
echo "OK asan deepseek_v4 warm start: sanitizers clean, $(grep -a -o 'device [0-9]* of [0-9]* routed experts' san.log | tail -1)"
rm -rf deepseek_v4_tiny_t deepseek_v4_tiny_e8 san.json
make clean >/dev/null 2>&1 || true
make deepseek-v4-clean >/dev/null 2>&1 || true
}
# Kimi K3: the routed experts on the shared tier (MXFP4 with ue8m0 scales, fmt 7,
# SiTU-GLU in the latent space), against Moonshot's vendor oracle and the CPU run.
# K3_IDOT=0 everywhere: the CPU's default int8-activation expert kernel is an
# approximation the device does not make (the oracle's engine_env sets it too).
family_kimi() {
make kimi_k3 VK=1
$PY tools/make_kimi_k3_tiny.py --output ./kimi_k3_tiny --force
# Moonshot's oracle (greedy, teacher forcing at every position, determinism, the
# bite) with the tier on, the shared experts where the default puts them (the CPU
# on Lavapipe) and on the device
COLI_USAGE=$PWD/k3.usage USAGE_SAVE=0 COLI_VULKAN=1 COLI_VK_TIER_SYNC=1 \
$PY tests/test_kimi_k3_tiny.py --binary ./kimi_k3 --fixture ./kimi_k3_tiny
COLI_USAGE=$PWD/k3.usage USAGE_SAVE=0 COLI_VULKAN=1 COLI_VK_TIER_SYNC=1 COLI_VK_DENSE=1 \
$PY tests/test_kimi_k3_tiny.py --binary ./kimi_k3 --fixture ./kimi_k3_tiny
k3ids() { $PY -c "import json,sys;print(' '.join(map(str,json.load(open('kimi_k3_tiny/ref.json'))['cases'][sys.argv[1]]['prompt_ids'])))" "$1"; }
# k3_gate <tag> <env...>: the tokens of the CPU run, the tier served some experts,
# the shared experts where they were asked to be (dense_where); EVICT=1: it evicted
k3_gate() {
local tag=$1 c; shift
for c in short chunk long; do
rm -f k3.usage
env "$@" COLI_USAGE=$PWD/k3.usage USAGE_SAVE=0 ./kimi_k3 kimi_k3_tiny --ids "$(k3ids $c)" --ngen 8 2>/dev/null | sed 's/ *TUNE.*//' > cpu.tok
env "$@" COLI_USAGE=$PWD/k3.usage USAGE_SAVE=0 COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 \
./kimi_k3 kimi_k3_tiny --ids "$(k3ids $c)" --ngen 8 2> vk.log | sed 's/ *TUNE.*//' > vk.tok
{ [ -s cpu.tok ] && cmp -s cpu.tok vk.tok; } || { cat cpu.tok vk.tok vk.log; fail "$tag $c: Vulkan tokens differ from the CPU"; }
[ "$(tier_count kimi_k3 vk.log)" -gt 0 ] || { cat vk.log; fail "$tag $c: no routed expert ran on the device"; }
if [ "${EVICT:-0}" = 1 ]; then
[ "$(tier_evictions kimi_k3 vk.log)" -gt 0 ] || { grep '\[VK\] tier' vk.log; fail "$tag $c: the budget forced no eviction"; }
grep -q ' failed 0 ' vk.log || { grep '\[VK\] tier' vk.log; fail "$tag $c: an upload failed"; }
fi
local where; where=$(dense_where kimi_k3 vk.log "$tag $c" "$@") || { echo "$where"; exit 1; }
echo "OK $tag $c: tokens = CPU, $(grep -a -o 'device [0-9]* of [0-9]* routed experts' vk.log | tail -1), $(grep -a -o 'evictions [0-9]*' vk.log | tail -1), shared experts on the $where"
done
}
local O="K3_BITS=32 K3_MLA_BITS=32 K3_HEAD_BITS=32 K3_IDOT=0 COLI_TEMP=0" b
k3_gate "kimi_k3 tier f32" $O
k3_gate "kimi_k3 tier f32, shared experts on the device" $O COLI_VK_DENSE=1
for b in 8 4; do # the shared experts as int8 rows (fmt 1) and int4-g64 (fmt 4) on the device
k3_gate "kimi_k3 tier K3_BITS=$b, shared experts on the device" K3_BITS=$b K3_MLA_BITS=$b K3_HEAD_BITS=$b K3_IDOT=0 COLI_TEMP=0 COLI_VK_DENSE=1
done
k3_gate "kimi_k3 tier, prefill one token at a time" $O K3_CHUNK=1
k3_gate "kimi_k3 tier, loads not pipelined" $O K3_PIPE=0
EVICT=1 k3_gate "kimi_k3 tier, a budget of two experts" $O COLI_VK_TIER_GB=0.000005
# the old switches: K3_VK=1 opens the device as COLI_VULKAN=1 does and K3_VK_GB caps
# the tier; K3_VK=0 keeps it closed whatever COLI_VULKAN says
rm -f k3.usage
env $O COLI_USAGE=$PWD/k3.usage USAGE_SAVE=0 ./kimi_k3 kimi_k3_tiny --ids "$(k3ids long)" --ngen 8 2>/dev/null | sed 's/ *TUNE.*//' > cpu.tok
env $O COLI_USAGE=$PWD/k3.usage USAGE_SAVE=0 COLI_VK_TIER_SYNC=1 K3_VK=1 K3_VK_GB=0.000005 \
./kimi_k3 kimi_k3_tiny --ids "$(k3ids long)" --ngen 8 2> vk.log | sed 's/ *TUNE.*//' > vk.tok
cmp -s cpu.tok vk.tok || { cat vk.log; fail "kimi_k3 K3_VK=1: tokens differ from the CPU"; }
grep -q 'K3_VK=1 read as COLI_VULKAN=1' vk.log && grep -q 'budget [0-9.]* KiB = 2 experts' vk.log &&
[ "$(tier_count kimi_k3 vk.log)" -gt 0 ] || { cat vk.log; fail "kimi_k3 K3_VK=1 K3_VK_GB: not the tier it asked for"; }
env $O COLI_USAGE=$PWD/k3.usage USAGE_SAVE=0 K3_VK=0 COLI_VULKAN=1 \
./kimi_k3 kimi_k3_tiny --ids "$(k3ids long)" --ngen 8 2> vk.log | sed 's/ *TUNE.*//' > vk.tok
cmp -s cpu.tok vk.tok && ! grep -q '^\[VK\]' vk.log || { cat vk.log; fail "kimi_k3 K3_VK=0: the device opened"; }
echo "OK kimi_k3 K3_VK=1 / K3_VK_GB / K3_VK=0: the shared tier's switches"
# a warm start from the history of the run before: the tier starts full, serves
# every routed expert of the same prompt, and the tokens stay the CPU's
rm -f k3.usage
env $O COLI_USAGE=$PWD/k3.usage COLI_VULKAN=1 COLI_VK_TIER_SYNC=1 ./kimi_k3 kimi_k3_tiny --ids "$(k3ids long)" --ngen 8 > /dev/null 2>&1
env $O COLI_USAGE=$PWD/k3.usage USAGE_SAVE=0 COLI_VULKAN=1 COLI_VK_TIER_SYNC=1 \
./kimi_k3 kimi_k3_tiny --ids "$(k3ids long)" --ngen 8 2> vk.log | sed 's/ *TUNE.*//' > vk.tok
cmp -s cpu.tok vk.tok || { cat vk.log; fail "kimi_k3 warm start: tokens differ from the CPU"; }
grep -q 'tier kimi_k3: warm start, [1-9]' vk.log || { cat vk.log; fail "kimi_k3: no warm start from the history"; }
echo "OK kimi_k3 warm start: tokens = CPU, $(grep -a -o 'warm start, [0-9]* experts' vk.log), $(grep -a -o 'device [0-9]* of [0-9]* routed experts' vk.log | tail -1)"
# COLI_VK_TIER=0: the shared experts alone on the device, as before the tier
env $O COLI_USAGE=$PWD/k3.usage USAGE_SAVE=0 COLI_VK_TIER=0 COLI_VULKAN=1 \
./kimi_k3 kimi_k3_tiny --ids "$(k3ids long)" --ngen 8 2> vk.log | sed 's/ *TUNE.*//' > vk.tok
cmp -s cpu.tok vk.tok || { cat vk.log; fail "kimi_k3 COLI_VK_TIER=0: tokens differ from the CPU"; }
! grep -q '^\[VK\] tier' vk.log || { cat vk.log; fail "kimi_k3 COLI_VK_TIER=0: the tier started"; }
need_gpu kimi_k3 vk.log "kimi_k3 COLI_VK_TIER=0"
echo "OK kimi_k3 COLI_VK_TIER=0: tokens = CPU, no tier, $(vk_count kimi_k3 vk.log) matmuls on the GPU"
}
# The Kimi K3 and MiMo expert tiers under ASan and UBSan, as qwen-sanitize does for
# the qwen engines: memory safety is the gate, and each run must still put routed
# experts on the device.
family_kimi_mimo_sanitize() {
make clean >/dev/null 2>&1 || true
make kimi_k3 mimo qwenimage VK=1 EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
$PY tools/make_kimi_k3_tiny.py --output ./kimi_k3_tiny --force
$PY tools/make_mimo_tiny.py --output ./mimo_tiny --force --vision
ksan() { # <engine> <tag> <env and argv...>
local eng=$1 tag=$2; shift 2
env ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1 OMP_NUM_THREADS=2 \
COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 "$@" > san.log 2>&1 || true
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "$tag: sanitizer diagnostic"; fi
[ "$(tier_count "$eng" san.log)" -gt 0 ] || { cat san.log; fail "$tag: no routed expert ran on the device"; }
echo "OK $tag: sanitizers clean, $(grep -a -o 'device [0-9]* of [0-9]* routed experts' san.log | tail -1), $(grep -a -o 'evictions [0-9]*' san.log | tail -1)"
}
local O="K3_BITS=32 K3_MLA_BITS=32 K3_HEAD_BITS=32 K3_IDOT=0 COLI_TEMP=0" K3IDS MIDS IMG GRID
K3IDS=$($PY -c "import json;print(' '.join(map(str,json.load(open('kimi_k3_tiny/ref.json'))['cases']['long']['prompt_ids'])))")
MIDS=$($PY -c "import json;print(' '.join(map(str,json.load(open('mimo_tiny/ref.json'))['cases']['long']['prompt_ids'])))")
IMG=$($PY -c "import json;print(' '.join(map(str,json.load(open('mimo_tiny/ref.json'))['image']['prompt_ids'])))")
GRID=$($PY -c "import json;i=json.load(open('mimo_tiny/ref.json'))['image'];print(i['grid_h'],i['grid_w'])")
rm -f k3.usage
ksan kimi_k3 "asan kimi_k3 tier" $O COLI_USAGE=$PWD/k3.usage ./kimi_k3 kimi_k3_tiny --ids "$K3IDS" --ngen 8
ksan kimi_k3 "asan kimi_k3 warm start, shared experts on the device" $O COLI_USAGE=$PWD/k3.usage COLI_VK_DENSE=1 ./kimi_k3 kimi_k3_tiny --ids "$K3IDS" --ngen 8
ksan kimi_k3 "asan kimi_k3 eviction, int8 shared experts" K3_BITS=8 K3_IDOT=0 COLI_USAGE=$PWD/k3e.usage USAGE_SAVE=0 COLI_VK_TIER_GB=0.000005 COLI_VK_DENSE=1 ./kimi_k3 kimi_k3_tiny --ids "$K3IDS" --ngen 8
ksan kimi_k3 "asan kimi_k3 prefill one token at a time, no pipeline" $O COLI_USAGE=$PWD/k3e.usage USAGE_SAVE=0 K3_CHUNK=1 K3_PIPE=0 ./kimi_k3 kimi_k3_tiny --ids "$K3IDS" --ngen 8
ksan mimo "asan mimo tier" MIMO_DENSE_BITS=32 COLI_TEMP=0 ./mimo mimo_tiny --ids "$MIDS" --ngen 6
ksan mimo "asan mimo eviction (MIMO_VK_EXPERTS=2)" MIMO_DENSE_BITS=32 COLI_TEMP=0 MIMO_VK_EXPERTS=2 ./mimo mimo_tiny --ids "$MIDS" --ngen 6
ksan mimo "asan mimo picture, native dense on the device" MIMO_DENSE_BITS=0 COLI_TEMP=0 COLI_VK_DENSE=1 ./mimo mimo_tiny --ids "$IMG" --ngen 6 --image mimo_tiny/patches.f32 --grid $GRID
ksan mimo "asan mimo prefill blocks of 3, cache 4" MIMO_DENSE_BITS=32 COLI_TEMP=0 MIMO_CHUNK=3 MIMO_CAP=4 ./mimo mimo_tiny --ids "$MIDS" --ngen 6
# qwenimage's DiT on the chain: the oracle, then the attention in slices
$PY tools/make_qwenimage_tiny.py qwenimage_tiny
local qa
for qa in COLI_VK_ATTN_SLICE=0 COLI_VK_ATTN_SLICE=60000 COLI_VK_QI_RESIDENT=0; do
env ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1 OMP_NUM_THREADS=2 \
COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_IMG_BITS=32 COLI_IMG_ACT8=0 $qa ./qwenimage --model qwenimage_tiny --ref qwenimage_tiny/ref \
> san.log 2>&1 || { cat san.log; fail "asan qwenimage chain $qa: the oracle"; }
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "asan qwenimage chain $qa: sanitizer diagnostic"; fi
grep -qa 'qwenimage chain: [0-9]* steps' san.log || { cat san.log; fail "asan qwenimage chain $qa: the chain never ran"; }
echo "OK asan qwenimage chain $qa: sanitizers clean, the oracle within tolerance"
done
make clean >/dev/null 2>&1 || true
}
# MiMo-V2.6's dense chain (mimo_chain.h, COLI_VK_CHAIN=1): every layer's attention with
# its sliding window (a ring of 8 rows on the fixture, which every case outgrows), the
# sink logits, partial RoPE with two thetas and the value scale on the device, the
# routed experts on the host. Gates, per configuration (mimo_chain_gate): the CPU run's
# generated tokens; the teacher-forced logits of every position (MIMO_LOGITS) within
# 1e-4 of the largest one; a "[VK] mimo chain: N forwards" line with N > 0.
mimo_ids() { # <case|image> <field>: a field of ref.json's case
$PY -c "import json,sys;r=json.load(open('mimo_tiny/ref.json'));c=r['image'] if sys.argv[1]=='image' else r['cases'][sys.argv[1]];print(' '.join(map(str,c[sys.argv[2]])))" "$1" "$2"
}
mimo_chain_gate() { # <tag> <case> <env...> (CHAINMODE: COLI_VK_CHAIN's value, default 1)
local tag=$1 c=$2 extra=() fw lg; shift 2
[ "$c" = image ] && extra=(--image mimo_tiny/patches.f32 --grid $MGRID)
env "$@" COLI_TEMP=0 ./mimo mimo_tiny --ids "$(mimo_ids $c prompt_ids)" --ngen 6 "${extra[@]}" > mimo-cpu.txt 2>/dev/null
env "$@" COLI_TEMP=0 COLI_VULKAN=1 COLI_VK_CHAIN=${CHAINMODE:-1} COLI_VK_TIER_SYNC=1 \
./mimo mimo_tiny --ids "$(mimo_ids $c prompt_ids)" --ngen 6 "${extra[@]}" > mimo-vk.txt 2> mimo-vk.err
[ -s mimo-cpu.txt ] && cmp -s mimo-cpu.txt mimo-vk.txt || { cat mimo-cpu.txt mimo-vk.txt mimo-vk.err; fail "$tag $c: the chain's tokens differ from the CPU"; }
fw=$(chain_count mimo mimo-vk.err)
[ "$fw" -gt 0 ] || { cat mimo-vk.err; fail "$tag $c: the chain never ran"; }
rm -f cpu.f32 vk.f32
env "$@" MIMO_LOGITS=cpu.f32 ./mimo mimo_tiny --ids "$(mimo_ids $c greedy_full_ids)" --ngen 0 "${extra[@]}" > /dev/null 2>&1
env "$@" MIMO_LOGITS=vk.f32 COLI_VULKAN=1 COLI_VK_CHAIN=${CHAINMODE:-1} COLI_VK_TIER_SYNC=1 \
./mimo mimo_tiny --ids "$(mimo_ids $c greedy_full_ids)" --ngen 0 "${extra[@]}" > /dev/null 2> mimo-vk.err
lg=$(logits_close cpu.f32 vk.f32) || { echo "$lg"; fail "$tag $c: logits"; }
echo "OK $tag $c: tokens = CPU, $lg, $fw chain forwards"
}
# mimo_lost_gate <tag> <case> <back> <env...>: the device "lost" (COLI_VK_CHAIN_FAULT) at
# the chain frame <back> frames before the end of the same run without a fault (the
# frames a device spends setting up differ: on Lavapipe its memory needs no zeroing
# frames). The host's caches hold whole steps only, so the CPU runs the rest of the
# step from where they end, with nothing to rebuild (a picture included): the CPU
# run's tokens all the same.
mimo_lost_gate() {
local tag=$1 c=$2 back=$3 extra=() frames; shift 3
[ "$c" = image ] && extra=(--image mimo_tiny/patches.f32 --grid $MGRID)
env "$@" COLI_TEMP=0 ./mimo mimo_tiny --ids "$(mimo_ids $c prompt_ids)" --ngen 6 "${extra[@]}" > mimo-cpu.txt 2>/dev/null
env "$@" COLI_TEMP=0 COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 \
./mimo mimo_tiny --ids "$(mimo_ids $c prompt_ids)" --ngen 6 "${extra[@]}" > /dev/null 2> mimo-vk.err
frames=$(sed -n 's/^\[VK\] mimo chain: [0-9]* forwards, \([0-9]*\) frames.*/\1/p' mimo-vk.err | tail -1)
[ "${frames:-0}" -gt "$back" ] || { cat mimo-vk.err; fail "$tag: the run had no frame $back before its end"; }
env "$@" COLI_TEMP=0 COLI_VK_CHAIN_FAULT=$((frames - back)) COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 \
./mimo mimo_tiny --ids "$(mimo_ids $c prompt_ids)" --ngen 6 "${extra[@]}" > mimo-vk.txt 2> mimo-vk.err
[ -s mimo-cpu.txt ] && cmp -s mimo-cpu.txt mimo-vk.txt || { cat mimo-cpu.txt mimo-vk.txt mimo-vk.err; fail "$tag: tokens differ from the CPU"; }
grep -q "the device was lost at position [0-9]" mimo-vk.err || { cat mimo-vk.err; fail "$tag: the device was never lost"; }
echo "OK $tag: tokens = CPU, $(grep -o 'the device was lost at position [0-9]*' mimo-vk.err)"
}
mimo_served_fixture() { # the fixture plus a byte tokenizer, for the serve tests
rm -rf mimo_tiny_served
$PY -c "import sys, shutil; sys.path.insert(0, 'tests'); import mimo_serve_fixture as f; shutil.copytree(f.served_fixture(), 'mimo_tiny_served')"
}
family_mimo_chain() {
make mimo tests/test_vk_chain VK=1
./tests/test_vk_chain shaders/qmatmul.spv | tee vk_chain.log
tail -1 vk_chain.log | grep -qx PASS || fail "the chain's ops"
$PY tools/make_mimo_tiny.py --output ./mimo_tiny --force --vision
MGRID=$($PY -c "import json;i=json.load(open('mimo_tiny/ref.json'))['image'];print(i['grid_h'],i['grid_w'])")
# Xiaomi's vendor oracle with the chain on: greedy and teacher-forced in every case,
# the native FP8/BF16 trunk, prefill in blocks of 3 and of 1, the picture
COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 $PY tests/mimo_tiny_harness.py --binary ./mimo --fixture ./mimo_tiny
# every dense form (MIMO_DENSE_BITS 32: f32, fmt 10; 0: the release's FP8 and BF16, fmt
# 12 and 11; 8: int8 rows, fmt 1), the prompt in one block (the tiled GEMM, the window
# gathered beside the block) and one token at a time (the ring read in place), text
# and picture, the routed experts on the tier
local bits ch c k
for bits in 32 0 8; do for ch in 64 1; do for c in short window long image; do
mimo_chain_gate "chain mimo bits=$bits MIMO_CHUNK=$ch" $c MIMO_DENSE_BITS=$bits MIMO_CHUNK=$ch
done; done; done
for c in short long; do
mimo_chain_gate "chain mimo blocks of 3" $c MIMO_DENSE_BITS=32 MIMO_CHUNK=3
mimo_chain_gate "chain mimo tier off" $c MIMO_DENSE_BITS=0 COLI_VK_TIER=0
mimo_chain_gate "chain mimo experts on the CPU (MIMO_VK_EXPERTS=0)" $c MIMO_DENSE_BITS=32 MIMO_VK_EXPERTS=0
mimo_chain_gate "chain mimo a block in chunks of 3 rows" $c MIMO_DENSE_BITS=32 COLI_VK_CHAIN_ROWS=3
mimo_chain_gate "chain mimo tiled GEMM from S=2" $c MIMO_DENSE_BITS=0 COLI_VK_GEMM_MIN_S=2
mimo_chain_gate "chain mimo the per-row GEMV" $c MIMO_DENSE_BITS=32 COLI_VK_CHAIN_GEMV=0
mimo_chain_gate "chain mimo eviction (MIMO_VK_EXPERTS=2)" $c MIMO_DENSE_BITS=32 MIMO_VK_EXPERTS=2
done
# the picture's tower on the device too (COLI_VK_DENSE=1: the per-matrix path for the
# tower, the chain for the layers)
mimo_chain_gate "chain mimo tower on the device" image MIMO_DENSE_BITS=0 COLI_VK_DENSE=1
# The window boundary, bit for bit: with every matrix on the per-row GEMV and the tier
# off the chain's logits are the same bytes whatever the blocks -- one row reading the
# ring in place, blocks of 3, of the window's 8 and of 9 rows gathering it, the whole
# prompt -- so a row's window is the same rows in the same order on every path.
# Keep the per-row attention too: blocked attention has its own reduction order;
# the default blocked path's numerical gates above remain enabled.
for c in window long image; do
local x=(); [ $c = image ] && x=(--image mimo_tiny/patches.f32 --grid $MGRID)
for ch in 64 1 3 8 9; do
COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER=0 COLI_VK_GEMM_MIN_S=0 COLI_VK_ATTN_BLOCK=0 MIMO_DENSE_BITS=32 MIMO_CHUNK=$ch MIMO_LOGITS=blk$ch.f32 \
./mimo mimo_tiny --ids "$(mimo_ids $c greedy_full_ids)" --ngen 0 "${x[@]}" > /dev/null 2> mimo-vk.err
[ "$(chain_count mimo mimo-vk.err)" -gt 0 ] || { cat mimo-vk.err; fail "chain mimo blocks $ch $c: the chain never ran"; }
[ $ch = 64 ] || cmp -s blk64.f32 blk$ch.f32 || fail "chain mimo $c: blocks of $ch are not the bytes of one block"
done
echo "OK chain mimo window boundary $c: blocks of 64, 1, 3, 8, 9 give the same logits bytes"
done
# COLI_VK_CHAIN=2: prompts on the device, decode on the CPU, the caches moving between them
for c in short long image; do CHAINMODE=2 mimo_chain_gate "chain mimo prompts only" $c MIMO_DENSE_BITS=32; done
# the device lost (frames counted back from the end of the run: 5 decode steps of 6
# frames follow the prompt): inside the last decode step; inside the first block of
# a prompt split in chunks of 3 rows (the CPU takes the step's remaining rows); inside
# a picture's prompt; inside a prompt fed one token at a time
mimo_lost_gate "chain mimo device lost mid-decode" long 3 MIMO_DENSE_BITS=32
mimo_lost_gate "chain mimo device lost inside a chunked prompt" long 100 MIMO_DENSE_BITS=32 COLI_VK_CHAIN_ROWS=3
mimo_lost_gate "chain mimo device lost inside a picture's prompt" image 33 MIMO_DENSE_BITS=0
mimo_lost_gate "chain mimo device lost, one token at a time" short 50 MIMO_DENSE_BITS=32 MIMO_CHUNK=1
# the prefix-reuse contract and Brio photos (the rings restored into the device's
# mirrors), compared bit for bit with a cold engine: the per-row GEMV and no tier, so
# a resumed prompt's rows get the bits of the same rows computed cold
COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER=0 COLI_VK_GEMM_MIN_S=0 COLI_VK_ATTN_BLOCK=0 $PY -m unittest tests.test_mimo_prefix_serve
# a serve session against the CPU's, frame for frame, in both modes and with the
# device lost mid-session. The fixture's logits run to 300 (its head is scaled for wide
# greedy margins), so its logprobs compare within 1e-3: about 3e-6 of the largest logit
mimo_served_fixture
for k in 1 2; do CHAIN_SERVE_TOL=1e-3 COLI_VK_CHAIN=$k $PY tests/vulkan_chain_serve.py ./mimo mimo_tiny_served OMP_NUM_THREADS=2; done
CHAIN_SERVE_TOL=1e-3 $PY tests/vulkan_chain_serve.py ./mimo mimo_tiny_served OMP_NUM_THREADS=2 COLI_VK_CHAIN_FAULT=60
}
# MiMo's chain under ASan and UBSan: memory safety is the gate (a sanitized build
# vectorizes differently, so tokens are not compared); each run must have run the chain.
family_mimo_chain_sanitize() {
make clean >/dev/null 2>&1 || true
make mimo tests/test_vk_chain VK=1 EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1
./tests/test_vk_chain shaders/qmatmul.spv > san.log 2>&1 || true
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log || ! tail -1 san.log | grep -qx PASS; then cat san.log; fail "asan: the chain's ops"; fi
echo "OK asan: the chain's ops"
$PY tools/make_mimo_tiny.py --output ./mimo_tiny --force --vision
MGRID=$($PY -c "import json;i=json.load(open('mimo_tiny/ref.json'))['image'];print(i['grid_h'],i['grid_w'])")
msan() { # <tag> <case> <env...>
local tag=$1 c=$2 x=(); shift 2
[ "$c" = image ] && x=(--image mimo_tiny/patches.f32 --grid $MGRID)
env OMP_NUM_THREADS=2 COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_TEMP=0 "$@" \
./mimo mimo_tiny --ids "$(mimo_ids $c prompt_ids)" --ngen 6 "${x[@]}" > san.log 2>&1 || true
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "$tag: sanitizer diagnostic"; fi
[ "$(chain_count mimo san.log)" -gt 0 ] || { cat san.log; fail "$tag: the chain never ran"; }
echo "OK $tag: sanitizers clean, $(chain_count mimo san.log) chain forwards"
}
msan "asan chain mimo f32, the tier" long MIMO_DENSE_BITS=32
msan "asan chain mimo native FP8/BF16, picture, tower on the device" image MIMO_DENSE_BITS=0 COLI_VK_DENSE=1
msan "asan chain mimo blocks of 3 in chunks of 2 rows" long MIMO_DENSE_BITS=32 MIMO_CHUNK=3 COLI_VK_CHAIN_ROWS=2
msan "asan chain mimo one token at a time, int8" window MIMO_DENSE_BITS=8 MIMO_CHUNK=1
msan "asan chain mimo experts on the CPU" short MIMO_DENSE_BITS=32 MIMO_VK_EXPERTS=0
# the device lost inside a prompt split in chunks of 3 rows: the frame 100 before the
# end of the same run without a fault (a device's setup frames differ)
msan "asan chain mimo frames to the end" long MIMO_DENSE_BITS=32 COLI_VK_CHAIN_ROWS=3
local frames
frames=$(sed -n 's/^\[VK\] mimo chain: [0-9]* forwards, \([0-9]*\) frames.*/\1/p' san.log | tail -1)
env OMP_NUM_THREADS=2 COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_TEMP=0 MIMO_DENSE_BITS=32 COLI_VK_CHAIN_ROWS=3 \
COLI_VK_CHAIN_FAULT=$((frames - 100)) ./mimo mimo_tiny --ids "$(mimo_ids long prompt_ids)" --ngen 6 > san.log 2>&1 || true
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "asan chain mimo device lost: sanitizer diagnostic"; fi
grep -q "the device was lost at position [1-9]" san.log || { cat san.log; fail "asan chain mimo device lost: not lost inside the prompt"; }
echo "OK asan chain mimo device lost: sanitizers clean, $(grep -o 'the device was lost at position [0-9]*' san.log)"
mimo_served_fixture
CHAIN_SERVE_TOL=1e-3 $PY tests/vulkan_chain_serve.py ./mimo mimo_tiny_served OMP_NUM_THREADS=2 > san.log 2>&1 || { cat san.log; fail "asan chain mimo serve"; }
echo "OK asan chain mimo serve: $(tail -1 san.log)"
make clean >/dev/null 2>&1 || true
}
# GLM-5.2 (colibri) and GLM-5.3 Flash (glm53) on the routed-expert tier: the fixtures.
# colibri: the bf16 oracle, whose experts the loader quantizes to the bits asked for
# (16: f32, fmt 10 on the device; 8: int8, fmt 1; 4: int4 per row, fmt 2; 3: int3-g64,
# fmt 5), the int4-g64 and E8/IQ3 containers of the parity gate, and the FP8 oracle
# converted to int4-g64 (its 32-wide down rows stay per row), int4 per row, int3-g64 and
# int4-g64 with an int3-g64 down. glm53: the int4-gs64 streaming container.
glm_fixtures() {
$PY tools/make_glm_oracle.py > /dev/null
$PY tools/make_glm_oracle.py --fmt4 > /dev/null
$PY tools/make_glm_oracle.py --fmt6 > /dev/null
mkdir -p glm_fp8 && (cd glm_fp8 && $PY ../tools/make_glm_oracle.py --fp8 > /dev/null)
local v
for v in "i4:" "i4r:--group-size 0" "i3:--xbits 3" "d3:--down-bits 3"; do
# shellcheck disable=SC2086
$PY tools/convert_fp8_to_int4.py --indir glm_fp8/glm_tiny --outdir glm_tiny_${v%%:*} \
--ebits 4 --io-bits 4 --n-layers 5 --min-free-gb 0 ${v#*:} > /dev/null
cp glm_fp8/ref_glm.json glm_tiny_${v%%:*}/
done
$PY tools/make_glm53_tiny.py --output glm53_tiny --force > /dev/null
$PY tools/make_glm53_streaming_pair.py --fixture glm53_tiny --output glm53_stream > /dev/null
}
# glm_tier <tag> <snap> <ref> <env...> -- <argv...>
# colibri with the tier against its CPU run: the same greedy tokens (or teacher-forced
# predictions, TF=1, mismatches included) and a "[VK] tier colibri run: device N" with
# N > 0; EVICT=1: evictions too. The history the tier warms from is <snap>/.coli_usage:
# none by default (the tier fills as experts pass by), HIST=1 writes one first from a
# greedy run (the warm start, and the device in a teacher-forced prefill). DEV2=1: the
# second device's registry served experts ("+ N vk" in the hit-rate line).
glm_tier() {
local tag=$1 snap=$2 ref=$3; shift 3
local envs=() pre=() e; while [ "$1" != "--" ]; do envs+=("$1"); shift; done; shift
for e in "${envs[@]}"; do [ "${e%%=*}" = TF ] || pre+=("$e"); done
rm -f "$snap/.coli_usage"
[ "${HIST:-0}" = 1 ] && { env "${pre[@]}" SNAP=$snap REF=$ref STATS=$snap/.coli_usage ./colibri "$@" > /dev/null 2>&1 || true; }
env "${envs[@]}" SNAP=$snap REF=$ref ./colibri "$@" > cpu.log 2>&1 || true
env "${envs[@]}" SNAP=$snap REF=$ref COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 ./colibri "$@" > vk.log 2>&1 || true
rm -f "$snap/.coli_usage"
grep -aE '^GLM C engine|^PREFILL|^\[ORACLE\] mismatch' cpu.log | sed 's/ | [0-9.]* pos\/s//' > cpu.tok || true
grep -aE '^GLM C engine|^PREFILL|^\[ORACLE\] mismatch' vk.log | sed 's/ | [0-9.]* pos\/s//' > vk.tok || true
{ [ -s cpu.tok ] && cmp -s cpu.tok vk.tok; } || { cat cpu.tok vk.tok; tail -20 vk.log; fail "$tag: the tier's tokens differ from the CPU"; }
if [ "${DEV2:-0}" = 1 ]; then
grep -qaE 'lru \+ [1-9][0-9]* vk /' vk.log || { cat vk.log; fail "$tag: the second device served no expert"; }
else
[ "$(tier_count colibri vk.log)" -gt 0 ] || { cat vk.log; fail "$tag: no routed expert ran on the device"; }
fi
if [ "${EVICT:-0}" = 1 ]; then
[ "$(tier_evictions colibri vk.log)" -gt 0 ] || { grep '\[VK\] tier' vk.log; fail "$tag: the budget forced no eviction"; }
[ "$(tier_failed colibri vk.log)" = 0 ] || { grep '\[VK\] tier' vk.log; fail "$tag: an upload failed (the budget shrank)"; }
fi
echo "OK $tag: tokens = CPU, $(grep -a -o 'device [0-9]* of [0-9]* routed experts' vk.log | tail -1)$(grep -a -o ' + [0-9]* vk' vk.log | tail -1), $(grep -a -o 'evictions [0-9]*' vk.log | tail -1), $(grep -a -o '(fmt [^)]*)' vk.log | head -1)"
}
# g53_tier <tag> <model dir> <env...> -- <argv...>: glm53, the same gate on its
# teacher_forcing and greedy lines; the history is a fresh COLI_USAGE file (HIST=1: one
# greedy run writes it first).
g53_tier() {
local tag=$1 model=$2; shift 2
local envs=(); while [ "$1" != "--" ]; do envs+=("$1"); shift; done; shift
rm -f g53.usage
[ "${HIST:-0}" = 1 ] && { env "${envs[@]}" COLI_USAGE=g53.usage ./glm53 --model $model "$@" > /dev/null 2>&1 || true; }
env "${envs[@]}" COLI_USAGE=g53.usage USAGE_SAVE=0 ./glm53 --model $model "$@" > cpu.log 2>&1 || true
env "${envs[@]}" COLI_USAGE=g53.usage USAGE_SAVE=0 COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 ./glm53 --model $model "$@" > vk.log 2>&1 || true
rm -f g53.usage
grep -aE '^teacher_forcing|^greedy' cpu.log > cpu.tok || true
grep -aE '^teacher_forcing|^greedy' vk.log > vk.tok || true
{ [ -s cpu.tok ] && cmp -s cpu.tok vk.tok; } || { cat cpu.tok vk.tok; tail -20 vk.log; fail "$tag: the tier's tokens differ from the CPU"; }
[ "$(tier_count glm53 vk.log)" -gt 0 ] || { cat vk.log; fail "$tag: no routed expert ran on the device"; }
if [ "${EVICT:-0}" = 1 ]; then
[ "$(tier_evictions glm53 vk.log)" -gt 0 ] || { grep '\[VK\] tier' vk.log; fail "$tag: the budget forced no eviction"; }
[ "$(tier_failed glm53 vk.log)" = 0 ] || { grep '\[VK\] tier' vk.log; fail "$tag: an upload failed (the budget shrank)"; }
fi
local where; where=$(dense_where glm53 vk.log "$tag" "${envs[@]}") || { echo "$where"; exit 1; }
echo "OK $tag: tokens = CPU, $(grep -a -o 'device [0-9]* of [0-9]* routed experts' vk.log | tail -1), $(grep -a -o 'evictions [0-9]*' vk.log | tail -1), trunk on the $where"
}
# GLM-5.2 and GLM-5.3 Flash on the routed-expert tier, in every expert format the two
# engines read, warm and cold, decode and teacher-forced prefill, under eviction (a
# budget of two experts, by COLI_VK_TIER_GB and by the deprecated COLI_VK_EXPERTS), with
# the trunk and the attention core on the device, beside COLI_VK_DEV2's second
# (logical) device, and with the tier off. IDOT=0 gives the CPU's int8 and int4-per-row
# experts the f32 activations the device uses (the CPU default rounds them to int8 on
# some ISAs), so the two sides differ by summation order only.
family_glm() {
make colibri glm53 VK=1
glm_fixtures
export OMP_NUM_THREADS=2 CAP_RAISE=0
local cap b
# the oracle with f32 experts (fmt 10): decode cold at a one-slot and a full cache,
# the teacher-forced prefill from a warm start
for cap in 1 64; do glm_tier "colibri f32 experts cap=$cap" glm_tiny ref_glm.json -- $cap 16 16; done
HIST=1 glm_tier "colibri f32 experts, prefill from a warm start" glm_tiny ref_glm.json TF=1 -- 64 16 16
# quantized at load: int8 (fmt 1), int4 per row (fmt 2), int3-g64 (fmt 5)
for b in 8 4 3; do
glm_tier "colibri ${b}-bit experts cap=2" glm_tiny ref_glm.json IDOT=0 -- 2 $b 4
HIST=1 glm_tier "colibri ${b}-bit experts, prefill warm" glm_tiny ref_glm.json IDOT=0 TF=1 -- 2 $b 4
done
# the containers: int4-g64 gate/up/down, int4-g64 with a per-row down, int4 per row,
# int3-g64, int4-g64 with an int3-g64 down
HIST=1 glm_tier "colibri int4-g64 container" glm_tiny_fmt4 glm_tiny_fmt4/ref_glm.json -- 2 16 16
HIST=1 glm_tier "colibri int4-g64 container, prefill" glm_tiny_fmt4 glm_tiny_fmt4/ref_glm.json TF=1 -- 2 16 16
local fx
for fx in i4 i4r i3 d3; do
HIST=1 glm_tier "colibri $fx container cap=1" glm_tiny_$fx glm_tiny_$fx/ref_glm.json IDOT=0 -- 1 4 4
HIST=1 glm_tier "colibri $fx container, prefill" glm_tiny_$fx glm_tiny_$fx/ref_glm.json IDOT=0 TF=1 -- 2 4 4
done
# PIPE's asynchronous loads and the PILOT prefetcher beside the tier
glm_tier "colibri PIPE=1" glm_tiny_i4 glm_tiny_i4/ref_glm.json IDOT=0 PIPE=1 -- 1 4 4
glm_tier "colibri PILOT=1" glm_tiny_i4 glm_tiny_i4/ref_glm.json IDOT=0 PILOT=1 -- 2 4 4
# the trunk and the MLA attention core on the device too
HIST=1 glm_tier "colibri tier + COLI_VK_DENSE=1 COLI_VK_ATTN=1" glm_tiny_i4 glm_tiny_i4/ref_glm.json IDOT=0 COLI_VK_DENSE=1 COLI_VK_ATTN=1 -- 2 4 4
# a budget of two experts must evict as the routing moves: COLI_VK_TIER_GB, and the
# deprecated COLI_VK_EXPERTS mapped onto the tier as a cap
EVICT=1 glm_tier "colibri, a budget of two experts" glm_tiny ref_glm.json COLI_VK_TIER_GB=0.0001 -- 64 16 16
EVICT=1 glm_tier "colibri, COLI_VK_EXPERTS=2" glm_tiny ref_glm.json COLI_VK_EXPERTS=2 -- 64 16 16
# COLI_VK_DEV2: a second logical device on Lavapipe holds what the capped tier does not
HIST=1 glm_tier "colibri tier + COLI_VK_DEV2" glm_tiny_i4r glm_tiny_i4r/ref_glm.json IDOT=0 COLI_VK_EXPERTS=4 COLI_VK_DEV2=0 -- 64 4 4
HIST=1 DEV2=1 glm_tier "colibri COLI_VK_DEV2 alone (COLI_VK_TIER=0)" glm_tiny_i4r glm_tiny_i4r/ref_glm.json IDOT=0 COLI_VK_TIER=0 COLI_VK_DEV2=0 -- 64 4 4
# E8/IQ3 (fmt 6) has no device form: the tier declines, the CPU computes them
SNAP=glm_tiny_fmt6 REF=glm_tiny_fmt6/ref_glm.json ./colibri 2 16 16 > cpu.log 2>&1 || true
SNAP=glm_tiny_fmt6 REF=glm_tiny_fmt6/ref_glm.json COLI_VULKAN=1 ./colibri 2 16 16 > vk.log 2>&1 || true
cmp -s <(grep -a '^GLM C engine' cpu.log) <(grep -a '^GLM C engine' vk.log) || { cat vk.log; fail "colibri fmt 6: tokens differ"; }
grep -qa 'tier colibri: experts in fmt 6/6/6' vk.log || { cat vk.log; fail "colibri fmt 6: the tier did not decline"; }
echo "OK colibri E8/IQ3 (fmt 6): the tier declines, tokens = CPU"
# COLI_VK_TIER=0: no tier, the trunk and the attention core on the device as before
SNAP=glm_tiny REF=ref_glm.json COLI_VK_DENSE=1 COLI_VK_ATTN=1 ./colibri 2 16 16 > cpu.log 2>&1 || true
SNAP=glm_tiny REF=ref_glm.json COLI_VK_DENSE=1 COLI_VK_ATTN=1 COLI_VK_TIER=0 COLI_VULKAN=1 ./colibri 2 16 16 > vk.log 2>&1 || true
cmp -s <(grep -a '^GLM C engine' cpu.log) <(grep -a '^GLM C engine' vk.log) || { cat vk.log; fail "colibri COLI_VK_TIER=0: tokens differ"; }
! grep -qa '^\[VK\] tier colibri' vk.log || { cat vk.log; fail "colibri COLI_VK_TIER=0: the tier started"; }
echo "OK colibri COLI_VK_TIER=0: tokens = CPU, no tier"
# the MTP head's layer as the tier's extra layer (COLI_VK_TIER_MTP=1, the default on a
# discrete GPU): its experts f32 and quantized at load, two drafts; Lavapipe shares the
# RAM, where the default keeps them on the CPU
$PY tools/make_glm_mtp_tiny.py --src glm_tiny --out glm_tiny_mtp > /dev/null && cp ref_glm.json glm_tiny_mtp/
glm_tier "colibri MTP head on the tier, f32" glm_tiny_mtp glm_tiny_mtp/ref_glm.json DRAFT=2 COLI_VK_TIER_MTP=1 -- 64 16 16
mtp_on_device "colibri MTP head on the tier, f32"
glm_tier "colibri MTP head on the tier, 4-bit" glm_tiny_mtp glm_tiny_mtp/ref_glm.json DRAFT=2 IDOT=0 COLI_VK_TIER_MTP=1 -- 2 4 4
mtp_on_device "colibri MTP head on the tier, 4-bit"
glm_tier "colibri MTP head, shared RAM default" glm_tiny_mtp glm_tiny_mtp/ref_glm.json DRAFT=2 -- 64 16 16
! grep -aq 'extra layers' vk.log || { grep -a '\[VK\] tier' vk.log; fail "colibri: the MTP head's layer went on the tier by default on shared RAM"; }
echo "OK colibri shared RAM: the MTP head's experts on the CPU by default"
# glm53: its int4-gs64 streaming container (swiglu_limit 10), dense matrices f32 and int4
local ids
ids=$($PY -c "print(','.join(str((i*37+5)%120+2) for i in range(100)))")
g53_tier "glm53 decode, cold" glm53_stream-i4 GLM53_BITS=32 -- --ids 5,7,9,11,13,17,19,23 --greedy 8
HIST=1 g53_tier "glm53 100-token prompt, warm (two blocks of rows)" glm53_stream-i4 GLM53_BITS=32 -- --ids $ids --greedy 4
HIST=1 g53_tier "glm53 prefill chunks of 7, one cache slot" glm53_stream-i4 GLM53_BITS=4 GLM53_PREFILL_CHUNK=7 GLM53_EXPERT_GB=0.000001 -- --ids $ids --greedy 4
HIST=1 g53_tier "glm53 trunk on the device (COLI_VK_DENSE=1)" glm53_stream-i4 GLM53_BITS=4 COLI_VK_DENSE=1 -- --ids $ids --greedy 4
EVICT=1 g53_tier "glm53, a budget of two experts" glm53_stream-i4 GLM53_BITS=32 COLI_VK_TIER_GB=0.00006 -- --ids $ids --greedy 6
# COLI_VK_TIER=0: the trunk alone, on the device by default, Lavapipe included
GLM53_BITS=4 COLI_USAGE=g53.usage USAGE_SAVE=0 ./glm53 --model glm53_stream-i4 --ids 5,7,9,11 --greedy 4 > cpu.log 2>&1 || true
GLM53_BITS=4 COLI_USAGE=g53.usage USAGE_SAVE=0 COLI_VK_TIER=0 COLI_VULKAN=1 ./glm53 --model glm53_stream-i4 --ids 5,7,9,11 --greedy 4 > vk.log 2>&1 || true
cmp -s <(grep -aE '^teacher_forcing|^greedy' cpu.log) <(grep -aE '^teacher_forcing|^greedy' vk.log) || { cat vk.log; fail "glm53 COLI_VK_TIER=0: tokens differ"; }
! grep -qa '^\[VK\] tier glm53' vk.log || { cat vk.log; fail "glm53 COLI_VK_TIER=0: the tier started"; }
need_gpu glm53 vk.log "glm53 COLI_VK_TIER=0"
echo "OK glm53 COLI_VK_TIER=0: tokens = CPU, no tier, $(vk_count glm53 vk.log) matmuls on the GPU"
unset OMP_NUM_THREADS CAP_RAISE
}
# The same engines under ASan and UBSan with the tier on: memory safety is the gate, and
# each run must still put experts on the device.
family_glm_sanitize() {
make clean >/dev/null 2>&1 || true
make colibri glm53 VK=1 EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
glm_fixtures
export OMP_NUM_THREADS=2 CAP_RAISE=0
gsan() { # <engine> <tag> <history: colibri snap, or - for glm53's COLI_USAGE> <env and argv...>
local eng=$1 tag=$2 snap=$3; shift 3
rm -f g53.usage; [ "$snap" = - ] || rm -f "$snap/.coli_usage"
if [ "${HIST:-0}" = 1 ]; then # a history from a greedy run (TF=1 writes none)
local a=() x; for x in "$@"; do [ "$x" = TF=1 ] || a+=("$x"); done
if [ "$snap" = - ]; then env COLI_USAGE=g53.usage "${a[@]}" > /dev/null 2>&1 || true
else env STATS=$snap/.coli_usage "${a[@]}" > /dev/null 2>&1 || true; fi
fi
env ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1 COLI_USAGE=g53.usage \
COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 "$@" > san.log 2>&1 || true
rm -f g53.usage; [ "$snap" = - ] || rm -f "$snap/.coli_usage"
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "$tag: sanitizer diagnostic"; fi
[ "$(tier_count $eng san.log)" -gt 0 ] || { cat san.log; fail "$tag: no routed expert ran on the device"; }
echo "OK $tag: sanitizers clean, $(grep -a -o 'device [0-9]* of [0-9]* routed experts' san.log | tail -1), $(grep -a -o 'evictions [0-9]*' san.log | tail -1)"
}
gsan colibri "asan colibri f32 cap=1" glm_tiny SNAP=glm_tiny REF=ref_glm.json ./colibri 1 16 16
HIST=1 gsan colibri "asan colibri prefill, warm" glm_tiny SNAP=glm_tiny REF=ref_glm.json TF=1 ./colibri 64 16 16
gsan colibri "asan colibri eviction" glm_tiny SNAP=glm_tiny REF=ref_glm.json COLI_VK_EXPERTS=2 ./colibri 64 16 16
HIST=1 gsan colibri "asan colibri int4-g64, trunk + attention on the device" glm_tiny_i4 SNAP=glm_tiny_i4 REF=glm_tiny_i4/ref_glm.json COLI_VK_DENSE=1 COLI_VK_ATTN=1 ./colibri 2 4 4
HIST=1 gsan colibri "asan colibri int3-g64 prefill" glm_tiny_i3 SNAP=glm_tiny_i3 REF=glm_tiny_i3/ref_glm.json TF=1 ./colibri 2 4 4
HIST=1 gsan colibri "asan colibri int3-g64 down, cap=1" glm_tiny_d3 SNAP=glm_tiny_d3 REF=glm_tiny_d3/ref_glm.json ./colibri 1 4 4
HIST=1 gsan colibri "asan colibri tier + COLI_VK_DEV2" glm_tiny_i4r SNAP=glm_tiny_i4r REF=glm_tiny_i4r/ref_glm.json COLI_VK_EXPERTS=4 COLI_VK_DEV2=0 ./colibri 64 4 4
gsan colibri "asan colibri int4 at load, PIPE=1" glm_tiny SNAP=glm_tiny REF=ref_glm.json PIPE=1 ./colibri 1 4 4
gsan colibri "asan colibri int8 at load, PILOT=1" glm_tiny SNAP=glm_tiny REF=ref_glm.json PILOT=1 ./colibri 2 8 8
$PY tools/make_glm_mtp_tiny.py --src glm_tiny --out glm_tiny_mtp > /dev/null && cp ref_glm.json glm_tiny_mtp/
gsan colibri "asan colibri MTP head on the tier" glm_tiny_mtp SNAP=glm_tiny_mtp REF=glm_tiny_mtp/ref_glm.json DRAFT=2 IDOT=0 COLI_VK_TIER_MTP=1 ./colibri 2 4 4
mtp_on_device "asan colibri MTP head on the tier" san.log
local ids
ids=$($PY -c "print(','.join(str((i*37+5)%120+2) for i in range(100)))")
gsan glm53 "asan glm53 decode" - GLM53_BITS=32 ./glm53 --model glm53_stream-i4 --ids 5,7,9,11,13,17,19,23 --greedy 20
HIST=1 gsan glm53 "asan glm53 100-token prompt, warm" - GLM53_BITS=32 ./glm53 --model glm53_stream-i4 --ids $ids --greedy 4
HIST=1 gsan glm53 "asan glm53 one slot, trunk on the device" - GLM53_BITS=4 GLM53_EXPERT_GB=0.000001 COLI_VK_DENSE=1 ./glm53 --model glm53_stream-i4 --ids $ids --greedy 4
gsan glm53 "asan glm53 eviction" - GLM53_BITS=32 COLI_VK_TIER_GB=0.00006 ./glm53 --model glm53_stream-i4 --ids $ids --greedy 6
unset OMP_NUM_THREADS CAP_RAISE
make clean >/dev/null 2>&1 || true
}
# The dense chain (vk_chain.c, COLI_VK_CHAIN=1): every layer recorded into one
# submission, the residual stream, the KV mirrors and the recurrent state on the device.
# Gates, per configuration: the CPU run's tokens (and its oracle, where the CPU passes
# it); the last logits within 1e-4 of the largest one where both runs multiply f32
# activations (tol 1); a "[VK] <engine> chain: N forwards" line with N > 0. Beside them:
# the chain's ops against CPU references (tests/test_vk_chain), qwen38's oracle targets
# with the chain on, the prefix-reuse contract, and serve sessions (pins, the prompt
# cache, the prefill read-out, MTP drafts) frame for frame against the CPU.
chain_count() {
local n
n=$(sed -n "s/^\[VK\] $1 chain: \([0-9][0-9]*\) forwards.*/\1/p" "$2" | tail -1)
echo "${n:-0}"
}
logits_close() { # <cpu.f32> <vk.f32>: max |diff| within 1e-4 of the largest |logit|
$PY - "$1" "$2" <<'PY'
import array, sys
a = array.array("f", open(sys.argv[1], "rb").read()); b = array.array("f", open(sys.argv[2], "rb").read())
d = max(abs(x - y) for x, y in zip(a, b)) if a and len(a) == len(b) else float("inf")
m = max(abs(x) for x in a) if a else 0.0
print(f"max |logit diff| {d:.2e} of {m:.2e}")
sys.exit(0 if d <= 1e-4 * m else 1)
PY
}
chain_gate() { # <engine> <tag> <tol 0|1> <env...> -- <argv...> (CPUENV: the CPU arm's own settings)
local eng=$1 tag=$2 tol=$3; shift 3
local envs=(); while [ "$1" != "--" ]; do envs+=("$1"); shift; done; shift
rm -f chain.usage cpu.f32 vk.f32
local rc_cpu=0
env "${envs[@]}" ${CPUENV:-} DUMP=cpu.f32 ./"$eng" "$@" > cpu.log 2>&1 || rc_cpu=$?
env "${envs[@]}" DUMP=vk.f32 COLI_USAGE=chain.usage COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=${CHAINMODE:-1} \
./"$eng" "$@" > vk.log 2>&1 || { [ $rc_cpu != 0 ] || { cat vk.log; fail "$tag: the chain misses the oracle the CPU passes"; }; }
same_tokens cpu.log vk.log "$tag"
[ "$(chain_count "$eng" vk.log)" -gt 0 ] || { cat vk.log; fail "$tag: the chain never ran"; }
local lg=""
if [ "$tol" = 1 ]; then lg=$(logits_close cpu.f32 vk.f32) || { echo "$lg"; fail "$tag: logits"; }; lg=", $lg"; fi
echo "OK $tag: tokens = CPU$lg, $(chain_count "$eng" vk.log) chain forwards"
}
# lost_gate <engine> <tag> <frame> <env...> -- <argv...>: the device "lost" at the given
# chain frame (COLI_VK_CHAIN_FAULT); the engine rebuilds the recurrent state on the CPU
# from the prefix record and continues there: the CPU run's tokens all the same.
lost_gate() {
local eng=$1 tag=$2 k=$3; shift 3
local envs=(); while [ "$1" != "--" ]; do envs+=("$1"); shift; done; shift
rm -f chain.usage
env "${envs[@]}" ./"$eng" "$@" > cpu.log 2>&1 || true
env "${envs[@]}" COLI_VK_CHAIN_FAULT=$k COLI_USAGE=chain.usage COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 \
./"$eng" "$@" > vk.log 2>&1 || true
same_tokens cpu.log vk.log "$tag"
grep -q "rebuilding the state of [1-9]" vk.log || { cat vk.log; fail "$tag: no state was rebuilt"; }
echo "OK $tag: tokens = CPU, $(grep -o 'rebuilding the state of [0-9]* positions' vk.log)"
}
family_qwen_chain() {
make qwen36 qwen38 tests/test_vk_chain VK=1
./tests/test_vk_chain shaders/qmatmul.spv | tee vk_chain.log
tail -1 vk_chain.log | grep -qx PASS || fail "the chain's ops"
$PY tools/make_qwen36_tiny.py --out qwen36_tiny --ref-mode full --emit-ref qwen36_tiny/ref_full.json
$PY tools/make_qwen36_tiny.py --geometry qwen3-coder-30b --out qwen3_coder_tiny --ref-mode full --emit-ref qwen3_coder_tiny/ref_full.json
$PY tools/make_qwen36_tiny.py --geometry qwen38-27b-dense --out qwen38_27b_tiny --ref-mode full --emit-ref qwen38_27b_tiny/ref_full.json
$PY tools/make_qwen36_tiny.py --geometry qwen38-2p4t --seed 3 --out qwen38_2p4t_tiny --ref-mode full --emit-ref qwen38_2p4t_tiny/ref_full.json
local fx cap caps
# qwen36: the hybrid, Qwen3-Coder (attention only, no gate, no shared expert), the
# 27B dense geometry (no routed experts: no host step at all) and the 2.4T one
for fx in qwen36_tiny qwen3_coder_tiny qwen38_27b_tiny qwen38_2p4t_tiny; do
$PY tools/convert_qwen36.py --model $fx --out ${fx}_c --ebits 8
caps="1 8"; [ $fx = qwen38_2p4t_tiny ] && caps=8
for cap in $caps; do
chain_gate qwen36 "chain qwen36 $fx f32 cap=$cap" 1 COLI_DENSE_I8=0 SNAP=${fx}_c -- $cap 8 $fx/ref_full.json
done
# int8 dense rows (fmt 1): the CPU arm with f32 activations (COLI_DENSE_IDOT=0), as the device
CPUENV=COLI_DENSE_IDOT=0 chain_gate qwen36 "chain qwen36 $fx int8" 1 SNAP=${fx}_c -- 8 8 $fx/ref_full.json
done
# the expert containers the tier reads, the tier off, prefill in chunks of 3 rows,
# the tiled GEMM inside the chain from S = 2
$PY tools/make_qwen36_tiny.py --out qwen36_tiny64 --ref-mode full --inter 64 --emit-ref qwen36_tiny64/ref_full.json
$PY tools/convert_qwen36.py --model qwen36_tiny64 --out qwen36_tiny64_c --ebits 4 --gs 64
$PY tools/convert_qwen36.py --model qwen36_tiny64 --out qwen36_tiny64_d8 --ebits 4 --gs 64 --down-bits 8
$PY tools/convert_qwen36.py --model qwen36_tiny --out qwen36_tiny_i4r --ebits 4
chain_gate qwen36 "chain qwen36 int4-g64 planar" 1 QWEN_EXPERT_ACT=f32 COLI_DENSE_I8=0 SNAP=qwen36_tiny64_c -- 8 4 qwen36_tiny64/ref_full.json
chain_gate qwen36 "chain qwen36 int4-g64 int8 activations" 0 COLI_DENSE_I8=0 SNAP=qwen36_tiny64_c -- 1 4 qwen36_tiny64/ref_full.json
chain_gate qwen36 "chain qwen36 mixed int4/int8" 1 COLI_DENSE_I8=0 SNAP=qwen36_tiny64_d8 -- 8 4 qwen36_tiny64/ref_full.json
chain_gate qwen36 "chain qwen36 int4 per row" 1 COLI_DENSE_I8=0 SNAP=qwen36_tiny_i4r -- 8 4 qwen36_tiny/ref_full.json
chain_gate qwen36 "chain qwen36 tier off" 1 COLI_VK_TIER=0 COLI_DENSE_I8=0 SNAP=qwen36_tiny_c -- 8 8 qwen36_tiny/ref_full.json
chain_gate qwen36 "chain qwen36 prefill in chunks of 3" 1 COLI_VK_CHAIN_ROWS=3 COLI_DENSE_I8=0 SNAP=qwen36_tiny_c -- 8 8 qwen36_tiny/ref_full.json
chain_gate qwen36 "chain qwen36 tiled GEMM" 1 COLI_VK_GEMM_MIN_S=2 COLI_DENSE_I8=0 SNAP=qwen36_tiny_c -- 8 8 qwen36_tiny/ref_full.json
chain_gate qwen36 "chain qwen36 the per-row GEMV" 1 COLI_VK_CHAIN_GEMV=0 COLI_DENSE_I8=0 SNAP=qwen36_tiny_c -- 8 8 qwen36_tiny/ref_full.json
# an image: the M-RoPE positions reach the chain as the host's cos/sin table,
# token-exact against transformers, on the CLI and through the IMAGE frame
$PY tools/make_qwen36_vl_tiny.py --out qwen38_27b_vl_tiny
$PY tools/convert_qwen36.py --model qwen38_27b_vl_tiny --out qwen38_27b_vl_tiny_c --ebits 8
chain_gate qwen36 "chain qwen36 vision" 1 COLI_DENSE_I8=0 SNAP=qwen38_27b_vl_tiny_c -- 8 8 qwen38_27b_vl_tiny/ref.json
QWEN36_VL_TINY=qwen38_27b_vl_tiny_c QWEN36_VL_REF=qwen38_27b_vl_tiny/ref.json COLI_VULKAN=1 COLI_VK_CHAIN=1 \
COLI_USAGE=$PWD/chain.usage $PY -m unittest tests.test_qwen36_vision_serve
# the device lost mid-decode: the state rebuilt on the CPU, the run finishes there
lost_late qwen36 "chain qwen36 device lost" 20 rebuild COLI_DENSE_I8=0 SNAP=qwen36_tiny_c -- 8 8 qwen36_tiny/ref_full.json
# the prefix-reuse contract with the state on the device, and a serve session
QWEN36_TINY=$PWD/qwen36_tiny_c COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_USAGE=$PWD/chain.usage $PY tests/test_qwen36_prefix_serve.py
$PY tests/vulkan_chain_serve.py ./qwen36 qwen36_tiny_c COLI_DENSE_I8=0
# COLI_VK_CHAIN=2, prompts on the device and decode on the CPU: the state crosses
# between them at every turn's first decode step and next prompt
CHAINMODE=2 chain_gate qwen36 "chain qwen36 prompts only" 1 COLI_DENSE_I8=0 SNAP=qwen36_tiny_c -- 8 8 qwen36_tiny/ref_full.json
COLI_VK_CHAIN=2 $PY tests/vulkan_chain_serve.py ./qwen36 qwen36_tiny_c COLI_DENSE_I8=0
# qwen38: every resident format, prefill batching, every expert form, MTP
$PY tools/make_qwen38_tiny.py --out qwen38_tiny
$PY tools/make_qwen38_tiny.py --out qwen38_tiny_fp8 --fp8-experts
$PY tools/make_qwen38_tiny.py --out qwen38_tiny_int4 --fp8-experts --int4-experts --expert-gain 3
local trunk batch bf16 fx ref f
for batch in 0 1; do for bf16 in 0 1; do
chain_gate qwen38 "chain qwen38 bf16=$bf16 batch=$batch" 1 OMP_NUM_THREADS=2 Q38_PREFILL_BATCH=$batch Q38_NATIVE_BF16=$bf16 SNAP=qwen38_tiny -- 4 8 qwen38_tiny/ref.json
done; done
# the int8 trunk (fmt 1): the CPU's integer kernel rounds activations, so tokens only
chain_gate qwen38 "chain qwen38 int8 trunk" 0 OMP_NUM_THREADS=2 Q38_TRUNK_MIN_KB=0 SNAP=qwen38_tiny -- 4 8 qwen38_tiny/ref.json
for fx in qwen38_tiny_fp8 qwen38_tiny_int4; do
ref=$fx/ref.json; [ $fx = qwen38_tiny_int4 ] && ref=$fx/ref_int4.json
for batch in 0 1; do chain_gate qwen38 "chain qwen38 $fx batch=$batch" 1 OMP_NUM_THREADS=2 Q38_PREFILL_BATCH=$batch SNAP=$fx -- 2 8 $ref; done
done
chain_gate qwen38 "chain qwen38 tier off" 1 OMP_NUM_THREADS=2 COLI_VK_TIER=0 SNAP=qwen38_tiny -- 4 8 qwen38_tiny/ref.json
chain_gate qwen38 "chain qwen38 prefill in chunks of 3" 1 OMP_NUM_THREADS=2 COLI_VK_CHAIN_ROWS=3 SNAP=qwen38_tiny -- 4 8 qwen38_tiny/ref.json
$PY tools/make_qwen38_tiny.py --out qwen38_tiny_mtp --mtp
$PY tools/make_edge_tiny_tokenizer.py --vocab-size 64 ./qwen38_tiny_mtp # serve speaks text
$PY tools/make_qwen38_tiny.py --out qwen38_tiny_fp8_mtp --fp8-experts --mtp
$PY tools/make_qwen38_tiny.py --out qwen38_tiny_int4_mtp --fp8-experts --int4-experts --expert-gain 3 --mtp
# MTP verify (S = 2) with the device's snapshot and rollback: drafts as they come,
# every draft rejected (a rollback per token), every one accepted, alternating
for fx in qwen38_tiny_mtp qwen38_tiny_fp8_mtp qwen38_tiny_int4_mtp; do for f in "" reject accept mixed; do
chain_gate qwen38 "chain qwen38 MTP $fx ${f:-drafting}" 1 OMP_NUM_THREADS=2 Q38_MTP=1 Q38_MTP_FORCE=$f SNAP=$fx -- 2 8 $fx/ref.json
done; done
# the device lost mid-decode, once between steps and once inside an MTP verify
lost_late qwen38 "chain qwen38 device lost" 10 rebuild OMP_NUM_THREADS=2 SNAP=qwen38_tiny -- 4 8 qwen38_tiny/ref.json
lost_late qwen38 "chain qwen38 device lost in a verify" 15 rebuild OMP_NUM_THREADS=2 Q38_MTP=1 Q38_MTP_FORCE=mixed SNAP=qwen38_tiny_mtp -- 2 8 qwen38_tiny_mtp/ref.json
# the oracle targets with the chain on (COLI_VK_TIER_BALANCE=0: the int4 target wants
# the same last logits from every expert path, and the balancer moves experts between
# the device and the CPU by measured times); the MTP harness, whose prompt-cache and pin
# checks compare a warm session's draft logits bit for bit with a fresh one's, with the
# tier off: the tier's residency differs between two sessions, with or without the chain
COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 COLI_VK_TIER_BALANCE=0 \
make qwen38-tiny-check qwen38-tiny-fp8-check qwen38-tiny-int4-check VK=1
COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER=0 make qwen38-tiny-mtp-check VK=1
$PY tests/vulkan_chain_serve.py ./qwen38 qwen38_tiny_mtp OMP_NUM_THREADS=2 Q38_MTP=1
$PY tests/vulkan_chain_serve.py ./qwen38 qwen38_tiny_mtp OMP_NUM_THREADS=2
CHAINMODE=2 chain_gate qwen38 "chain qwen38 prompts only, MTP" 1 OMP_NUM_THREADS=2 Q38_MTP=1 Q38_MTP_FORCE=mixed SNAP=qwen38_tiny_mtp -- 2 8 qwen38_tiny_mtp/ref.json
COLI_VK_CHAIN=2 $PY tests/vulkan_chain_serve.py ./qwen38 qwen38_tiny_mtp OMP_NUM_THREADS=2 Q38_MTP=1
}
# The chain under ASan and UBSan: memory safety is the gate (a sanitized build
# vectorizes differently, so tokens are not compared); each run must have run the chain.
family_qwen_chain_sanitize() {
make clean >/dev/null 2>&1 || true
make qwen36 qwen38 tests/test_vk_chain VK=1 EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1
./tests/test_vk_chain shaders/qmatmul.spv > san.log 2>&1 || true
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log || ! tail -1 san.log | grep -qx PASS; then cat san.log; fail "asan: the chain's ops"; fi
echo "OK asan: the chain's ops"
$PY tools/make_qwen36_tiny.py --out qwen36_tiny --ref-mode full --emit-ref qwen36_tiny/ref_full.json
$PY tools/convert_qwen36.py --model qwen36_tiny --out qwen36_tiny_c --ebits 8
$PY tools/make_qwen36_tiny.py --geometry qwen38-27b-dense --out qwen38_27b_tiny --ref-mode full --emit-ref qwen38_27b_tiny/ref_full.json
$PY tools/convert_qwen36.py --model qwen38_27b_tiny --out qwen38_27b_tiny_c --ebits 8
$PY tools/make_qwen36_tiny.py --out qwen36_tiny64 --ref-mode full --inter 64 --emit-ref qwen36_tiny64/ref_full.json
$PY tools/convert_qwen36.py --model qwen36_tiny64 --out qwen36_tiny64_c --ebits 4 --gs 64
$PY tools/make_qwen38_tiny.py --out qwen38_tiny
$PY tools/make_qwen38_tiny.py --out qwen38_tiny_int4_mtp --fp8-experts --int4-experts --expert-gain 3 --mtp
$PY tools/make_qwen38_tiny.py --out qwen38_tiny_mtp --mtp
$PY tools/make_edge_tiny_tokenizer.py --vocab-size 64 ./qwen38_tiny_mtp # serve speaks text
csan() { # <engine> <tag> <env and argv...>
local eng=$1 tag=$2; shift 2
rm -f chain.usage
env OMP_NUM_THREADS=2 COLI_USAGE=chain.usage COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 "$@" > san.log 2>&1 || true
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "$tag: sanitizer diagnostic"; fi
[ "$(chain_count "$eng" san.log)" -gt 0 ] || { cat san.log; fail "$tag: the chain never ran"; }
echo "OK $tag: sanitizers clean, $(chain_count "$eng" san.log) chain forwards"
}
csan qwen36 "asan chain qwen36 int8 experts" COLI_DENSE_I8=0 SNAP=qwen36_tiny_c ./qwen36 1 8 qwen36_tiny/ref_full.json
csan qwen36 "asan chain qwen36 int8 dense, chunks of 3" COLI_VK_CHAIN_ROWS=3 SNAP=qwen36_tiny_c ./qwen36 8 8 qwen36_tiny/ref_full.json
csan qwen36 "asan chain qwen36 int4-g64 planar" COLI_DENSE_I8=0 SNAP=qwen36_tiny64_c ./qwen36 8 4 qwen36_tiny64/ref_full.json
csan qwen36 "asan chain qwen36 dense model" COLI_DENSE_I8=0 SNAP=qwen38_27b_tiny_c ./qwen36 8 8 qwen38_27b_tiny/ref_full.json
csan qwen38 "asan chain qwen38 batch=0" Q38_PREFILL_BATCH=0 SNAP=qwen38_tiny ./qwen38 4 8 qwen38_tiny/ref.json
csan qwen38 "asan chain qwen38 int8 trunk, chunks of 3" Q38_TRUNK_MIN_KB=0 COLI_VK_CHAIN_ROWS=3 SNAP=qwen38_tiny ./qwen38 4 8 qwen38_tiny/ref.json
csan qwen38 "asan chain qwen38 MTP reject" Q38_MTP=1 Q38_MTP_FORCE=reject SNAP=qwen38_tiny_int4_mtp ./qwen38 2 8 qwen38_tiny_int4_mtp/ref.json
csan qwen38 "asan chain qwen38 MTP mixed" Q38_MTP=1 Q38_MTP_FORCE=mixed SNAP=qwen38_tiny_int4_mtp ./qwen38 2 8 qwen38_tiny_int4_mtp/ref.json
$PY tests/vulkan_chain_serve.py ./qwen38 qwen38_tiny_mtp Q38_MTP=1 > san.log 2>&1 || { cat san.log; fail "asan chain qwen38 serve"; }
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "asan chain qwen38 serve: sanitizer diagnostic"; fi
echo "OK asan chain qwen38 serve: $(tail -1 san.log)"
make clean >/dev/null 2>&1 || true
}
# Speculative decoding with deeper verifies (docs/speculative.md): the MTP head's drafts
# up to 3 per verify (qwen38, Q38_MTP_DRAFTS) and prompt-lookup drafts up to 5 (qwen38
# and qwen36, COLI_LOOKUP=1), a verify of up to 6 rows with the recurrent state copied
# after each row; and colibri's n-gram drafts from the same lookup and gate. tests/spec_drafts_harness.py (make spec-drafts-check) compares every
# run with drafts against the same run without, tokens and last logits byte for byte:
# - on the CPU (this VK=1 build with COLI_VULKAN unset);
# - in the dense chain, the tier off (with it on, the tier's residency follows the
# routed rows, the rejected drafts' included, and an expert on the device sums in
# another order than on the CPU, so the last bits depend on the history); and the
# chain in chunks of 3 rows (a verify of 6 rows split over two chunks), the
# per-matrix path (a verify's rows one GEMV at a time) and prompts only
# (COLI_VK_CHAIN=2: verifies on the per-matrix path);
# - against the CPU with the tier on (chain_gate: the CPU's tokens, logits within
# tolerance), and the device lost inside a deep verify (rebuilt on the CPU).
spec_fixtures() {
$PY tools/make_qwen38_tiny.py --out qwen38_tiny_mtp --mtp
$PY tools/make_edge_tiny_tokenizer.py --vocab-size 64 ./qwen38_tiny_mtp # serve speaks text
$PY tools/make_qwen38_tiny.py --out qwen38_tiny_int4_mtp --fp8-experts --int4-experts --expert-gain 3 --mtp
make qwen36-tiny-v128-generate
}
family_qwen_spec() {
make qwen36 qwen38 tests/test_spec_draft VK=1
make spec-drafts-check VK=1
local chain="--env COLI_VULKAN=1 --env COLI_VK_CHAIN=1 --env COLI_VK_TIER=0 --env COLI_VK_TIER_SYNC=1"
make spec-drafts-run VK=1 SPEC_ENV="$chain"
local h="$PY tests/spec_drafts_harness.py" e
for e in "--env COLI_VULKAN=1 --env COLI_VK_CHAIN=1 --env COLI_VK_TIER=0 --env COLI_VK_CHAIN_ROWS=3" \
"--env COLI_VULKAN=1 --env COLI_VK_CHAIN=0 --env COLI_VK_DENSE=1 --env COLI_VK_TIER=0" \
"--env COLI_VULKAN=1 --env COLI_VK_CHAIN=2 --env COLI_VK_TIER=0"; do
$h --engine ./qwen38 --kind qwen38 --fixture ./qwen38_tiny_mtp --mtp --quick $e
$h --engine ./qwen36 --kind qwen36 --fixture ./qwen36_tiny_v128_c --ref ./qwen36_tiny_v128/ref_full.json \
--cap 8 --n-new 64 --quick --env COLI_DENSE_I8=0 $e
done
# A six-row verify crosses several one-row KV chunks. Rollback must lower both
# the mirror and split-cache validity, including rejected rows already uploaded.
# Keep the same attention kernel in plain/speculative arms for byte comparisons.
e="$chain --env COLI_VK_KV_DEVICE_ROWS=8 --env COLI_VK_KV_BLOCK=2 --env COLI_VK_CHAIN_ROWS=1 --env COLI_VK_ATTN_BLOCK=0"
$h --engine ./qwen38 --kind qwen38 --fixture ./qwen38_tiny_mtp --mtp --serve --quick --require-kv-split $e
$h --engine ./qwen36 --kind qwen36 --fixture ./qwen36_tiny_v128_c --ref ./qwen36_tiny_v128/ref_full.json \
--cap 8 --n-new 64 --serve --quick --require-kv-split --env COLI_DENSE_I8=0 $e
# A completed MTP chunk has already returned its hidden streams. A loss in
# the next chunk must restart the CPU replay from the original embeddings.
$h --engine ./qwen38 --kind qwen38 --fixture ./qwen38_tiny_mtp --mtp --n-new 32 \
--device-loss --require-kv-split --env COLI_VK_DENSE_HOST=0 $e
local f
for f in cycle row2 row3; do
chain_gate qwen38 "spec chain qwen38 MTP depth 3 $f, tier on" 1 OMP_NUM_THREADS=2 Q38_MTP=1 Q38_MTP_DRAFTS=3 \
Q38_MTP_FORCE=$f SNAP=qwen38_tiny_mtp -- 2 8 qwen38_tiny_mtp/ref.json
chain_gate qwen36 "spec chain qwen36 lookup $f, tier on" 1 COLI_DENSE_I8=0 COLI_LOOKUP=1 COLI_SPEC_GATE=0 \
COLI_LOOKUP_FORCE=$f SNAP=qwen36_tiny_v128_c -- 8 8 qwen36_tiny_v128/ref_full.json
done
chain_gate qwen38 "spec chain qwen38 lookup row4, int4 experts, tier on" 1 OMP_NUM_THREADS=2 Q38_EXPERT_INT4=1 \
COLI_VK_TIER_BALANCE=0 COLI_LOOKUP=1 COLI_SPEC_GATE=0 COLI_LOOKUP_FORCE=row4 SNAP=qwen38_tiny_int4_mtp -- 2 8 \
qwen38_tiny_int4_mtp/ref_int4.json
# colibri (GLM-5.2): its n-gram drafts from spec_draft.h under COLI_LOOKUP=1, on the
# CPU and in the chain, against the oracle (every token) and against DRAFT=0
make colibri VK=1
$PY tools/make_glm_oracle.py > /dev/null
local v base
base=$(env OMP_NUM_THREADS=2 SNAP=glm_tiny REF=ref_glm.json DRAFT=0 ./colibri 64 16 16 2>/dev/null | grep -a '^GLM C engine')
for v in "" "COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER=0" "COLI_VULKAN=1 COLI_VK_CHAIN=1"; do
for e in "COLI_SPEC_GATE=0" "COLI_SPEC_GATE=0 DRAFT=2" ""; do
# shellcheck disable=SC2086
env OMP_NUM_THREADS=2 SNAP=glm_tiny REF=ref_glm.json COLI_USAGE=chain.usage COLI_LOOKUP=1 $e $v ./colibri 64 16 16 > vk.log 2>&1 \
|| { cat vk.log; fail "spec colibri lookup $e $v: misses the oracle"; }
[ "$(grep -a '^GLM C engine' vk.log)" = "$base" ] || { cat vk.log; fail "spec colibri lookup $e $v: tokens differ from DRAFT=0"; }
grep -q "colibri lookup\] acceptance" vk.log || { cat vk.log; fail "spec colibri lookup $e $v: no lookup line"; }
echo "OK spec colibri lookup ${e:-gated} ${v:-CPU}: oracle and DRAFT=0's tokens, $(grep -o 'acceptance [^|]*' vk.log | head -1)"
done
done
lost_late qwen38 "spec chain qwen38 device lost mid-decode, verifies of 4 rows" 15 rebuild OMP_NUM_THREADS=2 Q38_MTP=1 \
Q38_MTP_DRAFTS=3 Q38_MTP_FORCE=cycle SNAP=qwen38_tiny_mtp -- 2 8 qwen38_tiny_mtp/ref.json
lost_late qwen36 "spec chain qwen36 device lost mid-decode, lookup verifies" 20 rebuild COLI_DENSE_I8=0 COLI_LOOKUP=1 \
COLI_SPEC_GATE=0 COLI_LOOKUP_FORCE=cycle SNAP=qwen36_tiny_v128_c -- 8 8 qwen36_tiny_v128/ref_full.json
}
# The same under ASan and UBSan: every run of the harness's quick subset (depth 3
# rejected at rows 1 and 2 and cycling, lookup cycling and at row 3, a serve session
# per source) on the CPU and in the chain, sanitizer-clean and still equal to its own
# plain decoding.
family_qwen_spec_sanitize() {
make clean >/dev/null 2>&1 || true
make qwen36 qwen38 tests/test_spec_draft tests/test_qwen38_spec_alloc VK=1 EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1
./tests/test_spec_draft > san.log 2>&1 || true
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log || ! tail -1 san.log | grep -q "behave as"; then cat san.log; fail "asan: spec_draft.h"; fi
echo "OK asan: spec_draft.h"
./tests/test_qwen38_spec_alloc > san.log 2>&1 || { cat san.log; fail "asan: speculative allocation"; }
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "asan: speculative allocation"; fi
echo "OK asan: speculative allocation"
spec_fixtures
local h="$PY tests/spec_drafts_harness.py --sanitize --quick" e
for e in "" "--env COLI_VULKAN=1 --env COLI_VK_CHAIN=1 --env COLI_VK_TIER=0" \
"--require-kv-split --env COLI_VULKAN=1 --env COLI_VK_CHAIN=1 --env COLI_VK_TIER=0 --env COLI_VK_KV_DEVICE_ROWS=8 --env COLI_VK_KV_BLOCK=2 --env COLI_VK_CHAIN_ROWS=1 --env COLI_VK_ATTN_BLOCK=0"; do
$h --engine ./qwen38 --kind qwen38 --fixture ./qwen38_tiny_mtp --mtp --serve $e
$h --engine ./qwen38 --kind qwen38 --fixture ./qwen38_tiny_int4_mtp --ref ./qwen38_tiny_int4_mtp/ref_int4.json \
--mtp --env Q38_EXPERT_INT4=1 --env Q38_TRUNK_MIN_KB=0 $e
$h --engine ./qwen36 --kind qwen36 --fixture ./qwen36_tiny_v128_c --ref ./qwen36_tiny_v128/ref_full.json \
--cap 8 --n-new 64 --serve --env COLI_DENSE_I8=0 $e
done
$h --engine ./qwen38 --kind qwen38 --fixture ./qwen38_tiny_mtp --mtp --n-new 32 --device-loss \
--require-kv-split --env COLI_VULKAN=1 --env COLI_VK_CHAIN=1 --env COLI_VK_TIER=0 --env COLI_VK_DENSE_HOST=0 \
--env COLI_VK_KV_DEVICE_ROWS=8 --env COLI_VK_KV_BLOCK=2 --env COLI_VK_CHAIN_ROWS=1 --env COLI_VK_ATTN_BLOCK=0
make clean >/dev/null 2>&1 || true
}
# inkling's and olmoe's fixtures for their dense chain: the family inkling-olmoe's (the
# bf16 copy of the f32 snapshot, the dense-int4g64 container, the expert containers and
# the tokenizers), and Inkling at its real width (--wide: D = 6144, head dim 128).
inkling_olmoe_chain_fixtures() {
$PY tools/make_tiny_inkling.py tiny_inkling
$PY tools/make_tiny_inkling.py tiny_inkling_wide --wide
mkdir -p tiny_inkling_bf16
cp tiny_inkling/config.json tiny_inkling/generation_config.json tiny_inkling_bf16/
$PY - <<'EOF'
import json, struct, numpy as np
src, dst = "tiny_inkling/model.safetensors", "tiny_inkling_bf16/model.safetensors"
raw = open(src, "rb").read(); n = struct.unpack("<Q", raw[:8])[0]; hdr = json.loads(raw[8:8 + n])
out, blobs, off = {}, [], 0
for k, v in hdr.items():
if k == "__metadata__": out[k] = v; continue
a = np.frombuffer(raw[8 + n + v["data_offsets"][0]: 8 + n + v["data_offsets"][1]], np.float32)
u = a.view(np.uint32).astype(np.uint64)
b = ((u + 0x7FFF + ((u >> 16) & 1)) >> 16).astype(np.uint16).tobytes()
out[k] = {"dtype": "BF16", "shape": v["shape"], "data_offsets": [off, off + len(b)]}
blobs.append(b); off += len(b)
h = json.dumps(out).encode(); h += b" " * (-len(h) % 8)
with open(dst, "wb") as f:
f.write(struct.pack("<Q", len(h))); f.write(h); [f.write(b) for b in blobs]
EOF
rm -rf tiny_inkling_q && cp -r tiny_inkling tiny_inkling_q
$PY - tools/convert_inkling_dense_int4.py tiny_inkling_q <<'EOF'
import importlib.util, sys
spec = importlib.util.spec_from_file_location("conv", sys.argv[1])
conv = importlib.util.module_from_spec(spec); spec.loader.exec_module(conv)
conv.MIN_ELEMS = 0
conv.ATTN_BITS = 4
base = conv.classify
conv.classify = lambda n, s, d: "int8" if n.endswith(".mlp.down_proj.weight") else base(n, s, d)
sys.argv = ["convert_inkling_dense_int4.py", "--dir", sys.argv[2]]
conv.main()
EOF
mkdir -p tiny_inkling_q/dense-int4g64
mv tiny_inkling_q/dense-int4g64.safetensors tiny_inkling_q/dense-int4g64/dense.safetensors
$PY tools/make_olmoe_tiny.py --output olmoe_tiny --force
$PY tools/convert_olmoe_merged.py --model olmoe_tiny --out olmoe_tiny_c
inkling_olmoe_tier_fixtures
}
# chain_gate for a bf16 snapshot: where this CPU's bf16 dot rounds the activations
# (AVX512-BF16), the per-matrix path keeps bf16 on the CPU and so does the chain, which
# must then decline cleanly ("bf16 stays on the CPU") with the CPU's tokens.
chain_gate_bf16() { # <engine> <tag> <env...> -- <argv...>
local eng=$1 tag=$2; shift 2
local envs=(); while [ "$1" != "--" ]; do envs+=("$1"); shift; done; shift
env "${envs[@]}" COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 ./"$eng" "$@" > vk.log 2>&1 || true
if grep -qa 'bf16 stays on the CPU' vk.log; then
env "${envs[@]}" ./"$eng" "$@" > cpu.log 2>&1 || true
same_tokens cpu.log vk.log "$tag"
echo "OK $tag: tokens = CPU, the chain declined (this CPU's bf16 dot keeps bf16 on the CPU)"
else
chain_gate "$eng" "$tag" 1 "${envs[@]}" -- "$@"
fi
}
# lost_gate for olmoe: attention only, so nothing to rebuild; the CPU redoes the step
# the device was lost in from its first position and runs from there.
lost_gate_redo() { # <engine> <tag> <frame> <env...> -- <argv...>
local eng=$1 tag=$2 k=$3; shift 3
local envs=(); while [ "$1" != "--" ]; do envs+=("$1"); shift; done; shift
rm -f chain.usage
env "${envs[@]}" ./"$eng" "$@" > cpu.log 2>&1 || true
env "${envs[@]}" COLI_VK_CHAIN_FAULT=$k COLI_USAGE=chain.usage COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 \
./"$eng" "$@" > vk.log 2>&1 || true
same_tokens cpu.log vk.log "$tag"
grep -q "the CPU redoes the step from position [1-9]" vk.log || { cat vk.log; fail "$tag: no step was redone"; }
echo "OK $tag: tokens = CPU, $(grep -o 'redoes the step from position [0-9]*' vk.log)"
}
# The device lost `back` frames before the end of a run (a probe run counts the frames,
# so the fault lands mid-decode on any device, whatever its setup submits): lost_gate
# (the state rebuilt on the CPU) or lost_gate_redo (the step redone there).
lost_late() { # <engine> <tag> <frames back> rebuild|redo <env...> -- <argv...>
local eng=$1 tag=$2 back=$3 kind=$4; shift 4
local envs=(); while [ "$1" != "--" ]; do envs+=("$1"); shift; done; shift
env "${envs[@]}" COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 ./"$eng" "$@" > vk.log 2>&1 || true
local f; f=$(sed -n "s/^\[VK\] $eng chain: [0-9]* forwards, \([0-9]*\) frames.*/\1/p" vk.log | tail -1)
{ [ -n "$f" ] && [ "$f" -gt "$back" ]; } || { cat vk.log; fail "$tag: the probe run reports no frames"; }
if [ "$kind" = rebuild ]; then lost_gate "$eng" "$tag" $((f - back)) "${envs[@]}" -- "$@"
else lost_gate_redo "$eng" "$tag" $((f - back)) "${envs[@]}" -- "$@"; fi
}
# The dense chain (COLI_VK_CHAIN=1) of inkling and olmoe on Lavapipe, each configuration
# against the CPU run's tokens and logits (chain_gate: every forward's logits within 1e-4
# of the largest): inkling's resident forms (f32, bf16, the dense-int4g64 container's
# int8 and int4-g64), its expert containers and runtime quantizations, TOPP, the tier
# off, the trunk's device copies shared with the per-matrix path (COLI_VK_DENSE=1),
# prefill in chunks of 3, the tiled GEMM and the per-row GEMV inside the chain, its real
# width (D = 6144, beside the tier's expert batch at that width); olmoe's caps, PILOT's
# prefetch from the chain's residual rows (cap 1: the slot the forward pass reads is
# never the prefetcher's victim), an eviction budget and the same chain settings; both
# with prompts only (COLI_VK_CHAIN=2), the device lost mid-run, and the serve, prefix
# and dashboard tests with the chain on.
family_inkling_olmoe_chain() {
export OMP_NUM_THREADS=2 # tiny models: a wide team only waits for itself (and for Lavapipe's)
make inkling olmoe tests/test_vk_chain VK=1
./tests/test_vk_chain shaders/qmatmul.spv | tee vk_chain.log
tail -1 vk_chain.log | grep -qx PASS || fail "the chain's ops"
inkling_olmoe_chain_fixtures
local fx cap bits R=tiny_inkling/ref_inkling.json OR=olmoe_tiny/ref_olmoe.json
for fx in tiny_inkling tiny_inkling_q tiny_inkling_x-i4 tiny_inkling_x-i8; do
for cap in 1 8; do chain_gate inkling "chain inkling $fx cap=$cap" 1 SNAP=$fx -- $cap 0 $R; done
done
chain_gate_bf16 inkling "chain inkling bf16" SNAP=tiny_inkling_bf16 -- 8 0 $R
for bits in 4 8; do chain_gate inkling "chain inkling runtime int$bits" 1 SNAP=tiny_inkling -- 2 $bits $R; done
chain_gate inkling "chain inkling TOPP" 1 TOPP=0.3 SNAP=tiny_inkling_x-i4 -- 2 0 $R
chain_gate inkling "chain inkling tier off" 1 COLI_VK_TIER=0 SNAP=tiny_inkling -- 8 0 $R
chain_gate inkling "chain inkling trunk shared with the per-matrix path" 1 COLI_VK_DENSE=1 SNAP=tiny_inkling_q -- 8 0 $R
chain_gate inkling "chain inkling prefill in chunks of 3" 1 COLI_VK_CHAIN_ROWS=3 SNAP=tiny_inkling -- 8 0 $R
chain_gate inkling "chain inkling tiled GEMM" 1 COLI_VK_GEMM_MIN_S=2 SNAP=tiny_inkling -- 8 0 $R
chain_gate inkling "chain inkling the per-row GEMV" 1 COLI_VK_CHAIN_GEMV=0 SNAP=tiny_inkling -- 8 0 $R
CHAINMODE=2 chain_gate inkling "chain inkling prompts only" 1 SNAP=tiny_inkling -- 8 0 $R
# D = 6144: the CPU's tokens through the chain, and through the tier's expert batch
# (its gate/up shader stages x up to 6144 floats) with the chain off
chain_gate inkling "chain inkling D=6144" 1 SNAP=tiny_inkling_wide -- 8 0 tiny_inkling_wide/ref_inkling.json
tier_gate inkling "inkling tier D=6144" COLI_VK_CHAIN=0 SNAP=tiny_inkling_wide -- 8 0 tiny_inkling_wide/ref_inkling.json
# the device lost mid-decode, in the last step but one: in a shared-expert frame
# nobody waits for (20 frames before the end) and at a layer's router (19): the
# state rebuilt on the CPU from the prefix record, the run finished there
lost_late inkling "chain inkling device lost in a shared-expert frame" 20 rebuild SNAP=tiny_inkling -- 8 0 $R
lost_late inkling "chain inkling device lost at a router" 19 rebuild SNAP=tiny_inkling -- 8 0 $R
for cap in 1 8; do chain_gate olmoe "chain olmoe cap=$cap" 1 SNAP=olmoe_tiny_c -- $cap 8 $OR; done
chain_gate olmoe "chain olmoe 4-bit experts" 1 SNAP=olmoe_tiny_c -- 8 4 $OR
for cap in 1 2; do chain_gate olmoe "chain olmoe PILOT cap=$cap" 1 PILOT=1 WIDE=2 SNAP=olmoe_tiny_c -- $cap 8 $OR; done
chain_gate olmoe "chain olmoe PILOT=3" 1 PILOT=3 SNAP=olmoe_tiny_c -- 8 8 $OR
chain_gate olmoe "chain olmoe tier off" 1 COLI_VK_TIER=0 SNAP=olmoe_tiny_c -- 8 8 $OR
chain_gate olmoe "chain olmoe trunk shared with the per-matrix path" 1 COLI_VK_DENSE=1 SNAP=olmoe_tiny_c -- 8 8 $OR
chain_gate olmoe "chain olmoe a budget of three experts" 1 COLI_VK_TIER_GB=0.000025 SNAP=olmoe_tiny_c -- 8 8 $OR
chain_gate olmoe "chain olmoe prefill in chunks of 3" 1 COLI_VK_CHAIN_ROWS=3 SNAP=olmoe_tiny_c -- 8 8 $OR
chain_gate olmoe "chain olmoe tiled GEMM" 1 COLI_VK_GEMM_MIN_S=2 SNAP=olmoe_tiny_c -- 8 8 $OR
chain_gate olmoe "chain olmoe the per-row GEMV" 1 COLI_VK_CHAIN_GEMV=0 SNAP=olmoe_tiny_c -- 8 8 $OR
CHAINMODE=2 chain_gate olmoe "chain olmoe prompts only" 1 SNAP=olmoe_tiny_c -- 8 8 $OR
lost_late olmoe "chain olmoe device lost mid-decode" 7 redo SNAP=olmoe_tiny_c -- 8 8 $OR
# serve: a session of pins, prompt-cache extensions, a divergent prompt and logprobs
# against the CPU's, frame by frame (and with prompts only); the prefix-reuse and
# dashboard contracts with the chain on
$PY tests/vulkan_chain_serve.py ./inkling tiny_inkling INK_PREFIX_LOG=1
COLI_VK_CHAIN=2 $PY tests/vulkan_chain_serve.py ./inkling tiny_inkling INK_PREFIX_LOG=1
$PY tests/vulkan_chain_serve.py ./olmoe olmoe_tiny_c
$PY tests/vulkan_chain_serve.py ./olmoe olmoe_tiny_c PILOT=1 WIDE=2
COLI_VK_CHAIN=2 $PY tests/vulkan_chain_serve.py ./olmoe olmoe_tiny_c
# (these compare a warm session's logprobs with a cold one's to the last printed
# digit: no history file and COLI_VK_TIER_BALANCE=0, so both sessions split their
# experts between the device and the CPU alike. Use per-row attention on both:
# a cold prefill and a short suffix must share a reduction order for exact bits.)
COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 COLI_VK_TIER_BALANCE=0 COLI_VK_ATTN_BLOCK=0 INKLING_TINY=tiny_inkling \
$PY -m unittest tests.test_inkling_prefix_serve tests.test_inkling_dashboard_hits
COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 COLI_VK_TIER_BALANCE=0 COLI_VK_ATTN_BLOCK=0 OLMOE_TINY=olmoe_tiny_c \
$PY -m unittest tests.test_olmoe_prefix_serve tests.test_olmoe_dashboard_hits tests.test_brio_serve
}
# inkling's and olmoe's chain under ASan and UBSan: memory safety is the gate (a
# sanitized build vectorizes differently, so tokens are not compared); each run must
# have run the chain. detect_stack_use_after_return=0: ASan's fake stack does not keep
# the 64-byte alignment an AVX-512 build gives its locals.
family_inkling_olmoe_chain_sanitize() {
make clean >/dev/null 2>&1 || true
make inkling olmoe tests/test_vk_chain VK=1 EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1
./tests/test_vk_chain shaders/qmatmul.spv > san.log 2>&1 || true
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log || ! tail -1 san.log | grep -qx PASS; then cat san.log; fail "asan: the chain's ops"; fi
echo "OK asan: the chain's ops"
inkling_olmoe_chain_fixtures
csan() { # <engine> <tag> <env and argv...>
local eng=$1 tag=$2; shift 2
rm -f chain.usage
env OMP_NUM_THREADS=2 COLI_USAGE=chain.usage COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=${CHAINMODE:-1} "$@" > san.log 2>&1 || true
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "$tag: sanitizer diagnostic"; fi
[ "$(chain_count "$eng" san.log)" -gt 0 ] || { cat san.log; fail "$tag: the chain never ran"; }
echo "OK $tag: sanitizers clean, $(chain_count "$eng" san.log) chain forwards"
}
local R=tiny_inkling/ref_inkling.json OR=olmoe_tiny/ref_olmoe.json
csan inkling "asan chain inkling f32 cap=1" SNAP=tiny_inkling ./inkling 1 0 $R
csan inkling "asan chain inkling dense-int4g64, the trunk shared" COLI_VK_DENSE=1 SNAP=tiny_inkling_q ./inkling 8 0 $R
csan inkling "asan chain inkling TOPP, chunks of 3" TOPP=0.3 COLI_VK_CHAIN_ROWS=3 SNAP=tiny_inkling_x-i4 ./inkling 2 0 $R
csan inkling "asan chain inkling D=6144" SNAP=tiny_inkling_wide ./inkling 8 0 tiny_inkling_wide/ref_inkling.json
CHAINMODE=2 csan inkling "asan chain inkling prompts only" SNAP=tiny_inkling ./inkling 8 0 $R
csan inkling "asan chain inkling device lost" COLI_VK_CHAIN_FAULT=60 SNAP=tiny_inkling ./inkling 8 0 $R
csan olmoe "asan chain olmoe PILOT cap=1" PILOT=1 WIDE=2 SNAP=olmoe_tiny_c ./olmoe 1 8 $OR
csan olmoe "asan chain olmoe eviction, chunks of 3" COLI_VK_TIER_GB=0.000025 COLI_VK_CHAIN_ROWS=3 SNAP=olmoe_tiny_c ./olmoe 8 8 $OR
csan olmoe "asan chain olmoe device lost" COLI_VK_CHAIN_FAULT=20 SNAP=olmoe_tiny_c ./olmoe 8 8 $OR
local eng
for eng in inkling olmoe; do
local snap=tiny_inkling; [ $eng = olmoe ] && snap=olmoe_tiny_c
$PY tests/vulkan_chain_serve.py ./$eng $snap OMP_NUM_THREADS=2 INK_PREFIX_LOG=1 > san.log 2>&1 || { cat san.log; fail "asan chain $eng serve"; }
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "asan chain $eng serve: sanitizer diagnostic"; fi
echo "OK asan chain $eng serve: $(tail -1 san.log)"
done
make clean >/dev/null 2>&1 || true
}
# The dense chain of the MLA engines: GLM-5.2 (colibri) and GLM-5.3 Flash (glm53).
# mla_gate <engine> <tag> <tol 0|1> <env...> -- <argv...>: the CPU run's token lines
# (colibri: the generated tokens, the teacher-forced predictions and the oracle's
# mismatches; glm53: teacher_forcing and greedy), every logits row within 1e-4 of the
# largest |logit| (tol 1; DUMP= writes them), and a "[VK] <engine> chain: N forwards"
# line with N > 0. FAULT_BACK=k: the device is lost k frames before the end of the same
# run without a fault (COLI_VK_CHAIN_FAULT counted from that run's frames, so the loss
# lands in the same forward on every device, whatever frames its setup took), and the
# run must say so (and, for glm53 with REBUILD=1, rebuild the KDA state of some positions).
mla_toks() { # <engine> <log>
if [ "$1" = colibri ]; then grep -aE '^GLM C engine|^PREFILL|^\[ORACLE\] mismatch' "$2" | sed 's/ | [0-9.]* pos\/s//'
else grep -aE '^teacher_forcing|^greedy' "$2"; fi
}
mla_gate() {
local eng=$1 tag=$2 tol=$3; shift 3
local envs=(); while [ "$1" != "--" ]; do envs+=("$1"); shift; done; shift
rm -f chain.usage cpu.f32 vk.f32
env "${envs[@]}" COLI_USAGE=chain.usage USAGE_SAVE=0 DUMP=cpu.f32 ./"$eng" "$@" > cpu.log 2>&1 || true
if [ -n "${FAULT_BACK:-}" ]; then
rm -f chain.usage
env "${envs[@]}" COLI_USAGE=chain.usage USAGE_SAVE=0 COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 \
./"$eng" "$@" > vk.log 2>&1 || true
local frames; frames=$(sed -n "s/^\[VK\] $eng chain: [0-9]* forwards, \([0-9]*\) frames.*/\1/p" vk.log | tail -1)
[ -n "$frames" ] && [ "$frames" -gt "$FAULT_BACK" ] || { cat vk.log; fail "$tag: no fault-free run to count frames from"; }
envs+=("COLI_VK_CHAIN_FAULT=$((frames - FAULT_BACK + 1))")
fi
rm -f chain.usage
env "${envs[@]}" COLI_USAGE=chain.usage USAGE_SAVE=0 DUMP=vk.f32 COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 \
COLI_VK_CHAIN=${CHAINMODE:-1} ./"$eng" "$@" > vk.log 2>&1 || true
mla_toks "$eng" cpu.log > cpu.tok; mla_toks "$eng" vk.log > vk.tok
{ [ -s cpu.tok ] && cmp -s cpu.tok vk.tok; } || { cat cpu.tok vk.tok; tail -20 vk.log; fail "$tag: the chain's tokens differ from the CPU"; }
if [ -n "${FAULT_BACK:-}" ]; then
grep -q "$eng chain: the device was lost" vk.log || { cat vk.log; fail "$tag: no loss was handled"; }
if [ "${REBUILD:-0}" = 1 ]; then grep -q "rebuilding the state of [1-9]" vk.log || { cat vk.log; fail "$tag: no state was rebuilt"; }; fi
else
[ "$(chain_count "$eng" vk.log)" -gt 0 ] || { cat vk.log; fail "$tag: the chain never ran"; }
fi
local lg=""
if [ "$tol" = 1 ]; then lg=$(logits_close cpu.f32 vk.f32) || { echo "$lg"; fail "$tag: logits"; }; lg=", $lg"; fi
echo "OK $tag: tokens = CPU$lg, $(chain_count "$eng" vk.log) chain forwards$(grep -o 'rebuilding the state of [0-9]* positions' vk.log | sed 's/^/, /')"
}
glm_chain_fixtures() {
glm_fixtures
$PY tools/make_glm_mtp_tiny.py --src glm_tiny --out glm_tiny_mtp > /dev/null && cp ref_glm.json glm_tiny_mtp/
rm -rf glm_tiny_serve && cp -r glm_tiny glm_tiny_serve
$PY tools/make_edge_tiny_tokenizer.py --vocab-size 256 ./glm_tiny_serve > /dev/null # serve speaks text
rm -rf glm53_serve && cp -r glm53_stream-i4 glm53_serve
$PY tools/make_edge_tiny_tokenizer.py --vocab-size 128 ./glm53_serve > /dev/null
$PY tools/make_glm53_multimodal_tiny.py --output glm53_mm_tiny > /dev/null
rm -rf glm53_lim0 && cp -r glm53_stream-i4 glm53_lim0 # swiglu_limit 0: the tier declines, the chain does not
$PY - <<'PY'
import json
p = "glm53_lim0/config.json"; c = json.load(open(p))
(c["text_config"] if "text_config" in c else c)["swiglu_limit"] = 0.0
json.dump(c, open(p, "w"))
PY
}
# Every configuration the chain takes: colibri in every expert format (IDOT=0 where the
# CPU would round activations to int8: the device matches IDOT=0, as for the tier),
# decode and teacher-forced prefill in chunks, the DSA indexer's selection active
# (DSA_TOPK=4 on the 12-token prompt, and DSA_FORCE), n-gram and MTP drafts accepted and
# rejected (the MTP fixture's head is the last layer's copy), the tier off, the per-matrix
# trunk and attention core beside it, prompts only, a device lost mid-decode and in a
# prompt, serve sessions (pins, the prompt cache, two KV slots, the prefill read-out);
# glm53 with f32, int8 and int4 trunks, streamed and resident experts, an image, prefill
# chunks and chain chunks, swiglu_limit 0, a lost device whose KDA state is rebuilt on
# the CPU, serve sessions, and its pin-branch harness with the chain on.
family_glm_chain() {
make colibri glm53 tests/test_vk_chain VK=1
./tests/test_vk_chain shaders/qmatmul.spv | tee vk_chain.log
tail -1 vk_chain.log | grep -qx PASS || fail "the chain's ops"
glm_chain_fixtures
export OMP_NUM_THREADS=2 CAP_RAISE=0
local cap b fx d
for cap in 1 64; do mla_gate colibri "chain colibri f32 cap=$cap" 1 SNAP=glm_tiny REF=ref_glm.json -- $cap 16 16; done
mla_gate colibri "chain colibri f32 prefill" 1 SNAP=glm_tiny REF=ref_glm.json TF=1 -- 64 16 16
mla_gate colibri "chain colibri prefill in chunks of 3" 1 SNAP=glm_tiny REF=ref_glm.json TF=1 COLI_VK_CHAIN_ROWS=3 -- 64 16 16
for b in 8 4 3; do
mla_gate colibri "chain colibri ${b}-bit trunk and experts" 1 SNAP=glm_tiny REF=ref_glm.json IDOT=0 -- 2 $b $b
done
mla_gate colibri "chain colibri int4-g64 experts" 1 SNAP=glm_tiny_fmt4 REF=glm_tiny_fmt4/ref_glm.json -- 2 16 16
mla_gate colibri "chain colibri E8/IQ3 experts on the CPU" 1 SNAP=glm_tiny_fmt6 REF=glm_tiny_fmt6/ref_glm.json -- 2 16 16
for fx in i4 i4r i3 d3; do
mla_gate colibri "chain colibri $fx container" 1 SNAP=glm_tiny_$fx REF=glm_tiny_$fx/ref_glm.json IDOT=0 -- 1 4 4
done
mla_gate colibri "chain colibri i4 container, prefill" 1 SNAP=glm_tiny_i4 REF=glm_tiny_i4/ref_glm.json IDOT=0 TF=1 -- 2 4 4
# the DSA indexer's selection on the device: index_topk 4 against a 32-token context
mla_gate colibri "chain colibri DSA top-4" 1 SNAP=glm_tiny REF=ref_glm.json DSA_TOPK=4 -- 64 16 16
mla_gate colibri "chain colibri DSA top-4, prefill in chunks of 5" 1 SNAP=glm_tiny REF=ref_glm.json DSA_TOPK=4 TF=1 COLI_VK_CHAIN_ROWS=5 -- 64 16 16
mla_gate colibri "chain colibri DSA_FORCE" 1 SNAP=glm_tiny REF=ref_glm.json DSA_FORCE=1 -- 64 16 16
# drafts: n-gram, and the MTP head at depths 1 to 3 (accepted and rejected)
mla_gate colibri "chain colibri n-gram drafts" 1 SNAP=glm_tiny REF=ref_glm.json DRAFT=3 -- 64 16 16
for d in 1 2 3; do mla_gate colibri "chain colibri MTP depth $d" 1 SNAP=glm_tiny_mtp REF=glm_tiny_mtp/ref_glm.json DRAFT=$d -- 64 16 16; done
mla_gate colibri "chain colibri MTP with DSA top-4" 1 SNAP=glm_tiny_mtp REF=glm_tiny_mtp/ref_glm.json DSA_TOPK=4 -- 64 16 16
mla_gate colibri "chain colibri MTP, 4-bit" 1 SNAP=glm_tiny_mtp REF=glm_tiny_mtp/ref_glm.json IDOT=0 DRAFT=2 -- 2 4 4
# around the chain: the tier off, the per-matrix trunk and core, the GEMM and GEMV choices
mla_gate colibri "chain colibri tier off" 1 SNAP=glm_tiny REF=ref_glm.json COLI_VK_TIER=0 -- 64 16 16
mla_gate colibri "chain colibri beside COLI_VK_DENSE=1 COLI_VK_ATTN=1" 1 SNAP=glm_tiny_i4 REF=glm_tiny_i4/ref_glm.json IDOT=0 COLI_VK_DENSE=1 COLI_VK_ATTN=1 -- 2 4 4
# COLI_VK_DEV2 beside the chain: a second logical device holds what the capped tier does not
SNAP=glm_tiny_i4r REF=glm_tiny_i4r/ref_glm.json IDOT=0 STATS=glm_tiny_i4r/.coli_usage ./colibri 64 4 4 > /dev/null 2>&1 || true
mla_gate colibri "chain colibri beside COLI_VK_DEV2" 1 SNAP=glm_tiny_i4r REF=glm_tiny_i4r/ref_glm.json IDOT=0 COLI_VK_EXPERTS=4 COLI_VK_DEV2=0 -- 64 4 4
grep -qaE 'lru \+ [1-9][0-9]* vk /' vk.log || { cat vk.log; fail "chain colibri beside COLI_VK_DEV2: no expert on the devices"; }
rm -f glm_tiny_i4r/.coli_usage
mla_gate colibri "chain colibri tiled GEMM from 2 rows" 1 SNAP=glm_tiny REF=ref_glm.json TF=1 COLI_VK_GEMM_MIN_S=2 -- 64 16 16
mla_gate colibri "chain colibri the per-row GEMV" 1 SNAP=glm_tiny REF=ref_glm.json COLI_VK_CHAIN_GEMV=0 -- 64 16 16
CHAINMODE=2 mla_gate colibri "chain colibri prompts only, drafts" 1 SNAP=glm_tiny REF=ref_glm.json DRAFT=3 -- 64 16 16
# the device lost: mid-decode, in the prompt (the teacher-forced pass is one forward),
# between MTP drafts
FAULT_BACK=12 mla_gate colibri "chain colibri device lost mid-decode" 1 SNAP=glm_tiny REF=ref_glm.json -- 64 16 16
FAULT_BACK=3 mla_gate colibri "chain colibri device lost in the prompt" 1 SNAP=glm_tiny REF=ref_glm.json TF=1 -- 64 16 16
FAULT_BACK=12 mla_gate colibri "chain colibri device lost with MTP" 1 SNAP=glm_tiny_mtp REF=glm_tiny_mtp/ref_glm.json -- 64 16 16
# serve sessions frame for frame: pins, the prompt cache, the prefill read-out, two KV slots
CHAIN_SERVE_DIALECT=colibri $PY tests/vulkan_chain_serve.py ./colibri glm_tiny_serve SERVE_BATCH=1 IDOT=0
CHAIN_SERVE_DIALECT=colibri $PY tests/vulkan_chain_serve.py ./colibri glm_tiny_serve SERVE_BATCH=1 IDOT=0 DSA_TOPK=4 DRAFT=3
CHAIN_SERVE_SLOTS=2 CHAIN_SERVE_DIALECT=colibri $PY tests/vulkan_chain_serve.py ./colibri glm_tiny_serve SERVE_BATCH=1 IDOT=0 KV_SLOTS=2
COLI_VK_CHAIN=2 CHAIN_SERVE_DIALECT=colibri $PY tests/vulkan_chain_serve.py ./colibri glm_tiny_serve SERVE_BATCH=1 IDOT=0 DSA_TOPK=4
# several conversations at once (KV_SLOTS' batched decode): each row over its own
# conversation's mirror and DSA list; one mirror beside the chain's (evictions, and the
# steps it cannot hold on the CPU); the device lost in a multiplexed step; MUX=0
$PY tests/vulkan_chain_mux.py ./colibri glm_tiny_serve 3 SERVE_BATCH=1 IDOT=0
CHAIN_MUX_LOG=mux.log $PY tests/vulkan_chain_mux.py ./colibri glm_tiny_serve 3 SERVE_BATCH=1 IDOT=0 DSA_TOPK=4
CHAIN_MUX_EXPECT='[1-9][0-9]* evicted' $PY tests/vulkan_chain_mux.py ./colibri glm_tiny_serve 4 SERVE_BATCH=1 IDOT=0 COLI_VK_CHAIN_MUX=1
local fr; fr=$(grep -o '[0-9]* frames' mux.log | head -1 | cut -d' ' -f1)
CHAIN_MUX_EXPECT='lost in a multiplexed step' $PY tests/vulkan_chain_mux.py ./colibri glm_tiny_serve 3 SERVE_BATCH=1 IDOT=0 DSA_TOPK=4 \
COLI_VK_CHAIN_FAULT=$((fr / 2))
CHAIN_MUX_STEPS=0 $PY tests/vulkan_chain_mux.py ./colibri glm_tiny_serve 3 SERVE_BATCH=1 IDOT=0 COLI_VK_CHAIN_MUX=0
rm -f mux.log chain-mux.usage
# glm53
local ids bits
ids=$($PY -c "print(','.join(str((i*37+5)%120+2) for i in range(100)))")
mla_gate glm53 "chain glm53 decode" 1 GLM53_BITS=32 -- --model glm53_stream-i4 --ids 5,7,9,11,13,17,19,23 --greedy 8
for bits in 32 8 4; do
mla_gate glm53 "chain glm53 100-token prompt, ${bits}-bit trunk" 1 GLM53_BITS=$bits -- --model glm53_stream-i4 --ids $ids --greedy 4
done
mla_gate glm53 "chain glm53 prefill chunks of 7, one cache slot" 1 GLM53_BITS=4 GLM53_PREFILL_CHUNK=7 GLM53_EXPERT_GB=0.000001 -- --model glm53_stream-i4 --ids $ids --greedy 4
mla_gate glm53 "chain glm53 chain chunks of 3" 1 GLM53_BITS=32 COLI_VK_CHAIN_ROWS=3 -- --model glm53_stream-i4 --ids $ids --greedy 4
mla_gate glm53 "chain glm53 resident experts" 1 GLM53_BITS=32 -- --model glm53_tiny --ids $ids --greedy 6
mla_gate glm53 "chain glm53 tier off" 1 GLM53_BITS=32 COLI_VK_TIER=0 -- --model glm53_stream-i4 --ids $ids --greedy 6
mla_gate glm53 "chain glm53 an image" 1 GLM53_BITS=32 -- --model glm53_mm_tiny --ids 103,117,268,268,268,268,120,121 --patches glm53_mm_tiny/patches.f32 --grid 4x4 --greedy 4
mla_gate glm53 "chain glm53 swiglu_limit 0" 1 GLM53_BITS=32 -- --model glm53_lim0 --ids $ids --greedy 4
grep -qa 'tier glm53: swiglu_limit is 0' vk.log || { cat vk.log; fail "chain glm53 swiglu_limit 0: the tier did not decline"; }
CHAINMODE=2 mla_gate glm53 "chain glm53 prompts only" 1 GLM53_BITS=32 GLM53_PREFILL_CHUNK=16 -- --model glm53_stream-i4 --ids $ids --greedy 6
FAULT_BACK=7 REBUILD=1 mla_gate glm53 "chain glm53 device lost, the KDA state rebuilt" 1 GLM53_BITS=32 GLM53_PREFILL_CHUNK=16 -- --model glm53_stream-i4 --ids $ids --greedy 6
FAULT_BACK=5 REBUILD=1 mla_gate glm53 "chain glm53 device lost after an image" 1 GLM53_BITS=32 -- --model glm53_mm_tiny --ids 103,117,268,268,268,268,120,121 --patches glm53_mm_tiny/patches.f32 --grid 4x4 --greedy 4
CHAIN_SERVE_DIALECT=numeric $PY tests/vulkan_chain_serve.py ./glm53 glm53_serve GLM53_BITS=32
CHAIN_SERVE_DIALECT=numeric $PY tests/vulkan_chain_serve.py ./glm53 glm53_serve GLM53_BITS=4 COLI_VK_CHAIN_ROWS=3
CHAIN_SERVE_SLOTS=2 CHAIN_SERVE_DIALECT=numeric $PY tests/vulkan_chain_serve.py ./glm53 glm53_serve GLM53_BITS=32 KV_SLOTS=2
COLI_VK_CHAIN=2 CHAIN_SERVE_DIALECT=numeric $PY tests/vulkan_chain_serve.py ./glm53 glm53_serve GLM53_BITS=32
# a pin restored over rows another branch rewrote: the KDA state goes up from the pin,
# the MLA rows' watermark comes down (the tier off and the scores within 1e-5: two
# engine processes, see vulkan_partial_glm.sh; the per-row GEMV for every prompt
# length, so a cold prompt and a short resumed tail take the same kernels)
COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_GEMM_MIN_S=0 COLI_USAGE=$PWD/chain.usage COLI_VK_TIER=0 $PY tests/glm53_pin_branch_harness.py --binary ./glm53 --fixture glm53_mm_tiny --tol 1e-5
unset OMP_NUM_THREADS CAP_RAISE
}
# The same chains under ASan and UBSan: memory safety is the gate; each run must have run
# the chain (or handled the loss). detect_stack_use_after_return=0 as in every sanitized
# family (ASan's fake stack breaks 64-byte aligned AVX-512 locals).
family_glm_chain_sanitize() {
make clean >/dev/null 2>&1 || true
make colibri glm53 tests/test_vk_chain VK=1 EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1
./tests/test_vk_chain shaders/qmatmul.spv > san.log 2>&1 || true
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log || ! tail -1 san.log | grep -qx PASS; then cat san.log; fail "asan: the chain's ops"; fi
echo "OK asan: the chain's ops"
glm_chain_fixtures
export OMP_NUM_THREADS=2 CAP_RAISE=0
msan() { # <engine> <tag> <env and argv...>
local eng=$1 tag=$2; shift 2
rm -f chain.usage
env COLI_USAGE=chain.usage USAGE_SAVE=0 COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 "$@" > san.log 2>&1 || true
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "$tag: sanitizer diagnostic"; fi
grep -q "$eng chain: \([1-9][0-9]* forwards\|the device was lost\)" san.log || { cat san.log; fail "$tag: the chain never ran"; }
echo "OK $tag: sanitizers clean, $(chain_count "$eng" san.log) chain forwards"
}
local ids
ids=$($PY -c "print(','.join(str((i*37+5)%120+2) for i in range(100)))")
msan colibri "asan chain colibri f32 decode" SNAP=glm_tiny REF=ref_glm.json ./colibri 1 16 16
msan colibri "asan chain colibri DSA top-4 prefill, chunks of 3" SNAP=glm_tiny REF=ref_glm.json TF=1 DSA_TOPK=4 COLI_VK_CHAIN_ROWS=3 ./colibri 64 16 16
msan colibri "asan chain colibri i4 container" SNAP=glm_tiny_i4 REF=glm_tiny_i4/ref_glm.json IDOT=0 ./colibri 2 4 4
msan colibri "asan chain colibri MTP depth 2" SNAP=glm_tiny_mtp REF=glm_tiny_mtp/ref_glm.json DRAFT=2 ./colibri 64 16 16
msan colibri "asan chain colibri device lost" SNAP=glm_tiny REF=ref_glm.json COLI_VK_CHAIN_FAULT=30 ./colibri 64 16 16
msan glm53 "asan chain glm53 decode" GLM53_BITS=32 ./glm53 --model glm53_stream-i4 --ids 5,7,9,11,13,17,19,23 --greedy 8
msan glm53 "asan chain glm53 int4 trunk, chain chunks of 3" GLM53_BITS=4 COLI_VK_CHAIN_ROWS=3 ./glm53 --model glm53_stream-i4 --ids $ids --greedy 4
msan glm53 "asan chain glm53 an image" GLM53_BITS=32 ./glm53 --model glm53_mm_tiny --ids 103,117,268,268,268,268,120,121 --patches glm53_mm_tiny/patches.f32 --grid 4x4 --greedy 4
msan glm53 "asan chain glm53 device lost, rebuilt" GLM53_BITS=32 GLM53_PREFILL_CHUNK=16 COLI_VK_CHAIN_FAULT=20 ./glm53 --model glm53_stream-i4 --ids $ids --greedy 6
local args
for args in "./colibri glm_tiny_serve SERVE_BATCH=1 IDOT=0 KV_SLOTS=2" "./glm53 glm53_serve GLM53_BITS=32 KV_SLOTS=2"; do
# shellcheck disable=SC2086
CHAIN_SERVE_SLOTS=2 CHAIN_SERVE_DIALECT=$([ "${args%% *}" = ./colibri ] && echo colibri || echo numeric) \
$PY tests/vulkan_chain_serve.py $args > san.log 2>&1 || { cat san.log; fail "asan chain serve $args"; }
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "asan chain serve $args: sanitizer diagnostic"; fi
echo "OK asan chain serve ${args%% *}: $(tail -1 san.log)"
done
for args in "3 SERVE_BATCH=1 IDOT=0 DSA_TOPK=4" "4 SERVE_BATCH=1 IDOT=0 COLI_VK_CHAIN_MUX=1"; do
# shellcheck disable=SC2086
$PY tests/vulkan_chain_mux.py ./colibri glm_tiny_serve $args > san.log 2>&1 || { cat san.log; fail "asan chain mux $args"; }
echo "OK asan chain mux: $(tail -1 san.log)"
done
unset OMP_NUM_THREADS CAP_RAISE
make clean >/dev/null 2>&1 || true
}
# Kimi K3's dense chain (kimi_k3_chain.h). The tolerance: the tiny fixture amplifies
# rounding at a few positions. The CPU against itself, with only its RMSNorm's sum taken
# in float instead of double, moves the logits by up to 1.4e-4 of the largest one on the
# f32 trunk and 1.0e-3 on the 8-bit one, and the served logprobs by up to 4.1e-3 (int4
# trunk), at the positions where the chain moves them most (measured on Lavapipe: 1.8e-4
# and 4.5e-4; logprobs 6.1e-3): every logits row is held within 2e-3 of the largest
# |logit|, the logprobs of the serve sessions within 2e-2, the tokens exactly. With the CPU's int8 expert activations
# (K3_IDOT=1) a rounding that flips an int8 step moves the logits further (8e-3, the
# tokens unchanged): that configuration gates on its tokens (TOKENS=1).
k3c_ids() { # case "ids": the prompt K3C_IDS holds (a prompt past the oracle's)
if [ "$1" = ids ] && [ -n "${K3C_IDS:-}" ]; then echo "$K3C_IDS"; return; fi
$PY -c "import json,sys;print(' '.join(map(str,json.load(open('kimi_k3_tiny/ref.json'))['cases'][sys.argv[1]]['prompt_ids'])))" "$1"
}
k3c_close() { # <cpu.f32> <vk.f32>: max |diff| within 2e-3 of the largest |logit|
$PY - "$1" "$2" <<'PY'
import array, sys
a = array.array("f", open(sys.argv[1], "rb").read()); b = array.array("f", open(sys.argv[2], "rb").read())
d = max(abs(x - y) for x, y in zip(a, b)) if a and len(a) == len(b) else float("inf")
m = max(abs(x) for x in a) if a else 0.0
print(f"max |logit diff| {d / m if m else d:.1e} of the largest")
sys.exit(0 if d <= 2e-3 * m else 1)
PY
}
# k3c_gate <tag> <env...>: for each of the oracle's prompts (K3C_CASES, default all
# three), the CPU run's tokens, every logits row (K3_VAL_LOGITS: every prefill row and
# every decode step) within k3c_close (TOKENS=1: reported, not gated), and the chain
# ran. EVICT=1: the tier evicted.
# FAULT_BACK=k: the device is lost k frames before the end of the same run without a
# fault (counted from that run's frames, so on any device the loss lands in the same
# forward), and the run must say so; REBUILD=1: and rebuild the KDA state of some
# positions on the CPU.
k3c_gate() {
local tag=$1 c frames fault; shift
for c in ${K3C_CASES:-short chunk long}; do
rm -f k3c.usage cpu.f32 vk.f32
env "$@" COLI_USAGE=$PWD/k3c.usage USAGE_SAVE=0 K3_VAL_LOGITS=cpu.f32 ./kimi_k3 kimi_k3_tiny --ids "$(k3c_ids $c)" --ngen 8 \
2> cpu.log | sed 's/ *TUNE.*//' > cpu.tok
fault=""
if [ -n "${FAULT_BACK:-}" ]; then
env "$@" COLI_USAGE=$PWD/k3c.usage USAGE_SAVE=0 COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 \
./kimi_k3 kimi_k3_tiny --ids "$(k3c_ids $c)" --ngen 8 2> vk.log > /dev/null
frames=$(sed -n 's/^\[VK\] kimi_k3 chain: [0-9]* forwards, \([0-9]*\) frames.*/\1/p' vk.log | tail -1)
[ -n "$frames" ] && [ "$frames" -gt "$FAULT_BACK" ] || { cat vk.log; fail "$tag $c: no fault-free run to count frames from"; }
fault="COLI_VK_CHAIN_FAULT=$((frames - FAULT_BACK + 1))"
fi
env "$@" $fault COLI_USAGE=$PWD/k3c.usage USAGE_SAVE=0 K3_VAL_LOGITS=vk.f32 COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 \
COLI_VK_CHAIN=${CHAINMODE:-1} ./kimi_k3 kimi_k3_tiny --ids "$(k3c_ids $c)" --ngen 8 2> vk.log | sed 's/ *TUNE.*//' > vk.tok
{ [ -s cpu.tok ] && cmp -s cpu.tok vk.tok; } || { cat cpu.tok vk.tok; tail -20 vk.log; fail "$tag $c: the chain's tokens differ from the CPU"; }
if [ -n "$fault" ]; then
grep -q "kimi_k3 chain: the device was lost" vk.log || { cat vk.log; fail "$tag $c: no loss was handled"; }
if [ "${REBUILD:-0}" = 1 ]; then grep -q "rebuilding the state of [1-9]" vk.log || { cat vk.log; fail "$tag $c: no state was rebuilt"; }; fi
else
[ "$(chain_count kimi_k3 vk.log)" -gt 0 ] || { cat vk.log; fail "$tag $c: the chain never ran"; }
fi
if [ "${EVICT:-0}" = 1 ]; then
[ "$(tier_evictions kimi_k3 vk.log)" -gt 0 ] || { grep '\[VK\] tier' vk.log; fail "$tag $c: the budget forced no eviction"; }
grep -q ' failed 0 ' vk.log || { grep '\[VK\] tier' vk.log; fail "$tag $c: an upload failed"; }
fi
local lg; lg=$(k3c_close cpu.f32 vk.f32) || { echo "$lg"; [ "${TOKENS:-0}" = 1 ] || fail "$tag $c: logits"; }
echo "OK $tag $c: tokens = CPU, $lg, $(chain_count kimi_k3 vk.log) chain forwards$(grep -o 'rebuilding the state of [0-9]* positions\|the host.s state is current' vk.log | sed 's/^/, /')"
done
}
k3c_serve_fixture() { # the tiny fixture with the tokenizer the serve tests speak through
rm -rf kimi_k3_serve && mkdir kimi_k3_serve
cp kimi_k3_tiny/config.json kimi_k3_tiny/model.safetensors kimi_k3_serve/
cp tests/tok_kimi_tiny.json kimi_k3_serve/tokenizer.json
}
# Every configuration the chain takes: Moonshot's oracle with the chain on (the tier on,
# off, the shared experts on the device), every dense format (f32, int8 rows, int4-g64,
# mixed), the CPU's int8 expert activations with the tier off, prefill a token at a
# time and in chain chunks of 3, the tiled GEMM and the per-row GEMV, K3_TOPP, a tier
# budget that evicts, prompts only (the KDA state crossing between the device and the
# CPU at every decode), a device lost mid-decode (the KDA state rebuilt on the CPU), in
# the first prompt chunk (the host's state current) and in a later one (rebuilt); serve
# sessions frame for frame (pins, the prompt cache, prefix reuse, a prompt that diverges
# and one that starts over, the prefill read-out), with recurrent-state checkpoints
# (COLI_K3_CKPT: photos taken from the device's state and restored to it), in chain
# chunks, on prompts only, with a device lost mid-session; the checkpoint and dashboard
# harnesses with the chain on.
family_kimi_chain() {
make kimi_k3 tests/test_vk_chain VK=1
./tests/test_vk_chain shaders/qmatmul.spv | tee vk_chain.log
tail -1 vk_chain.log | grep -qx PASS || fail "the chain's ops"
$PY tools/make_kimi_k3_tiny.py --output ./kimi_k3_tiny --force
k3c_serve_fixture
export OMP_NUM_THREADS=2
local e b out frames
for e in COLI_VK_TIER=1 COLI_VK_TIER=0 COLI_VK_DENSE=1; do
env $e COLI_USAGE=$PWD/k3c.usage USAGE_SAVE=0 COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 \
$PY tests/test_kimi_k3_tiny.py --binary ./kimi_k3 --fixture ./kimi_k3_tiny
echo "OK chain kimi_k3 vendor oracle ($e)"
done
local O="K3_BITS=32 K3_MLA_BITS=32 K3_HEAD_BITS=32 K3_IDOT=0 COLI_TEMP=0"
k3c_gate "chain kimi_k3 f32" $O
k3c_gate "chain kimi_k3 f32, tier off" $O COLI_VK_TIER=0
k3c_gate "chain kimi_k3 f32, shared experts by COLI_VK_DENSE=1" $O COLI_VK_DENSE=1
for b in 8 4; do # int8 rows (fmt 1); int4-g64 (fmt 4) where a row is whole groups, int8 elsewhere
k3c_gate "chain kimi_k3 ${b}-bit trunk" K3_BITS=$b K3_MLA_BITS=$b K3_HEAD_BITS=$b K3_IDOT=0 COLI_TEMP=0
done
k3c_gate "chain kimi_k3 int4 KDA and MoE, int8 MLA and head" K3_BITS=4 K3_IDOT=0 COLI_TEMP=0
TOKENS=1 k3c_gate "chain kimi_k3 int8 expert activations, tier off" K3_BITS=32 K3_MLA_BITS=32 K3_HEAD_BITS=32 COLI_TEMP=0 COLI_VK_TIER=0
k3c_gate "chain kimi_k3 prefill one token at a time" $O K3_CHUNK=1
k3c_gate "chain kimi_k3 chain chunks of 3" $O COLI_VK_CHAIN_ROWS=3
k3c_gate "chain kimi_k3 tiled GEMM from 2 rows" $O COLI_VK_GEMM_MIN_S=2
k3c_gate "chain kimi_k3 the per-row GEMV" $O COLI_VK_CHAIN_GEMV=0
k3c_gate "chain kimi_k3 K3_TOPP=0.6" $O K3_TOPP=0.6
EVICT=1 k3c_gate "chain kimi_k3 a tier budget of two experts" $O COLI_VK_TIER_GB=0.000005
CHAINMODE=2 k3c_gate "chain kimi_k3 prompts only" $O
# the device lost: 9 frames a forward on this fixture (two per sparse layer and the
# head); the long prompt is three forwards (32, 32 and 8 rows), then 7 decode steps
FAULT_BACK=3 REBUILD=1 k3c_gate "chain kimi_k3 device lost mid-decode" $O
K3C_CASES=long FAULT_BACK=86 k3c_gate "chain kimi_k3 device lost in the first prompt chunk" $O K3_CHUNK=32
K3C_CASES=long FAULT_BACK=77 REBUILD=1 k3c_gate "chain kimi_k3 device lost in a later prompt chunk" $O K3_CHUNK=32
# in chain chunks of 5 rows (seven chunks a 32-row forward), the loss in the third
# chunk of the second forward: the state its first chunks advanced is not used
K3C_CASES=long FAULT_BACK=124 REBUILD=1 k3c_gate "chain kimi_k3 device lost inside a chunked forward" $O COLI_VK_CHAIN_ROWS=5
# serve sessions frame for frame, the KDA state across turns
local S="K3_BITS=32 K3_MLA_BITS=32 K3_HEAD_BITS=32 K3_IDOT=0 K3_PREFIX_LOG=1 USAGE_SAVE=0"
export CHAIN_SERVE_TOL=2e-2
rm -rf k3c_photos && mkdir k3c_photos
# shellcheck disable=SC2086
{
$PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve $S
out=$($PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve $S COLI_K3_CKPT=4); echo "$out"
$PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve $S COLI_K3_CKPT=2 COLI_K3_CKPT_DIR=$PWD/k3c_photos
$PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve K3_BITS=4 K3_IDOT=0 K3_PREFIX_LOG=1 USAGE_SAVE=0 COLI_VK_CHAIN_ROWS=3
COLI_VK_CHAIN=2 $PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve $S COLI_K3_CKPT=4
# a device lost mid-session, 134 frames before the end of the same session without a
# fault (on Lavapipe frame 200 of 334: after a photo brought the state to the host,
# the state of the 3 positions since then is rebuilt from it)
frames=$(echo "$out" | sed -n 's/.*kimi_k3 chain: [0-9]* forwards, \([0-9]*\) frames.*/\1/p')
[ -n "$frames" ] && [ "$frames" -gt 134 ] || fail "chain kimi_k3 serve: no fault-free session to count frames from"
CHAIN_SERVE_EXPECT='kimi_k3 chain: the device was lost' \
$PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve $S COLI_K3_CKPT=4 COLI_VK_CHAIN_FAULT=$((frames - 134))
}
unset CHAIN_SERVE_TOL
# recurrent-state checkpoints and the dashboard's lines with the chain on
COLI_USAGE=$PWD/k3c.usage USAGE_SAVE=0 COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 \
$PY tests/test_kimi_k3_ckpt.py --binary ./kimi_k3 --fixture ./kimi_k3_tiny
COLI_USAGE=$PWD/k3c.usage USAGE_SAVE=0 COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 \
$PY tests/test_kimi_k3_dashboard.py --binary ./kimi_k3 --fixture ./kimi_k3_tiny
unset OMP_NUM_THREADS
}
# The same chain under ASan and UBSan: memory safety is the gate; each run must have
# run the chain (or handled the loss).
family_kimi_chain_sanitize() {
make clean >/dev/null 2>&1 || true
make kimi_k3 tests/test_vk_chain VK=1 EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1
./tests/test_vk_chain shaders/qmatmul.spv > san.log 2>&1 || true
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log || ! tail -1 san.log | grep -qx PASS; then cat san.log; fail "asan: the chain's ops"; fi
echo "OK asan: the chain's ops"
$PY tools/make_kimi_k3_tiny.py --output ./kimi_k3_tiny --force
k3c_serve_fixture
export OMP_NUM_THREADS=2
k3san() { # <tag> <env and argv...>
local tag=$1; shift
rm -f k3c.usage
env COLI_USAGE=$PWD/k3c.usage USAGE_SAVE=0 COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 "$@" > san.log 2>&1 || true
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "$tag: sanitizer diagnostic"; fi
grep -q "kimi_k3 chain: \([1-9][0-9]* forwards\|the device was lost\)" san.log || { cat san.log; fail "$tag: the chain never ran"; }
echo "OK $tag: sanitizers clean, $(chain_count kimi_k3 san.log) chain forwards"
}
local O="K3_BITS=32 K3_MLA_BITS=32 K3_HEAD_BITS=32 K3_IDOT=0 COLI_TEMP=0" ids frames
ids=$(k3c_ids long)
# shellcheck disable=SC2086
{
k3san "asan chain kimi_k3 f32" env $O ./kimi_k3 kimi_k3_tiny --ids "$ids" --ngen 8
frames=$(sed -n 's/^\[VK\] kimi_k3 chain: [0-9]* forwards, \([0-9]*\) frames.*/\1/p' san.log | tail -1)
k3san "asan chain kimi_k3 int4 trunk, chain chunks of 3" env K3_BITS=4 K3_IDOT=0 COLI_VK_CHAIN_ROWS=3 ./kimi_k3 kimi_k3_tiny --ids "$ids" --ngen 8
k3san "asan chain kimi_k3 tier off, every logits row" env $O COLI_VK_TIER=0 K3_VAL_LOGITS=san.f32 ./kimi_k3 kimi_k3_tiny --ids "$ids" --ngen 8
k3san "asan chain kimi_k3 prompts only" env $O COLI_VK_CHAIN=2 ./kimi_k3 kimi_k3_tiny --ids "$ids" --ngen 8
# the device lost mid-decode: 3 frames before the end of the run above (a fixed frame
# number can land in the buffers' setup on a device whose memory is not mapped)
k3san "asan chain kimi_k3 device lost, rebuilt" env $O COLI_VK_CHAIN_FAULT=$((frames - 2)) ./kimi_k3 kimi_k3_tiny --ids "$ids" --ngen 8
grep -q "rebuilding the state of [1-9]" san.log || { cat san.log; fail "asan chain kimi_k3 device lost: no state was rebuilt"; }
}
CHAIN_SERVE_TOL=2e-2 $PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve K3_BITS=32 K3_MLA_BITS=32 K3_HEAD_BITS=32 K3_IDOT=0 \
K3_PREFIX_LOG=1 USAGE_SAVE=0 COLI_K3_CKPT=4 > san.log 2>&1 || { cat san.log; fail "asan chain kimi_k3 serve"; }
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "asan chain kimi_k3 serve: sanitizer diagnostic"; fi
echo "OK asan chain kimi_k3 serve: $(tail -1 san.log)"
unset OMP_NUM_THREADS
make clean >/dev/null 2>&1 || true
}
# DeepSeek V4.1 Flash (deepseek_v41) and DeepSeek V4 (deepseek_v4) on the dense chain
# (COLI_VK_CHAIN=1). v41_gate runs the CPU and then the chain with the same settings:
# the same exit code (the engine fails on a token off its reference), the same printed
# token stream, every logits row (DUMP) within 1e-4 of the largest, the chain ran (or,
# with FAULT_BACK=k, the device was lost k frames before the end of a fault-free run and
# the CPU took over).
v41_gate() { # <tag> <env...> -- <argv...>
local tag=$1; shift
local envs=(); while [ "$1" != "--" ]; do envs+=("$1"); shift; done; shift
local rc_cpu=0 rc_vk=0
rm -f chain.usage cpu.f32 vk.f32
env "${envs[@]}" DUMP=cpu.f32 ./deepseek_v41 "$@" > cpu.txt 2> cpu.log || rc_cpu=$?
if [ -n "${FAULT_BACK:-}" ]; then
env "${envs[@]}" COLI_USAGE=chain.usage COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 \
./deepseek_v41 "$@" > /dev/null 2> vk.log || true
local frames; frames=$(sed -n 's/^\[VK\] deepseek_v41 chain: [0-9]* forwards, \([0-9]*\) frames.*/\1/p' vk.log | tail -1)
[ -n "$frames" ] && [ "$frames" -gt "$FAULT_BACK" ] || { cat vk.log; fail "$tag: no fault-free run to count frames from"; }
envs+=("COLI_VK_CHAIN_FAULT=$((frames - FAULT_BACK + 1))")
rm -f chain.usage
fi
env "${envs[@]}" DUMP=vk.f32 COLI_USAGE=chain.usage COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 \
COLI_VK_CHAIN=${CHAINMODE:-1} ./deepseek_v41 "$@" > vk.txt 2> vk.log || rc_vk=$?
[ "$rc_cpu" = "$rc_vk" ] || { tail -20 vk.log; fail "$tag: exit $rc_vk, the CPU's $rc_cpu"; }
{ [ -s cpu.txt ] && cmp -s cpu.txt vk.txt; } || { cat cpu.txt vk.txt; tail -20 vk.log; fail "$tag: the chain's tokens differ from the CPU"; }
if [ -n "${FAULT_BACK:-}" ]; then
grep -q "deepseek_v41 chain: the device was lost" vk.log || { cat vk.log; fail "$tag: no loss was handled"; }
else
[ "$(chain_count deepseek_v41 vk.log)" -gt 0 ] || { cat vk.log; fail "$tag: the chain never ran"; }
fi
local lg; lg=$(logits_close cpu.f32 vk.f32) || { echo "$lg"; fail "$tag: logits"; }
echo "OK $tag: tokens = CPU (exit $rc_cpu), $lg, $(chain_count deepseek_v41 vk.log) chain forwards$(grep -q 'the device was lost' vk.log && echo ', the device lost and the CPU on')"
}
v41_chain_fixtures() {
$PY tools/make_dsv41_tiny.py --out dsv41_tiny --emit-ref dsv41_tiny/ref.json > /dev/null
$PY tools/make_dsv41_tiny.py --out dsv41_long --emit-ref dsv41_long/ref.json --prompt-len 40 --max-new 6 > /dev/null
}
# DeepSeek V4 (deepseek_v4) on the dense chain: the deepseek-chain family's V4 half. The
# functions use $PY, fail and chain_count from tests/vulkan_engines.sh and run from c/.
#
# v4_chain_gate <tag> <fixture> <case> <env...>: the CPU run (no COLI_VULKAN) and the
# chain run of one oracle case of the fixture (or ids:a,b,...:n, those ids and n new
# tokens), with the same settings and --record-oracle: the generated ids equal to the
# CPU's (and to the reference's greedy stream), every logits row (DUMP) within
# V4_CHAIN_TOL (5e-1) of its largest |logit|, the same draft attempts and acceptances as
# the CPU (V4_DRAFT), the chain ran, and, on the chain throughout, each decode row equal
# bit for bit to the teacher-forced row at its position. The teacher-forced argmaxes that
# moved are counted, not gated (the logits bound already covers them). FAULT_BACK=k: the
# device is lost k frames before the end of a fault-free run (counted without
# --record-oracle) and the CPU takes over; CHAINMODE=2: prompts only. Each run starts
# from no expert history (the fixture's .coli_usage removed), the tier's uploads awaited.
# Why 0.5: DeepSeek V4 rounds to bf16 after nearly every step and the input of every fp8
# matrix to E4M3 per block; a value the device's sums put on the other side of a bf16
# boundary can move a whole E4M3 block by a step and the indexer's top-k to another row.
# Measured worst on Lavapipe and a Radeon 780M: 0.32 (deepseek_v4_tiny_g2, long). The
# check that no driver blurs is v4_chain_same (and the decode-row check above).
v4_chain_prompt() { # <fixture> <case>: the case's prompt as the tiny vocabulary's text (ids:a,b,...: those ids)
$PY -c 'import json,sys; c=sys.argv[2]; ids=[int(t) for t in c[4:].split(":")[0].split(",")] if c.startswith("ids:") else json.load(open(sys.argv[1]+"/ref.json"))["cases"][c]["prompt_ids"]; print("".join("<t%03d>" % t for t in ids))' "$1" "$2"
}
v4_chain_logits() { # <cpu.f32> <vk.f32> <vocab>: the worst row's |diff| over its largest |logit|
$PY - "$1" "$2" "$3" "${V4_CHAIN_TOL:-5e-1}" <<'PY'
import array, sys
a = array.array("f", open(sys.argv[1], "rb").read()); b = array.array("f", open(sys.argv[2], "rb").read())
V, tol = int(sys.argv[3]), float(sys.argv[4])
if not a or len(a) != len(b): print(f"logits: {len(a)} and {len(b)} values"); sys.exit(1)
worst, same = 0.0, 0
for r in range(len(a) // V):
ra, rb = a[r * V:(r + 1) * V], b[r * V:(r + 1) * V]
d = max(abs(x - y) for x, y in zip(ra, rb)); m = max(abs(x) for x in ra)
same += d == 0
worst = max(worst, d / m if m else d)
print(f"logits: {same} of {len(a) // V} rows identical, worst row {worst:.2e} of its largest")
sys.exit(0 if worst <= tol else 1)
PY
}
# v4_chain_same <tag> <fixture> <case> <A> <B> <env...>: the chain twice, with the settings
# A and then B (space-separated KEY=VALUE lists) on top of env: the same ids and every
# logits row the same bits. The chain's arithmetic does not depend on how the rows are
# cut (every matrix takes the per-row GEMV, every other op is per row or in the CPU's
# row order) nor on which side of the tier computed an expert, so chunks of 1 or 3, the
# tier off or a draft's rows cut differently give the default run's bits; a state the
# device kept stale across forwards (a rejected draft longer than a chunk) shows here
# even where its logits stay within the CPU's tolerance.
v4_chain_same() {
local tag=$1 fx=$2 c=$3 A=$4 B=$5; shift 5
local p mt side
p=$(v4_chain_prompt "$fx" "$c")
mt=$($PY -c 'import json,sys; c=sys.argv[2]; print(c.split(":")[2] if c.startswith("ids:") else json.load(open(sys.argv[1]+"/ref.json"))["cases"][c]["max_new_tokens"])' "$fx" "$c")
for side in a b; do
rm -f "$fx/.coli_usage" "v4-$side.f32"
# shellcheck disable=SC2046
env "$@" $([ $side = a ] && echo "$A" || echo "$B") DUMP=v4-$side.f32 COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 \
./deepseek_v4 "./$fx" "$p" --raw-prompt --max-tokens "$mt" --record-oracle v4-$side.json > /dev/null 2> v4-$side.err ||
{ tail -20 v4-$side.err; fail "$tag: run $side"; }
[ "$(chain_count deepseek_v4 v4-$side.err)" -gt 0 ] || { cat v4-$side.err; fail "$tag: the chain never ran ($side)"; }
done
rm -f "$fx/.coli_usage"
cmp -s v4-a.f32 v4-b.f32 && $PY -c 'import json,sys; sys.exit(json.load(open("v4-a.json"))["full_ids"] != json.load(open("v4-b.json"))["full_ids"])' ||
{ v4_chain_logits v4-a.f32 v4-b.f32 "$($PY -c 'import json,sys; print(json.load(open(sys.argv[1]+"/config.json"))["vocab_size"])' "$fx")"; fail "$tag: $A and $B give different bits"; }
echo "OK $tag (${c%%:*}): $A and $B: the same ids and the same logits bit for bit ($(($(wc -c < v4-a.f32) / 4)) values), $(chain_count deepseek_v4 v4-b.err) chain forwards"
}
v4_chain_gate() {
local tag=$1 fx=$2 c=$3; shift 3
local p mt lg frames fault=()
p=$(v4_chain_prompt "$fx" "$c")
mt=$($PY -c 'import json,sys; c=sys.argv[2]; print(c.split(":")[2] if c.startswith("ids:") else json.load(open(sys.argv[1]+"/ref.json"))["cases"][c]["max_new_tokens"])' "$fx" "$c")
rm -f "$fx/.coli_usage" cpu.f32 vk.f32 v4-cpu.json v4-vk.json
env "$@" DUMP=cpu.f32 ./deepseek_v4 "./$fx" "$p" --raw-prompt --max-tokens "$mt" --record-oracle v4-cpu.json > /dev/null 2> v4-cpu.err ||
{ cat v4-cpu.err; fail "$tag: CPU run"; }
if [ -n "${FAULT_BACK:-}" ]; then
rm -f "$fx/.coli_usage"
env "$@" COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 ./deepseek_v4 "./$fx" "$p" --raw-prompt --max-tokens "$mt" > /dev/null 2> v4-vk.err || true
frames=$(sed -n 's/^\[VK\] deepseek_v4 chain: [0-9]* forwards, \([0-9]*\) frames.*/\1/p' v4-vk.err | tail -1)
[ -n "$frames" ] && [ "$frames" -gt "$FAULT_BACK" ] || { cat v4-vk.err; fail "$tag: no fault-free run to count frames from"; }
fault=("COLI_VK_CHAIN_FAULT=$((frames - FAULT_BACK + 1))")
elif [ -n "${FAULT2_BACK:-}" ]; then # the second device's frames (vulkan_layers_dev2.sh); 0: its first, the setup
fault=("COLI_VK_CHAIN_FAULT2=1")
if [ "$FAULT2_BACK" -gt 0 ]; then
rm -f "$fx/.coli_usage"
env "$@" COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 ./deepseek_v4 "./$fx" "$p" --raw-prompt --max-tokens "$mt" > /dev/null 2> v4-vk.err || true
frames=$(sed -n 's/^\[VK\] deepseek_v4 dev2 chain: [0-9]* forwards, \([0-9]*\) frames.*/\1/p' v4-vk.err | tail -1)
[ -n "$frames" ] && [ "$frames" -gt "$FAULT2_BACK" ] || { cat v4-vk.err; fail "$tag: no fault-free run to count frames from"; }
fault=("COLI_VK_CHAIN_FAULT2=$((frames - FAULT2_BACK + 1))")
fi
fi
rm -f "$fx/.coli_usage"
env "$@" "${fault[@]}" DUMP=vk.f32 COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=${CHAINMODE:-1} \
./deepseek_v4 "./$fx" "$p" --raw-prompt --max-tokens "$mt" --record-oracle v4-vk.json > /dev/null 2> v4-vk.err ||
{ tail -20 v4-vk.err; fail "$tag: chain run"; }
rm -f "$fx/.coli_usage"
$PY - "$fx" "$c" <<'PY' || { tail -20 v4-vk.err; fail "$tag: the chain's session differs from the CPU's"; }
import json, sys
a, b = json.load(open("v4-cpu.json")), json.load(open("v4-vk.json"))
ref = a["full_ids"] if sys.argv[2].startswith("ids:") else json.load(open(sys.argv[1] + "/ref.json"))["cases"][sys.argv[2]]["greedy_full_ids"]
ok = a["full_ids"] == b["full_ids"] == ref and len(a["tf_pred"]) == len(b["tf_pred"])
if not ok: print("CPU", a["full_ids"], "chain", b["full_ids"], "ref", ref)
open("v4-tf.txt", "w").write(str(sum(x != y for x, y in zip(a["tf_pred"], b["tf_pred"]))))
sys.exit(0 if ok else 1)
PY
if [ "${EVICT:-0}" = 1 ]; then # the budget forced evictions
[ "$(tier_evictions deepseek_v4 v4-vk.err)" -gt 0 ] || { grep '\[VK\] tier' v4-vk.err; fail "$tag: the budget forced no eviction"; }
fi
# drafts (V4_DRAFT): the same attempts, drafted and accepted tokens as the CPU's
[ "$(grep -ao 'v4_dspark attempts=[0-9]* drafted=[0-9]* accepted=[0-9]*' v4-cpu.err)" = \
"$(grep -ao 'v4_dspark attempts=[0-9]* drafted=[0-9]* accepted=[0-9]*' v4-vk.err)" ] || { grep -a v4_dspark v4-cpu.err v4-vk.err; fail "$tag: the drafts went otherwise"; }
if [ -n "${FAULT_BACK:-}" ]; then
grep -q "deepseek_v4 chain: the device was lost" v4-vk.err || { cat v4-vk.err; fail "$tag: no loss was handled"; }
elif [ -n "${FAULT2_BACK:-}" ]; then
grep -q "deepseek_v4 dev2 chain: \(0 of [0-9]* layers on the device: the chain stays off (layer 0 did not reach the device: \)\?the device was lost" v4-vk.err ||
{ cat v4-vk.err; fail "$tag: the second device's loss was not taken over"; }
else
[ "$(chain_count deepseek_v4 v4-vk.err)" -gt 0 ] || { cat v4-vk.err; fail "$tag: the chain never ran"; }
fi
lg=$(v4_chain_logits cpu.f32 vk.f32 "$($PY -c 'import json,sys; print(json.load(open(sys.argv[1]+"/config.json"))["vocab_size"])' "$fx")") ||
{ echo "$lg"; fail "$tag: logits"; }
# an oracle case on the chain throughout: its decode rows are the teacher-forced
# forward's rows at the same positions, bit for bit (as on the CPU)
if [ "${c#ids:}" = "$c" ] && [ "${CHAINMODE:-1}" = 1 ] && [ -z "${FAULT_BACK:-}${FAULT2_BACK:-}" ] && [ "${*#*V4_DRAFT}" = "$*" ]; then
$PY - vk.f32 "$fx" "$c" <<'PY' || fail "$tag: a decode row differs from the teacher-forced row at its position"
import array, json, sys
v = array.array("f", open(sys.argv[1], "rb").read())
V = json.load(open(sys.argv[2] + "/config.json"))["vocab_size"]
case = json.load(open(sys.argv[2] + "/ref.json"))["cases"][sys.argv[3]]
P, mt = len(case["prompt_ids"]), case["max_new_tokens"]
row = lambda r: v[r * V:(r + 1) * V]
for j in range(mt): # session rows: the prompt's last position, then each decode step
if row(j) != row(mt + P - 1 + j): sys.exit(1)
PY
lg="$lg, decode rows = teacher-forced rows"
fi
echo "OK $tag (${c%%:*}): ids = CPU$(case $c in ids:*) ;; *) echo ' = reference';; esac), $(cat v4-tf.txt) teacher-forced argmaxes moved, $lg, $(chain_count deepseek_v4 v4-vk.err) chain forwards$(grep -q 'the device was lost' v4-vk.err && echo ', the device lost and the CPU on')$(grep -ao 'v4_dspark attempts=[0-9]* drafted=[0-9]* accepted=[0-9]*' v4-vk.err | tail -1 | sed 's/^/, /')$([ "${EVICT:-0}" = 1 ] && grep -ao 'evictions [0-9]*' v4-vk.err | tail -1 | sed 's/^/, /')"
}
# v4_chain_fixtures: the 4-expert fixture (a copy), the 8-expert one (pinned rows16
# experts), and three more geometries: 8 heads in 2 output groups (wo_a per group), a
# window of 4 with ratios 4 and 2 (the window and the rings roll over more often), and
# DeepSeek V4's indexer of 64 heads of 128 (the scores past the shared staging).
v4_chain_fixtures() {
$PY tools/make_deepseek_v4_tiny.py --output deepseek_v4_tiny_t --force > /dev/null
local spec
for spec in "deepseek_v4_tiny_e8 EXPERTS=8" "deepseek_v4_tiny_g2 HEADS=8 O_GROUPS=2" \
"deepseek_v4_tiny_w4 SLIDING=4 COMPRESS_RATIOS=[0,4,2] HCA=2" "deepseek_v4_tiny_ix INDEX_HEADS=64 INDEX_DIM=128"; do
# shellcheck disable=SC2086
$PY - tools/make_deepseek_v4_tiny.py $spec <<'PY' > /dev/null
import importlib.util, sys
spec = importlib.util.spec_from_file_location("gen", sys.argv[1])
gen = importlib.util.module_from_spec(spec); spec.loader.exec_module(gen)
for kv in sys.argv[3:]:
k, v = kv.split("=", 1)
if k != "HCA": setattr(gen, k, eval(v)); continue
hf = gen.make_hf_config # the heavily compressed layers' ratio
def patched(C, hf=hf, r=int(v)):
cfg = hf(C); cfg.compress_rates["heavily_compressed_attention"] = r; return cfg
gen.make_hf_config = patched
sys.argv = ["make_deepseek_v4_tiny.py", "--output", sys.argv[2], "--force"]
gen.main()
PY
done
}
# v4_chain_serve: the brio served prompt's per-position logprob echoes with the chain
# against the CPU's (pinned, then a prompt that extends the pin, then max_tokens 0),
# and the prefix-reuse contract (tests/test_deepseek_v4_prefix.py) with the chain on.
v4_chain_serve() { # <fixture>
$PY - "$1" <<'PY' || fail "deepseek_v4 chain served: the chain's echoes differ from the CPU's"
import json, os, sys
from pathlib import Path
sys.path.insert(0, "tests")
import test_deepseek_v4_brio as b
fx = Path(sys.argv[1])
case = json.load(open(fx / "ref.json"))["cases"]["long"]
ids = case["prompt_ids"]
def turns(chain):
for k in ("COLI_VULKAN", "COLI_VK_CHAIN", "COLI_VK_TIER_SYNC"): os.environ.pop(k, None)
try: os.remove(fx / ".coli_usage")
except FileNotFoundError: pass
if chain: os.environ.update(COLI_VULKAN="1", COLI_VK_CHAIN=os.environ.get("V4_CHAIN_MODE", "1"), COLI_VK_TIER_SYNC="1")
s = b.Serve(Path("deepseek_v4").resolve(), fx)
out = []
try:
out.append(s.submit(b.token_prompt(ids[:40]), 3, logprobs=4, pin=True))
out.append(s.submit(b.token_prompt(ids[:40] + ids[40:60]), 4, logprobs=4))
out.append(s.submit(b.token_prompt(ids), 0, logprobs=5))
out.append(s.submit(b.token_prompt(ids[:30]), 4))
finally:
s.close()
err = s.process.stderr.read().decode(errors="replace")
return out, err
cpu, err_cpu = turns(False)
dev, err = turns(True)
for text in (err_cpu, err): # a sanitized build reports into the engine's stderr
if "ERROR: AddressSanitizer" in text or "runtime error:" in text: print(text[-4000:]); sys.exit("sanitizer diagnostic")
if "deepseek_v4 chain:" not in err or " forwards" not in err:
print(err[-3000:]); sys.exit("the chain never ran")
import re
if os.environ.get("V4_SERVE_EXPECT") and not re.search(os.environ["V4_SERVE_EXPECT"], err):
print(err[-3000:]); sys.exit("no line matches V4_SERVE_EXPECT")
worst, n, same = 0.0, 0, 0
for x, y in zip(cpu, dev):
if [d[0] for d in x.data] != [d[0] for d in y.data] or sorted(x.echoes) != sorted(y.echoes) or x.reuse != y.reuse:
print("CPU", x.data, x.echoes.keys(), x.reuse, "chain", y.data, y.echoes.keys(), y.reuse); sys.exit(1)
for p in x.echoes:
n += 1; same += x.echoes[p] == y.echoes[p]
if x.echoes[p]["token"] != y.echoes[p]["token"]: sys.exit(1)
worst = max(worst, abs(x.echoes[p]["lp"] - y.echoes[p]["lp"]))
fw = [l for l in err.splitlines() if "deepseek_v4 chain:" in l and "forwards" in l][-1]
print(f"OK deepseek_v4 chain served: the pin, its extension, a read-only prompt and a shorter one: texts and reuse = CPU, "
f"{same} of {n} logprob echoes identical, worst |delta| {worst:.2e}; {fw.split('] ', 1)[1][:48]}")
sys.exit(0 if worst <= 0.5 else 1)
PY
}
# v4_chain_san <tag> <env and argv...>: a sanitized build's chain run (FAULT_BACK=k: the
# loss k frames before the end of a fault-free run): no sanitizer diagnostic, and the
# chain ran (or handled the loss).
v4_chain_san() {
local tag=$1; shift
local fault=() frames
if [ -n "${FAULT_BACK:-}" ]; then
rm -f deepseek_v4_tiny_*/.coli_usage
env COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 "$@" > san.log 2>&1 || true
frames=$(sed -n 's/^\[VK\] deepseek_v4 chain: [0-9]* forwards, \([0-9]*\) frames.*/\1/p' san.log | tail -1)
[ -n "$frames" ] && [ "$frames" -gt "$FAULT_BACK" ] || { cat san.log; fail "$tag: no fault-free run to count frames from"; }
fault=("COLI_VK_CHAIN_FAULT=$((frames - FAULT_BACK + 1))")
fi
rm -f deepseek_v4_tiny_*/.coli_usage
env COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 "${fault[@]}" "$@" > san.log 2>&1 || true
rm -f deepseek_v4_tiny_*/.coli_usage
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "$tag: sanitizer diagnostic"; fi
grep -q "deepseek_v4 chain: \([1-9][0-9]* forwards\|the device was lost\)" san.log || { cat san.log; fail "$tag: the chain never ran"; }
echo "OK $tag: sanitizers clean, $(chain_count deepseek_v4 san.log) chain forwards$(grep -q 'the device was lost' san.log && echo ', the device lost')"
}
# The V4 half of the family: the engine, its fixtures, then every configuration: 4 and 8
# experts (pinned rows16), the three oracle cases (the long one rolls the window of 8 and
# both compressors over many times), 2 output groups, a window of 4 with ratios 4 and 2,
# V4's indexer of 64 heads of 128, eviction, the tier off, the per-matrix trunk beside,
# prefill and chain chunks, the per-row GEMV, V4_IDX_IDENTITY, prompts only, n-gram
# drafts accepted and rejected (V4_MTP on a one-layer MTP checkpoint runs target-only),
# the chain against itself, a device lost mid-decode, in the prompt and between drafts,
# served turns (a pin, its extension, a read-only prompt), and the engine's own serve
# tests with the chain on.
v4_chain_family() {
make deepseek-v4 VK=1
v4_chain_fixtures
local c fx
for c in short compressed long; do
v4_chain_gate "chain deepseek_v4 4 experts" deepseek_v4_tiny_t $c
v4_chain_gate "chain deepseek_v4 8 experts, pinned rows16" deepseek_v4_tiny_e8 $c
done
for c in short long; do
v4_chain_gate "chain deepseek_v4 2 output groups" deepseek_v4_tiny_g2 $c
v4_chain_gate "chain deepseek_v4 a window of 4, ratios 4 and 2" deepseek_v4_tiny_w4 $c
v4_chain_gate "chain deepseek_v4 an indexer of 64 heads of 128" deepseek_v4_tiny_ix $c
done
EVICT=1 v4_chain_gate "chain deepseek_v4 a budget of three experts" deepseek_v4_tiny_e8 long COLI_VK_TIER_GB=0.00009
v4_chain_gate "chain deepseek_v4 tier off" deepseek_v4_tiny_t long COLI_VK_TIER=0 COLI_VK_DENSE=0
v4_chain_gate "chain deepseek_v4 beside the per-matrix trunk" deepseek_v4_tiny_t long COLI_VK_DENSE=1
v4_chain_gate "chain deepseek_v4 prefill chunks of 7" deepseek_v4_tiny_t long V4_PREFILL_CHUNK=7
v4_chain_gate "chain deepseek_v4 chain chunks of 3" deepseek_v4_tiny_e8 long COLI_VK_CHAIN_ROWS=3
v4_chain_gate "chain deepseek_v4 chunks of 5, a window of 4" deepseek_v4_tiny_w4 long COLI_VK_CHAIN_ROWS=5
v4_chain_gate "chain deepseek_v4 the per-row GEMV" deepseek_v4_tiny_t long COLI_VK_CHAIN_GEMV=0
v4_chain_gate "chain deepseek_v4 V4_IDX_IDENTITY=1" deepseek_v4_tiny_t long V4_IDX_IDENTITY=1
CHAINMODE=2 v4_chain_gate "chain deepseek_v4 prompts only" deepseek_v4_tiny_t long
# n-gram drafts on prompts whose tails repeat: accepted and rejected (A), every one
# rejected (R), a mix over three attempts (M)
local A=ids:20,21,22,23,24,25,26,27,28,29,20,21,22:12 R=ids:20,21,22,23,50,51,52,20,21,22:12
local M=ids:30,31,32,33,34,35,60,61,30,31,32,33,34,35,36,37,38,39,40,41,30,31,32:12
v4_chain_gate "chain deepseek_v4 n-gram drafts accepted and rejected" deepseek_v4_tiny_t $A V4_DRAFT=4 V4_NGRAM_PARTIAL_KEEP=1
v4_chain_gate "chain deepseek_v4 an n-gram draft rejected" deepseek_v4_tiny_t $R V4_DRAFT=4 V4_NGRAM_PARTIAL_KEEP=1
v4_chain_gate "chain deepseek_v4 n-gram drafts, a window of 4" deepseek_v4_tiny_w4 $M V4_DRAFT=4 V4_NGRAM_PARTIAL_KEEP=1
v4_chain_gate "chain deepseek_v4 n-gram drafts, 8 experts" deepseek_v4_tiny_e8 $M V4_DRAFT=4 V4_NGRAM_PARTIAL_KEEP=1
CHAINMODE=2 v4_chain_gate "chain deepseek_v4 prompts only, drafts" deepseek_v4_tiny_w4 $M V4_DRAFT=4 V4_NGRAM_PARTIAL_KEEP=1
v4_chain_gate "chain deepseek_v4 V4_MTP=1 (one MTP layer: target only)" deepseek_v4_tiny_t long V4_MTP=1 V4_DRAFT=3
# the chain against itself
for fx in deepseek_v4_tiny_t deepseek_v4_tiny_w4 deepseek_v4_tiny_g2; do
v4_chain_same "chain deepseek_v4 chunks, $fx" $fx long "X=1" "COLI_VK_CHAIN_ROWS=1"
v4_chain_same "chain deepseek_v4 prefill chunks, $fx" $fx long "X=1" "V4_PREFILL_CHUNK=3"
done
v4_chain_same "chain deepseek_v4 the tier off" deepseek_v4_tiny_e8 long "X=1" "COLI_VK_TIER=0 COLI_VK_DENSE=0"
v4_chain_same "chain deepseek_v4 a draft rejected, chunks of 2" deepseek_v4_tiny_w4 $R "X=1" "COLI_VK_CHAIN_ROWS=2" V4_DRAFT=4 V4_NGRAM_PARTIAL_KEEP=1
v4_chain_same "chain deepseek_v4 drafts, chunks of 1" deepseek_v4_tiny_w4 $M "X=1" "COLI_VK_CHAIN_ROWS=1" V4_DRAFT=4 V4_NGRAM_PARTIAL_KEEP=1
v4_chain_same "chain deepseek_v4 drafts, chunks of 3, 8 experts" deepseek_v4_tiny_e8 $A "X=1" "COLI_VK_CHAIN_ROWS=3" V4_DRAFT=4 V4_NGRAM_PARTIAL_KEEP=1
FAULT_BACK=3 v4_chain_gate "chain deepseek_v4 device lost mid-decode" deepseek_v4_tiny_t long
FAULT_BACK=40 v4_chain_gate "chain deepseek_v4 device lost in the prompt" deepseek_v4_tiny_t long COLI_VK_CHAIN_ROWS=7
FAULT_BACK=10 v4_chain_gate "chain deepseek_v4 device lost between drafts" deepseek_v4_tiny_w4 $M V4_DRAFT=4 V4_NGRAM_PARTIAL_KEEP=1
v4_chain_serve deepseek_v4_tiny_t
V4_CHAIN_MODE=2 v4_chain_serve deepseek_v4_tiny_t
local t
for t in test_deepseek_v4_prefix test_deepseek_v4_brio test_deepseek_v4_tiny; do
COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 COLI_USAGE=$PWD/chain.usage \
$PY tests/$t.py --binary "$PWD/deepseek_v4" --fixture "$PWD/deepseek_v4_tiny_t" > v4-test.log 2>&1 ||
{ cat v4-test.log; fail "deepseek_v4 $t with the chain"; }
echo "OK deepseek_v4 $t with the chain: $(tail -1 v4-test.log)"
done
rm -rf deepseek_v4_tiny_t deepseek_v4_tiny_e8 deepseek_v4_tiny_g2 deepseek_v4_tiny_w4 deepseek_v4_tiny_ix chain.usage
rm -f cpu.f32 vk.f32 v4-*.f32 v4-*.json v4-*.err v4-tf.txt v4-test.log
}
# The same under ASan and UBSan (LTO off, as family_deepseek_sanitize builds it): no
# diagnostic, and each run ran the chain (or handled the loss); the serve tests with
# UBSan halting on its first report.
v4_chain_sanitize_runs() {
local p t
p=$(v4_chain_prompt deepseek_v4_tiny_t long)
mk() { v4_chain_prompt x "ids:$1"; }
v4_chain_san "asan chain deepseek_v4 long, 4 experts" ./deepseek_v4 ./deepseek_v4_tiny_t "$p" --raw-prompt --max-tokens 4 --record-oracle san.json
v4_chain_san "asan chain deepseek_v4 8 experts, pinned rows16, eviction" COLI_VK_TIER_GB=0.00009 ./deepseek_v4 ./deepseek_v4_tiny_e8 "$p" --raw-prompt --max-tokens 4 --record-oracle san.json
v4_chain_san "asan chain deepseek_v4 chunks of 3, a window of 4" COLI_VK_CHAIN_ROWS=3 ./deepseek_v4 ./deepseek_v4_tiny_w4 "$(v4_chain_prompt deepseek_v4_tiny_w4 long)" --raw-prompt --max-tokens 4 --record-oracle san.json
v4_chain_san "asan chain deepseek_v4 2 output groups" ./deepseek_v4 ./deepseek_v4_tiny_g2 "$(v4_chain_prompt deepseek_v4_tiny_g2 long)" --raw-prompt --max-tokens 4
v4_chain_san "asan chain deepseek_v4 an indexer of 64 heads of 128" ./deepseek_v4 ./deepseek_v4_tiny_ix "$(v4_chain_prompt deepseek_v4_tiny_ix long)" --raw-prompt --max-tokens 4
v4_chain_san "asan chain deepseek_v4 drafts rejected, chunks of 2" V4_DRAFT=4 V4_NGRAM_PARTIAL_KEEP=1 COLI_VK_CHAIN_ROWS=2 ./deepseek_v4 ./deepseek_v4_tiny_w4 "$(mk 20,21,22,23,50,51,52,20,21,22)" --raw-prompt --max-tokens 12
v4_chain_san "asan chain deepseek_v4 drafts" V4_DRAFT=4 V4_NGRAM_PARTIAL_KEEP=1 ./deepseek_v4 ./deepseek_v4_tiny_w4 "$(mk 30,31,32,33,34,35,60,61,30,31,32,33,34,35,36,37,38,39,40,41,30,31,32)" --raw-prompt --max-tokens 12
FAULT_BACK=10 v4_chain_san "asan chain deepseek_v4 device lost" ./deepseek_v4 ./deepseek_v4_tiny_t "$p" --raw-prompt --max-tokens 4
v4_chain_serve deepseek_v4_tiny_t > san.log 2>&1 || { cat san.log; fail "asan chain deepseek_v4 served"; }
echo "OK asan chain deepseek_v4 served: $(tail -1 san.log | cut -c1-120)"
for t in test_deepseek_v4_prefix test_deepseek_v4_brio; do
UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 $PY tests/$t.py --binary $PWD/deepseek_v4 --fixture $PWD/deepseek_v4_tiny_t > san.log 2>&1 || { cat san.log; fail "asan $t with the chain"; }
echo "OK asan $t with the chain: $(tail -1 san.log)"
done
}
v4_chain_family_sanitize() {
local SAN="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
make deepseek-v4-clean >/dev/null 2>&1 || true
make deepseek-v4 VK=1 LTO=0 EXTRA_CFLAGS="$SAN" EXTRA_LDFLAGS="$SAN"
v4_chain_fixtures
export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1
v4_chain_sanitize_runs
rm -rf deepseek_v4_tiny_t deepseek_v4_tiny_e8 deepseek_v4_tiny_g2 deepseek_v4_tiny_w4 deepseek_v4_tiny_ix san.json san.log
make deepseek-v4-clean >/dev/null 2>&1 || true
}
# deepseek_v41 in every configuration the chain takes: the expert cache from one slot
# to all (cap 1, 2, 8), the 8-token and the 40-token prompt (the window ring of 8 and
# the ratio-2 groups roll over many times; the candidate blocks and the published index
# keys of the ratio-1 group), DSpark drafts accepted (1, 3) and rejected (2, 4, 5:
# undo rows on the device's copies), prompts in chunks of 3 and 7 (the device's ring
# wraps), prompts only (decode on the CPU between them), the tier off, the per-matrix
# trunk beside, the tiled GEMM from two rows, the per-row GEMV, V41_INDEX_OWNER, a
# device lost mid-decode, in a prompt and between drafts; serve sessions frame for
# frame (pins, the prompt cache, the prefill read-out, DSpark, prompts only), images
# on the wire, and the engine's own serve tests with the chain on. Then deepseek_v4
# (v4_chain_family above).
family_deepseek_chain() {
make deepseek_v41 tests/test_vk_chain VK=1
./tests/test_vk_chain shaders/qmatmul.spv | tee vk_chain.log
tail -1 vk_chain.log | grep -qx PASS || fail "the chain's ops"
v41_chain_fixtures
export OMP_NUM_THREADS=2
local cap f
for cap in 1 2 8; do v41_gate "chain deepseek_v41 cap=$cap" SNAP=dsv41_tiny -- $cap dsv41_tiny/ref.json; done
for cap in 2 8; do v41_gate "chain deepseek_v41 40-token prompt cap=$cap" SNAP=dsv41_long -- $cap dsv41_long/ref.json; done
for f in 1 2 3 4 5; do v41_gate "chain deepseek_v41 DSpark spec=$f" SNAP=dsv41_tiny V41_DSPARK=1 V41_SPEC_FORCE=$f -- 8 dsv41_tiny/ref.json; done
v41_gate "chain deepseek_v41 DSpark spec=5, one cache slot" SNAP=dsv41_tiny V41_DSPARK=1 V41_SPEC_FORCE=5 -- 1 dsv41_tiny/ref.json
v41_gate "chain deepseek_v41 prompt in chunks of 3" SNAP=dsv41_long COLI_VK_CHAIN_ROWS=3 -- 8 dsv41_long/ref.json
v41_gate "chain deepseek_v41 prompt in chunks of 7" SNAP=dsv41_long COLI_VK_CHAIN_ROWS=7 -- 2 dsv41_long/ref.json
CHAINMODE=2 v41_gate "chain deepseek_v41 prompts only" SNAP=dsv41_long -- 8 dsv41_long/ref.json
CHAINMODE=2 v41_gate "chain deepseek_v41 prompts only, DSpark spec=2" SNAP=dsv41_tiny V41_DSPARK=1 V41_SPEC_FORCE=2 -- 8 dsv41_tiny/ref.json
v41_gate "chain deepseek_v41 tier off" SNAP=dsv41_long COLI_VK_TIER=0 -- 8 dsv41_long/ref.json
v41_gate "chain deepseek_v41 beside the per-matrix trunk" SNAP=dsv41_long COLI_VK_DENSE=1 -- 8 dsv41_long/ref.json
v41_gate "chain deepseek_v41 tiled GEMM from 2 rows" SNAP=dsv41_long COLI_VK_GEMM_MIN_S=2 -- 8 dsv41_long/ref.json
v41_gate "chain deepseek_v41 the per-row GEMV" SNAP=dsv41_tiny COLI_VK_CHAIN_GEMV=0 -- 8 dsv41_tiny/ref.json
v41_gate "chain deepseek_v41 V41_INDEX_OWNER=1" SNAP=dsv41_long V41_INDEX_OWNER=1 -- 8 dsv41_long/ref.json
FAULT_BACK=5 v41_gate "chain deepseek_v41 device lost mid-decode" SNAP=dsv41_long -- 8 dsv41_long/ref.json
FAULT_BACK=100 v41_gate "chain deepseek_v41 device lost in the prompt" SNAP=dsv41_long COLI_VK_CHAIN_ROWS=7 -- 8 dsv41_long/ref.json
FAULT_BACK=8 v41_gate "chain deepseek_v41 device lost between drafts" SNAP=dsv41_tiny V41_DSPARK=1 V41_SPEC_FORCE=5 -- 8 dsv41_tiny/ref.json
$PY tests/vulkan_chain_serve.py ./deepseek_v41 dsv41_tiny V41_DSPARK=0
$PY tests/vulkan_chain_serve.py ./deepseek_v41 dsv41_tiny V41_DSPARK=1
COLI_VK_CHAIN=2 $PY tests/vulkan_chain_serve.py ./deepseek_v41 dsv41_tiny V41_DSPARK=1 COLI_VK_CHAIN_ROWS=3
$PY tests/vulkan_chain_v41_image.py ./deepseek_v41 dsv41_tiny V41_DSPARK=0
COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 COLI_USAGE=$PWD/chain.usage \
$PY -m unittest tests.test_dsv41_prefix_serve tests.test_dsv41_dspark_serve
rm -f chain.usage chain-serve.usage chain-image.usage
v4_chain_family
unset OMP_NUM_THREADS
}
# The same chain under ASan and UBSan: memory safety is the gate; each run must have run
# the chain (or handled the loss), with the ASAN_OPTIONS of every sanitized family.
family_deepseek_chain_sanitize() {
local SAN="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
make clean >/dev/null 2>&1 || true
make deepseek_v41 tests/test_vk_chain VK=1 EXTRA_CFLAGS="$SAN"
export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1
./tests/test_vk_chain shaders/qmatmul.spv > san.log 2>&1 || true
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log || ! tail -1 san.log | grep -qx PASS; then cat san.log; fail "asan: the chain's ops"; fi
echo "OK asan: the chain's ops"
v41_chain_fixtures
export OMP_NUM_THREADS=2
dsan() { # <tag> <env and argv...> (FAULT_BACK=k: the loss k frames before the end of a fault-free run)
local tag=$1; shift
local fault=()
if [ -n "${FAULT_BACK:-}" ]; then
rm -f chain.usage
env COLI_USAGE=chain.usage COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 "$@" > san.log 2>&1 || true
local frames; frames=$(sed -n 's/^\[VK\] deepseek_v41 chain: [0-9]* forwards, \([0-9]*\) frames.*/\1/p' san.log | tail -1)
[ -n "$frames" ] && [ "$frames" -gt "$FAULT_BACK" ] || { cat san.log; fail "$tag: no fault-free run to count frames from"; }
fault=("COLI_VK_CHAIN_FAULT=$((frames - FAULT_BACK + 1))")
fi
rm -f chain.usage
env COLI_USAGE=chain.usage COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 "${fault[@]}" "$@" > san.log 2>&1 || true
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "$tag: sanitizer diagnostic"; fi
grep -q "deepseek_v41 chain: \([1-9][0-9]* forwards\|the device was lost\)" san.log || { cat san.log; fail "$tag: the chain never ran"; }
echo "OK $tag: sanitizers clean, $(chain_count deepseek_v41 san.log) chain forwards"
}
dsan "asan chain deepseek_v41 40-token prompt" SNAP=dsv41_long ./deepseek_v41 8 dsv41_long/ref.json
dsan "asan chain deepseek_v41 DSpark spec=5, one cache slot" SNAP=dsv41_tiny V41_DSPARK=1 V41_SPEC_FORCE=5 ./deepseek_v41 1 dsv41_tiny/ref.json
dsan "asan chain deepseek_v41 prompt in chunks of 3" SNAP=dsv41_long COLI_VK_CHAIN_ROWS=3 ./deepseek_v41 2 dsv41_long/ref.json
FAULT_BACK=20 dsan "asan chain deepseek_v41 device lost" SNAP=dsv41_long ./deepseek_v41 8 dsv41_long/ref.json
$PY tests/vulkan_chain_serve.py ./deepseek_v41 dsv41_tiny V41_DSPARK=1 > san.log 2>&1 || { cat san.log; fail "asan chain deepseek_v41 serve"; }
echo "OK asan chain serve deepseek_v41: $(tail -1 san.log)"
$PY tests/vulkan_chain_v41_image.py ./deepseek_v41 dsv41_tiny V41_DSPARK=0 > san.log 2>&1 || { cat san.log; fail "asan chain deepseek_v41 images"; }
echo "OK asan chain images deepseek_v41: $(tail -1 san.log)"
rm -f chain.usage chain-serve.usage chain-image.usage
v4_chain_family_sanitize
unset OMP_NUM_THREADS
make clean >/dev/null 2>&1 || true
}
# ---- the dense weights on the device only (COLI_VK_DENSE_HOST=0; docs/vulkan.md,
# "Dense weights on the device only"): one file per engine group,
# tests/vulkan_dense_only_<group>.sh, each defining dho_family_<group> and
# dho_family_<group>_sanitize (run as dense-only-<group> and dense-only-<group>-sanitize).
#
# dho_dropped <log>: matrices the engine placed on the device only (its startup line)
dho_dropped() {
local n; n=$(sed -n 's/^\[VK\] [a-z0-9_]*: \([0-9][0-9]*\) dense matrices on the device only.*/\1/p' "$1" | tail -1)
echo "${n:-0}"
}
# dho_reloaded <log>: host copies read back from disk for the CPU (the exit line)
dho_reloaded() {
local n; n=$(sed -n 's/^\[VK\] [a-z0-9_]*: dense weights at exit: .*), \([0-9][0-9]*\) read back from disk.*/\1/p' "$1" | tail -1)
echo "${n:--1}"
}
# dho_rss <log>: the resident set the exit line reports, in GiB
dho_rss() { sed -n 's/^\[VK\] [a-z0-9_]*: dense weights at exit: .*; RSS \([0-9.]*\) GiB$/\1/p' "$1" | tail -1; }
# dho_gate <engine> <tag> <tol 0|1|2> <env...> -- <argv...>: one configuration three
# times, on the CPU, on the device keeping its host copies (COLI_VK_DENSE_HOST=1) and on
# the device only (=0), the tier deterministic (COLI_VK_TIER_SYNC=1 COLI_VK_TIER_BALANCE=0)
# and each run from a fresh history. Gates: the CPU's tokens; matrices dropped and none read
# back; with the chain asked for (CHAINMODE, default 1; 0 = the per-matrix path alone with
# COLI_VK_DENSE=1), chain forwards; tol 2: the logits the bytes of the host-copy run (what
# runs on the device is the same), 1: within 1e-4 of the CPU's largest logit (logits_close),
# 0: tokens only. CPUENV: the CPU arm's own settings; DUMPENV: the variable the engine
# writes its logits to (default DUMP).
dho_gate() {
local eng=$1 tag=$2 tol=$3; shift 3
local envs=(); while [ "$1" != "--" ]; do envs+=("$1"); shift; done; shift
local dv=${DUMPENV:-DUMP} mode=${CHAINMODE:-1}
local vk=(COLI_VK_TIER_SYNC=1 COLI_VK_TIER_BALANCE=0 COLI_VULKAN=1 COLI_VK_CHAIN=$mode)
[ "$mode" = 0 ] && vk+=(COLI_VK_DENSE=1)
rm -f chain.usage cpu.f32 vk.f32 host.f32
env "${envs[@]}" ${CPUENV:-} $dv=cpu.f32 ./"$eng" "$@" > cpu.log 2>&1 || true
env "${envs[@]}" "${vk[@]}" COLI_USAGE=chain.usage COLI_VK_DENSE_HOST=1 $dv=host.f32 ./"$eng" "$@" > host.log 2>&1 || true
rm -f chain.usage
env "${envs[@]}" "${vk[@]}" COLI_USAGE=chain.usage COLI_VK_DENSE_HOST=0 $dv=vk.f32 ./"$eng" "$@" > vk.log 2>&1 || true
rm -f chain.usage
same_tokens cpu.log vk.log "$tag"
same_tokens cpu.log host.log "$tag (host copies)"
local n; n=$(dho_dropped vk.log)
[ "$n" -gt 0 ] || { cat vk.log; fail "$tag: no dense matrix went to the device only"; }
[ "$(dho_reloaded vk.log)" = 0 ] || { grep '^\[VK\]' vk.log; fail "$tag: a matrix was read back from disk without a lost device"; }
if [ "$mode" != 0 ]; then [ "$(chain_count "$eng" vk.log)" -gt 0 ] || { cat vk.log; fail "$tag: the chain did not stay on"; }; fi
local lg=""
case $tol in
2) cmp -s host.f32 vk.f32 || fail "$tag: the logits differ from the run that keeps its host copies"; lg=", logits = the host-copy run's" ;;
1) lg=$(logits_close cpu.f32 vk.f32) || { echo "$lg"; fail "$tag: logits"; }; lg=", $lg" ;;
esac
echo "OK $tag: tokens = CPU$lg, $n matrices on the device only, $(chain_count "$eng" vk.log) chain forwards"
}
# dho_lost <engine> <tag> <frames back> <env...> -- <argv...>: device only, the device lost
# `back` frames before the end of a run (COLI_VK_CHAIN_FAULT, counted on a probe run): the
# CPU's tokens, and the matrices the CPU then needed read back from disk.
dho_lost() {
local eng=$1 tag=$2 back=$3; shift 3
local envs=(); while [ "$1" != "--" ]; do envs+=("$1"); shift; done; shift
local vk=(COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_DENSE_HOST=0)
rm -f chain.usage
env "${envs[@]}" ./"$eng" "$@" > cpu.log 2>&1 || true
env "${envs[@]}" "${vk[@]}" COLI_USAGE=chain.usage ./"$eng" "$@" > vk.log 2>&1 || true
local f; f=$(sed -n "s/^\[VK\] $eng chain: [0-9]* forwards, \([0-9]*\) frames.*/\1/p" vk.log | tail -1)
{ [ -n "$f" ] && [ "$f" -gt "$back" ]; } || { cat vk.log; fail "$tag: the probe run reports no frames"; }
rm -f chain.usage
env "${envs[@]}" "${vk[@]}" COLI_VK_CHAIN_FAULT=$((f - back)) COLI_USAGE=chain.usage ./"$eng" "$@" > vk.log 2>&1 || true
rm -f chain.usage
same_tokens cpu.log vk.log "$tag"
local r; r=$(dho_reloaded vk.log)
[ "$r" -gt 0 ] || { grep '^\[VK\]' vk.log; fail "$tag: nothing was read back for the CPU"; }
echo "OK $tag: tokens = CPU after the loss, $r matrices read back from disk"
}
# dho_rss_gate <engine> <tag> <MiB at least> <env...> -- <argv...>: the resident set at
# exit falls by most of what was dropped (the device copies are host memory on a CPU
# device, the host copies are not there twice): RSS(host copies) - RSS(device only) is at
# least the given MiB.
dho_rss_gate() {
local eng=$1 tag=$2 mib=$3; shift 3
local envs=(); while [ "$1" != "--" ]; do envs+=("$1"); shift; done; shift
local vk=(COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=${CHAINMODE:-1})
rm -f chain.usage
env "${envs[@]}" "${vk[@]}" COLI_USAGE=chain.usage COLI_VK_DENSE_HOST=1 ./"$eng" "$@" > host.log 2>&1 || true
rm -f chain.usage
env "${envs[@]}" "${vk[@]}" COLI_USAGE=chain.usage COLI_VK_DENSE_HOST=0 ./"$eng" "$@" > vk.log 2>&1 || true
rm -f chain.usage
local a b; a=$(dho_rss host.log); b=$(dho_rss vk.log)
{ [ -n "$a" ] && [ -n "$b" ]; } || { grep '^\[VK\]' host.log vk.log; fail "$tag: no exit line with the resident set"; }
$PY - "$a" "$b" "$mib" <<'PY' || { grep '^\[VK\].*dense' host.log vk.log; fail "$tag: the resident set did not fall"; }
import sys
a, b, mib = float(sys.argv[1]), float(sys.argv[2]), float(sys.argv[3])
print(f"RSS at exit {a:.2f} GiB with the host copies, {b:.2f} GiB on the device only")
sys.exit(0 if (a - b) * 1024 >= mib else 1)
PY
echo "OK $tag: $(grep -o 'given back.*' vk.log | head -1)"
}
for f in tests/vulkan_dense_only_*.sh; do [ -e "$f" ] && . "$f"; done
# ---- the partial chain (docs/vulkan.md, "A partial chain"): the first N layers on the
# device, the rest and the head on the CPU. One file per engine group,
# tests/vulkan_partial_<group>.sh, each defining ptl_family_<group> and
# ptl_family_<group>_sanitize (run as partial-<group> and partial-<group>-sanitize);
# tests/vulkan_partial_deepseek.sh is the template. The helpers read the engine's lines:
# [VK] <engine> chain fit: free F B, reserve R B, fixed X B (...), tail T B, layers b0 .. B, matrices m0 .. B
# [VK] <engine> chain: N of L layers on the device ... (the last one is the final N)
# [VK] <engine> chain: N of L layers placed: M B of matrices on the device (the fit counted M' B ...), ...
#
# ptl_n <engine> <log>: the final N (-1 without a line)
ptl_n() {
local n; n=$(sed -n "s/^\[VK\] $1 chain: \([0-9][0-9]*\) of [0-9][0-9]* layers on the device.*/\1/p" "$2" | tail -1)
echo "${n:--1}"
}
# ptl_L <engine> <log>: L, the model's layers, from the same line
ptl_L() {
local n; n=$(sed -n "s/^\[VK\] $1 chain: [0-9][0-9]* of \([0-9][0-9]*\) layers on the device.*/\1/p" "$2" | tail -1)
echo "${n:--1}"
}
# ptl_calc <what> <engine> <log> [k]: arithmetic on the fit line (a probe run's for "cap")
# predict N as vkc_fit's rule gives it from the line's numbers
# cap the COLI_VK_DEVICE_CAP_MB (MiB, a fraction) under which exactly k layers fit:
# the bytes the probe held at the fit (its cap PTL_PROBE_CAP_MB less its free),
# the reserve, the engine's fixed bytes, the pools' granularity at the new cap,
# k layers, and half of layer k (k < L) or the tail too (k = L). Probe with the
# same settings and a cap of 256 or less: up to 256 MiB the pools' blocks (and so
# what is held at the fit) do not depend on the cap
# fault the COLI_VK_STAGED_FAULT count to aim inside layer k's setup: the second time
# the point is reached within it (the placed line's marks; a first one there may
# be a fresh block's zero fill, which only skips the fill)
# matrices M and M' of the placed line, "M M'"
ptl_calc() {
$PY - "$@" <<'PY'
import os, re, sys
what, eng, log = sys.argv[1], sys.argv[2], sys.argv[3]
k = int(sys.argv[4]) if len(sys.argv) > 4 else 0
lines = open(log, errors="replace").read().splitlines()
fit = [l for l in lines if l.startswith(f"[VK] {eng} chain fit: ")]
if what in ("predict", "cap"):
if not fit: sys.exit("no fit line")
f = fit[-1]
num = lambda key: int(re.search(key + r" (\d+) B", f)[1])
free, res, fixed, tail = num("free"), num("reserve"), num("fixed"), num("tail")
layers = [int(x) for x in re.search(r"layers((?: \d+)*) B", f)[1].split()]
L, room = len(layers), max(0, free - res)
if what == "predict":
if fixed + sum(layers) <= room: print(L); sys.exit(0)
n, acc = 0, fixed
while n < L and acc + layers[n] <= room: acc += layers[n]; n += 1
print(n); sys.exit(0)
# the pools' part of fixed depends on the cap (coli_vk_block_bytes): recompute it
def blk(cap, d):
b = 64 << 10
while b < cap // 4096: b <<= 1
return min(b, d)
pools = lambda cap: blk(cap, 256 << 20) + 3 * blk(cap, 64 << 20) + 4 * (4 << 20)
engine = int(re.search(r"the engine's (\d+) B", f)[1])
held = int(float(os.environ["PTL_PROBE_CAP_MB"]) * 1048576) - free
cap = held + free
for _ in range(4):
cap = held + res + engine + pools(cap) + sum(layers[:k]) + (layers[k] // 2 if k < L else tail + 4096)
print(f"{cap / 1048576:.6f}")
elif what == "fault":
p = [l for l in lines if re.match(rf"\[VK\] {eng} chain: \d+ of \d+ layers placed", l)]
m = re.search(r"reached ([\d,]+) times", p[-1] if p else "")
if not m: sys.exit("no marks on the placed line")
c = [int(x) for x in m[1].split(",")]
lo = c[k - 1] if k else 0
if c[k] - lo < 2: sys.exit(f"layer {k} reaches the point {c[k] - lo} times")
print(lo + 2)
elif what == "matrices":
p = [l for l in lines if re.match(rf"\[VK\] {eng} chain: \d+ of \d+ layers placed", l)]
m = re.search(r"placed: (\d+) B of matrices on the device \(the fit counted (\d+) B", p[-1] if p else "")
if not m: sys.exit("no placed line")
print(m[1], m[2])
PY
}
# ptl_check_n <engine> <log> <want> <tag>: the final N is want, and the line says L
ptl_check_n() {
local n; n=$(ptl_n "$1" "$2")
[ "$n" = "$3" ] || { grep -E "^\[VK\] $1 chain" "$2"; fail "$4: N is $n, not $3"; }
}
# ptl_check_placed <engine> <log> <tag>: the matrices on the device after setup are the N
# layers' as the fit counted them, nothing of a layer that did not complete
ptl_check_placed() {
local mm; mm=$(ptl_calc matrices "$1" "$2") || { grep -E "^\[VK\] $1 chain" "$2"; fail "$3: no placed line"; }
set -- $mm "$3"
[ "$1" = "$2" ] || fail "$3: $1 B of matrices on the device after setup, the N layers' are $2 B"
}
# Each group's helpers carry its own prefix: a script that redefines a function sourced
# before it would silently change another group's tests (partial-dsk's ptl_plan once).
for f in tests/vulkan_partial_*.sh; do
[ -e "$f" ] || continue
for fn in $(sed -n 's/^\([A-Za-z_][A-Za-z_0-9]*\)() .*/\1/p' "$f"); do
if declare -F "$fn" >/dev/null; then echo "$f redefines $fn, already defined" >&2; exit 2; fi
done
. "$f"
done
. tests/vulkan_dev2.sh # the expert tier on two devices
. tests/vulkan_layers_dev2.sh # the chain's layers on two devices
# ---- big prompt chunks and expert streaming (docs/vulkan.md, "Big prompt chunks and
# expert streaming") ----
# Every engine with the chain, on a prompt longer than its usual block, against its CPU
# run with the same snapshot and settings:
# default the chunk from the budget (the whole prompt in one chunk here) and the cold
# experts the rule streams: the CPU's tokens and logits, the chunk's line,
# and one tier step of the whole prompt;
# forced a small chunk cap and a forced streaming threshold (PF_FORCE: chunks of
# PF_ROWS rows, a tier of a few experts, cold experts with 3 rows or more
# streamed through four slots in sub-batches of 24 rows, so experts are cut
# into parts): the CPU's tokens and logits, streamed experts counted, and the
# later chunks' prefetch from the earlier ones' routing (PF_PREFETCH=1);
# staged the forced configuration with COLI_VK_STAGED=1 (the staging slots filled
# through host images): the same, nothing resident left in host memory;
# off COLI_VK_CHAIN_ROWS=512 COLI_VK_TIER_STREAM=0 COLI_VK_ATTN_BLOCK=0, the
# build before's behaviour: the CPU's tokens, nothing streamed.
# <family>-sanitize runs the forced configurations under ASan and UBSan.
PF_FORCE="COLI_VK_TIER_STREAM_ROWS=3 COLI_VK_TIER_STREAM_SLOTS=4 COLI_VK_TIER_STREAM_HALF=24"
PF_OFF="COLI_VK_CHAIN_ROWS=512 COLI_VK_TIER_STREAM=0 COLI_VK_ATTN_BLOCK=0"
stream_count() { # <engine> <log>: cold experts streamed in the run (the last report)
local n; n=$(sed -n "s/^\[VK\] tier $1 run: .* stream: [0-9]* steps, \([0-9]*\) cold experts.*/\1/p" "$2" | tail -1)
echo "${n:-0}"
}
prefetch_count() {
local n; n=$(sed -n "s/^\[VK\] tier $1 run: .* prefetched \([0-9]*\) (\([0-9]*\) used).*/\2/p" "$2" | tail -1)
echo "${n:-0}"
}
# pf_check <engine> <log> <tag> <kind> [rows]: what the run must show besides the CPU's tokens
pf_check() {
local eng=$1 log=$2 tag=$3 kind=$4 rows=${5:-}
case $kind in
default)
grep -q "^\[VK\] $eng chain: prompt chunks of up to 8192 rows" "$log" || { grep '\[VK\]' "$log"; fail "$tag: the chunk is not the budget's"; }
if [ -n "$rows" ]; then grep -q "^\[VK\] tier $eng stream: a step of $rows rows" "$log" ||
{ grep '\[VK\] tier' "$log"; fail "$tag: no tier step of the whole prompt ($rows rows)"; }; fi ;;
forced|staged)
[ "$(stream_count "$eng" "$log")" -gt 0 ] || { grep '\[VK\] tier' "$log"; fail "$tag: nothing streamed"; }
if [ "${PF_PREFETCH:-0}" = 1 ]; then [ "$(prefetch_count "$eng" "$log")" -gt 0 ] ||
{ grep '\[VK\] tier' "$log"; fail "$tag: no prefetched expert was used"; }; fi
if [ $kind = staged ]; then staged_check "$log" "$tag"; fi ;;
off)
! grep -q "^\[VK\] tier $eng stream:" "$log" || { grep '\[VK\] tier' "$log"; fail "$tag: streamed with the feature off"; } ;;
esac
echo " $tag: $(grep -o 'stream: [0-9]* steps, [0-9]* cold experts[^|]*' "$log" | tail -1 | cut -c1-160)"
}
# pf_runs <gate> <engine> <tag> <log> <rows> <budget env> <env...>: the four runs above.
# <gate> is a command line (eval'd: its quotes hold) taking <tag> <tol> <env...> after it.
pf_runs() {
local g=$1 eng=$2 tag=$3 log=$4 rows=$5 bud=$6; shift 6
local rw="COLI_VK_CHAIN_ROWS=${PF_ROWS:-64}"
eval "$g"' "pf $tag default" 1 "$@"'; pf_check "$eng" "$log" "pf $tag default" default "$rows"
eval "$g"' "pf $tag forced" 1 $PF_FORCE $bud $rw "$@"'; pf_check "$eng" "$log" "pf $tag forced" forced
eval "$g"' "pf $tag staged" 1 $PF_FORCE $bud $rw COLI_VK_STAGED=1 "$@"'; pf_check "$eng" "$log" "pf $tag staged" staged
eval "$g"' "pf $tag off" 1 $PF_OFF "$@"'; pf_check "$eng" "$log" "pf $tag off" off
}
# the engines' gates in pf_runs' calling convention: <fixed args...> <tag> <tol> <env...>
pf_chain() { # <engine> <argv as one string> <tag> <tol> <env...>
local eng=$1 argv=$2 tag=$3 tol=$4; shift 4
# shellcheck disable=SC2086
chain_gate "$eng" "$tag" "$tol" "$@" -- $argv
}
pf_mla() { # <engine> <argv as one string> <tag> <tol> <env...>
local eng=$1 argv=$2 tag=$3 tol=$4; shift 4
# shellcheck disable=SC2086
mla_gate "$eng" "$tag" "$tol" "$@" -- $argv
}
pf_ids() { $PY -c "print(','.join(str($2 + (i * 37) % $3) for i in range($1)))"; }
# a long prompt's ref.json for the engines whose oracle reads one: the ids and a few more
pf_ref() { # <out> <n> <first id> <span>
$PY - "$1" "$2" "$3" "$4" <<'PY'
import json, sys
out, n, lo, span = sys.argv[1], int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4])
p = [lo + (i * 37) % span for i in range(n)]
full = p + [lo] * 8
json.dump({"prompt_ids": p, "full_ids": full, "tf_pred": full[1:] + [lo]}, open(out, "w"))
PY
}
# MiMo on a prompt of ids (its cases are at most 72 tokens): tokens with --ngen 6, every
# position's logits with --ngen 0, against the CPU
pf_mimo() { # <ids> <tag> <tol> <env...>
local ids=$1 tag=$2; shift 3
env "$@" COLI_TEMP=0 ./mimo mimo_tiny --ids "$ids" --ngen 6 > mimo-cpu.txt 2>/dev/null
env "$@" COLI_TEMP=0 COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 ./mimo mimo_tiny --ids "$ids" --ngen 6 > mimo-vk.txt 2> mimo-vk.err
[ -s mimo-cpu.txt ] && cmp -s mimo-cpu.txt mimo-vk.txt || { cat mimo-cpu.txt mimo-vk.txt mimo-vk.err; fail "$tag: tokens differ from the CPU"; }
rm -f cpu.f32 vk.f32
env "$@" MIMO_LOGITS=cpu.f32 ./mimo mimo_tiny --ids "$ids" --ngen 0 > /dev/null 2>&1
env "$@" MIMO_LOGITS=vk.f32 COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 ./mimo mimo_tiny --ids "$ids" --ngen 0 > /dev/null 2> mimo-vk2.err
local lg; lg=$(logits_close cpu.f32 vk.f32) || { echo "$lg"; fail "$tag: logits"; }
echo "OK $tag: tokens = CPU, $lg, $(chain_count mimo mimo-vk.err) chain forwards"
}
pf_k3() { # <ids> <tag> <tol> <env...>: k3c_gate on one prompt of ids
local ids=$1 tag=$2; shift 3
K3C_IDS="$ids" K3C_CASES=ids k3c_gate "$tag" "$@"
}
pf_v41() { # <snap> <tag> <tol> <env...>
local snap=$1 tag=$2; shift 3
v41_gate "$tag" SNAP="$snap" "$@" -- 2 "$snap"/ref.json
}
pf_v4() { # <fixture> <case> <tag> <tol> <env...>
local fx=$1 c=$2 tag=$3; shift 4
v4_chain_gate "$tag" "$fx" "$c" "$@"
}
family_prefill_qwen() {
make qwen36 qwen38 tests/test_vk_chain tests/test_vk_tier VK=1
./tests/test_vk_tier shaders/qmatmul.spv | tee vk_tier.log
tail -1 vk_tier.log | grep -qx PASS || fail "the tier (its streaming section)"
./tests/test_vk_chain shaders/qmatmul.spv | tee vk_chain.log
tail -1 vk_chain.log | grep -qx PASS || fail "the chain's ops (the blocked attention)"
$PY tools/make_qwen36_tiny.py --out qwen36_tiny --ref-mode full --emit-ref qwen36_tiny/ref_full.json
$PY tools/convert_qwen36.py --model qwen36_tiny --out qwen36_tiny_c --ebits 8
$PY tools/make_qwen36_tiny.py --out qwen36_tiny64 --ref-mode full --inter 64 --emit-ref qwen36_tiny64/ref_full.json
$PY tools/convert_qwen36.py --model qwen36_tiny64 --out qwen36_tiny64_c --ebits 4 --gs 64
pf_ref qwen36_tiny/ref_long.json 300 1 250
local ids; ids=$(pf_ids 110 1 60)
$PY tools/make_qwen38_tiny.py --out qwen38_long_int4 --prompt-ids "$ids" --fp8-experts --int4-experts --expert-gain 3
$PY tools/make_qwen38_tiny.py --out qwen38_long --prompt-ids "$ids"
export OMP_NUM_THREADS=2
pf_runs "pf_chain qwen36 '8 8 qwen36_tiny/ref_long.json'" qwen36 "qwen36 int8 experts" vk.log 300 COLI_VK_TIER_GB=0.00015 COLI_DENSE_I8=0 SNAP=qwen36_tiny_c
pf_runs "pf_chain qwen36 '2 4 qwen36_tiny/ref_long.json'" qwen36 "qwen36 planar int4-g64, cap 2" vk.log 300 COLI_VK_TIER_GB=0.0003 QWEN_EXPERT_ACT=f32 COLI_DENSE_I8=0 SNAP=qwen36_tiny64_c
pf_runs "pf_chain qwen38 '4 8 qwen38_long/ref.json'" qwen38 "qwen38 bf16" vk.log 110 COLI_VK_TIER_GB=0.00005 SNAP=qwen38_long
pf_runs "pf_chain qwen38 '2 8 qwen38_long_int4/ref_int4.json'" qwen38 "qwen38 int4 sidecar" vk.log 110 COLI_VK_TIER_GB=0.00005 SNAP=qwen38_long_int4
PF_PREFETCH=1 chain_gate qwen36 "pf qwen36 prefetch across chunks" 1 $PF_FORCE COLI_VK_TIER_GB=0.00015 COLI_VK_CHAIN_ROWS=100 COLI_DENSE_I8=0 SNAP=qwen36_tiny_c -- 8 8 qwen36_tiny/ref_long.json
PF_PREFETCH=1 pf_check qwen36 vk.log "pf qwen36 prefetch across chunks" forced
PF_PREFETCH=1 chain_gate qwen38 "pf qwen38 prefetch across chunks" 1 $PF_FORCE COLI_VK_TIER_GB=0.00005 COLI_VK_CHAIN_ROWS=40 SNAP=qwen38_long_int4 -- 2 8 qwen38_long_int4/ref_int4.json
PF_PREFETCH=1 pf_check qwen38 vk.log "pf qwen38 prefetch across chunks" forced
# the CPU's prefill batching off (Q38_PREFILL_BATCH=0): the chain's MoE step still whole
chain_gate qwen38 "pf qwen38 batch=0" 1 Q38_PREFILL_BATCH=0 COLI_VK_TIER_GB=0.00005 COLI_VK_TIER_STREAM_ROWS=3 SNAP=qwen38_long -- 4 8 qwen38_long/ref.json
# the device lost while a chunk's streamed sub-batches are in flight: the state rebuilt
# on the CPU, the run finishes there
lost_late qwen36 "pf qwen36 device lost in a streamed prompt" 60 rebuild $PF_FORCE COLI_VK_TIER_GB=0.00015 COLI_VK_CHAIN_ROWS=64 COLI_DENSE_I8=0 SNAP=qwen36_tiny_c -- 8 8 qwen36_tiny/ref_long.json
unset OMP_NUM_THREADS
}
family_prefill_qwen_sanitize() {
make clean >/dev/null 2>&1 || true
make qwen36 qwen38 tests/test_vk_tier VK=1 EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1
./tests/test_vk_tier shaders/qmatmul.spv > san.log 2>&1 || true
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log || ! tail -1 san.log | grep -qx PASS; then cat san.log; fail "asan: the tier"; fi
echo "OK asan: the tier, its streaming section included"
COLI_VK_STAGED=1 ./tests/test_vk_tier shaders/qmatmul.spv > san.log 2>&1 || true
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log || ! tail -1 san.log | grep -qx PASS; then cat san.log; fail "asan: the tier staged"; fi
echo "OK asan: the tier staged"
$PY tools/make_qwen36_tiny.py --out qwen36_tiny --ref-mode full --emit-ref qwen36_tiny/ref_full.json
$PY tools/convert_qwen36.py --model qwen36_tiny --out qwen36_tiny_c --ebits 8
pf_ref qwen36_tiny/ref_long.json 300 1 250
local ids; ids=$(pf_ids 110 1 60)
$PY tools/make_qwen38_tiny.py --out qwen38_long_int4 --prompt-ids "$ids" --fp8-experts --int4-experts --expert-gain 3
export OMP_NUM_THREADS=2
pfsan qwen36 "asan pf qwen36 forced" COLI_VK_TIER_GB=0.00015 COLI_VK_CHAIN_ROWS=64 COLI_DENSE_I8=0 SNAP=qwen36_tiny_c ./qwen36 8 8 qwen36_tiny/ref_long.json
pfsan qwen36 "asan pf qwen36 forced, staged" COLI_VK_STAGED=1 COLI_VK_TIER_GB=0.00015 COLI_VK_CHAIN_ROWS=64 COLI_DENSE_I8=0 SNAP=qwen36_tiny_c ./qwen36 8 8 qwen36_tiny/ref_long.json
pfsan qwen38 "asan pf qwen38 forced" COLI_VK_TIER_GB=0.00005 COLI_VK_CHAIN_ROWS=40 SNAP=qwen38_long_int4 ./qwen38 2 8 qwen38_long_int4/ref_int4.json
unset OMP_NUM_THREADS
make clean >/dev/null 2>&1 || true
}
# pfsan <engine> <tag> <env and argv...>: the forced configuration under the sanitizers
pfsan() {
local eng=$1 tag=$2; shift 2
rm -f chain.usage
# shellcheck disable=SC2086
env $PF_FORCE COLI_USAGE=chain.usage COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 "$@" > san.log 2>&1 || true
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "$tag: sanitizer diagnostic"; fi
[ "$(chain_count "$eng" san.log)" -gt 0 ] || { cat san.log; fail "$tag: the chain never ran"; }
[ "$(stream_count "$eng" san.log)" -gt 0 ] || { grep '\[VK\]' san.log; fail "$tag: nothing streamed"; }
echo "OK $tag: sanitizers clean, $(chain_count "$eng" san.log) chain forwards, $(stream_count "$eng" san.log) experts streamed"
}
family_prefill_inkling_olmoe() {
export OMP_NUM_THREADS=2
make inkling olmoe VK=1
$PY tools/make_tiny_inkling.py tiny_inkling
$PY tools/make_olmoe_tiny.py --output olmoe_tiny --force
$PY tools/convert_olmoe_merged.py --model olmoe_tiny --out olmoe_tiny_c
pf_ref tiny_inkling/ref_long.json 200 1 250
pf_ref olmoe_tiny/ref_long.json 200 1 120
pf_runs "pf_chain inkling '8 0 tiny_inkling/ref_long.json'" inkling "inkling f32" vk.log 200 COLI_VK_TIER_GB=0.0001 SNAP=tiny_inkling
pf_runs "pf_chain inkling '2 8 tiny_inkling/ref_long.json'" inkling "inkling runtime int8, cap 2" vk.log 200 COLI_VK_TIER_GB=0.0001 SNAP=tiny_inkling
pf_runs "pf_chain olmoe '8 8 olmoe_tiny/ref_long.json'" olmoe "olmoe" vk.log 200 COLI_VK_TIER_GB=0.0001 SNAP=olmoe_tiny_c
pf_runs "pf_chain olmoe '1 4 olmoe_tiny/ref_long.json'" olmoe "olmoe 4-bit experts, cap 1" vk.log 200 COLI_VK_TIER_GB=0.0001 SNAP=olmoe_tiny_c
PF_PREFETCH=1 chain_gate olmoe "pf olmoe prefetch across chunks" 1 $PF_FORCE COLI_VK_TIER_GB=0.0001 COLI_VK_CHAIN_ROWS=64 SNAP=olmoe_tiny_c -- 8 8 olmoe_tiny/ref_long.json
PF_PREFETCH=1 pf_check olmoe vk.log "pf olmoe prefetch across chunks" forced
chain_gate olmoe "pf olmoe PILOT" 1 PILOT=1 WIDE=2 $PF_FORCE COLI_VK_TIER_GB=0.0001 SNAP=olmoe_tiny_c -- 2 8 olmoe_tiny/ref_long.json
pf_check olmoe vk.log "pf olmoe PILOT" forced
# sanitizers
make clean >/dev/null 2>&1 || true
make inkling olmoe VK=1 EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1
pfsan inkling "asan pf inkling forced" COLI_VK_TIER_GB=0.0001 COLI_VK_CHAIN_ROWS=64 SNAP=tiny_inkling ./inkling 2 8 tiny_inkling/ref_long.json
pfsan olmoe "asan pf olmoe forced, staged" COLI_VK_STAGED=1 COLI_VK_TIER_GB=0.0001 COLI_VK_CHAIN_ROWS=64 SNAP=olmoe_tiny_c ./olmoe 1 8 olmoe_tiny/ref_long.json
unset OMP_NUM_THREADS
make clean >/dev/null 2>&1 || true
}
family_prefill_mimo_kimi() {
export OMP_NUM_THREADS=2
make mimo kimi_k3 VK=1
$PY tools/make_mimo_tiny.py --output ./mimo_tiny --force --vision
$PY tools/make_kimi_k3_tiny.py --output ./kimi_k3_tiny --force
local mids kids
mids=$(pf_ids 200 1 300 | tr ',' ' '); kids=$(pf_ids 200 1 300 | tr ',' ' ')
pf_runs "pf_mimo '$mids'" mimo "mimo f32" mimo-vk.err 200 MIMO_VK_EXPERTS=6 MIMO_DENSE_BITS=32
pf_runs "pf_mimo '$mids'" mimo "mimo fp8/bf16" mimo-vk.err 200 MIMO_VK_EXPERTS=6 MIMO_DENSE_BITS=0
local O="K3_BITS=32 K3_MLA_BITS=32 K3_HEAD_BITS=32 K3_IDOT=0 COLI_TEMP=0"
# shellcheck disable=SC2086
pf_runs "pf_k3 '$kids'" kimi_k3 "kimi_k3 f32" vk.log 200 COLI_VK_TIER_GB=0.0005 $O
PF_PREFETCH=1 pf_mimo "$mids" "pf mimo prefetch across chunks" 1 $PF_FORCE MIMO_VK_EXPERTS=6 MIMO_DENSE_BITS=32 COLI_VK_CHAIN_ROWS=50
PF_PREFETCH=1 pf_check mimo mimo-vk.err "pf mimo prefetch across chunks" forced
# shellcheck disable=SC2086
PF_PREFETCH=1 pf_k3 "$kids" "pf kimi_k3 prefetch across chunks" 1 $PF_FORCE COLI_VK_TIER_GB=0.0005 COLI_VK_CHAIN_ROWS=50 $O
PF_PREFETCH=1 pf_check kimi_k3 vk.log "pf kimi_k3 prefetch across chunks" forced
make clean >/dev/null 2>&1 || true
make mimo kimi_k3 VK=1 EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1
pfsan mimo "asan pf mimo forced" MIMO_VK_EXPERTS=6 MIMO_DENSE_BITS=32 COLI_VK_CHAIN_ROWS=50 COLI_TEMP=0 ./mimo mimo_tiny --ids "$mids" --ngen 4
# shellcheck disable=SC2086
pfsan kimi_k3 "asan pf kimi_k3 forced, staged" COLI_VK_STAGED=1 COLI_VK_TIER_GB=0.0005 COLI_VK_CHAIN_ROWS=50 $O ./kimi_k3 kimi_k3_tiny --ids "$kids" --ngen 4
unset OMP_NUM_THREADS
make clean >/dev/null 2>&1 || true
}
family_prefill_glm() {
export OMP_NUM_THREADS=2 CAP_RAISE=0
make colibri glm53 VK=1
glm_fixtures
pf_ref glm_tiny/ref_long.json 200 1 250
local ids; ids=$(pf_ids 200 2 120)
pf_runs "pf_mla colibri '64 16 16'" colibri "colibri f32, teacher-forced" vk.log 208 COLI_VK_TIER_GB=0.0003 SNAP=glm_tiny REF=glm_tiny/ref_long.json TF=1
pf_runs "pf_mla colibri '2 16 16'" colibri "colibri int4-g64 experts" vk.log 200 COLI_VK_TIER_GB=0.0012 SNAP=glm_tiny_fmt4 REF=glm_tiny/ref_long.json
pf_runs "pf_mla glm53 '--model glm53_stream-i4 --ids $ids --greedy 4'" glm53 "glm53 f32 trunk" vk.log 200 COLI_VK_TIER_GB=0.0002 GLM53_BITS=32
pf_runs "pf_mla glm53 '--model glm53_stream-i4 --ids $ids --greedy 4'" glm53 "glm53 int4 trunk" vk.log 200 COLI_VK_TIER_GB=0.0002 GLM53_BITS=4
PF_PREFETCH=1 mla_gate colibri "pf colibri prefetch across chunks" 1 $PF_FORCE COLI_VK_TIER_GB=0.0003 COLI_VK_CHAIN_ROWS=64 SNAP=glm_tiny REF=glm_tiny/ref_long.json TF=1 -- 2 16 16
PF_PREFETCH=1 pf_check colibri vk.log "pf colibri prefetch across chunks" forced
PF_PREFETCH=1 mla_gate glm53 "pf glm53 prefetch across chunks" 1 $PF_FORCE COLI_VK_TIER_GB=0.0002 COLI_VK_CHAIN_ROWS=64 GLM53_BITS=4 GLM53_PREFILL_CHUNK=200 -- --model glm53_stream-i4 --ids "$ids" --greedy 4
PF_PREFETCH=1 pf_check glm53 vk.log "pf glm53 prefetch across chunks" forced
make clean >/dev/null 2>&1 || true
make colibri glm53 VK=1 EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1
pfsan colibri "asan pf colibri forced" COLI_VK_TIER_GB=0.0003 COLI_VK_CHAIN_ROWS=64 SNAP=glm_tiny REF=glm_tiny/ref_long.json TF=1 ./colibri 2 16 16
pfsan glm53 "asan pf glm53 forced, staged" COLI_VK_STAGED=1 COLI_VK_TIER_GB=0.0002 COLI_VK_CHAIN_ROWS=64 GLM53_BITS=4 ./glm53 --model glm53_stream-i4 --ids "$ids" --greedy 4
unset OMP_NUM_THREADS CAP_RAISE
make clean >/dev/null 2>&1 || true
}
family_prefill_deepseek() {
export OMP_NUM_THREADS=2
make deepseek_v41 VK=1
make deepseek-v4 VK=1
$PY tools/make_dsv41_tiny.py --out dsv41_vlong --emit-ref dsv41_vlong/ref.json --prompt-len 160 --max-new 6 > /dev/null
pf_v4_fixture
local ids; ids=$(pf_ids 300 3 120)
pf_runs "pf_v41 dsv41_vlong" deepseek_v41 "deepseek_v41" vk.log 160 COLI_VK_TIER_GB=0.0005
pf_runs "pf_v4 deepseek_v4_tiny_p ids:$ids:4" deepseek_v4 "deepseek_v4" v4-vk.err 300 COLI_VK_TIER_GB=0.0005
PF_PREFETCH=1 v41_gate "pf deepseek_v41 prefetch across chunks" SNAP=dsv41_vlong $PF_FORCE COLI_VK_TIER_GB=0.0005 COLI_VK_CHAIN_ROWS=64 -- 2 dsv41_vlong/ref.json
PF_PREFETCH=1 pf_check deepseek_v41 vk.log "pf deepseek_v41 prefetch across chunks" forced
PF_PREFETCH=1 v4_chain_gate "pf deepseek_v4 prefetch across chunks" deepseek_v4_tiny_p "ids:$ids:4" $PF_FORCE COLI_VK_TIER_GB=0.0005 COLI_VK_CHAIN_ROWS=100
PF_PREFETCH=1 pf_check deepseek_v4 v4-vk.err "pf deepseek_v4 prefetch across chunks" forced
make clean >/dev/null 2>&1 || true
make deepseek-v4-clean >/dev/null 2>&1 || true
local SAN="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
make deepseek_v41 VK=1 EXTRA_CFLAGS="$SAN"
make deepseek-v4 VK=1 LTO=0 EXTRA_CFLAGS="$SAN" EXTRA_LDFLAGS="$SAN"
export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1
pfsan deepseek_v41 "asan pf deepseek_v41 forced" COLI_VK_TIER_GB=0.0005 COLI_VK_CHAIN_ROWS=64 SNAP=dsv41_vlong ./deepseek_v41 2 dsv41_vlong/ref.json
pfsan deepseek_v4 "asan pf deepseek_v4 forced, staged" COLI_VK_STAGED=1 COLI_VK_TIER_GB=0.0005 COLI_VK_CHAIN_ROWS=100 ./deepseek_v4 ./deepseek_v4_tiny_p \
"$(v4_chain_prompt deepseek_v4_tiny_p "ids:$ids:4")" --raw-prompt --max-tokens 4
unset OMP_NUM_THREADS
make clean >/dev/null 2>&1 || true
make deepseek-v4-clean >/dev/null 2>&1 || true
}
# DeepSeek V4's tiny model at 512 positions, 8 experts: room for a prompt past its 128-row block
pf_v4_fixture() {
$PY - tools/make_deepseek_v4_tiny.py <<'PY' > /dev/null
import importlib.util, sys
spec = importlib.util.spec_from_file_location("gen", sys.argv[1])
gen = importlib.util.module_from_spec(spec); spec.loader.exec_module(gen)
gen.EXPERTS = 8; gen.MAX_POSITIONS = 512
sys.argv = ["make_deepseek_v4_tiny.py", "--output", "deepseek_v4_tiny_p", "--force"]
gen.main()
PY
}
# ---- the KV cache split between the device and the host (vk_kvsplit.h) ----------------
# Past the device's budget each chain engine keeps a few blocks of every attention
# layer's cache on the device and attends over the rest on the CPU, the two parts merged
# through their softmax statistics. The tiny fixtures emulate a small device with
# COLI_VK_KV_DEVICE_ROWS (rows a layer keeps) and COLI_VK_KV_BLOCK (positions a block),
# over prompts and generations long enough that most steps have a host part.
# kv_hostparts <engine> <log>: M from "[VK] <engine> chain: KV split: N layer steps, M with a host part"
kv_hostparts() {
local n
n=$(sed -n "s/^\[VK\] $1 chain: KV split: [0-9]* layer steps, \([0-9]*\) with a host part.*/\1/p" "$2" | tail -1)
echo "${n:-0}"
}
# kv_devcold <engine> <log>: N from "| the host's part on the device: N layer steps"
# (COLI_VK_KV_COLD=device), 0 without it
kv_devcold() {
local n
n=$(sed -n "s/^\[VK\] $1 chain: KV split: .* the host's part on the device: \([0-9]*\) layer steps.*/\1/p" "$2" | tail -1)
echo "${n:-0}"
}
# kv_cold_check <engine> <tag> <log>: with COLI_VK_KV_COLD=device exported (the
# kv-split-cold families), the host's part must have run on the device; with read-based
# pins (COLI_VK_KV_PIN=1) it stays on the CPU by design, and the engine says so
kv_cold_check() {
[ "${COLI_VK_KV_COLD:-}" = device ] || return 0
if [ "${COLI_VK_KV_PIN:-0}" = 1 ]; then
grep -qa "COLI_VK_KV_COLD=device needs .* no pins; the CPU computes the host's part" "$3" || { cat "$3"; fail "$2: pins on, and no line that the CPU computes the host's part"; }
return 0
fi
[ "$(kv_devcold "$1" "$3")" -gt 0 ] || { cat "$3"; fail "$2: the host's part never ran on the device"; }
}
# kv_gate <engine> <tag> <tol 0|1> <rows> <block> <env...> -- <argv...>: chain_gate (the
# CPU's tokens, logits within 1e-4 of the largest with tol 1) with the split on, which
# must have run steps with a host part.
kv_gate() {
local eng=$1 tag=$2 tol=$3 rows=$4 blk=$5; shift 5
chain_gate "$eng" "$tag" "$tol" COLI_VK_KV_DEVICE_ROWS=$rows COLI_VK_KV_BLOCK=$blk "$@"
[ "$(kv_hostparts "$eng" vk.log)" -gt 0 ] || { cat vk.log; fail "$tag: the split never ran a host part"; }
kv_cold_check "$eng" "$tag" vk.log
echo " $tag: $(grep -a "KV split:" vk.log | tail -1 | sed 's/.*KV split: //')"
}
# kv_san <engine> <tag> <env and argv...>: a sanitized run under the split (the build is
# the caller's): no diagnostic, and steps with a host part
kv_san() {
local eng=$1 tag=$2; shift 2
rm -f chain.usage
env OMP_NUM_THREADS=2 COLI_USAGE=chain.usage COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 "$@" > san.log 2>&1 || true
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "$tag: sanitizer diagnostic"; fi
[ "$(kv_hostparts "$eng" san.log)" -gt 0 ] || { cat san.log; fail "$tag: the split never ran a host part"; }
kv_cold_check "$eng" "$tag" san.log
echo "OK $tag: sanitizers clean, $(kv_hostparts "$eng" san.log) layer steps with a host part"
}
# qwen36: the hybrid and the dense geometry, a 40-token prompt and 40 new tokens
kv_qwen36_fixtures() {
local P; P=$($PY -c "print(','.join(str(3 + i * 7 % 60) for i in range(40)))")
$PY tools/make_qwen36_tiny.py --out qwen36_kv --ref-mode full --prompt-ids "$P" --max-new 40 --emit-ref qwen36_kv/ref_full.json
$PY tools/convert_qwen36.py --model qwen36_kv --out qwen36_kv_c --ebits 8
$PY tools/make_qwen36_tiny.py --geometry qwen38-27b-dense --out qwen38_27b_kv --ref-mode full --prompt-ids "$P" --max-new 40 \
--emit-ref qwen38_27b_kv/ref_full.json
$PY tools/convert_qwen36.py --model qwen38_27b_kv --out qwen38_27b_kv_c --ebits 8
$PY tools/make_qwen36_tiny.py --out qwen36_tiny --ref-mode full --emit-ref qwen36_tiny/ref_full.json
$PY tools/convert_qwen36.py --model qwen36_tiny --out qwen36_tiny_c --ebits 8
}
kv_split_qwen36() {
kv_qwen36_fixtures
local R=qwen36_kv/ref_full.json
kv_gate qwen36 "kv qwen36 16 rows on the device" 1 16 4 COLI_DENSE_I8=0 SNAP=qwen36_kv_c -- 8 8 $R
kv_gate qwen36 "kv qwen36 two blocks on the device" 1 8 4 COLI_DENSE_I8=0 SNAP=qwen36_kv_c -- 8 8 $R
kv_gate qwen36 "kv qwen36 prefill in chunks of 3" 1 16 4 COLI_VK_CHAIN_ROWS=3 COLI_DENSE_I8=0 SNAP=qwen36_kv_c -- 8 8 $R
kv_gate qwen36 "kv qwen36 cap 1, blocks of 8" 1 24 8 COLI_DENSE_I8=0 SNAP=qwen36_kv_c -- 1 8 $R
kv_gate qwen36 "kv qwen36 tier off, tiled GEMM" 1 24 4 COLI_VK_TIER=0 COLI_VK_GEMM_MIN_S=2 COLI_DENSE_I8=0 SNAP=qwen36_kv_c -- 8 8 $R
CPUENV=COLI_DENSE_IDOT=0 kv_gate qwen36 "kv qwen36 int8 trunk" 1 16 4 SNAP=qwen36_kv_c -- 8 8 $R
kv_gate qwen36 "kv qwen36 dense model" 1 16 4 COLI_DENSE_I8=0 SNAP=qwen38_27b_kv_c -- 8 8 qwen38_27b_kv/ref_full.json
CHAINMODE=2 kv_gate qwen36 "kv qwen36 prompts only" 1 16 4 COLI_DENSE_I8=0 SNAP=qwen36_kv_c -- 8 8 $R
# COLI_VK_KV_SPLIT=0: the whole mirrors, as before the split
chain_gate qwen36 "kv qwen36 split off" 1 COLI_VK_KV_SPLIT=0 COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 COLI_DENSE_I8=0 SNAP=qwen36_kv_c -- 8 8 $R
if grep -qa 'KV cache split' vk.log; then cat vk.log; fail "kv qwen36 split off: the split ran"; fi
# the device lost mid-decode with the split on: the DeltaNet state rebuilt on the CPU
lost_late qwen36 "kv qwen36 device lost" 20 rebuild COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 COLI_DENSE_I8=0 SNAP=qwen36_kv_c -- 8 8 $R
# serve sessions (pins, prompt-cache extensions, a divergent prompt, logprobs) and the
# prefix-reuse contract across the split
QWEN36_TINY=$PWD/qwen36_tiny_c COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4 \
COLI_USAGE=$PWD/chain.usage $PY tests/test_qwen36_prefix_serve.py
CHAIN_SERVE_EXPECT='KV split: [0-9]+ layer steps, [1-9][0-9]* with a host part' \
$PY tests/vulkan_chain_serve.py ./qwen36 qwen36_tiny_c COLI_DENSE_I8=0 COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4
}
kv_split_qwen36_sanitize() {
kv_qwen36_fixtures
kv_san qwen36 "asan kv qwen36" COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4 COLI_DENSE_I8=0 SNAP=qwen36_kv_c ./qwen36 8 8 qwen36_kv/ref_full.json
kv_san qwen36 "asan kv qwen36 chunks of 3" COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 COLI_VK_CHAIN_ROWS=3 SNAP=qwen36_kv_c \
./qwen36 8 8 qwen36_kv/ref_full.json
kv_san qwen36 "asan kv qwen36 dense model" COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 COLI_DENSE_I8=0 SNAP=qwen38_27b_kv_c \
./qwen36 8 8 qwen38_27b_kv/ref_full.json
$PY -c "import sys; sys.path.insert(0, 'tests'); from prefix_serve_harness import ensure_byte_tokenizer as t; from pathlib import Path; t(Path('qwen36_tiny_c'))"
CHAIN_SERVE_EXPECT='KV split: [0-9]+ layer steps, [1-9][0-9]* with a host part' \
$PY tests/vulkan_chain_serve.py ./qwen36 qwen36_tiny_c COLI_DENSE_I8=0 COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4 > san.log 2>&1 || { cat san.log; fail "asan kv qwen36 serve"; }
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "asan kv qwen36 serve: sanitizer diagnostic"; fi
echo "OK asan kv qwen36 serve: $(tail -1 san.log)"
}
# qwen38: a 40-token prompt and 40 new tokens (the generation the transformers reference
# and the CPU agree on), with and without the MTP head; the serve fixture with a tokenizer
kv_qwen38_fixtures() {
local P; P=$($PY -c "print(','.join(str(3 + (i * 9 + i * i * 2) % 60) for i in range(40)))")
$PY tools/make_qwen38_tiny.py --out qwen38_kv --prompt-ids "$P" --max-new 40
$PY tools/make_qwen38_tiny.py --out qwen38_kv_mtp --prompt-ids "$P" --max-new 40 --mtp
$PY tools/make_qwen38_tiny.py --out qwen38_tiny_mtp --mtp
$PY tools/make_edge_tiny_tokenizer.py --vocab-size 64 ./qwen38_tiny_mtp # serve speaks text
}
kv_split_qwen38() {
kv_qwen38_fixtures
local R=qwen38_kv/ref.json M=qwen38_kv_mtp/ref.json batch f
for batch in 0 1; do
kv_gate qwen38 "kv qwen38 16 rows on the device, batch=$batch" 1 16 4 Q38_PREFILL_BATCH=$batch SNAP=qwen38_kv -- 4 8 $R
done
kv_gate qwen38 "kv qwen38 two blocks on the device" 1 8 4 SNAP=qwen38_kv -- 4 8 $R
kv_gate qwen38 "kv qwen38 prefill in chunks of 3" 1 16 4 COLI_VK_CHAIN_ROWS=3 SNAP=qwen38_kv -- 4 8 $R
# QSA's lists with blocks pinned by their reads (steps of 4 rows leave the window three
# of six slots): the device's part and the host's take the listed positions
COLI_VK_KV_PIN=1 kv_gate qwen38 "kv qwen38 blocks pinned by QSA's reads" 1 24 4 COLI_VK_CHAIN_ROWS=4 SNAP=qwen38_kv -- 4 8 $R
grep -qa 'KV split: .* [1-9][0-9]* blocks pinned by reads' vk.log || { cat vk.log; fail "kv qwen38: no block was pinned"; }
kv_gate qwen38 "kv qwen38 int8 trunk" 0 16 4 Q38_TRUNK_MIN_KB=0 SNAP=qwen38_kv -- 4 8 $R
kv_gate qwen38 "kv qwen38 tier off" 1 16 4 COLI_VK_TIER=0 SNAP=qwen38_kv -- 4 8 $R
# MTP verify (S = 2) and its rollback across the split: drafts as they come, every
# draft rejected, every one accepted, alternating
for f in "" reject accept mixed; do
kv_gate qwen38 "kv qwen38 MTP ${f:-drafting}" 1 16 4 Q38_MTP=1 Q38_MTP_FORCE=$f SNAP=qwen38_kv_mtp -- 2 8 $M
done
CHAINMODE=2 kv_gate qwen38 "kv qwen38 prompts only" 1 16 4 SNAP=qwen38_kv -- 4 8 $R
CHAINMODE=2 kv_gate qwen38 "kv qwen38 prompts only, MTP" 1 16 4 Q38_MTP=1 Q38_MTP_FORCE=mixed SNAP=qwen38_kv_mtp -- 2 8 $M
chain_gate qwen38 "kv qwen38 split off" 1 COLI_VK_KV_SPLIT=0 COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 SNAP=qwen38_kv -- 4 8 $R
if grep -qa 'KV cache split' vk.log; then cat vk.log; fail "kv qwen38 split off: the split ran"; fi
# the device lost with the split on, between steps and inside an MTP verify: the
# DeltaNet, conv and PLE states rebuilt on the CPU from the prefix record
lost_late qwen38 "kv qwen38 device lost" 10 rebuild COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 SNAP=qwen38_kv -- 4 8 $R
lost_late qwen38 "kv qwen38 device lost in a verify" 15 rebuild COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 Q38_MTP=1 \
Q38_MTP_FORCE=mixed SNAP=qwen38_kv_mtp -- 2 8 $M
# serve sessions (pins, prompt-cache extensions, a divergent prompt, logprobs) with
# and without MTP, and prompts only
local E='KV split: [0-9]+ layer steps, [1-9][0-9]* with a host part'
CHAIN_SERVE_EXPECT="$E" $PY tests/vulkan_chain_serve.py ./qwen38 qwen38_tiny_mtp Q38_MTP=1 COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4
CHAIN_SERVE_EXPECT="$E" $PY tests/vulkan_chain_serve.py ./qwen38 qwen38_tiny_mtp COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4
CHAIN_SERVE_EXPECT="$E" COLI_VK_CHAIN=2 $PY tests/vulkan_chain_serve.py ./qwen38 qwen38_tiny_mtp Q38_MTP=1 \
COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4
}
kv_split_qwen38_sanitize() {
kv_qwen38_fixtures
kv_san qwen38 "asan kv qwen38" COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4 SNAP=qwen38_kv ./qwen38 4 8 qwen38_kv/ref.json
COLI_VK_KV_PIN=1 kv_san qwen38 "asan kv qwen38 pinned blocks, chunks of 4" COLI_VK_KV_DEVICE_ROWS=24 COLI_VK_KV_BLOCK=4 COLI_VK_CHAIN_ROWS=4 \
SNAP=qwen38_kv ./qwen38 4 8 qwen38_kv/ref.json
kv_san qwen38 "asan kv qwen38 MTP mixed" COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 Q38_MTP=1 Q38_MTP_FORCE=mixed \
SNAP=qwen38_kv_mtp ./qwen38 2 8 qwen38_kv_mtp/ref.json
CHAIN_SERVE_EXPECT='KV split: [0-9]+ layer steps, [1-9][0-9]* with a host part' \
$PY tests/vulkan_chain_serve.py ./qwen38 qwen38_tiny_mtp Q38_MTP=1 COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4 \
> san.log 2>&1 || { cat san.log; fail "asan kv qwen38 serve"; }
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "asan kv qwen38 serve: sanitizer diagnostic"; fi
echo "OK asan kv qwen38 serve: $(tail -1 san.log)"
}
# olmoe: the tiny checkpoint and, from it, a 40-token prompt's 40-token generation by
# transformers (the oracle the engine's run checks)
kv_olmoe_fixtures() {
$PY tools/make_olmoe_tiny.py --output olmoe_tiny --force
$PY tools/convert_olmoe_merged.py --model olmoe_tiny --out olmoe_tiny_c
$PY tools/make_edge_tiny_tokenizer.py --vocab-size 128 olmoe_tiny_c
$PY - <<'PY'
import json, torch
from transformers import OlmoeForCausalLM
torch.set_num_threads(1)
m = OlmoeForCausalLM.from_pretrained("olmoe_tiny").eval().float()
prompt = [(17 + i * 21 + (i * i) % 13) % 125 + 2 for i in range(40)]
with torch.no_grad():
g = m.generate(torch.tensor([prompt]), max_new_tokens=40, do_sample=False, use_cache=True)[0].tolist()
json.dump({"prompt_ids": prompt, "full_ids": g}, open("olmoe_tiny/ref_olmoe_long.json", "w"))
PY
}
kv_split_olmoe() {
kv_olmoe_fixtures
local R=olmoe_tiny/ref_olmoe_long.json
kv_gate olmoe "kv olmoe 16 rows on the device" 1 16 4 SNAP=olmoe_tiny_c -- 8 8 $R
kv_gate olmoe "kv olmoe two blocks on the device" 1 8 4 SNAP=olmoe_tiny_c -- 8 8 $R
kv_gate olmoe "kv olmoe prefill in chunks of 3" 1 16 4 COLI_VK_CHAIN_ROWS=3 SNAP=olmoe_tiny_c -- 8 8 $R
kv_gate olmoe "kv olmoe cap 1, blocks of 8" 1 24 8 SNAP=olmoe_tiny_c -- 1 8 $R
kv_gate olmoe "kv olmoe 4-bit experts" 1 16 4 SNAP=olmoe_tiny_c -- 8 4 $R
kv_gate olmoe "kv olmoe PILOT cap 2" 1 16 4 PILOT=1 WIDE=2 SNAP=olmoe_tiny_c -- 2 8 $R
kv_gate olmoe "kv olmoe tier off, tiled GEMM" 1 24 4 COLI_VK_TIER=0 COLI_VK_GEMM_MIN_S=2 SNAP=olmoe_tiny_c -- 8 8 $R
CHAINMODE=2 kv_gate olmoe "kv olmoe prompts only" 1 16 4 SNAP=olmoe_tiny_c -- 8 8 $R
chain_gate olmoe "kv olmoe split off" 1 COLI_VK_KV_SPLIT=0 COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 SNAP=olmoe_tiny_c -- 8 8 $R
if grep -qa 'KV cache split' vk.log; then cat vk.log; fail "kv olmoe split off: the split ran"; fi
# the device lost mid-decode with the split on: the CPU redoes the step (nothing to rebuild)
lost_late olmoe "kv olmoe device lost" 7 redo COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 SNAP=olmoe_tiny_c -- 8 8 $R
# serve sessions and the prefix-reuse and dashboard contracts across the split
CHAIN_SERVE_EXPECT='KV split: [0-9]+ layer steps, [1-9][0-9]* with a host part' \
$PY tests/vulkan_chain_serve.py ./olmoe olmoe_tiny_c COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4
CHAIN_SERVE_EXPECT='KV split: [0-9]+ layer steps, [1-9][0-9]* with a host part' \
$PY tests/vulkan_chain_serve.py ./olmoe olmoe_tiny_c PILOT=1 WIDE=2 COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4
COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 COLI_VK_TIER_BALANCE=0 COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4 \
OLMOE_TINY=olmoe_tiny_c $PY -m unittest tests.test_olmoe_prefix_serve tests.test_olmoe_dashboard_hits
}
kv_split_olmoe_sanitize() {
kv_olmoe_fixtures
local R=olmoe_tiny/ref_olmoe_long.json
kv_san olmoe "asan kv olmoe" COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4 SNAP=olmoe_tiny_c ./olmoe 8 8 $R
kv_san olmoe "asan kv olmoe chunks of 3, PILOT" COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 COLI_VK_CHAIN_ROWS=3 PILOT=1 WIDE=2 \
SNAP=olmoe_tiny_c ./olmoe 2 8 $R
CHAIN_SERVE_EXPECT='KV split: [0-9]+ layer steps, [1-9][0-9]* with a host part' \
$PY tests/vulkan_chain_serve.py ./olmoe olmoe_tiny_c COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4 > san.log 2>&1 || { cat san.log; fail "asan kv olmoe serve"; }
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "asan kv olmoe serve: sanitizer diagnostic"; fi
echo "OK asan kv olmoe serve: $(tail -1 san.log)"
}
# inkling: the tiny model with a 40-token prompt and 40 new tokens (its global layer
# attends over all 80 positions, the bias bank reaching back 32), its expert containers,
# the dense-int4g64 container and the model at D = 6144
kv_inkling_fixtures() {
$PY tools/make_tiny_inkling.py tiny_inkling
$PY - tiny_inkling <<'PY'
import json, sys, torch
from transformers import InklingForCausalLM
d = sys.argv[1]
m = InklingForCausalLM.from_pretrained(d).eval().float()
prompt = [(3 + i * 7) % 250 for i in range(40)]
with torch.no_grad():
full = m.generate(torch.tensor([prompt]), max_new_tokens=40, do_sample=False, use_cache=True)[0].tolist()
tf = m(torch.tensor([full])).logits[0].argmax(-1).tolist()
json.dump({"prompt_ids": prompt, "full_ids": full, "tf_pred": tf}, open(f"{d}/ref_long.json", "w"))
PY
$PY - <<'PY'
import importlib.util, os, shutil
from safetensors.torch import load_file, save_file
spec = importlib.util.spec_from_file_location("conv", "tools/convert_inkling_int4.py")
conv = importlib.util.module_from_spec(spec); spec.loader.exec_module(conv)
t = load_file("tiny_inkling/model.safetensors")
if "model.embed_tokens.embed_norm.weight" in t:
t["model.embed_norm.weight"] = t.pop("model.embed_tokens.embed_norm.weight")
for d in ("tiny_inkling_hf", "tiny_inkling_x-tml", "tiny_inkling_x-pass", "tiny_inkling_x-i4", "tiny_inkling_x-i8"):
shutil.rmtree(d, ignore_errors=True)
os.makedirs("tiny_inkling_hf")
save_file(t, "tiny_inkling_hf/model.safetensors")
shutil.copy("tiny_inkling/config.json", "tiny_inkling_hf/")
conv.selftest_e2e("tiny_inkling_hf", "tiny_inkling_x") # tiny_inkling_x-i4: int4 experts
conv.convert_dir("tiny_inkling_x-tml", "tiny_inkling_x-i8", 8) # int8 experts
PY
rm -rf tiny_inkling_q && cp -r tiny_inkling tiny_inkling_q
$PY - tools/convert_inkling_dense_int4.py tiny_inkling_q <<'PY'
import importlib.util, sys
spec = importlib.util.spec_from_file_location("conv", sys.argv[1])
conv = importlib.util.module_from_spec(spec); spec.loader.exec_module(conv)
conv.MIN_ELEMS = 0
conv.ATTN_BITS = 4
base = conv.classify
conv.classify = lambda n, s, d: "int8" if n.endswith(".mlp.down_proj.weight") else base(n, s, d)
sys.argv = ["convert_inkling_dense_int4.py", "--dir", sys.argv[2]]
conv.main()
PY
mkdir -p tiny_inkling_q/dense-int4g64
mv tiny_inkling_q/dense-int4g64.safetensors tiny_inkling_q/dense-int4g64/dense.safetensors
$PY tools/make_tiny_inkling.py tiny_inkling_wide --wide
$PY -c 'import sys; from pathlib import Path; sys.path.insert(0, "."); from tests.test_inkling_prefix_serve import ensure_tokenizer; ensure_tokenizer(Path("tiny_inkling"))'
}
kv_split_inkling() {
kv_inkling_fixtures
local R=tiny_inkling/ref_long.json fx
kv_gate inkling "kv inkling 16 rows on the device" 1 16 4 SNAP=tiny_inkling -- 8 0 $R
kv_gate inkling "kv inkling two blocks on the device" 1 8 4 SNAP=tiny_inkling -- 8 0 $R
kv_gate inkling "kv inkling cap 1, blocks of 8" 1 24 8 SNAP=tiny_inkling -- 1 0 $R
for fx in tiny_inkling_q tiny_inkling_x-i4 tiny_inkling_x-i8; do
kv_gate inkling "kv inkling $fx" 1 16 4 SNAP=$fx -- 8 0 $R
done
kv_gate inkling "kv inkling runtime int4" 1 16 4 SNAP=tiny_inkling -- 2 4 $R
kv_gate inkling "kv inkling TOPP" 1 16 4 TOPP=0.3 SNAP=tiny_inkling_x-i4 -- 2 0 $R
kv_gate inkling "kv inkling tier off" 1 16 4 COLI_VK_TIER=0 SNAP=tiny_inkling -- 8 0 $R
kv_gate inkling "kv inkling prefill in chunks of 3" 1 16 4 COLI_VK_CHAIN_ROWS=3 SNAP=tiny_inkling -- 8 0 $R
kv_gate inkling "kv inkling tiled GEMM" 1 24 4 COLI_VK_GEMM_MIN_S=2 SNAP=tiny_inkling -- 8 0 $R
CHAINMODE=2 kv_gate inkling "kv inkling prompts only" 1 16 4 SNAP=tiny_inkling -- 8 0 $R
kv_gate inkling "kv inkling D=6144" 1 8 4 SNAP=tiny_inkling_wide -- 8 0 tiny_inkling_wide/ref_inkling.json
# COLI_VK_KV_SPLIT=0: the whole mirrors, as before the split
chain_gate inkling "kv inkling split off" 1 COLI_VK_KV_SPLIT=0 COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 SNAP=tiny_inkling -- 8 0 $R
if grep -qa 'KV cache split' vk.log; then cat vk.log; fail "kv inkling split off: the split ran"; fi
# the device lost mid-decode with the split on: K/V and convolution states rebuilt on the CPU
lost_late inkling "kv inkling device lost" 20 rebuild COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 SNAP=tiny_inkling -- 8 0 $R
# serve sessions (pins, prompt-cache extensions, a divergent prompt, logprobs), prompts
# only, and the prefix-reuse and dashboard contracts, across the split
CHAIN_SERVE_EXPECT='KV split: [0-9]+ layer steps, [1-9][0-9]* with a host part' \
$PY tests/vulkan_chain_serve.py ./inkling tiny_inkling INK_PREFIX_LOG=1 COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4
COLI_VK_CHAIN=2 CHAIN_SERVE_EXPECT='KV split: [0-9]+ layer steps, [1-9][0-9]* with a host part' \
$PY tests/vulkan_chain_serve.py ./inkling tiny_inkling INK_PREFIX_LOG=1 COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4
COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 COLI_VK_TIER_BALANCE=0 COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4 \
INKLING_TINY=tiny_inkling $PY -m unittest tests.test_inkling_prefix_serve tests.test_inkling_dashboard_hits
}
kv_split_inkling_sanitize() {
kv_inkling_fixtures
local R=tiny_inkling/ref_long.json
kv_san inkling "asan kv inkling" COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4 SNAP=tiny_inkling ./inkling 8 0 $R
kv_san inkling "asan kv inkling chunks of 3" COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 COLI_VK_CHAIN_ROWS=3 SNAP=tiny_inkling \
./inkling 8 0 $R
kv_san inkling "asan kv inkling int4 experts" COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 SNAP=tiny_inkling_x-i4 ./inkling 2 0 $R
CHAIN_SERVE_EXPECT='KV split: [0-9]+ layer steps, [1-9][0-9]* with a host part' \
$PY tests/vulkan_chain_serve.py ./inkling tiny_inkling INK_PREFIX_LOG=1 COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4 \
> san.log 2>&1 || { cat san.log; fail "asan kv inkling serve"; }
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "asan kv inkling serve: sanitizer diagnostic"; fi
echo "OK asan kv inkling serve: $(tail -1 san.log)"
}
# mimo: its full-attention layers split (the windowed ones keep their rings); the tiny
# fixture's long case (a 72-token prompt) and the picture's, each with 40 new tokens
kv_mimo_fixtures() {
$PY tools/make_mimo_tiny.py --output ./mimo_tiny --force --vision
MGRID=$($PY -c "import json;i=json.load(open('mimo_tiny/ref.json'))['image'];print(i['grid_h'],i['grid_w'])")
mimo_served_fixture
}
# mimo_kv_gate <tag> <case> <rows> <block> <env...>: the case's prompt and 40 new tokens,
# the CPU's tokens, then every position's logits over the CPU's whole sequence within
# 1e-4 of the largest (CHAINMODE: COLI_VK_CHAIN's value); steps with a host part
mimo_kv_gate() {
local tag=$1 c=$2 rows=$3 blk=$4 extra=() P lg; shift 4
[ "$c" = image ] && extra=(--image mimo_tiny/patches.f32 --grid $MGRID)
P=$(mimo_ids $c prompt_ids)
local kv=(COLI_VULKAN=1 COLI_VK_CHAIN=${CHAINMODE:-1} COLI_VK_TIER_SYNC=1 COLI_VK_KV_DEVICE_ROWS=$rows COLI_VK_KV_BLOCK=$blk)
env "$@" COLI_TEMP=0 ./mimo mimo_tiny --ids "$P" --ngen 40 "${extra[@]}" > mimo-cpu.txt 2>/dev/null
env "$@" "${kv[@]}" COLI_TEMP=0 ./mimo mimo_tiny --ids "$P" --ngen 40 "${extra[@]}" > mimo-vk.txt 2> mimo-vk.err
[ -s mimo-cpu.txt ] && cmp -s mimo-cpu.txt mimo-vk.txt || { cat mimo-cpu.txt mimo-vk.txt mimo-vk.err; fail "$tag $c: the chain's tokens differ from the CPU"; }
[ "$(kv_hostparts mimo mimo-vk.err)" -gt 0 ] || { cat mimo-vk.err; fail "$tag $c: the split never ran a host part"; }
local hp; hp=$(grep -a "KV split:" mimo-vk.err | tail -1 | sed 's/.*KV split: //')
rm -f cpu.f32 vk.f32
env "$@" MIMO_LOGITS=cpu.f32 ./mimo mimo_tiny --ids "$P $(cat mimo-cpu.txt)" --ngen 0 "${extra[@]}" > /dev/null 2>&1
env "$@" "${kv[@]}" MIMO_LOGITS=vk.f32 ./mimo mimo_tiny --ids "$P $(cat mimo-cpu.txt)" --ngen 0 "${extra[@]}" > /dev/null 2> mimo-vk.err
lg=$(logits_close cpu.f32 vk.f32) || { echo "$lg"; fail "$tag $c: logits"; }
echo "OK $tag $c: tokens = CPU, $lg; $hp"
}
kv_split_mimo() {
kv_mimo_fixtures
local bits c
for bits in 32 0 8; do mimo_kv_gate "kv mimo bits=$bits, 16 rows on the device" long 16 4 MIMO_DENSE_BITS=$bits; done
mimo_kv_gate "kv mimo two blocks on the device" long 8 4 MIMO_DENSE_BITS=32
mimo_kv_gate "kv mimo a picture" image 16 4 MIMO_DENSE_BITS=0
mimo_kv_gate "kv mimo one token at a time" long 16 4 MIMO_DENSE_BITS=32 MIMO_CHUNK=1
mimo_kv_gate "kv mimo blocks of 3" long 16 4 MIMO_DENSE_BITS=32 MIMO_CHUNK=3
mimo_kv_gate "kv mimo chunks of 3 rows" long 24 8 MIMO_DENSE_BITS=32 COLI_VK_CHAIN_ROWS=3
mimo_kv_gate "kv mimo tier off, tiled GEMM" long 16 4 MIMO_DENSE_BITS=0 COLI_VK_TIER=0 COLI_VK_GEMM_MIN_S=2
CHAINMODE=2 mimo_kv_gate "kv mimo prompts only" long 16 4 MIMO_DENSE_BITS=32
# COLI_VK_KV_SPLIT=0: the whole mirrors, as before the split
mimo_chain_gate "kv mimo split off" long MIMO_DENSE_BITS=32 COLI_VK_KV_SPLIT=0 COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4
if grep -qa 'KV cache split' mimo-vk.err; then cat mimo-vk.err; fail "kv mimo split off: the split ran"; fi
# the device lost with the split on (mid-decode; inside a prompt in chunks of 3): the
# host's caches hold whole steps, the CPU runs on from where they end
mimo_lost_gate "kv mimo device lost mid-decode" long 3 MIMO_DENSE_BITS=32 COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4
mimo_lost_gate "kv mimo device lost inside a chunked prompt" long 100 MIMO_DENSE_BITS=32 COLI_VK_CHAIN_ROWS=3 \
COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4
# serve sessions (photos restored, prompt-cache extensions, a divergent prompt,
# logprobs) frame for frame within the family's 1e-3, both modes and a lost device
local k
for k in 1 2; do
CHAIN_SERVE_EXPECT='KV split: [0-9]+ layer steps, [1-9][0-9]* with a host part' CHAIN_SERVE_TOL=1e-3 COLI_VK_CHAIN=$k \
$PY tests/vulkan_chain_serve.py ./mimo mimo_tiny_served OMP_NUM_THREADS=2 COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4
done
CHAIN_SERVE_TOL=1e-3 $PY tests/vulkan_chain_serve.py ./mimo mimo_tiny_served OMP_NUM_THREADS=2 COLI_VK_CHAIN_FAULT=60 \
COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4
}
kv_split_mimo_sanitize() {
kv_mimo_fixtures
local P; P=$(mimo_ids long prompt_ids)
kv_san mimo "asan kv mimo" COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4 MIMO_DENSE_BITS=32 COLI_TEMP=0 ./mimo mimo_tiny --ids "$P" --ngen 20
kv_san mimo "asan kv mimo blocks of 3, the FP8 trunk" COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 MIMO_DENSE_BITS=0 MIMO_CHUNK=3 \
COLI_TEMP=0 ./mimo mimo_tiny --ids "$P" --ngen 10
CHAIN_SERVE_EXPECT='KV split: [0-9]+ layer steps, [1-9][0-9]* with a host part' CHAIN_SERVE_TOL=1e-3 \
$PY tests/vulkan_chain_serve.py ./mimo mimo_tiny_served OMP_NUM_THREADS=2 COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4 \
> san.log 2>&1 || { cat san.log; fail "asan kv mimo serve"; }
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "asan kv mimo serve: sanitizer diagnostic"; fi
echo "OK asan kv mimo serve: $(tail -1 san.log)"
}
# kv_mla_gate <engine> <tag> <tol 0|1> <rows> <block> <env...> -- <argv...>: mla_gate (the
# CPU's tokens, logits within 1e-4 with tol 1; FAULT_BACK, REBUILD as there) with the
# split on, which must have run steps with a host part (PINNED=1: and pinned blocks)
kv_mla_gate() {
local eng=$1 tag=$2 tol=$3 rows=$4 blk=$5; shift 5
mla_gate "$eng" "$tag" "$tol" COLI_VK_KV_DEVICE_ROWS=$rows COLI_VK_KV_BLOCK=$blk "$@"
[ "$(kv_hostparts "$eng" vk.log)" -gt 0 ] || { cat vk.log; fail "$tag: the split never ran a host part"; }
if [ "${PINNED:-0}" = 1 ]; then grep -qa 'KV split: .* [1-9][0-9]* blocks pinned by reads' vk.log || { cat vk.log; fail "$tag: no block was pinned"; }; fi
kv_cold_check "$eng" "$tag" vk.log
echo " $tag: $(grep -a "KV split:" vk.log | tail -1 | sed 's/.*KV split: //')"
}
# colibri (GLM-5.2): the 32-token oracle over 8 or 16 rows on the device; DSA's lists
kv_split_colibri() {
[ -f glm_tiny_serve/tokenizer.json ] || glm_chain_fixtures
export CAP_RAISE=0
local G="SNAP=glm_tiny REF=ref_glm.json"
kv_mla_gate colibri "kv colibri decode, 8 rows on the device" 1 8 4 $G -- 64 16 16
kv_mla_gate colibri "kv colibri cap 1, blocks of 2" 1 8 2 $G -- 1 16 16
kv_mla_gate colibri "kv colibri prefill in chunks of 3" 1 8 4 $G TF=1 COLI_VK_CHAIN_ROWS=3 -- 64 16 16
kv_mla_gate colibri "kv colibri 4-bit trunk and experts" 1 8 4 $G IDOT=0 -- 2 4 4
kv_mla_gate colibri "kv colibri i4 container" 1 8 4 SNAP=glm_tiny_i4 REF=glm_tiny_i4/ref_glm.json IDOT=0 -- 1 4 4
# DSA's lists: the device skips the positions it does not hold, the host takes them,
# and the blocks the lists read most are pinned (chunks of 3 leave room for them)
COLI_VK_KV_PIN=1 PINNED=1 kv_mla_gate colibri "kv colibri DSA top-4, pinned blocks" 1 16 4 $G DSA_TOPK=4 COLI_VK_CHAIN_ROWS=3 -- 64 16 16
kv_mla_gate colibri "kv colibri DSA top-4, prefill in chunks of 5" 1 8 4 $G DSA_TOPK=4 TF=1 COLI_VK_CHAIN_ROWS=5 -- 64 16 16
kv_mla_gate colibri "kv colibri DSA_FORCE" 1 8 4 $G DSA_FORCE=1 -- 64 16 16
# drafts: the verify rows over the split, rejected rows rewritten
kv_mla_gate colibri "kv colibri n-gram drafts" 1 8 4 $G DRAFT=3 -- 64 16 16
kv_mla_gate colibri "kv colibri MTP depth 2" 1 8 4 SNAP=glm_tiny_mtp REF=glm_tiny_mtp/ref_glm.json DRAFT=2 -- 64 16 16
kv_mla_gate colibri "kv colibri tier off" 1 8 4 $G COLI_VK_TIER=0 -- 64 16 16
# prompts only: the prompt and the drafts' verify rows on the device, decode on the CPU
CHAINMODE=2 kv_mla_gate colibri "kv colibri prompts only, drafts" 1 8 4 $G DRAFT=3 -- 64 16 16
# COLI_VK_KV_SPLIT=0: the whole mirrors, as before the split
mla_gate colibri "kv colibri split off" 1 COLI_VK_KV_SPLIT=0 COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4 $G -- 64 16 16
if grep -qa 'KV cache split' vk.log; then cat vk.log; fail "kv colibri split off: the split ran"; fi
FAULT_BACK=12 kv_mla_gate colibri "kv colibri device lost mid-decode" 1 8 4 $G -- 64 16 16
# serve sessions frame for frame: pins, the prompt cache, the prefill read-out, two KV
# slots taking the device in turn, DSA's lists with drafts
local E='KV split: [0-9]+ layer steps, [1-9][0-9]* with a host part' K="COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4"
# shellcheck disable=SC2086
{
CHAIN_SERVE_EXPECT="$E" CHAIN_SERVE_DIALECT=colibri $PY tests/vulkan_chain_serve.py ./colibri glm_tiny_serve SERVE_BATCH=1 IDOT=0 $K
CHAIN_SERVE_EXPECT="$E" CHAIN_SERVE_DIALECT=colibri $PY tests/vulkan_chain_serve.py ./colibri glm_tiny_serve SERVE_BATCH=1 IDOT=0 DSA_TOPK=4 DRAFT=3 $K
CHAIN_SERVE_EXPECT="$E" CHAIN_SERVE_SLOTS=2 CHAIN_SERVE_DIALECT=colibri $PY tests/vulkan_chain_serve.py ./colibri glm_tiny_serve SERVE_BATCH=1 IDOT=0 KV_SLOTS=2 $K
# several conversations at once with the split on: those steps on the CPU
CHAIN_MUX_EXPECT="$E" CHAIN_MUX_STEPS=0 $PY tests/vulkan_chain_mux.py ./colibri glm_tiny_serve 3 SERVE_BATCH=1 IDOT=0 DSA_TOPK=4 $K
}
unset CAP_RAISE
}
kv_split_colibri_sanitize() {
[ -f glm_tiny_serve/tokenizer.json ] || glm_chain_fixtures
export CAP_RAISE=0
local K="COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4 USAGE_SAVE=0"
# shellcheck disable=SC2086
{
kv_san colibri "asan kv colibri decode" $K SNAP=glm_tiny REF=ref_glm.json ./colibri 64 16 16
COLI_VK_KV_PIN=1 kv_san colibri "asan kv colibri DSA top-4, pinned, chunks of 3" COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 USAGE_SAVE=0 SNAP=glm_tiny \
REF=ref_glm.json DSA_TOPK=4 COLI_VK_CHAIN_ROWS=3 ./colibri 64 16 16
kv_san colibri "asan kv colibri MTP depth 2" $K SNAP=glm_tiny_mtp REF=glm_tiny_mtp/ref_glm.json DRAFT=2 ./colibri 64 16 16
CHAIN_SERVE_EXPECT='KV split: [0-9]+ layer steps, [1-9][0-9]* with a host part' CHAIN_SERVE_SLOTS=2 CHAIN_SERVE_DIALECT=colibri \
$PY tests/vulkan_chain_serve.py ./colibri glm_tiny_serve SERVE_BATCH=1 IDOT=0 KV_SLOTS=2 $K > san.log 2>&1 \
|| { cat san.log; fail "asan kv colibri serve"; }
}
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "asan kv colibri serve: sanitizer diagnostic"; fi
echo "OK asan kv colibri serve: $(tail -1 san.log)"
unset CAP_RAISE
}
# glm53 (GLM-5.3 Flash): its MLA layer's latent rows split, the k-pooled lists through
# the split, the KDA layers untouched; the 100-token prompt and 8-token decodes
kv_split_glm53() {
[ -f glm53_serve/tokenizer.json ] || glm_chain_fixtures
local ids bits M="--model glm53_stream-i4"
ids=$($PY -c "print(','.join(str((i*37+5)%120+2) for i in range(100)))")
kv_mla_gate glm53 "kv glm53 decode, 8 rows on the device" 1 8 4 GLM53_BITS=32 -- $M --ids 5,7,9,11,13,17,19,23 --greedy 8
for bits in 32 8 4; do
kv_mla_gate glm53 "kv glm53 100-token prompt, ${bits}-bit trunk" 1 16 4 GLM53_BITS=$bits -- $M --ids $ids --greedy 8
done
COLI_VK_KV_PIN=1 PINNED=1 kv_mla_gate glm53 "kv glm53 chain chunks of 3, pinned blocks" 1 16 4 GLM53_BITS=32 COLI_VK_CHAIN_ROWS=3 -- $M --ids $ids --greedy 8
kv_mla_gate glm53 "kv glm53 prefill chunks of 7, one cache slot" 1 16 4 GLM53_BITS=4 GLM53_PREFILL_CHUNK=7 GLM53_EXPERT_GB=0.000001 \
-- $M --ids $ids --greedy 4
kv_mla_gate glm53 "kv glm53 blocks of 2" 1 6 2 GLM53_BITS=32 -- $M --ids $ids --greedy 6
kv_mla_gate glm53 "kv glm53 resident experts" 1 16 4 GLM53_BITS=32 -- --model glm53_tiny --ids $ids --greedy 6
kv_mla_gate glm53 "kv glm53 tier off" 1 16 4 GLM53_BITS=32 COLI_VK_TIER=0 -- $M --ids $ids --greedy 6
kv_mla_gate glm53 "kv glm53 an image" 1 4 2 GLM53_BITS=32 -- --model glm53_mm_tiny --ids 103,117,268,268,268,268,120,121 \
--patches glm53_mm_tiny/patches.f32 --grid 4x4 --greedy 4
CHAINMODE=2 kv_mla_gate glm53 "kv glm53 prompts only" 1 16 4 GLM53_BITS=32 GLM53_PREFILL_CHUNK=16 -- $M --ids $ids --greedy 6
# COLI_VK_KV_SPLIT=0: the whole mirrors, as before the split
mla_gate glm53 "kv glm53 split off" 1 COLI_VK_KV_SPLIT=0 COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 GLM53_BITS=32 -- $M --ids $ids --greedy 4
if grep -qa 'KV cache split' vk.log; then cat vk.log; fail "kv glm53 split off: the split ran"; fi
FAULT_BACK=7 REBUILD=1 kv_mla_gate glm53 "kv glm53 device lost, the KDA state rebuilt" 1 16 4 GLM53_BITS=32 GLM53_PREFILL_CHUNK=16 \
-- $M --ids $ids --greedy 6
# serve sessions frame for frame (pins, the prompt cache, read-outs), two sessions
# taking the device in turn; a pin restored over rows another branch rewrote
local E='KV split: [0-9]+ layer steps, [1-9][0-9]* with a host part' K="COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4"
# shellcheck disable=SC2086
{
CHAIN_SERVE_EXPECT="$E" CHAIN_SERVE_DIALECT=numeric $PY tests/vulkan_chain_serve.py ./glm53 glm53_serve GLM53_BITS=32 $K
CHAIN_SERVE_EXPECT="$E" CHAIN_SERVE_DIALECT=numeric $PY tests/vulkan_chain_serve.py ./glm53 glm53_serve GLM53_BITS=4 COLI_VK_CHAIN_ROWS=3 $K
CHAIN_SERVE_EXPECT="$E" CHAIN_SERVE_SLOTS=2 CHAIN_SERVE_DIALECT=numeric $PY tests/vulkan_chain_serve.py ./glm53 glm53_serve GLM53_BITS=32 KV_SLOTS=2 $K
env $K COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_USAGE=$PWD/chain.usage COLI_VK_TIER=0 $PY tests/glm53_pin_branch_harness.py --binary ./glm53 --fixture glm53_mm_tiny --tol 1e-5
}
}
kv_split_glm53_sanitize() {
[ -f glm53_serve/tokenizer.json ] || glm_chain_fixtures
local ids K="COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 USAGE_SAVE=0"
ids=$($PY -c "print(','.join(str((i*37+5)%120+2) for i in range(100)))")
# shellcheck disable=SC2086
{
kv_san glm53 "asan kv glm53 100-token prompt" $K GLM53_BITS=32 ./glm53 --model glm53_stream-i4 --ids $ids --greedy 6
COLI_VK_KV_PIN=1 kv_san glm53 "asan kv glm53 int4 trunk, chunks of 3, pinned" $K GLM53_BITS=4 COLI_VK_CHAIN_ROWS=3 ./glm53 --model glm53_stream-i4 --ids $ids --greedy 6
kv_san glm53 "asan kv glm53 device lost, rebuilt" $K GLM53_BITS=32 GLM53_PREFILL_CHUNK=16 COLI_VK_CHAIN_FAULT=20 ./glm53 \
--model glm53_stream-i4 --ids $ids --greedy 6
CHAIN_SERVE_EXPECT='KV split: [0-9]+ layer steps, [1-9][0-9]* with a host part' CHAIN_SERVE_SLOTS=2 CHAIN_SERVE_DIALECT=numeric \
$PY tests/vulkan_chain_serve.py ./glm53 glm53_serve GLM53_BITS=32 KV_SLOTS=2 COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4 > san.log 2>&1 \
|| { cat san.log; fail "asan kv glm53 serve"; }
}
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "asan kv glm53 serve: sanitizer diagnostic"; fi
echo "OK asan kv glm53 serve: $(tail -1 san.log)"
}
# kv_k3c_gate <tag> <rows> <block> <env...>: k3c_gate (kimi's three cases, logits within
# 2e-3 of the largest) with the split on; the last case run (long, 72 prompt tokens)
# must have had steps with a host part
kv_k3c_gate() {
local tag=$1 rows=$2 blk=$3; shift 3
k3c_gate "$tag" "$@" COLI_VK_KV_DEVICE_ROWS=$rows COLI_VK_KV_BLOCK=$blk
[ "$(kv_hostparts kimi_k3 vk.log)" -gt 0 ] || { cat vk.log; fail "$tag: the split never ran a host part"; }
echo " $tag: $(grep -a "KV split:" vk.log | tail -1 | sed 's/.*KV split: //')"
}
# kimi_k3: its gated MLA layers (NoPE) split, the KDA layers untouched
kv_split_kimi() {
[ -f kimi_k3_tiny/model.safetensors ] || $PY tools/make_kimi_k3_tiny.py --output ./kimi_k3_tiny --force
k3c_serve_fixture
local O="K3_BITS=32 K3_MLA_BITS=32 K3_HEAD_BITS=32 K3_IDOT=0 COLI_TEMP=0" b
# shellcheck disable=SC2086
{
kv_k3c_gate "kv kimi_k3 f32, 16 rows on the device" 16 4 $O
kv_k3c_gate "kv kimi_k3 f32, blocks of 2" 8 2 $O
kv_k3c_gate "kv kimi_k3 f32, tier off" 16 4 $O COLI_VK_TIER=0
for b in 8 4; do kv_k3c_gate "kv kimi_k3 ${b}-bit trunk" 16 4 K3_BITS=$b K3_MLA_BITS=$b K3_HEAD_BITS=$b K3_IDOT=0 COLI_TEMP=0; done
kv_k3c_gate "kv kimi_k3 prefill one token at a time" 16 4 $O K3_CHUNK=1
kv_k3c_gate "kv kimi_k3 chain chunks of 3" 16 4 $O COLI_VK_CHAIN_ROWS=3
CHAINMODE=2 kv_k3c_gate "kv kimi_k3 prompts only" 16 4 $O
# COLI_VK_KV_SPLIT=0: the whole mirrors, as before the split
K3C_CASES=long k3c_gate "kv kimi_k3 split off" $O COLI_VK_KV_SPLIT=0 COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4
if grep -qa 'KV cache split' vk.log; then cat vk.log; fail "kv kimi_k3 split off: the split ran"; fi
# the device lost: mid-decode (the KDA state rebuilt), and inside the long prompt
FAULT_BACK=3 REBUILD=1 kv_k3c_gate "kv kimi_k3 device lost mid-decode" 16 4 $O
K3C_CASES=long FAULT_BACK=150 kv_k3c_gate "kv kimi_k3 device lost in the prompt" 16 4 $O
}
# serve sessions frame for frame with recurrent-state checkpoints, and the checkpoint
# tests, across the split
local S="K3_BITS=32 K3_MLA_BITS=32 K3_HEAD_BITS=32 K3_IDOT=0 K3_PREFIX_LOG=1 USAGE_SAVE=0" K="COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4"
local E='KV split: [0-9]+ layer steps, [1-9][0-9]* with a host part'
# shellcheck disable=SC2086
{
CHAIN_SERVE_TOL=2e-2 CHAIN_SERVE_EXPECT="$E" $PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve $S $K
CHAIN_SERVE_TOL=2e-2 CHAIN_SERVE_EXPECT="$E" $PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve $S COLI_K3_CKPT=4 $K
CHAIN_SERVE_TOL=2e-2 CHAIN_SERVE_EXPECT="$E" $PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve K3_BITS=4 K3_IDOT=0 K3_PREFIX_LOG=1 \
USAGE_SAVE=0 COLI_VK_CHAIN_ROWS=3 $K
env $K COLI_USAGE=$PWD/k3c.usage USAGE_SAVE=0 COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 \
$PY tests/test_kimi_k3_ckpt.py --binary ./kimi_k3 --fixture ./kimi_k3_tiny
}
}
kv_split_kimi_sanitize() {
[ -f kimi_k3_tiny/model.safetensors ] || $PY tools/make_kimi_k3_tiny.py --output ./kimi_k3_tiny --force
k3c_serve_fixture
local O="K3_BITS=32 K3_MLA_BITS=32 K3_HEAD_BITS=32 K3_IDOT=0 COLI_TEMP=0 USAGE_SAVE=0" K="COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4" ids
ids=$(k3c_ids long)
# shellcheck disable=SC2086
{
kv_san kimi_k3 "asan kv kimi_k3 f32" $K $O ./kimi_k3 kimi_k3_tiny --ids "$ids" --ngen 8
kv_san kimi_k3 "asan kv kimi_k3 int4 trunk, chunks of 3" $K K3_BITS=4 K3_IDOT=0 USAGE_SAVE=0 COLI_VK_CHAIN_ROWS=3 ./kimi_k3 kimi_k3_tiny \
--ids "$ids" --ngen 8
kv_san kimi_k3 "asan kv kimi_k3 prompts only" $K $O COLI_VK_CHAIN=2 ./kimi_k3 kimi_k3_tiny --ids "$ids" --ngen 8
CHAIN_SERVE_TOL=2e-2 CHAIN_SERVE_EXPECT='KV split: [0-9]+ layer steps, [1-9][0-9]* with a host part' \
$PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve K3_BITS=32 K3_MLA_BITS=32 K3_HEAD_BITS=32 K3_IDOT=0 K3_PREFIX_LOG=1 \
USAGE_SAVE=0 COLI_K3_CKPT=4 COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4 > san.log 2>&1 || { cat san.log; fail "asan kv kimi_k3 serve"; }
}
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "asan kv kimi_k3 serve: sanitizer diagnostic"; fi
echo "OK asan kv kimi_k3 serve: $(tail -1 san.log)"
}
# deepseek_v41: the compressed rows of its kv_source layers split (ratios 2 and 1 in the
# fixtures; the window ring and the index keys stay whole on the device). kv_v41_gate:
# v41_gate (the CPU's tokens, every forward's logits within 1e-4 of the largest) under an
# emulated small device, which must have run attention steps with a host part.
kv_v41_gate() { # <tag> <rows> <block> <env...> -- <argv...>
local tag=$1 rows=$2 blk=$3; shift 3
v41_gate "$tag" COLI_VK_KV_DEVICE_ROWS=$rows COLI_VK_KV_BLOCK=$blk "$@"
if [ -z "${FAULT_BACK:-}" ]; then
[ "$(kv_hostparts deepseek_v41 vk.log)" -gt 0 ] || { cat vk.log; fail "$tag: the split never ran a host part"; }
fi
echo " $tag: $(grep -a "KV split:" vk.log | tail -1 | sed 's/.*KV split: //')"
}
kv_split_deepseek_v41() {
v41_chain_fixtures
local cap f K=(COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=2)
for cap in 2 8; do kv_v41_gate "kv deepseek_v41 40-token prompt cap=$cap" 16 4 SNAP=dsv41_long -- $cap dsv41_long/ref.json; done
kv_v41_gate "kv deepseek_v41 four blocks of 2" 8 2 SNAP=dsv41_long -- 8 dsv41_long/ref.json
COLI_VK_KV_PIN=1 kv_v41_gate "kv deepseek_v41 eight blocks of 4, four pinned" 32 4 SNAP=dsv41_long -- 8 dsv41_long/ref.json
kv_v41_gate "kv deepseek_v41 12-token prompt" 8 2 SNAP=dsv41_tiny -- 8 dsv41_tiny/ref.json
# DSpark: a verify is one chunk, and a chunk's new compressed rows fit the window (four
# blocks of 4 take a verify of 4 rows, eight blocks one of 12)
for f in 1 3; do kv_v41_gate "kv deepseek_v41 DSpark spec=$f" 16 4 SNAP=dsv41_long V41_DSPARK=1 V41_SPEC_FORCE=$f -- 8 dsv41_long/ref.json; done
kv_v41_gate "kv deepseek_v41 DSpark spec=5" 32 4 SNAP=dsv41_long V41_DSPARK=1 V41_SPEC_FORCE=5 -- 8 dsv41_long/ref.json
kv_v41_gate "kv deepseek_v41 prompt in chunks of 3" 16 4 SNAP=dsv41_long COLI_VK_CHAIN_ROWS=3 -- 8 dsv41_long/ref.json
CHAINMODE=2 kv_v41_gate "kv deepseek_v41 prompts only" 8 2 SNAP=dsv41_long -- 8 dsv41_long/ref.json
kv_v41_gate "kv deepseek_v41 tier off" 16 4 SNAP=dsv41_long COLI_VK_TIER=0 -- 8 dsv41_long/ref.json
kv_v41_gate "kv deepseek_v41 V41_INDEX_OWNER=1" 16 4 SNAP=dsv41_long V41_INDEX_OWNER=1 -- 8 dsv41_long/ref.json
# COLI_VK_KV_SPLIT=0: the whole mirrors, as before the split
v41_gate "kv deepseek_v41 split off" SNAP=dsv41_long COLI_VK_KV_SPLIT=0 "${K[@]}" -- 8 dsv41_long/ref.json
if grep -qa 'KV cache split' vk.log; then cat vk.log; fail "kv deepseek_v41 split off: the split ran"; fi
FAULT_BACK=5 kv_v41_gate "kv deepseek_v41 device lost mid-decode" 8 2 SNAP=dsv41_long -- 8 dsv41_long/ref.json
FAULT_BACK=60 kv_v41_gate "kv deepseek_v41 device lost in the prompt" 8 2 SNAP=dsv41_long -- 8 dsv41_long/ref.json
CHAIN_SERVE_EXPECT='KV split: [0-9]+ layer steps, [1-9][0-9]* with a host part' \
$PY tests/vulkan_chain_serve.py ./deepseek_v41 dsv41_tiny V41_DSPARK=0 "${K[@]}"
CHAIN_SERVE_EXPECT='KV split: [0-9]+ layer steps, [1-9][0-9]* with a host part' \
$PY tests/vulkan_chain_serve.py ./deepseek_v41 dsv41_tiny V41_DSPARK=1 "${K[@]}"
env "${K[@]}" COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 COLI_USAGE=$PWD/chain.usage \
COLI_DSV41_FIXTURE=$PWD/dsv41_tiny $PY -m unittest tests.test_dsv41_prefix_serve tests.test_dsv41_dspark_serve
rm -f chain.usage chain-serve.usage
}
kv_split_deepseek_v41_sanitize() {
v41_chain_fixtures
local K=(COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=2)
kv_san deepseek_v41 "asan kv deepseek_v41 40-token prompt" "${K[@]}" SNAP=dsv41_long ./deepseek_v41 8 dsv41_long/ref.json
COLI_VK_KV_PIN=1 kv_san deepseek_v41 "asan kv deepseek_v41 pinned blocks, chunks of 3" COLI_VK_KV_DEVICE_ROWS=32 COLI_VK_KV_BLOCK=4 \
COLI_VK_CHAIN_ROWS=3 SNAP=dsv41_long ./deepseek_v41 2 dsv41_long/ref.json
kv_san deepseek_v41 "asan kv deepseek_v41 DSpark spec=5" COLI_VK_KV_DEVICE_ROWS=32 COLI_VK_KV_BLOCK=4 SNAP=dsv41_long \
V41_DSPARK=1 V41_SPEC_FORCE=5 ./deepseek_v41 8 dsv41_long/ref.json
CHAIN_SERVE_EXPECT='KV split: [0-9]+ layer steps, [1-9][0-9]* with a host part' \
$PY tests/vulkan_chain_serve.py ./deepseek_v41 dsv41_tiny V41_DSPARK=1 "${K[@]}" > san.log 2>&1 || { cat san.log; fail "asan kv deepseek_v41 serve"; }
echo "OK asan kv deepseek_v41 serve: $(tail -1 san.log)"
rm -f chain.usage chain-serve.usage
}
# DeepSeek's split keeps sparse attention in its original order on the device:
# only selected missing compressed rows are staged from RAM. The exact chunk and
# decode-versus-teacher-forced checks therefore apply unchanged, including V4 bf16.
kv_v4_gate() { # <tag> <rows> <block> <fixture> <case> <env...>
local tag=$1 rows=$2 blk=$3 fx=$4 c=$5; shift 5
v4_chain_gate "$tag" "$fx" "$c" COLI_VK_KV_DEVICE_ROWS=$rows COLI_VK_KV_BLOCK=$blk "$@"
if [ -z "${FAULT_BACK:-}" ]; then
[ "$(kv_hostparts deepseek_v4 v4-vk.err)" -gt 0 ] || { cat v4-vk.err; fail "$tag: the split never ran a host part"; }
fi
echo " $tag: $(grep -a "KV split:" v4-vk.err | tail -1 | sed 's/.*KV split: //')"
}
kv_split_deepseek_v4() {
v4_chain_fixtures
local c fx
# the compressed case makes 6 rows at ratio 4 (four blocks of 1 on the device: a split
# holds three blocks at least), the long one 19 (four blocks of 2)
for fx in deepseek_v4_tiny_t deepseek_v4_tiny_e8; do
kv_v4_gate "kv deepseek_v4 $fx compressed" 4 1 $fx compressed
kv_v4_gate "kv deepseek_v4 $fx long" 8 2 $fx long
done
COLI_VK_KV_PIN=1 kv_v4_gate "kv deepseek_v4 eight blocks of 2, four pinned" 16 2 deepseek_v4_tiny_t long
kv_v4_gate "kv deepseek_v4 2 output groups" 8 2 deepseek_v4_tiny_g2 long
kv_v4_gate "kv deepseek_v4 a window of 4, ratios 4 and 2" 8 2 deepseek_v4_tiny_w4 long
kv_v4_gate "kv deepseek_v4 an indexer of 64 heads of 128" 8 2 deepseek_v4_tiny_ix long
kv_v4_gate "kv deepseek_v4 tier off" 8 2 deepseek_v4_tiny_t long COLI_VK_TIER=0 COLI_VK_DENSE=0
kv_v4_gate "kv deepseek_v4 chain chunks of 3" 16 2 deepseek_v4_tiny_e8 long COLI_VK_CHAIN_ROWS=3
kv_v4_gate "kv deepseek_v4 prefill chunks of 7" 16 2 deepseek_v4_tiny_t long V4_PREFILL_CHUNK=7
CHAINMODE=2 kv_v4_gate "kv deepseek_v4 prompts only" 8 2 deepseek_v4_tiny_t long
# n-gram drafts (V4_DRAFT): accepted and rejected, every one rejected, a mix
local A=ids:20,21,22,23,24,25,26,27,28,29,20,21,22:12 R=ids:20,21,22,23,50,51,52,20,21,22:12
local M=ids:30,31,32,33,34,35,60,61,30,31,32,33,34,35,36,37,38,39,40,41,30,31,32:12
kv_v4_gate "kv deepseek_v4 n-gram drafts accepted and rejected" 4 1 deepseek_v4_tiny_t $A V4_DRAFT=4 V4_NGRAM_PARTIAL_KEEP=1
kv_v4_gate "kv deepseek_v4 an n-gram draft rejected" 4 1 deepseek_v4_tiny_t $R V4_DRAFT=4 V4_NGRAM_PARTIAL_KEEP=1
kv_v4_gate "kv deepseek_v4 n-gram drafts, a window of 4" 4 2 deepseek_v4_tiny_w4 $M V4_DRAFT=4 V4_NGRAM_PARTIAL_KEEP=1
# The same model arithmetic with full mirrors, a split, different chunking and
# read-based pins. These comparisons are bit-exact and have no logit tolerance.
local K=(COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=2)
v4_chain_same "kv deepseek_v4 split = full mirror" deepseek_v4_tiny_t long "COLI_VK_KV_SPLIT=0" "COLI_VK_KV_SPLIT=1" "${K[@]}"
v4_chain_same "kv deepseek_v4 chunks 1 = 3" deepseek_v4_tiny_t long "COLI_VK_CHAIN_ROWS=1" "COLI_VK_CHAIN_ROWS=3" "${K[@]}"
v4_chain_same "kv deepseek_v4 pinned residency" deepseek_v4_tiny_e8 long "COLI_VK_KV_PIN=0" "COLI_VK_KV_PIN=1" "${K[@]}"
v4_chain_same "kv deepseek_v4 draft chunks" deepseek_v4_tiny_w4 $M "COLI_VK_CHAIN_ROWS=1" "COLI_VK_CHAIN_ROWS=3" "${K[@]}" V4_DRAFT=4 V4_NGRAM_PARTIAL_KEEP=1
# COLI_VK_KV_SPLIT=0: the whole mirrors, as before the split
v4_chain_gate "kv deepseek_v4 split off" deepseek_v4_tiny_t long COLI_VK_KV_SPLIT=0 COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=2
if grep -qa 'KV cache split' v4-vk.err; then cat v4-vk.err; fail "kv deepseek_v4 split off: the split ran"; fi
FAULT_BACK=3 kv_v4_gate "kv deepseek_v4 device lost mid-decode" 8 2 deepseek_v4_tiny_t long
FAULT_BACK=40 kv_v4_gate "kv deepseek_v4 device lost in the prompt" 8 2 deepseek_v4_tiny_t long
# serve: a pin, its extension, a read-only prompt, a shorter one (texts and reuse =
# CPU, logprobs within 0.5)
COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=2 v4_chain_serve deepseek_v4_tiny_t
rm -rf deepseek_v4_tiny_t deepseek_v4_tiny_e8 deepseek_v4_tiny_g2 deepseek_v4_tiny_w4 deepseek_v4_tiny_ix chain.usage
rm -f cpu.f32 vk.f32 v4-*.f32 v4-*.json v4-*.err v4-tf.txt
}
kv_split_deepseek_v4_sanitize() {
# the V4 sanitized build as its chain family makes it (no LTO, the sanitizers at the link)
local SAN="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
make deepseek-v4-clean >/dev/null 2>&1 || true
make deepseek-v4 VK=1 LTO=0 EXTRA_CFLAGS="$SAN" EXTRA_LDFLAGS="$SAN"
v4_chain_fixtures
local p K=(COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=2)
p=$(v4_chain_prompt deepseek_v4_tiny_t long)
kv_v4_san() { # <tag> <env and argv...>: v4_chain_san, and steps with a host part
v4_chain_san "$@"
[ -n "${FAULT_BACK:-}" ] || [ "$(kv_hostparts deepseek_v4 san.log)" -gt 0 ] || { cat san.log; fail "$1: the split never ran a host part"; }
}
kv_v4_san "asan kv deepseek_v4 long, 4 experts" "${K[@]}" ./deepseek_v4 ./deepseek_v4_tiny_t "$p" --raw-prompt --max-tokens 4
kv_v4_san "asan kv deepseek_v4 chunks of 3, a window of 4" "${K[@]}" COLI_VK_CHAIN_ROWS=3 ./deepseek_v4 ./deepseek_v4_tiny_w4 \
"$(v4_chain_prompt deepseek_v4_tiny_w4 long)" --raw-prompt --max-tokens 4
COLI_VK_KV_PIN=1 kv_v4_san "asan kv deepseek_v4 pinned blocks, 64 index heads" COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=2 ./deepseek_v4 \
./deepseek_v4_tiny_ix "$(v4_chain_prompt deepseek_v4_tiny_ix long)" --raw-prompt --max-tokens 4
kv_v4_san "asan kv deepseek_v4 drafts" COLI_VK_KV_DEVICE_ROWS=4 COLI_VK_KV_BLOCK=1 V4_DRAFT=4 V4_NGRAM_PARTIAL_KEEP=1 ./deepseek_v4 \
./deepseek_v4_tiny_w4 "$(v4_chain_prompt x ids:30,31,32,33,34,35,60,61,30,31,32,33,34,35,36,37,38,39,40,41,30,31,32)" --raw-prompt --max-tokens 12
FAULT_BACK=10 kv_v4_san "asan kv deepseek_v4 device lost" "${K[@]}" ./deepseek_v4 ./deepseek_v4_tiny_t "$p" --raw-prompt --max-tokens 4
COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=2 v4_chain_serve deepseek_v4_tiny_t > san.log 2>&1 || { cat san.log; fail "asan kv deepseek_v4 served"; }
echo "OK asan kv deepseek_v4 served: $(tail -1 san.log | cut -c1-120)"
rm -rf deepseek_v4_tiny_t deepseek_v4_tiny_e8 deepseek_v4_tiny_g2 deepseek_v4_tiny_w4 deepseek_v4_tiny_ix san.json san.log
make deepseek-v4-clean >/dev/null 2>&1 || true
}
# kv-split: every chain engine but DeepSeek's (whose fixtures want another transformers)
family_kv_split() {
export OMP_NUM_THREADS=2
make qwen36 qwen38 olmoe inkling mimo colibri glm53 kimi_k3 tests/test_vk_chain VK=1
./tests/test_vk_chain shaders/qmatmul.spv | tee vk_chain.log
tail -1 vk_chain.log | grep -qx PASS || fail "the chain's ops"
grep -a 'kvs ' vk_chain.log
kv_split_qwen36; kv_split_qwen38; kv_split_olmoe; kv_split_inkling; kv_split_mimo
kv_split_colibri; kv_split_glm53; kv_split_kimi
unset OMP_NUM_THREADS
}
family_kv_split_sanitize() {
make clean >/dev/null 2>&1 || true
make qwen36 qwen38 olmoe inkling mimo colibri glm53 kimi_k3 tests/test_vk_chain VK=1 EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1
./tests/test_vk_chain shaders/qmatmul.spv > san.log 2>&1 || true
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log || ! tail -1 san.log | grep -qx PASS; then cat san.log; fail "asan: the chain's ops"; fi
echo "OK asan: the chain's ops, the split KV cache's among them"
kv_split_qwen36_sanitize; kv_split_qwen38_sanitize; kv_split_olmoe_sanitize; kv_split_inkling_sanitize
kv_split_mimo_sanitize; kv_split_colibri_sanitize; kv_split_glm53_sanitize; kv_split_kimi_sanitize
make clean >/dev/null 2>&1 || true
}
# COLI_VK_KV_COLD=device: every gate of the two families above again, with the host's
# part of each split layer's attention on the device too (vkc_kv_shadow). The tokens and
# logits are held to the CPU's as before, and each gate must have run the host's part on
# the device (kv_cold_check). DeepSeek's sparse forms keep it on the CPU.
family_kv_split_cold() {
export COLI_VK_KV_COLD=device
family_kv_split
unset COLI_VK_KV_COLD
}
family_kv_split_cold_sanitize() {
export COLI_VK_KV_COLD=device
family_kv_split_sanitize
unset COLI_VK_KV_COLD
}
family_kv_split_deepseek() {
export OMP_NUM_THREADS=2
make deepseek_v41 deepseek-v4 VK=1
kv_split_deepseek_v41; kv_split_deepseek_v4
unset OMP_NUM_THREADS
}
family_kv_split_deepseek_sanitize() {
make clean >/dev/null 2>&1 || true
# deepseek_v4's sanitized build is its own (no LTO, the sanitizers at the link too):
# kv_split_deepseek_v4_sanitize makes it, as v4_chain_family_sanitize does
make deepseek_v41 VK=1 EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1
kv_split_deepseek_v41_sanitize; kv_split_deepseek_v4_sanitize
make clean >/dev/null 2>&1 || true
}
# The decision engines (docs/vulkan.md, "The decision engines"): laya and gliner_decide
# with their forward on the device (COLI_VK_CHAIN=1, chain_enc.comp) and matrix by matrix
# (COLI_VK_CHAIN=0), Clef's DECIDE on qwen36's per-matrix path and its chain, in f32, int8
# and f16 (fmt 14), the rows read in place and copied. Each configuration against the CPU
# (tests/vulkan_decide_compare.py: every case of the fixture, the same decisions, the
# probabilities within 1e-5 and the logits within 1e-4 of the largest, matmuls on the
# device); the tiny oracles through the device; the device opened with every path off,
# byte for byte the CPU's; a device lost mid-run (the rest of the forwards on the CPU);
# the serve tests and TypeSafe's SDKs (where installed) through the device, checked by
# the engines' own lines. SAN=1 (decide-sanitize): a sanitized build, the comparisons
# and the serve tests, UBSan halting on its first report.
decide_vk_lines() { # <log> <tag>: every decision engine's forwards ran on the device
local e
for e in laya gliner_decide; do
grep -qE "^\[VK\] $e chain: [1-9][0-9]* forwards on the device" "$1" || { tail -50 "$1"; fail "$2: $e never ran on the device"; }
done
grep -qE "^\[VK\] qwen36 chain: [1-9][0-9]* forwards" "$1" || { tail -50 "$1"; fail "$2: Clef never ran on the device"; }
}
family_decide() {
local SANF=""
if [ -n "${SAN:-}" ]; then
SANF="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
make clean >/dev/null 2>&1 || true
export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1
fi
make laya gliner_decide qwen36 tests/test_qwen36_weight_alloc VK=1 EXTRA_CFLAGS="$SANF"
COLI_VULKAN=0 ./tests/test_qwen36_weight_alloc
COLI_VULKAN=1 ./tests/test_qwen36_weight_alloc
make laya-tiny-generate gliner-decide-tiny-generate clef-tiny-generate
export OMP_NUM_THREADS=2
local c e
local -a cmp=(
"laya COLI_VK_CHAIN=1" "laya COLI_VK_CHAIN=0" "gliner COLI_VK_CHAIN=1" "gliner COLI_VK_CHAIN=0"
"laya COLI_VK_CHAIN=1 COLI_VK_STAGED=1" "gliner COLI_VK_CHAIN=1 COLI_VK_STAGED=1"
"laya COLI_VK_CHAIN=1 COLI_VK_CHAIN_FAULT=10" "gliner COLI_VK_CHAIN=1 COLI_VK_CHAIN_FAULT=10"
"clef" "clef COLI_VK_CHAIN=1"
"clef COLI_VK_CHAIN=1 COLI_VK_CHAIN_ROWS=1 COLI_VK_CHAIN_FAULT=10 COLI_VK_DENSE_HOST=0"
"clef COLI_DENSE_I8=1 COLI_DENSE_BITS=16 COLI_VK_CHAIN=1 COLI_VK_CHAIN_ROWS=1 COLI_VK_CHAIN_FAULT=10 COLI_VK_DENSE_HOST=0 COLI_VK_IMPORT=0"
"clef COLI_DENSE_I8=1 COLI_DENSE_IDOT=0 COLI_VK_CHAIN=1 COLI_VK_DENSE_HOST=0"
"clef COLI_DENSE_I8=1 COLI_DENSE_BITS=16 COLI_VK_CHAIN=1 COLI_VK_DENSE_HOST=0"
"clef COLI_DENSE_I8=1 COLI_DENSE_IDOT=0" "clef COLI_DENSE_I8=1 COLI_DENSE_IDOT=0 COLI_VK_CHAIN=1"
"clef COLI_DENSE_I8=1 COLI_DENSE_IDOT=0 COLI_VK_IMPORT=0"
"clef COLI_DENSE_I8=1 COLI_DENSE_BITS=16" "clef COLI_DENSE_I8=1 COLI_DENSE_BITS=16 COLI_VK_CHAIN=1"
"clef COLI_DENSE_I8=1 COLI_DENSE_BITS=16 COLI_VK_IMPORT=0")
for c in "${cmp[@]}"; do
# shellcheck disable=SC2086
$PY tests/vulkan_decide_compare.py $c > cmp.log 2>&1 || { cat cmp.log; fail "decide: $c"; }
tail -1 cmp.log
done
for e in laya gliner clef; do
$PY tests/vulkan_decide_compare.py --off $e COLI_VK_CHAIN=0 COLI_VK_DENSE=0 > cmp.log 2>&1 || { cat cmp.log; fail "decide: $e with every path off"; }
tail -1 cmp.log
done
[ -n "${SAN:-}" ] || for c in 0 1; do
COLI_VULKAN=1 COLI_VK_CHAIN=$c LAYA_TINY_REQUIRED=1 GLINER_DECIDE_TINY_REQUIRED=1 CLEF_TINY_REQUIRED=1 \
$PY -m unittest tests.test_laya_tiny tests.test_gliner_decide_tiny tests.test_clef_tiny > oracle.log 2>&1 \
|| { cat oracle.log; fail "decide: the tiny oracles through the device, COLI_VK_CHAIN=$c"; }
echo "OK decide: the tiny oracles (Laya, GLiNER2.5-Decide, Clef) through the device, COLI_VK_CHAIN=$c"
done
COLI_VULKAN=1 COLI_VK_CHAIN=1 LAYA_TINY_REQUIRED=1 GLINER_DECIDE_TINY_REQUIRED=1 CLEF_TINY_REQUIRED=1 \
$PY -m unittest tests.test_decision_serve > serve.log 2>&1 || { tail -80 serve.log; fail "decide: the serve tests through the device"; }
decide_vk_lines serve.log "decide serve"
echo "OK decide: the serve tests (tests.test_decision_serve) through the device: $(tail -1 serve.log)"
if [ -z "${SAN:-}" ] && $PY -c "import typesafe_sdk" 2>/dev/null && [ -d "${JEV_TS_SDK_DIR:-tools/jev-sdk}/node_modules" ]; then
make mimo VK=1 mimo-tiny-generate
rm -f sdk-serve.log
COLI_VULKAN=1 COLI_VK_CHAIN=1 JEV_SDK_REQUIRED=1 JEV_SDK_SERVE_LOG=sdk-serve.log \
$PY -m unittest tests.test_jev_sdk > sdk.log 2>&1 || { tail -80 sdk.log; fail "decide: TypeSafe's SDKs through the device"; }
decide_vk_lines sdk-serve.log "decide SDKs"
echo "OK decide: TypeSafe's Python and TypeScript SDKs (tests.test_jev_sdk) through the device"
elif [ -z "${SAN:-}" ]; then
echo "SKIP decide: TypeSafe's SDKs are not installed (tools/requirements-jev-sdk.txt, npm ci in tools/jev-sdk)"
fi
rm -f cmp.log oracle.log serve.log sdk.log sdk-serve.log
unset OMP_NUM_THREADS
[ -z "${SAN:-}" ] || make clean >/dev/null 2>&1 || true
}
# Several conversations at once (KV_SLOTS, tests/serve_mux_check.py): every engine that
# decodes its serve's conversations together, each request's frames against the same
# request served alone. On the CPU bit for bit (with KV_SLOTS 4, and 3 with prompts three
# times longer), then with the routed-expert tier on the device within 1e-4 (a batch's
# rows go through the device's GEMM, which sums in another order than one row's).
# SAN=1: the same under ASan and UBSan, no diagnostic. One VK=1 build per engine runs
# both: without COLI_VULKAN it is the CPU engine. colibri and glm53 have theirs in
# glm-chain (vulkan_chain_mux.py), with the dense chain.
mux_gate() { # <engine> <snapshot> <slots> <env...>
local eng=$1 snap=$2 n=$3 out; shift 3
local mx=(MUX_LONG=${MUX_LONG:-1} MUX_TOL=${MUX_TOL:-0})
if [ "${SAN:-0}" = 1 ]; then
env "${mx[@]}" ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1 \
$PY tests/serve_mux_check.py ./$eng $snap $n OMP_NUM_THREADS=2 "$@" > san.log 2>&1 || { cat san.log; fail "asan mux $eng $n $*"; }
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "asan mux $eng $n $*: sanitizer diagnostic"; fi
out=$(tail -1 san.log)
else
out=$(env "${mx[@]}" $PY tests/serve_mux_check.py ./$eng $snap $n "$@") || { echo "$out"; fail "mux $eng $n $*"; }
fi
echo "${SAN:+asan }$out${MUX_LONG:+ (prompts $MUX_LONG times longer)}${1:+ [$*]}"
}
mux_engine() { # <engine> <snapshot> <env...>; VKENV: more environment for the Vulkan gate, VKTOL its bound
local eng=$1 snap=$2 vk=(${VKENV:-}); shift 2
mux_gate "$eng" "$snap" 4 "$@"
MUX_LONG=3 mux_gate "$eng" "$snap" 3 "$@"
MUX_TOL=${VKTOL:-1e-4} mux_gate "$eng" "$snap" 4 COLI_VULKAN=1 COLI_VK_TIER_SYNC=1 COLI_VK_TIER_BALANCE=0 COLI_VK_CHAIN=0 "${vk[@]}" "$@"
}
mux_build() { # <make targets...>
if [ "${SAN:-0}" = 1 ]; then
local F="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
make "$@" VK=1 LTO=0 EXTRA_CFLAGS="$F" EXTRA_LDFLAGS="$F"
else
make "$@" VK=1
fi
}
byte_tokenizer() { # <fixture> <vocabulary>: a one-token-per-byte tokenizer in place of its own
rm -f "$1/tokenizer.json"
$PY -c "import sys; sys.path.insert(0, 'tests'); from pathlib import Path; from prefix_serve_harness import ensure_byte_tokenizer as t; t(Path('$1'), $2)"
}
family_mux() {
make clean >/dev/null 2>&1 || true
mux_build qwen36 qwen38 olmoe inkling kimi_k3 mimo
$PY tools/make_qwen36_tiny.py --out qwen36_tiny --ref-mode full --emit-ref qwen36_tiny/ref_full.json
$PY tools/convert_qwen36.py --model qwen36_tiny --out qwen36_tiny_c --ebits 8
byte_tokenizer qwen36_tiny_c 256
mux_engine qwen36 qwen36_tiny_c
$PY tools/make_qwen38_tiny.py --out qwen38_tiny
$PY tools/make_edge_tiny_tokenizer.py --vocab-size 64 ./qwen38_tiny
mux_engine qwen38 qwen38_tiny
$PY tools/make_olmoe_tiny.py --output olmoe_tiny --force
$PY tools/convert_olmoe_merged.py --model olmoe_tiny --out olmoe_tiny_c
$PY tools/make_edge_tiny_tokenizer.py --vocab-size 128 olmoe_tiny_c
mux_engine olmoe olmoe_tiny_c
$PY tools/make_tiny_inkling.py tiny_inkling
$PY -c 'import sys; from pathlib import Path; sys.path.insert(0, "."); from tests.test_inkling_prefix_serve import ensure_tokenizer; ensure_tokenizer(Path("tiny_inkling"))'
mux_engine inkling tiny_inkling
$PY tools/make_kimi_k3_tiny.py --output ./kimi_k3_tiny --force
byte_tokenizer kimi_k3_tiny 256
# the tier's experts compute as the CPU's with K3_IDOT=0 (float activations); with
# the default int8 ones an expert's result depends on whether it is resident, which
# the reference sessions and the shared one reach in another order
VKENV=K3_IDOT=0 mux_engine kimi_k3 kimi_k3_tiny
$PY tools/make_mimo_tiny.py --output ./mimo_tiny --force --vision
mimo_served_fixture
# MiMo's logprobs reach -270: the device's sums move their last digits by up to 1.5e-4
# (6e-7 of the value), so its gate takes the family's 1e-3, as vulkan_chain_serve.py's
VKTOL=1e-3 mux_engine mimo mimo_tiny_served
make clean >/dev/null 2>&1 || true
}
family_mux_deepseek() {
make clean >/dev/null 2>&1 || true
mux_build deepseek_v41 deepseek-v4
$PY tools/make_dsv41_tiny.py --out dsv41_tiny --emit-ref dsv41_tiny/ref.json
mux_engine deepseek_v41 dsv41_tiny
# V4's fixture speaks <tNNN> tokens; the check's prompts are bytes
$PY tools/make_deepseek_v4_tiny.py --output deepseek_v4_mux --force
byte_tokenizer deepseek_v4_mux 128
mux_engine deepseek_v4 deepseek_v4_mux
make clean >/dev/null 2>&1 || true
}
case "${1:-}" in
shader) shader_formats ;;
qwen) family_qwen ;;
qwen-sanitize) family_qwen_sanitize ;;
inkling-olmoe) family_inkling_olmoe ;;
inkling-olmoe-sanitize) family_inkling_olmoe_sanitize ;;
mimo-qwenimage) family_mimo_qwenimage ;;
deepseek) family_deepseek ;;
deepseek-sanitize) family_deepseek_sanitize ;;
kimi) family_kimi ;;
kimi-mimo-sanitize) family_kimi_mimo_sanitize ;;
mimo-chain) family_mimo_chain ;;
mimo-chain-sanitize) family_mimo_chain_sanitize ;;
glm) family_glm ;;
glm-sanitize) family_glm_sanitize ;;
qwen-chain) family_qwen_chain ;;
qwen-chain-sanitize) family_qwen_chain_sanitize ;;
qwen-spec) family_qwen_spec ;;
qwen-spec-sanitize) family_qwen_spec_sanitize ;;
inkling-olmoe-chain) family_inkling_olmoe_chain ;;
inkling-olmoe-chain-sanitize) family_inkling_olmoe_chain_sanitize ;;
glm-chain) family_glm_chain ;;
glm-chain-sanitize) family_glm_chain_sanitize ;;
kimi-chain) family_kimi_chain ;;
kimi-chain-sanitize) family_kimi_chain_sanitize ;;
deepseek-chain) family_deepseek_chain ;;
deepseek-chain-sanitize) family_deepseek_chain_sanitize ;;
prefill-qwen) family_prefill_qwen ;;
prefill-qwen-sanitize) family_prefill_qwen_sanitize ;;
prefill-inkling-olmoe) family_prefill_inkling_olmoe ;;
prefill-mimo-kimi) family_prefill_mimo_kimi ;;
prefill-glm) family_prefill_glm ;;
prefill-deepseek) family_prefill_deepseek ;;
kv-split) family_kv_split ;;
kv-split-sanitize) family_kv_split_sanitize ;;
kv-split-cold) family_kv_split_cold ;;
kv-split-cold-sanitize) family_kv_split_cold_sanitize ;;
layers-dev2) family_layers_dev2 ;;
layers-dev2-sanitize) family_layers_dev2_sanitize ;;
layers-dev2-mla) family_layers_dev2_mla ;;
layers-dev2-mla-sanitize) family_layers_dev2_mla_sanitize ;;
layers-dev2-deepseek) family_layers_dev2_deepseek ;;
layers-dev2-deepseek-sanitize) family_layers_dev2_deepseek_sanitize ;;
layers-dev2-*) e=${1#layers-dev2-} # one engine's gates (the engine already built)
declare -F "ld2_$e" >/dev/null || { echo "no layers-dev2 engine $e" >&2; exit 2; }
OMP_NUM_THREADS=2 "ld2_$e" ;;
kv-split-deepseek) family_kv_split_deepseek ;;
kv-split-deepseek-sanitize) family_kv_split_deepseek_sanitize ;;
staged) family_staged ;;
*-staged) # COLI_VK_STAGED unset for the window's run, then the whole family staged
if [ "$1" = qwen-staged ]; then env -u COLI_VK_STAGED bash tests/vulkan_engines.sh staged-window; fi
COLI_VK_STAGED=1 bash tests/vulkan_engines.sh "${1%-staged}" ;;
staged-window) staged_window ;;
staged-faults) family_staged_faults ;;
staged-faults-sanitize) SAN=1 family_staged_faults ;;
decide) family_decide ;;
mux) family_mux ;;
mux-sanitize) SAN=1 family_mux ;;
mux-deepseek) family_mux_deepseek ;;
mux-deepseek-sanitize) SAN=1 family_mux_deepseek ;;
dev2) family_dev2 ;;
dev2-deepseek-kimi-mimo) family_dev2_deepseek_kimi_mimo ;;
dev2-sanitize) family_dev2_sanitize ;;
decide-sanitize) SAN=1 family_decide ;;
dense-only-*) g=${1#dense-only-}; fn=dho_family_${g//-/_}
declare -F "$fn" >/dev/null || { echo "no dense-only group ${g}" >&2; exit 2; }
"$fn" ;;
partial-*) g=${1#partial-}; fn=ptl_family_${g//-/_}
declare -F "$fn" >/dev/null || { echo "no partial-chain group ${g}" >&2; exit 2; }
"$fn" ;;
*) echo "usage: $0 mux|mux-sanitize|mux-deepseek|mux-deepseek-sanitize|decide|decide-sanitize|staged|<family>-staged|shader|qwen|qwen-sanitize|inkling-olmoe|inkling-olmoe-sanitize|mimo-qwenimage|deepseek|deepseek-sanitize|kimi|kimi-mimo-sanitize|glm|glm-sanitize|qwen-chain|qwen-chain-sanitize|qwen-spec|qwen-spec-sanitize|mimo-chain|mimo-chain-sanitize|inkling-olmoe-chain|inkling-olmoe-chain-sanitize|glm-chain|glm-chain-sanitize|kimi-chain|kimi-chain-sanitize|deepseek-chain|deepseek-chain-sanitize|dense-only-<group>[-sanitize]|partial-<group>[-sanitize]|prefill-qwen|prefill-qwen-sanitize|prefill-inkling-olmoe|prefill-mimo-kimi|prefill-glm|prefill-deepseek|kv-split|kv-split-sanitize|kv-split-cold|kv-split-cold-sanitize|layers-dev2[-mla|-deepseek][-sanitize]|layers-dev2-<engine>|kv-split-deepseek|kv-split-deepseek-sanitize|dev2|dev2-deepseek-kimi-mimo|dev2-sanitize" >&2; exit 2 ;;
esac