Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
# Sourced by tests/vulkan_engines.sh: the dense chain's layers on two devices (docs/vulkan.md,
# "Layers on two devices"). Lavapipe is opened twice (COLI_VK_DEV2=0: a second logical device
# on the same physical one, the test mode); COLI_VK_CHAIN_LAYERS puts the first layers on
# the primary device, COLI_VK_CHAIN_LAYERS2 the next ones on the second, the CPU the rest.
#
# layers-dev2 qwen36, qwen38, olmoe, mimo, inkling against their CPU runs
# layers-dev2-mla GLM-5.2 (colibri), GLM-5.3, Kimi K3
# layers-dev2-deepseek DeepSeek V4.1 and V4
# layers-dev2[-mla|-deepseek]-sanitize their paths under ASan and UBSan
# layers-dev2-<engine> one engine's gates (the engine built)
#
# Each gate (ld2_gate; ld2_mla for the MLA engines) is chain_gate's or mla_gate's (the
# CPU's tokens, logits within 1e-4 of the largest) with the split forced, and the second
# device's chain must have run with neither device lost.
# ld2_gate <engine> <tag> <n0> <n1> <env...> -- <argv...>
ld2_gate() {
local eng=$1 tag=$2 n0=$3 n1=$4; shift 4
chain_gate "$eng" "$tag" 1 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=$n0 COLI_VK_CHAIN_LAYERS2=$n1 COLI_VK_TIER_BALANCE=0 "$@"
ld2_ran "$eng" "$tag" vk.log
}
# ld2_mla <engine> <tag> <n0> <n1> <env...> -- <argv...>
ld2_mla() {
local eng=$1 tag=$2 n0=$3 n1=$4; shift 4
mla_gate "$eng" "$tag" 1 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=$n0 COLI_VK_CHAIN_LAYERS2=$n1 COLI_VK_TIER_BALANCE=0 "$@"
ld2_ran "$eng" "$tag" vk.log
}
# ld2_ran <engine> <tag> <log>: the second device's chain ran, no device was lost
ld2_ran() {
local f; f=$(ld2_forwards "$1" "$3")
[ "$f" -gt 0 ] || { cat "$3"; fail "$2: the second device's chain never ran"; }
! grep -qa 'the device is lost\|the device was lost' "$3" || { grep -a 'lost' "$3"; fail "$2: a device was lost"; }
echo " $2: the second device's chain ran $f forwards"
}
# ld2_forwards <engine> <log>: N from "[VK] <engine> dev2 chain: N forwards"
ld2_forwards() {
local n; n=$(sed -n "s/^\[VK\] $1 dev2 chain: \([0-9]*\) forwards.*/\1/p" "$2" | tail -1)
echo "${n:-0}"
}
# ld2_lost <engine> <tag> <n0> <n1> <frame> <env...> -- <argv...>: the second device lost at
# its given frame (COLI_VK_CHAIN_FAULT2): the CPU's tokens; at its first frame (the setup's
# parameter upload) LD2_EXPECT=setup: its layers stay on the CPU, nothing to rebuild; past
# it, the state rebuilt on the CPU (LD2_EXPECT=redo, an engine without recurrent state:
# the CPU redoes the step)
ld2_lost() {
local eng=$1 tag=$2 n0=$3 n1=$4 k=$5; shift 5
local envs=(); while [ "$1" != "--" ]; do envs+=("$1"); shift; done; shift
rm -f chain.usage
env "${envs[@]}" ./"$eng" "$@" > cpu.log 2>&1 || true
env "${envs[@]}" COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=$n0 COLI_VK_CHAIN_LAYERS2=$n1 COLI_VK_CHAIN_FAULT2=$k \
COLI_USAGE=chain.usage COLI_VK_TIER_SYNC=1 COLI_VK_TIER_BALANCE=0 COLI_VULKAN=1 COLI_VK_CHAIN=1 ./"$eng" "$@" > vk.log 2>&1 || true
case $eng in
colibri|glm53) mla_toks "$eng" cpu.log > cpu.tok; mla_toks "$eng" vk.log > vk.tok
{ [ -s cpu.tok ] && cmp -s cpu.tok vk.tok; } || { cat cpu.tok vk.tok; tail -20 vk.log; fail "$tag: the tokens differ from the CPU"; } ;;
*) same_tokens cpu.log vk.log "$tag" ;;
esac
grep -q "COLI_VK_CHAIN_FAULT2" vk.log || { cat vk.log; fail "$tag: the second device's fault never fired"; }
if [ "${LD2_EXPECT:-}" = setup ]; then
grep -q "^\[VK\] $eng dev2 chain: 0 of [0-9]* layers on the device: the chain stays off (layer 0 did not reach the device: the device was lost" vk.log ||
{ cat vk.log; fail "$tag: the second device's layers did not go to the CPU"; }
echo "OK $tag: tokens = CPU, the second device's layers on the CPU from the start"
return 0
fi
if [ "${LD2_EXPECT:-}" = again ]; then # the KV cache is the host's: the CPU runs the forward again
grep -q "^\[VK\] $eng dev2 chain: the device was lost; the CPU runs this forward again" vk.log ||
{ cat vk.log; fail "$tag: the CPU did not take the forward over"; }
echo "OK $tag: tokens = CPU, the CPU ran the forward again"
return 0
fi
if [ "${LD2_EXPECT:-}" = redo ]; then
grep -q "the CPU redoes the step" vk.log || { cat vk.log; fail "$tag: the CPU did not redo the step"; }
echo "OK $tag: tokens = CPU, the CPU redid the step"
return 0
fi
grep -q "rebuilding the state of [1-9]" vk.log || { cat vk.log; fail "$tag: no state was rebuilt"; }
echo "OK $tag: tokens = CPU, $(grep -o 'rebuilding the state of [0-9]* positions' vk.log | head -1)"
}
ld2_qwen36() {
[ -d qwen36_tiny_c ] || { $PY tools/make_qwen36_tiny.py --out qwen36_tiny --ref-mode full --emit-ref qwen36_tiny/ref_full.json
$PY tools/convert_qwen36.py --model qwen36_tiny --out qwen36_tiny_c --ebits 8; }
[ -d qwen36_kv_c ] || kv_qwen36_fixtures
local R=qwen36_tiny/ref_full.json T="SNAP=qwen36_tiny_c COLI_DENSE_I8=0"
ld2_gate qwen36 "ld2 qwen36 3 + 5, the head on the second device" 3 5 $T -- 8 8 $R
ld2_gate qwen36 "ld2 qwen36 2 + 3, the CPU the rest and the head" 2 3 $T -- 8 8 $R
ld2_gate qwen36 "ld2 qwen36 1 + 7, prompt in chunks of 3" 1 7 $T COLI_VK_CHAIN_ROWS=3 -- 8 8 $R
ld2_gate qwen36 "ld2 qwen36 cap 1" 4 4 $T -- 1 8 $R
CPUENV=COLI_DENSE_IDOT=0 ld2_gate qwen36 "ld2 qwen36 int8 trunk" 4 4 SNAP=qwen36_tiny_c -- 8 8 $R
ld2_gate qwen36 "ld2 qwen36 experts on both devices too" 3 5 $T COLI_VK_TIER_GB=0.00002 -- 8 8 $R
CHAINMODE=2 ld2_gate qwen36 "ld2 qwen36 prompts only" 3 5 $T -- 8 8 $R
ld2_gate qwen36 "ld2 qwen36 dense model" 2 4 SNAP=qwen38_27b_kv_c COLI_DENSE_I8=0 -- 8 8 qwen38_27b_kv/ref_full.json
# the KV cache split on both devices' attention layers
ld2_gate qwen36 "ld2 qwen36 KV split on both" 4 4 COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 SNAP=qwen36_kv_c COLI_DENSE_I8=0 \
-- 8 8 qwen36_kv/ref_full.json
[ "$(kv_hostparts qwen36 vk.log)" -gt 0 ] && [ "$(kv_hostparts "qwen36 dev2" vk.log)" -gt 0 ] ||
{ cat vk.log; fail "ld2 qwen36 KV split on both: a device's split ran no host part"; }
# with COLI_VK_KV_COLD=device the primary attends its host part on the device; the
# second device's stays on the CPU (the import is the primary's)
ld2_gate qwen36 "ld2 qwen36 KV split on both, the host's part on the device" 4 4 COLI_VK_KV_COLD=device COLI_VK_KV_DEVICE_ROWS=16 \
COLI_VK_KV_BLOCK=4 SNAP=qwen36_kv_c COLI_DENSE_I8=0 -- 8 8 qwen36_kv/ref_full.json
[ "$(kv_devcold qwen36 vk.log)" -gt 0 ] || { cat vk.log; fail "ld2 qwen36 COLI_VK_KV_COLD=device: the primary's host part never ran on the device"; }
{ grep -q "^\[VK\] qwen36 dev2 chain: no shadow for layer" vk.log && [ "$(kv_hostparts "qwen36 dev2" vk.log)" -gt 0 ]; } ||
{ cat vk.log; fail "ld2 qwen36 COLI_VK_KV_COLD=device: the second device's host part did not stay on the CPU"; }
# prompt-lookup drafts verified across both devices (rollbacks of both chains' copies)
local f
for f in accept mixed cycle row1 row3; do
ld2_gate qwen36 "ld2 qwen36 lookup $f" 3 5 COLI_DENSE_I8=0 COLI_LOOKUP=1 COLI_SPEC_GATE=0 COLI_LOOKUP_FORCE=$f \
SNAP=qwen36_kv_c -- 8 8 qwen36_kv/ref_full.json
done
# the second device lost: at its first frame, and mid-decode
LD2_EXPECT=setup ld2_lost qwen36 "ld2 qwen36 second device lost at its setup" 3 5 1 $T -- 8 8 $R
ld2_lost qwen36 "ld2 qwen36 second device lost mid-decode" 3 5 40 SNAP=qwen36_kv_c COLI_DENSE_I8=0 -- 8 8 qwen36_kv/ref_full.json
# the prefix-reuse contract with the state on both devices, and serve sessions (pins,
# prompt-cache extensions, a divergent prompt, logprobs)
QWEN36_TINY=$PWD/qwen36_tiny_c COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=3 COLI_VK_CHAIN_LAYERS2=5 \
COLI_USAGE=$PWD/chain.usage $PY tests/test_qwen36_prefix_serve.py
CHAIN_SERVE_EXPECT='qwen36 dev2 chain: [1-9][0-9]* forwards' \
$PY tests/vulkan_chain_serve.py ./qwen36 qwen36_tiny_c COLI_DENSE_I8=0 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=3 COLI_VK_CHAIN_LAYERS2=5
}
ld2_qwen38() {
[ -d qwen38_tiny ] || $PY tools/make_qwen38_tiny.py --out qwen38_tiny
[ -d qwen38_tiny_fp8 ] || $PY tools/make_qwen38_tiny.py --out qwen38_tiny_fp8 --fp8-experts
[ -d qwen38_kv_mtp ] || kv_qwen38_fixtures
local R=qwen38_tiny/ref.json T="OMP_NUM_THREADS=2 SNAP=qwen38_tiny" f b
for b in 0 1; do
ld2_gate qwen38 "ld2 qwen38 2 + 2, the head on the second device, batch=$b" 2 2 $T Q38_PREFILL_BATCH=$b -- 4 8 $R
done
ld2_gate qwen38 "ld2 qwen38 1 + 2, the CPU the rest and the head" 1 2 $T -- 4 8 $R
ld2_gate qwen38 "ld2 qwen38 1 + 3, prompt in chunks of 3" 1 3 $T COLI_VK_CHAIN_ROWS=3 -- 4 8 $R
ld2_gate qwen38 "ld2 qwen38 bf16 rows" 2 2 $T Q38_NATIVE_BF16=1 -- 4 8 $R
ld2_gate qwen38 "ld2 qwen38 fp8 experts" 2 2 OMP_NUM_THREADS=2 SNAP=qwen38_tiny_fp8 -- 2 8 qwen38_tiny_fp8/ref.json
ld2_gate qwen38 "ld2 qwen38 experts on both devices too" 2 2 $T COLI_VK_TIER_GB=0.00002 -- 4 8 $R
CHAINMODE=2 ld2_gate qwen38 "ld2 qwen38 prompts only" 2 2 $T -- 4 8 $R
# MTP drafts verified across both devices (both chains' copies rolled back)
for f in "" accept reject mixed; do
ld2_gate qwen38 "ld2 qwen38 MTP ${f:-drafting}" 2 2 OMP_NUM_THREADS=2 Q38_MTP=1 Q38_MTP_FORCE=$f SNAP=qwen38_tiny_mtp -- 2 8 qwen38_tiny_mtp/ref.json
done
# the KV split on both devices' QSA layers (layers 1 and 3), with MTP
ld2_gate qwen38 "ld2 qwen38 KV split on both" 2 2 COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 SNAP=qwen38_kv -- 4 8 qwen38_kv/ref.json
[ "$(kv_hostparts qwen38 vk.log)" -gt 0 ] && [ "$(kv_hostparts "qwen38 dev2" vk.log)" -gt 0 ] ||
{ cat vk.log; fail "ld2 qwen38 KV split on both: a device's split ran no host part"; }
ld2_gate qwen38 "ld2 qwen38 KV split on both, MTP mixed" 2 2 COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 Q38_MTP=1 \
Q38_MTP_FORCE=mixed SNAP=qwen38_kv_mtp -- 4 8 qwen38_kv_mtp/ref.json
# the second device lost: at its setup, mid-decode, and in a verify
LD2_EXPECT=setup ld2_lost qwen38 "ld2 qwen38 second device lost at its setup" 2 2 1 $T -- 4 8 $R
ld2_lost qwen38 "ld2 qwen38 second device lost mid-decode" 2 2 20 $T -- 4 8 $R
ld2_lost qwen38 "ld2 qwen38 second device lost in a verify" 2 2 15 OMP_NUM_THREADS=2 Q38_MTP=1 Q38_MTP_FORCE=mixed \
SNAP=qwen38_tiny_mtp -- 2 8 qwen38_tiny_mtp/ref.json
# serve sessions with MTP and without
CHAIN_SERVE_EXPECT='qwen38 dev2 chain: [1-9][0-9]* forwards' \
$PY tests/vulkan_chain_serve.py ./qwen38 qwen38_tiny_mtp Q38_MTP=1 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=2 COLI_VK_CHAIN_LAYERS2=2
CHAIN_SERVE_EXPECT='qwen38 dev2 chain: [1-9][0-9]* forwards' \
$PY tests/vulkan_chain_serve.py ./qwen38 qwen38_tiny_mtp COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=1 COLI_VK_CHAIN_LAYERS2=2
}
ld2_olmoe() {
[ -d olmoe_tiny_c ] || { $PY tools/make_olmoe_tiny.py --output olmoe_tiny; $PY tools/convert_olmoe_merged.py --model olmoe_tiny --out olmoe_tiny_c; }
[ -f olmoe_tiny/ref_olmoe_long.json ] || kv_olmoe_fixtures
local OR=olmoe_tiny/ref_olmoe.json T=SNAP=olmoe_tiny_c cap
ld2_gate olmoe "ld2 olmoe 2 + 2, the head on the second device" 2 2 $T -- 8 8 $OR
ld2_gate olmoe "ld2 olmoe 1 + 2, the CPU the rest and the head" 1 2 $T -- 8 8 $OR
ld2_gate olmoe "ld2 olmoe 1 + 3, prompt in chunks of 3" 1 3 $T COLI_VK_CHAIN_ROWS=3 -- 8 8 $OR
for cap in 1 2; do ld2_gate olmoe "ld2 olmoe PILOT cap=$cap" 2 2 PILOT=1 WIDE=2 $T -- $cap 8 $OR; done
ld2_gate olmoe "ld2 olmoe 4-bit experts" 2 2 $T -- 8 4 $OR
ld2_gate olmoe "ld2 olmoe experts on both devices too" 2 2 $T COLI_VK_TIER_GB=0.000025 -- 8 8 $OR
CHAINMODE=2 ld2_gate olmoe "ld2 olmoe prompts only" 2 2 $T -- 8 8 $OR
ld2_gate olmoe "ld2 olmoe KV split on both" 2 2 COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 $T -- 8 8 olmoe_tiny/ref_olmoe_long.json
[ "$(kv_hostparts olmoe vk.log)" -gt 0 ] && [ "$(kv_hostparts "olmoe dev2" vk.log)" -gt 0 ] ||
{ cat vk.log; fail "ld2 olmoe KV split on both: a device's split ran no host part"; }
LD2_EXPECT=setup ld2_lost olmoe "ld2 olmoe second device lost at its setup" 2 2 1 $T -- 8 8 $OR
LD2_EXPECT=redo ld2_lost olmoe "ld2 olmoe second device lost mid-decode" 2 2 8 $T -- 8 8 $OR
CHAIN_SERVE_EXPECT='olmoe dev2 chain: [1-9][0-9]* forwards' \
$PY tests/vulkan_chain_serve.py ./olmoe olmoe_tiny_c COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=2 COLI_VK_CHAIN_LAYERS2=2
CHAIN_SERVE_EXPECT='olmoe dev2 chain: [1-9][0-9]* forwards' \
$PY tests/vulkan_chain_serve.py ./olmoe olmoe_tiny_c PILOT=1 WIDE=2 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=1 COLI_VK_CHAIN_LAYERS2=2
}
# mimo: its own gate (mimo_chain_gate: the vendor fixture's cases, tokens and logits), with
# the split forced; the second device's chain must have run in the logits run
ld2_mimo_gate() { # <tag> <case> <n0> <n1> <env...>
local tag=$1 c=$2 n0=$3 n1=$4; shift 4
mimo_chain_gate "$tag" "$c" COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=$n0 COLI_VK_CHAIN_LAYERS2=$n1 COLI_VK_TIER_BALANCE=0 "$@"
ld2_ran mimo "$tag $c" mimo-vk.err
}
ld2_mimo() {
$PY tools/make_mimo_tiny.py --output ./mimo_tiny --force --vision
MGRID=$($PY -c "import json;i=json.load(open('mimo_tiny/ref.json'))['image'];print(i['grid_h'],i['grid_w'])")
local c
for c in $($PY -c "import json;print(' '.join(json.load(open('mimo_tiny/ref.json'))['cases']))"); do
ld2_mimo_gate "ld2 mimo 3 + 3, the head on the second device" $c 3 3 MIMO_DENSE_BITS=32
ld2_mimo_gate "ld2 mimo 1 + 2, the CPU the rest and the head" $c 1 2 MIMO_DENSE_BITS=32
done
ld2_mimo_gate "ld2 mimo 2 + 4, blocks of 3" image 2 4 MIMO_DENSE_BITS=32 MIMO_CHUNK=3
ld2_mimo_gate "ld2 mimo 1 + 5, the release's FP8 and BF16" image 1 5 MIMO_DENSE_BITS=0
ld2_mimo_gate "ld2 mimo experts on both devices too" image 3 3 MIMO_DENSE_BITS=32 MIMO_VK_EXPERTS=2 COLI_VK_TIER_GB=0.00002
ld2_mimo_gate "ld2 mimo KV split on both" image 3 3 MIMO_DENSE_BITS=32 COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=2
CHAINMODE=2 ld2_mimo_gate "ld2 mimo prompts only" image 3 3 MIMO_DENSE_BITS=32
# the second device lost at its third frame: the host's caches hold whole steps, the CPU
# runs the rest from where they end
local x=(--image mimo_tiny/patches.f32 --grid $MGRID) ids
ids=$(mimo_ids image prompt_ids)
COLI_TEMP=0 MIMO_DENSE_BITS=32 ./mimo mimo_tiny --ids "$ids" --ngen 6 "${x[@]}" > mimo-cpu.txt 2>/dev/null
COLI_TEMP=0 MIMO_DENSE_BITS=32 COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=3 \
COLI_VK_CHAIN_LAYERS2=3 COLI_VK_CHAIN_FAULT2=3 ./mimo mimo_tiny --ids "$ids" --ngen 6 "${x[@]}" > mimo-vk.txt 2> mimo-vk.err
cmp -s mimo-cpu.txt mimo-vk.txt || { cat mimo-cpu.txt mimo-vk.txt mimo-vk.err; fail "ld2 mimo second device lost: tokens differ from the CPU"; }
grep -q "COLI_VK_CHAIN_FAULT2" mimo-vk.err && grep -q "the device was lost at position" mimo-vk.err ||
{ cat mimo-vk.err; fail "ld2 mimo second device lost: the loss was not taken over"; }
echo "OK ld2 mimo second device lost: tokens = CPU, $(grep -o 'the device was lost at position [0-9]*' mimo-vk.err)"
}
ld2_inkling() {
[ -f tiny_inkling/ref_long.json ] || kv_inkling_fixtures
local R=tiny_inkling/ref_inkling.json T=SNAP=tiny_inkling b
ld2_gate inkling "ld2 inkling 3 + 5, lm_head on the second device" 3 5 $T -- 8 0 $R
ld2_gate inkling "ld2 inkling 2 + 2, the CPU the rest and lm_head" 2 2 $T -- 8 0 $R
ld2_gate inkling "ld2 inkling 1 + 7, prompt in chunks of 3" 1 7 $T COLI_VK_CHAIN_ROWS=3 -- 8 0 $R
for b in 4 8; do ld2_gate inkling "ld2 inkling runtime int$b" 3 5 $T -- 2 $b $R; done
ld2_gate inkling "ld2 inkling experts on both devices too" 3 5 $T COLI_VK_TIER_GB=0.00002 -- 8 0 $R
CHAINMODE=2 ld2_gate inkling "ld2 inkling prompts only" 3 5 $T -- 8 0 $R
# the global layer's KV split (the fixture has one global layer, 5): on the second
# device, then on the first
ld2_gate inkling "ld2 inkling KV split on the second device" 4 4 COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 $T -- 8 0 tiny_inkling/ref_long.json
[ "$(kv_hostparts "inkling dev2" vk.log)" -gt 0 ] || { cat vk.log; fail "ld2 inkling KV split on the second device: no host part"; }
ld2_gate inkling "ld2 inkling KV split on the first device" 6 2 COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 $T -- 8 0 tiny_inkling/ref_long.json
[ "$(kv_hostparts inkling vk.log)" -gt 0 ] || { cat vk.log; fail "ld2 inkling KV split on the first device: no host part"; }
LD2_EXPECT=setup ld2_lost inkling "ld2 inkling second device lost at its setup" 3 5 1 $T -- 8 0 $R
ld2_lost inkling "ld2 inkling second device lost mid-decode" 3 5 30 $T -- 8 0 $R
CHAIN_SERVE_EXPECT='inkling dev2 chain: [1-9][0-9]* forwards' \
$PY tests/vulkan_chain_serve.py ./inkling tiny_inkling INK_PREFIX_LOG=1 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=3 COLI_VK_CHAIN_LAYERS2=5
}
# colibri (GLM-5.2): the head stays on the host. glm_tiny_shx shares indexers (full,
# shared, full, shared, shared): with DSA top-4 the selection of the layer before the
# second device's first one crosses to it.
ld2_colibri() {
[ -f glm_tiny_serve/tokenizer.json ] || glm_chain_fixtures
[ -d glm_tiny_shx ] || ptl_glm_shx glm_tiny glm_tiny_shx
export CAP_RAISE=0
local G="SNAP=glm_tiny REF=ref_glm.json USAGE_SAVE=0" X="SNAP=glm_tiny_shx REF=ref_glm.json USAGE_SAVE=0" s
ld2_mla colibri "ld2 colibri 2 + 3, every layer on the devices" 2 3 $G -- 64 16 16
ld2_mla colibri "ld2 colibri 1 + 2, the CPU the rest" 1 2 $G -- 64 16 16
ld2_mla colibri "ld2 colibri cap 1" 3 2 $G -- 1 16 16
ld2_mla colibri "ld2 colibri prefill in chunks of 3" 2 3 $G TF=1 COLI_VK_CHAIN_ROWS=3 -- 64 16 16
ld2_mla colibri "ld2 colibri 4-bit trunk and experts" 2 3 $G IDOT=0 -- 2 4 4
ld2_mla colibri "ld2 colibri i4 container" 2 3 SNAP=glm_tiny_i4 REF=glm_tiny_i4/ref_glm.json USAGE_SAVE=0 IDOT=0 -- 1 4 4
ld2_mla colibri "ld2 colibri DSA top-4" 2 3 $G DSA_TOPK=4 -- 64 16 16
ld2_mla colibri "ld2 colibri DSA_FORCE" 2 3 $G DSA_FORCE=1 -- 64 16 16
for s in "1 2" "3 2"; do
ld2_mla colibri "ld2 colibri DSA top-4, a shared indexer first on the second device, ${s/ / + }" ${s% *} ${s#* } $X DSA_TOPK=4 -- 64 16 16
ld2_mla colibri "ld2 colibri DSA top-4, a shared indexer first on the second device, ${s/ / + }, prefill in chunks of 5" \
${s% *} ${s#* } $X DSA_TOPK=4 TF=1 COLI_VK_CHAIN_ROWS=5 -- 64 16 16
done
ld2_mla colibri "ld2 colibri n-gram drafts" 2 3 $G DRAFT=3 -- 64 16 16
ld2_mla colibri "ld2 colibri MTP depth 2" 2 3 SNAP=glm_tiny_mtp REF=glm_tiny_mtp/ref_glm.json USAGE_SAVE=0 DRAFT=2 -- 64 16 16
ld2_mla colibri "ld2 colibri MTP with DSA top-4" 1 2 SNAP=glm_tiny_mtp REF=glm_tiny_mtp/ref_glm.json USAGE_SAVE=0 DSA_TOPK=4 -- 64 16 16
ld2_mla colibri "ld2 colibri experts on both devices too" 2 3 $G COLI_VK_TIER_GB=0.00002 -- 64 16 16
CHAINMODE=2 ld2_mla colibri "ld2 colibri prompts only, drafts" 2 3 $G DRAFT=3 -- 64 16 16
ld2_mla colibri "ld2 colibri KV split on both" 2 3 COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4 $G DSA_TOPK=4 -- 64 16 16
[ "$(kv_hostparts colibri vk.log)" -gt 0 ] && [ "$(kv_hostparts "colibri dev2" vk.log)" -gt 0 ] ||
{ cat vk.log; fail "ld2 colibri KV split on both: a device's split ran no host part"; }
# the second device lost: at its setup, in the prompt (one forward), mid-decode
LD2_EXPECT=setup ld2_lost colibri "ld2 colibri second device lost at its setup" 2 2 1 $G -- 64 16 16
LD2_EXPECT=again ld2_lost colibri "ld2 colibri second device lost in the prompt" 2 2 2 $G TF=1 -- 64 16 16
LD2_EXPECT=again ld2_lost colibri "ld2 colibri second device lost mid-decode" 2 2 10 $G -- 64 16 16
LD2_EXPECT=again ld2_lost colibri "ld2 colibri second device lost with MTP" 2 2 10 SNAP=glm_tiny_mtp REF=glm_tiny_mtp/ref_glm.json \
USAGE_SAVE=0 DRAFT=2 -- 64 16 16
# serve sessions: pins, the prompt cache, the prefill read-out, two KV slots
CHAIN_SERVE_EXPECT='colibri dev2 chain: [1-9][0-9]* forwards' CHAIN_SERVE_DIALECT=colibri \
$PY tests/vulkan_chain_serve.py ./colibri glm_tiny_serve SERVE_BATCH=1 IDOT=0 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=2 COLI_VK_CHAIN_LAYERS2=3
CHAIN_SERVE_EXPECT='colibri dev2 chain: [1-9][0-9]* forwards' CHAIN_SERVE_DIALECT=colibri \
$PY tests/vulkan_chain_serve.py ./colibri glm_tiny_serve SERVE_BATCH=1 IDOT=0 DSA_TOPK=4 DRAFT=3 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=1 COLI_VK_CHAIN_LAYERS2=2
CHAIN_SERVE_EXPECT='colibri dev2 chain: [1-9][0-9]* forwards' CHAIN_SERVE_SLOTS=2 CHAIN_SERVE_DIALECT=colibri \
$PY tests/vulkan_chain_serve.py ./colibri glm_tiny_serve SERVE_BATCH=1 IDOT=0 KV_SLOTS=2 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=2 COLI_VK_CHAIN_LAYERS2=2
# several conversations at once on both devices; a shared indexer first on the second
# (the primary's rows' DSA lists cross), and the CPU's after it
CHAIN_MUX_EXPECT='colibri dev2 chain: [1-9][0-9]* multiplexed' \
$PY tests/vulkan_chain_mux.py ./colibri glm_tiny_serve 3 SERVE_BATCH=1 IDOT=0 DSA_TOPK=4 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=2 COLI_VK_CHAIN_LAYERS2=3
ptl_glm_shx glm_tiny_serve glm_tiny_shx_serve
for s in "1 2" "3 2"; do
CHAIN_MUX_EXPECT='colibri dev2 chain: [1-9][0-9]* multiplexed' $PY tests/vulkan_chain_mux.py ./colibri glm_tiny_shx_serve 3 \
SERVE_BATCH=1 IDOT=0 DSA_TOPK=4 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=${s% *} COLI_VK_CHAIN_LAYERS2=${s#* }
done
rm -rf glm_tiny_shx_serve chain-mux.usage
}
# glm53 (GLM-5.3 Flash): the six-layer fixture, KDA and MLA alternating (the KDA state on
# both devices); the head stays on the host
ld2_glm53() {
[ -d glm53_mm_tiny ] || glm_chain_fixtures
[ -d glm53_l6s-i4 ] || { ptl_g53_l6 glm53_l6; $PY tools/make_glm53_streaming_pair.py --fixture glm53_l6 --output glm53_l6s > /dev/null; }
[ -f glm53_l6_serve/tokenizer.json ] || { rm -rf glm53_l6_serve && cp -r glm53_l6s-i4 glm53_l6_serve
$PY tools/make_edge_tiny_tokenizer.py --vocab-size 128 ./glm53_l6_serve > /dev/null; }
local ids F="--model glm53_l6s-i4" G="GLM53_BITS=32 USAGE_SAVE=0" s
local M="--model glm53_mm_tiny --ids 103,117,268,268,268,268,120,121 --patches glm53_mm_tiny/patches.f32 --grid 4x4"
ids=$($PY -c "print(','.join(str((i*37+5)%120+2) for i in range(100)))")
for s in "2 2" "3 3" "1 5" "4 1"; do
ld2_mla glm53 "ld2 glm53 ${s/ / + }" ${s% *} ${s#* } $G -- $F --ids $ids --greedy 6
done
ld2_mla glm53 "ld2 glm53 decode" 3 3 $G -- $F --ids 5,7,9,11,13,17,19,23 --greedy 8
ld2_mla glm53 "ld2 glm53 4-bit trunk" 2 3 GLM53_BITS=4 USAGE_SAVE=0 -- $F --ids $ids --greedy 4
ld2_mla glm53 "ld2 glm53 chain chunks of 3" 2 2 $G COLI_VK_CHAIN_ROWS=3 -- $F --ids $ids --greedy 4
ld2_mla glm53 "ld2 glm53 prefill chunks of 7, one cache slot" 3 2 GLM53_BITS=4 USAGE_SAVE=0 GLM53_PREFILL_CHUNK=7 \
GLM53_EXPERT_GB=0.000001 -- $F --ids $ids --greedy 4
ld2_mla glm53 "ld2 glm53 experts on both devices too" 3 3 $G COLI_VK_TIER_GB=0.0002 -- $F --ids $ids --greedy 6
ld2_mla glm53 "ld2 glm53 an image" 2 1 $G -- $M --greedy 4
CHAINMODE=2 ld2_mla glm53 "ld2 glm53 prompts only" 3 3 $G GLM53_PREFILL_CHUNK=16 -- $F --ids $ids --greedy 6
ld2_mla glm53 "ld2 glm53 KV split on both" 2 2 COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 $G -- $F --ids $ids --greedy 8
[ "$(kv_hostparts glm53 vk.log)" -gt 0 ] && [ "$(kv_hostparts "glm53 dev2" vk.log)" -gt 0 ] ||
{ cat vk.log; fail "ld2 glm53 KV split on both: a device's split ran no host part"; }
# the second device lost: at its setup, mid-prompt and mid-decode (both devices' KDA
# state rebuilt from the primary's record), after an image
LD2_EXPECT=setup ld2_lost glm53 "ld2 glm53 second device lost at its setup" 3 3 1 $G -- $F --ids $ids --greedy 6
ld2_lost glm53 "ld2 glm53 second device lost in a prompt chunk" 3 3 12 $G GLM53_PREFILL_CHUNK=16 -- $F --ids $ids --greedy 6
ld2_lost glm53 "ld2 glm53 second device lost mid-decode" 3 3 40 $G -- $F --ids $ids --greedy 6
ld2_lost glm53 "ld2 glm53 second device lost after an image" 2 1 6 $G -- $M --greedy 4
# a pin's branch over the image fixture, its KDA state on both devices (layer 2 on the second)
rm -f chain.usage
COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_GEMM_MIN_S=0 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=2 COLI_VK_CHAIN_LAYERS2=1 \
COLI_USAGE=$PWD/chain.usage COLI_VK_TIER=0 \
$PY tests/glm53_pin_branch_harness.py --binary ./glm53 --fixture glm53_mm_tiny --tol 1e-5
rm -f chain.usage
# serve sessions: pins, the prompt cache, two KV slots
CHAIN_SERVE_EXPECT='glm53 dev2 chain: [1-9][0-9]* forwards' CHAIN_SERVE_DIALECT=numeric \
$PY tests/vulkan_chain_serve.py ./glm53 glm53_l6_serve GLM53_BITS=32 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=3 COLI_VK_CHAIN_LAYERS2=3
CHAIN_SERVE_EXPECT='glm53 dev2 chain: [1-9][0-9]* forwards' CHAIN_SERVE_DIALECT=numeric \
$PY tests/vulkan_chain_serve.py ./glm53 glm53_l6_serve GLM53_BITS=4 COLI_VK_CHAIN_ROWS=3 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=2 COLI_VK_CHAIN_LAYERS2=2
CHAIN_SERVE_EXPECT='glm53 dev2 chain: [1-9][0-9]* forwards' CHAIN_SERVE_SLOTS=2 CHAIN_SERVE_DIALECT=numeric \
$PY tests/vulkan_chain_serve.py ./glm53 glm53_l6_serve GLM53_BITS=32 KV_SLOTS=2 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=3 COLI_VK_CHAIN_LAYERS2=3
}
# kimi_k3: k3c_gate (the CPU's tokens, every logits row within 2e-3 of the largest) for
# each of the oracle's prompts, with the split forced; the second device's chain ran
ld2_k3_gate() { # <tag> <n0> <n1> <env...>
local tag=$1 n0=$2 n1=$3 c; shift 3
for c in ${K3C_CASES:-short chunk long}; do
K3C_CASES=$c k3c_gate "$tag" COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=$n0 COLI_VK_CHAIN_LAYERS2=$n1 COLI_VK_TIER_BALANCE=0 "$@"
ld2_ran kimi_k3 "$tag $c" vk.log
done
}
# ld2_k3_lost <tag> <n0> <n1> <back> <env...>: the long prompt, the second device lost
# <back> of its frames before the end of the same run without a fault (back = 0: at its
# first frame, its setup); the CPU's tokens, and LD2_EXPECT=setup (its layers on the CPU
# from the start), rebuild (both devices' KDA state rebuilt) or current (the host's state
# current: nothing to rebuild)
ld2_k3_lost() {
local tag=$1 n0=$2 n1=$3 back=$4 frames k ids; shift 4
local D2=(COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=$n0 COLI_VK_CHAIN_LAYERS2=$n1 COLI_VK_TIER_BALANCE=0 COLI_USAGE=$PWD/k3c.usage USAGE_SAVE=0)
ids=$(k3c_ids long)
rm -f k3c.usage
env "$@" COLI_USAGE=$PWD/k3c.usage USAGE_SAVE=0 ./kimi_k3 kimi_k3_tiny --ids "$ids" --ngen 8 2> cpu.log | sed 's/ *TUNE.*//' > cpu.tok
k=1
if [ "$back" -gt 0 ]; then
env "$@" "${D2[@]}" COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 ./kimi_k3 kimi_k3_tiny --ids "$ids" --ngen 8 2> vk.log > /dev/null
frames=$(sed -n 's/^\[VK\] kimi_k3 dev2 chain: [0-9]* forwards, \([0-9]*\) frames.*/\1/p' vk.log | tail -1)
[ -n "$frames" ] && [ "$frames" -gt "$back" ] || { cat vk.log; fail "$tag: no fault-free run to count frames from"; }
k=$((frames - back + 1))
fi
env "$@" "${D2[@]}" COLI_VK_CHAIN_FAULT2=$k COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 \
./kimi_k3 kimi_k3_tiny --ids "$ids" --ngen 8 2> vk.log | sed 's/ *TUNE.*//' > vk.tok
{ [ -s cpu.tok ] && cmp -s cpu.tok vk.tok; } || { cat cpu.tok vk.tok; tail -20 vk.log; fail "$tag: the tokens differ from the CPU"; }
grep -q "COLI_VK_CHAIN_FAULT2" vk.log || { cat vk.log; fail "$tag: the second device's fault never fired"; }
case ${LD2_EXPECT:-rebuild} in
setup) grep -q "^\[VK\] kimi_k3 dev2 chain: 0 of [0-9]* layers on the device: the chain stays off (layer 0 did not reach the device: the device was lost" vk.log ||
{ cat vk.log; fail "$tag: the second device's layers did not go to the CPU"; } ;;
rebuild) grep -q "kimi_k3 dev2 chain: the device was lost; rebuilding the state of [1-9]" vk.log || { cat vk.log; fail "$tag: no state was rebuilt"; } ;;
current) grep -q "kimi_k3 dev2 chain: the device was lost; the host's state is current" vk.log || { cat vk.log; fail "$tag: the loss was not taken over"; } ;;
esac
echo "OK $tag: tokens = CPU (fault at the second device's frame $k), $(grep -o 'rebuilding the state of [0-9]* positions\|the host.s state is current\|the chain stays off' vk.log | head -1)"
}
ld2_kimi_k3() {
$PY tools/make_kimi_k3_tiny.py --output ./kimi_k3_tiny --force
k3c_serve_fixture
local O="K3_BITS=32 K3_MLA_BITS=32 K3_HEAD_BITS=32 K3_IDOT=0 COLI_TEMP=0" s b
for s in "2 4" "1 2" "3 1" "2 2"; do ld2_k3_gate "ld2 kimi_k3 f32 ${s/ / + }" ${s% *} ${s#* } $O; done
for b in 8 4; do
ld2_k3_gate "ld2 kimi_k3 ${b}-bit trunk" 2 4 K3_BITS=$b K3_MLA_BITS=$b K3_HEAD_BITS=$b K3_IDOT=0 COLI_TEMP=0
done
ld2_k3_gate "ld2 kimi_k3 prefill one token at a time" 1 3 $O K3_CHUNK=1
ld2_k3_gate "ld2 kimi_k3 chain chunks of 3" 2 4 $O COLI_VK_CHAIN_ROWS=3
ld2_k3_gate "ld2 kimi_k3 tier off" 2 4 $O COLI_VK_TIER=0
ld2_k3_gate "ld2 kimi_k3 shared experts by COLI_VK_DENSE=1" 2 4 $O COLI_VK_DENSE=1
ld2_k3_gate "ld2 kimi_k3 K3_TOPP=0.6" 2 2 $O K3_TOPP=0.6
CHAINMODE=2 ld2_k3_gate "ld2 kimi_k3 prompts only" 2 4 $O
K3C_CASES=long ld2_k3_gate "ld2 kimi_k3 KV split on both" 3 3 $O COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4
[ "$(kv_hostparts kimi_k3 vk.log)" -gt 0 ] && [ "$(kv_hostparts "kimi_k3 dev2" vk.log)" -gt 0 ] ||
{ cat vk.log; fail "ld2 kimi_k3 KV split on both: a device's split ran no host part"; }
# the second device lost (a setup frame, then 9 frames a forward; the long prompt in
# forwards of 32, 32 and 8 rows, then 7 decode steps): at its setup, in the last decode
# step, in the second prompt forward, in the third chain chunk of the second prompt
# forward in chunks of 5 (the state its first chunks advanced not used); both devices'
# KDA state rebuilt from the token ids
LD2_EXPECT=setup ld2_k3_lost "ld2 kimi_k3 second device lost at its setup" 2 4 0 $O K3_CHUNK=32
ld2_k3_lost "ld2 kimi_k3 second device lost mid-decode" 2 4 3 $O K3_CHUNK=32
ld2_k3_lost "ld2 kimi_k3 second device lost in a later prompt chunk" 2 4 77 $O K3_CHUNK=32
ld2_k3_lost "ld2 kimi_k3 second device lost inside a chunked forward" 2 4 122 $O K3_CHUNK=32 COLI_VK_CHAIN_ROWS=5
# serve sessions frame for frame, the KDA state across turns on both devices: photos
# taken from both and restored to both
local SV="K3_BITS=32 K3_MLA_BITS=32 K3_HEAD_BITS=32 K3_IDOT=0 K3_PREFIX_LOG=1 USAGE_SAVE=0 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=2 COLI_VK_CHAIN_LAYERS2=4"
export CHAIN_SERVE_TOL=2e-2 CHAIN_SERVE_EXPECT='kimi_k3 dev2 chain: [1-9][0-9]* forwards'
# shellcheck disable=SC2086
{
$PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve $SV
$PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve $SV COLI_K3_CKPT=4
$PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve $SV COLI_K3_CKPT=2 COLI_VK_CHAIN_ROWS=3
}
unset CHAIN_SERVE_TOL CHAIN_SERVE_EXPECT
}
# deepseek_v41: v41_gate (the CPU's tokens and exit, every forward's logits within 1e-4 of
# the largest) with the split forced at a layer the second device may start at (the
# fixture's 1, 3 and 5: its first compressed layer owns its caches and runs its indexer);
# the second device's chain ran
ld2_v41_gate() { # <tag> <n0> <n1> <env...> -- <argv...>
local tag=$1 n0=$2 n1=$3; shift 3
v41_gate "$tag" COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=$n0 COLI_VK_CHAIN_LAYERS2=$n1 COLI_VK_TIER_BALANCE=0 "$@"
ld2_ran deepseek_v41 "$tag" vk.log
}
# ld2_v41_lost <tag> <n0> <n1> <back> <env...> -- <argv...>: the second device lost <back>
# of its frames before the end of the same run without a fault (0: at its first frame,
# its setup); the CPU's tokens, and its layers on the CPU from the start (back 0) or from
# the forward it was lost in
ld2_v41_lost() {
local tag=$1 n0=$2 n1=$3 back=$4 frames k=1; shift 4
local envs=(); while [ "$1" != "--" ]; do envs+=("$1"); shift; done; shift
local D2=(COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=$n0 COLI_VK_CHAIN_LAYERS2=$n1 COLI_VK_TIER_BALANCE=0 COLI_USAGE=chain.usage COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1)
rm -f chain.usage
env "${envs[@]}" ./deepseek_v41 "$@" > cpu.txt 2> cpu.log || true
if [ "$back" -gt 0 ]; then
env "${envs[@]}" "${D2[@]}" ./deepseek_v41 "$@" > /dev/null 2> vk.log || true
frames=$(sed -n 's/^\[VK\] deepseek_v41 dev2 chain: [0-9]* forwards, \([0-9]*\) frames.*/\1/p' vk.log | tail -1)
[ -n "$frames" ] && [ "$frames" -gt "$back" ] || { cat vk.log; fail "$tag: no fault-free run to count frames from"; }
k=$((frames - back + 1))
fi
rm -f chain.usage
env "${envs[@]}" "${D2[@]}" COLI_VK_CHAIN_FAULT2=$k ./deepseek_v41 "$@" > vk.txt 2> vk.log || true
{ [ -s cpu.txt ] && cmp -s cpu.txt vk.txt; } || { cat cpu.txt vk.txt; tail -20 vk.log; fail "$tag: the tokens differ from the CPU"; }
grep -q "COLI_VK_CHAIN_FAULT2" vk.log || { cat vk.log; fail "$tag: the second device's fault never fired"; }
if [ "$back" = 0 ]; then
grep -q "^\[VK\] deepseek_v41 dev2 chain: 0 of [0-9]* layers on the device: the chain stays off (layer 0 did not reach the device: the device was lost" vk.log ||
{ cat vk.log; fail "$tag: the second device's layers did not go to the CPU"; }
else
grep -q "deepseek_v41 dev2 chain: the device was lost; the CPU runs its layers" vk.log || { cat vk.log; fail "$tag: the loss was not taken over"; }
fi
rm -f chain.usage
echo "OK $tag: tokens = CPU (fault at the second device's frame $k)"
}
ld2_deepseek_v41() {
v41_chain_fixtures
local s f cap T=(SNAP=dsv41_long) A=(-- 8 dsv41_long/ref.json)
for s in "1 2" "1 5" "3 3" "5 1"; do ld2_v41_gate "ld2 deepseek_v41 ${s/ / + }" ${s% *} ${s#* } "${T[@]}" "${A[@]}"; done
for cap in 1 2; do ld2_v41_gate "ld2 deepseek_v41 cap=$cap" 1 4 SNAP=dsv41_tiny -- $cap dsv41_tiny/ref.json; done
# a layer the second device may not start at: forced there, it stays off; the fit's
# choice comes down to the last layer it may start at (a device for 2 layers: 1)
v41_gate "ld2 deepseek_v41 forced at layer 2" COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=2 COLI_VK_CHAIN_LAYERS2=2 "${T[@]}" "${A[@]}"
grep -q "layer 2 reads what the layers before it make in a forward" vk.log && [ "$(ld2_forwards deepseek_v41 vk.log)" = 0 ] ||
{ cat vk.log; fail "ld2 deepseek_v41 forced at layer 2: the second device did not stay off"; }
ptl_v41_probe ptl-probe.log COLI_VK_DEVICE_CAP_MB=256 COLI_VK_TIER_RESERVE_GB=0.04 "${T[@]}" "${A[@]}"
cap=$(PTL_PROBE_CAP_MB=256 ptl_calc cap deepseek_v41 ptl-probe.log 2) || fail "ld2 deepseek_v41 the fit: no cap from the probe"
v41_gate "ld2 deepseek_v41 the fit at layer 2 (cap $cap MiB)" COLI_VK_DEV2=0 COLI_VK_DEVICE_CAP_MB=$cap COLI_VK_TIER_RESERVE_GB=0.04 \
COLI_VK_TIER_BALANCE=0 "${T[@]}" "${A[@]}"
grep -q "deepseek_v41 chain: 1 of 6 layers on the device: the second device's layers start at layer 1" vk.log ||
{ grep -a '^\[VK\]' vk.log; fail "ld2 deepseek_v41 the fit at layer 2: it did not come down to layer 1"; }
ld2_ran deepseek_v41 "ld2 deepseek_v41 the fit at layer 2" vk.log
# DSpark drafts accepted (1, 3) and rejected (2, 4, 5), the targets on the second device
for f in 1 2 3 4 5; do ld2_v41_gate "ld2 deepseek_v41 DSpark spec=$f" 1 5 SNAP=dsv41_tiny V41_DSPARK=1 V41_SPEC_FORCE=$f -- 8 dsv41_tiny/ref.json; done
ld2_v41_gate "ld2 deepseek_v41 DSpark spec=5, one cache slot" 3 3 SNAP=dsv41_tiny V41_DSPARK=1 V41_SPEC_FORCE=5 -- 1 dsv41_tiny/ref.json
ld2_v41_gate "ld2 deepseek_v41 prompt in chunks of 3" 1 5 "${T[@]}" COLI_VK_CHAIN_ROWS=3 "${A[@]}"
ld2_v41_gate "ld2 deepseek_v41 prompt in chunks of 7" 3 3 "${T[@]}" COLI_VK_CHAIN_ROWS=7 -- 2 dsv41_long/ref.json
CHAINMODE=2 ld2_v41_gate "ld2 deepseek_v41 prompts only" 1 5 "${T[@]}" "${A[@]}"
ld2_v41_gate "ld2 deepseek_v41 tier off" 3 3 "${T[@]}" COLI_VK_TIER=0 "${A[@]}"
ld2_v41_gate "ld2 deepseek_v41 beside the per-matrix trunk" 3 3 "${T[@]}" COLI_VK_DENSE=1 "${A[@]}"
ld2_v41_gate "ld2 deepseek_v41 V41_INDEX_OWNER=1" 1 5 "${T[@]}" V41_INDEX_OWNER=1 "${A[@]}"
ld2_v41_gate "ld2 deepseek_v41 experts on both devices too" 3 3 "${T[@]}" COLI_VK_TIER_GB=0.00002 "${A[@]}"
# the compressed rows split on both devices' kv_source layers (1 on the first, 3 on the second)
ld2_v41_gate "ld2 deepseek_v41 KV split on both" 3 3 COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=2 "${T[@]}" "${A[@]}"
[ "$(kv_hostparts deepseek_v41 vk.log)" -gt 0 ] && [ "$(kv_hostparts "deepseek_v41 dev2" vk.log)" -gt 0 ] ||
{ cat vk.log; fail "ld2 deepseek_v41 KV split on both: a device's split ran no host part"; }
# the second device lost: at its setup, mid-decode, in the prompt, between drafts
ld2_v41_lost "ld2 deepseek_v41 second device lost at its setup" 1 5 0 "${T[@]}" "${A[@]}"
ld2_v41_lost "ld2 deepseek_v41 second device lost mid-decode" 1 5 5 "${T[@]}" "${A[@]}"
ld2_v41_lost "ld2 deepseek_v41 second device lost in the prompt" 3 3 60 "${T[@]}" COLI_VK_CHAIN_ROWS=7 "${A[@]}"
ld2_v41_lost "ld2 deepseek_v41 second device lost between drafts" 1 5 8 SNAP=dsv41_tiny V41_DSPARK=1 V41_SPEC_FORCE=5 -- 8 dsv41_tiny/ref.json
# serve sessions frame for frame, DSpark, prompts only; images on the wire
export CHAIN_SERVE_EXPECT='deepseek_v41 dev2 chain: [1-9][0-9]* forwards'
$PY tests/vulkan_chain_serve.py ./deepseek_v41 dsv41_tiny V41_DSPARK=0 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=1 COLI_VK_CHAIN_LAYERS2=5
$PY tests/vulkan_chain_serve.py ./deepseek_v41 dsv41_tiny V41_DSPARK=1 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=3 COLI_VK_CHAIN_LAYERS2=3
COLI_VK_CHAIN=2 $PY tests/vulkan_chain_serve.py ./deepseek_v41 dsv41_tiny V41_DSPARK=1 COLI_VK_CHAIN_ROWS=3 COLI_VK_DEV2=0 \
COLI_VK_CHAIN_LAYERS=1 COLI_VK_CHAIN_LAYERS2=5
unset CHAIN_SERVE_EXPECT
$PY tests/vulkan_chain_v41_image.py ./deepseek_v41 dsv41_tiny V41_DSPARK=0 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=1 COLI_VK_CHAIN_LAYERS2=5
rm -f chain.usage chain-serve.usage chain-image.usage ptl-probe.log
}
# deepseek_v4: v4_chain_gate (the CPU's and the reference's ids, the teacher-forced rows,
# the drafts, every logits row within V4_CHAIN_TOL) with the split forced; the second
# device's chain ran (FAULT2_BACK=k: lost k of its frames before the end of a fault-free
# run, 0 at its setup: the CPU runs its layers)
ld2_v4_gate() { # <tag> <n0> <n1> <fixture> <case> <env...>
local tag=$1 n0=$2 n1=$3 fx=$4 c=$5; shift 5
v4_chain_gate "$tag" "$fx" "$c" COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=$n0 COLI_VK_CHAIN_LAYERS2=$n1 COLI_VK_TIER_BALANCE=0 "$@"
[ -n "${FAULT2_BACK:-}" ] || ld2_ran deepseek_v4 "$tag" v4-vk.err
}
ld2_deepseek_v4() {
v4_chain_fixtures
local c fx A=ids:20,21,22,23,24,25,26,27,28,29,20,21,22:12 R=ids:20,21,22,23,50,51,52,20,21,22:12
local M=ids:30,31,32,33,34,35,60,61,30,31,32,33,34,35,36,37,38,39,40,41,30,31,32:12
for c in short compressed long; do
ld2_v4_gate "ld2 deepseek_v4 1 + 2" 1 2 deepseek_v4_tiny_t $c
ld2_v4_gate "ld2 deepseek_v4 2 + 1" 2 1 deepseek_v4_tiny_t $c
done
ld2_v4_gate "ld2 deepseek_v4 8 experts" 1 2 deepseek_v4_tiny_e8 long
ld2_v4_gate "ld2 deepseek_v4 2 output groups" 1 2 deepseek_v4_tiny_g2 long
ld2_v4_gate "ld2 deepseek_v4 a window of 4, ratios 4 and 2" 1 2 deepseek_v4_tiny_w4 long
ld2_v4_gate "ld2 deepseek_v4 an indexer of 64 heads of 128" 1 2 deepseek_v4_tiny_ix long
ld2_v4_gate "ld2 deepseek_v4 experts on both devices too" 1 2 deepseek_v4_tiny_e8 long COLI_VK_TIER_GB=0.00009
ld2_v4_gate "ld2 deepseek_v4 tier off" 1 2 deepseek_v4_tiny_t long COLI_VK_TIER=0 COLI_VK_DENSE=0
ld2_v4_gate "ld2 deepseek_v4 beside the per-matrix trunk" 1 2 deepseek_v4_tiny_t long COLI_VK_DENSE=1
ld2_v4_gate "ld2 deepseek_v4 prefill chunks of 7" 2 1 deepseek_v4_tiny_t long V4_PREFILL_CHUNK=7
ld2_v4_gate "ld2 deepseek_v4 chain chunks of 3" 1 2 deepseek_v4_tiny_e8 long COLI_VK_CHAIN_ROWS=3
ld2_v4_gate "ld2 deepseek_v4 V4_IDX_IDENTITY=1" 1 2 deepseek_v4_tiny_t long V4_IDX_IDENTITY=1
CHAINMODE=2 ld2_v4_gate "ld2 deepseek_v4 prompts only" 1 2 deepseek_v4_tiny_t long
ld2_v4_gate "ld2 deepseek_v4 n-gram drafts accepted and rejected" 1 2 deepseek_v4_tiny_t $A V4_DRAFT=4 V4_NGRAM_PARTIAL_KEEP=1
ld2_v4_gate "ld2 deepseek_v4 an n-gram draft rejected" 2 1 deepseek_v4_tiny_t $R V4_DRAFT=4 V4_NGRAM_PARTIAL_KEEP=1
ld2_v4_gate "ld2 deepseek_v4 n-gram drafts, a window of 4" 1 2 deepseek_v4_tiny_w4 $M V4_DRAFT=4 V4_NGRAM_PARTIAL_KEEP=1
ld2_v4_gate "ld2 deepseek_v4 V4_MTP=1" 1 2 deepseek_v4_tiny_t long V4_MTP=1 V4_DRAFT=3
# the compressed rows split on both devices (layer 1 ratio 4 on the first, layer 2 ratio 8 on the second)
ld2_v4_gate "ld2 deepseek_v4 KV split on both" 2 1 deepseek_v4_tiny_t long COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=2
[ "$(kv_hostparts deepseek_v4 v4-vk.err)" -gt 0 ] && [ "$(kv_hostparts "deepseek_v4 dev2" v4-vk.err)" -gt 0 ] ||
{ cat v4-vk.err; fail "ld2 deepseek_v4 KV split on both: a device's split ran no host part"; }
# the second device lost: at its setup, mid-decode, in the prompt, between drafts
FAULT2_BACK=0 ld2_v4_gate "ld2 deepseek_v4 second device lost at its setup" 1 2 deepseek_v4_tiny_t long
FAULT2_BACK=3 ld2_v4_gate "ld2 deepseek_v4 second device lost mid-decode" 1 2 deepseek_v4_tiny_t long
FAULT2_BACK=30 ld2_v4_gate "ld2 deepseek_v4 second device lost in the prompt" 1 2 deepseek_v4_tiny_t long COLI_VK_CHAIN_ROWS=7
FAULT2_BACK=10 ld2_v4_gate "ld2 deepseek_v4 second device lost between drafts" 1 2 deepseek_v4_tiny_w4 $M V4_DRAFT=4 V4_NGRAM_PARTIAL_KEEP=1
# served: the pin, its extension, a read-only prompt and a shorter one
COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=1 COLI_VK_CHAIN_LAYERS2=2 V4_SERVE_EXPECT='deepseek_v4 dev2 chain: [1-9][0-9]* forwards' \
v4_chain_serve deepseek_v4_tiny_t
rm -rf deepseek_v4_tiny_t deepseek_v4_tiny_e8 deepseek_v4_tiny_g2 deepseek_v4_tiny_w4 deepseek_v4_tiny_ix chain.usage
rm -f cpu.f32 vk.f32 v4-*.f32 v4-*.json v4-*.err v4-tf.txt
}
family_layers_dev2() {
export OMP_NUM_THREADS=2
make qwen36 qwen38 olmoe mimo inkling tests/test_vk_chain VK=1
COLI_VK_DEV2=0 ./tests/test_vk_chain shaders/qmatmul.spv | tee vk_chain.log
tail -1 vk_chain.log | grep -qx PASS || fail "the chain's ops on two devices"
ld2_qwen36
ld2_qwen38
ld2_olmoe
ld2_mimo
ld2_inkling
unset OMP_NUM_THREADS
}
family_layers_dev2_mla() {
export OMP_NUM_THREADS=2
make colibri glm53 kimi_k3 VK=1
ld2_colibri
ld2_glm53
ld2_kimi_k3
unset OMP_NUM_THREADS CAP_RAISE
}
family_layers_dev2_deepseek() {
export OMP_NUM_THREADS=2
make deepseek_v41 VK=1
make deepseek-v4 VK=1
ld2_deepseek_v41
ld2_deepseek_v4
unset OMP_NUM_THREADS
}
family_layers_dev2_sanitize() {
make clean >/dev/null 2>&1 || true
make qwen36 qwen38 olmoe mimo inkling VK=1 EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1
[ -d qwen36_kv_c ] || kv_qwen36_fixtures
kv_san qwen36 "asan ld2 qwen36 KV split on both" COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=4 COLI_VK_CHAIN_LAYERS2=4 \
COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 COLI_DENSE_I8=0 SNAP=qwen36_kv_c ./qwen36 8 8 qwen36_kv/ref_full.json
[ "$(ld2_forwards qwen36 san.log)" -gt 0 ] || { cat san.log; fail "asan ld2 qwen36: the second device's chain never ran"; }
[ -d qwen38_kv_mtp ] || kv_qwen38_fixtures
kv_san qwen38 "asan ld2 qwen38 KV split on both, MTP mixed" COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=2 COLI_VK_CHAIN_LAYERS2=2 \
COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 Q38_MTP=1 Q38_MTP_FORCE=mixed SNAP=qwen38_kv_mtp ./qwen38 4 8 qwen38_kv_mtp/ref.json
[ "$(ld2_forwards qwen38 san.log)" -gt 0 ] || { cat san.log; fail "asan ld2 qwen38: the second device's chain never ran"; }
[ -f olmoe_tiny/ref_olmoe_long.json ] || kv_olmoe_fixtures
kv_san olmoe "asan ld2 olmoe KV split on both, PILOT" COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=2 COLI_VK_CHAIN_LAYERS2=2 \
COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 PILOT=1 WIDE=2 SNAP=olmoe_tiny_c ./olmoe 2 8 olmoe_tiny/ref_olmoe_long.json
[ "$(ld2_forwards olmoe san.log)" -gt 0 ] || { cat san.log; fail "asan ld2 olmoe: the second device's chain never ran"; }
[ -f tiny_inkling/ref_long.json ] || kv_inkling_fixtures
# the fixture's one global layer (5) on the first device, where kv_san counts the host parts
kv_san inkling "asan ld2 inkling KV split on the first device, layers 6 and 7 on the second" COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=6 COLI_VK_CHAIN_LAYERS2=2 \
COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 SNAP=tiny_inkling ./inkling 8 0 tiny_inkling/ref_long.json
[ "$(ld2_forwards inkling san.log)" -gt 0 ] || { cat san.log; fail "asan ld2 inkling: the second device's chain never ran"; }
make clean >/dev/null 2>&1 || true
}
# ld2_san <engine> <tag> <env and argv...>: a sanitized build's run with the split: no
# sanitizer diagnostic, and the second device's chain ran (LD2_SAN_LOST=1: its loss was
# taken over)
ld2_san() {
local eng=$1 tag=$2; shift 2
rm -f chain.usage
env OMP_NUM_THREADS=2 COLI_USAGE=chain.usage COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_DEV2=0 COLI_VK_TIER_BALANCE=0 \
"$@" > san.log 2>&1 || true
rm -f chain.usage
if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "$tag: sanitizer diagnostic"; fi
if [ "${LD2_SAN_LOST:-0}" = 1 ]; then
grep -q "COLI_VK_CHAIN_FAULT2" san.log && grep -q "the device was lost" san.log || { cat san.log; fail "$tag: the loss was not taken over"; }
echo "OK $tag: sanitizers clean, the second device's loss taken over"
else
[ "$(ld2_forwards "$eng" san.log)" -gt 0 ] || { cat san.log; fail "$tag: the second device's chain never ran"; }
echo "OK $tag: sanitizers clean, the second device's chain ran $(ld2_forwards "$eng" san.log) forwards"
fi
}
family_layers_dev2_mla_sanitize() {
local SAN="-fsanitize=address,undefined -fno-omit-frame-pointer -g" ids
make clean >/dev/null 2>&1 || true
make colibri glm53 kimi_k3 VK=1 EXTRA_CFLAGS="$SAN"
export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1 CAP_RAISE=0
[ -f glm_tiny_serve/tokenizer.json ] || glm_chain_fixtures
[ -d glm_tiny_shx ] || ptl_glm_shx glm_tiny glm_tiny_shx
[ -d glm53_l6s-i4 ] || { ptl_g53_l6 glm53_l6; $PY tools/make_glm53_streaming_pair.py --fixture glm53_l6 --output glm53_l6s > /dev/null; }
$PY tools/make_kimi_k3_tiny.py --output ./kimi_k3_tiny --force
ids=$($PY -c "print(','.join(str((i*37+5)%120+2) for i in range(100)))")
local G=(SNAP=glm_tiny REF=ref_glm.json USAGE_SAVE=0) K=(COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4)
ld2_san colibri "asan ld2 colibri KV split on both, DSA top-4" COLI_VK_CHAIN_LAYERS=2 COLI_VK_CHAIN_LAYERS2=3 "${K[@]}" "${G[@]}" DSA_TOPK=4 \
./colibri 64 16 16
ld2_san colibri "asan ld2 colibri a shared indexer first on the second device, chunks of 5" COLI_VK_CHAIN_LAYERS=1 COLI_VK_CHAIN_LAYERS2=2 \
SNAP=glm_tiny_shx REF=ref_glm.json USAGE_SAVE=0 DSA_TOPK=4 TF=1 COLI_VK_CHAIN_ROWS=5 ./colibri 64 16 16
LD2_SAN_LOST=1 ld2_san colibri "asan ld2 colibri second device lost mid-decode" COLI_VK_CHAIN_LAYERS=2 COLI_VK_CHAIN_LAYERS2=2 \
COLI_VK_CHAIN_FAULT2=10 "${G[@]}" ./colibri 64 16 16
ptl_glm_shx glm_tiny_serve glm_tiny_shx_serve # several conversations at once, the DSA lists crossing
$PY tests/vulkan_chain_mux.py ./colibri glm_tiny_shx_serve 3 SERVE_BATCH=1 IDOT=0 DSA_TOPK=4 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=1 \
COLI_VK_CHAIN_LAYERS2=2 > san.log 2>&1 || { cat san.log; fail "asan ld2 colibri several conversations at once"; }
echo "OK asan ld2 colibri several conversations at once: $(tail -1 san.log)"
rm -rf glm_tiny_shx_serve chain-mux.usage
ld2_san glm53 "asan ld2 glm53 KV split on both" COLI_VK_CHAIN_LAYERS=2 COLI_VK_CHAIN_LAYERS2=2 COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 \
GLM53_BITS=32 USAGE_SAVE=0 ./glm53 --model glm53_l6s-i4 --ids $ids --greedy 8
LD2_SAN_LOST=1 ld2_san glm53 "asan ld2 glm53 second device lost, the KDA state rebuilt" COLI_VK_CHAIN_LAYERS=3 COLI_VK_CHAIN_LAYERS2=3 \
COLI_VK_CHAIN_FAULT2=40 GLM53_BITS=32 USAGE_SAVE=0 ./glm53 --model glm53_l6s-i4 --ids $ids --greedy 6
local O=(K3_BITS=32 K3_MLA_BITS=32 K3_HEAD_BITS=32 K3_IDOT=0 COLI_TEMP=0 USAGE_SAVE=0)
ld2_san kimi_k3 "asan ld2 kimi_k3 KV split on both" COLI_VK_CHAIN_LAYERS=3 COLI_VK_CHAIN_LAYERS2=3 COLI_VK_KV_DEVICE_ROWS=16 \
COLI_VK_KV_BLOCK=4 "${O[@]}" ./kimi_k3 kimi_k3_tiny --ids "$(k3c_ids long)" --ngen 8
LD2_SAN_LOST=1 ld2_san kimi_k3 "asan ld2 kimi_k3 second device lost in a prompt forward" COLI_VK_CHAIN_LAYERS=2 COLI_VK_CHAIN_LAYERS2=4 \
COLI_VK_CHAIN_FAULT2=15 K3_CHUNK=32 "${O[@]}" ./kimi_k3 kimi_k3_tiny --ids "$(k3c_ids long)" --ngen 8
unset CAP_RAISE
make clean >/dev/null 2>&1 || true
}
family_layers_dev2_deepseek_sanitize() {
local SAN="-fsanitize=address,undefined -fno-omit-frame-pointer -g" p
make clean >/dev/null 2>&1 || true
make deepseek_v41 VK=1 EXTRA_CFLAGS="$SAN"
export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1
v41_chain_fixtures
ld2_san deepseek_v41 "asan ld2 deepseek_v41 DSpark spec=5" COLI_VK_CHAIN_LAYERS=1 COLI_VK_CHAIN_LAYERS2=5 SNAP=dsv41_tiny V41_DSPARK=1 \
V41_SPEC_FORCE=5 ./deepseek_v41 8 dsv41_tiny/ref.json
ld2_san deepseek_v41 "asan ld2 deepseek_v41 KV split on both, chunks of 3" COLI_VK_CHAIN_LAYERS=3 COLI_VK_CHAIN_LAYERS2=3 \
COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=2 COLI_VK_CHAIN_ROWS=3 SNAP=dsv41_long ./deepseek_v41 8 dsv41_long/ref.json
LD2_SAN_LOST=1 ld2_san deepseek_v41 "asan ld2 deepseek_v41 second device lost" COLI_VK_CHAIN_LAYERS=1 COLI_VK_CHAIN_LAYERS2=5 \
COLI_VK_CHAIN_FAULT2=40 SNAP=dsv41_long ./deepseek_v41 8 dsv41_long/ref.json
make clean >/dev/null 2>&1 || true
# the V4 sanitized build as its chain family makes it (no LTO, the sanitizers at the link)
make deepseek-v4-clean >/dev/null 2>&1 || true
make deepseek-v4 VK=1 LTO=0 EXTRA_CFLAGS="$SAN" EXTRA_LDFLAGS="$SAN"
v4_chain_fixtures
p=$(v4_chain_prompt deepseek_v4_tiny_t long)
ld2_san deepseek_v4 "asan ld2 deepseek_v4 KV split on both" COLI_VK_CHAIN_LAYERS=2 COLI_VK_CHAIN_LAYERS2=1 COLI_VK_KV_DEVICE_ROWS=8 \
COLI_VK_KV_BLOCK=2 ./deepseek_v4 ./deepseek_v4_tiny_t "$p" --raw-prompt --max-tokens 4
ld2_san deepseek_v4 "asan ld2 deepseek_v4 drafts, a window of 4" COLI_VK_CHAIN_LAYERS=1 COLI_VK_CHAIN_LAYERS2=2 V4_DRAFT=4 \
V4_NGRAM_PARTIAL_KEEP=1 ./deepseek_v4 ./deepseek_v4_tiny_w4 \
"$(v4_chain_prompt x ids:30,31,32,33,34,35,60,61,30,31,32,33,34,35,36,37,38,39,40,41,30,31,32)" --raw-prompt --max-tokens 12
LD2_SAN_LOST=1 ld2_san deepseek_v4 "asan ld2 deepseek_v4 second device lost" COLI_VK_CHAIN_LAYERS=1 COLI_VK_CHAIN_LAYERS2=2 \
COLI_VK_CHAIN_FAULT2=8 ./deepseek_v4 ./deepseek_v4_tiny_t "$p" --raw-prompt --max-tokens 4
rm -rf deepseek_v4_tiny_t deepseek_v4_tiny_e8 deepseek_v4_tiny_g2 deepseek_v4_tiny_w4 deepseek_v4_tiny_ix san.log
make deepseek-v4-clean >/dev/null 2>&1 || true
}