Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
40.8 KB
Raw Blame
# Sourced by tests/vulkan_engines.sh: the partial chain (docs/vulkan.md, "A partial chain")
# on DeepSeek V4.1 Flash (deepseek_v41) and Kimi K3 (kimi_k3): the first N layers on the
# device, the CPU the rest and the head (partial-dsk, partial-dsk-sanitize; the template
# is tests/vulkan_partial_deepseek.sh).
#
# Every configuration runs the engine's chain gate (v41_gate: the CPU's tokens and exit,
# every logits row within 1e-4 of the largest; k3c_gate: the CPU's tokens, every logits
# row within 2e-3) with the partial chain's settings on top, then checks the lines (ptl_*
# helpers in tests/vulkan_engines.sh):
#   - COLI_VK_CHAIN_LAYERS=k for every k of the six-layer fixtures: N = k, the matrices on
#     the device after setup are those k layers' (the placed line), and with k < L the
#     per-matrix path put nothing more there (the resident matrices at exit = at setup).
#     V4.1's cuts at 2 and 4 hand the CPU's layers the index list and the candidate mask a
#     chain layer made; Kimi K3's at 1 and 3 cut inside an AttnRes block;
#   - COLI_VK_DEVICE_CAP_MB aimed at k layers from a probe's fit line (and at every layer
#     but the head): the line's N, the rule's from the run's own numbers, and coli plan's
#     (resource_plan.py: free, per-layer and fixed bytes too);
#   - a staged upload failing inside layer k's setup: N = k, nothing of layer k on the
#     device; with the dense weights on the device only, nothing read back from disk;
#   - the forward paths with N < L: prompt chunks, expert streaming, DSpark drafts (V4.1),
#     prompts only, the tier off, the per-matrix path beside, the KV split, a lost device
#     (V4.1: the forward again on the CPU; Kimi K3: the chain layers' KDA state rebuilt),
#     serve sessions with prefix reuse (and Kimi K3's recurrent-state photos), images;
#   - the dense weights on the device only with N < L: only the N layers' matrices give back
#     their host copies, none is read back on a healthy run (after a loss, only theirs);
#   - everything fitting: N = L, the same bits with and without COLI_VK_CHAIN_LAYERS=L.

# ---- deepseek_v41 ----------------------------------------------------------------------
# ptl_v41_resident <log>: the matrices the device holds at setup and at exit, "S X"
ptl_v41_resident() {
  local s x
  s=$(sed -n 's/^\[VK\] deepseek_v41 chain: [0-9]* layers on the device (.*, \([0-9]*\) matrices (.* on the device$/\1/p' "$1" | tail -1)
  x=$(sed -n 's/^\[VK\] deepseek_v41: \([0-9]*\) resident matrices on the device.*/\1/p' "$1" | tail -1)
  echo "${s:-0} ${x:-0}"
}
# ptl_v41_off <tag> <env...> -- <argv...>: N = 0, the chain off: the CPU's tokens and exit,
# the logits within 1e-4, no chain forward, no dense matrix on the device
ptl_v41_off() {
  local tag=$1; shift
  local envs=(); while [ "$1" != "--" ]; do envs+=("$1"); shift; done; shift
  local rc_cpu=0 rc_vk=0
  rm -f chain.usage cpu.f32 vk.f32
  env "${envs[@]}" DUMP=cpu.f32 ./deepseek_v41 "$@" > cpu.txt 2> cpu.log || rc_cpu=$?
  env "${envs[@]}" DUMP=vk.f32 COLI_USAGE=chain.usage COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=${CHAINMODE:-1} \
    ./deepseek_v41 "$@" > vk.txt 2> vk.log || rc_vk=$?
  [ "$rc_cpu" = "$rc_vk" ] && [ -s cpu.txt ] && cmp -s cpu.txt vk.txt || { cat cpu.txt vk.txt; tail -20 vk.log; fail "$tag: tokens"; }
  ptl_check_n deepseek_v41 vk.log 0 "$tag"
  [ "$(chain_count deepseek_v41 vk.log)" = 0 ] || { grep '^\[VK\]' vk.log; fail "$tag: the chain ran"; }
  if grep -q '^\[VK\] deepseek_v41 chain: [0-9]* of [0-9]* layers placed' vk.log; then ptl_check_placed deepseek_v41 vk.log "$tag"; fi
  set -- $(ptl_v41_resident vk.log)
  [ "$2" = 0 ] || { grep '^\[VK\]' vk.log; fail "$tag: $2 dense matrices on the device with the chain off"; }
  local lg; lg=$(logits_close cpu.f32 vk.f32) || { echo "$lg"; fail "$tag: logits"; }
  echo "OK $tag: N = 0, the chain off, tokens = CPU (exit $rc_cpu), $lg, no dense matrix on the device"
}
# ptl_v41 <tag> <k|-> <env...> -- <argv...>: v41_gate with COLI_VK_CHAIN_LAYERS=k (-: the
# fit decides), then N = PTL_WANT (default k), the placed check, and with N < L nothing
# more on the device at exit than at setup
ptl_v41() {
  local tag=$1 k=$2; shift 2
  local want=${PTL_WANT:-$k} e=()
  [ "$k" != - ] && e=(COLI_VK_CHAIN_LAYERS=$k)
  if [ "$want" = 0 ]; then ptl_v41_off "$tag" "${e[@]}" "$@"; return; fi
  v41_gate "$tag" "${e[@]}" "$@"
  ptl_check_n deepseek_v41 vk.log "$want" "$tag"
  ptl_check_placed deepseek_v41 vk.log "$tag"
  local L; L=$(ptl_L deepseek_v41 vk.log)
  if [ "$want" -lt "$L" ] && [ -z "${FAULT_BACK:-}" ]; then
    set -- $(ptl_v41_resident vk.log)
    [ "$1" = "$2" ] || { grep '^\[VK\]' vk.log; fail "$tag: $1 matrices on the device at setup, $2 at exit: the CPU's layers went up"; }
  fi
  echo "   N = $want of $L, $(ptl_calc matrices deepseek_v41 vk.log | cut -d' ' -f1) B of matrices on the device after setup"
}
# ptl_v41_probe <log> <env...> -- <argv...>: one chained run, its stderr kept
ptl_v41_probe() {
  local log=$1; shift
  local envs=(); while [ "$1" != "--" ]; do envs+=("$1"); shift; done; shift
  rm -f chain.usage
  env "${envs[@]}" COLI_USAGE=chain.usage COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=${CHAINMODE:-1} \
    ./deepseek_v41 "$@" > /dev/null 2> "$log" || true
  rm -f chain.usage
  grep -q '^\[VK\] deepseek_v41 chain fit: ' "$log" || { tail -20 "$log"; fail "probe run: no fit line"; }
}
# ptl_plan <engine> <family> <tag> <snapshot> <log> <env...>: coli plan's prediction
# (resource_plan.py, vk_chain_fit) for the same device and settings: the engine's free
# bytes, per-layer bytes, fixed bytes and N, from the checkpoint's header and config alone
ptl_plan() {
  local eng=$1 fam=$2 tag=$3 snap=$4 log=$5; shift 5
  $PY - "$eng" "$fam" "$snap" "$log" "$@" <<'PY' || fail "$tag: coli plan predicts otherwise"
import re, sys
sys.path.insert(0, ".")
import resource_plan as rp
eng, fam, snap, log = sys.argv[1:5]
env = dict(a.split("=", 1) for a in sys.argv[5:] if "=" in a)
env.update(COLI_VULKAN="1", COLI_VK_CHAIN=env.get("COLI_VK_CHAIN", "1"))
lines = open(log, errors="replace").read().splitlines()
f = [l for l in lines if l.startswith(f"[VK] {eng} chain fit: ")][-1]
num = lambda key: int(re.search(key + r" (\d+) B", f)[1])
layers = [int(x) for x in re.search(r"layers((?: \d+)*) B", f)[1].split()]
n = int(re.findall(rf"\[VK\] {eng} chain: (\d+) of \d+ layers on the device", "\n".join(lines))[0])
fit = rp.vk_chain_fit(rp.analyze_model(snap), fam, env, {"type": "cpu"})
ok = (fit["layers"], fit["fixed"], fit["free"], fit["n"]) == (layers, num("fixed"), num("free"), n)
print(f"   coli plan: N = {fit['n']} of {fit['L']} ({'as the engine' if ok else 'the engine: %d' % n}), "
      f"layers {'= the engine' if fit['layers'] == layers else fit['layers']}, fixed {fit['fixed']} B, free {fit['free']} B")
sys.exit(0 if ok else 1)
PY
}
# ptl_notail <engine> <probe log>: the COLI_VK_DEVICE_CAP_MB under which every layer fits
# but not the tail (ptl_calc cap's arithmetic, half of the tail instead of all of it)
ptl_notail() {
  $PY - "$1" "$2" <<'PY'
import os, re, sys
eng, log = sys.argv[1], sys.argv[2]
f = [l for l in open(log, errors="replace").read().splitlines() if l.startswith(f"[VK] {eng} chain fit: ")][-1]
num = lambda key: int(re.search(key + r" (\d+) B", f)[1])
free, res, tail, engine = num("free"), num("reserve"), num("tail"), int(re.search(r"the engine's (\d+) B", f)[1])
layers = [int(x) for x in re.search(r"layers((?: \d+)*) B", f)[1].split()]
if tail < 2: sys.exit("no tail")
def blk(cap, d):
    b = 64 << 10
    while b < cap // 4096: b <<= 1
    return min(b, d)
pools = lambda cap: blk(cap, 256 << 20) + 3 * blk(cap, 64 << 20) + 4 * (4 << 20)
held = int(float(os.environ["PTL_PROBE_CAP_MB"]) * 1048576) - free
cap = held + free
for _ in range(4):
    cap = held + res + engine + pools(cap) + sum(layers) + tail // 2
print(f"{cap / 1048576:.6f}")
PY
}
# the snapshot of a run's settings (SNAP=...)
ptl_snap() { local a; for a in "$@"; do case $a in SNAP=*) echo "${a#SNAP=}"; return;; esac; done; }
# ptl_v41_cap <tag> <k|notail> <env...> -- <argv...>: COLI_VK_DEVICE_CAP_MB aimed at k layers
# (notail: every layer but not the tail) from a probe at 256 MiB with the same settings;
# the reserve holds the first chunk's scratch and the frames' staging after the tier took
# the rest
ptl_v41_cap() {
  local tag=$1 k=$2; shift 2
  local R=(COLI_VK_TIER_RESERVE_GB=0.04) cap pred envs=() snap
  while [ "$1" != "--" ]; do envs+=("$1"); shift; done; shift
  snap=$(ptl_snap "${envs[@]}")
  ptl_v41_probe ptl-probe.log COLI_VK_DEVICE_CAP_MB=256 "${R[@]}" "${envs[@]}" -- "$@"
  if [ "$k" = notail ]; then
    cap=$(PTL_PROBE_CAP_MB=256 ptl_notail deepseek_v41 ptl-probe.log) || fail "$tag: no cap from the probe"
    k=$(ptl_L deepseek_v41 ptl-probe.log)
  else
    cap=$(PTL_PROBE_CAP_MB=256 ptl_calc cap deepseek_v41 ptl-probe.log "$k") || fail "$tag: no cap from the probe"
  fi
  PTL_WANT=$k ptl_v41 "$tag (cap $cap MiB)" - COLI_VK_DEVICE_CAP_MB="$cap" "${R[@]}" "${envs[@]}" -- "$@"
  pred=$(ptl_calc predict deepseek_v41 vk.log)
  [ "$pred" = "$k" ] || { grep '^\[VK\] deepseek_v41 chain' vk.log; fail "$tag: the rule gives $pred from the run's own numbers"; }
  ptl_plan deepseek_v41 deepseek_v41 "$tag" "$snap" vk.log COLI_VK_DEVICE_CAP_MB="$cap" "${R[@]}" "${envs[@]}"
}
# ptl_v41_fault <tag> <k> <env...> -- <argv...>: a staged upload fails inside layer k's
# setup (every layer asked for): the chain keeps the layers before it
ptl_v41_fault() {
  local tag=$1 k=$2; shift 2
  local envs=() L n
  while [ "$1" != "--" ]; do envs+=("$1"); shift; done; shift
  ptl_v41_probe ptl-probe.log COLI_VK_STAGED=1 COLI_VK_STAGED_FAULT=submit:1000000 COLI_VK_CHAIN_LAYERS=99 "${envs[@]}" -- "$@"
  L=$(ptl_L deepseek_v41 ptl-probe.log)
  n=$(ptl_calc fault deepseek_v41 ptl-probe.log "$k") || { grep '^\[VK\] deepseek_v41 chain' ptl-probe.log; fail "$tag: no fault count from the probe"; }
  PTL_WANT=$k ptl_v41 "$tag (submit #$n)" "$L" COLI_VK_STAGED=1 COLI_VK_STAGED_FAULT=submit:"$n" "${envs[@]}" -- "$@"
  grep -q "deepseek_v41 chain: $k of $L layers on the device.*layer $k did not reach the device" vk.log ||
    { grep '^\[VK\] deepseek_v41 chain' vk.log; fail "$tag: no line for layer $k's failure"; }
}
# ptl_v41_droppable <snapshot> <k>: the matrices of layers 0..k-1 the device holds alone with
# the dense weights on the device only (v41_dho_bytes' set), from the checkpoint's header
ptl_v41_droppable() {
  $PY - "$1" "$2" <<'PY'
import json, re, struct, sys
with open(sys.argv[1] + "/model.safetensors", "rb") as f:
    h = json.loads(f.read(struct.unpack("<Q", f.read(8))[0]))
pat = re.compile(r"layers\.(\d+)\.(?:attn\.(?:wq_a|wq_b|wkv|wo_a|wo_b|compressor\.(?:wkv|wgate)|indexer\.(?:wk|wq_b|weights_proj))"
                 r"|ffn\.shared_experts\.w[123]|engram\.wkv)\.weight")
print(sum(1 for name in h if (m := pat.fullmatch(name)) and int(m[1]) < int(sys.argv[2])))
PY
}
# ptl_v41_dho <tag> <k> <env...> -- <argv...>: the dense weights on the device only with
# N = k: the k layers' matrices alone give back their host copies (the count from the
# header), none comes back on a healthy run (after a loss: some of theirs, no other)
ptl_v41_dho() {
  local tag=$1 k=$2; shift 2
  local snap; snap=$(ptl_snap "$@")
  ptl_v41 "$tag" "$k" COLI_VK_DENSE_HOST=0 "$@"
  local n want L r
  n=$(dho_dropped vk.log); want=$(ptl_v41_droppable "$snap" "$k"); L=$(ptl_L deepseek_v41 vk.log); r=$(dho_reloaded vk.log)
  [ "$n" = "$want" ] || { grep '^\[VK\]' vk.log; fail "$tag: $n host copies dropped, the $k layers have $want"; }
  if [ "$k" -lt "$L" ]; then
    grep -q "dense matrices on the device only, .* (the $k of $L layers on the device; the $((L - k)) on the CPU keep theirs)" vk.log ||
      { grep '^\[VK\]' vk.log; fail "$tag: the placed line does not say the $k of $L layers"; }
  fi
  if [ -n "${FAULT_BACK:-}" ]; then
    [ "$r" -gt 0 ] && [ "$r" -le "$n" ] || { grep '^\[VK\]' vk.log; fail "$tag: $r matrices read back after the loss ($n on the device only)"; }
  else [ "$r" = 0 ] || { grep '^\[VK\]' vk.log; fail "$tag: $r matrices read back from disk on a healthy run"; }; fi
  echo "   $n host copies dropped (the $k layers' $want), $r read back"
}
# ptl_v41_same <tag> <a> <b> <env...> -- <argv...>: the chain's logits under settings a and b
# (each one word, X=1 for none) the same bits, the tier deterministic
ptl_v41_same() {
  local tag=$1 a=$2 b=$3; shift 3
  local envs=(); while [ "$1" != "--" ]; do envs+=("$1"); shift; done; shift
  local D=(COLI_VK_TIER_SYNC=1 COLI_VK_TIER_BALANCE=0 COLI_VK_TIER_STREAM_ROWS=16 COLI_VULKAN=1 COLI_VK_CHAIN=${CHAINMODE:-1})
  rm -f chain.usage same-a.f32 same-b.f32
  env "${envs[@]}" "${D[@]}" $a COLI_USAGE=chain.usage DUMP=same-a.f32 ./deepseek_v41 "$@" > /dev/null 2> same-a.log || true
  rm -f chain.usage
  env "${envs[@]}" "${D[@]}" $b COLI_USAGE=chain.usage DUMP=same-b.f32 ./deepseek_v41 "$@" > /dev/null 2> same-b.log || true
  rm -f chain.usage
  [ -s same-a.f32 ] && cmp -s same-a.f32 same-b.f32 || { grep '^\[VK\] deepseek_v41 chain' same-a.log same-b.log; fail "$tag: the logits differ"; }
  echo "OK $tag: the same bits ($a; $b)"
}

ptl_v41_cases() {
  local k f L=SNAP=dsv41_long T=SNAP=dsv41_tiny
  # ---- N forced: every k of the six layers (2: the index list crosses, 4: the candidate mask)
  for k in 0 1 2 3 4 5 6; do ptl_v41 "partial deepseek_v41 COLI_VK_CHAIN_LAYERS=$k" $k $L -- 8 dsv41_long/ref.json; done
  ptl_v41 "partial deepseek_v41 12-token prompt, 3 of 6 layers" 3 $T -- 8 dsv41_tiny/ref.json
  ptl_v41 "partial deepseek_v41 one cache slot, 2 of 6 layers" 2 $L -- 1 dsv41_long/ref.json
  # ---- everything fits: every layer, the same bits as asking for all of them
  PTL_WANT=6 ptl_v41 "partial deepseek_v41 the fit, everything fitting" - $L -- 8 dsv41_long/ref.json
  ptl_v41_same "partial deepseek_v41 the fit = every layer asked for" X=1 COLI_VK_CHAIN_LAYERS=6 $L -- 8 dsv41_long/ref.json
  # ---- a small device: N from the budget
  for k in 1 3 5 0; do ptl_v41_cap "partial deepseek_v41 a device for $k layers" $k $L -- 8 dsv41_long/ref.json; done
  # every layer and not the head: the per-matrix path (COLI_VK_DENSE) leaves it on the CPU
  ptl_v41_cap "partial deepseek_v41 a device for every layer, not the head" notail $L COLI_VK_DENSE=1 -- 8 dsv41_long/ref.json
  grep -q 'deepseek_v41 chain: 6 of 6 layers on the device (.*), 0 on the CPU, the head and what goes with it' vk.log ||
    { grep '^\[VK\] deepseek_v41 chain' vk.log; fail "partial deepseek_v41 every layer, not the head: no line for the head on the CPU"; }
  # ---- a staged upload failing in the middle of the setup
  for k in 0 2 4; do ptl_v41_fault "partial deepseek_v41 an upload failing in layer $k" $k $L -- 8 dsv41_long/ref.json; done
  # ---- the forward paths with N < L
  ptl_v41 "partial deepseek_v41 prompt in chunks of 3" 2 $L COLI_VK_CHAIN_ROWS=3 -- 8 dsv41_long/ref.json
  ptl_v41 "partial deepseek_v41 prompt in chunks of 7" 4 $L COLI_VK_CHAIN_ROWS=7 -- 2 dsv41_long/ref.json
  ptl_v41 "partial deepseek_v41 cold experts streamed" 3 SNAP=dsv41_vlong $PF_FORCE COLI_VK_TIER_GB=0.0005 COLI_VK_CHAIN_ROWS=64 -- 2 dsv41_vlong/ref.json
  pf_check deepseek_v41 vk.log "partial deepseek_v41 cold experts streamed" forced
  for k in 2 4; do for f in 1 3 5; do
    ptl_v41 "partial deepseek_v41 DSpark spec=$f, $k of 6 layers" $k $T V41_DSPARK=1 V41_SPEC_FORCE=$f -- 8 dsv41_tiny/ref.json
  done; done
  CHAINMODE=2 ptl_v41 "partial deepseek_v41 prompts only" 3 $L -- 8 dsv41_long/ref.json
  CHAINMODE=2 ptl_v41 "partial deepseek_v41 prompts only, DSpark spec=2" 2 $T V41_DSPARK=1 V41_SPEC_FORCE=2 -- 8 dsv41_tiny/ref.json
  ptl_v41 "partial deepseek_v41 tier off" 3 $L COLI_VK_TIER=0 -- 8 dsv41_long/ref.json
  ptl_v41 "partial deepseek_v41 beside the per-matrix trunk" 3 $L COLI_VK_DENSE=1 -- 8 dsv41_long/ref.json
  ptl_v41 "partial deepseek_v41 the per-row GEMV" 3 $T COLI_VK_CHAIN_GEMV=0 -- 8 dsv41_tiny/ref.json
  ptl_v41 "partial deepseek_v41 V41_INDEX_OWNER=1" 4 $L V41_INDEX_OWNER=1 -- 8 dsv41_long/ref.json
  kv_v41_gate "partial deepseek_v41 the KV split" 8 2 COLI_VK_CHAIN_LAYERS=3 $L -- 8 dsv41_long/ref.json
  ptl_check_n deepseek_v41 vk.log 3 "partial deepseek_v41 the KV split"
  kv_v41_gate "partial deepseek_v41 the KV split, DSpark spec=3" 16 4 COLI_VK_CHAIN_LAYERS=4 $L V41_DSPARK=1 V41_SPEC_FORCE=3 -- 8 dsv41_long/ref.json
  ptl_check_n deepseek_v41 vk.log 4 "partial deepseek_v41 the KV split, DSpark spec=3"
  FAULT_BACK=5 ptl_v41 "partial deepseek_v41 device lost mid-decode" 3 $L -- 8 dsv41_long/ref.json
  FAULT_BACK=60 ptl_v41 "partial deepseek_v41 device lost in the prompt" 3 $L COLI_VK_CHAIN_ROWS=7 -- 8 dsv41_long/ref.json
  FAULT_BACK=8 ptl_v41 "partial deepseek_v41 device lost between drafts" 2 $T V41_DSPARK=1 V41_SPEC_FORCE=5 -- 8 dsv41_tiny/ref.json
  # ---- the dense weights on the device only
  for k in 1 3 6; do ptl_v41_dho "partial deepseek_v41 device only, $k of 6 layers" $k $L -- 8 dsv41_long/ref.json; done
  ptl_v41_dho "partial deepseek_v41 device only, DSpark spec=3" 2 $T V41_DSPARK=1 V41_SPEC_FORCE=3 -- 8 dsv41_tiny/ref.json
  ptl_v41_dho "partial deepseek_v41 device only, staged" 3 $L COLI_VK_STAGED=1 -- 8 dsv41_long/ref.json
  CHAINMODE=2 ptl_v41_dho "partial deepseek_v41 device only, prompts only" 3 $L -- 8 dsv41_long/ref.json
  FAULT_BACK=5 ptl_v41_dho "partial deepseek_v41 device only, device lost mid-decode" 3 $L -- 8 dsv41_long/ref.json
  ptl_v41_fault "partial deepseek_v41 device only, an upload failing in layer 3" 3 $L COLI_VK_DENSE_HOST=0 -- 8 dsv41_long/ref.json
  [ "$(dho_reloaded vk.log)" = 0 ] || { grep '^\[VK\]' vk.log; fail "partial deepseek_v41 device only, an upload failing: read back from disk"; }
  # the trunk's placement asks with N from the budget (the fit runs there, before any upload)
  ptl_v41_cap "partial deepseek_v41 device only, a device for 3 layers" 3 $L COLI_VK_DENSE_HOST=0 -- 8 dsv41_long/ref.json
  [ "$(dho_dropped vk.log)" = "$(ptl_v41_droppable dsv41_long 3)" ] && [ "$(dho_reloaded vk.log)" = 0 ] ||
    { grep '^\[VK\]' vk.log; fail "partial deepseek_v41 device only, a device for 3 layers: the host copies dropped are not the 3 layers'"; }
  ptl_v41_same "partial deepseek_v41 device only = host copies" COLI_VK_DENSE_HOST=1 COLI_VK_DENSE_HOST=0 COLI_VK_CHAIN_LAYERS=3 $L -- 8 dsv41_long/ref.json
  # ---- serve sessions, prefix reuse and images with N < L
  CHAIN_SERVE_EXPECT='deepseek_v41 chain: 2 of 6 layers on the device' \
    $PY tests/vulkan_chain_serve.py ./deepseek_v41 dsv41_tiny V41_DSPARK=1 COLI_VK_CHAIN_LAYERS=2
  CHAIN_SERVE_EXPECT='deepseek_v41 chain: 4 of 6 layers on the device' \
    $PY tests/vulkan_chain_serve.py ./deepseek_v41 dsv41_tiny V41_DSPARK=0 COLI_VK_CHAIN_LAYERS=4
  CHAIN_SERVE_EXPECT='deepseek_v41 chain: 3 of 6 layers on the device' COLI_VK_CHAIN=2 \
    $PY tests/vulkan_chain_serve.py ./deepseek_v41 dsv41_tiny V41_DSPARK=1 COLI_VK_CHAIN_ROWS=3 COLI_VK_CHAIN_LAYERS=3
  CHAIN_SERVE_EXPECT='dense matrices on the device only, .*the 3 of 6 layers on the device' COLI_VK_DENSE_HOST=0 \
    $PY tests/vulkan_chain_serve.py ./deepseek_v41 dsv41_tiny V41_DSPARK=1 COLI_VK_CHAIN_LAYERS=3
  $PY tests/vulkan_chain_v41_image.py ./deepseek_v41 dsv41_tiny V41_DSPARK=0 COLI_VK_CHAIN_LAYERS=3
  COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_CHAIN_LAYERS=3 COLI_VK_TIER_SYNC=1 COLI_USAGE=$PWD/chain.usage \
    $PY -m unittest tests.test_dsv41_prefix_serve tests.test_dsv41_dspark_serve
  rm -f chain.usage chain-serve.usage chain-image.usage
}

# ---- kimi_k3 ---------------------------------------------------------------------------
PTL_K3_O="K3_BITS=32 K3_MLA_BITS=32 K3_HEAD_BITS=32 K3_IDOT=0 COLI_TEMP=0"
ptl_k3_resident() {
  local s x
  s=$(sed -n 's/^\[VK\] kimi_k3 chain: [0-9]* layers on the device (.*, \([0-9]*\) matrices (.* on the device$/\1/p' "$1" | tail -1)
  x=$(sed -n 's/^\[VK\] kimi_k3: \([0-9]*\) resident matrices on the device.*/\1/p' "$1" | tail -1)
  echo "${s:-0} ${x:-0}"
}
# ptl_k3_off <tag> <case> <env...>: N = 0, the chain off: the CPU's tokens, every logits row
# within k3c_close, no chain forward, no dense matrix on the device
ptl_k3_off() {
  local tag=$1 c=$2; shift 2
  rm -f k3c.usage cpu.f32 vk.f32
  env "$@" COLI_USAGE=$PWD/k3c.usage USAGE_SAVE=0 K3_VAL_LOGITS=cpu.f32 ./kimi_k3 kimi_k3_tiny --ids "$(k3c_ids $c)" --ngen 8 \
    2> cpu.log | sed 's/ *TUNE.*//' > cpu.tok
  env "$@" COLI_USAGE=$PWD/k3c.usage USAGE_SAVE=0 K3_VAL_LOGITS=vk.f32 COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=${CHAINMODE:-1} \
    ./kimi_k3 kimi_k3_tiny --ids "$(k3c_ids $c)" --ngen 8 2> vk.log | sed 's/ *TUNE.*//' > vk.tok
  { [ -s cpu.tok ] && cmp -s cpu.tok vk.tok; } || { cat cpu.tok vk.tok; tail -20 vk.log; fail "$tag $c: tokens"; }
  ptl_check_n kimi_k3 vk.log 0 "$tag"
  [ "$(chain_count kimi_k3 vk.log)" = 0 ] || { grep '^\[VK\]' vk.log; fail "$tag: the chain ran"; }
  if grep -q '^\[VK\] kimi_k3 chain: [0-9]* of [0-9]* layers placed' vk.log; then ptl_check_placed kimi_k3 vk.log "$tag"; fi
  set -- $(ptl_k3_resident vk.log)
  [ "$2" = 0 ] || { grep '^\[VK\]' vk.log; fail "$tag: $2 dense matrices on the device with the chain off"; }
  local lg; lg=$(k3c_close cpu.f32 vk.f32) || { echo "$lg"; fail "$tag $c: logits"; }
  echo "OK $tag $c: N = 0, the chain off, tokens = CPU, $lg, no dense matrix on the device"
}
# ptl_k3 <tag> <case> <k|-> <env...>: k3c_gate on one oracle case with COLI_VK_CHAIN_LAYERS=k
# (-: the fit decides), then N = PTL_WANT (default k), the placed check, and with N < L
# nothing more on the device at exit than at setup
ptl_k3() {
  local tag=$1 c=$2 k=$3; shift 3
  local want=${PTL_WANT:-$k} e=()
  [ "$k" != - ] && e=(COLI_VK_CHAIN_LAYERS=$k)
  if [ "$want" = 0 ]; then ptl_k3_off "$tag" "$c" "${e[@]}" "$@"; return; fi
  K3C_CASES=$c k3c_gate "$tag" "${e[@]}" "$@"
  ptl_check_n kimi_k3 vk.log "$want" "$tag"
  ptl_check_placed kimi_k3 vk.log "$tag"
  local L; L=$(ptl_L kimi_k3 vk.log)
  if [ "$want" -lt "$L" ] && [ -z "${FAULT_BACK:-}" ]; then
    set -- $(ptl_k3_resident vk.log)
    [ "$1" = "$2" ] || { grep '^\[VK\]' vk.log; fail "$tag: $1 matrices on the device at setup, $2 at exit: the CPU's layers went up"; }
  fi
  echo "   N = $want of $L, $(ptl_calc matrices kimi_k3 vk.log | cut -d' ' -f1) B of matrices on the device after setup"
}
# ptl_k3_probe <log> <case> <env...>: one chained run, its stderr kept
ptl_k3_probe() {
  local log=$1 c=$2; shift 2
  rm -f k3c.usage
  env "$@" COLI_USAGE=$PWD/k3c.usage USAGE_SAVE=0 COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=${CHAINMODE:-1} \
    ./kimi_k3 kimi_k3_tiny --ids "$(k3c_ids $c)" --ngen 8 > /dev/null 2> "$log" || true
  grep -q '^\[VK\] kimi_k3 chain fit: ' "$log" || { tail -20 "$log"; fail "probe run: no fit line"; }
}
# ptl_k3_cap <tag> <case> <k|notail> <env...>: COLI_VK_DEVICE_CAP_MB aimed at k layers (notail:
# every layer, not the head) from a probe at 256 MiB with the same settings
ptl_k3_cap() {
  local tag=$1 c=$2 k=$3; shift 3
  local R=(COLI_VK_TIER_RESERVE_GB=0.04) cap pred
  ptl_k3_probe ptl-probe.log "$c" COLI_VK_DEVICE_CAP_MB=256 "${R[@]}" "$@"
  if [ "$k" = notail ]; then
    cap=$(PTL_PROBE_CAP_MB=256 ptl_notail kimi_k3 ptl-probe.log) || fail "$tag: no cap from the probe"
    k=$(ptl_L kimi_k3 ptl-probe.log)
  else
    cap=$(PTL_PROBE_CAP_MB=256 ptl_calc cap kimi_k3 ptl-probe.log "$k") || fail "$tag: no cap from the probe"
  fi
  PTL_WANT=$k ptl_k3 "$tag (cap $cap MiB)" "$c" - COLI_VK_DEVICE_CAP_MB="$cap" "${R[@]}" "$@"
  pred=$(ptl_calc predict kimi_k3 vk.log)
  [ "$pred" = "$k" ] || { grep '^\[VK\] kimi_k3 chain' vk.log; fail "$tag: the rule gives $pred from the run's own numbers"; }
  ptl_plan kimi_k3 kimi "$tag" kimi_k3_tiny vk.log COLI_VK_DEVICE_CAP_MB="$cap" "${R[@]}" "$@"
}
# ptl_k3_fault <tag> <case> <k> <env...>: a staged upload fails inside layer k's setup
ptl_k3_fault() {
  local tag=$1 c=$2 k=$3; shift 3
  local L n
  ptl_k3_probe ptl-probe.log "$c" COLI_VK_STAGED=1 COLI_VK_STAGED_FAULT=submit:1000000 COLI_VK_CHAIN_LAYERS=99 "$@"
  L=$(ptl_L kimi_k3 ptl-probe.log)
  n=$(ptl_calc fault kimi_k3 ptl-probe.log "$k") || { grep '^\[VK\] kimi_k3 chain' ptl-probe.log; fail "$tag: no fault count from the probe"; }
  PTL_WANT=$k ptl_k3 "$tag (submit #$n)" "$c" "$L" COLI_VK_STAGED=1 COLI_VK_STAGED_FAULT=submit:"$n" "$@"
  grep -q "kimi_k3 chain: $k of $L layers on the device.*layer $k did not reach the device" vk.log ||
    { grep '^\[VK\] kimi_k3 chain' vk.log; fail "$tag: no line for layer $k's failure"; }
}
# ptl_k3_droppable <k>: the matrices of layers 0..k-1 the device holds alone with the
# dense weights on the device only and the chain on: every W (k3_dho_each) and the f32
# routers, decay pairs and beta projections the chain holds (k3_dho_place)
ptl_k3_droppable() {
  $PY - kimi_k3_tiny "$1" <<'PY'
import json, re, struct, sys
with open(sys.argv[1] + "/model.safetensors", "rb") as f:
    h = json.loads(f.read(struct.unpack("<Q", f.read(8))[0]))
pat = re.compile(r"model\.layers\.(\d+)\.(?:self_attn\.(?:[qkvog]_proj|q_a_proj|q_b_proj|kv_a_proj_with_mqa|kv_b_proj|f_a_proj|f_b_proj|b_proj)"
                 r"|block_sparse_moe\.(?:gate|routed_expert_down_proj|routed_expert_up_proj|shared_experts\.(?:gate|up|down)_proj)"
                 r"|mlp\.(?:gate|up|down)_proj)\.weight")
print(sum(1 for name in h if (m := pat.fullmatch(name)) and int(m[1]) < int(sys.argv[2])))
PY
}
# ptl_k3_dho <tag> <case> <k> <env...>: the dense weights on the device only with N = k
ptl_k3_dho() {
  local tag=$1 c=$2 k=$3; shift 3
  ptl_k3 "$tag" "$c" "$k" COLI_VK_DENSE_HOST=0 "$@"
  local n want L r
  n=$(dho_dropped vk.log); want=$(ptl_k3_droppable "$k"); L=$(ptl_L kimi_k3 vk.log); r=$(dho_reloaded vk.log)
  # RELOAD_MLA=1 (prompts only: decode runs on the CPU, whose MLA attention reads kv_b by
  # row): each MLA layer on the device reads its kv_b back once, as in the dense-only family
  local reloads=0
  [ "${RELOAD_MLA:-0}" = 1 ] && reloads=$(sed -n 's/^\[VK\] kimi_k3 chain: [0-9]* layers on the device ([0-9]* KDA, \([0-9]*\) MLA.*/\1/p' vk.log | tail -1)
  [ "$k" = "$L" ] && want=$((want + 1))   # the head, with every layer
  [ "$n" = "$want" ] || { grep '^\[VK\]' vk.log; fail "$tag: $n host copies dropped, the $k layers have $want"; }
  if [ "$k" -lt "$L" ]; then
    grep -q "dense matrices on the device only, .* (the $k of $L layers on the device; the $((L - k)) on the CPU keep theirs)" vk.log ||
      { grep '^\[VK\]' vk.log; fail "$tag: the placed line does not say the $k of $L layers"; }
  fi
  if [ -n "${FAULT_BACK:-}" ]; then
    [ "$r" -gt 0 ] && [ "$r" -le "$n" ] || { grep '^\[VK\]' vk.log; fail "$tag: $r matrices read back after the loss ($n on the device only)"; }
  else [ "$r" = "${reloads:-0}" ] || { grep '^\[VK\]' vk.log; fail "$tag: $r matrices read back from disk on a healthy run, expected ${reloads:-0}"; }; fi
  echo "   $n host copies dropped (the $k layers' $want), $r read back"
}
# ptl_k3_same <tag> <case> <a> <b> <env...>: the chain's logits under settings a and b the
# same bits, the tier deterministic
ptl_k3_same() {
  local tag=$1 c=$2 a=$3 b=$4; shift 4
  local D=(COLI_VK_TIER_SYNC=1 COLI_VK_TIER_BALANCE=0 COLI_VK_TIER_STREAM_ROWS=16 COLI_VULKAN=1 COLI_VK_CHAIN=${CHAINMODE:-1})
  rm -f k3c.usage same-a.f32 same-b.f32
  env "$@" "${D[@]}" $a COLI_USAGE=$PWD/k3c.usage USAGE_SAVE=0 K3_VAL_LOGITS=same-a.f32 ./kimi_k3 kimi_k3_tiny --ids "$(k3c_ids $c)" --ngen 8 > /dev/null 2> same-a.log || true
  env "$@" "${D[@]}" $b COLI_USAGE=$PWD/k3c.usage USAGE_SAVE=0 K3_VAL_LOGITS=same-b.f32 ./kimi_k3 kimi_k3_tiny --ids "$(k3c_ids $c)" --ngen 8 > /dev/null 2> same-b.log || true
  [ -s same-a.f32 ] && cmp -s same-a.f32 same-b.f32 || { grep '^\[VK\] kimi_k3 chain' same-a.log same-b.log; fail "$tag: the logits differ"; }
  echo "OK $tag: the same bits ($a; $b)"
}

ptl_k3_cases() {
  local O=$PTL_K3_O k c out frames
  # ---- N forced: every k (1, 3, 5 cut inside an AttnRes block of 2 layers)
  for k in 0 1 2 3 4 5 6; do ptl_k3 "partial kimi_k3 COLI_VK_CHAIN_LAYERS=$k" long $k $O; done
  for c in short chunk; do ptl_k3 "partial kimi_k3 COLI_VK_CHAIN_LAYERS=3" $c 3 $O; done
  # ---- everything fits: every layer and the head, the same bits as asking for all of them
  PTL_WANT=6 ptl_k3 "partial kimi_k3 the fit, everything fitting" long - $O
  ptl_k3_same "partial kimi_k3 the fit = every layer asked for" long X=1 COLI_VK_CHAIN_LAYERS=6 $O
  # ---- a small device: N from the budget
  for k in 1 3 5 0; do ptl_k3_cap "partial kimi_k3 a device for $k layers" long $k $O; done
  # every layer and not the head: the chain hands the CPU its final rows, the CPU the head
  ptl_k3_cap "partial kimi_k3 a device for every layer, not the head" long notail $O
  grep -q 'kimi_k3 chain: 6 of 6 layers on the device (.*), 0 on the CPU, the head and what goes with it' vk.log ||
    { grep '^\[VK\] kimi_k3 chain' vk.log; fail "partial kimi_k3 every layer, not the head: no line for the head on the CPU"; }
  # ---- a staged upload failing in the middle of the setup
  for k in 0 2 4; do ptl_k3_fault "partial kimi_k3 an upload failing in layer $k" long $k $O; done
  # ---- the forward paths with N < L
  ptl_k3 "partial kimi_k3 chain chunks of 3" long 3 $O COLI_VK_CHAIN_ROWS=3
  ptl_k3 "partial kimi_k3 prefill one token at a time" long 2 $O K3_CHUNK=1
  K3C_IDS=$(pf_ids 200 1 300 | tr ',' ' ') ptl_k3 "partial kimi_k3 cold experts streamed" ids 3 $O $PF_FORCE COLI_VK_TIER_GB=0.0005 COLI_VK_CHAIN_ROWS=50
  pf_check kimi_k3 vk.log "partial kimi_k3 cold experts streamed" forced
  for k in 1 4; do ptl_k3 "partial kimi_k3 int4 KDA and MoE, int8 MLA and head, $k of 6 layers" long $k K3_BITS=4 K3_IDOT=0 COLI_TEMP=0; done
  ptl_k3 "partial kimi_k3 tier off" long 3 $O COLI_VK_TIER=0
  ptl_k3 "partial kimi_k3 shared experts by COLI_VK_DENSE=1" long 3 $O COLI_VK_DENSE=1
  ptl_k3 "partial kimi_k3 K3_TOPP=0.6" long 3 $O K3_TOPP=0.6
  CHAINMODE=2 ptl_k3 "partial kimi_k3 prompts only" long 3 $O
  K3C_CASES=long kv_k3c_gate "partial kimi_k3 the KV split" 8 2 COLI_VK_CHAIN_LAYERS=3 $O
  ptl_check_n kimi_k3 vk.log 3 "partial kimi_k3 the KV split"
  FAULT_BACK=3 REBUILD=1 ptl_k3 "partial kimi_k3 device lost mid-decode" long 3 $O
  FAULT_BACK=29 ptl_k3 "partial kimi_k3 device lost in the first prompt chunk" long 3 $O K3_CHUNK=32
  FAULT_BACK=26 REBUILD=1 ptl_k3 "partial kimi_k3 device lost in a later prompt chunk" long 3 $O K3_CHUNK=32
  # ---- the dense weights on the device only
  for k in 1 3 6; do ptl_k3_dho "partial kimi_k3 device only, $k of 6 layers" long $k $O; done
  ptl_k3_dho "partial kimi_k3 device only, staged" long 3 $O COLI_VK_STAGED=1
  CHAINMODE=2 RELOAD_MLA=1 ptl_k3_dho "partial kimi_k3 device only, prompts only" long 3 $O
  FAULT_BACK=3 REBUILD=1 ptl_k3_dho "partial kimi_k3 device only, device lost mid-decode" long 3 $O
  # device only: the dense weights go up through the per-matrix placement before the chain
  # (k3_dho_place), not with each chain layer as deepseek_v41's; an upload failing there
  # keeps that matrix's host copy and the chain puts it up itself, so the run keeps every
  # layer: N = L, one host copy fewer dropped, nothing read back, the CPU's tokens
  local all fn L
  ptl_k3_probe ptl-probe.log long COLI_VK_STAGED=1 COLI_VK_STAGED_FAULT=submit:1000000 COLI_VK_CHAIN_LAYERS=99 $O COLI_VK_DENSE_HOST=0
  all=$(dho_dropped ptl-probe.log); L=$(ptl_L kimi_k3 ptl-probe.log)
  fn=$(ptl_calc fault kimi_k3 ptl-probe.log 3) || { grep '^\[VK\] kimi_k3 chain' ptl-probe.log; fail "partial kimi_k3 device only: no fault count from the probe"; }
  PTL_WANT=$L ptl_k3 "partial kimi_k3 device only, an upload failing in layer 3's placement (submit #$fn)" long "$L" \
    COLI_VK_STAGED=1 COLI_VK_STAGED_FAULT=submit:"$fn" $O COLI_VK_DENSE_HOST=0
  grep -q "staged upload failed (submit: .*): a matrix stays on the CPU" vk.log && [ "$(dho_dropped vk.log)" = $((all - 1)) ] &&
    [ "$(dho_reloaded vk.log)" = 0 ] ||
    { grep '^\[VK\]' vk.log; fail "partial kimi_k3 device only, an upload failing: not one host copy kept with every layer on the device"; }
  ptl_k3_cap "partial kimi_k3 device only, a device for 3 layers" long 3 $O COLI_VK_DENSE_HOST=0
  [ "$(dho_dropped vk.log)" = "$(ptl_k3_droppable 3)" ] && [ "$(dho_reloaded vk.log)" = 0 ] ||
    { grep -E '^\[VK\]|^\[K3\]' vk.log; fail "partial kimi_k3 device only, a device for 3 layers: the host copies dropped are not the 3 layers'"; }
  grep -q '^\[K3\] dense weights on the device only: ' vk.log ||
    { grep '^\[K3\]' vk.log; fail "partial kimi_k3 device only: the expert cache's plan did not count the dropped copies"; }
  ptl_k3_same "partial kimi_k3 device only = host copies" long COLI_VK_DENSE_HOST=1 COLI_VK_DENSE_HOST=0 COLI_VK_CHAIN_LAYERS=3 $O
  # ---- serve sessions frame for frame with N < L: pins, the prompt cache, prefix reuse,
  # recurrent-state photos taken from the chain layers' device state and restored to it
  local S="K3_BITS=32 K3_MLA_BITS=32 K3_HEAD_BITS=32 K3_IDOT=0 K3_PREFIX_LOG=1 USAGE_SAVE=0"
  export CHAIN_SERVE_TOL=2e-2
  # shellcheck disable=SC2086
  {
    CHAIN_SERVE_EXPECT='kimi_k3 chain: 2 of 6 layers on the device' $PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve $S COLI_VK_CHAIN_LAYERS=2
    out=$(CHAIN_SERVE_EXPECT='kimi_k3 chain: 3 of 6 layers on the device' \
      $PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve $S COLI_K3_CKPT=4 COLI_VK_CHAIN_LAYERS=3); echo "$out"
    CHAIN_SERVE_EXPECT='kimi_k3 chain: 4 of 6 layers on the device' \
      $PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve K3_BITS=4 K3_IDOT=0 K3_PREFIX_LOG=1 USAGE_SAVE=0 COLI_VK_CHAIN_ROWS=3 COLI_VK_CHAIN_LAYERS=4
    CHAIN_SERVE_EXPECT='kimi_k3 chain: 3 of 6 layers on the device' COLI_VK_CHAIN=2 \
      $PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve $S COLI_K3_CKPT=4 COLI_VK_CHAIN_LAYERS=3
    CHAIN_SERVE_EXPECT='dense matrices on the device only, .*the 3 of 6 layers on the device' COLI_VK_DENSE_HOST=0 \
      $PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve $S COLI_K3_CKPT=2 COLI_VK_CHAIN_LAYERS=3
    # a device lost mid-session, 40 frames before the end of the same session without a fault
    frames=$(echo "$out" | sed -n 's/.*kimi_k3 chain: [0-9]* forwards, \([0-9]*\) frames.*/\1/p')
    [ -n "$frames" ] && [ "$frames" -gt 40 ] || fail "partial kimi_k3 serve: no fault-free session to count frames from"
    CHAIN_SERVE_EXPECT='kimi_k3 chain: the device was lost' \
      $PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve $S COLI_K3_CKPT=4 COLI_VK_CHAIN_LAYERS=3 COLI_VK_CHAIN_FAULT=$((frames - 40))
  }
  unset CHAIN_SERVE_TOL
  # recurrent-state checkpoints and the dashboard's lines with N < L
  COLI_USAGE=$PWD/k3c.usage USAGE_SAVE=0 COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_CHAIN_LAYERS=3 COLI_VK_TIER_SYNC=1 \
    $PY tests/test_kimi_k3_ckpt.py --binary ./kimi_k3 --fixture ./kimi_k3_tiny
  COLI_USAGE=$PWD/k3c.usage USAGE_SAVE=0 COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_CHAIN_LAYERS=3 COLI_VK_TIER_SYNC=1 \
    $PY tests/test_kimi_k3_dashboard.py --binary ./kimi_k3 --fixture ./kimi_k3_tiny
  rm -f k3c.usage chain-serve.usage
}

ptl_dsk_fixtures() {
  v41_chain_fixtures
  $PY tools/make_dsv41_tiny.py --out dsv41_vlong --emit-ref dsv41_vlong/ref.json --prompt-len 160 --max-new 6 > /dev/null
  $PY tools/make_kimi_k3_tiny.py --output ./kimi_k3_tiny --force
  k3c_serve_fixture
}
ptl_family_dsk() {
  export OMP_NUM_THREADS=2
  make deepseek_v41 kimi_k3 VK=1
  ptl_dsk_fixtures
  ptl_v41_cases
  ptl_k3_cases
  rm -rf dsv41_vlong kimi_k3_serve
  rm -f cpu.f32 vk.f32 cpu.txt vk.txt cpu.tok vk.tok same-a.* same-b.* ptl-probe.log
  unset OMP_NUM_THREADS
}

# The same under ASan and UBSan: no diagnostic, the chain ran with the N the line says
# (or handled the loss), the matrices placed the N layers' exactly.
ptl_dsk_san() {  # <engine> <tag> <want N|lost> <env and argv...>
  local eng=$1 tag=$2 want=$3; shift 3
  rm -f chain.usage k3c.usage
  env COLI_USAGE=$PWD/chain.usage USAGE_SAVE=0 COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 "$@" > san.log 2>&1 || true
  if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "$tag: sanitizer diagnostic"; fi
  if [ "$want" = lost ]; then
    grep -q "$eng chain: the device was lost" san.log || { cat san.log; fail "$tag: no loss was handled"; }
  else
    grep -q "$eng chain: [1-9][0-9]* forwards" san.log || { cat san.log; fail "$tag: the chain never ran"; }
    ptl_check_n "$eng" san.log "$want" "$tag"
    ptl_check_placed "$eng" san.log "$tag"
  fi
  echo "OK $tag: sanitizers clean, N = $(ptl_n "$eng" san.log), $(chain_count "$eng" san.log) chain forwards"
}
ptl_family_dsk_sanitize() {
  local SAN="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
  make clean >/dev/null 2>&1 || true
  make deepseek_v41 kimi_k3 VK=1 EXTRA_CFLAGS="$SAN"
  export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1 OMP_NUM_THREADS=2
  ptl_dsk_fixtures
  local cap n frames ids O=$PTL_K3_O
  # ---- deepseek_v41
  local V=(SNAP=dsv41_long ./deepseek_v41 8 dsv41_long/ref.json)
  ptl_dsk_san deepseek_v41 "asan partial deepseek_v41 3 of 6 layers" 3 COLI_VK_CHAIN_LAYERS=3 "${V[@]}"
  ptl_dsk_san deepseek_v41 "asan partial deepseek_v41 2 of 6 layers, chunks of 3" 2 COLI_VK_CHAIN_LAYERS=2 COLI_VK_CHAIN_ROWS=3 "${V[@]}"
  ptl_dsk_san deepseek_v41 "asan partial deepseek_v41 4 of 6 layers, DSpark spec=5" 4 COLI_VK_CHAIN_LAYERS=4 SNAP=dsv41_tiny \
    V41_DSPARK=1 V41_SPEC_FORCE=5 ./deepseek_v41 1 dsv41_tiny/ref.json
  ptl_v41_probe ptl-probe.log COLI_VK_DEVICE_CAP_MB=256 COLI_VK_TIER_RESERVE_GB=0.04 SNAP=dsv41_long -- 8 dsv41_long/ref.json
  cap=$(PTL_PROBE_CAP_MB=256 ptl_calc cap deepseek_v41 ptl-probe.log 2)
  ptl_dsk_san deepseek_v41 "asan partial deepseek_v41 a device for 2 layers" 2 COLI_VK_DEVICE_CAP_MB=$cap COLI_VK_TIER_RESERVE_GB=0.04 "${V[@]}"
  ptl_v41_probe ptl-probe.log COLI_VK_STAGED=1 COLI_VK_STAGED_FAULT=submit:1000000 COLI_VK_CHAIN_LAYERS=6 SNAP=dsv41_long -- 8 dsv41_long/ref.json
  n=$(ptl_calc fault deepseek_v41 ptl-probe.log 3)
  ptl_dsk_san deepseek_v41 "asan partial deepseek_v41 an upload failing in layer 3" 3 COLI_VK_STAGED=1 COLI_VK_STAGED_FAULT=submit:$n COLI_VK_CHAIN_LAYERS=6 "${V[@]}"
  ptl_dsk_san deepseek_v41 "asan partial deepseek_v41 device only, an upload failing in layer 3" 3 COLI_VK_DENSE_HOST=0 COLI_VK_STAGED=1 COLI_VK_STAGED_FAULT=submit:$n COLI_VK_CHAIN_LAYERS=6 "${V[@]}"
  ptl_dsk_san deepseek_v41 "asan partial deepseek_v41 device only, 3 of 6 layers" 3 COLI_VK_CHAIN_LAYERS=3 COLI_VK_DENSE_HOST=0 "${V[@]}"
  ptl_dsk_san deepseek_v41 "asan partial deepseek_v41 the KV split" 3 COLI_VK_CHAIN_LAYERS=3 COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=2 "${V[@]}"
  ptl_dsk_san deepseek_v41 "asan partial deepseek_v41 device lost" lost COLI_VK_CHAIN_LAYERS=3 COLI_VK_CHAIN_FAULT=30 "${V[@]}"
  CHAIN_SERVE_EXPECT='deepseek_v41 chain: 2 of 6 layers on the device' \
    $PY tests/vulkan_chain_serve.py ./deepseek_v41 dsv41_tiny V41_DSPARK=1 COLI_VK_CHAIN_LAYERS=2 > san.log 2>&1 || { cat san.log; fail "asan partial deepseek_v41 serve"; }
  if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "asan partial deepseek_v41 serve: sanitizer diagnostic"; fi
  echo "OK asan partial deepseek_v41 serve: $(tail -1 san.log | cut -c1-120)"
  # ---- kimi_k3
  ids=$(k3c_ids long)
  local K=(./kimi_k3 kimi_k3_tiny --ids "$ids" --ngen 8)
  # shellcheck disable=SC2086
  {
    ptl_dsk_san kimi_k3 "asan partial kimi_k3 3 of 6 layers" 3 $O COLI_VK_CHAIN_LAYERS=3 "${K[@]}"
    frames=$(sed -n 's/^\[VK\] kimi_k3 chain: [0-9]* forwards, \([0-9]*\) frames.*/\1/p' san.log | tail -1)
    ptl_dsk_san kimi_k3 "asan partial kimi_k3 1 of 6 layers, int4 trunk, chunks of 3" 1 K3_BITS=4 K3_IDOT=0 COLI_VK_CHAIN_LAYERS=1 COLI_VK_CHAIN_ROWS=3 "${K[@]}"
    ptl_k3_probe ptl-probe.log long COLI_VK_DEVICE_CAP_MB=256 COLI_VK_TIER_RESERVE_GB=0.04 $O
    cap=$(PTL_PROBE_CAP_MB=256 ptl_calc cap kimi_k3 ptl-probe.log 2)
    ptl_dsk_san kimi_k3 "asan partial kimi_k3 a device for 2 layers" 2 COLI_VK_DEVICE_CAP_MB=$cap COLI_VK_TIER_RESERVE_GB=0.04 $O "${K[@]}"
    ptl_k3_probe ptl-probe.log long COLI_VK_STAGED=1 COLI_VK_STAGED_FAULT=submit:1000000 COLI_VK_CHAIN_LAYERS=6 $O
    n=$(ptl_calc fault kimi_k3 ptl-probe.log 3)
    ptl_dsk_san kimi_k3 "asan partial kimi_k3 an upload failing in layer 3" 3 COLI_VK_STAGED=1 COLI_VK_STAGED_FAULT=submit:$n COLI_VK_CHAIN_LAYERS=6 $O "${K[@]}"
    ptl_dsk_san kimi_k3 "asan partial kimi_k3 device only, 3 of 6 layers" 3 COLI_VK_CHAIN_LAYERS=3 COLI_VK_DENSE_HOST=0 $O "${K[@]}"
    ptl_dsk_san kimi_k3 "asan partial kimi_k3 the KV split" 3 COLI_VK_CHAIN_LAYERS=3 COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=2 $O "${K[@]}"
    # the device lost mid-decode: 2 frames before the end of the run above, the chain
    # layers' KDA state rebuilt on the CPU
    ptl_dsk_san kimi_k3 "asan partial kimi_k3 device lost, rebuilt" lost COLI_VK_CHAIN_LAYERS=3 COLI_VK_CHAIN_FAULT=$((frames - 2)) $O "${K[@]}"
    grep -q "rebuilding the state of [1-9]" san.log || { cat san.log; fail "asan partial kimi_k3 device lost: no state was rebuilt"; }
  }
  CHAIN_SERVE_TOL=2e-2 CHAIN_SERVE_EXPECT='kimi_k3 chain: 2 of 6 layers on the device' $PY tests/vulkan_chain_serve.py ./kimi_k3 kimi_k3_serve \
    K3_BITS=32 K3_MLA_BITS=32 K3_HEAD_BITS=32 K3_IDOT=0 K3_PREFIX_LOG=1 USAGE_SAVE=0 COLI_K3_CKPT=4 COLI_VK_CHAIN_LAYERS=2 > san.log 2>&1 ||
    { cat san.log; fail "asan partial kimi_k3 serve"; }
  if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "asan partial kimi_k3 serve: sanitizer diagnostic"; fi
  echo "OK asan partial kimi_k3 serve: $(tail -1 san.log | cut -c1-120)"
  rm -rf dsv41_vlong kimi_k3_serve
  rm -f san.log ptl-probe.log chain.usage k3c.usage chain-serve.usage
  unset OMP_NUM_THREADS
  make clean >/dev/null 2>&1 || true
}