Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

vk: layers on two devices: CI legs, sanitizer runs, docs

JustVugg3b5f3d0a50a8
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 3be0a443..9d12a78d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -386,6 +386,28 @@ jobs:
- family: kv-split-deepseek-sanitize
req: c/tools/requirements-deepseek-v4-tiny.txt
extra: sympy pillow
+ # the dense chain's layers on two devices (Lavapipe opened twice, COLI_VK_DEV2=0):
+ # every chain engine against its CPU run, split at every kind of layer, the state
+ # crossing between the devices, the second device lost; the same under ASan and
+ # UBSan; GLM and Kimi K3, and DeepSeek, in legs of their own
+ - family: layers-dev2
+ req: c/tools/requirements-qwen38-tiny.txt
+ extra: numpy pillow
+ - family: layers-dev2-sanitize
+ req: c/tools/requirements-qwen38-tiny.txt
+ extra: numpy pillow
+ - family: layers-dev2-mla
+ req: c/tools/requirements-qwen38-tiny.txt
+ extra: numpy pillow
+ - family: layers-dev2-mla-sanitize
+ req: c/tools/requirements-qwen38-tiny.txt
+ extra: numpy pillow
+ - family: layers-dev2-deepseek
+ req: c/tools/requirements-deepseek-v4-tiny.txt
+ extra: sympy pillow
+ - family: layers-dev2-deepseek-sanitize
+ req: c/tools/requirements-deepseek-v4-tiny.txt
+ extra: sympy pillow
# the decision engines: Laya's and GLiNER2.5-Decide's forward on the device and
# matrix by matrix, Clef's DECIDE on qwen36's paths in f32, int8 and f16, the
# tiny oracles, the serve tests and TypeSafe's SDKs through the device; and the
diff --git a/c/tests/vulkan_engines.sh b/c/tests/vulkan_engines.sh
index c67087cf..546777ce 100755
--- a/c/tests/vulkan_engines.sh
+++ b/c/tests/vulkan_engines.sh
@@ -4198,6 +4198,10 @@ case "${1:-}" in
kv-split-sanitize) family_kv_split_sanitize ;;
layers-dev2) family_layers_dev2 ;;
layers-dev2-sanitize) family_layers_dev2_sanitize ;;
+ layers-dev2-mla) family_layers_dev2_mla ;;
+ layers-dev2-mla-sanitize) family_layers_dev2_mla_sanitize ;;
+ layers-dev2-deepseek) family_layers_dev2_deepseek ;;
+ layers-dev2-deepseek-sanitize) family_layers_dev2_deepseek_sanitize ;;
layers-dev2-*) e=${1#layers-dev2-} # one engine's gates (the engine already built)
declare -F "ld2_$e" >/dev/null || { echo "no layers-dev2 engine $e" >&2; exit 2; }
OMP_NUM_THREADS=2 "ld2_$e" ;;
@@ -4221,5 +4225,5 @@ case "${1:-}" in
partial-*) g=${1#partial-}; fn=ptl_family_${g//-/_}
declare -F "$fn" >/dev/null || { echo "no partial-chain group ${g}" >&2; exit 2; }
"$fn" ;;
- *) echo "usage: $0 decide|decide-sanitize|staged|<family>-staged|shader|qwen|qwen-sanitize|inkling-olmoe|inkling-olmoe-sanitize|mimo-qwenimage|deepseek|deepseek-sanitize|kimi|kimi-mimo-sanitize|glm|glm-sanitize|qwen-chain|qwen-chain-sanitize|qwen-spec|qwen-spec-sanitize|mimo-chain|mimo-chain-sanitize|inkling-olmoe-chain|inkling-olmoe-chain-sanitize|glm-chain|glm-chain-sanitize|kimi-chain|kimi-chain-sanitize|deepseek-chain|deepseek-chain-sanitize|dense-only-<group>[-sanitize]|partial-<group>[-sanitize]|prefill-qwen|prefill-qwen-sanitize|prefill-inkling-olmoe|prefill-mimo-kimi|prefill-glm|prefill-deepseek|kv-split|kv-split-sanitize|layers-dev2|layers-dev2-sanitize|kv-split-deepseek|kv-split-deepseek-sanitize|dev2|dev2-deepseek-kimi-mimo|dev2-sanitize" >&2; exit 2 ;;
+ *) echo "usage: $0 decide|decide-sanitize|staged|<family>-staged|shader|qwen|qwen-sanitize|inkling-olmoe|inkling-olmoe-sanitize|mimo-qwenimage|deepseek|deepseek-sanitize|kimi|kimi-mimo-sanitize|glm|glm-sanitize|qwen-chain|qwen-chain-sanitize|qwen-spec|qwen-spec-sanitize|mimo-chain|mimo-chain-sanitize|inkling-olmoe-chain|inkling-olmoe-chain-sanitize|glm-chain|glm-chain-sanitize|kimi-chain|kimi-chain-sanitize|deepseek-chain|deepseek-chain-sanitize|dense-only-<group>[-sanitize]|partial-<group>[-sanitize]|prefill-qwen|prefill-qwen-sanitize|prefill-inkling-olmoe|prefill-mimo-kimi|prefill-glm|prefill-deepseek|kv-split|kv-split-sanitize|layers-dev2[-mla|-deepseek][-sanitize]|layers-dev2-<engine>|kv-split-deepseek|kv-split-deepseek-sanitize|dev2|dev2-deepseek-kimi-mimo|dev2-sanitize" >&2; exit 2 ;;
esac
diff --git a/c/tests/vulkan_layers_dev2.sh b/c/tests/vulkan_layers_dev2.sh
index c0038204..cd3b3015 100644
--- a/c/tests/vulkan_layers_dev2.sh
+++ b/c/tests/vulkan_layers_dev2.sh
@@ -3,8 +3,11 @@
# on the same physical one, the test mode); COLI_VK_CHAIN_LAYERS puts the first layers on
# the primary device, COLI_VK_CHAIN_LAYERS2 the next ones on the second, the CPU the rest.
#
-# layers-dev2 every chain engine against its own CPU run
-# layers-dev2-sanitize the same paths under ASan and UBSan
+# layers-dev2 qwen36, qwen38, olmoe, mimo, inkling against their CPU runs
+# layers-dev2-mla GLM-5.2 (colibri), GLM-5.3, Kimi K3
+# layers-dev2-deepseek DeepSeek V4.1 and V4
+# layers-dev2[-mla|-deepseek]-sanitize their paths under ASan and UBSan
+# layers-dev2-<engine> one engine's gates (the engine built)
#
# Each gate (ld2_gate; ld2_mla for the MLA engines) is chain_gate's or mla_gate's (the
# CPU's tokens, logits within 1e-4 of the largest) with the split forced, and the second
@@ -522,8 +525,7 @@ ld2_deepseek_v4() {
family_layers_dev2() {
export OMP_NUM_THREADS=2
- make qwen36 qwen38 olmoe mimo inkling colibri glm53 kimi_k3 deepseek_v41 tests/test_vk_chain VK=1
- make deepseek-v4 VK=1
+ make qwen36 qwen38 olmoe mimo inkling tests/test_vk_chain VK=1
COLI_VK_DEV2=0 ./tests/test_vk_chain shaders/qmatmul.spv | tee vk_chain.log
tail -1 vk_chain.log | grep -qx PASS || fail "the chain's ops on two devices"
ld2_qwen36
@@ -531,9 +533,20 @@ family_layers_dev2() {
ld2_olmoe
ld2_mimo
ld2_inkling
+ unset OMP_NUM_THREADS
+}
+family_layers_dev2_mla() {
+ export OMP_NUM_THREADS=2
+ make colibri glm53 kimi_k3 VK=1
ld2_colibri
ld2_glm53
ld2_kimi_k3
+ unset OMP_NUM_THREADS CAP_RAISE
+}
+family_layers_dev2_deepseek() {
+ export OMP_NUM_THREADS=2
+ make deepseek_v41 VK=1
+ make deepseek-v4 VK=1
ld2_deepseek_v41
ld2_deepseek_v4
unset OMP_NUM_THREADS
@@ -560,3 +573,78 @@ family_layers_dev2_sanitize() {
[ "$(ld2_forwards inkling san.log)" -gt 0 ] || { cat san.log; fail "asan ld2 inkling: the second device's chain never ran"; }
make clean >/dev/null 2>&1 || true
}
+# ld2_san <engine> <tag> <env and argv...>: a sanitized build's run with the split: no
+# sanitizer diagnostic, and the second device's chain ran (LD2_SAN_LOST=1: its loss was
+# taken over)
+ld2_san() {
+ local eng=$1 tag=$2; shift 2
+ rm -f chain.usage
+ env OMP_NUM_THREADS=2 COLI_USAGE=chain.usage COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_DEV2=0 COLI_VK_TIER_BALANCE=0 \
+ "$@" > san.log 2>&1 || true
+ rm -f chain.usage
+ if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "$tag: sanitizer diagnostic"; fi
+ if [ "${LD2_SAN_LOST:-0}" = 1 ]; then
+ grep -q "COLI_VK_CHAIN_FAULT2" san.log && grep -q "the device was lost" san.log || { cat san.log; fail "$tag: the loss was not taken over"; }
+ echo "OK $tag: sanitizers clean, the second device's loss taken over"
+ else
+ [ "$(ld2_forwards "$eng" san.log)" -gt 0 ] || { cat san.log; fail "$tag: the second device's chain never ran"; }
+ echo "OK $tag: sanitizers clean, the second device's chain ran $(ld2_forwards "$eng" san.log) forwards"
+ fi
+}
+family_layers_dev2_mla_sanitize() {
+ local SAN="-fsanitize=address,undefined -fno-omit-frame-pointer -g" ids
+ make clean >/dev/null 2>&1 || true
+ make colibri glm53 kimi_k3 VK=1 EXTRA_CFLAGS="$SAN"
+ export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1 CAP_RAISE=0
+ [ -f glm_tiny_serve/tokenizer.json ] || glm_chain_fixtures
+ [ -d glm_tiny_shx ] || ptl_glm_shx glm_tiny glm_tiny_shx
+ [ -d glm53_l6s-i4 ] || { ptl_g53_l6 glm53_l6; $PY tools/make_glm53_streaming_pair.py --fixture glm53_l6 --output glm53_l6s > /dev/null; }
+ $PY tools/make_kimi_k3_tiny.py --output ./kimi_k3_tiny --force
+ ids=$($PY -c "print(','.join(str((i*37+5)%120+2) for i in range(100)))")
+ local G=(SNAP=glm_tiny REF=ref_glm.json USAGE_SAVE=0) K=(COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=4)
+ ld2_san colibri "asan ld2 colibri KV split on both, DSA top-4" COLI_VK_CHAIN_LAYERS=2 COLI_VK_CHAIN_LAYERS2=3 "${K[@]}" "${G[@]}" DSA_TOPK=4 \
+ ./colibri 64 16 16
+ ld2_san colibri "asan ld2 colibri a shared indexer first on the second device, chunks of 5" COLI_VK_CHAIN_LAYERS=1 COLI_VK_CHAIN_LAYERS2=2 \
+ SNAP=glm_tiny_shx REF=ref_glm.json USAGE_SAVE=0 DSA_TOPK=4 TF=1 COLI_VK_CHAIN_ROWS=5 ./colibri 64 16 16
+ LD2_SAN_LOST=1 ld2_san colibri "asan ld2 colibri second device lost mid-decode" COLI_VK_CHAIN_LAYERS=2 COLI_VK_CHAIN_LAYERS2=2 \
+ COLI_VK_CHAIN_FAULT2=10 "${G[@]}" ./colibri 64 16 16
+ ld2_san glm53 "asan ld2 glm53 KV split on both" COLI_VK_CHAIN_LAYERS=2 COLI_VK_CHAIN_LAYERS2=2 COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 \
+ GLM53_BITS=32 USAGE_SAVE=0 ./glm53 --model glm53_l6s-i4 --ids $ids --greedy 8
+ LD2_SAN_LOST=1 ld2_san glm53 "asan ld2 glm53 second device lost, the KDA state rebuilt" COLI_VK_CHAIN_LAYERS=3 COLI_VK_CHAIN_LAYERS2=3 \
+ COLI_VK_CHAIN_FAULT2=40 GLM53_BITS=32 USAGE_SAVE=0 ./glm53 --model glm53_l6s-i4 --ids $ids --greedy 6
+ local O=(K3_BITS=32 K3_MLA_BITS=32 K3_HEAD_BITS=32 K3_IDOT=0 COLI_TEMP=0 USAGE_SAVE=0)
+ ld2_san kimi_k3 "asan ld2 kimi_k3 KV split on both" COLI_VK_CHAIN_LAYERS=3 COLI_VK_CHAIN_LAYERS2=3 COLI_VK_KV_DEVICE_ROWS=16 \
+ COLI_VK_KV_BLOCK=4 "${O[@]}" ./kimi_k3 kimi_k3_tiny --ids "$(k3c_ids long)" --ngen 8
+ LD2_SAN_LOST=1 ld2_san kimi_k3 "asan ld2 kimi_k3 second device lost in a prompt forward" COLI_VK_CHAIN_LAYERS=2 COLI_VK_CHAIN_LAYERS2=4 \
+ COLI_VK_CHAIN_FAULT2=15 K3_CHUNK=32 "${O[@]}" ./kimi_k3 kimi_k3_tiny --ids "$(k3c_ids long)" --ngen 8
+ unset CAP_RAISE
+ make clean >/dev/null 2>&1 || true
+}
+family_layers_dev2_deepseek_sanitize() {
+ local SAN="-fsanitize=address,undefined -fno-omit-frame-pointer -g" p
+ make clean >/dev/null 2>&1 || true
+ make deepseek_v41 VK=1 EXTRA_CFLAGS="$SAN"
+ export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1
+ v41_chain_fixtures
+ ld2_san deepseek_v41 "asan ld2 deepseek_v41 DSpark spec=5" COLI_VK_CHAIN_LAYERS=1 COLI_VK_CHAIN_LAYERS2=5 SNAP=dsv41_tiny V41_DSPARK=1 \
+ V41_SPEC_FORCE=5 ./deepseek_v41 8 dsv41_tiny/ref.json
+ ld2_san deepseek_v41 "asan ld2 deepseek_v41 KV split on both, chunks of 3" COLI_VK_CHAIN_LAYERS=3 COLI_VK_CHAIN_LAYERS2=3 \
+ COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=2 COLI_VK_CHAIN_ROWS=3 SNAP=dsv41_long ./deepseek_v41 8 dsv41_long/ref.json
+ LD2_SAN_LOST=1 ld2_san deepseek_v41 "asan ld2 deepseek_v41 second device lost" COLI_VK_CHAIN_LAYERS=1 COLI_VK_CHAIN_LAYERS2=5 \
+ COLI_VK_CHAIN_FAULT2=40 SNAP=dsv41_long ./deepseek_v41 8 dsv41_long/ref.json
+ make clean >/dev/null 2>&1 || true
+ # the V4 sanitized build as its chain family makes it (no LTO, the sanitizers at the link)
+ make deepseek-v4-clean >/dev/null 2>&1 || true
+ make deepseek-v4 VK=1 LTO=0 EXTRA_CFLAGS="$SAN" EXTRA_LDFLAGS="$SAN"
+ v4_chain_fixtures
+ p=$(v4_chain_prompt deepseek_v4_tiny_t long)
+ ld2_san deepseek_v4 "asan ld2 deepseek_v4 KV split on both" COLI_VK_CHAIN_LAYERS=2 COLI_VK_CHAIN_LAYERS2=1 COLI_VK_KV_DEVICE_ROWS=8 \
+ COLI_VK_KV_BLOCK=2 ./deepseek_v4 ./deepseek_v4_tiny_t "$p" --raw-prompt --max-tokens 4
+ ld2_san deepseek_v4 "asan ld2 deepseek_v4 drafts, a window of 4" COLI_VK_CHAIN_LAYERS=1 COLI_VK_CHAIN_LAYERS2=2 V4_DRAFT=4 \
+ V4_NGRAM_PARTIAL_KEEP=1 ./deepseek_v4 ./deepseek_v4_tiny_w4 \
+ "$(v4_chain_prompt x ids:30,31,32,33,34,35,60,61,30,31,32,33,34,35,36,37,38,39,40,41,30,31,32)" --raw-prompt --max-tokens 12
+ LD2_SAN_LOST=1 ld2_san deepseek_v4 "asan ld2 deepseek_v4 second device lost" COLI_VK_CHAIN_LAYERS=1 COLI_VK_CHAIN_LAYERS2=2 \
+ COLI_VK_CHAIN_FAULT2=8 ./deepseek_v4 ./deepseek_v4_tiny_t "$p" --raw-prompt --max-tokens 4
+ rm -rf deepseek_v4_tiny_t deepseek_v4_tiny_e8 deepseek_v4_tiny_g2 deepseek_v4_tiny_w4 deepseek_v4_tiny_ix san.log
+ make deepseek-v4-clean >/dev/null 2>&1 || true
+}
diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md
index c662c850..9ed508b6 100644
--- a/docs/ENVIRONMENT.md
+++ b/docs/ENVIRONMENT.md
@@ -262,6 +262,9 @@ With `COLI_VULKAN=1` every MoE engine (qwen36, qwen38, inkling, olmoe, kimi_k3,
| `COLI_VK_CHAIN_ROWS` | from the budget, up to 8192 | Prompt rows per chunk of the chain's prefill. Unset or `auto`: the most rows whose scratch fits half the device memory free at the engine's first forward (also bounded by available physical RAM and Windows commit headroom), up to `COLI_VK_CHAIN_ROWS_MAX`; MiMo, GLM-5.3, Kimi K3 and DeepSeek V4 then hand the chain blocks of that chunk instead of `MIMO_CHUNK`, `GLM53_PREFILL_CHUNK`, `K3_CHUNK` and the 128-row block, when those are unset. Set: that many rows, the engines' own blocks as before. See [vulkan.md](vulkan.md#big-prompt-chunks-and-expert-streaming). |
| `COLI_VK_CHAIN_ROWS_MAX` | `8192` | The most rows the budget's chunk takes. |
| `COLI_VK_CHAIN_LAYERS` | from the device's free memory | How many layers the chain places on the device, the first N of them; the CPU runs the rest and the head (the partial chain, [vulkan.md](vulkan.md#a-partial-chain)). Unset or `auto`: the most layers from layer 0 that fit the device's free memory less `COLI_VK_TIER_RESERVE_GB`, the chain's fixed buffers and one prompt chunk's scratch; every layer (the chain as before) when they all fit. `n`: n layers, at most every one; `0`: the chain off, nothing of the dense part on the device. The `[VK] <engine> chain: N of L layers on the device ...` line says what was decided and why (`COLI_VK_CHAIN_LAYERS=n` when forced); `[VK] <engine> chain fit:` gives the numbers. With `COLI_VK_DENSE_HOST=0` (or its rule) only the N layers give back their host copies, and the RAM plan and `coli plan` count only those. Engines not yet on the partial chain keep every layer. |
+| `COLI_VK_CHAIN_LAYERS2` | from the second device's free memory | With `COLI_VK_DEV2` and a partial chain: how many of the layers after the primary's the second device's chain places ([vulkan.md](vulkan.md#layers-on-two-devices)). Unset or `auto`: the most that fit that device's free memory less `COLI_VK_TIER_RESERVE_GB` and the chain's fixed buffers; `n`: n layers. The head stays on the host. The `[VK] <engine> dev2 chain: n of m layers on the device ...` line says what was decided. DeepSeek V4.1 starts the second device only at a layer that reads nothing the layers before it make (the primary's fit comes down to one). |
+| `COLI_VK_CHAIN_DEV2` | `1` | `0` keeps the dense chain's layers off the second device (`COLI_VK_DEV2` then holds experts only). |
+| `COLI_VK_CHAIN_FAULT2` | unset | Tests: the n-th frame on the second device's chain fails as a lost device's would (frame 1 is its setup: its layers stay on the CPU). Both chains go off; the CPU runs from there, the recurrent state rebuilt where a device held it. |
| `COLI_VK_DEVICE_CAP_MB` | unset | Tests: the device holds at most this many MiB of device-local memory (a fraction is taken). An allocation past it fails as out of device memory, and every budget the engines read (`VK_EXT_memory_budget`'s, the expert tier's, the chunk's, the partial chain's fit) reports the cap and this process's bytes, so Lavapipe behaves like a small card. The second device (`COLI_VK_DEV2`) gets the same cap. A `[VK] COLI_VK_DEVICE_CAP_MB=...` line says it is on. |
| `COLI_VK_ATTN_BLOCK` | `16` | Rows from which the chain's attention takes the blocked shader (`chain_attnb.comp`: K and V read once per block of rows); `0` never. Below it, and at decode, `chain_attn.comp` as before. |
| `COLI_VK_ATTN_SLICE` | `4294967296` | Rows x positions x heads x head dim past which the chain's attention ops are recorded in slices of rows, a submission each, so no submission of a big prompt chunk runs past a driver's job timeout; `0` never. |
diff --git a/docs/vulkan.md b/docs/vulkan.md
index e204ea5a..d48432dd 100644
--- a/docs/vulkan.md
+++ b/docs/vulkan.md
@@ -634,6 +634,8 @@ one of each.
### A second device (`COLI_VK_DEV2`)
+The dense chain can put layers there too: [Layers on two devices](#layers-on-two-devices).
+
A machine with two GPUs (a V100 beside a GTX 1070, an RX 9070 beside an RX 580) can
give the tier the memory of both. `COLI_VK_DEV2=auto` takes the best GPU that is not the
primary device (a discrete card before an integrated one); `COLI_VK_DEV2=<index>` takes
@@ -2748,6 +2750,70 @@ KDA state rebuilt from the prefix record), serve sessions with prefix reuse, Kim
recurrent-state photos and V4.1's images with N < L; the dense weights on the device only
with N < L.
+### Layers on two devices
+
+With a second GPU (`COLI_VK_DEV2`) the layers the primary device leaves do not have to go
+to the CPU: a second chain takes them on that device. The primary keeps its first N
+layers (its fit, as above); the second device's chain takes the layers from N on with a
+fit of its own over that device's free memory (`COLI_VK_CHAIN_LAYERS2` forces how many);
+the CPU runs what is left, and the head stays on the host. Every chain engine does it:
+qwen36, qwen38, OLMoE, MiMo, Inkling, GLM-5.2 (colibri), GLM-5.3, Kimi K3, DeepSeek V4.1
+and DeepSeek V4. It needs a partial chain on the primary: when every layer fits there,
+the second device holds experts only, as before.
+
+A forward runs every row through the primary's layers, brings them back, and runs them
+through the second device's layers from there: what crosses is what the CPU's next layer
+would have read. That is the residual (the hc_mult streams on the mHC engines; Kimi K3's
+AttnRes prefix, block snapshots and their count), and on GLM-5.2 the DSA selection of the
+primary's last full layer when the second device's first layer shares its indexer. The
+second device's matrices are copies of its own, which the per-matrix path never reads.
+Each device's KV mirror, KV split and watermarks are its own.
+
+Recurrent state (qwen36's and qwen38's DeltaNet, Inkling's convolution rings, GLM-5.3's
+and Kimi K3's KDA) stays where its layers run. The host's copy is made current for both
+devices or for neither, the second device's read first, so a read that fails leaves the
+primary's as it was.
+
+DeepSeek V4.1's second device starts only at a layer that reads nothing the layers
+before it make in a forward. Its first compressed layer owns its compressed rows and runs
+its own indexer, and no candidate mask crosses. On V4.1 Flash those are layers 2, 8, 14
+and 20. The fit comes down to the last such layer, with a line that says so; a forced
+`COLI_VK_CHAIN_LAYERS` that is not one stays, and the second device stays off.
+
+The lines (Kimi K3's six-layer fixture, Lavapipe opened twice: `COLI_VK_DEV2=0`):
+
+```
+[VK] kimi_k3 chain: 2 of 6 layers on the device (0.5 MiB), 4 on the CPU, the head and what goes with it (0.2 MiB) on the CPU (COLI_VK_CHAIN_LAYERS=2)
+[VK] kimi_k3 dev2 chain fit: free 23872688128 B, reserve 1073741824 B, fixed 489255168 B (...), tail 0 B, layers 190280 304512 304512 304512 B, ...
+[VK] kimi_k3 dev2 chain: 4 of 4 layers on the device (1.1 MiB), 0 on the CPU (COLI_VK_CHAIN_LAYERS2=4)
+[VK] kimi_k3 chain: layers 2..5 on the second device (1 KDA, 3 MLA, 0 dense MLP), 3 AttnRes blocks, ...
+[VK] kimi_k3 dev2 chain: 8 forwards, 41 frames (...), ...
+```
+
+**A lost second device.** Losing either device turns both chains off. An engine whose
+state is the host's (OLMoE, MiMo, GLM-5.2, DeepSeek V4.1 and V4) has the CPU run the
+forward again, or run the second device's layers from the primary's output. An engine with
+recurrent state rebuilds the state its devices held from its record, as on one device.
+`COLI_VK_CHAIN_FAULT2=n` fakes the loss at the second device's n-th frame; frame 1 is its
+setup, whose layers then stay on the CPU from the start.
+
+`COLI_VK_CHAIN_DEV2=0` keeps the layers off the second device (its experts stay there).
+
+Tested on Lavapipe only, opened twice: every split of every engine's fixtures against its
+own CPU run. The families are `layers-dev2` (qwen36, qwen38, OLMoE, MiMo, Inkling),
+`layers-dev2-mla` (colibri, GLM-5.3, Kimi K3) and `layers-dev2-deepseek`, each also under
+ASan and UBSan. They cover:
+
+- the tokens, and every logits row within each engine's chain tolerance;
+- prompt chunks, drafts and MTP accepted and rejected;
+- prompts only, the KV split on both devices, experts on both devices;
+- the second device lost at its setup, in a prompt and mid-decode;
+- serve sessions with pins and the prompt cache.
+
+Two real GPUs have not been measured: none is available here. So whether a forward is
+faster than the same layers on one device and the CPU depends on the card and the link,
+and is not verified. The rows cross through host memory once per forward, chunk by chunk.
+
## Correctness
- `gcc -O3 -DVK_TEST backend_vulkan.c -o test_vk -lvulkan -lm && ./test_vk