Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
vulkan: inkling's and MiMo's head failing to go up keeps every layer on the device
JustVuggbec7030541e7
diff --git a/c/inkling_chain.h b/c/inkling_chain.hindex 103b0c55..b1c4a070 100644--- a/c/inkling_chain.h+++ b/c/inkling_chain.h@@ -389,7 +389,14 @@ static InkChain *inkc_setup(Model *m) {int nl = inkc_layers(m);if (fit && !g_inkc_placed) { vkc_fit_placed("inkling", &g_inkc_fit); g_inkc_placed = 1; } /* placed here, lm_head next */if (fit && vkc_fit_partial(&g_inkc_fit)) inkc_refuse_from(m, nl);- if (!what && nl == L && (!fit || g_inkc_fit.tail) && !(ch->t_lm = inkc_tensor(m->lm_head, D, c->unpad_vocab))) what = "lm_head";+ if (!what && nl == L && (!fit || g_inkc_fit.tail) && !(ch->t_lm = inkc_tensor(m->lm_head, D, c->unpad_vocab))) {+ if (!fit) what = "lm_head";+ else { /* the fit's tail did not go up: every layer here, the head on the CPU */+ g_inkc_fit.tail = 0;+ inkc_refuse_from(m, L);+ fprintf(stderr, "[VK] inkling chain: lm_head did not reach the device; it runs on the CPU, the layers here\n");+ }+ }if (what || !nl) {if (what) {int bf16 = m->L[0].q.h && !m->L[0].q.q4 && !ink_vk_fmt(&m->L[0].q);diff --git a/c/mimo_chain.h b/c/mimo_chain.hindex aab6a0f3..a02ba4ff 100644--- a/c/mimo_chain.h+++ b/c/mimo_chain.h@@ -150,6 +150,10 @@ static MimoChain *mc_setup(Model *m) {(c->moe[i] || (mc_ctensor(&l->gate) && mc_ctensor(&l->up) && mc_ctensor(&l->down)));}int full = nl == L && (!g_mc_fit.L || g_mc_fit.tail);+ if (ok && full && g_mc_fit.L && !mc_ctensor(&m->head)) { /* the fit's tail did not go up: the head on the CPU */+ full = 0; g_mc_fit.tail = 0;+ fprintf(stderr, "[VK] mimo chain: the head did not reach the device; it runs on the CPU, the layers here\n");+ }ok = ok && (!full || mc_ctensor(&m->head));if (!ok) { fprintf(stderr, "[VK] mimo chain: a dense matrix did not reach the device; per-matrix path\n"); return NULL; }size_t n = 0;diff --git a/c/tests/vulkan_partial_inkling-mimo.sh b/c/tests/vulkan_partial_inkling-mimo.shindex 72e87504..89d279d5 100644--- a/c/tests/vulkan_partial_inkling-mimo.sh+++ b/c/tests/vulkan_partial_inkling-mimo.sh@@ -94,16 +94,29 @@ ptl_cap() {ptl_check_placed "$eng" vk.log "$tag"echo " $tag: COLI_VK_DEVICE_CAP_MB=$cap: $(grep -a "^\[VK\] $eng chain: [0-9]* of [0-9]* layers on the device" vk.log | tail -1 | sed "s/^\[VK\] $eng chain: //")"}+# ptl_head_fault <engine> <probe log>: the COLI_VK_STAGED_FAULT=submit:n n of the head's+# upload, the first time the point is reached after the last layer's setup (the probe's+# placed line, as for ptl_calc fault; one weight block holds the fixture, so no zero fill)+ptl_head_fault() {+ sed -n "s/^\[VK\] $1 chain: [0-9]* of [0-9]* layers placed:.*reached \([0-9,]*\) times.*/\1/p" "$2" | tail -1 |+ awk -F, '{ print $NF + 1 }'+}# The reserve the capped runs keep for a chunk's scratch and the frames' staging (the tier# takes the rest of the device): 0.04 GiB.PTL_RESERVE=0.04# ptl_plan <engine> <tag> <fixture> <log> <env...>: coli plan's prediction (resource_plan.py,# vk_chain_fit) for the same device and settings, from the checkpoint's header and config-# alone: the engine's free bytes, per-layer bytes, fixed bytes, tail and N. The tiny-# inkling's config says model_type inkling_text, which the family registry does not-# take: the plan reads a copy that says inkling (the real checkpoint's), the tensors linked.+# alone: the engine's free bytes, per-layer bytes, fixed bytes and N. The tiny inkling's+# config says model_type inkling_text, which the family registry does not take: the plan+# reads a copy that says inkling (the real checkpoint's), the tensors linked. The plan+# counts a buffer's alignment as 256 bytes, Lavapipe's: a driver that aligns them wider+# (Dozen: 64 KiB) gives a tiny fixture's layers other bytes, so only Lavapipe compares.ptl_plan() {local eng=$1 tag=$2 fx=$3 log=$4; shift 4+ case "${VK_ICD_FILENAMES:-}" in+ *lvp_icd*) ;;+ *) echo " coli plan: not compared on this driver (its buffers' alignment is not the plan's 256 bytes)"; return 0 ;;+ esac$PY - "$eng" "$fx" "$log" "$@" <<'PY' || fail "$tag: coli plan predicts otherwise"import json, os, re, shutil, sys, tempfilesys.path.insert(0, ".")@@ -200,6 +213,17 @@ ptl_inkling() {env COLI_VK_DENSE_HOST=$h COLI_VK_STAGED=1 COLI_VK_STAGED_FAULT=submit:1000000 COLI_VK_CHAIN_LAYERS=8 COLI_USAGE=chain.usage \COLI_VK_TIER_SYNC=1 COLI_VULKAN=1 COLI_VK_CHAIN=1 SNAP=tiny_inkling ./inkling 8 0 $R > ptl-fprobe.log 2>&1 || truefor k in 0 3 7; do ink_fault "partial inkling upload failing in layer $k (COLI_VK_DENSE_HOST=$h)" $k ptl-fprobe.log COLI_VK_DENSE_HOST=$h; done+ if [ $h = 1 ]; then # lm_head's upload failing: every layer on the device, the head on the CPU+ f=$(ptl_head_fault inkling ptl-fprobe.log)+ rm -f chain.usage+ env COLI_VK_STAGED=1 COLI_VK_STAGED_FAULT=submit:$f COLI_VK_CHAIN_LAYERS=8 COLI_USAGE=chain.usage COLI_VK_TIER_SYNC=1 \+ COLI_VULKAN=1 COLI_VK_CHAIN=1 SNAP=tiny_inkling ./inkling 8 0 $R > vk.log 2>&1 || true+ grep -qa "lm_head did not reach the device; it runs on the CPU" vk.log || { cat vk.log; fail "partial inkling head fault: not the head's"; }+ same_tokens cpu.log vk.log "partial inkling head fault"+ ptl_check_n inkling vk.log 8 "partial inkling head fault"+ ptl_check_placed inkling vk.log "partial inkling head fault"+ echo "OK partial inkling lm_head's upload failing (submit #$f): tokens = CPU, every layer on the device, the head on the CPU"+ fiif [ $h = 0 ]; then # the dropped host copies are the 7 layers' only, none read back[ "$(dho_dropped vk.log)" = 56 ] || { grep -a '^\[VK\]' vk.log; fail "partial inkling fault, device only: $(dho_dropped vk.log) matrices dropped, not the 7 layers' 56"; }[ "$(dho_reloaded vk.log)" = 0 ] || { grep -a '^\[VK\]' vk.log; fail "partial inkling fault, device only: a matrix read back"; }@@ -340,6 +364,16 @@ ptl_mimo_all() {env COLI_VK_DENSE_HOST=$h COLI_VK_STAGED=1 COLI_VK_STAGED_FAULT=submit:1000000 COLI_VK_CHAIN_LAYERS=6 COLI_TEMP=0 COLI_VULKAN=1 \COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 MIMO_DENSE_BITS=32 ./mimo mimo_tiny --ids "$P" --ngen 6 > /dev/null 2> ptl-fprobe.logfor k in 0 2 5; do mimo_fault "partial mimo upload failing in layer $k (COLI_VK_DENSE_HOST=$h)" $k ptl-fprobe.log COLI_VK_DENSE_HOST=$h; done+ if [ $h = 1 ]; then # the head's upload failing: every layer on the device, the head on the CPU+ f=$(ptl_head_fault mimo ptl-fprobe.log)+ env COLI_VK_STAGED=1 COLI_VK_STAGED_FAULT=submit:$f COLI_VK_CHAIN_LAYERS=6 COLI_TEMP=0 COLI_VULKAN=1 COLI_VK_CHAIN=1 \+ COLI_VK_TIER_SYNC=1 MIMO_DENSE_BITS=32 ./mimo mimo_tiny --ids "$P" --ngen 6 > mimo-vk.txt 2> vk.log+ grep -qa "the head did not reach the device; it runs on the CPU" vk.log || { cat vk.log; fail "partial mimo head fault: not the head's"; }+ [ -s mimo-cpu.txt ] && cmp -s mimo-cpu.txt mimo-vk.txt || { cat mimo-cpu.txt mimo-vk.txt vk.log; fail "partial mimo head fault: tokens differ from the CPU"; }+ ptl_check_n mimo vk.log 6 "partial mimo head fault"+ ptl_check_placed mimo vk.log "partial mimo head fault"+ echo "OK partial mimo the head's upload failing (submit #$f): tokens = CPU, every layer on the device, the head on the CPU"+ fiif [ $h = 0 ]; then # the dropped host copies are the 5 layers' only (layer 0's five, two a layer after it)[ "$(dho_dropped vk.log)" = 13 ] || { grep -a '^\[VK\]' vk.log; fail "partial mimo fault, device only: $(dho_dropped vk.log) matrices dropped, not the 5 layers' 13"; }[ "$(dho_reloaded vk.log)" = 0 ] || { grep -a '^\[VK\]' vk.log; fail "partial mimo fault, device only: a matrix read back"; }