Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
tests: staged uploads failing at every point, and CI legs
JustVugg4ea9c267f728
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 5b35f57a..2325c3f3 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -189,6 +189,13 @@ jobs:req: c/tools/requirements-qwen38-tiny.txt- family: qwen-chain-stagedreq: c/tools/requirements-qwen38-tiny.txt+ # staged uploads failing at every point they can (COLI_VK_STAGED_FAULT): qwen36's+ # matrices, tier and chain, colibri's KV mirror; the CPU's tokens, and the same+ # under ASan and UBSan+ - family: staged-faults+ req: c/tools/requirements-qwen38-tiny.txt+ - family: staged-faults-sanitize+ req: c/tools/requirements-qwen38-tiny.txt- family: inkling-olmoereq: c/tools/oracle-requirements.txt# the routed-expert tier of inkling and olmoe under ASan and UBSandiff --git a/c/tests/vulkan_engines.sh b/c/tests/vulkan_engines.shindex f6777402..2c4d3888 100755--- a/c/tests/vulkan_engines.sh+++ b/c/tests/vulkan_engines.sh@@ -13,6 +13,7 @@# bash tests/vulkan_engines.sh staged # staged uploads: the same bits as mapped memory, the decision# bash tests/vulkan_engines.sh <family>-staged # a family with COLI_VK_STAGED=1 (qwen-staged: and# # an engine under an emulated small window)+# bash tests/vulkan_engines.sh staged-faults | staged-faults-sanitize # staged uploads failing## Needs libvulkan-dev, glslc and mesa-vulkan-drivers, plus the Python packages of# the family's tiny fixtures (see the vulkan-engines job in .github/workflows/ci.yml).@@ -123,6 +124,104 @@ staged_window() {echo "OK window qwen36: staged on its own, $(grep -o 'resident data in host memory: .*' vk.log)"}+# Staged uploads failing (COLI_VK_STAGED_FAULT=<point>[:n], the n-th time) at every point+# they can: the uploader's staging buffer (stage), the KV mirror's (pwstage), a weight+# pool's device-local block (block), a KV mirror or norm-weight buffer (kvbuf), a command+# buffer's begin or end (record), a submit, a fence wait (the device is then lost), a tier+# expert's commit after its first matrix (commit). qwen36 with its trunk's matrices on the+# device one by one, its tier awaited (COLI_VK_TIER_SYNC=1) and with the uploader thread+# free after a warm start, and its chain; colibri's attention core for the KV mirror. Every+# run: no crash and no sanitizer report, the fault fired, the CPU's tokens. A matrix that+# failed stays on the CPU (one fewer resident); an expert whose commit failed stays on the+# CPU and the tier's budget stands; a lost device takes everything to the CPU, the chain+# rebuilding its state there. SAN=1 (staged-faults-sanitize): a sanitized build.+fault_run() { # <tag> <fault> <cpu tokens> <env...> -- <command...>+ local tag=$1 fault=$2 ref=$3 rc=0; shift 3+ local envs=(); while [ "$1" != "--" ]; do envs+=("$1"); shift; done; shift+ if [ -f hist.usage ]; then cp hist.usage run.usage; else rm -f run.usage; fi+ env "${envs[@]}" COLI_USAGE=run.usage COLI_VULKAN=1 COLI_VK_STAGED=1 ${fault:+COLI_VK_STAGED_FAULT=$fault} "$@" > vk.log 2>&1 || rc=$?+ if [ $rc -ge 128 ] || grep -qE "ERROR: AddressSanitizer|runtime error:" vk.log; then cat vk.log; fail "$tag: crashed (exit $rc)"; fi+ [ -z "$fault" ] || grep -q "COLI_VK_STAGED_FAULT: ${fault%%:*} " vk.log || { cat vk.log; fail "$tag: the fault never fired"; }+ grep -a 'C engine' vk.log > vk.tok || true+ { [ -s "$ref" ] && cmp -s "$ref" vk.tok; } || { cat "$ref" vk.tok; fail "$tag: tokens differ from the CPU's"; }+}+fault_num() { # <sed expression> <log>: the first number it extracts, 0 without one+ local n; n=$(sed -n "$1" "$2" | tail -1); echo "${n:-0}"+}+family_staged_faults() {+ if [ "${SAN:-0}" = 1 ]; then+ make clean >/dev/null 2>&1 || true+ make qwen36 colibri VK=1 EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"+ export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1+ else+ make qwen36 colibri VK=1+ fi+ $PY tools/make_qwen36_tiny.py --out qwen36_tiny --ref-mode full --emit-ref qwen36_tiny/ref_full.json+ $PY tools/convert_qwen36.py --model qwen36_tiny --out qwen36_tiny_c --ebits 8+ mkdir -p glm_fp8 && (cd glm_fp8 && $PY ../tools/make_glm_oracle.py --fp8 > /dev/null)+ $PY tools/convert_fp8_to_int4.py --indir glm_fp8/glm_tiny --outdir glm_tiny_i4 --ebits 4 --io-bits 4 \+ --n-layers 5 --min-free-gb 0 > /dev/null+ cp glm_fp8/ref_glm.json glm_tiny_i4/+ local q=(COLI_DENSE_I8=0 SNAP=qwen36_tiny_c ./qwen36 8 8 qwen36_tiny/ref_full.json)+ local g=(IDOT=0 SNAP=glm_tiny_i4 REF=glm_tiny_i4/ref_glm.json ./colibri 2 4 4)+ env "${q[@]}" > cpu.log 2>&1 || true; grep -a 'C engine' cpu.log > q.tok || true+ env "${g[@]}" > cpu.log 2>&1 || true; grep -a 'C engine' cpu.log > g.tok || true+ local f n S+ rm -f hist.usage+ # the trunk's matrices one by one (no tier, no chain)+ local D=(COLI_VK_TIER=0 COLI_VK_CHAIN=0 COLI_VK_DENSE=1)+ fault_run "faults: qwen36 matrices, none" "" q.tok "${D[@]}" -- "${q[@]}"+ n=$(fault_num 's/.*matmuls on the GPU (\([0-9]*\) matrices.*/\1/p' vk.log)+ for f in stage:1 block:1 record:4 submit:4 wait:1 wait:4; do+ fault_run "faults: qwen36 matrices, $f" $f q.tok "${D[@]}" -- "${q[@]}"+ case $f in block:1|record:4|submit:4)+ [ "$(fault_num 's/.*matmuls on the GPU (\([0-9]*\) matrices.*/\1/p' vk.log)" = $((n - 1)) ] || {+ grep '\[VK\]' vk.log; fail "faults: qwen36 matrices, $f: not exactly one matrix left on the CPU"; } ;;+ wait:*) grep -q 'the device is lost' vk.log || { cat vk.log; fail "faults: qwen36 matrices, $f: the device was not lost"; } ;;+ esac+ echo "OK faults: qwen36 matrices, $f: tokens = CPU, $(grep -ao 'staged upload failed[^:]*: [^)]*): .*' vk.log | head -1 | sed 's/.*): //')"+ done+ # the tier awaited: the trunk on the CPU, every upload at the next step+ local T=(COLI_VK_CHAIN=0 COLI_VK_DENSE=0 COLI_VK_TIER_SYNC=1 COLI_VK_TIER_BALANCE=0)+ fault_run "faults: qwen36 tier, none" "" q.tok "${T[@]}" -- "${q[@]}"+ local budget; budget=$(fault_num 's/.*resident [0-9]* (budget \([0-9]*\),.*/\1/p' vk.log)+ cp run.usage hist.usage.next+ for f in stage:1 block:1 record:3 record:4 submit:2 wait:2 commit:1 commit:3; do+ fault_run "faults: qwen36 tier, $f" $f q.tok "${T[@]}" -- "${q[@]}"+ case $f in commit:*|record:*|submit:*)+ [ "$(fault_num 's/.*resident [0-9]* (budget \([0-9]*\),.*/\1/p' vk.log)" = "$budget" ] || {+ grep '\[VK\] tier' vk.log; fail "faults: qwen36 tier, $f: the budget shrank"; } ;;+ esac+ echo "OK faults: qwen36 tier, $f: tokens = CPU, $(grep -ao 'resident [0-9]* (budget [0-9]*' vk.log | tail -1), $(grep -ao 'failed [0-9]* |' vk.log | tail -1 | tr -d '|')"+ done+ # the tier's uploader thread free: a warm start from a history (vkt_put, any thread),+ # then promotions as experts pass+ mv hist.usage.next hist.usage+ T=(COLI_VK_CHAIN=0 COLI_VK_DENSE=0)+ for f in stage:1 block:1 record:3 submit:2 wait:3 commit:1 commit:5; do+ fault_run "faults: qwen36 tier, uploader thread, $f" $f q.tok "${T[@]}" -- "${q[@]}"+ echo "OK faults: qwen36 tier, uploader thread, $f: tokens = CPU, $(grep -ao 'resident [0-9]* (budget [0-9]*' vk.log | tail -1)"+ done+ rm -f hist.usage+ # the chain: a failure before it starts declines it (or leaves one matrix behind), a+ # lost device in the middle has the CPU rebuild the state+ local C=(COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1)+ fault_run "faults: qwen36 chain, none" "" q.tok "${C[@]}" -- "${q[@]}"+ S=$(fault_num 's/.*copies (\([0-9]*\) submits).*/\1/p' vk.log)+ for f in stage:1 block:1 record:4 submit:4 commit:1 wait:4 wait:$((S - 2)); do+ fault_run "faults: qwen36 chain, $f" $f q.tok "${C[@]}" -- "${q[@]}"+ [ $f != wait:$((S - 2)) ] || grep -q 'rebuilding the state of [1-9]' vk.log || { cat vk.log; fail "faults: qwen36 chain, $f: no state rebuilt on the CPU"; }+ echo "OK faults: qwen36 chain, $f: tokens = CPU, $(grep -ao 'qwen36 chain: [0-9]* forwards\|rebuilding the state of [0-9]* positions' vk.log | tr '\n' ' ')"+ done+ # colibri's attention core: the KV mirror's buffers and its staging+ local A=(COLI_VK_DENSE=1 COLI_VK_ATTN=1 COLI_VK_TIER_SYNC=1)+ for f in pwstage:1 kvbuf:1 kvbuf:3 wait:3; do+ fault_run "faults: colibri attention, $f" $f g.tok "${A[@]}" -- "${g[@]}"+ echo "OK faults: colibri attention, $f: tokens = CPU"+ done+ if [ "${SAN:-0}" = 1 ]; then make clean >/dev/null 2>&1 || true; fi+}+# tier_count <engine> <log>: N from the last "[VK] tier <engine> run: device N of M" line;# tier_evictions <engine> <log>: the evictions of that line; tier_failed: its failed# uploads (an eviction case must have none: a promotion that displaces a resident@@ -1968,5 +2067,7 @@ case "${1:-}" inif [ "$1" = qwen-staged ]; then env -u COLI_VK_STAGED bash tests/vulkan_engines.sh staged-window; fiCOLI_VK_STAGED=1 bash tests/vulkan_engines.sh "${1%-staged}" ;;staged-window) staged_window ;;+ staged-faults) family_staged_faults ;;+ staged-faults-sanitize) SAN=1 family_staged_faults ;;*) echo "usage: $0 staged|<family>-staged|shader|qwen|qwen-sanitize|inkling-olmoe|inkling-olmoe-sanitize|mimo-qwenimage|deepseek|deepseek-sanitize|kimi|kimi-mimo-sanitize|glm|glm-sanitize|qwen-chain|qwen-chain-sanitize|mimo-chain|mimo-chain-sanitize|inkling-olmoe-chain|inkling-olmoe-chain-sanitize|glm-chain|glm-chain-sanitize" >&2; exit 2 ;;esacdiff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.mdindex e0bd052f..eb402aec 100644--- a/docs/ENVIRONMENT.md+++ b/docs/ENVIRONMENT.md@@ -229,6 +229,7 @@ Per-drive byte counts are reported in a `MIRROR:` stats line. Combine with `DIRE| `COLI_VK_GEMM_TILE` | measured | `bm,bn,bk,tm,tn[,pf]`: one tile for every width of the fp32 GEMM instead of the measured pair. Tuning only (`COLI_VK_TEST_GEMM_BENCH`). || `COLI_VK_COOP_TILE` | measured | `bm,bn,wm,wn,bk`: one tile for every width of the cooperative-matrix GEMM. Tuning only. || `COLI_VK_STAGED` | auto | Staged uploads: resident data (the weights, the expert tier, the MLA KV mirror, `COLI_VK_DEV2`'s experts) copied into device-local memory the host does not map, through a host staging buffer, instead of written through a mapping of the host-visible device-local memory. `1` on, `0` off. Unset: on when the host-visible device-local heap is under a quarter of the largest device-local heap, as on a discrete card without Resizable BAR (about 256 MB of 8 GB, where the mapped path fails or spills to system RAM); off on a card with Resizable BAR, an integrated GPU or Lavapipe. See [vulkan.md](vulkan.md#memory-placement-without-resizable-bar). |+| `COLI_VK_STAGED_FAULT` | unset | Tests: `<point>[:n]` makes the n-th staged upload (the first by default) fail at `stage` (the staging buffer), `pwstage` (the KV mirror's), `block` (a device-local block), `kvbuf` (a KV mirror or norm-weight buffer), `record` (a command buffer's begin or end), `submit`, `wait` (a fence wait: the device is then lost) or `commit` (a tier expert's commit after its first matrix); a `[VK] COLI_VK_STAGED_FAULT` line says when it fired. || `COLI_VK_HOST_VISIBLE_CAP_MB` | unset | Tests: treat the host-visible device-local heap as at most this many MiB in the `COLI_VK_STAGED` decision, so a device with Resizable BAR or unified memory takes the decision a card without it would (`246` emulates an RTX 3070 on its launch VBIOS). |### The routed-expert tier (`vk_tier.c`, every MoE engine)diff --git a/docs/vulkan.md b/docs/vulkan.mdindex a282ff18..7087f53c 100644--- a/docs/vulkan.md+++ b/docs/vulkan.md@@ -1249,6 +1249,21 @@ differs from the main or the tier queue's, the staged tensors' buffers are creatmain queue in the same command buffer as their reader; a row written again before that(a rewound cache) first sends what is pending, so no two pending copies overlap.+**When an upload fails.** A staging buffer that cannot be had at startup leaves the+mapped path on (with a line saying so). A device-local block or buffer the driver refuses+is out of memory: that matrix stays on the CPU, the tier takes fewer experts, the KV+mirror's layer runs its attention on the CPU. A command buffer that would not record or a+submit refused fails that upload only: the uploader waits for what it had sent, frees the+tensors and starts the next upload clean; the matrix stays on the CPU, and an expert whose+commit failed stays on the CPU with the tier's budget intact (it may be promoted again). A+fence wait that fails means the copy may still run: the device is taken as lost, as for+every other wait, so the dense matrices, the tier (its batches stop) and the chain (at its+next frame, rebuilding the state on the CPU) all move to the CPU. `COLI_VK_STAGED_FAULT`+injects each of these, and `tests/vulkan_engines.sh staged-faults` (with+`staged-faults-sanitize` under ASan and UBSan) runs qwen36's matrices, its tier awaited and+with the uploader thread free, its chain, and colibri's KV mirror through every point,+gated on the CPU's tokens.+**A fresh device-local block is zero-filled** (`vkCmdFillBuffer`) before its firsttensor. On an RX 580 (RADV, Polaris) the author of #1338, where this approach comes from,measured results that differed slightly from run to run when read from a block the GPU had