Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
ci: several conversations at once on every engine (mux families)
JustVuggf51380b9258b
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 70076767..266c8937 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -426,6 +426,22 @@ jobs:- family: decide-sanitizereq: c/tools/requirements-jev-sdk.txtextra: -r c/tools/oracle-requirements.txt+ # several conversations at once (KV_SLOTS, tests/serve_mux_check.py): each engine's+ # requests decoded together, frame for frame against the same requests served+ # alone, on the CPU and with the expert tier; the same under ASan and UBSan;+ # DeepSeek's in legs of their own (their fixtures pin another transformers)+ - family: mux+ req: c/tools/requirements-qwen38-tiny.txt+ extra: numpy pillow+ - family: mux-sanitize+ req: c/tools/requirements-qwen38-tiny.txt+ extra: numpy pillow+ - family: mux-deepseek+ req: c/tools/requirements-deepseek-v4-tiny.txt+ extra: sympy pillow+ - family: mux-deepseek-sanitize+ req: c/tools/requirements-deepseek-v4-tiny.txt+ extra: sympy pillowsteps:- uses: actions/checkout@v4- uses: actions/setup-python@v5diff --git a/c/tests/vulkan_engines.sh b/c/tests/vulkan_engines.shindex 92139bb4..9022e9da 100755--- a/c/tests/vulkan_engines.sh+++ b/c/tests/vulkan_engines.sh@@ -28,6 +28,8 @@# the first N layers on the device (COLI_VK_CHAIN_LAYERS, COLI_VK_DEVICE_CAP_MB),# tests/vulkan_partial_<group>.sh# bash tests/vulkan_engines.sh decide | decide-sanitize # Laya, GLiNER2.5-Decide and Clef's DECIDE+# bash tests/vulkan_engines.sh mux | mux-sanitize | mux-deepseek | mux-deepseek-sanitize # several+# conversations at once (KV_SLOTS), every engine but colibri and glm53 (glm-chain has theirs)# bash tests/vulkan_engines.sh dev2 | dev2-deepseek-kimi-mimo | dev2-sanitize # the expert tier on# two devices (COLI_VK_DEV2), every MoE engine, tests/vulkan_dev2.sh#@@ -4215,6 +4217,85 @@ family_decide() {[ -z "${SAN:-}" ] || make clean >/dev/null 2>&1 || true}+# Several conversations at once (KV_SLOTS, tests/serve_mux_check.py): every engine that+# decodes its serve's conversations together, each request's frames against the same+# request served alone. On the CPU bit for bit (with KV_SLOTS 4, and 3 with prompts three+# times longer), then with the routed-expert tier on the device within 1e-4 (a batch's+# rows go through the device's GEMM, which sums in another order than one row's).+# SAN=1: the same under ASan and UBSan, no diagnostic. One VK=1 build per engine runs+# both: without COLI_VULKAN it is the CPU engine. colibri and glm53 have theirs in+# glm-chain (vulkan_chain_mux.py), with the dense chain.+mux_gate() { # <engine> <snapshot> <slots> <env...>+ local eng=$1 snap=$2 n=$3 out; shift 3+ local mx=(MUX_LONG=${MUX_LONG:-1} MUX_TOL=${MUX_TOL:-0})+ if [ "${SAN:-0}" = 1 ]; then+ env "${mx[@]}" ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1 \+ $PY tests/serve_mux_check.py ./$eng $snap $n OMP_NUM_THREADS=2 "$@" > san.log 2>&1 || { cat san.log; fail "asan mux $eng $n $*"; }+ if grep -qE "ERROR: AddressSanitizer|runtime error:" san.log; then cat san.log; fail "asan mux $eng $n $*: sanitizer diagnostic"; fi+ out=$(tail -1 san.log)+ else+ out=$(env "${mx[@]}" $PY tests/serve_mux_check.py ./$eng $snap $n "$@") || { echo "$out"; fail "mux $eng $n $*"; }+ fi+ echo "${SAN:+asan }$out${MUX_LONG:+ (prompts $MUX_LONG times longer)}${1:+ [$*]}"+}+mux_engine() { # <engine> <snapshot> <env...>; VKENV: more environment for the Vulkan gate+ local eng=$1 snap=$2 vk=(${VKENV:-}); shift 2+ mux_gate "$eng" "$snap" 4 "$@"+ MUX_LONG=3 mux_gate "$eng" "$snap" 3 "$@"+ MUX_TOL=1e-4 mux_gate "$eng" "$snap" 4 COLI_VULKAN=1 COLI_VK_TIER_SYNC=1 COLI_VK_TIER_BALANCE=0 COLI_VK_CHAIN=0 "${vk[@]}" "$@"+}+mux_build() { # <make targets...>+ if [ "${SAN:-0}" = 1 ]; then+ local F="-fsanitize=address,undefined -fno-omit-frame-pointer -g"+ make "$@" VK=1 LTO=0 EXTRA_CFLAGS="$F" EXTRA_LDFLAGS="$F"+ else+ make "$@" VK=1+ fi+}+byte_tokenizer() { # <fixture> <vocabulary>: a one-token-per-byte tokenizer in place of its own+ rm -f "$1/tokenizer.json"+ $PY -c "import sys; sys.path.insert(0, 'tests'); from pathlib import Path; from prefix_serve_harness import ensure_byte_tokenizer as t; t(Path('$1'), $2)"+}+family_mux() {+ make clean >/dev/null 2>&1 || true+ mux_build qwen36 qwen38 olmoe inkling kimi_k3 mimo+ $PY tools/make_qwen36_tiny.py --out qwen36_tiny --ref-mode full --emit-ref qwen36_tiny/ref_full.json+ $PY tools/convert_qwen36.py --model qwen36_tiny --out qwen36_tiny_c --ebits 8+ byte_tokenizer qwen36_tiny_c 256+ mux_engine qwen36 qwen36_tiny_c+ $PY tools/make_qwen38_tiny.py --out qwen38_tiny+ $PY tools/make_edge_tiny_tokenizer.py --vocab-size 64 ./qwen38_tiny+ mux_engine qwen38 qwen38_tiny+ $PY tools/make_olmoe_tiny.py --output olmoe_tiny --force+ $PY tools/convert_olmoe_merged.py --model olmoe_tiny --out olmoe_tiny_c+ $PY tools/make_edge_tiny_tokenizer.py --vocab-size 128 olmoe_tiny_c+ mux_engine olmoe olmoe_tiny_c+ $PY tools/make_tiny_inkling.py tiny_inkling+ $PY -c 'import sys; from pathlib import Path; sys.path.insert(0, "."); from tests.test_inkling_prefix_serve import ensure_tokenizer; ensure_tokenizer(Path("tiny_inkling"))'+ mux_engine inkling tiny_inkling+ $PY tools/make_kimi_k3_tiny.py --output ./kimi_k3_tiny --force+ byte_tokenizer kimi_k3_tiny 256+ # the tier's experts compute as the CPU's with K3_IDOT=0 (float activations); with+ # the default int8 ones an expert's result depends on whether it is resident, which+ # the reference sessions and the shared one reach in another order+ VKENV=K3_IDOT=0 mux_engine kimi_k3 kimi_k3_tiny+ $PY tools/make_mimo_tiny.py --output ./mimo_tiny --force --vision+ mimo_served_fixture+ mux_engine mimo mimo_tiny_served+ make clean >/dev/null 2>&1 || true+}+family_mux_deepseek() {+ make clean >/dev/null 2>&1 || true+ mux_build deepseek_v41 deepseek-v4+ $PY tools/make_dsv41_tiny.py --out dsv41_tiny --emit-ref dsv41_tiny/ref.json+ mux_engine deepseek_v41 dsv41_tiny+ # V4's fixture speaks <tNNN> tokens; the check's prompts are bytes+ $PY tools/make_deepseek_v4_tiny.py --output deepseek_v4_mux --force+ byte_tokenizer deepseek_v4_mux 128+ mux_engine deepseek_v4 deepseek_v4_mux+ make clean >/dev/null 2>&1 || true+}+case "${1:-}" inshader) shader_formats ;;qwen) family_qwen ;;@@ -4271,6 +4352,10 @@ case "${1:-}" instaged-faults) family_staged_faults ;;staged-faults-sanitize) SAN=1 family_staged_faults ;;decide) family_decide ;;+ mux) family_mux ;;+ mux-sanitize) SAN=1 family_mux ;;+ mux-deepseek) family_mux_deepseek ;;+ mux-deepseek-sanitize) SAN=1 family_mux_deepseek ;;dev2) family_dev2 ;;dev2-deepseek-kimi-mimo) family_dev2_deepseek_kimi_mimo ;;dev2-sanitize) family_dev2_sanitize ;;@@ -4281,5 +4366,5 @@ case "${1:-}" inpartial-*) g=${1#partial-}; fn=ptl_family_${g//-/_}declare -F "$fn" >/dev/null || { echo "no partial-chain group ${g}" >&2; exit 2; }"$fn" ;;- *) echo "usage: $0 decide|decide-sanitize|staged|<family>-staged|shader|qwen|qwen-sanitize|inkling-olmoe|inkling-olmoe-sanitize|mimo-qwenimage|deepseek|deepseek-sanitize|kimi|kimi-mimo-sanitize|glm|glm-sanitize|qwen-chain|qwen-chain-sanitize|qwen-spec|qwen-spec-sanitize|mimo-chain|mimo-chain-sanitize|inkling-olmoe-chain|inkling-olmoe-chain-sanitize|glm-chain|glm-chain-sanitize|kimi-chain|kimi-chain-sanitize|deepseek-chain|deepseek-chain-sanitize|dense-only-<group>[-sanitize]|partial-<group>[-sanitize]|prefill-qwen|prefill-qwen-sanitize|prefill-inkling-olmoe|prefill-mimo-kimi|prefill-glm|prefill-deepseek|kv-split|kv-split-sanitize|kv-split-cold|kv-split-cold-sanitize|layers-dev2[-mla|-deepseek][-sanitize]|layers-dev2-<engine>|kv-split-deepseek|kv-split-deepseek-sanitize|dev2|dev2-deepseek-kimi-mimo|dev2-sanitize" >&2; exit 2 ;;+ *) echo "usage: $0 mux|mux-sanitize|mux-deepseek|mux-deepseek-sanitize|decide|decide-sanitize|staged|<family>-staged|shader|qwen|qwen-sanitize|inkling-olmoe|inkling-olmoe-sanitize|mimo-qwenimage|deepseek|deepseek-sanitize|kimi|kimi-mimo-sanitize|glm|glm-sanitize|qwen-chain|qwen-chain-sanitize|qwen-spec|qwen-spec-sanitize|mimo-chain|mimo-chain-sanitize|inkling-olmoe-chain|inkling-olmoe-chain-sanitize|glm-chain|glm-chain-sanitize|kimi-chain|kimi-chain-sanitize|deepseek-chain|deepseek-chain-sanitize|dense-only-<group>[-sanitize]|partial-<group>[-sanitize]|prefill-qwen|prefill-qwen-sanitize|prefill-inkling-olmoe|prefill-mimo-kimi|prefill-glm|prefill-deepseek|kv-split|kv-split-sanitize|kv-split-cold|kv-split-cold-sanitize|layers-dev2[-mla|-deepseek][-sanitize]|layers-dev2-<engine>|kv-split-deepseek|kv-split-deepseek-sanitize|dev2|dev2-deepseek-kimi-mimo|dev2-sanitize" >&2; exit 2 ;;esac