Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(vulkan): count dense-chain work and dispatch glm53 f32 residents
JustVugg4b10e014920a
diff --git a/c/Makefile b/c/Makefileindex 403514e7..a77b8c3b 100644--- a/c/Makefile+++ b/c/Makefile@@ -529,7 +529,7 @@ TEST_EXCLUDE = test_uring test_deepseek_v4 test_v4_ownership test_v4_serve_framitest_gsgemv_sse41 test_qgemv_sse41 test_olmoe_dot_i8_16_sse41 test_st_f16_bf16_simd_sse41 test_qwen38_tier_engine \test_e8x4g64_loader test_qwenimage_vae test_vk_tier \test_i4_grouped_sse41 test_i4_grouped_sse41_o1 test_i4_grouped_sse41_no_contract-TEST_EXCLUDE += test_vk_chain+TEST_EXCLUDE += test_vk_chain test_glm53_vk_f32TEST_BINS = $(addprefix tests/,$(addsuffix $(EXE),$(filter-out $(TEST_EXCLUDE),$(TEST_RULES))))ifneq (,$(LINUX))TEST_BINS += tests/test_uring$(EXE)@@ -1985,6 +1985,12 @@ tests/test_vk_chain$(EXE): tests/test_vk_chain.c $(VK_OBJ) $(VK_SPV)vk-chain-check: tests/test_vk_chain$(EXE)./tests/test_vk_chain$(EXE) shaders/qmatmul.spv+# GLM-5.3's resident f32 dispatch and its dense memory reservation (#1863).+# Run by vulkan_engines.sh shader; requires a real or software Vulkan device.+tests/test_glm53_vk_f32$(EXE): tests/test_glm53_vk_f32.c glm53.c $(VK_OBJ) $(VK_SPV)+ @[ "$(VK)" = 1 ] || { echo "tests/test_glm53_vk_f32 needs VK=1" >&2; exit 1; }+ $(CC) $(CFLAGS) $< $(VK_OBJ) -o $@ $(LDFLAGS)+tests/test_rans$(EXE): tests/test_rans.c$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)diff --git a/c/glm53.c b/c/glm53.cindex d539a069..63611172 100644--- a/c/glm53.c+++ b/c/glm53.c@@ -989,11 +989,13 @@ static void mv(float *out, const Mat *w, const float *x) {}#endif#ifdef COLI_VULKAN- if (g_vk_dense && w->resident && (w->fmt == 1 || w->fmt == 4)) {+ if (g_vk_dense && w->resident && (w->fmt == 0 || w->fmt == 1 || w->fmt == 4)) {Mat *mutable_w = (Mat *)w;if (coli_vk_matmul((ColiVkTensor **)&mutable_w->vk, out, x,+ w->fmt == 0 ? (const void *)w->f :w->fmt == 4 ? (const void *)w->q4 : (const void *)w->q8,- w->s, w->fmt, 1, w->columns, w->rows, w->gs))+ w->fmt == 0 ? NULL : w->s, w->fmt == 0 ? 10 : w->fmt,+ 1, w->columns, w->rows, w->gs))return;}#endif@@ -1017,14 +1019,16 @@ static void mm(float *out, const Mat *w, const float *x, int S) {gpu |= g_metal_ready && w->resident && (w->fmt == 1 || w->fmt == 4);#endif#ifdef COLI_VULKAN- if (S > 1 && g_vk_dense && w->resident && (w->fmt == 1 || w->fmt == 4)) {+ if (S > 1 && g_vk_dense && w->resident && (w->fmt == 0 || w->fmt == 1 || w->fmt == 4)) {Mat *mutable_w = (Mat *)w;if (coli_vk_matmul((ColiVkTensor **)&mutable_w->vk, out, x,+ w->fmt == 0 ? (const void *)w->f :w->fmt == 4 ? (const void *)w->q4 : (const void *)w->q8,- w->s, w->fmt, S, w->columns, w->rows, w->gs))+ w->fmt == 0 ? NULL : w->s, w->fmt == 0 ? 10 : w->fmt,+ S, w->columns, w->rows, w->gs))return;}- gpu |= g_vk_dense && w->resident && (w->fmt == 1 || w->fmt == 4);+ gpu |= g_vk_dense && w->resident && (w->fmt == 0 || w->fmt == 1 || w->fmt == 4);#endifif (S == 1 || gpu) {for (int t = 0; t < S; t++)@@ -4037,8 +4041,9 @@ static int glm53_in_ram(void *ctx, int layer, int eid) {/* Bytes the resident matrices will take on the device: they upload at first use,* after the tier has taken its budget. */static size_t glm53_mat_dev_bytes(const Mat *w) {- if (!w->resident || (w->fmt != 1 && w->fmt != 4) || w->rows < 1 || w->columns < 1) return 0;+ if (!w->resident || (w->fmt != 0 && w->fmt != 1 && w->fmt != 4) || w->rows < 1 || w->columns < 1) return 0;const size_t r = (size_t)w->rows, cl = (size_t)w->columns;+ if (w->fmt == 0) return r * cl * sizeof(float);return w->fmt == 1 ? r * cl + r * 4 : r * ((cl + 1) / 2) + r * ((cl + w->gs - 1) / w->gs) * 4;}static size_t glm53_dense_dev_bytes(const GModel *m) {diff --git a/c/tests/test_glm53_vk_f32.c b/c/tests/test_glm53_vk_f32.cnew file mode 100644index 00000000..eacfe997--- /dev/null+++ b/c/tests/test_glm53_vk_f32.c@@ -0,0 +1,56 @@+/* The production GLM-5.3 dispatch must upload resident f32 matrices for both+ * decode and prefill, and reserve their bytes before sizing the expert tier.+ * A numerical comparison alone would pass when every call falls back to CPU. */+#define main glm53_main_unused+#include "../glm53.c"+#undef main++int main(void) {+ enum { I = 64, O = 128, S = 64 };+ float weights[I * O], x[S * I], cpu[S * O], gpu[S * O];+ for (int i = 0; i < I * O; i++) weights[i] = (float)(i % 19 - 9) / 19.f;+ for (int i = 0; i < S * I; i++) x[i] = (float)(i % 17 - 8) / 17.f;+ Mat w = {.fmt = 0, .rows = O, .columns = I, .resident = 1, .f = weights};+ if (glm53_mat_dev_bytes(&w) != sizeof(weights)) {+ fputs("FAIL: f32 residents missing from the dense GPU budget\n", stderr);+ return 1;+ }+ if (!coli_vk_init("shaders/qmatmul.spv")) {+ fputs("FAIL: no Vulkan device\n", stderr);+ return 1;+ }+ int failed = 0;+ const int rows[] = {1, 5, S};+ for (size_t k = 0; k < sizeof(rows) / sizeof(rows[0]); k++) {+ const int n = rows[k];+ g_vk_dense = 0;+ mm(cpu, &w, x, n);+ g_vk_dense = 1;+ unsigned long long before = coli_vk_matmul_calls();+ mm(gpu, &w, x, n);+ if (coli_vk_matmul_calls() == before) {+ fprintf(stderr, "FAIL: S=%d f32 dispatch did no Vulkan matmul\n", n);+ failed = 1;+ }+ for (int i = 0; i < n * O; i++) {+ if (!isfinite(gpu[i]) || fabsf(cpu[i] - gpu[i]) > 1e-5f * (1.f + fabsf(cpu[i]))) {+ fprintf(stderr, "FAIL: S=%d output[%d]: CPU=%g Vulkan=%g\n", n, i, cpu[i], gpu[i]);+ failed = 1;+ break;+ }+ }+ }+ /* Nonresident weights still belong to the CPU and take no dense budget. */+ w.resident = 0;+ unsigned long long before = coli_vk_matmul_calls();+ mm(gpu, &w, x, S);+ if (coli_vk_matmul_calls() != before || glm53_mat_dev_bytes(&w) != 0 ||+ memcmp(cpu, gpu, sizeof(cpu))) {+ fputs("FAIL: nonresident f32 matrix changed placement or result\n", stderr);+ failed = 1;+ }+ coli_vk_tensor_free((ColiVkTensor *)w.vk);+ coli_vk_shutdown();+ if (!failed) puts("PASS: glm53 f32 Vulkan decode, prefill and dense budget");+ return failed;+}diff --git a/c/tests/test_vulkan_engine_counts.py b/c/tests/test_vulkan_engine_counts.pynew file mode 100644index 00000000..0978599f--- /dev/null+++ b/c/tests/test_vulkan_engine_counts.py@@ -0,0 +1,70 @@+"""Exercise the shell harness's actual counters without requiring a GPU."""+from pathlib import Path+import re+import shutil+import subprocess+import tempfile+import unittest+++SCRIPT = Path(__file__).with_name("vulkan_engines.sh").read_text()+# Load only these production helpers: sourcing the full script runs a family.+HELPERS = re.search(r"^fail\(\).*?$", SCRIPT, re.M).group() + "\n"+for name in ("vk_count", "need_gpu"):+ HELPERS += re.search(rf"^{name}\(\).*?^\}}", SCRIPT, re.M | re.S).group() + "\n"++[email protected](shutil.which("bash"), "the Vulkan harness requires bash")+class VulkanEngineCounts(unittest.TestCase):+ def run_helper(self, log, engine="qwen36", helper="vk_count"):+ with tempfile.TemporaryDirectory() as directory:+ path = Path(directory) / "run.log"+ path.write_text(log)+ return subprocess.run(+ ["bash", "-eu", "-o", "pipefail", "-c",+ HELPERS + f'{helper} "$1" "$2" regression',+ "counts-test", engine, str(path)],+ capture_output=True, text=True, check=False,+ )++ def test_legacy_per_matrix_count(self):+ result = self.run_helper("[VK] qwen36: 7 matmuls on the GPU (73 matrices resident)\n")+ self.assertEqual(result.returncode, 0, result.stderr)+ self.assertEqual(result.stdout.strip(), "7")++ def test_chain_only_satisfies_gpu_gate(self):+ for engine, count in (("qwen36", 1168), ("olmoe", 168), ("mimo", 96)):+ with self.subTest(engine=engine):+ log = (f"[VK] {engine}: 0 matmuls on the GPU\n"+ f"[VK] {engine} chain: 6 forwards, 63 frames (360 ops, "+ f"{count} matmuls, 4 tiled GEMM), timings\n")+ result = self.run_helper(log, engine)+ self.assertEqual(result.returncode, 0, result.stderr)+ self.assertEqual(result.stdout.strip(), str(count))+ self.assertEqual(self.run_helper(log, engine, "need_gpu").returncode, 0)++ def test_latest_cumulative_reports_are_added_once(self):+ log = ("[VK] qwen36: 1 matmuls on the GPU\n"+ "[VK] qwen36 chain: 1 forwards, 3 frames (50 ops, 3 matmuls, 1 tiled GEMM)\n"+ "[VK] qwen36: 2 matmuls on the GPU\n"+ "[VK] qwen36 chain: 2 forwards, 6 frames (100 ops, 9 matmuls, 2 tiled GEMM)\n")+ result = self.run_helper(log)+ self.assertEqual(result.returncode, 0, result.stderr)+ self.assertEqual(result.stdout.strip(), "11")++ def test_unrelated_engine_does_not_satisfy_gpu_gate(self):+ log = ("[VK] olmoe: 999 matmuls on the GPU\n"+ "[VK] olmoe chain: 1 forwards, 3 frames (50 ops, 99 matmuls, 0 tiled GEMM)\n")+ self.assertEqual(self.run_helper(log).stdout.strip(), "0")+ self.assertNotEqual(self.run_helper(log, helper="need_gpu").returncode, 0)++ def test_empty_and_zero_logs_still_fail_gpu_gate(self):+ for log in ("", "[VK] qwen36: 0 matmuls on the GPU\n",+ "[VK] qwen36 chain: 1 forwards, 3 frames (50 ops, 0 matmuls, 0 tiled GEMM)\n"):+ with self.subTest(log=log):+ self.assertEqual(self.run_helper(log).stdout.strip(), "0")+ self.assertNotEqual(self.run_helper(log, helper="need_gpu").returncode, 0)+++if __name__ == "__main__":+ unittest.main()diff --git a/c/tests/vulkan_engines.sh b/c/tests/vulkan_engines.shindex f945b8c0..19031b1b 100755--- a/c/tests/vulkan_engines.sh+++ b/c/tests/vulkan_engines.sh@@ -19,7 +19,7 @@# back as the CPU computes it. Each configuration is gated on two things:# - the Vulkan run gives the tokens of the CPU run with the same snapshot and# settings (and, where the engine has one, passes its own oracle);-# - its "[VK] <engine>: N matmuls on the GPU" line has N > 0, because a hook that+# - its per-matrix and dense-chain matmul counts sum to N > 0, because a hook that# declines every matrix would otherwise pass the first gate trivially. A# configuration of the routed-expert tier with the dense trunk on the CPU (the# default on Lavapipe while the tier is on, see dense_where) gates on its count of@@ -37,11 +37,13 @@ fail() { echo "FAIL: $*"; exit 1; }# (-march=native) GCC keeps 64-byte aligned locals there and stores them with# vmovdqa64, which faults (c/Makefile, ASAN_ENV, has the details).-# vk_count <engine> <log>: N from the last "[VK] <engine>: N matmuls on the GPU" line+# vk_count <engine> <log>: the latest per-matrix and dense-chain matmul counts.+# The counters are independent and cumulative; sum their last reports, not turns.vk_count() {- local n+ local n chainn=$(sed -n "s/^\[VK\] $1: \([0-9][0-9]*\) matmuls on the GPU.*/\1/p" "$2" | tail -1)- echo "${n:-0}"+ chain=$(sed -n "s/^\[VK\] $1 chain: .* ops, \([0-9][0-9]*\) matmuls,.*/\1/p" "$2" | tail -1)+ echo "$(( ${n:-0} + ${chain:-0} ))"}need_gpu() { # <engine> <log> <tag>[ "$(vk_count "$1" "$2")" -gt 0 ] || { cat "$2"; fail "$3: no matmul ran on the device"; }@@ -56,12 +58,13 @@ same_tokens() { # <cpu log> <vk log> <tag>: the engines' "C engine" token lines# then the expert batch and the weight pool in the same harness, and the routed-expert# tier (vk_tier.c) on a synthetic model in every source format.shader_formats() {- make tests/test_vk_tier VK=1 # every shader too: the harness's expert batch needs them+ make tests/test_vk_tier tests/test_glm53_vk_f32 VK=1 # every shader too: the harness's expert batch needs themcc -O2 -pthread -DVK_TEST backend_vulkan.c -o vk_test -lvulkan -lmCOLI_VK_TEST_MATMUL_ONLY=1 ./vk_test shaders/qmatmul.spv | tee vk_test.logtail -1 vk_test.log | grep -qx PASS || fail "qmatmul format cases"./tests/test_vk_tier shaders/qmatmul.spv | tee vk_tier.logtail -1 vk_tier.log | grep -qx PASS || fail "routed-expert tier"+ ./tests/test_glm53_vk_f32}# tier_count <engine> <log>: N from the last "[VK] tier <engine> run: device N of M" line;