Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(vulkan): count dense-chain work and dispatch glm53 f32 residents

JustVugg4b10e014920a
diff --git a/c/Makefile b/c/Makefile
index 403514e7..a77b8c3b 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -529,7 +529,7 @@ TEST_EXCLUDE = test_uring test_deepseek_v4 test_v4_ownership test_v4_serve_frami
test_gsgemv_sse41 test_qgemv_sse41 test_olmoe_dot_i8_16_sse41 test_st_f16_bf16_simd_sse41 test_qwen38_tier_engine \
test_e8x4g64_loader test_qwenimage_vae test_vk_tier \
test_i4_grouped_sse41 test_i4_grouped_sse41_o1 test_i4_grouped_sse41_no_contract
-TEST_EXCLUDE += test_vk_chain
+TEST_EXCLUDE += test_vk_chain test_glm53_vk_f32
TEST_BINS = $(addprefix tests/,$(addsuffix $(EXE),$(filter-out $(TEST_EXCLUDE),$(TEST_RULES))))
ifneq (,$(LINUX))
TEST_BINS += tests/test_uring$(EXE)
@@ -1985,6 +1985,12 @@ tests/test_vk_chain$(EXE): tests/test_vk_chain.c $(VK_OBJ) $(VK_SPV)
vk-chain-check: tests/test_vk_chain$(EXE)
./tests/test_vk_chain$(EXE) shaders/qmatmul.spv
+# GLM-5.3's resident f32 dispatch and its dense memory reservation (#1863).
+# Run by vulkan_engines.sh shader; requires a real or software Vulkan device.
+tests/test_glm53_vk_f32$(EXE): tests/test_glm53_vk_f32.c glm53.c $(VK_OBJ) $(VK_SPV)
+ @[ "$(VK)" = 1 ] || { echo "tests/test_glm53_vk_f32 needs VK=1" >&2; exit 1; }
+ $(CC) $(CFLAGS) $< $(VK_OBJ) -o $@ $(LDFLAGS)
+
tests/test_rans$(EXE): tests/test_rans.c
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
diff --git a/c/glm53.c b/c/glm53.c
index d539a069..63611172 100644
--- a/c/glm53.c
+++ b/c/glm53.c
@@ -989,11 +989,13 @@ static void mv(float *out, const Mat *w, const float *x) {
}
#endif
#ifdef COLI_VULKAN
- if (g_vk_dense && w->resident && (w->fmt == 1 || w->fmt == 4)) {
+ if (g_vk_dense && w->resident && (w->fmt == 0 || w->fmt == 1 || w->fmt == 4)) {
Mat *mutable_w = (Mat *)w;
if (coli_vk_matmul((ColiVkTensor **)&mutable_w->vk, out, x,
+ w->fmt == 0 ? (const void *)w->f :
w->fmt == 4 ? (const void *)w->q4 : (const void *)w->q8,
- w->s, w->fmt, 1, w->columns, w->rows, w->gs))
+ w->fmt == 0 ? NULL : w->s, w->fmt == 0 ? 10 : w->fmt,
+ 1, w->columns, w->rows, w->gs))
return;
}
#endif
@@ -1017,14 +1019,16 @@ static void mm(float *out, const Mat *w, const float *x, int S) {
gpu |= g_metal_ready && w->resident && (w->fmt == 1 || w->fmt == 4);
#endif
#ifdef COLI_VULKAN
- if (S > 1 && g_vk_dense && w->resident && (w->fmt == 1 || w->fmt == 4)) {
+ if (S > 1 && g_vk_dense && w->resident && (w->fmt == 0 || w->fmt == 1 || w->fmt == 4)) {
Mat *mutable_w = (Mat *)w;
if (coli_vk_matmul((ColiVkTensor **)&mutable_w->vk, out, x,
+ w->fmt == 0 ? (const void *)w->f :
w->fmt == 4 ? (const void *)w->q4 : (const void *)w->q8,
- w->s, w->fmt, S, w->columns, w->rows, w->gs))
+ w->fmt == 0 ? NULL : w->s, w->fmt == 0 ? 10 : w->fmt,
+ S, w->columns, w->rows, w->gs))
return;
}
- gpu |= g_vk_dense && w->resident && (w->fmt == 1 || w->fmt == 4);
+ gpu |= g_vk_dense && w->resident && (w->fmt == 0 || w->fmt == 1 || w->fmt == 4);
#endif
if (S == 1 || gpu) {
for (int t = 0; t < S; t++)
@@ -4037,8 +4041,9 @@ static int glm53_in_ram(void *ctx, int layer, int eid) {
/* Bytes the resident matrices will take on the device: they upload at first use,
* after the tier has taken its budget. */
static size_t glm53_mat_dev_bytes(const Mat *w) {
- if (!w->resident || (w->fmt != 1 && w->fmt != 4) || w->rows < 1 || w->columns < 1) return 0;
+ if (!w->resident || (w->fmt != 0 && w->fmt != 1 && w->fmt != 4) || w->rows < 1 || w->columns < 1) return 0;
const size_t r = (size_t)w->rows, cl = (size_t)w->columns;
+ if (w->fmt == 0) return r * cl * sizeof(float);
return w->fmt == 1 ? r * cl + r * 4 : r * ((cl + 1) / 2) + r * ((cl + w->gs - 1) / w->gs) * 4;
}
static size_t glm53_dense_dev_bytes(const GModel *m) {
diff --git a/c/tests/test_glm53_vk_f32.c b/c/tests/test_glm53_vk_f32.c
new file mode 100644
index 00000000..eacfe997
--- /dev/null
+++ b/c/tests/test_glm53_vk_f32.c
@@ -0,0 +1,56 @@
+/* The production GLM-5.3 dispatch must upload resident f32 matrices for both
+ * decode and prefill, and reserve their bytes before sizing the expert tier.
+ * A numerical comparison alone would pass when every call falls back to CPU. */
+#define main glm53_main_unused
+#include "../glm53.c"
+#undef main
+
+int main(void) {
+ enum { I = 64, O = 128, S = 64 };
+ float weights[I * O], x[S * I], cpu[S * O], gpu[S * O];
+ for (int i = 0; i < I * O; i++) weights[i] = (float)(i % 19 - 9) / 19.f;
+ for (int i = 0; i < S * I; i++) x[i] = (float)(i % 17 - 8) / 17.f;
+ Mat w = {.fmt = 0, .rows = O, .columns = I, .resident = 1, .f = weights};
+ if (glm53_mat_dev_bytes(&w) != sizeof(weights)) {
+ fputs("FAIL: f32 residents missing from the dense GPU budget\n", stderr);
+ return 1;
+ }
+ if (!coli_vk_init("shaders/qmatmul.spv")) {
+ fputs("FAIL: no Vulkan device\n", stderr);
+ return 1;
+ }
+ int failed = 0;
+ const int rows[] = {1, 5, S};
+ for (size_t k = 0; k < sizeof(rows) / sizeof(rows[0]); k++) {
+ const int n = rows[k];
+ g_vk_dense = 0;
+ mm(cpu, &w, x, n);
+ g_vk_dense = 1;
+ unsigned long long before = coli_vk_matmul_calls();
+ mm(gpu, &w, x, n);
+ if (coli_vk_matmul_calls() == before) {
+ fprintf(stderr, "FAIL: S=%d f32 dispatch did no Vulkan matmul\n", n);
+ failed = 1;
+ }
+ for (int i = 0; i < n * O; i++) {
+ if (!isfinite(gpu[i]) || fabsf(cpu[i] - gpu[i]) > 1e-5f * (1.f + fabsf(cpu[i]))) {
+ fprintf(stderr, "FAIL: S=%d output[%d]: CPU=%g Vulkan=%g\n", n, i, cpu[i], gpu[i]);
+ failed = 1;
+ break;
+ }
+ }
+ }
+ /* Nonresident weights still belong to the CPU and take no dense budget. */
+ w.resident = 0;
+ unsigned long long before = coli_vk_matmul_calls();
+ mm(gpu, &w, x, S);
+ if (coli_vk_matmul_calls() != before || glm53_mat_dev_bytes(&w) != 0 ||
+ memcmp(cpu, gpu, sizeof(cpu))) {
+ fputs("FAIL: nonresident f32 matrix changed placement or result\n", stderr);
+ failed = 1;
+ }
+ coli_vk_tensor_free((ColiVkTensor *)w.vk);
+ coli_vk_shutdown();
+ if (!failed) puts("PASS: glm53 f32 Vulkan decode, prefill and dense budget");
+ return failed;
+}
diff --git a/c/tests/test_vulkan_engine_counts.py b/c/tests/test_vulkan_engine_counts.py
new file mode 100644
index 00000000..0978599f
--- /dev/null
+++ b/c/tests/test_vulkan_engine_counts.py
@@ -0,0 +1,70 @@
+"""Exercise the shell harness's actual counters without requiring a GPU."""
+from pathlib import Path
+import re
+import shutil
+import subprocess
+import tempfile
+import unittest
+
+
+SCRIPT = Path(__file__).with_name("vulkan_engines.sh").read_text()
+# Load only these production helpers: sourcing the full script runs a family.
+HELPERS = re.search(r"^fail\(\).*?$", SCRIPT, re.M).group() + "\n"
+for name in ("vk_count", "need_gpu"):
+ HELPERS += re.search(rf"^{name}\(\).*?^\}}", SCRIPT, re.M | re.S).group() + "\n"
+
+
[email protected](shutil.which("bash"), "the Vulkan harness requires bash")
+class VulkanEngineCounts(unittest.TestCase):
+ def run_helper(self, log, engine="qwen36", helper="vk_count"):
+ with tempfile.TemporaryDirectory() as directory:
+ path = Path(directory) / "run.log"
+ path.write_text(log)
+ return subprocess.run(
+ ["bash", "-eu", "-o", "pipefail", "-c",
+ HELPERS + f'{helper} "$1" "$2" regression',
+ "counts-test", engine, str(path)],
+ capture_output=True, text=True, check=False,
+ )
+
+ def test_legacy_per_matrix_count(self):
+ result = self.run_helper("[VK] qwen36: 7 matmuls on the GPU (73 matrices resident)\n")
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(result.stdout.strip(), "7")
+
+ def test_chain_only_satisfies_gpu_gate(self):
+ for engine, count in (("qwen36", 1168), ("olmoe", 168), ("mimo", 96)):
+ with self.subTest(engine=engine):
+ log = (f"[VK] {engine}: 0 matmuls on the GPU\n"
+ f"[VK] {engine} chain: 6 forwards, 63 frames (360 ops, "
+ f"{count} matmuls, 4 tiled GEMM), timings\n")
+ result = self.run_helper(log, engine)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(result.stdout.strip(), str(count))
+ self.assertEqual(self.run_helper(log, engine, "need_gpu").returncode, 0)
+
+ def test_latest_cumulative_reports_are_added_once(self):
+ log = ("[VK] qwen36: 1 matmuls on the GPU\n"
+ "[VK] qwen36 chain: 1 forwards, 3 frames (50 ops, 3 matmuls, 1 tiled GEMM)\n"
+ "[VK] qwen36: 2 matmuls on the GPU\n"
+ "[VK] qwen36 chain: 2 forwards, 6 frames (100 ops, 9 matmuls, 2 tiled GEMM)\n")
+ result = self.run_helper(log)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(result.stdout.strip(), "11")
+
+ def test_unrelated_engine_does_not_satisfy_gpu_gate(self):
+ log = ("[VK] olmoe: 999 matmuls on the GPU\n"
+ "[VK] olmoe chain: 1 forwards, 3 frames (50 ops, 99 matmuls, 0 tiled GEMM)\n")
+ self.assertEqual(self.run_helper(log).stdout.strip(), "0")
+ self.assertNotEqual(self.run_helper(log, helper="need_gpu").returncode, 0)
+
+ def test_empty_and_zero_logs_still_fail_gpu_gate(self):
+ for log in ("", "[VK] qwen36: 0 matmuls on the GPU\n",
+ "[VK] qwen36 chain: 1 forwards, 3 frames (50 ops, 0 matmuls, 0 tiled GEMM)\n"):
+ with self.subTest(log=log):
+ self.assertEqual(self.run_helper(log).stdout.strip(), "0")
+ self.assertNotEqual(self.run_helper(log, helper="need_gpu").returncode, 0)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/c/tests/vulkan_engines.sh b/c/tests/vulkan_engines.sh
index f945b8c0..19031b1b 100755
--- a/c/tests/vulkan_engines.sh
+++ b/c/tests/vulkan_engines.sh
@@ -19,7 +19,7 @@
# back as the CPU computes it. Each configuration is gated on two things:
# - the Vulkan run gives the tokens of the CPU run with the same snapshot and
# settings (and, where the engine has one, passes its own oracle);
-# - its "[VK] <engine>: N matmuls on the GPU" line has N > 0, because a hook that
+# - its per-matrix and dense-chain matmul counts sum to N > 0, because a hook that
# declines every matrix would otherwise pass the first gate trivially. A
# configuration of the routed-expert tier with the dense trunk on the CPU (the
# default on Lavapipe while the tier is on, see dense_where) gates on its count of
@@ -37,11 +37,13 @@ fail() { echo "FAIL: $*"; exit 1; }
# (-march=native) GCC keeps 64-byte aligned locals there and stores them with
# vmovdqa64, which faults (c/Makefile, ASAN_ENV, has the details).
-# vk_count <engine> <log>: N from the last "[VK] <engine>: N matmuls on the GPU" line
+# vk_count <engine> <log>: the latest per-matrix and dense-chain matmul counts.
+# The counters are independent and cumulative; sum their last reports, not turns.
vk_count() {
- local n
+ local n chain
n=$(sed -n "s/^\[VK\] $1: \([0-9][0-9]*\) matmuls on the GPU.*/\1/p" "$2" | tail -1)
- echo "${n:-0}"
+ chain=$(sed -n "s/^\[VK\] $1 chain: .* ops, \([0-9][0-9]*\) matmuls,.*/\1/p" "$2" | tail -1)
+ echo "$(( ${n:-0} + ${chain:-0} ))"
}
need_gpu() { # <engine> <log> <tag>
[ "$(vk_count "$1" "$2")" -gt 0 ] || { cat "$2"; fail "$3: no matmul ran on the device"; }
@@ -56,12 +58,13 @@ same_tokens() { # <cpu log> <vk log> <tag>: the engines' "C engine" token lines
# then the expert batch and the weight pool in the same harness, and the routed-expert
# tier (vk_tier.c) on a synthetic model in every source format.
shader_formats() {
- make tests/test_vk_tier VK=1 # every shader too: the harness's expert batch needs them
+ make tests/test_vk_tier tests/test_glm53_vk_f32 VK=1 # every shader too: the harness's expert batch needs them
cc -O2 -pthread -DVK_TEST backend_vulkan.c -o vk_test -lvulkan -lm
COLI_VK_TEST_MATMUL_ONLY=1 ./vk_test shaders/qmatmul.spv | tee vk_test.log
tail -1 vk_test.log | grep -qx PASS || fail "qmatmul format cases"
./tests/test_vk_tier shaders/qmatmul.spv | tee vk_tier.log
tail -1 vk_tier.log | grep -qx PASS || fail "routed-expert tier"
+ ./tests/test_glm53_vk_f32
}
# tier_count <engine> <log>: N from the last "[VK] tier <engine> run: device N of M" line;