Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

vulkan: one entry point for every engine, and the engines link the backend

JustVugg703facb60dea
diff --git a/c/Makefile b/c/Makefile
index 21c5988f..4168f64b 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -1181,8 +1181,8 @@ tests/bench_cuda_resident_batch$(EXE): tests/bench_cuda_resident_batch.cu backen
# the GPU sits idle. Same guard #783 put on kimi_k3, which since gaining an
# MXFP4 expert path no longer needs it. tests/test_makefile_cuda_scope.py
# asserts this shape.
-olmoe$(EXE): olmoe.c
- $(CC) $(NOCUDA_CFLAGS) olmoe.c -o olmoe$(EXE) $(NOCUDA_LDFLAGS)
+olmoe$(EXE): olmoe.c $(VK_OBJ) $(VK_SPV)
+ $(CC) $(NOCUDA_CFLAGS) olmoe.c $(VK_OBJ) -o olmoe$(EXE) $(NOCUDA_LDFLAGS)
# Qwen3.6-35B-A3B engine (hybrid Gated Attention + Gated DeltaNet + streaming
# MoE). With CUDA=1 the optional VRAM expert tier (qwen36_tier.c) is compiled
@@ -1236,8 +1236,8 @@ qpack.o: qpack.c .build-config
$(CC) $(QWEN36_CFLAGS) -c qpack.c -o $@
# Rebuild when CUDA_DLL changes; otherwise an existing CPU-only executable can
# be reported as up to date despite selecting the Windows CUDA DLL tier.
-qwen36$(EXE): qwen36.c $(QWEN36_TIER_OBJ) $(QWEN36_QPACK_OBJ) $(CUDA_OBJ) $(METAL_OBJ) .build-config
- $(CC) $(QWEN36_CFLAGS) qwen36.c $(QWEN36_TIER_OBJ) $(QWEN36_QPACK_OBJ) $(CUDA_OBJ) $(METAL_OBJ) -o qwen36$(EXE) $(QWEN36_LDFLAGS)
+qwen36$(EXE): qwen36.c $(QWEN36_TIER_OBJ) $(QWEN36_QPACK_OBJ) $(CUDA_OBJ) $(METAL_OBJ) $(VK_OBJ) $(VK_SPV) .build-config
+ $(CC) $(QWEN36_CFLAGS) qwen36.c $(QWEN36_TIER_OBJ) $(QWEN36_QPACK_OBJ) $(CUDA_OBJ) $(METAL_OBJ) $(VK_OBJ) -o qwen36$(EXE) $(QWEN36_LDFLAGS)
# Qwen-Image-2.1 text-to-image (docs/qwen-image.md). It reads the diffusers
# checkpoint as it is: no conversion step. qi_gemm.h is its one matrix kernel.
@@ -1245,8 +1245,8 @@ ifneq ($(EXE),)
.PHONY: qwenimage
qwenimage: qwenimage$(EXE)
endif
-qwenimage$(EXE): qwenimage.c .build-config
- $(CC) $(CFLAGS) qwenimage.c -o qwenimage$(EXE) $(LDFLAGS)
+qwenimage$(EXE): qwenimage.c $(VK_OBJ) $(VK_SPV) .build-config
+ $(CC) $(CFLAGS) qwenimage.c $(VK_OBJ) -o qwenimage$(EXE) $(LDFLAGS)
# In make check: every format and ragged edges against the naive product, and
# the int8 x int8 VNNI path against the exact integer sum.
@@ -1256,8 +1256,8 @@ tests/test_qi_gemm$(EXE): tests/test_qi_gemm.c
# DeepSeek V4.1 Flash: one file, like every other portable engine. The fp4 experts
# stream from the official checkpoint and quant.h's mxfp4 kernel reads them as they
# are, so there is no conversion target here to go with it.
-deepseek_v41$(EXE): deepseek_v41.c .build-config
- $(CC) $(CFLAGS) deepseek_v41.c -o deepseek_v41$(EXE) $(LDFLAGS)
+deepseek_v41$(EXE): deepseek_v41.c $(VK_OBJ) $(VK_SPV) .build-config
+ $(CC) $(CFLAGS) deepseek_v41.c $(VK_OBJ) -o deepseek_v41$(EXE) $(LDFLAGS)
# Xiaomi MiMo-V2.6 (Flash 309B, Pro 1.02T): one file plus its ViT header, reading the
# release as it is -- MXFP4 experts streamed per layer, FP8/BF16 dense resident.
@@ -1265,8 +1265,8 @@ ifneq ($(EXE),)
.PHONY: mimo
mimo: mimo$(EXE)
endif
-mimo$(EXE): mimo.c .build-config
- $(CC) $(CFLAGS) mimo.c -o mimo$(EXE) $(LDFLAGS)
+mimo$(EXE): mimo.c $(VK_OBJ) $(VK_SPV) .build-config
+ $(CC) $(CFLAGS) mimo.c $(VK_OBJ) -o mimo$(EXE) $(LDFLAGS)
# Tiny MiMo fixture + vendor oracle, the Kimi K3 pattern: the generator is numpy (and
# Pillow for the test picture) and deterministic; ref.json is committed and comes from
@@ -1283,8 +1283,8 @@ mimo-tiny-check: mimo-tiny-generate mimo$(EXE)
# loader and SERVE=1 protocol. With CUDA=1 it links the same expert tier as
# qwen36 (fp8 streaming mode: hot experts get VRAM copies, the RAM LRU stays);
# without it the tier header's inline stubs keep the build toolkit-free.
-qwen38$(EXE): qwen38.c $(QWEN36_TIER_OBJ) $(CUDA_OBJ)
- $(CC) $(QWEN36_CFLAGS) qwen38.c $(QWEN36_TIER_OBJ) $(CUDA_OBJ) -o qwen38$(EXE) $(QWEN36_LDFLAGS)
+qwen38$(EXE): qwen38.c $(QWEN36_TIER_OBJ) $(CUDA_OBJ) $(VK_OBJ) $(VK_SPV)
+ $(CC) $(QWEN36_CFLAGS) qwen38.c $(QWEN36_TIER_OBJ) $(CUDA_OBJ) $(VK_OBJ) -o qwen38$(EXE) $(QWEN36_LDFLAGS)
.PHONY: qwen38-tiny-generate qwen38-tiny-check
qwen38-tiny-generate:
@@ -1404,8 +1404,8 @@ tests/bench_qwen36_dense_batch$(EXE): tests/bench_qwen36_dense_batch.c qwen36.c
$(QWEN36_TIER_OBJ) $(CUDA_OBJ)
$(CC) $(QWEN36_CFLAGS) $< $(QWEN36_TIER_OBJ) $(CUDA_OBJ) -o $@ $(QWEN36_LDFLAGS)
-inkling$(EXE): inkling.c $(INK_CUDA_OBJ) $(METAL_OBJ)
- $(CC) $(CFLAGS) inkling.c $(INK_CUDA_OBJ) $(METAL_OBJ) -o inkling$(EXE) $(LDFLAGS)
+inkling$(EXE): inkling.c $(INK_CUDA_OBJ) $(METAL_OBJ) $(VK_OBJ) $(VK_SPV)
+ $(CC) $(CFLAGS) inkling.c $(INK_CUDA_OBJ) $(METAL_OBJ) $(VK_OBJ) -o inkling$(EXE) $(LDFLAGS)
# ENGINES WITHOUT A CUDA BACKEND (#783).
# kimi_k3.c contains no COLI_CUDA code at all — its GPU path is Vulkan. But
diff --git a/c/backend_vulkan.c b/c/backend_vulkan.c
index 9b9d913c..33346eee 100644
--- a/c/backend_vulkan.c
+++ b/c/backend_vulkan.c
@@ -13,6 +13,10 @@
#include <stdlib.h>
#include <string.h>
#include <time.h>
+#include <sys/stat.h>
+#ifdef __linux__
+#include <unistd.h>
+#endif
static double vk_now(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec*1000.0 + t.tv_nsec/1e6; }
#define VKCHECK(x, what) do { VkResult _r = (x); if (_r != VK_SUCCESS) { \
@@ -584,6 +588,8 @@ static void vkprof_tick(void) {
fprintf(stderr, "[VK_PROF sub] n=%ld | submit %.0f | wait %.0f ms\n", g_vsub_n, g_vsub_ms, g_vwait_ms);
}
+static unsigned long long g_vk_matmul_calls; /* successful coli_vk_matmul calls */
+
int coli_vk_matmul(ColiVkTensor **tensor, float *y, const float *x,
const void *weights, const float *scales,
int fmt, int S, int I, int O, int gs) {
@@ -661,9 +667,48 @@ int coli_vk_matmul(ColiVkTensor **tensor, float *y, const float *x,
fprintf(stderr, "[VK_PROF dense] n=%ld | memcpy_x %.0f | desc %.0f | record %.0f | submit %.0f | wait %.0f | memcpy_y %.0f ms\n",
p_n, p_x, p_desc, p_rec, p_sub, p_wait, p_y);
}
+ g_vk_matmul_calls++;
return 1;
}
+unsigned long long coli_vk_matmul_calls(void) { return g_vk_matmul_calls; }
+
+/* The shader path every engine resolves the same way (#523): COLI_VK_SHADERS may be the
+ * qmatmul.spv file or the directory holding it; unset, the shaders/ directory next to the
+ * binary (the build layout), then the historical path relative to the working directory. */
+const char *coli_vk_shader_path(char *buf, size_t n) {
+ const char *env = getenv("COLI_VK_SHADERS");
+ struct stat st;
+ if (env && *env) {
+ if (!stat(env, &st) && S_ISDIR(st.st_mode)) { snprintf(buf, n, "%s/qmatmul.spv", env); return buf; }
+ return env;
+ }
+#ifdef __linux__
+ ssize_t k = readlink("/proc/self/exe", buf, n - 1);
+ if (k > 0) {
+ buf[k] = 0;
+ char *sl = strrchr(buf, '/');
+ if (sl && (size_t)(sl + 1 - buf) + sizeof("shaders/qmatmul.spv") <= n) {
+ strcpy(sl + 1, "shaders/qmatmul.spv");
+ if (!stat(buf, &st)) return buf;
+ }
+ }
+#endif
+ return "shaders/qmatmul.spv";
+}
+
+int coli_vk_init_env(const char *engine) {
+ const char *on = getenv("COLI_VULKAN");
+ if (!on || !atoi(on)) return 0;
+ char buf[1024];
+ const char *spv = coli_vk_shader_path(buf, sizeof buf);
+ int ok = coli_vk_init(spv) && coli_vk_available();
+ if (ok) fprintf(stderr, "[VK] %s: resident matrices on the GPU\n", engine);
+ else fprintf(stderr, "[VK] %s: no usable Vulkan device (shaders %s), running on the CPU\n",
+ engine, spv);
+ return ok;
+}
+
/* Fused first half of the expert MLP: hidden = silu(gate(x)) * up(x), computed in ONE
* dispatch that reads x once for both projections. gate/up are resident (uploaded on
* first call). D = input (hidden) dim, I = moe_inter. Returns 0 -> caller falls back. */
diff --git a/c/backend_vulkan.h b/c/backend_vulkan.h
index e673b37d..ed1fe983 100644
--- a/c/backend_vulkan.h
+++ b/c/backend_vulkan.h
@@ -133,6 +133,16 @@ size_t coli_vk_tensor_bytes(const ColiVkTensor *t);
* the host), 0 otherwise or when no device is selected. */
int coli_vk_device_integrated(void);
+/* For engines: COLI_VULKAN=1 opens the device with the shaders found by
+ * coli_vk_shader_path() and prints one line naming the engine; 0 (and one line) when
+ * Vulkan is not asked for or no device is usable, so the engine stays on the CPU. */
+int coli_vk_init_env(const char *engine);
+/* COLI_VK_SHADERS (the .spv or its directory), else shaders/ next to the binary, else
+ * shaders/ in the working directory. buf holds the result when it is not a literal. */
+const char *coli_vk_shader_path(char *buf, size_t n);
+/* How many coli_vk_matmul calls ran on the device: a check that a path is really used. */
+unsigned long long coli_vk_matmul_calls(void);
+
#ifdef __cplusplus
}
#endif