Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
vulkan: one entry point for every engine, and the engines link the backend
JustVugg703facb60dea
diff --git a/c/Makefile b/c/Makefileindex 21c5988f..4168f64b 100644--- a/c/Makefile+++ b/c/Makefile@@ -1181,8 +1181,8 @@ tests/bench_cuda_resident_batch$(EXE): tests/bench_cuda_resident_batch.cu backen# the GPU sits idle. Same guard #783 put on kimi_k3, which since gaining an# MXFP4 expert path no longer needs it. tests/test_makefile_cuda_scope.py# asserts this shape.-olmoe$(EXE): olmoe.c- $(CC) $(NOCUDA_CFLAGS) olmoe.c -o olmoe$(EXE) $(NOCUDA_LDFLAGS)+olmoe$(EXE): olmoe.c $(VK_OBJ) $(VK_SPV)+ $(CC) $(NOCUDA_CFLAGS) olmoe.c $(VK_OBJ) -o olmoe$(EXE) $(NOCUDA_LDFLAGS)# Qwen3.6-35B-A3B engine (hybrid Gated Attention + Gated DeltaNet + streaming# MoE). With CUDA=1 the optional VRAM expert tier (qwen36_tier.c) is compiled@@ -1236,8 +1236,8 @@ qpack.o: qpack.c .build-config$(CC) $(QWEN36_CFLAGS) -c qpack.c -o $@# Rebuild when CUDA_DLL changes; otherwise an existing CPU-only executable can# be reported as up to date despite selecting the Windows CUDA DLL tier.-qwen36$(EXE): qwen36.c $(QWEN36_TIER_OBJ) $(QWEN36_QPACK_OBJ) $(CUDA_OBJ) $(METAL_OBJ) .build-config- $(CC) $(QWEN36_CFLAGS) qwen36.c $(QWEN36_TIER_OBJ) $(QWEN36_QPACK_OBJ) $(CUDA_OBJ) $(METAL_OBJ) -o qwen36$(EXE) $(QWEN36_LDFLAGS)+qwen36$(EXE): qwen36.c $(QWEN36_TIER_OBJ) $(QWEN36_QPACK_OBJ) $(CUDA_OBJ) $(METAL_OBJ) $(VK_OBJ) $(VK_SPV) .build-config+ $(CC) $(QWEN36_CFLAGS) qwen36.c $(QWEN36_TIER_OBJ) $(QWEN36_QPACK_OBJ) $(CUDA_OBJ) $(METAL_OBJ) $(VK_OBJ) -o qwen36$(EXE) $(QWEN36_LDFLAGS)# Qwen-Image-2.1 text-to-image (docs/qwen-image.md). It reads the diffusers# checkpoint as it is: no conversion step. qi_gemm.h is its one matrix kernel.@@ -1245,8 +1245,8 @@ ifneq ($(EXE),).PHONY: qwenimageqwenimage: qwenimage$(EXE)endif-qwenimage$(EXE): qwenimage.c .build-config- $(CC) $(CFLAGS) qwenimage.c -o qwenimage$(EXE) $(LDFLAGS)+qwenimage$(EXE): qwenimage.c $(VK_OBJ) $(VK_SPV) .build-config+ $(CC) $(CFLAGS) qwenimage.c $(VK_OBJ) -o qwenimage$(EXE) $(LDFLAGS)# In make check: every format and ragged edges against the naive product, and# the int8 x int8 VNNI path against the exact integer sum.@@ -1256,8 +1256,8 @@ tests/test_qi_gemm$(EXE): tests/test_qi_gemm.c# DeepSeek V4.1 Flash: one file, like every other portable engine. The fp4 experts# stream from the official checkpoint and quant.h's mxfp4 kernel reads them as they# are, so there is no conversion target here to go with it.-deepseek_v41$(EXE): deepseek_v41.c .build-config- $(CC) $(CFLAGS) deepseek_v41.c -o deepseek_v41$(EXE) $(LDFLAGS)+deepseek_v41$(EXE): deepseek_v41.c $(VK_OBJ) $(VK_SPV) .build-config+ $(CC) $(CFLAGS) deepseek_v41.c $(VK_OBJ) -o deepseek_v41$(EXE) $(LDFLAGS)# Xiaomi MiMo-V2.6 (Flash 309B, Pro 1.02T): one file plus its ViT header, reading the# release as it is -- MXFP4 experts streamed per layer, FP8/BF16 dense resident.@@ -1265,8 +1265,8 @@ ifneq ($(EXE),).PHONY: mimomimo: mimo$(EXE)endif-mimo$(EXE): mimo.c .build-config- $(CC) $(CFLAGS) mimo.c -o mimo$(EXE) $(LDFLAGS)+mimo$(EXE): mimo.c $(VK_OBJ) $(VK_SPV) .build-config+ $(CC) $(CFLAGS) mimo.c $(VK_OBJ) -o mimo$(EXE) $(LDFLAGS)# Tiny MiMo fixture + vendor oracle, the Kimi K3 pattern: the generator is numpy (and# Pillow for the test picture) and deterministic; ref.json is committed and comes from@@ -1283,8 +1283,8 @@ mimo-tiny-check: mimo-tiny-generate mimo$(EXE)# loader and SERVE=1 protocol. With CUDA=1 it links the same expert tier as# qwen36 (fp8 streaming mode: hot experts get VRAM copies, the RAM LRU stays);# without it the tier header's inline stubs keep the build toolkit-free.-qwen38$(EXE): qwen38.c $(QWEN36_TIER_OBJ) $(CUDA_OBJ)- $(CC) $(QWEN36_CFLAGS) qwen38.c $(QWEN36_TIER_OBJ) $(CUDA_OBJ) -o qwen38$(EXE) $(QWEN36_LDFLAGS)+qwen38$(EXE): qwen38.c $(QWEN36_TIER_OBJ) $(CUDA_OBJ) $(VK_OBJ) $(VK_SPV)+ $(CC) $(QWEN36_CFLAGS) qwen38.c $(QWEN36_TIER_OBJ) $(CUDA_OBJ) $(VK_OBJ) -o qwen38$(EXE) $(QWEN36_LDFLAGS).PHONY: qwen38-tiny-generate qwen38-tiny-checkqwen38-tiny-generate:@@ -1404,8 +1404,8 @@ tests/bench_qwen36_dense_batch$(EXE): tests/bench_qwen36_dense_batch.c qwen36.c$(QWEN36_TIER_OBJ) $(CUDA_OBJ)$(CC) $(QWEN36_CFLAGS) $< $(QWEN36_TIER_OBJ) $(CUDA_OBJ) -o $@ $(QWEN36_LDFLAGS)-inkling$(EXE): inkling.c $(INK_CUDA_OBJ) $(METAL_OBJ)- $(CC) $(CFLAGS) inkling.c $(INK_CUDA_OBJ) $(METAL_OBJ) -o inkling$(EXE) $(LDFLAGS)+inkling$(EXE): inkling.c $(INK_CUDA_OBJ) $(METAL_OBJ) $(VK_OBJ) $(VK_SPV)+ $(CC) $(CFLAGS) inkling.c $(INK_CUDA_OBJ) $(METAL_OBJ) $(VK_OBJ) -o inkling$(EXE) $(LDFLAGS)# ENGINES WITHOUT A CUDA BACKEND (#783).# kimi_k3.c contains no COLI_CUDA code at all — its GPU path is Vulkan. Butdiff --git a/c/backend_vulkan.c b/c/backend_vulkan.cindex 9b9d913c..33346eee 100644--- a/c/backend_vulkan.c+++ b/c/backend_vulkan.c@@ -13,6 +13,10 @@#include <stdlib.h>#include <string.h>#include <time.h>+#include <sys/stat.h>+#ifdef __linux__+#include <unistd.h>+#endifstatic double vk_now(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec*1000.0 + t.tv_nsec/1e6; }#define VKCHECK(x, what) do { VkResult _r = (x); if (_r != VK_SUCCESS) { \@@ -584,6 +588,8 @@ static void vkprof_tick(void) {fprintf(stderr, "[VK_PROF sub] n=%ld | submit %.0f | wait %.0f ms\n", g_vsub_n, g_vsub_ms, g_vwait_ms);}+static unsigned long long g_vk_matmul_calls; /* successful coli_vk_matmul calls */+int coli_vk_matmul(ColiVkTensor **tensor, float *y, const float *x,const void *weights, const float *scales,int fmt, int S, int I, int O, int gs) {@@ -661,9 +667,48 @@ int coli_vk_matmul(ColiVkTensor **tensor, float *y, const float *x,fprintf(stderr, "[VK_PROF dense] n=%ld | memcpy_x %.0f | desc %.0f | record %.0f | submit %.0f | wait %.0f | memcpy_y %.0f ms\n",p_n, p_x, p_desc, p_rec, p_sub, p_wait, p_y);}+ g_vk_matmul_calls++;return 1;}+unsigned long long coli_vk_matmul_calls(void) { return g_vk_matmul_calls; }++/* The shader path every engine resolves the same way (#523): COLI_VK_SHADERS may be the+ * qmatmul.spv file or the directory holding it; unset, the shaders/ directory next to the+ * binary (the build layout), then the historical path relative to the working directory. */+const char *coli_vk_shader_path(char *buf, size_t n) {+ const char *env = getenv("COLI_VK_SHADERS");+ struct stat st;+ if (env && *env) {+ if (!stat(env, &st) && S_ISDIR(st.st_mode)) { snprintf(buf, n, "%s/qmatmul.spv", env); return buf; }+ return env;+ }+#ifdef __linux__+ ssize_t k = readlink("/proc/self/exe", buf, n - 1);+ if (k > 0) {+ buf[k] = 0;+ char *sl = strrchr(buf, '/');+ if (sl && (size_t)(sl + 1 - buf) + sizeof("shaders/qmatmul.spv") <= n) {+ strcpy(sl + 1, "shaders/qmatmul.spv");+ if (!stat(buf, &st)) return buf;+ }+ }+#endif+ return "shaders/qmatmul.spv";+}++int coli_vk_init_env(const char *engine) {+ const char *on = getenv("COLI_VULKAN");+ if (!on || !atoi(on)) return 0;+ char buf[1024];+ const char *spv = coli_vk_shader_path(buf, sizeof buf);+ int ok = coli_vk_init(spv) && coli_vk_available();+ if (ok) fprintf(stderr, "[VK] %s: resident matrices on the GPU\n", engine);+ else fprintf(stderr, "[VK] %s: no usable Vulkan device (shaders %s), running on the CPU\n",+ engine, spv);+ return ok;+}+/* Fused first half of the expert MLP: hidden = silu(gate(x)) * up(x), computed in ONE* dispatch that reads x once for both projections. gate/up are resident (uploaded on* first call). D = input (hidden) dim, I = moe_inter. Returns 0 -> caller falls back. */diff --git a/c/backend_vulkan.h b/c/backend_vulkan.hindex e673b37d..ed1fe983 100644--- a/c/backend_vulkan.h+++ b/c/backend_vulkan.h@@ -133,6 +133,16 @@ size_t coli_vk_tensor_bytes(const ColiVkTensor *t);* the host), 0 otherwise or when no device is selected. */int coli_vk_device_integrated(void);+/* For engines: COLI_VULKAN=1 opens the device with the shaders found by+ * coli_vk_shader_path() and prints one line naming the engine; 0 (and one line) when+ * Vulkan is not asked for or no device is usable, so the engine stays on the CPU. */+int coli_vk_init_env(const char *engine);+/* COLI_VK_SHADERS (the .spv or its directory), else shaders/ next to the binary, else+ * shaders/ in the working directory. buf holds the result when it is not a literal. */+const char *coli_vk_shader_path(char *buf, size_t n);+/* How many coli_vk_matmul calls ran on the device: a check that a path is really used. */+unsigned long long coli_vk_matmul_calls(void);+#ifdef __cplusplus}#endif