Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
mimo, qwenimage: the float dense matrices on Vulkan too (fmt 10, 11, 12)
JustVuggbe553ee725e8
diff --git a/c/mimo.c b/c/mimo.cindex 72749cf4..21193b7a 100644--- a/c/mimo.c+++ b/c/mimo.c@@ -65,10 +65,11 @@#include <immintrin.h>#endif#ifdef COLI_VULKAN-/* VK=1 and COLI_VULKAN=1: the int8 dense matrices (MIMO_DENSE_BITS=8) run on- * the GPU, and with MIMO_VK_EXPERTS=N up to N routed experts are kept there as- * MXFP4. Everything else, and every matrix whose upload fails, stays on the- * CPU path below. The backend has one command buffer: main thread only. */+/* VK=1 and COLI_VULKAN=1: the dense matrices of the trunk and of the vision+ * tower run on the GPU in whatever form MIMO_DENSE_BITS gave them (FP8, BF16,+ * int8 or f32), and with MIMO_VK_EXPERTS=N up to N routed experts are kept+ * there as MXFP4. The router, and every matrix whose upload fails, stays on+ * the CPU path below. The backend has one command buffer: main thread only. */#include "backend_vulkan.h"static int g_vk_ready;#endif@@ -257,7 +258,7 @@ typedef struct {int fmt, O, I, nblk;void *w; /* f32 / bf16 / e4m3 / int8 */float *s; /* FP8: [O][nblk]; I8: [O] */- void *vk; /* COLI_VULKAN: the device copy of an I8 matrix, uploaded at start-up */+ void *vk; /* COLI_VULKAN: the device copy, uploaded at start-up */int vk_off; /* its upload failed: this matrix stays on the CPU */} DW;@@ -314,9 +315,15 @@ static void matmul_fp8_rows(float *y, const float *x, const uint8_t *q8, const f}#ifdef COLI_VULKAN-/* The shader's fmt 1 is this layout exactly: int8 [O, I] and one f32 scale per- * row, applied after the row's sum, as matmul_q does. FP8 and BF16 have no- * shader format and stay on the CPU. */+/* Each form is one of the shader's formats as it is stored, so nothing is+ * converted on the way up:+ * DW_F32 fmt 10, f32 [O, I], no scales+ * DW_BF16 fmt 11, bf16 [O, I] (the low half of a word is the even column)+ * DW_FP8 fmt 12, e4m3 [O, I] with the per-(row, 128-column block) scales+ * rs[O][nblk] that dw_load_fp8 expanded: a group size of 128+ * DW_I8 fmt 1, int8 [O, I] and one f32 scale per row after the sum+ * Every one is f32 weights times f32 activations, as on the CPU; what differs+ * is the order of the sums (the CPU's FP8 kernel adds its blocks in double). */static int vk_main_thread(void) {#ifdef _OPENMPreturn !omp_in_parallel();@@ -325,12 +332,32 @@ static int vk_main_thread(void) {#endif}+static int dw_vk_fmt(const DW *d, int *gs, const float **sc) {+ *gs = 0; *sc = NULL;+ switch (d->fmt) {+ case DW_F32: return 10;+ case DW_BF16: return 11;+ case DW_FP8: *gs = 128; *sc = d->s; return 12;+ case DW_I8: *sc = d->s; return 1;+ }+ return -1;+}++static int dw_upload(DW *d) {+ int gs; const float *sc;+ int fmt = dw_vk_fmt(d, &gs, &sc);+ if (d->vk || d->vk_off || !d->w) return d->vk != NULL;+ if (fmt < 0 || !coli_vk_tensor_ensure((ColiVkTensor **)&d->vk, d->w, sc, fmt, d->I, d->O, gs)) d->vk_off = 1;+ return d->vk != NULL;+}+static int dw_matmul_vk(float *y, const float *x, int S, const DW *d) {- if (!g_vk_ready || d->fmt != DW_I8 || d->vk_off || !vk_main_thread()) return 0;+ if (!g_vk_ready || d->vk_off || !vk_main_thread()) return 0;DW *dev = (DW *)d; /* the device copy is a cache inside a read-only matrix */- if (coli_vk_matmul((ColiVkTensor **)&dev->vk, y, x, d->w, d->s, 1, S, d->I, d->O, 0)) return 1;- if (!dev->vk) dev->vk_off = 1;- return 0;+ if (!dw_upload(dev)) return 0;+ int gs; const float *sc;+ int fmt = dw_vk_fmt(d, &gs, &sc);+ return coli_vk_matmul((ColiVkTensor **)&dev->vk, y, x, d->w, sc, fmt, S, d->I, d->O, gs);}#endif@@ -880,24 +907,6 @@ static int g_vkx_full; /* an upload failed or the budget is reachestatic long g_vkx_resident;static unsigned long long g_vkx_calls;-/* The dense int8 matrices go up first, at start-up, so the experts take only- * what they leave; one that does not fit stays on the CPU. */-static void vk_dense_upload(Model *m) {- DW *all[5 * MIMO_MAX_LAYERS + 1];- int n = 0;- for (int li = 0; li < m->c.n_layers; li++) {- Layer *l = &m->L[li];- all[n++] = &l->qkv; all[n++] = &l->o;- if (!m->c.moe[li]) { all[n++] = &l->gate; all[n++] = &l->up; all[n++] = &l->down; }- }- all[n++] = &m->head;- for (int k = 0; k < n; k++) {- DW *d = all[k];- if (d->fmt == DW_I8 && !d->vk &&- !coli_vk_tensor_ensure((ColiVkTensor **)&d->vk, d->w, d->s, 1, d->I, d->O, 0)) d->vk_off = 1;- }-}-/* the device's own budget, when it reports one, with half a GB left over */static int vk_budget_full(void) {double used = 0, budget = 0;@@ -1200,6 +1209,29 @@ typedef struct Vision Vision;static Vision *g_vision;#include "mimo_vision.h"+#ifdef COLI_VULKAN+/* The dense matrices go up first, at start-up, the trunk before the tower, so+ * the experts take only what they leave; one that does not fit stays on the+ * CPU. Returns how many went up. */+static int vk_dense_upload(Model *m) {+ int n = 0;+ for (int li = 0; li < m->c.n_layers; li++) {+ Layer *l = &m->L[li];+ n += dw_upload(&l->qkv) + dw_upload(&l->o);+ if (!m->c.moe[li]) n += dw_upload(&l->gate) + dw_upload(&l->up) + dw_upload(&l->down);+ }+ n += dw_upload(&m->head);+ if (g_vision) {+ Vision *v = g_vision;+ n += dw_upload(&v->embed) + dw_upload(&v->fc1) + dw_upload(&v->fc2);+ for (int i = 0; i < v->depth; i++)+ n += dw_upload(&v->b[i].qkv) + dw_upload(&v->b[i].proj) + dw_upload(&v->b[i].gate) ++ dw_upload(&v->b[i].up) + dw_upload(&v->b[i].down);+ }+ return n;+}+#endif+/* --------------------------------------------------------------- sampling ---- */static int argmax(const float *x, int n) {@@ -1510,7 +1542,12 @@ int main(int argc, char **argv) {#ifdef COLI_VULKAN/* after the weights, so a missing device costs one line and nothing else */g_vk_ready = coli_vk_init_env("mimo");- if (g_vk_ready) vk_dense_upload(m);+ if (g_vk_ready) {+ int up = vk_dense_upload(m);+ size_t used = 0, count = 0;+ coli_vk_mem_info(&used, &count);+ fprintf(stderr, "[VK] mimo: %d dense matrices on the GPU (%.2f GB)\n", up, used / 1e9);+ }int vk_experts = env_int("MIMO_VK_EXPERTS", 0);if (g_vk_ready && vk_experts > 0) {g_vkx = xcalloc((size_t)c->n_layers * c->n_experts, sizeof(VkExpert), "GPU experts");diff --git a/c/qwenimage.c b/c/qwenimage.cindex c78e5f4c..7c37475e 100644--- a/c/qwenimage.c+++ b/c/qwenimage.c@@ -50,8 +50,8 @@#endif#endif#ifdef COLI_VULKAN-/* VK=1 and COLI_VULKAN=1: the DiT's int8 matrices run on the GPU, all the- * image tokens of a product in one call. One command buffer: main thread only. */+/* VK=1 and COLI_VULKAN=1: the DiT's matrices run on the GPU, all the image+ * tokens of a product in one call. One command buffer: main thread only. */#include "backend_vulkan.h"static int g_vk_ready, g_vk_tried;#endif@@ -165,19 +165,23 @@ static float *vec_load(shards *S, const char *name, int n){st_read_f32_cap(S, name, v, n, 1);return v;}-/* Y[M][N] = X . W^T on the GPU, for every row of X in one call. The shader's- * fmt 1 is QI_I8 as it is stored (int8 [N][K], one f32 scale per row), and the- * activations stay f32, so the result is the CPU's f32-activation product up to- * summation order. COLI_IMG_ACT8 is a CPU kernel: a matrix the GPU takes does- * not quantize its activations. 0 when the product stays on the CPU. */+/* Y[M][N] = X . W^T on the GPU, for every row of X in one call. Each storage+ * format is one of the shader's as it is stored: QI_I8 is fmt 1 (int8 [N][K],+ * one f32 scale per row), QI_BF16 fmt 11, QI_F32 fmt 10. The activations stay+ * f32, so the result is the CPU's f32-activation product up to summation+ * order. COLI_IMG_ACT8 is a CPU kernel: a matrix the GPU takes does not+ * quantize its activations. 0 when the product stays on the CPU. */static int qi_vk_linear(float *y, const float *x, int M, const Lin *l){#ifdef COLI_VULKAN- if (!g_vk_ready || !l->gpu || l->vk_off || l->m.fmt != QI_I8 || l->m.ld || M < 1) return 0;+ if (!g_vk_ready || !l->gpu || l->vk_off || l->m.ld || M < 1) return 0;#ifdef _OPENMPif (omp_in_parallel()) return 0;#endif+ int fmt = l->m.fmt == QI_I8 ? 1 : l->m.fmt == QI_BF16 ? 11 : l->m.fmt == QI_F32 ? 10 : -1;+ if (fmt < 0) return 0;Lin *dev = (Lin *)l; /* the device copy is a cache inside a read-only matrix */- if (coli_vk_matmul((ColiVkTensor **)&dev->vk, y, x, l->m.w, l->m.sc, 1, M, l->m.K, l->m.N, 0)) return 1;+ if (coli_vk_matmul((ColiVkTensor **)&dev->vk, y, x, l->m.w, fmt == 1 ? l->m.sc : NULL, fmt,+ M, l->m.K, l->m.N, 0)) return 1;if (!dev->vk) dev->vk_off = 1;#endif(void)y; (void)x; (void)M; (void)l;@@ -544,7 +548,8 @@ static void dit_load(Dit *d, const char *model){lin_bytes(&B->gate) + lin_bytes(&B->proj) + lin_bytes(&B->out);}st_destroy(&S);- d->mod.gpu = d->txt1.gpu = d->txt2.gpu = 1;+ d->img_in.gpu = d->t1.gpu = d->t2.gpu = d->mod.gpu = d->norm_out.gpu = d->proj_out.gpu = 1;+ d->txt1.gpu = d->txt2.gpu = 1;d->loaded = 1;fprintf(stderr, "[qwenimage] transformer: %d blocks, %.2f GB resident, %.1f s\n",d->layers, (bytes + lin_bytes(&d->mod) + lin_bytes(&d->txt1) + lin_bytes(&d->txt2)) / 1e9,@@ -568,9 +573,9 @@ static void dit_temb(Dit *d, float t, float *temb){e[i] = cosf(a); e[half + i] = sinf(a);}float *h = fmalloc(d->dim);- qi_gemm(h, e, 1, &d->t1.m, NULL);+ if (!qi_vk_linear(h, e, 1, &d->t1)) qi_gemm(h, e, 1, &d->t1.m, NULL);for (int i = 0; i < d->dim; i++) h[i] = silu(h[i]);- qi_gemm(temb, h, 1, &d->t2.m, NULL);+ if (!qi_vk_linear(temb, h, 1, &d->t2)) qi_gemm(temb, h, 1, &d->t2.m, NULL);free(h);}/* mod[4*dim] = [scale1 | gate1 | scale2 | gate2], outs[dim] = the final norm's scale */@@ -579,7 +584,7 @@ static void dit_modulation(Dit *d, float t, float *mod, float *outs){dit_temb(d, t, temb);for (int i = 0; i < d->dim; i++) s[i] = silu(temb[i]);if (!qi_vk_linear(mod, s, 1, &d->mod)) qi_gemm(mod, s, 1, &d->mod.m, NULL);- qi_gemm(outs, s, 1, &d->norm_out.m, NULL);+ if (!qi_vk_linear(outs, s, 1, &d->norm_out)) qi_gemm(outs, s, 1, &d->norm_out.m, NULL);free(temb); free(s);}diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.mdindex cd68865c..7d8d1a15 100644--- a/docs/ENVIRONMENT.md+++ b/docs/ENVIRONMENT.md@@ -512,7 +512,7 @@ Read **only** by `c/mimo.c` (and `MIMO_MAX_IMAGE_TOKENS` by the gateway). See [m| `MIMO_TRACE` | unset | MiMo, oracle: dump the residual after every sublayer of the first block. || `MIMO_DIRS` | unset | MiMo: extra directories holding shards. || `MIMO_STATS` | unset | MiMo: report the vision tower's time per picture. |-| `COLI_VULKAN` | `0` | MiMo, `VK=1` build: the int8 dense matrices (`MIMO_DENSE_BITS=8`) run on the GPU. FP8, BF16, f32 and the vision tower stay on the CPU. |+| `COLI_VULKAN` | `0` | MiMo, `VK=1` build: the dense matrices of the trunk and of the vision tower run on the GPU in their `MIMO_DENSE_BITS` form (FP8 and BF16, int8 or f32). The router stays on the CPU. || `MIMO_VK_EXPERTS` | `0` | MiMo, with `COLI_VULKAN=1`: keep up to this many routed experts (MXFP4) on the GPU once they have run from the RAM cache; never evicted, capped by the device's memory budget. |## OLMoE engine (`olmoe`)diff --git a/docs/mimo.md b/docs/mimo.mdindex 5d0a9076..741e7f5b 100644--- a/docs/mimo.md+++ b/docs/mimo.md@@ -221,5 +221,5 @@ picture's path at the start of the line, and `coli web` with the upload button:| `MIMO_LOGITS` | unset | Oracle: dump every prompt position's logits (f32) to this file. || `MIMO_TRACE` | unset | Oracle: dump the residual after every sublayer of the first block. || `MIMO_DIRS` | unset | Extra directories holding shards (multi-disk). |-| `COLI_VULKAN` | 0 | 1, in a `make mimo VK=1` build: the int8 dense matrices (`MIMO_DENSE_BITS=8`: qkv, o_proj, the dense MLP, lm_head) are uploaded at start-up and run on the GPU. FP8, BF16 and f32 have no shader format and stay on the CPU, and so does the vision tower, which is BF16 in every configuration but the oracle's f32. |+| `COLI_VULKAN` | 0 | 1, in a `make mimo VK=1` build: the dense matrices (qkv, o_proj, the dense MLP, lm_head, and the vision tower's) are uploaded at start-up and run on the GPU in the form `MIMO_DENSE_BITS` gave them: FP8 with its 128-column block scales and BF16 by default, int8 with 8, f32 with 32. The router stays on the CPU, so the experts chosen and read from disk are the CPU's. A matrix that does not fit stays on the CPU. || `MIMO_VK_EXPERTS` | 0 | With `COLI_VULKAN=1`: up to this many routed experts are copied to the GPU, as the release's MXFP4, the first time they run from the RAM cache, and stay there for the rest of the run; uploads stop at this count or when the device's memory budget (less half a GB) is reached. 0 keeps every expert on the CPU. |diff --git a/docs/qwen-image.md b/docs/qwen-image.mdindex 84cf5bf3..bb3d0665 100644--- a/docs/qwen-image.md+++ b/docs/qwen-image.md@@ -260,14 +260,15 @@ Measured on a server with 8 Zen 4 cores (AVX-512, 8 OpenMP threads), 768x512,scores 30.1 dB with them and 35.6 dB without, and by eye the two cannot betold apart.-On a GPU, a `make qwenimage VK=1` build with `COLI_VULKAN=1` runs the-transformer's int8 matrices (the default `COLI_IMG_BITS=8`) through the shared-Vulkan backend, every image token of a product in one call. The activations+On a GPU, a `make qwenimage VK=1` build with `COLI_VULKAN=1` runs every+matrix of the transformer through the shared Vulkan backend, in the storage+`COLI_IMG_BITS` gave it (int8 by default, bf16 or f32) and with every image+token of a product in one call: the blocks, the prompt's input layers, the+timestep embedding, the modulation and the output layers. The activationsstay f32 there, so `COLI_IMG_ACT8` does not apply to those matrices and theresult is the CPU's `COLI_IMG_ACT8=0` run up to summation order. The text-encoder, the VAE, the f32 timestep and output layers, attention and anything-stored in bf16 or f32 stay on the CPU. The run ends with one line saying how-many products the GPU took. No GPU timing has been measured yet: the path has+encoder (run once per prompt), the VAE and attention stay on the CPU. The+run ends with one line saying how many products the GPU took. No GPU timing has been measured yet: the path hasbeen checked on Lavapipe, a software driver, which tests the numbers and notthe speed.