Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
glm: VK dense offload (COLI_VK_DENSE) — attention projections + shared expert on Vulkan
Steve Markgraf7c2f45f93cb6
diff --git a/c/colibri.c b/c/colibri.cindex 06148ef5..711783f9 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -293,6 +293,8 @@ static uint64_t g_last_repin;static int g_vulkan; /* COLI_VULKAN=1: compute routed experts on the Vulkan tier */static int g_vk_budget; /* COLI_VK_EXPERTS: max experts uploaded to VK VRAM (default 1024) */static int g_vk_resident; /* how many experts currently VK-resident */+static int g_vk_dense; /* COLI_VK_DENSE=1: run the resident dense matmuls (attention+ * projections + shared expert) on Vulkan too */#endif#ifdef COLI_CUDAstatic int g_cuda_enabled;@@ -333,6 +335,14 @@ static void qt_vk_reset(QT *t){if(t->vk){ coli_vk_tensor_free(t->vk); t->vk=NULL; }t->vk_eligible=0;}+/* Dense matmul on the Vulkan tier: y[S,O] = x[S,I] @ dequant(t)^T. Uploads the resident+ * int4/int8 weight once (t->vk), then reuses it. Returns 0 (caller runs the CPU matmul)+ * when VK-dense is off, the format is unsupported, or upload/compute fails. */+static int vk_matmul_qt(QT *t, float *y, const float *x, int S){+ if(!g_vk_dense || (t->fmt!=1 && t->fmt!=2)) return 0;+ const void *w = t->fmt==1 ? (const void*)t->q8 : (const void*)t->q4;+ return coli_vk_matmul(&t->vk, y, x, w, t->s, t->fmt, S, t->I, t->O);+}#endif#ifdef COLI_CUDAstatic int qt_cuda_upload(QT *t){@@ -2496,10 +2506,19 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int ppipe_done=attn_pipe_prefill(m,l,layer,x,0,S,pos_base,out,NULL);#endifif(!pipe_done){+#ifdef COLI_VULKAN+ if(!vk_matmul_qt(&l->q_a, QR, x, S))+#endifmatmul_qt_ex(QR, x, &l->q_a, S, 0);for(int s=0;s<S;s++){ float *qr=QR+(int64_t)s*c->q_lora;rmsnorm(qr, qr, l->q_a_ln, c->q_lora, c->eps); } /* q_b legge il residuo NORMATO */+#ifdef COLI_VULKAN+ if(!vk_matmul_qt(&l->q_b, Q, QR, S))+#endifmatmul_qt_ex(Q, QR, &l->q_b, S, 0);+#ifdef COLI_VULKAN+ if(!vk_matmul_qt(&l->kv_a, comp, x, S))+#endifmatmul_qt_ex(comp, x, &l->kv_a, S, 0);}if(!pipe_done) for(int s=0;s<S;s++){@@ -2783,7 +2802,12 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p}}m->t_acore+=now_s()-tac; double tao=now_s();- if(!cuda_projected){matmul_qt(out, ctx, &l->o, S);} m->t_aout+=now_s()-tao;+ if(!cuda_projected){+#ifdef COLI_VULKAN+ if(!vk_matmul_qt(&l->o, out, ctx, S))+#endif+ matmul_qt(out, ctx, &l->o, S);+ } m->t_aout+=now_s()-tao;free(ctx); free(Q); free(QR); free(comp); free(sc_all);m->t_attn += now_s()-ta0;return;@@ -3800,9 +3824,18 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int#endifif(!shared_cuda){sg=falloc((int64_t)S*sI);su=falloc((int64_t)S*sI);+#ifdef COLI_VULKAN+ if(!vk_matmul_qt(&l->sh_gate, sg, x, S))+#endifmatmul_qt(sg, x, &l->sh_gate, S);+#ifdef COLI_VULKAN+ if(!vk_matmul_qt(&l->sh_up, su, x, S))+#endifmatmul_qt(su, x, &l->sh_up, S);for(int64_t z=0;z<(int64_t)S*sI;z++) sg[z]=siluf(sg[z])*su[z];+#ifdef COLI_VULKAN+ if(!vk_matmul_qt(&l->sh_down, hh, sg, S))+#endifmatmul_qt(hh, sg, &l->sh_down, S);}if(shared_cuda!=2) for(int64_t z=0;z<(int64_t)S*D;z++) out[z]+=hh[z];@@ -7155,7 +7188,9 @@ int main(int argc, char **argv){g_vulkan = coli_vk_init(spv);if(!g_vulkan){ fprintf(stderr,"[VK] Vulkan backend unavailable (need libvulkan + shaders/qmatmul{,_gate_up}.spv)\n"); return 2; }g_vk_budget = getenv("COLI_VK_EXPERTS") ? atoi(getenv("COLI_VK_EXPERTS")) : 1024;- fprintf(stderr,"[VK] expert tier active: routed int4 experts on the GPU (budget %d)\n", g_vk_budget);+ g_vk_dense = getenv("COLI_VK_DENSE") ? atoi(getenv("COLI_VK_DENSE")) : 0;+ fprintf(stderr,"[VK] expert tier active: routed int4 experts on the GPU (budget %d)%s\n",+ g_vk_budget, g_vk_dense ? " + dense projections + shared expert" : "");}#endif#ifdef COLI_CUDA