Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
vulkan: fmt=7 MXFP4 decode (e2m1 nibbles + per-32 group scales) for Kimi K3 experts
Steve Markgrafd4bf04139fa6
diff --git a/c/backend_vulkan.c b/c/backend_vulkan.cindex 9d22d2df..d779607d 100644--- a/c/backend_vulkan.c+++ b/c/backend_vulkan.c@@ -211,7 +211,8 @@ static int rowwords(int fmt, int I) {* (one f32 per 64-input group). upload_tensor and tensor_free must agree on this. */static size_t scale_floats(int fmt, int I, int O, int gs) {if (fmt == 5) return (size_t)O * (((size_t)I + 63) / 64);- if (fmt == 4) return (size_t)O * (((size_t)I + gs - 1) / gs); // per-group [O,ng]+ if (fmt == 4 || fmt == 7)+ return (size_t)O * (((size_t)I + gs - 1) / gs); // per-group [O,ng]return (size_t)O;}@@ -506,11 +507,11 @@ static int arena_suballoc(size_t bytes, VkBuffer *buf, void **ptr) {static int upload_tensor(ColiVkTensor **out, const void *weights, const float *scales,int fmt, int I, int O, int gs) {if (*out) return (*out)->fmt == fmt && (*out)->I == I && (*out)->O == O;- if (fmt != 1 && fmt != 2 && fmt != 5 &&- !(fmt == 4 && gs >= 8 && gs % 8 == 0)) return 0; /* fmt=4: word-aligned groups only */+ if (fmt != 1 && fmt != 2 && fmt != 5 && /* fmt=4/7: word-aligned groups only */+ !((fmt == 4 || fmt == 7) && gs >= 8 && gs % 8 == 0)) return 0;ColiVkTensor *t = calloc(1, sizeof(*t));if (!t) return 0;- t->fmt = fmt; t->I = I; t->O = O; t->rowWords = rowwords(fmt, I); t->gs = fmt == 4 ? gs : 0;+ t->fmt = fmt; t->I = I; t->O = O; t->rowWords = rowwords(fmt, I); t->gs = (fmt == 4 || fmt == 7) ? gs : 0;size_t stride = (size_t)t->rowWords * 4; // padded row bytessize_t cpu_rb = fmt == 1 ? (size_t)I: fmt == 5 ? ((size_t)I + 63) / 64 * 24 : (size_t)(I + 1) / 2;@@ -923,7 +924,7 @@ static int upload_tensor_d2(ColiVkTensor **out, const void *weights, const float!(fmt == 4 && gs >= 8 && gs % 8 == 0)) return 0;ColiVkTensor *t = calloc(1, sizeof(*t));if (!t) return 0;- t->fmt = fmt; t->I = I; t->O = O; t->rowWords = rowwords(fmt, I); t->gs = fmt == 4 ? gs : 0;+ t->fmt = fmt; t->I = I; t->O = O; t->rowWords = rowwords(fmt, I); t->gs = (fmt == 4 || fmt == 7) ? gs : 0;t->dev = 1;size_t stride = (size_t)t->rowWords * 4;size_t cpu_rb = fmt == 1 ? (size_t)Idiff --git a/c/shaders/qmatmul.comp b/c/shaders/qmatmul.compindex a4220585..25de4f20 100644--- a/c/shaders/qmatmul.comp+++ b/c/shaders/qmatmul.comp@@ -46,6 +46,15 @@ float i8(uint word, int lane) {float i4(uint word, int lane) {return float(int((word >> (uint(lane) * 4u)) & 0xfu) - 8);}+// fmt=7 MXFP4 (Kimi K3 experts): e2m1 nibble, bit3 = sign, LOW nibble = even+// column (same packing order as i4). Scales arrive as f32 (host pre-expands+// the ue8m0 exponents), one per 32-input group.+float mx4(uint word, int lane) {+ uint n = (word >> (uint(lane) * 4u)) & 0xfu;+ const float lut[8] = float[8](0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0);+ float v = lut[n & 7u];+ return (n & 8u) != 0u ? -v : v;+}void main() {int s = int(gl_WorkGroupID.y);@@ -97,16 +106,22 @@ void main() {}sum += a * scale[sb + uint(g)];}- } else if (p.fmt == 4) { // grouped int4 (#298 semantics): nibble decode, one scale- // per gs inputs β gs % 8 == 0 (host-gated), so a packed word never straddles- // a group and its scale multiplies the 8-wide partial. Per-row scale must NOT run.+ } else if (p.fmt == 4 || p.fmt == 7) { // grouped int4 (#298 semantics) or MXFP4:+ // nibble decode, one scale per gs inputs β gs % 8 == 0 (host-gated), so a+ // packed word never straddles a group and its scale multiplies the 8-wide+ // partial. Per-row scale must NOT run.int words = (p.I + 7) / 8, ng = (p.I + p.gs - 1) / p.gs;uint sb = uint(o) * uint(ng);for (int wi = lane; wi < words; wi += sgsize) {uint pk = w[rowBase + uint(wi)]; int i0 = wi * 8;float a = 0.0;- for (int k = 0; k < 8; k++) { int i = i0 + k;- if (i < p.I) a += (staged ? xsh[i] : x[xoff + i]) * i4(pk, k); }+ if (p.fmt == 4) {+ for (int k = 0; k < 8; k++) { int i = i0 + k;+ if (i < p.I) a += (staged ? xsh[i] : x[xoff + i]) * i4(pk, k); }+ } else {+ for (int k = 0; k < 8; k++) { int i = i0 + k;+ if (i < p.I) a += (staged ? xsh[i] : x[xoff + i]) * mx4(pk, k); }+ }sum += a * scale[sb + uint(i0 / p.gs)];}} else { // int4@@ -120,6 +135,6 @@ void main() {}}float tot = subgroupAdd(sum);- if (lane == 0) y[s * p.O + o] = (p.fmt == 5 || p.fmt == 4) ? tot : tot * scale[o];+ if (lane == 0) y[s * p.O + o] = (p.fmt == 5 || p.fmt == 4 || p.fmt == 7) ? tot : tot * scale[o];}}diff --git a/c/shaders/qmatmul.spv b/c/shaders/qmatmul.spvnew file mode 100644index 00000000..f6488614Binary files /dev/null and b/c/shaders/qmatmul.spv differdiff --git a/c/tests/test_vk_mxfp4.c b/c/tests/test_vk_mxfp4.cnew file mode 100644index 00000000..e5faf176--- /dev/null+++ b/c/tests/test_vk_mxfp4.c@@ -0,0 +1,43 @@+/* fmt=7 (MXFP4) Vulkan matmul vs the CPU reference kernel (quant.h).+ * Random e2m1 nibbles + ue8m0 group exponents, K3 expert dims.+ * Skips (exit 0) when no Vulkan device is available. */+#include <stdio.h>+#include <stdlib.h>+#include <math.h>+#include <stdint.h>+#include "../quant.h"+#include "../backend_vulkan.h"++int main(void){+ if(!coli_vk_init("shaders/qmatmul.spv")||!coli_vk_available()){+ fprintf(stderr,"vk-mxfp4: no Vulkan device β skipped\n"); return 0;+ }+ srand(7);+ int S=2, I=3584, O=3072; /* K3 expert w1/w3 shape */+ int rb=(I+1)/2, ng=(I+31)/32;+ uint8_t *q4=malloc((size_t)O*rb), *e8=malloc((size_t)O*ng);+ float *x=malloc((size_t)S*I*sizeof(float));+ float *yc=calloc((size_t)S*O,sizeof(float)), *yg=calloc((size_t)S*O,sizeof(float));+ float *sc=malloc((size_t)O*ng*sizeof(float));+ for(size_t i=0;i<(size_t)O*rb;i++) q4[i]=(uint8_t)rand();+ for(size_t i=0;i<(size_t)O*ng;i++){ e8[i]=(uint8_t)(120+rand()%12); sc[i]=mx4_scale(e8[i]); }+ for(int i=0;i<S*I;i++) x[i]=(float)(rand()%2001-1000)/500.f;++ matmul_mxfp4(yc,x,q4,e8,S,I,O);++ ColiVkTensor *t=NULL;+ if(!coli_vk_matmul(&t,yg,x,q4,sc,7,S,I,O,32)){+ fprintf(stderr,"vk-mxfp4: FAIL coli_vk_matmul fmt=7 refused\n"); return 1;+ }+ double num=0,den=0,mx=0;+ for(int i=0;i<S*O;i++){+ double d=yg[i]-yc[i]; num+=d*d; den+=(double)yc[i]*yc[i];+ double r=fabs(d)/(fabs(yc[i])+1e-6); if(r>mx)mx=r;+ }+ double rel=sqrt(num/(den+1e-30));+ printf("vk-mxfp4: rel_l2 %.3e max_rel %.3e (S=%d I=%d O=%d)\n",rel,mx,S,I,O);+ if(rel>1e-5){ fprintf(stderr,"vk-mxfp4: FAIL rel_l2 %.3e\n",rel); return 1; }+ printf("vk-mxfp4: OK\n");+ coli_vk_shutdown();+ return 0;+}