Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

vulkan: fmt=7 MXFP4 decode (e2m1 nibbles + per-32 group scales) for Kimi K3 experts

Steve Markgrafd4bf04139fa6
diff --git a/c/backend_vulkan.c b/c/backend_vulkan.c
index 9d22d2df..d779607d 100644
--- a/c/backend_vulkan.c
+++ b/c/backend_vulkan.c
@@ -211,7 +211,8 @@ static int rowwords(int fmt, int I) {
* (one f32 per 64-input group). upload_tensor and tensor_free must agree on this. */
static size_t scale_floats(int fmt, int I, int O, int gs) {
if (fmt == 5) return (size_t)O * (((size_t)I + 63) / 64);
- if (fmt == 4) return (size_t)O * (((size_t)I + gs - 1) / gs); // per-group [O,ng]
+ if (fmt == 4 || fmt == 7)
+ return (size_t)O * (((size_t)I + gs - 1) / gs); // per-group [O,ng]
return (size_t)O;
}
@@ -506,11 +507,11 @@ static int arena_suballoc(size_t bytes, VkBuffer *buf, void **ptr) {
static int upload_tensor(ColiVkTensor **out, const void *weights, const float *scales,
int fmt, int I, int O, int gs) {
if (*out) return (*out)->fmt == fmt && (*out)->I == I && (*out)->O == O;
- if (fmt != 1 && fmt != 2 && fmt != 5 &&
- !(fmt == 4 && gs >= 8 && gs % 8 == 0)) return 0; /* fmt=4: word-aligned groups only */
+ if (fmt != 1 && fmt != 2 && fmt != 5 && /* fmt=4/7: word-aligned groups only */
+ !((fmt == 4 || fmt == 7) && gs >= 8 && gs % 8 == 0)) return 0;
ColiVkTensor *t = calloc(1, sizeof(*t));
if (!t) return 0;
- t->fmt = fmt; t->I = I; t->O = O; t->rowWords = rowwords(fmt, I); t->gs = fmt == 4 ? gs : 0;
+ t->fmt = fmt; t->I = I; t->O = O; t->rowWords = rowwords(fmt, I); t->gs = (fmt == 4 || fmt == 7) ? gs : 0;
size_t stride = (size_t)t->rowWords * 4; // padded row bytes
size_t cpu_rb = fmt == 1 ? (size_t)I
: fmt == 5 ? ((size_t)I + 63) / 64 * 24 : (size_t)(I + 1) / 2;
@@ -923,7 +924,7 @@ static int upload_tensor_d2(ColiVkTensor **out, const void *weights, const float
!(fmt == 4 && gs >= 8 && gs % 8 == 0)) return 0;
ColiVkTensor *t = calloc(1, sizeof(*t));
if (!t) return 0;
- t->fmt = fmt; t->I = I; t->O = O; t->rowWords = rowwords(fmt, I); t->gs = fmt == 4 ? gs : 0;
+ t->fmt = fmt; t->I = I; t->O = O; t->rowWords = rowwords(fmt, I); t->gs = (fmt == 4 || fmt == 7) ? gs : 0;
t->dev = 1;
size_t stride = (size_t)t->rowWords * 4;
size_t cpu_rb = fmt == 1 ? (size_t)I
diff --git a/c/shaders/qmatmul.comp b/c/shaders/qmatmul.comp
index a4220585..25de4f20 100644
--- a/c/shaders/qmatmul.comp
+++ b/c/shaders/qmatmul.comp
@@ -46,6 +46,15 @@ float i8(uint word, int lane) {
float i4(uint word, int lane) {
return float(int((word >> (uint(lane) * 4u)) & 0xfu) - 8);
}
+// fmt=7 MXFP4 (Kimi K3 experts): e2m1 nibble, bit3 = sign, LOW nibble = even
+// column (same packing order as i4). Scales arrive as f32 (host pre-expands
+// the ue8m0 exponents), one per 32-input group.
+float mx4(uint word, int lane) {
+ uint n = (word >> (uint(lane) * 4u)) & 0xfu;
+ const float lut[8] = float[8](0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0);
+ float v = lut[n & 7u];
+ return (n & 8u) != 0u ? -v : v;
+}
void main() {
int s = int(gl_WorkGroupID.y);
@@ -97,16 +106,22 @@ void main() {
}
sum += a * scale[sb + uint(g)];
}
- } else if (p.fmt == 4) { // grouped int4 (#298 semantics): nibble decode, one scale
- // per gs inputs β€” gs % 8 == 0 (host-gated), so a packed word never straddles
- // a group and its scale multiplies the 8-wide partial. Per-row scale must NOT run.
+ } else if (p.fmt == 4 || p.fmt == 7) { // grouped int4 (#298 semantics) or MXFP4:
+ // nibble decode, one scale per gs inputs β€” gs % 8 == 0 (host-gated), so a
+ // packed word never straddles a group and its scale multiplies the 8-wide
+ // partial. Per-row scale must NOT run.
int words = (p.I + 7) / 8, ng = (p.I + p.gs - 1) / p.gs;
uint sb = uint(o) * uint(ng);
for (int wi = lane; wi < words; wi += sgsize) {
uint pk = w[rowBase + uint(wi)]; int i0 = wi * 8;
float a = 0.0;
- for (int k = 0; k < 8; k++) { int i = i0 + k;
- if (i < p.I) a += (staged ? xsh[i] : x[xoff + i]) * i4(pk, k); }
+ if (p.fmt == 4) {
+ for (int k = 0; k < 8; k++) { int i = i0 + k;
+ if (i < p.I) a += (staged ? xsh[i] : x[xoff + i]) * i4(pk, k); }
+ } else {
+ for (int k = 0; k < 8; k++) { int i = i0 + k;
+ if (i < p.I) a += (staged ? xsh[i] : x[xoff + i]) * mx4(pk, k); }
+ }
sum += a * scale[sb + uint(i0 / p.gs)];
}
} else { // int4
@@ -120,6 +135,6 @@ void main() {
}
}
float tot = subgroupAdd(sum);
- if (lane == 0) y[s * p.O + o] = (p.fmt == 5 || p.fmt == 4) ? tot : tot * scale[o];
+ if (lane == 0) y[s * p.O + o] = (p.fmt == 5 || p.fmt == 4 || p.fmt == 7) ? tot : tot * scale[o];
}
}
diff --git a/c/shaders/qmatmul.spv b/c/shaders/qmatmul.spv
new file mode 100644
index 00000000..f6488614
Binary files /dev/null and b/c/shaders/qmatmul.spv differ
diff --git a/c/tests/test_vk_mxfp4.c b/c/tests/test_vk_mxfp4.c
new file mode 100644
index 00000000..e5faf176
--- /dev/null
+++ b/c/tests/test_vk_mxfp4.c
@@ -0,0 +1,43 @@
+/* fmt=7 (MXFP4) Vulkan matmul vs the CPU reference kernel (quant.h).
+ * Random e2m1 nibbles + ue8m0 group exponents, K3 expert dims.
+ * Skips (exit 0) when no Vulkan device is available. */
+#include <stdio.h>
+#include <stdlib.h>
+#include <math.h>
+#include <stdint.h>
+#include "../quant.h"
+#include "../backend_vulkan.h"
+
+int main(void){
+ if(!coli_vk_init("shaders/qmatmul.spv")||!coli_vk_available()){
+ fprintf(stderr,"vk-mxfp4: no Vulkan device β€” skipped\n"); return 0;
+ }
+ srand(7);
+ int S=2, I=3584, O=3072; /* K3 expert w1/w3 shape */
+ int rb=(I+1)/2, ng=(I+31)/32;
+ uint8_t *q4=malloc((size_t)O*rb), *e8=malloc((size_t)O*ng);
+ float *x=malloc((size_t)S*I*sizeof(float));
+ float *yc=calloc((size_t)S*O,sizeof(float)), *yg=calloc((size_t)S*O,sizeof(float));
+ float *sc=malloc((size_t)O*ng*sizeof(float));
+ for(size_t i=0;i<(size_t)O*rb;i++) q4[i]=(uint8_t)rand();
+ for(size_t i=0;i<(size_t)O*ng;i++){ e8[i]=(uint8_t)(120+rand()%12); sc[i]=mx4_scale(e8[i]); }
+ for(int i=0;i<S*I;i++) x[i]=(float)(rand()%2001-1000)/500.f;
+
+ matmul_mxfp4(yc,x,q4,e8,S,I,O);
+
+ ColiVkTensor *t=NULL;
+ if(!coli_vk_matmul(&t,yg,x,q4,sc,7,S,I,O,32)){
+ fprintf(stderr,"vk-mxfp4: FAIL coli_vk_matmul fmt=7 refused\n"); return 1;
+ }
+ double num=0,den=0,mx=0;
+ for(int i=0;i<S*O;i++){
+ double d=yg[i]-yc[i]; num+=d*d; den+=(double)yc[i]*yc[i];
+ double r=fabs(d)/(fabs(yc[i])+1e-6); if(r>mx)mx=r;
+ }
+ double rel=sqrt(num/(den+1e-30));
+ printf("vk-mxfp4: rel_l2 %.3e max_rel %.3e (S=%d I=%d O=%d)\n",rel,mx,S,I,O);
+ if(rel>1e-5){ fprintf(stderr,"vk-mxfp4: FAIL rel_l2 %.3e\n",rel); return 1; }
+ printf("vk-mxfp4: OK\n");
+ coli_vk_shutdown();
+ return 0;
+}