Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
vulkan: f32, bf16 and fp8-e4m3 weights in qmatmul
JustVugg46e632003057
diff --git a/c/backend_vulkan.c b/c/backend_vulkan.cindex 33346eee..1c9f454e 100644--- a/c/backend_vulkan.c+++ b/c/backend_vulkan.c@@ -214,17 +214,24 @@ static int scratch_reserve_mt(Scratch *s, size_t bytes, uint32_t memtype) {}static int scratch_reserve(Scratch *s, size_t bytes) { return scratch_reserve_mt(s, bytes, G.memtype); }+/* Bytes of one weight row on the CPU side. fmt 10 (f32) and 11 (bf16) are the plain+ * float weights the newer engines keep resident; they carry no scales. */+static size_t cpu_row_bytes(int fmt, int I) {+ return fmt == 1 || fmt == 12 ? (size_t)I // int8, fp8: one byte per weight+ : fmt == 5 ? ((size_t)I + 63) / 64 * 24 // int3-g64: 24B per 64-group+ : fmt == 10 ? (size_t)I * 4+ : fmt == 11 ? (size_t)I * 2+ : (size_t)(I + 1) / 2;+}static int rowwords(int fmt, int I) {- size_t rb = fmt == 1 ? (size_t)I // bytes/row on CPU side- : fmt == 5 ? ((size_t)I + 63) / 64 * 24 // int3-g64: 24B per 64-group- : (size_t)(I + 1) / 2;- return (int)((rb + 3) / 4); // padded to uint32 (24|4: exact)+ return (int)((cpu_row_bytes(fmt, I) + 3) / 4); // padded to uint32 (24|4: exact)}/* Scale floats per tensor: per-row formats carry O, int3-g64 carries O*ceil(I/64)* (one f32 per 64-input group). upload_tensor and tensor_free must agree on this. */static size_t scale_floats(int fmt, int I, int O, int gs) {+ if (fmt == 10 || fmt == 11) return 1; // unused by the shader; bound anywayif (fmt == 5) return (size_t)O * (((size_t)I + 63) / 64);- if (fmt == 4 || fmt == 7)+ if (fmt == 4 || fmt == 7 || fmt == 12)return (size_t)O * (((size_t)I + gs - 1) / gs); // per-group [O,ng]return (size_t)O;}@@ -520,14 +527,15 @@ static int arena_suballoc(size_t bytes, VkBuffer *buf, void **ptr) {static int upload_tensor(ColiVkTensor **out, const void *weights, const float *scales,int fmt, int I, int O, int gs) {if (*out) return (*out)->fmt == fmt && (*out)->I == I && (*out)->O == O;- if (fmt != 1 && fmt != 2 && fmt != 5 && /* fmt=4/7: word-aligned groups only */- !((fmt == 4 || fmt == 7) && gs >= 8 && gs % 8 == 0)) return 0;+ if (fmt != 1 && fmt != 2 && fmt != 5 && fmt != 10 && fmt != 11 && /* fmt=4/7: word-aligned groups only */+ !((fmt == 4 || fmt == 7) && gs >= 8 && gs % 8 == 0) &&+ !(fmt == 12 && gs >= 4 && gs % 4 == 0)) return 0; /* fp8: 4 per word */ColiVkTensor *t = calloc(1, sizeof(*t));if (!t) return 0;- t->fmt = fmt; t->I = I; t->O = O; t->rowWords = rowwords(fmt, I); t->gs = (fmt == 4 || fmt == 7) ? gs : 0;+ t->fmt = fmt; t->I = I; t->O = O; t->rowWords = rowwords(fmt, I);+ t->gs = (fmt == 4 || fmt == 7 || fmt == 12) ? gs : 0;size_t stride = (size_t)t->rowWords * 4; // padded row bytes- size_t cpu_rb = fmt == 1 ? (size_t)I- : fmt == 5 ? ((size_t)I + 63) / 64 * 24 : (size_t)(I + 1) / 2;+ size_t cpu_rb = cpu_row_bytes(fmt, I);size_t sfl = scale_floats(fmt, I, O, gs); // fmt=5: O*ceil(I/64) group scalest->wbytes = stride * (size_t)O;void *wptr;@@ -540,7 +548,8 @@ static int upload_tensor(ColiVkTensor **out, const void *weights, const float *sif (!arena_suballoc(sfl * sizeof(float), &t->sbuf, &sptr)) {vkDestroyBuffer(G.dev, t->wbuf, NULL); free(t); return 0;}- memcpy(sptr, scales, sfl * sizeof(float));+ if (fmt == 10 || fmt == 11) ((float *)sptr)[0] = 1.0f; /* float weights: no scales */+ else memcpy(sptr, scales, sfl * sizeof(float));// Counters are touched concurrently: frees run from expert_load under// `#pragma omp parallel`, so RMW them atomically (torn counts otherwise).__atomic_add_fetch(&G.used_bytes, t->wbytes + sfl * sizeof(float), __ATOMIC_RELAXED);@@ -703,7 +712,7 @@ int coli_vk_init_env(const char *engine) {char buf[1024];const char *spv = coli_vk_shader_path(buf, sizeof buf);int ok = coli_vk_init(spv) && coli_vk_available();- if (ok) fprintf(stderr, "[VK] %s: resident matrices on the GPU\n", engine);+ if (ok) fprintf(stderr, "[VK] %s: device ready\n", engine);else fprintf(stderr, "[VK] %s: no usable Vulkan device (shaders %s), running on the CPU\n",engine, spv);return ok;@@ -1662,15 +1671,24 @@ static double now(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &tsstatic int g_ref_gs = 64; /* fmt=4 group size the harness cases use */static size_t ref_rowbytes(int fmt, int I) {- return fmt == 1 ? (size_t)I : fmt == 5 ? (size_t)((I + 63) / 64) * 24 : (size_t)(I + 1) / 2;+ return fmt == 1 || fmt == 12 ? (size_t)I : fmt == 5 ? (size_t)((I + 63) / 64) * 24+ : fmt == 10 ? (size_t)I * 4 : fmt == 11 ? (size_t)I * 2 : (size_t)(I + 1) / 2;}static size_t ref_scales(int fmt, int I, int O) { // scale COUNT (per-group for fmt 4/5)+ if (fmt == 10 || fmt == 11) return 1;if (fmt == 5) return (size_t)O * (size_t)((I + 63) / 64);- if (fmt == 4) return (size_t)O * (size_t)((I + g_ref_gs - 1) / g_ref_gs);+ if (fmt == 4 || fmt == 12) return (size_t)O * (size_t)((I + g_ref_gs - 1) / g_ref_gs);return (size_t)O;}static float deq(const uint8_t *row, int fmt, int i) {if (fmt == 1) { int b = ((const int8_t *)row)[i]; return (float)b; }+ if (fmt == 10) { float f; memcpy(&f, row + (size_t)i * 4, 4); return f; }+ if (fmt == 12) { uint8_t b = row[i]; int e = (b >> 3) & 15, m = b & 7; /* E4M3_LUT semantics */+ if ((b & 0x7f) == 0x7f) return NAN;+ float v = e == 0 ? m / 512.0f : ldexpf(1.0f + m / 8.0f, e - 7);+ return (b & 0x80) ? -v : v; }+ if (fmt == 11) { uint16_t h; memcpy(&h, row + (size_t)i * 2, 2); uint32_t u = (uint32_t)h << 16;+ float f; memcpy(&f, &u, 4); return f; }if (fmt == 5) { // int3-g64: 16B low plane (2 bits) + 8B high plane (1 bit), v+4const uint8_t *lo = row + (size_t)(i >> 6) * 24, *hi = lo + 16; int j = i & 63;unsigned u = ((lo[j >> 2] >> ((j & 3) * 2)) & 3u) | (((hi[j >> 3] >> (j & 7)) & 1u) << 2);@@ -1681,10 +1699,10 @@ static float deq(const uint8_t *row, int fmt, int i) {static void cpu_ref(float *y, const float *x, const uint8_t *w, const float *sc,int fmt, int S, int I, int O) {size_t rb = ref_rowbytes(fmt, I);- int gw2 = fmt == 4 ? g_ref_gs : 64, ng = (I + gw2 - 1) / gw2;+ int gw2 = (fmt == 4 || fmt == 12) ? g_ref_gs : 64, ng = (I + gw2 - 1) / gw2;for (int s = 0; s < S; s++) for (int o = 0; o < O; o++) {double sum = 0; const uint8_t *row = w + (size_t)o * rb;- if (fmt == 5 || fmt == 4) { // per-group scales fold inside the sum+ if (fmt == 5 || fmt == 4 || fmt == 12) { // per-group scales fold inside the sumfor (int g = 0; g < ng; g++) {double a = 0; int end = (g + 1) * gw2 < I ? (g + 1) * gw2 : I;for (int i = g * gw2; i < end; i++) a += x[s * I + i] * deq(row, fmt, i);@@ -1693,7 +1711,7 @@ static void cpu_ref(float *y, const float *x, const uint8_t *w, const float *sc,y[s * O + o] = (float)sum;} else {for (int i = 0; i < I; i++) sum += x[s * I + i] * deq(row, fmt, i);- y[s * O + o] = (float)(sum * sc[o]);+ y[s * O + o] = (float)(fmt == 10 || fmt == 11 ? sum : sum * sc[o]);}}}@@ -1725,6 +1743,14 @@ static int run_case(int fmt, int S, int I, int O, int iters) {float *yc = malloc((size_t)S * O * sizeof(float));for (int i = 0; i < S * I; i++) x[i] = (float)((rand() % 200 - 100) / 100.0);for (size_t i = 0; i < rb * O; i++) w[i] = rand() & 0xff;+ if (fmt == 12) /* no NaN bytes: they would poison the comparison */+ for (size_t i = 0; i < rb * O; i++) if ((w[i] & 0x7f) == 0x7f) w[i] ^= 1;+ if (fmt == 10 || fmt == 11) /* float weights: random bytes could be NaN/Inf */+ for (size_t i = 0; i < (size_t)I * O; i++) {+ float f = (float)((rand() % 2001 - 1000) / 1000.0);+ if (fmt == 10) memcpy(w + i * 4, &f, 4);+ else { uint32_t u; memcpy(&u, &f, 4); uint16_t h = (uint16_t)(u >> 16); memcpy(w + i * 2, &h, 2); }+ }for (size_t o = 0; o < nsc; o++) sc[o] = 0.01f + (rand() % 100) / 10000.0f;ColiVkTensor *t = NULL;@@ -2130,6 +2156,23 @@ int main(int argc, char **argv) {bad |= run_case(5, 8, 6144, 2048, 20); // int3 batchbad |= run_case(5, 1, 100, 64, 20); // partial tail group (I%64 != 0)bad |= run_case(5, 1, 16384, 6144, 20); // int3 o_proj shape (unstaged path)+ /* f32 (fmt=10) and bf16 (fmt=11) weights: staged, batched, unstaged, odd widths */+ bad |= run_case(10, 1, 6144, 1536, 20);+ bad |= run_case(10, 8, 2048, 512, 10);+ bad |= run_case(10, 1, 16384, 512, 5);+ bad |= run_case(10, 3, 37, 5, 5);+ bad |= run_case(11, 1, 6144, 1536, 20);+ bad |= run_case(11, 8, 2048, 512, 10);+ bad |= run_case(11, 1, 16384, 512, 5);+ bad |= run_case(11, 3, 37, 5, 5); // odd I: the last word holds one bf16+ /* fp8 e4m3 (fmt=12), grouped scales: 64 (harness default), then 32 and 128 below */+ bad |= run_case(12, 1, 6144, 1536, 20);+ bad |= run_case(12, 8, 2048, 512, 10);+ bad |= run_case(12, 1, 16384, 512, 5);+ bad |= run_case(12, 3, 100, 5, 5); // partial tail group+ g_ref_gs = 32; bad |= run_case(12, 2, 4096, 256, 5);+ g_ref_gs = 128; bad |= run_case(12, 2, 4096, 256, 5);+ g_ref_gs = 64;/* Batched (amortized) throughput on the int4 expert shapes — the real expert-tier pattern. */{int I = 6144, O = 2048; /* our gate/up dims */diff --git a/c/backend_vulkan.h b/c/backend_vulkan.hindex ed1fe983..07cb9616 100644--- a/c/backend_vulkan.h+++ b/c/backend_vulkan.h@@ -35,6 +35,8 @@ int coli_vk_mem_budget(double *used_gb, double *budget_gb);/* y[S,O] = (x[S,I] @ dequant(W[O,I])^T) * scale[O].* fmt matches QT in glm.c: 1=int8, 2=int4. (0=f32,3=int2 fall back to CPU.)+ * fmt 10 = plain f32 weights and 11 = bf16 weights (low half = even column): no+ * scales, pass NULL. Numbered apart from QT's 0 (f32), which keeps falling back.* First call uploads W+scales; later calls reuse the resident copy.* Returns 1 on success, 0 if unavailable / unsupported fmt. */int coli_vk_matmul(ColiVkTensor **tensor,diff --git a/c/shaders/qmatmul.comp b/c/shaders/qmatmul.compindex 25de4f20..c55461c5 100644--- a/c/shaders/qmatmul.comp+++ b/c/shaders/qmatmul.comp@@ -26,7 +26,7 @@ layout(std430, binding = 2) readonly buffer Sc { float scale[]; };layout(std430, binding = 3) writeonly buffer Y { float y[]; };layout(push_constant) uniform P {- int fmt; // 1=int8, 2=int4, 4=grouped int4, 5=int3-g64+ int fmt; // 1=int8, 2=int4, 4=grouped int4, 5=int3-g64, 7=MXFP4, 10=f32, 11=bf16, 12=fp8-e4m3 groupedint S, I, O;int rowWords; // uint32 words per weight rowint gs; // fmt=4 group size (multiple of 8; host gates otherwise)@@ -56,6 +56,17 @@ float mx4(uint word, int lane) {return (n & 8u) != 0u ? -v : v;}+// fmt=12 fp8 e4m3 (OCP E4M3-FN, the CPU's E4M3_LUT): sign, 4-bit exponent with bias 7,+// 3-bit mantissa, subnormals at exponent 0, NaN only at 0x7F/0xFF. Decoded arithmetically:+// building the float from its bits would make subnormals that a GPU may flush to zero.+float e4m3(uint word, int lane) {+ uint b = (word >> (uint(lane) * 8u)) & 0xffu;+ if ((b & 0x7fu) == 0x7fu) return uintBitsToFloat(0x7fc00000u);+ uint e = (b >> 3) & 0xfu, m = b & 7u;+ float v = e == 0u ? float(m) * 0.001953125 : ldexp(1.0 + float(m) * 0.125, int(e) - 7);+ return (b & 0x80u) != 0u ? -v : v;+}+void main() {int s = int(gl_WorkGroupID.y);int tid = int(gl_LocalInvocationID.x);@@ -85,6 +96,28 @@ void main() {uint pk = w[rowBase + uint(wi)]; int i0 = wi * 4;for (int k = 0; k < 4; k++) { int i = i0 + k; if (i < p.I) sum += x[xoff + i] * i8(pk, k); }}+ } else if (p.fmt == 10) { // f32 weights, one per word: no dequant, no scale+ for (int i = lane; i < p.I; i += sgsize)+ sum += (staged ? xsh[i] : x[xoff + i]) * uintBitsToFloat(w[rowBase + uint(i)]);+ } else if (p.fmt == 11) { // bf16 weights, two per word, LOW half = even column+ int words = (p.I + 1) / 2;+ for (int wi = lane; wi < words; wi += sgsize) {+ uint pk = w[rowBase + uint(wi)]; int i0 = wi * 2;+ sum += (staged ? xsh[i0] : x[xoff + i0]) * uintBitsToFloat((pk & 0xffffu) << 16);+ if (i0 + 1 < p.I)+ sum += (staged ? xsh[i0 + 1] : x[xoff + i0 + 1]) * uintBitsToFloat(pk & 0xffff0000u);+ }+ } else if (p.fmt == 12) { // fp8 e4m3, 4 per word, one f32 scale per gs inputs+ // gs % 4 == 0 (host-gated): a word never straddles a group.+ int words = (p.I + 3) / 4, ng = (p.I + p.gs - 1) / p.gs;+ uint sb = uint(o) * uint(ng);+ for (int wi = lane; wi < words; wi += sgsize) {+ uint pk = w[rowBase + uint(wi)]; int i0 = wi * 4;+ float a = 0.0;+ for (int k = 0; k < 4; k++) { int i = i0 + k;+ if (i < p.I) a += (staged ? xsh[i] : x[xoff + i]) * e4m3(pk, k); }+ sum += a * scale[sb + uint(i0 / p.gs)];+ }} else if (p.fmt == 5) { // int3-g64: lanes stride the 16-value LOW-PLANE words;// all 16 values of one low word share a group, so its scale multiplies the// 16-wide partial directly and the per-row scale at the end must NOT run.@@ -135,6 +168,7 @@ void main() {}}float tot = subgroupAdd(sum);- if (lane == 0) y[s * p.O + o] = (p.fmt == 5 || p.fmt == 4 || p.fmt == 7) ? tot : tot * scale[o];+ if (lane == 0) y[s * p.O + o] = (p.fmt == 5 || p.fmt == 4 || p.fmt == 7 || p.fmt == 10 || p.fmt == 11 || p.fmt == 12)+ ? tot : tot * scale[o];}}diff --git a/c/shaders/qmatmul.spv b/c/shaders/qmatmul.spvindex f6488614..a55e168b 100644Binary files a/c/shaders/qmatmul.spv and b/c/shaders/qmatmul.spv differ