Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

vulkan: f32, bf16 and fp8-e4m3 weights in qmatmul

JustVugg46e632003057
diff --git a/c/backend_vulkan.c b/c/backend_vulkan.c
index 33346eee..1c9f454e 100644
--- a/c/backend_vulkan.c
+++ b/c/backend_vulkan.c
@@ -214,17 +214,24 @@ static int scratch_reserve_mt(Scratch *s, size_t bytes, uint32_t memtype) {
}
static int scratch_reserve(Scratch *s, size_t bytes) { return scratch_reserve_mt(s, bytes, G.memtype); }
+/* Bytes of one weight row on the CPU side. fmt 10 (f32) and 11 (bf16) are the plain
+ * float weights the newer engines keep resident; they carry no scales. */
+static size_t cpu_row_bytes(int fmt, int I) {
+ return fmt == 1 || fmt == 12 ? (size_t)I // int8, fp8: one byte per weight
+ : fmt == 5 ? ((size_t)I + 63) / 64 * 24 // int3-g64: 24B per 64-group
+ : fmt == 10 ? (size_t)I * 4
+ : fmt == 11 ? (size_t)I * 2
+ : (size_t)(I + 1) / 2;
+}
static int rowwords(int fmt, int I) {
- size_t rb = fmt == 1 ? (size_t)I // bytes/row on CPU side
- : fmt == 5 ? ((size_t)I + 63) / 64 * 24 // int3-g64: 24B per 64-group
- : (size_t)(I + 1) / 2;
- return (int)((rb + 3) / 4); // padded to uint32 (24|4: exact)
+ return (int)((cpu_row_bytes(fmt, I) + 3) / 4); // padded to uint32 (24|4: exact)
}
/* Scale floats per tensor: per-row formats carry O, int3-g64 carries O*ceil(I/64)
* (one f32 per 64-input group). upload_tensor and tensor_free must agree on this. */
static size_t scale_floats(int fmt, int I, int O, int gs) {
+ if (fmt == 10 || fmt == 11) return 1; // unused by the shader; bound anyway
if (fmt == 5) return (size_t)O * (((size_t)I + 63) / 64);
- if (fmt == 4 || fmt == 7)
+ if (fmt == 4 || fmt == 7 || fmt == 12)
return (size_t)O * (((size_t)I + gs - 1) / gs); // per-group [O,ng]
return (size_t)O;
}
@@ -520,14 +527,15 @@ static int arena_suballoc(size_t bytes, VkBuffer *buf, void **ptr) {
static int upload_tensor(ColiVkTensor **out, const void *weights, const float *scales,
int fmt, int I, int O, int gs) {
if (*out) return (*out)->fmt == fmt && (*out)->I == I && (*out)->O == O;
- if (fmt != 1 && fmt != 2 && fmt != 5 && /* fmt=4/7: word-aligned groups only */
- !((fmt == 4 || fmt == 7) && gs >= 8 && gs % 8 == 0)) return 0;
+ if (fmt != 1 && fmt != 2 && fmt != 5 && fmt != 10 && fmt != 11 && /* fmt=4/7: word-aligned groups only */
+ !((fmt == 4 || fmt == 7) && gs >= 8 && gs % 8 == 0) &&
+ !(fmt == 12 && gs >= 4 && gs % 4 == 0)) return 0; /* fp8: 4 per word */
ColiVkTensor *t = calloc(1, sizeof(*t));
if (!t) return 0;
- t->fmt = fmt; t->I = I; t->O = O; t->rowWords = rowwords(fmt, I); t->gs = (fmt == 4 || fmt == 7) ? gs : 0;
+ t->fmt = fmt; t->I = I; t->O = O; t->rowWords = rowwords(fmt, I);
+ t->gs = (fmt == 4 || fmt == 7 || fmt == 12) ? gs : 0;
size_t stride = (size_t)t->rowWords * 4; // padded row bytes
- size_t cpu_rb = fmt == 1 ? (size_t)I
- : fmt == 5 ? ((size_t)I + 63) / 64 * 24 : (size_t)(I + 1) / 2;
+ size_t cpu_rb = cpu_row_bytes(fmt, I);
size_t sfl = scale_floats(fmt, I, O, gs); // fmt=5: O*ceil(I/64) group scales
t->wbytes = stride * (size_t)O;
void *wptr;
@@ -540,7 +548,8 @@ static int upload_tensor(ColiVkTensor **out, const void *weights, const float *s
if (!arena_suballoc(sfl * sizeof(float), &t->sbuf, &sptr)) {
vkDestroyBuffer(G.dev, t->wbuf, NULL); free(t); return 0;
}
- memcpy(sptr, scales, sfl * sizeof(float));
+ if (fmt == 10 || fmt == 11) ((float *)sptr)[0] = 1.0f; /* float weights: no scales */
+ else memcpy(sptr, scales, sfl * sizeof(float));
// Counters are touched concurrently: frees run from expert_load under
// `#pragma omp parallel`, so RMW them atomically (torn counts otherwise).
__atomic_add_fetch(&G.used_bytes, t->wbytes + sfl * sizeof(float), __ATOMIC_RELAXED);
@@ -703,7 +712,7 @@ int coli_vk_init_env(const char *engine) {
char buf[1024];
const char *spv = coli_vk_shader_path(buf, sizeof buf);
int ok = coli_vk_init(spv) && coli_vk_available();
- if (ok) fprintf(stderr, "[VK] %s: resident matrices on the GPU\n", engine);
+ if (ok) fprintf(stderr, "[VK] %s: device ready\n", engine);
else fprintf(stderr, "[VK] %s: no usable Vulkan device (shaders %s), running on the CPU\n",
engine, spv);
return ok;
@@ -1662,15 +1671,24 @@ static double now(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts
static int g_ref_gs = 64; /* fmt=4 group size the harness cases use */
static size_t ref_rowbytes(int fmt, int I) {
- return fmt == 1 ? (size_t)I : fmt == 5 ? (size_t)((I + 63) / 64) * 24 : (size_t)(I + 1) / 2;
+ return fmt == 1 || fmt == 12 ? (size_t)I : fmt == 5 ? (size_t)((I + 63) / 64) * 24
+ : fmt == 10 ? (size_t)I * 4 : fmt == 11 ? (size_t)I * 2 : (size_t)(I + 1) / 2;
}
static size_t ref_scales(int fmt, int I, int O) { // scale COUNT (per-group for fmt 4/5)
+ if (fmt == 10 || fmt == 11) return 1;
if (fmt == 5) return (size_t)O * (size_t)((I + 63) / 64);
- if (fmt == 4) return (size_t)O * (size_t)((I + g_ref_gs - 1) / g_ref_gs);
+ if (fmt == 4 || fmt == 12) return (size_t)O * (size_t)((I + g_ref_gs - 1) / g_ref_gs);
return (size_t)O;
}
static float deq(const uint8_t *row, int fmt, int i) {
if (fmt == 1) { int b = ((const int8_t *)row)[i]; return (float)b; }
+ if (fmt == 10) { float f; memcpy(&f, row + (size_t)i * 4, 4); return f; }
+ if (fmt == 12) { uint8_t b = row[i]; int e = (b >> 3) & 15, m = b & 7; /* E4M3_LUT semantics */
+ if ((b & 0x7f) == 0x7f) return NAN;
+ float v = e == 0 ? m / 512.0f : ldexpf(1.0f + m / 8.0f, e - 7);
+ return (b & 0x80) ? -v : v; }
+ if (fmt == 11) { uint16_t h; memcpy(&h, row + (size_t)i * 2, 2); uint32_t u = (uint32_t)h << 16;
+ float f; memcpy(&f, &u, 4); return f; }
if (fmt == 5) { // int3-g64: 16B low plane (2 bits) + 8B high plane (1 bit), v+4
const uint8_t *lo = row + (size_t)(i >> 6) * 24, *hi = lo + 16; int j = i & 63;
unsigned u = ((lo[j >> 2] >> ((j & 3) * 2)) & 3u) | (((hi[j >> 3] >> (j & 7)) & 1u) << 2);
@@ -1681,10 +1699,10 @@ static float deq(const uint8_t *row, int fmt, int i) {
static void cpu_ref(float *y, const float *x, const uint8_t *w, const float *sc,
int fmt, int S, int I, int O) {
size_t rb = ref_rowbytes(fmt, I);
- int gw2 = fmt == 4 ? g_ref_gs : 64, ng = (I + gw2 - 1) / gw2;
+ int gw2 = (fmt == 4 || fmt == 12) ? g_ref_gs : 64, ng = (I + gw2 - 1) / gw2;
for (int s = 0; s < S; s++) for (int o = 0; o < O; o++) {
double sum = 0; const uint8_t *row = w + (size_t)o * rb;
- if (fmt == 5 || fmt == 4) { // per-group scales fold inside the sum
+ if (fmt == 5 || fmt == 4 || fmt == 12) { // per-group scales fold inside the sum
for (int g = 0; g < ng; g++) {
double a = 0; int end = (g + 1) * gw2 < I ? (g + 1) * gw2 : I;
for (int i = g * gw2; i < end; i++) a += x[s * I + i] * deq(row, fmt, i);
@@ -1693,7 +1711,7 @@ static void cpu_ref(float *y, const float *x, const uint8_t *w, const float *sc,
y[s * O + o] = (float)sum;
} else {
for (int i = 0; i < I; i++) sum += x[s * I + i] * deq(row, fmt, i);
- y[s * O + o] = (float)(sum * sc[o]);
+ y[s * O + o] = (float)(fmt == 10 || fmt == 11 ? sum : sum * sc[o]);
}
}
}
@@ -1725,6 +1743,14 @@ static int run_case(int fmt, int S, int I, int O, int iters) {
float *yc = malloc((size_t)S * O * sizeof(float));
for (int i = 0; i < S * I; i++) x[i] = (float)((rand() % 200 - 100) / 100.0);
for (size_t i = 0; i < rb * O; i++) w[i] = rand() & 0xff;
+ if (fmt == 12) /* no NaN bytes: they would poison the comparison */
+ for (size_t i = 0; i < rb * O; i++) if ((w[i] & 0x7f) == 0x7f) w[i] ^= 1;
+ if (fmt == 10 || fmt == 11) /* float weights: random bytes could be NaN/Inf */
+ for (size_t i = 0; i < (size_t)I * O; i++) {
+ float f = (float)((rand() % 2001 - 1000) / 1000.0);
+ if (fmt == 10) memcpy(w + i * 4, &f, 4);
+ else { uint32_t u; memcpy(&u, &f, 4); uint16_t h = (uint16_t)(u >> 16); memcpy(w + i * 2, &h, 2); }
+ }
for (size_t o = 0; o < nsc; o++) sc[o] = 0.01f + (rand() % 100) / 10000.0f;
ColiVkTensor *t = NULL;
@@ -2130,6 +2156,23 @@ int main(int argc, char **argv) {
bad |= run_case(5, 8, 6144, 2048, 20); // int3 batch
bad |= run_case(5, 1, 100, 64, 20); // partial tail group (I%64 != 0)
bad |= run_case(5, 1, 16384, 6144, 20); // int3 o_proj shape (unstaged path)
+ /* f32 (fmt=10) and bf16 (fmt=11) weights: staged, batched, unstaged, odd widths */
+ bad |= run_case(10, 1, 6144, 1536, 20);
+ bad |= run_case(10, 8, 2048, 512, 10);
+ bad |= run_case(10, 1, 16384, 512, 5);
+ bad |= run_case(10, 3, 37, 5, 5);
+ bad |= run_case(11, 1, 6144, 1536, 20);
+ bad |= run_case(11, 8, 2048, 512, 10);
+ bad |= run_case(11, 1, 16384, 512, 5);
+ bad |= run_case(11, 3, 37, 5, 5); // odd I: the last word holds one bf16
+ /* fp8 e4m3 (fmt=12), grouped scales: 64 (harness default), then 32 and 128 below */
+ bad |= run_case(12, 1, 6144, 1536, 20);
+ bad |= run_case(12, 8, 2048, 512, 10);
+ bad |= run_case(12, 1, 16384, 512, 5);
+ bad |= run_case(12, 3, 100, 5, 5); // partial tail group
+ g_ref_gs = 32; bad |= run_case(12, 2, 4096, 256, 5);
+ g_ref_gs = 128; bad |= run_case(12, 2, 4096, 256, 5);
+ g_ref_gs = 64;
/* Batched (amortized) throughput on the int4 expert shapes — the real expert-tier pattern. */
{
int I = 6144, O = 2048; /* our gate/up dims */
diff --git a/c/backend_vulkan.h b/c/backend_vulkan.h
index ed1fe983..07cb9616 100644
--- a/c/backend_vulkan.h
+++ b/c/backend_vulkan.h
@@ -35,6 +35,8 @@ int coli_vk_mem_budget(double *used_gb, double *budget_gb);
/* y[S,O] = (x[S,I] @ dequant(W[O,I])^T) * scale[O].
* fmt matches QT in glm.c: 1=int8, 2=int4. (0=f32,3=int2 fall back to CPU.)
+ * fmt 10 = plain f32 weights and 11 = bf16 weights (low half = even column): no
+ * scales, pass NULL. Numbered apart from QT's 0 (f32), which keeps falling back.
* First call uploads W+scales; later calls reuse the resident copy.
* Returns 1 on success, 0 if unavailable / unsupported fmt. */
int coli_vk_matmul(ColiVkTensor **tensor,
diff --git a/c/shaders/qmatmul.comp b/c/shaders/qmatmul.comp
index 25de4f20..c55461c5 100644
--- a/c/shaders/qmatmul.comp
+++ b/c/shaders/qmatmul.comp
@@ -26,7 +26,7 @@ layout(std430, binding = 2) readonly buffer Sc { float scale[]; };
layout(std430, binding = 3) writeonly buffer Y { float y[]; };
layout(push_constant) uniform P {
- int fmt; // 1=int8, 2=int4, 4=grouped int4, 5=int3-g64
+ int fmt; // 1=int8, 2=int4, 4=grouped int4, 5=int3-g64, 7=MXFP4, 10=f32, 11=bf16, 12=fp8-e4m3 grouped
int S, I, O;
int rowWords; // uint32 words per weight row
int gs; // fmt=4 group size (multiple of 8; host gates otherwise)
@@ -56,6 +56,17 @@ float mx4(uint word, int lane) {
return (n & 8u) != 0u ? -v : v;
}
+// fmt=12 fp8 e4m3 (OCP E4M3-FN, the CPU's E4M3_LUT): sign, 4-bit exponent with bias 7,
+// 3-bit mantissa, subnormals at exponent 0, NaN only at 0x7F/0xFF. Decoded arithmetically:
+// building the float from its bits would make subnormals that a GPU may flush to zero.
+float e4m3(uint word, int lane) {
+ uint b = (word >> (uint(lane) * 8u)) & 0xffu;
+ if ((b & 0x7fu) == 0x7fu) return uintBitsToFloat(0x7fc00000u);
+ uint e = (b >> 3) & 0xfu, m = b & 7u;
+ float v = e == 0u ? float(m) * 0.001953125 : ldexp(1.0 + float(m) * 0.125, int(e) - 7);
+ return (b & 0x80u) != 0u ? -v : v;
+}
+
void main() {
int s = int(gl_WorkGroupID.y);
int tid = int(gl_LocalInvocationID.x);
@@ -85,6 +96,28 @@ void main() {
uint pk = w[rowBase + uint(wi)]; int i0 = wi * 4;
for (int k = 0; k < 4; k++) { int i = i0 + k; if (i < p.I) sum += x[xoff + i] * i8(pk, k); }
}
+ } else if (p.fmt == 10) { // f32 weights, one per word: no dequant, no scale
+ for (int i = lane; i < p.I; i += sgsize)
+ sum += (staged ? xsh[i] : x[xoff + i]) * uintBitsToFloat(w[rowBase + uint(i)]);
+ } else if (p.fmt == 11) { // bf16 weights, two per word, LOW half = even column
+ int words = (p.I + 1) / 2;
+ for (int wi = lane; wi < words; wi += sgsize) {
+ uint pk = w[rowBase + uint(wi)]; int i0 = wi * 2;
+ sum += (staged ? xsh[i0] : x[xoff + i0]) * uintBitsToFloat((pk & 0xffffu) << 16);
+ if (i0 + 1 < p.I)
+ sum += (staged ? xsh[i0 + 1] : x[xoff + i0 + 1]) * uintBitsToFloat(pk & 0xffff0000u);
+ }
+ } else if (p.fmt == 12) { // fp8 e4m3, 4 per word, one f32 scale per gs inputs
+ // gs % 4 == 0 (host-gated): a word never straddles a group.
+ int words = (p.I + 3) / 4, ng = (p.I + p.gs - 1) / p.gs;
+ uint sb = uint(o) * uint(ng);
+ for (int wi = lane; wi < words; wi += sgsize) {
+ uint pk = w[rowBase + uint(wi)]; int i0 = wi * 4;
+ float a = 0.0;
+ for (int k = 0; k < 4; k++) { int i = i0 + k;
+ if (i < p.I) a += (staged ? xsh[i] : x[xoff + i]) * e4m3(pk, k); }
+ sum += a * scale[sb + uint(i0 / p.gs)];
+ }
} else if (p.fmt == 5) { // int3-g64: lanes stride the 16-value LOW-PLANE words;
// all 16 values of one low word share a group, so its scale multiplies the
// 16-wide partial directly and the per-row scale at the end must NOT run.
@@ -135,6 +168,7 @@ void main() {
}
}
float tot = subgroupAdd(sum);
- if (lane == 0) y[s * p.O + o] = (p.fmt == 5 || p.fmt == 4 || p.fmt == 7) ? tot : tot * scale[o];
+ if (lane == 0) y[s * p.O + o] = (p.fmt == 5 || p.fmt == 4 || p.fmt == 7 || p.fmt == 10 || p.fmt == 11 || p.fmt == 12)
+ ? tot : tot * scale[o];
}
}
diff --git a/c/shaders/qmatmul.spv b/c/shaders/qmatmul.spv
index f6488614..a55e168b 100644
Binary files a/c/shaders/qmatmul.spv and b/c/shaders/qmatmul.spv differ