Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fp8: Metal mm_gemv kernel for fmt=7 native FP8-e4m3 passthrough

monotophic6ca61a1ba267
diff --git a/c/backend_metal.h b/c/backend_metal.h
index 0c79ca2d..84baccfa 100644
--- a/c/backend_metal.h
+++ b/c/backend_metal.h
@@ -29,9 +29,15 @@ int coli_metal_mem_info(size_t *used_bytes, size_t *total_bytes);
/*
* y[S,O] = (x[S,I] @ W[O,I]^T) * scale[o]. fmt=4 (grouped int4) instead folds a
* PER-GROUP scale into the accumulation -- see the shader comment in backend_metal.mm.
+ * fmt=7 (fp8 passthrough -- see colibri.c) likewise folds its per-128x128-block
+ * scale into the accumulation.
* fmt matches QT in colibri.c: 0=f32, 1=int8, 2=int4(packed), 3=int2(packed),
* 4=int4(packed, same layout as 2)+per-group scale (gs = group size along I; scale
- * array is [O, ceil(I/gs)] floats instead of [O]). gs is ignored for fmt!=4 (pass 0).
+ * array is [O, ceil(I/gs)] floats instead of [O]),
+ * 7=fp8-e4m3 (one raw byte/element, same layout as fmt=1) + per-128x128-
+ * block scale (scale array is [ceil(O/128),ceil(I/128)] floats; no group-size
+ * parameter needed -- the block is a fixed 128x128, not caller-configurable).
+ * gs is ignored for fmt!=4 (pass 0).
* The first successful call wraps W and its scales in GPU-visible buffers (sized from
* fmt+gs); later calls reuse them (weights are assumed stable at the same address).
* Returns 1 on success, 0 if Metal is unavailable or fmt is invalid.
diff --git a/c/backend_metal.mm b/c/backend_metal.mm
index f13e7432..3df08c8b 100644
--- a/c/backend_metal.mm
+++ b/c/backend_metal.mm
@@ -13,6 +13,16 @@
// nibble layout as fmt=2, but scale is PER-GROUP (gsz elements of I), buffer(1) holds
// [O, ceil(I/gsz)] floats indexed scale[o*ng+g] -- so unlike fmt 1-3, the group scale is
// folded into the accumulation itself (see the fmt==4 branch), not applied once at the end.
+// fmt=7 (native FP8-e4m3 passthrough -- see colibri.c): raw byte
+// layout identical to fmt=1 (one e4m3 byte per element, no packing), but the scale is
+// per-128x128 BLOCK of [O,I] -- buffer(1) holds [ceil(O/128),ceil(I/128)] floats indexed
+// scale[(o/128)*nblkI+i/128]. Folded into acc like fmt=4's per-group scale, for the
+// same reason: it is not constant across one output row. e4m3->float is bit manipulation
+// in-kernel (sign/exp/mantissa, OCP E4M3-FN: exp==0 subnormal, exp==0xF&&mant==0x7 is the
+// only NaN code) -- BW-bound kernel, decode ALU is free, no LUT texture. Must byte-match
+// quant.h's E4M3_LUT/e4m3_decode (the CPU reference) including the NaN policy, which is
+// why the metal-test suite runs all 256 byte codes through this exact kernel path (not
+// just spot values) -- see run_fp8_lut().
static const char *SHADER = R"METAL(
#include <metal_stdlib>
using namespace metal;
@@ -75,6 +85,25 @@ kernel void mm_gemv(device const uchar* w [[buffer(0)]], // raw weight by
acc += float(int(b>>4)-8) * xr[i+1] * sc1;
}
}
+ } else if (fmt == 7) { // fp8 e4m3 passthrough: one raw byte per
+ // element (like fmt=1), scale per 128x128
+ // block folded into acc (like a grouped fmt).
+ int nblkI = (I + 127) / 128;
+ device const uchar* wr = w + (long)o * I;
+ device const float* scl = scale + (long)(o/128) * nblkI;
+ for (int i = slane; i < I; i += 32) {
+ uchar b = wr[i];
+ uint sign = b >> 7, exp = (b >> 3) & 0xF, mant = b & 0x7;
+ float wv;
+ if (exp == 0xF && mant == 0x7) {
+ wv = as_type<float>(0x7fc00000u); // qNaN -- matches quant.h's e4m3_decode
+ } else {
+ float mag = (exp == 0) ? (float(mant) * 0.001953125f) // subnormal: mant*2^-9
+ : (1.0f + float(mant)*0.125f) * exp2(float(int(exp) - 7));
+ wv = sign ? -mag : mag;
+ }
+ acc += wv * xr[i] * scl[i/128];
+ }
} else { // f32
device const float* wr = (device const float*)(w) + (long)o * I;
device const float4* w4 = (device const float4*)wr;
@@ -82,8 +111,9 @@ kernel void mm_gemv(device const uchar* w [[buffer(0)]], // raw weight by
for (int i = I8*8 + slane; i < I; i += 32) acc += wr[i] * xr[i];
}
acc = simd_sum(acc);
- // fmt==4 already folded the per-group scale into acc above -- do not scale again.
- if (slane == 0) y[row] = (fmt == 4) ? acc : acc * scale[o];
+ // fmt==4 (per-group) and fmt==7 (per-block) already folded their scale into acc
+ // above -- do not scale again.
+ if (slane == 0) y[row] = (fmt == 4 || fmt == 7) ? acc : acc * scale[o];
}
// Batched bindless expert GEMV: each row gr belongs to expert erow[gr], whose weight and
@@ -447,11 +477,20 @@ static size_t fmt_bytes(int fmt, int I, int O) {
if (fmt == 2) return (size_t)O * ((I+1)/2);
if (fmt == 3) return (size_t)O * ((I+3)/4);
if (fmt == 4) return (size_t)O * ((I+1)/2); // grouped int4: identical packed-nibble layout to fmt=2
+ if (fmt == 7) return (size_t)O * I; // fp8 e4m3: one raw byte/element, same as fmt=1
return (size_t)O * I * sizeof(float);
}
// Grouped-int4 (fmt=4) scale-array size: one f32 per gsz-element group, per row -> O*ceil(I/gsz).
+// fp8 (fmt=7) scale-array size: one f32 per 128x128 BLOCK -> ceil(O/128)*ceil(I/128) (2D,
+// not per-row -- quant.h isn't included here, so the ceil-div is inlined rather than sharing
+// colibri.c's qt_scale_bytes/quant.h's fp8_nblk). The block is a fixed 128x128, so gs is
+// ignored for fmt==7. f32 is this build's implemented scale
+// ENCODING for fmt=7 (see quant.h/colibri.c) -- this file has no reason to know that a
+// UE8M0 encoding exists at all: qt_resolve_fmt refuses it on the CPU read path before any
+// tensor in that encoding could ever reach this Metal-side sizing helper.
static size_t fmt_scale_bytes(int fmt, int I, int O, int gs) {
if (fmt == 4) return (size_t)O * ((I + gs - 1) / gs) * sizeof(float);
+ if (fmt == 7) return (size_t)((O + 127) / 128) * (size_t)((I + 127) / 128) * sizeof(float);
return (size_t)O * sizeof(float);
}
@@ -619,7 +658,9 @@ extern "C" int coli_metal_mem_info(size_t *used, size_t *total) {
extern "C" int coli_metal_matmul(ColiMetalTensor **tp, float *y, const float *x,
const void *weights, const float *scales,
int fmt, int S, int I, int O, int gs) {
- if (!g_dev || fmt < 0 || fmt > 4) return 0;
+ /* fmt==7 (fp8 passthrough) is an explicit allow-list entry, not folded into the 0..4
+ * contiguous range check below: it is not adjacent to it. */
+ if (!g_dev || fmt < 0 || (fmt > 4 && fmt != 7)) return 0;
@autoreleasepool {
ColiMetalTensor *t = *tp;
if (!t) {
diff --git a/c/tests/test_backend_metal.mm b/c/tests/test_backend_metal.mm
index 7d8abfdb..59c0f0cf 100644
--- a/c/tests/test_backend_metal.mm
+++ b/c/tests/test_backend_metal.mm
@@ -10,7 +10,7 @@
#include <cmath>
#include <vector>
-enum { F32=0, I8=1, I4=2, I2=3, I4G=4 };
+enum { F32=0, I8=1, I4=2, I2=3, I4G=4, FP8=7 };
static void cpu_ref(int fmt, const void *W, const float *s, const float *x,
float *y, int S, int I, int O) {
@@ -80,6 +80,41 @@ static void cpu_ref_grouped(const uint8_t *q4, const float *scale, const float *
}
}
+// ---- fmt=7 (native FP8-e4m3 passthrough -- see colibri.c): own CPU reference +
+// harness --------------------------------------------------------------------------
+// Independent reference decode (bit manipulation, NOT quant.h's E4M3_LUT -- this file
+// stays self-contained like the rest of its CPU references) for OCP E4M3-FN: exp==0 is
+// subnormal, exp==0xF&&mant==0x7 is the only NaN code (both signs), else normal with
+// bias 7. Must match quant.h's e4m3_decode AND the mm_gemv fmt==7 branch's in-kernel
+// bit manipulation exactly -- see run_fp8_lut() below for the exhaustive 256-code check
+// that proves the GPU kernel agrees with this reference (and by transitivity with the
+// CPU LUT, which was cross-checked byte-for-byte against torch.float8_e4m3fn offline).
+static float ref_e4m3(uint8_t b) {
+ unsigned sign=(b>>7)&1, exp=(b>>3)&0xF, mant=b&0x7;
+ if (exp==0xF && mant==0x7) return NAN;
+ float val = (exp==0) ? (float)mant*(1.0f/8.0f)*powf(2.0f,1.0f-7.0f)
+ : (1.0f+(float)mant*(1.0f/8.0f))*powf(2.0f,(float)exp-7.0f);
+ return sign ? -val : val;
+}
+static int fp8_nblk(int n){ return (n+127)/128; }
+
+static void cpu_ref_fp8(const uint8_t *q8, const float *bscale, const float *x,
+ double *y, double *mag, int S, int I, int O) {
+ int nblkI = fp8_nblk(I);
+ for (int o=0;o<O;o++){
+ const uint8_t *w = q8 + (size_t)o*I;
+ const float *scl = bscale + (size_t)(o/128)*nblkI;
+ for (int s=0;s<S;s++){
+ const float *xs = x + (size_t)s*I; double a=0, m=0;
+ for (int i=0;i<I;i++){
+ double term = (double)ref_e4m3(w[i]) * (double)xs[i] * (double)scl[i/128];
+ a += term; m += fabs(term);
+ }
+ y[(size_t)s*O+o] = a; mag[(size_t)s*O+o] = m;
+ }
+ }
+}
+
// Tolerance: magnitude-relative (error / sum of |terms|), NOT the ymax-relative-max-
// abs-error convention run() above uses. Reason: a dot product over many groups can
// land near zero from signed-term cancellation, and a fixed-point-ish absolute GPU/CPU
@@ -133,6 +168,107 @@ static int run_grouped(int O, int I, int gs, int S, int outlier, const char *nam
return ok?0:1;
}
+// Tolerance/oracle convention identical to run_grouped() above (magnitude-relative for
+// the block-scale accumulation, avoiding cancellation-noise false positives on
+// near-zero results).
+static int run_fp8(int O, int I, int S, const char *name) {
+ int nblkO=fp8_nblk(O), nblkI=fp8_nblk(I), nblk=nblkO*nblkI;
+ std::vector<uint8_t> W((size_t)O*I);
+ std::vector<float> scale((size_t)nblk), x((size_t)S*I), yg((size_t)S*O);
+ std::vector<double> yr((size_t)S*O), mag((size_t)S*O);
+ srand(5150 + I*7 + O*3 + S*13);
+ for (auto &b : W) { do { b = (uint8_t)(rand()&0xFF); } while (b==0x7F || b==0xFF); } // exclude NaN codes
+ // distinct, easily-distinguishable scale per block (stride-audit idiom, same as
+ // test_fp8_passthrough.c's CPU version): a swapped o/128 vs i/128 stride, or a wrong
+ // nblkI in the shader's indexing, lands on a DIFFERENT block's scale and shows up as
+ // a loud numeric mismatch rather than passing by luck.
+ for (int b=0;b<nblk;b++) scale[b] = (float)(b+1)*0.01f*((b&1)?-1.f:1.f);
+ for (auto &v : x) v = ((rand()%2000)-1000)/1000.0f;
+ cpu_ref_fp8(W.data(), scale.data(), x.data(), yr.data(), mag.data(), S, I, O);
+ ColiMetalTensor *t=nullptr;
+ if (!coli_metal_matmul(&t, yg.data(), x.data(), W.data(), scale.data(), FP8, S, I, O, 0)) {
+ printf(" %-42s FAIL (matmul returned 0)\n", name); return 1; }
+ double worst=0; int bad=0;
+ for (size_t i=0; i<(size_t)S*O; i++) {
+ double d = fabs((double)yg[i] - yr[i]);
+ double rel = mag[i] > 1e-30 ? d/mag[i] : d;
+ if (rel > worst) worst = rel;
+ if (rel > 1e-4) bad++;
+ }
+ int ok = (bad == 0);
+ printf(" %-42s worst_rel=%.2e (I=%d O=%d nblkO=%d nblkI=%d S=%d) %s\n",
+ name, worst, I, O, nblkO, nblkI, S, ok?"ok":"*** MISMATCH");
+ coli_metal_tensor_free(t);
+ return ok?0:1;
+}
+
+// Exhaustive LUT-exactness check run THROUGH the actual GPU kernel: O=256,I=1 means
+// each of the 256 output rows has exactly ONE weight byte, set to that row's own index
+// (0..255) -- so row o's dequant is exactly e4m3_decode(o). x=[1.0] and both blocks'
+// scale=1.0 isolate the decode from the block-scale/accumulation logic entirely. This
+// mirrors test_fp8_passthrough.c's CPU LUT-exactness test (test_lut), but exercised
+// through mm_gemv's in-kernel bit manipulation instead of quant.h's table -- proving
+// the two independently-written decoders agree on all 256 codes, including both NaN
+// codes and the sign of zero.
+static int run_fp8_lut(const char *name) {
+ enum { O=256, I=1 };
+ std::vector<uint8_t> W(O*I); for (int b=0;b<O;b++) W[b]=(uint8_t)b;
+ std::vector<float> scale(fp8_nblk(O)*fp8_nblk(I), 1.0f); // nblkO=2,nblkI=1 -> both blocks scale=1
+ std::vector<float> x(I, 1.0f), yg(O);
+ ColiMetalTensor *t=nullptr;
+ if (!coli_metal_matmul(&t, yg.data(), x.data(), W.data(), scale.data(), FP8, 1, I, O, 0)) {
+ printf(" %-42s FAIL (matmul returned 0)\n", name); return 1; }
+ int bad=0;
+ for (int b=0;b<O;b++) {
+ float ref = ref_e4m3((uint8_t)b);
+ if (b==0x7F || b==0xFF) { if (!std::isnan(yg[b])) bad++; continue; }
+ if (yg[b] != ref) bad++;
+ }
+ int ok = (bad==0);
+ printf(" %-42s %d/256 mismatches %s\n", name, bad, ok?"ok":"*** MISMATCH");
+ coli_metal_tensor_free(t);
+ return ok?0:1;
+}
+
+// Confirms the documented "GEMM path optional this build" claim is actually true, not
+// just asserted in a comment: coli_metal_gemm must refuse fmt=7 (return 0, the CPU-
+// fallback signal) so matmul_qt_ex's caller-side allowlist exclusion is backed by a
+// second, independent gate inside the Metal backend itself.
+static int run_fp8_gemm_gate(const char *name) {
+ uint8_t w[8]={0}; float s[1]={1.0f}, x[8]={0}, y[1]={0};
+ int rc = coli_metal_gemm(y, x, w, s, FP8, 1, 8, 1, 0);
+ int ok = (rc == 0);
+ printf(" %-42s rc=%d (expect 0/CPU-fallback) %s\n", name, rc, ok?"ok":"*** MISMATCH (should have refused)");
+ return ok?0:1;
+}
+
+// colibri.c's moe() has a THIRD fmt=7-adjacent Metal entry point besides the two
+// guarded above (bind_gemv's attn/layer-decode shaders and coli_metal_gemm) -- the
+// batched routed-expert dispatch via coli_metal_moe_block[_begin] (colibri.c's
+// MB_BUILD macro). MB_BUILD's own pointer-selection ternary does NOT special-case
+// fmt=7: if a layer's shared expert is fmt=7 and MB_BUILD's TRY_SH path picks it
+// up with no routed expert already fixing `mfmt`, it would submit the WRONG pointer
+// (q4, NULL/stale for an fmt=7 tensor whose weights live in q8) tagged as fmt=7.
+// This is safe ANYWAY, but only incidentally: moe_submit() (backend_metal.mm) gates
+// `fmt != 1 && fmt != 2` as its very FIRST statement, before any of g/u/d/gs/us/ds is
+// dereferenced or even resolve()'d -- so an fmt=7 submission is refused before the
+// bad pointer would ever be read, no matter what garbage MB_BUILD packed into it. This
+// test uses deliberately-invalid weight/scale pointers (never dereferenced if the gate
+// holds) to prove the fence BY TEST rather than leaving it an artifact of moe_submit's
+// fmt allowlist happening not to include 7 (yet) -- same discipline
+// run_fp8_gemm_gate above applies to coli_metal_gemm's analogous exclusion.
+static int run_fp8_moe_gate(const char *name) {
+ const void *bad = (const void*)(uintptr_t)0xdeadbeef; // must NEVER be dereferenced
+ const void *g[1] = {bad}, *u[1] = {bad}, *d[1] = {bad};
+ const float *gs[1] = {(const float*)bad}, *us[1] = {(const float*)bad}, *ds[1] = {(const float*)bad};
+ float xg[8]={0}, out[8]={0}, rw[1]={1.0f};
+ int xoff[1]={0}, nr[1]={1}, rows[1]={0};
+ int rc = coli_metal_moe_block(1, 8, 8, FP8, g, u, d, gs, us, ds, xg, xoff, nr, rows, rw, out, 1);
+ int ok = (rc == 0);
+ printf(" %-42s rc=%d (expect 0/CPU-fallback) %s\n", name, rc, ok?"ok":"*** MISMATCH (should have refused)");
+ return ok?0:1;
+}
+
static float deq4(const uint8_t* w,int i){ uint8_t b=w[i>>1]; int v=(i&1)?(b>>4):(b&0xF); return (float)(v-8); }
static size_t roundpg(size_t n){ size_t p=16384; return ((n+p-1)/p)*p; }
@@ -463,6 +599,22 @@ int main(void) {
// tail-clipping regime grouped scales exist for), verified end-to-end through the GPU.
fail |= run_grouped(5, 512, 64,3,1, "grouped I=512(mult64) outlier-heavy S=3");
fail |= run_grouped(5, 201, 64,2,1, "grouped I=201(non-mult-64) outlier-heavy S=2");
+ printf("Metal fmt=7 native FP8-e4m3 passthrough tests:\n");
+ fail |= run_fp8_lut("fp8 LUT exactness (256/256 codes via GPU kernel)");
+ fail |= run_fp8(2048,6144,1, "fp8 gate/up-shaped O=2048 I=6144 (spec example) S=1");
+ fail |= run_fp8(6144,2048,1, "fp8 down-shaped O=6144 I=2048 S=1");
+ fail |= run_fp8(2048,6144,4, "fp8 gate/up-shaped O=2048 I=6144 S=4");
+ // block edges: O,I not multiples of 128 (the partial-tile clamp)
+ fail |= run_fp8(130, 200, 2, "fp8 block edges: O,I both non-mult-128");
+ fail |= run_fp8(129, 128, 1, "fp8 block edges: O just over 128, I exact");
+ fail |= run_fp8(128, 129, 1, "fp8 block edges: O exact, I just over 128");
+ fail |= run_fp8(1, 1, 1, "fp8 degenerate 1x1 (single sub-block)");
+ // non-square block grid (nblkO=3 != nblkI=48) with a distinct scale per block: a
+ // swapped o/128 vs i/128 index, or a wrong nblkI stride, lands on the wrong block's
+ // scale and this shape/scale choice makes that numerically loud.
+ fail |= run_fp8(384, 6144, 3, "fp8 non-square block grid nblkO=3 nblkI=48 (stride audit)");
+ fail |= run_fp8_gemm_gate("fp8 GEMM entry explicitly gated off (coli_metal_gemm refuses)");
+ fail |= run_fp8_moe_gate("fp8 MB_BUILD/moe_submit entry gated off (shared-expert fmt=7 hazard)");
printf("Metal batched moe_block tests:\n");
fail |= run_moe({1,1,1,1,1,1,1,1}, "moe decode nb=8");
fail |= run_moe({3,1,4,2,1,5}, "moe ragged nb=6");