Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

metal: fmt=6 (E8/IQ3) expert path β€” GPU decode + on-GPU FWHT rotation

Michael Denyercdd6ed4df947
diff --git a/c/backend_metal.mm b/c/backend_metal.mm
index f13e7432..489ad01f 100644
--- a/c/backend_metal.mm
+++ b/c/backend_metal.mm
@@ -17,6 +17,43 @@ static const char *SHADER = R"METAL(
#include <metal_stdlib>
using namespace metal;
+// fmt=6 E8/IQ3 magnitude grid β€” generated from quant.h e8_grid (must stay identical;
+// the metal-test oracle compares against the CPU decoder, so drift fails the build).
+constant uchar4 E8G[256] = {
+ uchar4(4,4,4,4),uchar4(20,4,4,4),uchar4(36,4,4,4),uchar4(12,12,4,4),uchar4(28,12,4,4),uchar4(62,12,4,4),uchar4(4,20,4,4),uchar4(20,20,4,4),
+ uchar4(12,28,4,4),uchar4(20,36,4,4),uchar4(28,62,4,4),uchar4(44,62,4,4),uchar4(12,4,12,4),uchar4(28,4,12,4),uchar4(4,12,12,4),uchar4(20,12,12,4),
+ uchar4(12,20,12,4),uchar4(44,20,12,4),uchar4(4,28,12,4),uchar4(20,28,12,4),uchar4(12,36,12,4),uchar4(36,44,12,4),uchar4(4,62,12,4),uchar4(4,4,20,4),
+ uchar4(20,4,20,4),uchar4(36,4,20,4),uchar4(12,12,20,4),uchar4(4,20,20,4),uchar4(20,20,20,4),uchar4(12,28,20,4),uchar4(28,28,20,4),uchar4(62,28,20,4),
+ uchar4(12,44,20,4),uchar4(62,44,20,4),uchar4(44,62,20,4),uchar4(12,4,28,4),uchar4(62,4,28,4),uchar4(4,12,28,4),uchar4(20,12,28,4),uchar4(44,20,28,4),
+ uchar4(4,62,28,4),uchar4(28,12,36,4),uchar4(62,28,36,4),uchar4(36,36,36,4),uchar4(62,44,36,4),uchar4(28,62,36,4),uchar4(44,62,36,4),uchar4(12,4,44,4),
+ uchar4(62,4,44,4),uchar4(20,28,44,4),uchar4(20,44,44,4),uchar4(44,28,52,4),uchar4(36,52,52,4),uchar4(4,12,62,4),uchar4(36,12,62,4),uchar4(52,12,62,4),
+ uchar4(28,36,62,4),uchar4(12,52,62,4),uchar4(12,4,4,12),uchar4(28,4,4,12),uchar4(4,12,4,12),uchar4(20,12,4,12),uchar4(12,20,4,12),uchar4(28,20,4,12),
+ uchar4(4,28,4,12),uchar4(20,28,4,12),uchar4(36,28,4,12),uchar4(62,36,4,12),uchar4(4,44,4,12),uchar4(4,4,12,12),uchar4(20,4,12,12),uchar4(12,12,12,12),
+ uchar4(4,20,12,12),uchar4(20,20,12,12),uchar4(12,4,20,12),uchar4(28,4,20,12),uchar4(4,12,20,12),uchar4(20,12,20,12),uchar4(12,20,20,12),uchar4(4,28,20,12),
+ uchar4(20,62,20,12),uchar4(4,4,28,12),uchar4(20,4,28,12),uchar4(4,20,28,12),uchar4(12,28,28,12),uchar4(52,36,28,12),uchar4(52,52,28,12),uchar4(12,4,36,12),
+ uchar4(44,4,36,12),uchar4(4,44,36,12),uchar4(4,20,44,12),uchar4(36,20,44,12),uchar4(52,36,44,12),uchar4(12,62,44,12),uchar4(44,4,52,12),uchar4(20,20,62,12),
+ uchar4(4,36,62,12),uchar4(4,4,4,20),uchar4(20,4,4,20),uchar4(12,12,4,20),uchar4(28,12,4,20),uchar4(4,20,4,20),uchar4(20,20,4,20),uchar4(52,20,4,20),
+ uchar4(12,28,4,20),uchar4(20,36,4,20),uchar4(12,4,12,20),uchar4(28,4,12,20),uchar4(44,4,12,20),uchar4(4,12,12,20),uchar4(20,12,12,20),uchar4(12,20,12,20),
+ uchar4(4,28,12,20),uchar4(28,52,12,20),uchar4(62,52,12,20),uchar4(4,62,12,20),uchar4(4,4,20,20),uchar4(20,4,20,20),uchar4(12,12,20,20),uchar4(62,12,20,20),
+ uchar4(4,20,20,20),uchar4(20,20,20,20),uchar4(62,28,20,20),uchar4(4,36,20,20),uchar4(44,44,20,20),uchar4(12,4,28,20),uchar4(4,12,28,20),uchar4(36,12,28,20),
+ uchar4(4,62,28,20),uchar4(36,62,28,20),uchar4(44,28,36,20),uchar4(28,44,36,20),uchar4(28,4,44,20),uchar4(62,20,44,20),uchar4(12,36,44,20),uchar4(36,62,44,20),
+ uchar4(12,4,62,20),uchar4(28,4,62,20),uchar4(52,12,62,20),uchar4(44,36,62,20),uchar4(12,4,4,28),uchar4(4,12,4,28),uchar4(20,12,4,28),uchar4(12,20,4,28),
+ uchar4(28,20,4,28),uchar4(4,44,4,28),uchar4(44,52,4,28),uchar4(20,62,4,28),uchar4(4,4,12,28),uchar4(20,4,12,28),uchar4(4,20,12,28),uchar4(12,28,12,28),
+ uchar4(36,36,12,28),uchar4(52,36,12,28),uchar4(12,4,20,28),uchar4(28,4,20,28),uchar4(4,12,20,28),uchar4(44,20,20,28),uchar4(20,44,20,28),uchar4(20,62,20,28),
+ uchar4(12,12,28,28),uchar4(28,28,28,28),uchar4(4,28,36,28),uchar4(62,36,36,28),uchar4(20,62,36,28),uchar4(4,4,44,28),uchar4(52,4,44,28),uchar4(20,20,44,28),
+ uchar4(44,44,44,28),uchar4(36,12,52,28),uchar4(52,28,52,28),uchar4(28,52,52,28),uchar4(28,28,62,28),uchar4(4,52,62,28),uchar4(36,4,4,36),uchar4(62,12,4,36),
+ uchar4(44,28,4,36),uchar4(62,28,4,36),uchar4(28,44,4,36),uchar4(62,44,4,36),uchar4(36,62,12,36),uchar4(4,20,20,36),uchar4(62,28,20,36),uchar4(4,36,20,36),
+ uchar4(4,52,20,36),uchar4(52,52,20,36),uchar4(62,4,28,36),uchar4(44,36,28,36),uchar4(36,4,36,36),uchar4(12,44,36,36),uchar4(36,52,36,36),uchar4(44,20,44,36),
+ uchar4(28,36,44,36),uchar4(4,62,44,36),uchar4(44,4,62,36),uchar4(4,12,62,36),uchar4(20,12,62,36),uchar4(4,28,62,36),uchar4(20,12,4,44),uchar4(12,36,4,44),
+ uchar4(4,62,4,44),uchar4(4,4,12,44),uchar4(52,4,12,44),uchar4(52,20,12,44),uchar4(44,44,12,44),uchar4(36,12,20,44),uchar4(20,28,20,44),uchar4(20,62,20,44),
+ uchar4(20,4,28,44),uchar4(28,44,28,44),uchar4(4,12,36,44),uchar4(28,20,36,44),uchar4(62,20,36,44),uchar4(20,62,36,44),uchar4(20,4,44,44),uchar4(12,28,44,44),
+ uchar4(4,44,52,44),uchar4(36,20,62,44),uchar4(20,36,62,44),uchar4(36,20,4,52),uchar4(36,36,4,52),uchar4(52,36,4,52),uchar4(36,52,4,52),uchar4(12,20,12,52),
+ uchar4(12,52,12,52),uchar4(62,12,20,52),uchar4(36,52,20,52),uchar4(4,28,28,52),uchar4(52,28,28,52),uchar4(36,36,36,52),uchar4(44,4,44,52),uchar4(20,44,44,52),
+ uchar4(28,28,52,52),uchar4(28,4,62,52),uchar4(12,20,62,52),uchar4(28,4,4,62),uchar4(44,4,4,62),uchar4(62,4,4,62),uchar4(4,12,4,62),uchar4(20,28,4,62),
+ uchar4(20,44,4,62),uchar4(52,20,12,62),uchar4(4,36,12,62),uchar4(20,12,20,62),uchar4(44,36,20,62),uchar4(20,44,20,62),uchar4(4,4,28,62),uchar4(44,12,28,62),
+ uchar4(28,28,28,62),uchar4(4,52,28,62),uchar4(12,20,36,62),uchar4(12,36,36,62),uchar4(4,4,44,62),uchar4(20,4,44,62),uchar4(36,20,44,62),uchar4(4,28,52,62)
+};
+
kernel void mm_gemv(device const uchar* w [[buffer(0)]], // raw weight bytes
device const float* scale [[buffer(1)]], // [O] (fmt<4) or [O,ceil(I/gsz)] (fmt==4)
device const float* x [[buffer(2)]], // [S,I]
@@ -112,13 +149,59 @@ kernel void moe_gemv(device const ulong* waddr [[buffer(0)]], device const ulong
float4 w1=float4(float(int(b.z&0xF)-8),float(int(b.z>>4)-8),float(int(b.w&0xF)-8),float(int(b.w>>4)-8));
acc+=dot(w0,x4[2*c])+dot(w1,x4[2*c+1]); }
for(int i=K8*8+slane;i<K;i+=32){ uchar b=w[i>>1]; int v=(i&1)?(b>>4):(b&0xF); acc+=float(v-8)*xr[i]; }
+ } else if (fmt == 6) { // E8/IQ3: 98B per 256 weights, scales in-block
+ long rb=((long)(K+255)/256)*98; // host guards K%256==0 (GLM dims are)
+ device const uchar* w=(device const uchar*)(waddr[e])+(long)o*rb;
+ int nsub=K/32; // one 32-weight sub-block per lane step
+ for(int s6=slane;s6<nsub;s6+=32){
+ int b=s6>>3, ib=s6&7; device const uchar* blk=w+(long)b*98;
+ uint word = uint(blk[64+ib*4]) | (uint(blk[65+ib*4])<<8)
+ | (uint(blk[66+ib*4])<<16) | (uint(blk[67+ib*4])<<24);
+ ushort dh = ushort(blk[96]) | (ushort(blk[97])<<8);
+ float db = float(as_type<half>(dh)) * (0.5f + float((word>>28)&0xFu)) * 0.5f;
+ device const uchar* idx = blk + ib*8;
+ device const float4* xs = (device const float4*)(xr + s6*32);
+ for(int l=0;l<4;l++){
+ uint sv=(word>>(7*l))&0x7Fu;
+ float4 m0=float4(E8G[idx[l*2+0]]), m1=float4(E8G[idx[l*2+1]]);
+ float4 sA=float4((sv&1u)?-1.0f:1.0f,(sv&2u)?-1.0f:1.0f,(sv&4u)?-1.0f:1.0f,(sv&8u)?-1.0f:1.0f);
+ float4 sB=float4((sv&16u)?-1.0f:1.0f,(sv&32u)?-1.0f:1.0f,(sv&64u)?-1.0f:1.0f,
+ (popcount(sv)&1u)?-1.0f:1.0f); // j=7 closes by odd parity
+ acc += (dot(m0*sA,xs[l*2]) + dot(m1*sB,xs[l*2+1])) * (0.5f*db);
+ }
+ }
} else { device const char* w=(device const char*)(waddr[e])+(long)o*K;
device const char4* w4=(device const char4*)w;
for(int c=slane;c<K8;c+=32) acc+=dot(float4(w4[2*c]),x4[2*c])+dot(float4(w4[2*c+1]),x4[2*c+1]);
for(int i=K8*8+slane;i<K;i+=32) acc+=float(w[i])*xr[i];
}
acc=simd_sum(acc);
- if(slane==0) yout[row]=acc*sc[o];
+ if(slane==0) yout[row] = (fmt==6) ? acc : acc*sc[o]; // fmt=6: scales live in-block
+}
+
+// fmt=6 activation rotation for the GPU-resident down-projection input: one FWHT
+// tile per dispatch (block-diagonal tiling and sign stream match quant.h e8_rot_rows;
+// signs are regenerated host-side with e8_signs and passed in). One threadgroup per
+// row; tile fits threadgroup memory (n <= 4096).
+kernel void moe_fwht(device float* v [[buffer(0)]], device const uchar* signs [[buffer(1)]],
+ constant int& dim [[buffer(2)]], constant int& off [[buffer(3)]],
+ constant int& n [[buffer(4)]],
+ uint tg [[threadgroup_position_in_grid]],
+ uint t [[thread_position_in_threadgroup]],
+ uint nt [[threads_per_threadgroup]]) {
+ threadgroup float sh[4096];
+ device float* row = v + (long)tg*dim + off;
+ for (int i=int(t);i<n;i+=int(nt)){ float x=row[i]; if((signs[i>>3]>>(i&7))&1u) x=-x; sh[i]=x; }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ for (int len=1;len<n;len<<=1){
+ for (int j=int(t);j<n/2;j+=int(nt)){
+ int blk=j/len, k=j%len, i0=blk*(len<<1)+k;
+ float a=sh[i0], b=sh[i0+len]; sh[i0]=a+b; sh[i0+len]=a-b;
+ }
+ threadgroup_barrier(mem_flags::mem_threadgroup);
+ }
+ float s=rsqrt(float(n));
+ for (int i=int(t);i<n;i+=int(nt)) row[i]=sh[i]*s;
}
kernel void moe_silu(device float* g [[buffer(0)]], device const float* u [[buffer(1)]],
uint i [[thread_position_in_grid]]) { float v=g[i]; g[i]=(v/(1.0f+exp(-v)))*u[i]; }
@@ -309,7 +392,30 @@ struct ColiMetalTensor {
static id<MTLDevice> g_dev;
static id<MTLCommandQueue> g_queue;
-static id<MTLComputePipelineState> g_gemv, g_moe_gemv, g_moe_silu;
+static id<MTLComputePipelineState> g_gemv, g_moe_gemv, g_moe_silu, g_moe_fwht;
+
+// fmt=6: sign-bit buffers for the GPU FWHT, one per tile size, cached forever (a
+// handful of sizes). The xorshift64* draw replicates quant.h e8_signs exactly β€”
+// the metal-test fmt=6 oracle compares end-to-end against quant.h, so any drift
+// between the two copies fails the build.
+static void e8_signs_local(uint8_t *bits, int n) {
+ uint64_t s = 417u + (uint64_t)n;
+ for (int i = 0; i < (n+7)/8; i++) {
+ s ^= s>>12; s ^= s<<25; s ^= s>>27;
+ bits[i] = (uint8_t)((s*2685821657736338717ULL)>>56);
+ }
+}
+static id<MTLBuffer> fwht_signs(int n) {
+ static std::mutex mtx; static std::vector<std::pair<int, id<MTLBuffer>>> cache;
+ std::lock_guard<std::mutex> lk(mtx);
+ for (auto &p : cache) if (p.first == n) return p.second;
+ std::vector<uint8_t> bits((n+7)/8);
+ e8_signs_local(bits.data(), n);
+ id<MTLBuffer> b = [g_dev newBufferWithBytes:bits.data() length:bits.size()
+ options:MTLResourceStorageModeShared];
+ cache.push_back({n, b});
+ return b;
+}
static id<MTLComputePipelineState> g_a_rms, g_a_rope, g_a_copy, g_a_qabs, g_a_score, g_a_smax, g_a_clat, g_a_ctx;
static id<MTLComputePipelineState> g_a_add, g_r_router, g_r_top8, g_r_top8p;
static int g_rtop8_par = 1; // COLI_RTOP8 (default ON); COLI_RTOP8=0 opts out to the
@@ -481,6 +587,7 @@ extern "C" int coli_metal_init(void) {
g_gemv = [g_dev newComputePipelineStateWithFunction:[lib newFunctionWithName:@"mm_gemv"] error:&err];
g_moe_gemv = [g_dev newComputePipelineStateWithFunction:[lib newFunctionWithName:@"moe_gemv"] error:&err];
g_moe_silu = [g_dev newComputePipelineStateWithFunction:[lib newFunctionWithName:@"moe_silu"] error:&err];
+ g_moe_fwht = [g_dev newComputePipelineStateWithFunction:[lib newFunctionWithName:@"moe_fwht"] error:&err];
auto P=[&](const char*n){ return [g_dev newComputePipelineStateWithFunction:[lib newFunctionWithName:@(n)] error:&err]; };
g_a_rms=P("a_rmsnorm"); g_a_rope=P("a_rope"); g_a_copy=P("a_copy");
g_a_qabs=P("a_qabs"); g_a_score=P("a_score"); g_a_smax=P("a_smax"); g_a_clat=P("a_clat"); g_a_ctx=P("a_ctx");
@@ -501,7 +608,7 @@ extern "C" int coli_metal_init(void) {
"r_top8 in use\n", (unsigned long)[g_r_top8p threadExecutionWidth]);
g_rtop8_par = 0;
}
- if (!g_gemv || !g_moe_gemv || !g_moe_silu || !g_a_rms || !g_a_rope || !g_a_copy ||
+ if (!g_gemv || !g_moe_gemv || !g_moe_silu || !g_moe_fwht || !g_a_rms || !g_a_rope || !g_a_copy ||
!g_a_qabs || !g_a_score || !g_a_smax || !g_a_clat || !g_a_ctx) {
fprintf(stderr, "[metal] pipeline failed\n"); g_dev = nil; return 0; }
// E5 experiment: COLI_METAL_RESSET=1 -- see g_resset_obj comment above.
@@ -995,7 +1102,17 @@ static id<MTLCommandBuffer> moe_submit(int nb, int D, int Iinter, int fmt,
const float *const *gs, const float *const *us, const float *const *ds,
const float *xg, const int *xoff, const int *nr, int R,
id<MTLBuffer> xg_buf, id<MTLBuffer> gg_buf, id<MTLBuffer> uu_buf, id<MTLBuffer> hh_buf) {
- if (!g_dev || (fmt != 1 && fmt != 2)) return nil;
+ if (!g_dev || (fmt != 1 && fmt != 2 && fmt != 6)) return nil;
+ if (fmt == 6) { /* e8 kernel assumes clean block tiling, and every FWHT tile of the
+ * down input (CPU tiling rule, e8_rot_rows) must fit threadgroup mem */
+ if ((D & 255) || (Iinter & 31)) return nil;
+ for (int off = 0; off < Iinter; ) {
+ int rem = Iinter - off, n = rem & (-rem);
+ while (n > 32768) n >>= 1;
+ if (n > 4096) return nil;
+ off += n;
+ }
+ }
if (g_resset_enabled) { // E5: commit any pending slab adds before we may skip useResource:
double t0 = mnow(); resset_flush(); g_t_resset_flush += mnow() - t0; // METAL-RESSET line
}
@@ -1047,6 +1164,23 @@ static id<MTLCommandBuffer> moe_submit(int nb, int D, int Iinter, int fmt,
[e setBuffer:gg_buf offset:0 atIndex:0];[e setBuffer:uu_buf offset:0 atIndex:1];
[e dispatchThreads:MTLSizeMake((size_t)R*Iinter,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)];
[e memoryBarrierWithScope:MTLBarrierScopeBuffers];
+ if (fmt == 6) { /* rotate the down-projection input in place: same block-diagonal
+ * tiling as quant.h e8_rot_rows (largest power of two dividing the
+ * remainder, capped at 4096 for threadgroup memory) */
+ int off = 0;
+ while (off < Iinter) {
+ int rem = Iinter - off, n = rem & (-rem);
+ while (n > 32768) n >>= 1; /* CPU tiling rule (e8_rot_rows); sizes pre-validated above */
+ id<MTLBuffer> sb = fwht_signs(n);
+ [e setComputePipelineState:g_moe_fwht];
+ [e setBuffer:gg_buf offset:0 atIndex:0];[e setBuffer:sb offset:0 atIndex:1];
+ [e setBytes:&Iinter length:4 atIndex:2];[e setBytes:&off length:4 atIndex:3];
+ [e setBytes:&n length:4 atIndex:4];
+ [e dispatchThreadgroups:MTLSizeMake((size_t)R,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)];
+ off += n;
+ }
+ [e memoryBarrierWithScope:MTLBarrierScopeBuffers];
+ }
gemv(bad,bsd,gg_buf,hh_buf,D,Iinter,Iinter); // down
g_t_setup += mnow() - ts_start;
[e endEncoding];[cb commit];
diff --git a/c/colibri.c b/c/colibri.c
index 65acbf30..0c92c22c 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -3127,6 +3127,18 @@ static int router_best_or_fallback(int best, int kk, int E, int layer){
return kk<E ? kk : 0;
}
+#ifdef COLI_METAL
+/* Rotate the Metal-staged gate/up input rows for fmt=6 (Q^T x). Duplicate rows
+ * (same source s) are copied from the first rotated instance: O(p^2) scan, p<=65. */
+static void metal_stage_rot_e8(float *mxg, const int *mrows, int p, int D){
+ for(int i=0;i<p;i++){
+ int j=-1; for(int k=0;k<i;k++) if(mrows[k]==mrows[i]){ j=k; break; }
+ if(j>=0) memcpy(mxg+(int64_t)i*D, mxg+(int64_t)j*D, (size_t)D*sizeof(float));
+ else e8_rot_rows(mxg+(int64_t)i*D, 1, D);
+ }
+}
+#endif
+
static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int with_shared){
if(g_pilot_real){ /* barriera cross-layer: prendi possesso di QUESTO layer e aspetta
* l'eventuale load-pilota in volo sullo stesso layer (dopodiche' il
@@ -3536,6 +3548,10 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
* the first block) to the GPU BEFORE loading the missed experts from disk, so the
* preads run while the GPU computes; the missed subset follows in a second submit.
* Per-subset CPU fallback on unresolved slab / bad fmt / GPU fault. */
+ /* fmt=6 stores W@Q, so the staged gate/up input must be rotated (Q^T x) before
+ * upload. Rotate each distinct source row once, copy to duplicates β€” a decode
+ * batch's rows all share s, so this is one FWHT per token in practice. The down
+ * input is rotated on-GPU by moe_fwht inside moe_submit. */
int is_miss[64]={0}; ColiMetalMoeHandle *mh=NULL;
int cpu_res=1, cpu_miss=1, mh_shared=0, nbb=0, Rtot=0, mfmt=-1, sh_in=0;
const void *MG[65],*MU[65],*MD[65]; const float *MGS[65],*MUS[65],*MDS[65];
@@ -3581,6 +3597,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
MB_BUILD(0, base==0 && !g_pre_sh);
if(nbb>0){
double t0=now_s();
+ if(mfmt==6) metal_stage_rot_e8(mxg,mrows,Rtot,D);
mh=coli_metal_moe_block_begin(nbb,D,I,mfmt,MG,MU,MD,MGS,MUS,MDS,mxg,xoffb,nrb,mrows,mrw);
m->t_emm += now_s()-t0;
if(mh){ cpu_res=0; mh_shared=sh_in; }
@@ -3643,6 +3660,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
MB_BUILD(1, 0); /* missed experts, now loaded */
if(nbb>0){
double t0=now_s();
+ if(mfmt==6) metal_stage_rot_e8(mxg,mrows,Rtot,D);
if(coli_metal_moe_block(nbb,D,I,mfmt,MG,MU,MD,MGS,MUS,MDS,mxg,xoffb,nrb,mrows,mrw,out,S)) cpu_miss=0;
m->t_emm += now_s()-t0;
} else cpu_miss=0;
diff --git a/c/quant.h b/c/quant.h
index d244d1b1..4c952339 100644
--- a/c/quant.h
+++ b/c/quant.h
@@ -804,12 +804,12 @@ typedef struct { int8_t *xq; size_t xq_cap; float *sx; size_t sx_cap; } QScratch
static _Thread_local QScratch g_qscratch;
static void quant_scratch(size_t xn, size_t sn, int8_t **xq, float **sx){
if(xn>g_qscratch.xq_cap){
- int8_t *p=realloc(g_qscratch.xq,xn);
+ int8_t *p=(int8_t*)realloc(g_qscratch.xq,xn);
if(!p){ fprintf(stderr,"OOM quant scratch\n"); exit(1); }
g_qscratch.xq=p; g_qscratch.xq_cap=xn;
}
if(sn>g_qscratch.sx_cap){
- float *p=realloc(g_qscratch.sx,sn*sizeof(float));
+ float *p=(float*)realloc(g_qscratch.sx,sn*sizeof(float));
if(!p){ fprintf(stderr,"OOM quant scales\n"); exit(1); }
g_qscratch.sx=p; g_qscratch.sx_cap=sn;
}
diff --git a/c/tests/test_backend_metal.mm b/c/tests/test_backend_metal.mm
index 7d8abfdb..189118e5 100644
--- a/c/tests/test_backend_metal.mm
+++ b/c/tests/test_backend_metal.mm
@@ -177,6 +177,60 @@ static int run_moe(const std::vector<int>& nrv, const char* name) {
return pass?0:1;
}
+// ---- fmt=6 (E8/IQ3) moe_block vs the engine's own scalar decoder ----
+// Mirrors the engine split exactly: the caller pre-rotates the staged gate/up input
+// (colibri.c metal_stage_rot_e8), the GPU rotates the down input (moe_fwht), and
+// matmul_e8/e8_rot_rows from quant.h are the reference for both.
+#define COLI_QUANT_TEST_ONLY 1
+#include "../quant.h"
+static int run_moe_e8(const std::vector<int>& nrv, const char* name) {
+ const int D=6144, I=1536, fmt=6; // I=1536 exercises the 512+1024 FWHT tiles
+ int64_t rbG=e8_rowbytes(D), rbD=e8_rowbytes(I); int nb=(int)nrv.size();
+ int R=0; std::vector<int> xoff(nb),nr(nrv); for(int e=0;e<nb;e++){ xoff[e]=R; R+=nrv[e]; }
+ srand(6666+nb);
+ std::vector<void*> slab(nb), fslab(nb);
+ std::vector<const void*> g(nb),u(nb),d(nb); std::vector<const float*> gs(nb),us(nb),ds(nb);
+ size_t wlen=roundpg((size_t)I*rbG*2 + (size_t)D*rbD), flen=roundpg(3*sizeof(float));
+ for(int e=0;e<nb;e++){
+ posix_memalign(&slab[e],16384,wlen); posix_memalign(&fslab[e],16384,flen);
+ uint8_t* sp=(uint8_t*)slab[e];
+ for(size_t i=0;i<(size_t)I*rbG*2+(size_t)D*rbD;i++) sp[i]=(uint8_t)(rand()&0xFF);
+ // clamp every block scale to a sane positive fp16 (random fp16 can be inf/nan)
+ for(size_t off=0; off+98<=(size_t)I*rbG*2+(size_t)D*rbD; off+=98){
+ uint16_t dh=(uint16_t)(0x2C00 | (rand()&0x3FF)); memcpy(sp+off+96,&dh,2); }
+ float* fp=(float*)fslab[e]; fp[0]=fp[1]=fp[2]=1.f; // .qs tags: resolvable, unused
+ g[e]=sp; u[e]=sp+(size_t)I*rbG; d[e]=sp+(size_t)I*rbG*2;
+ gs[e]=fp; us[e]=fp+1; ds[e]=fp+2;
+ coli_metal_register(slab[e],wlen); coli_metal_register(fslab[e],flen);
+ }
+ std::vector<float> xg((size_t)R*D); for(auto&v:xg) v=((rand()%2000)-1000)/1000.f;
+ std::vector<int> rows(R); std::vector<float> rw(R);
+ for(int gr=0;gr<R;gr++){ rows[gr]=0; rw[gr]=0.1f+(rand()%100)/100.f; }
+ int S=1;
+ // CPU reference from the unrotated input, engine semantics
+ std::vector<float> refout((size_t)S*D,0.f), xrot(D), gg(I),uu(I),hh(D);
+ for(int e=0;e<nb;e++) for(int r=0;r<nr[e];r++){ int gr=xoff[e]+r;
+ memcpy(xrot.data(), &xg[(size_t)gr*D], D*sizeof(float)); e8_rot_rows(xrot.data(),1,D);
+ matmul_e8(gg.data(),xrot.data(),(const uint8_t*)g[e],NULL,1,D,I);
+ matmul_e8(uu.data(),xrot.data(),(const uint8_t*)u[e],NULL,1,D,I);
+ for(int o=0;o<I;o++){ float v=gg[o]; gg[o]=(v/(1.f+expf(-v)))*uu[o]; }
+ e8_rot_rows(gg.data(),1,I);
+ matmul_e8(hh.data(),gg.data(),(const uint8_t*)d[e],NULL,1,I,D);
+ float* os=&refout[(size_t)rows[gr]*D]; for(int o=0;o<D;o++) os[o]+=rw[gr]*hh[o];
+ }
+ // GPU input: pre-rotated, as colibri.c stages it
+ std::vector<float> xg_gpu(xg);
+ for(int gr=0;gr<R;gr++) e8_rot_rows(&xg_gpu[(size_t)gr*D],1,D);
+ std::vector<float> gout((size_t)S*D,0.f);
+ int ok = coli_metal_moe_block(nb,D,I,fmt,g.data(),u.data(),d.data(),gs.data(),us.data(),ds.data(),
+ xg_gpu.data(),xoff.data(),nr.data(),rows.data(),rw.data(),gout.data(),S);
+ double maxabs=0,ymax=0; for(size_t i=0;i<gout.size();i++){ maxabs=fmax(maxabs,fabs(gout[i]-refout[i])); ymax=fmax(ymax,fabs(refout[i])); }
+ double nerr=maxabs/(ymax+1e-9); int pass = ok && nerr<1e-4;
+ printf(" %-22s R=%d nerr=%.2e %s\n", name, R, nerr, pass?"ok":"*** MISMATCH");
+ for(int e=0;e<nb;e++){ coli_metal_unregister(slab[e]); coli_metal_unregister(fslab[e]); free(slab[e]); free(fslab[e]); }
+ return pass?0:1;
+}
+
// ---- fused decode attention vs a CPU reference replicating glm.c's exact math ----
// GLM-5.2 dims (hardcoded in the backend): hidden=6144 H=64 q_lora=2048 kv_lora=512
// nope=192 rope=64 vh=256; theta=10000 ascale=1/16 eps=1e-5.
@@ -466,6 +520,9 @@ int main(void) {
printf("Metal batched moe_block tests:\n");
fail |= run_moe({1,1,1,1,1,1,1,1}, "moe decode nb=8");
fail |= run_moe({3,1,4,2,1,5}, "moe ragged nb=6");
+ printf("Metal fmt=6 (E8/IQ3) moe_block tests:\n");
+ fail |= run_moe_e8({1,1,1,1,1,1,1,1}, "e8 decode nb=8");
+ fail |= run_moe_e8({3,1,4,2,1,5}, "e8 ragged nb=6");
printf("Metal large-batch gemm test:\n");
{ // registered int4 weights, S=64: coli_metal_gemm vs cpu_ref
srand(77); int O=2048,I=6144,S=64,rb=(I+1)/2;
diff --git a/c/tools/convert_fp8_to_int4.py b/c/tools/convert_fp8_to_int4.py
index 5bc78dee..c64b294e 100644
--- a/c/tools/convert_fp8_to_int4.py
+++ b/c/tools/convert_fp8_to_int4.py
@@ -280,9 +280,17 @@ def _rowwise(fn, w, *args):
qs.append(q); ss.append(s)
return np.concatenate(qs), np.concatenate(ss)
+E8_JOBS = 1 # --jobs: parallel e8 encodes per shard
+
+def _e8_job(item):
+ name, w = item
+ q, s = quant_e8(w)
+ return name, q, s
+
def convert_shard(path, out_dict, n_layers, ebits, io_bits, xbits,
keep_mtp=False, keep_idx=False, group_size=0, bits_map=None):
from safetensors import safe_open
+ e8_jobs = [] # deferred: encoded in a pool after the scan
with safe_open(path, framework="pt") as f:
keys = set(f.keys())
for name in f.keys():
@@ -309,8 +317,11 @@ def convert_shard(path, out_dict, n_layers, ebits, io_bits, xbits,
out_dict[name] = w.astype(np.float32); continue
if bits == E8:
# fmt=6 E8/IQ3 β€” routed-expert projections only, enforced in main().
- # Already row-blocked inside iq3_pack.encode.
- q, s = quant_e8(w)
+ # Already row-blocked inside iq3_pack.encode. The python codec is
+ # slow (~5.5s per expert matrix), so encodes are deferred and run
+ # across a process pool after the shard scan (--jobs).
+ e8_jobs.append((name, w))
+ continue
elif bits == 3:
# int3-g64 (fmt=5): inherently group-64, distinct from grouped-int4.
q, s = _rowwise(quant_int3_g64, w)
@@ -321,6 +332,16 @@ def convert_shard(path, out_dict, n_layers, ebits, io_bits, xbits,
quant_int4 if bits <= 4 else quant_int8, w, bits)
out_dict[name] = q
out_dict[name + ".qs"] = s
+ if e8_jobs:
+ if E8_JOBS > 1:
+ from multiprocessing import get_context
+ with get_context("spawn").Pool(E8_JOBS) as pool: # spawn: safe after BLAS threads
+ for name, q, s in pool.imap(_e8_job, e8_jobs, chunksize=1):
+ out_dict[name] = q; out_dict[name + ".qs"] = s
+ else:
+ for item in e8_jobs:
+ name, q, s = _e8_job(item)
+ out_dict[name] = q; out_dict[name + ".qs"] = s
def free_gb(p): return shutil.disk_usage(p).free / 1e9
@@ -419,6 +440,11 @@ def main():
"concurrently; the main process still writes and checkpoints in "
"shard order, so output and out-NNNNN numbering are identical. "
"No effect on the --repo disk-safe path.")
+ ap.add_argument("--jobs", type=int, default=1,
+ help="parallel worker processes for the e8 encode WITHIN a shard (the python codec "
+ "is ~5.5s per expert matrix single-threaded; other quant modes are fast and "
+ "stay serial). Works on both --indir and the --repo disk-safe path. "
+ "Untested in combination with --workers>1; use one or the other.")
ap.add_argument("--selftest", action="store_true")
ap.add_argument("--selftest-nvfp4", action="store_true",
help="unit-test del dequant NVFP4 (LUT e2m1 + round-trip), nessun download / no network")
@@ -430,6 +456,8 @@ def main():
"repository (~756 GB of traffic) to retain only a few GB. Resumable per shard. "
"Recommended: --ebits 8.")
a = ap.parse_args()
+ global E8_JOBS
+ E8_JOBS = max(1, a.jobs)
if a.ebits is None:
# testa MTP a int4 = acceptance ~0-4% (misurato, issue #8): il draft sbaglia sempre
# e la speculazione non parte mai. A int8: 39-59%, 2.2-2.8 token/forward.