Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
metal: fmt=6 (E8/IQ3) expert path β GPU decode + on-GPU FWHT rotation
Michael Denyercdd6ed4df947
diff --git a/c/backend_metal.mm b/c/backend_metal.mmindex f13e7432..489ad01f 100644--- a/c/backend_metal.mm+++ b/c/backend_metal.mm@@ -17,6 +17,43 @@ static const char *SHADER = R"METAL(#include <metal_stdlib>using namespace metal;+// fmt=6 E8/IQ3 magnitude grid β generated from quant.h e8_grid (must stay identical;+// the metal-test oracle compares against the CPU decoder, so drift fails the build).+constant uchar4 E8G[256] = {+ uchar4(4,4,4,4),uchar4(20,4,4,4),uchar4(36,4,4,4),uchar4(12,12,4,4),uchar4(28,12,4,4),uchar4(62,12,4,4),uchar4(4,20,4,4),uchar4(20,20,4,4),+ uchar4(12,28,4,4),uchar4(20,36,4,4),uchar4(28,62,4,4),uchar4(44,62,4,4),uchar4(12,4,12,4),uchar4(28,4,12,4),uchar4(4,12,12,4),uchar4(20,12,12,4),+ uchar4(12,20,12,4),uchar4(44,20,12,4),uchar4(4,28,12,4),uchar4(20,28,12,4),uchar4(12,36,12,4),uchar4(36,44,12,4),uchar4(4,62,12,4),uchar4(4,4,20,4),+ uchar4(20,4,20,4),uchar4(36,4,20,4),uchar4(12,12,20,4),uchar4(4,20,20,4),uchar4(20,20,20,4),uchar4(12,28,20,4),uchar4(28,28,20,4),uchar4(62,28,20,4),+ uchar4(12,44,20,4),uchar4(62,44,20,4),uchar4(44,62,20,4),uchar4(12,4,28,4),uchar4(62,4,28,4),uchar4(4,12,28,4),uchar4(20,12,28,4),uchar4(44,20,28,4),+ uchar4(4,62,28,4),uchar4(28,12,36,4),uchar4(62,28,36,4),uchar4(36,36,36,4),uchar4(62,44,36,4),uchar4(28,62,36,4),uchar4(44,62,36,4),uchar4(12,4,44,4),+ uchar4(62,4,44,4),uchar4(20,28,44,4),uchar4(20,44,44,4),uchar4(44,28,52,4),uchar4(36,52,52,4),uchar4(4,12,62,4),uchar4(36,12,62,4),uchar4(52,12,62,4),+ uchar4(28,36,62,4),uchar4(12,52,62,4),uchar4(12,4,4,12),uchar4(28,4,4,12),uchar4(4,12,4,12),uchar4(20,12,4,12),uchar4(12,20,4,12),uchar4(28,20,4,12),+ uchar4(4,28,4,12),uchar4(20,28,4,12),uchar4(36,28,4,12),uchar4(62,36,4,12),uchar4(4,44,4,12),uchar4(4,4,12,12),uchar4(20,4,12,12),uchar4(12,12,12,12),+ uchar4(4,20,12,12),uchar4(20,20,12,12),uchar4(12,4,20,12),uchar4(28,4,20,12),uchar4(4,12,20,12),uchar4(20,12,20,12),uchar4(12,20,20,12),uchar4(4,28,20,12),+ uchar4(20,62,20,12),uchar4(4,4,28,12),uchar4(20,4,28,12),uchar4(4,20,28,12),uchar4(12,28,28,12),uchar4(52,36,28,12),uchar4(52,52,28,12),uchar4(12,4,36,12),+ uchar4(44,4,36,12),uchar4(4,44,36,12),uchar4(4,20,44,12),uchar4(36,20,44,12),uchar4(52,36,44,12),uchar4(12,62,44,12),uchar4(44,4,52,12),uchar4(20,20,62,12),+ uchar4(4,36,62,12),uchar4(4,4,4,20),uchar4(20,4,4,20),uchar4(12,12,4,20),uchar4(28,12,4,20),uchar4(4,20,4,20),uchar4(20,20,4,20),uchar4(52,20,4,20),+ uchar4(12,28,4,20),uchar4(20,36,4,20),uchar4(12,4,12,20),uchar4(28,4,12,20),uchar4(44,4,12,20),uchar4(4,12,12,20),uchar4(20,12,12,20),uchar4(12,20,12,20),+ uchar4(4,28,12,20),uchar4(28,52,12,20),uchar4(62,52,12,20),uchar4(4,62,12,20),uchar4(4,4,20,20),uchar4(20,4,20,20),uchar4(12,12,20,20),uchar4(62,12,20,20),+ uchar4(4,20,20,20),uchar4(20,20,20,20),uchar4(62,28,20,20),uchar4(4,36,20,20),uchar4(44,44,20,20),uchar4(12,4,28,20),uchar4(4,12,28,20),uchar4(36,12,28,20),+ uchar4(4,62,28,20),uchar4(36,62,28,20),uchar4(44,28,36,20),uchar4(28,44,36,20),uchar4(28,4,44,20),uchar4(62,20,44,20),uchar4(12,36,44,20),uchar4(36,62,44,20),+ uchar4(12,4,62,20),uchar4(28,4,62,20),uchar4(52,12,62,20),uchar4(44,36,62,20),uchar4(12,4,4,28),uchar4(4,12,4,28),uchar4(20,12,4,28),uchar4(12,20,4,28),+ uchar4(28,20,4,28),uchar4(4,44,4,28),uchar4(44,52,4,28),uchar4(20,62,4,28),uchar4(4,4,12,28),uchar4(20,4,12,28),uchar4(4,20,12,28),uchar4(12,28,12,28),+ uchar4(36,36,12,28),uchar4(52,36,12,28),uchar4(12,4,20,28),uchar4(28,4,20,28),uchar4(4,12,20,28),uchar4(44,20,20,28),uchar4(20,44,20,28),uchar4(20,62,20,28),+ uchar4(12,12,28,28),uchar4(28,28,28,28),uchar4(4,28,36,28),uchar4(62,36,36,28),uchar4(20,62,36,28),uchar4(4,4,44,28),uchar4(52,4,44,28),uchar4(20,20,44,28),+ uchar4(44,44,44,28),uchar4(36,12,52,28),uchar4(52,28,52,28),uchar4(28,52,52,28),uchar4(28,28,62,28),uchar4(4,52,62,28),uchar4(36,4,4,36),uchar4(62,12,4,36),+ uchar4(44,28,4,36),uchar4(62,28,4,36),uchar4(28,44,4,36),uchar4(62,44,4,36),uchar4(36,62,12,36),uchar4(4,20,20,36),uchar4(62,28,20,36),uchar4(4,36,20,36),+ uchar4(4,52,20,36),uchar4(52,52,20,36),uchar4(62,4,28,36),uchar4(44,36,28,36),uchar4(36,4,36,36),uchar4(12,44,36,36),uchar4(36,52,36,36),uchar4(44,20,44,36),+ uchar4(28,36,44,36),uchar4(4,62,44,36),uchar4(44,4,62,36),uchar4(4,12,62,36),uchar4(20,12,62,36),uchar4(4,28,62,36),uchar4(20,12,4,44),uchar4(12,36,4,44),+ uchar4(4,62,4,44),uchar4(4,4,12,44),uchar4(52,4,12,44),uchar4(52,20,12,44),uchar4(44,44,12,44),uchar4(36,12,20,44),uchar4(20,28,20,44),uchar4(20,62,20,44),+ uchar4(20,4,28,44),uchar4(28,44,28,44),uchar4(4,12,36,44),uchar4(28,20,36,44),uchar4(62,20,36,44),uchar4(20,62,36,44),uchar4(20,4,44,44),uchar4(12,28,44,44),+ uchar4(4,44,52,44),uchar4(36,20,62,44),uchar4(20,36,62,44),uchar4(36,20,4,52),uchar4(36,36,4,52),uchar4(52,36,4,52),uchar4(36,52,4,52),uchar4(12,20,12,52),+ uchar4(12,52,12,52),uchar4(62,12,20,52),uchar4(36,52,20,52),uchar4(4,28,28,52),uchar4(52,28,28,52),uchar4(36,36,36,52),uchar4(44,4,44,52),uchar4(20,44,44,52),+ uchar4(28,28,52,52),uchar4(28,4,62,52),uchar4(12,20,62,52),uchar4(28,4,4,62),uchar4(44,4,4,62),uchar4(62,4,4,62),uchar4(4,12,4,62),uchar4(20,28,4,62),+ uchar4(20,44,4,62),uchar4(52,20,12,62),uchar4(4,36,12,62),uchar4(20,12,20,62),uchar4(44,36,20,62),uchar4(20,44,20,62),uchar4(4,4,28,62),uchar4(44,12,28,62),+ uchar4(28,28,28,62),uchar4(4,52,28,62),uchar4(12,20,36,62),uchar4(12,36,36,62),uchar4(4,4,44,62),uchar4(20,4,44,62),uchar4(36,20,44,62),uchar4(4,28,52,62)+};+kernel void mm_gemv(device const uchar* w [[buffer(0)]], // raw weight bytesdevice const float* scale [[buffer(1)]], // [O] (fmt<4) or [O,ceil(I/gsz)] (fmt==4)device const float* x [[buffer(2)]], // [S,I]@@ -112,13 +149,59 @@ kernel void moe_gemv(device const ulong* waddr [[buffer(0)]], device const ulongfloat4 w1=float4(float(int(b.z&0xF)-8),float(int(b.z>>4)-8),float(int(b.w&0xF)-8),float(int(b.w>>4)-8));acc+=dot(w0,x4[2*c])+dot(w1,x4[2*c+1]); }for(int i=K8*8+slane;i<K;i+=32){ uchar b=w[i>>1]; int v=(i&1)?(b>>4):(b&0xF); acc+=float(v-8)*xr[i]; }+ } else if (fmt == 6) { // E8/IQ3: 98B per 256 weights, scales in-block+ long rb=((long)(K+255)/256)*98; // host guards K%256==0 (GLM dims are)+ device const uchar* w=(device const uchar*)(waddr[e])+(long)o*rb;+ int nsub=K/32; // one 32-weight sub-block per lane step+ for(int s6=slane;s6<nsub;s6+=32){+ int b=s6>>3, ib=s6&7; device const uchar* blk=w+(long)b*98;+ uint word = uint(blk[64+ib*4]) | (uint(blk[65+ib*4])<<8)+ | (uint(blk[66+ib*4])<<16) | (uint(blk[67+ib*4])<<24);+ ushort dh = ushort(blk[96]) | (ushort(blk[97])<<8);+ float db = float(as_type<half>(dh)) * (0.5f + float((word>>28)&0xFu)) * 0.5f;+ device const uchar* idx = blk + ib*8;+ device const float4* xs = (device const float4*)(xr + s6*32);+ for(int l=0;l<4;l++){+ uint sv=(word>>(7*l))&0x7Fu;+ float4 m0=float4(E8G[idx[l*2+0]]), m1=float4(E8G[idx[l*2+1]]);+ float4 sA=float4((sv&1u)?-1.0f:1.0f,(sv&2u)?-1.0f:1.0f,(sv&4u)?-1.0f:1.0f,(sv&8u)?-1.0f:1.0f);+ float4 sB=float4((sv&16u)?-1.0f:1.0f,(sv&32u)?-1.0f:1.0f,(sv&64u)?-1.0f:1.0f,+ (popcount(sv)&1u)?-1.0f:1.0f); // j=7 closes by odd parity+ acc += (dot(m0*sA,xs[l*2]) + dot(m1*sB,xs[l*2+1])) * (0.5f*db);+ }+ }} else { device const char* w=(device const char*)(waddr[e])+(long)o*K;device const char4* w4=(device const char4*)w;for(int c=slane;c<K8;c+=32) acc+=dot(float4(w4[2*c]),x4[2*c])+dot(float4(w4[2*c+1]),x4[2*c+1]);for(int i=K8*8+slane;i<K;i+=32) acc+=float(w[i])*xr[i];}acc=simd_sum(acc);- if(slane==0) yout[row]=acc*sc[o];+ if(slane==0) yout[row] = (fmt==6) ? acc : acc*sc[o]; // fmt=6: scales live in-block+}++// fmt=6 activation rotation for the GPU-resident down-projection input: one FWHT+// tile per dispatch (block-diagonal tiling and sign stream match quant.h e8_rot_rows;+// signs are regenerated host-side with e8_signs and passed in). One threadgroup per+// row; tile fits threadgroup memory (n <= 4096).+kernel void moe_fwht(device float* v [[buffer(0)]], device const uchar* signs [[buffer(1)]],+ constant int& dim [[buffer(2)]], constant int& off [[buffer(3)]],+ constant int& n [[buffer(4)]],+ uint tg [[threadgroup_position_in_grid]],+ uint t [[thread_position_in_threadgroup]],+ uint nt [[threads_per_threadgroup]]) {+ threadgroup float sh[4096];+ device float* row = v + (long)tg*dim + off;+ for (int i=int(t);i<n;i+=int(nt)){ float x=row[i]; if((signs[i>>3]>>(i&7))&1u) x=-x; sh[i]=x; }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ for (int len=1;len<n;len<<=1){+ for (int j=int(t);j<n/2;j+=int(nt)){+ int blk=j/len, k=j%len, i0=blk*(len<<1)+k;+ float a=sh[i0], b=sh[i0+len]; sh[i0]=a+b; sh[i0+len]=a-b;+ }+ threadgroup_barrier(mem_flags::mem_threadgroup);+ }+ float s=rsqrt(float(n));+ for (int i=int(t);i<n;i+=int(nt)) row[i]=sh[i]*s;}kernel void moe_silu(device float* g [[buffer(0)]], device const float* u [[buffer(1)]],uint i [[thread_position_in_grid]]) { float v=g[i]; g[i]=(v/(1.0f+exp(-v)))*u[i]; }@@ -309,7 +392,30 @@ struct ColiMetalTensor {static id<MTLDevice> g_dev;static id<MTLCommandQueue> g_queue;-static id<MTLComputePipelineState> g_gemv, g_moe_gemv, g_moe_silu;+static id<MTLComputePipelineState> g_gemv, g_moe_gemv, g_moe_silu, g_moe_fwht;++// fmt=6: sign-bit buffers for the GPU FWHT, one per tile size, cached forever (a+// handful of sizes). The xorshift64* draw replicates quant.h e8_signs exactly β+// the metal-test fmt=6 oracle compares end-to-end against quant.h, so any drift+// between the two copies fails the build.+static void e8_signs_local(uint8_t *bits, int n) {+ uint64_t s = 417u + (uint64_t)n;+ for (int i = 0; i < (n+7)/8; i++) {+ s ^= s>>12; s ^= s<<25; s ^= s>>27;+ bits[i] = (uint8_t)((s*2685821657736338717ULL)>>56);+ }+}+static id<MTLBuffer> fwht_signs(int n) {+ static std::mutex mtx; static std::vector<std::pair<int, id<MTLBuffer>>> cache;+ std::lock_guard<std::mutex> lk(mtx);+ for (auto &p : cache) if (p.first == n) return p.second;+ std::vector<uint8_t> bits((n+7)/8);+ e8_signs_local(bits.data(), n);+ id<MTLBuffer> b = [g_dev newBufferWithBytes:bits.data() length:bits.size()+ options:MTLResourceStorageModeShared];+ cache.push_back({n, b});+ return b;+}static id<MTLComputePipelineState> g_a_rms, g_a_rope, g_a_copy, g_a_qabs, g_a_score, g_a_smax, g_a_clat, g_a_ctx;static id<MTLComputePipelineState> g_a_add, g_r_router, g_r_top8, g_r_top8p;static int g_rtop8_par = 1; // COLI_RTOP8 (default ON); COLI_RTOP8=0 opts out to the@@ -481,6 +587,7 @@ extern "C" int coli_metal_init(void) {g_gemv = [g_dev newComputePipelineStateWithFunction:[lib newFunctionWithName:@"mm_gemv"] error:&err];g_moe_gemv = [g_dev newComputePipelineStateWithFunction:[lib newFunctionWithName:@"moe_gemv"] error:&err];g_moe_silu = [g_dev newComputePipelineStateWithFunction:[lib newFunctionWithName:@"moe_silu"] error:&err];+ g_moe_fwht = [g_dev newComputePipelineStateWithFunction:[lib newFunctionWithName:@"moe_fwht"] error:&err];auto P=[&](const char*n){ return [g_dev newComputePipelineStateWithFunction:[lib newFunctionWithName:@(n)] error:&err]; };g_a_rms=P("a_rmsnorm"); g_a_rope=P("a_rope"); g_a_copy=P("a_copy");g_a_qabs=P("a_qabs"); g_a_score=P("a_score"); g_a_smax=P("a_smax"); g_a_clat=P("a_clat"); g_a_ctx=P("a_ctx");@@ -501,7 +608,7 @@ extern "C" int coli_metal_init(void) {"r_top8 in use\n", (unsigned long)[g_r_top8p threadExecutionWidth]);g_rtop8_par = 0;}- if (!g_gemv || !g_moe_gemv || !g_moe_silu || !g_a_rms || !g_a_rope || !g_a_copy ||+ if (!g_gemv || !g_moe_gemv || !g_moe_silu || !g_moe_fwht || !g_a_rms || !g_a_rope || !g_a_copy ||!g_a_qabs || !g_a_score || !g_a_smax || !g_a_clat || !g_a_ctx) {fprintf(stderr, "[metal] pipeline failed\n"); g_dev = nil; return 0; }// E5 experiment: COLI_METAL_RESSET=1 -- see g_resset_obj comment above.@@ -995,7 +1102,17 @@ static id<MTLCommandBuffer> moe_submit(int nb, int D, int Iinter, int fmt,const float *const *gs, const float *const *us, const float *const *ds,const float *xg, const int *xoff, const int *nr, int R,id<MTLBuffer> xg_buf, id<MTLBuffer> gg_buf, id<MTLBuffer> uu_buf, id<MTLBuffer> hh_buf) {- if (!g_dev || (fmt != 1 && fmt != 2)) return nil;+ if (!g_dev || (fmt != 1 && fmt != 2 && fmt != 6)) return nil;+ if (fmt == 6) { /* e8 kernel assumes clean block tiling, and every FWHT tile of the+ * down input (CPU tiling rule, e8_rot_rows) must fit threadgroup mem */+ if ((D & 255) || (Iinter & 31)) return nil;+ for (int off = 0; off < Iinter; ) {+ int rem = Iinter - off, n = rem & (-rem);+ while (n > 32768) n >>= 1;+ if (n > 4096) return nil;+ off += n;+ }+ }if (g_resset_enabled) { // E5: commit any pending slab adds before we may skip useResource:double t0 = mnow(); resset_flush(); g_t_resset_flush += mnow() - t0; // METAL-RESSET line}@@ -1047,6 +1164,23 @@ static id<MTLCommandBuffer> moe_submit(int nb, int D, int Iinter, int fmt,[e setBuffer:gg_buf offset:0 atIndex:0];[e setBuffer:uu_buf offset:0 atIndex:1];[e dispatchThreads:MTLSizeMake((size_t)R*Iinter,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)];[e memoryBarrierWithScope:MTLBarrierScopeBuffers];+ if (fmt == 6) { /* rotate the down-projection input in place: same block-diagonal+ * tiling as quant.h e8_rot_rows (largest power of two dividing the+ * remainder, capped at 4096 for threadgroup memory) */+ int off = 0;+ while (off < Iinter) {+ int rem = Iinter - off, n = rem & (-rem);+ while (n > 32768) n >>= 1; /* CPU tiling rule (e8_rot_rows); sizes pre-validated above */+ id<MTLBuffer> sb = fwht_signs(n);+ [e setComputePipelineState:g_moe_fwht];+ [e setBuffer:gg_buf offset:0 atIndex:0];[e setBuffer:sb offset:0 atIndex:1];+ [e setBytes:&Iinter length:4 atIndex:2];[e setBytes:&off length:4 atIndex:3];+ [e setBytes:&n length:4 atIndex:4];+ [e dispatchThreadgroups:MTLSizeMake((size_t)R,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)];+ off += n;+ }+ [e memoryBarrierWithScope:MTLBarrierScopeBuffers];+ }gemv(bad,bsd,gg_buf,hh_buf,D,Iinter,Iinter); // downg_t_setup += mnow() - ts_start;[e endEncoding];[cb commit];diff --git a/c/colibri.c b/c/colibri.cindex 65acbf30..0c92c22c 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -3127,6 +3127,18 @@ static int router_best_or_fallback(int best, int kk, int E, int layer){return kk<E ? kk : 0;}+#ifdef COLI_METAL+/* Rotate the Metal-staged gate/up input rows for fmt=6 (Q^T x). Duplicate rows+ * (same source s) are copied from the first rotated instance: O(p^2) scan, p<=65. */+static void metal_stage_rot_e8(float *mxg, const int *mrows, int p, int D){+ for(int i=0;i<p;i++){+ int j=-1; for(int k=0;k<i;k++) if(mrows[k]==mrows[i]){ j=k; break; }+ if(j>=0) memcpy(mxg+(int64_t)i*D, mxg+(int64_t)j*D, (size_t)D*sizeof(float));+ else e8_rot_rows(mxg+(int64_t)i*D, 1, D);+ }+}+#endif+static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int with_shared){if(g_pilot_real){ /* barriera cross-layer: prendi possesso di QUESTO layer e aspetta* l'eventuale load-pilota in volo sullo stesso layer (dopodiche' il@@ -3536,6 +3548,10 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int* the first block) to the GPU BEFORE loading the missed experts from disk, so the* preads run while the GPU computes; the missed subset follows in a second submit.* Per-subset CPU fallback on unresolved slab / bad fmt / GPU fault. */+ /* fmt=6 stores W@Q, so the staged gate/up input must be rotated (Q^T x) before+ * upload. Rotate each distinct source row once, copy to duplicates β a decode+ * batch's rows all share s, so this is one FWHT per token in practice. The down+ * input is rotated on-GPU by moe_fwht inside moe_submit. */int is_miss[64]={0}; ColiMetalMoeHandle *mh=NULL;int cpu_res=1, cpu_miss=1, mh_shared=0, nbb=0, Rtot=0, mfmt=-1, sh_in=0;const void *MG[65],*MU[65],*MD[65]; const float *MGS[65],*MUS[65],*MDS[65];@@ -3581,6 +3597,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intMB_BUILD(0, base==0 && !g_pre_sh);if(nbb>0){double t0=now_s();+ if(mfmt==6) metal_stage_rot_e8(mxg,mrows,Rtot,D);mh=coli_metal_moe_block_begin(nbb,D,I,mfmt,MG,MU,MD,MGS,MUS,MDS,mxg,xoffb,nrb,mrows,mrw);m->t_emm += now_s()-t0;if(mh){ cpu_res=0; mh_shared=sh_in; }@@ -3643,6 +3660,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intMB_BUILD(1, 0); /* missed experts, now loaded */if(nbb>0){double t0=now_s();+ if(mfmt==6) metal_stage_rot_e8(mxg,mrows,Rtot,D);if(coli_metal_moe_block(nbb,D,I,mfmt,MG,MU,MD,MGS,MUS,MDS,mxg,xoffb,nrb,mrows,mrw,out,S)) cpu_miss=0;m->t_emm += now_s()-t0;} else cpu_miss=0;diff --git a/c/quant.h b/c/quant.hindex d244d1b1..4c952339 100644--- a/c/quant.h+++ b/c/quant.h@@ -804,12 +804,12 @@ typedef struct { int8_t *xq; size_t xq_cap; float *sx; size_t sx_cap; } QScratchstatic _Thread_local QScratch g_qscratch;static void quant_scratch(size_t xn, size_t sn, int8_t **xq, float **sx){if(xn>g_qscratch.xq_cap){- int8_t *p=realloc(g_qscratch.xq,xn);+ int8_t *p=(int8_t*)realloc(g_qscratch.xq,xn);if(!p){ fprintf(stderr,"OOM quant scratch\n"); exit(1); }g_qscratch.xq=p; g_qscratch.xq_cap=xn;}if(sn>g_qscratch.sx_cap){- float *p=realloc(g_qscratch.sx,sn*sizeof(float));+ float *p=(float*)realloc(g_qscratch.sx,sn*sizeof(float));if(!p){ fprintf(stderr,"OOM quant scales\n"); exit(1); }g_qscratch.sx=p; g_qscratch.sx_cap=sn;}diff --git a/c/tests/test_backend_metal.mm b/c/tests/test_backend_metal.mmindex 7d8abfdb..189118e5 100644--- a/c/tests/test_backend_metal.mm+++ b/c/tests/test_backend_metal.mm@@ -177,6 +177,60 @@ static int run_moe(const std::vector<int>& nrv, const char* name) {return pass?0:1;}+// ---- fmt=6 (E8/IQ3) moe_block vs the engine's own scalar decoder ----+// Mirrors the engine split exactly: the caller pre-rotates the staged gate/up input+// (colibri.c metal_stage_rot_e8), the GPU rotates the down input (moe_fwht), and+// matmul_e8/e8_rot_rows from quant.h are the reference for both.+#define COLI_QUANT_TEST_ONLY 1+#include "../quant.h"+static int run_moe_e8(const std::vector<int>& nrv, const char* name) {+ const int D=6144, I=1536, fmt=6; // I=1536 exercises the 512+1024 FWHT tiles+ int64_t rbG=e8_rowbytes(D), rbD=e8_rowbytes(I); int nb=(int)nrv.size();+ int R=0; std::vector<int> xoff(nb),nr(nrv); for(int e=0;e<nb;e++){ xoff[e]=R; R+=nrv[e]; }+ srand(6666+nb);+ std::vector<void*> slab(nb), fslab(nb);+ std::vector<const void*> g(nb),u(nb),d(nb); std::vector<const float*> gs(nb),us(nb),ds(nb);+ size_t wlen=roundpg((size_t)I*rbG*2 + (size_t)D*rbD), flen=roundpg(3*sizeof(float));+ for(int e=0;e<nb;e++){+ posix_memalign(&slab[e],16384,wlen); posix_memalign(&fslab[e],16384,flen);+ uint8_t* sp=(uint8_t*)slab[e];+ for(size_t i=0;i<(size_t)I*rbG*2+(size_t)D*rbD;i++) sp[i]=(uint8_t)(rand()&0xFF);+ // clamp every block scale to a sane positive fp16 (random fp16 can be inf/nan)+ for(size_t off=0; off+98<=(size_t)I*rbG*2+(size_t)D*rbD; off+=98){+ uint16_t dh=(uint16_t)(0x2C00 | (rand()&0x3FF)); memcpy(sp+off+96,&dh,2); }+ float* fp=(float*)fslab[e]; fp[0]=fp[1]=fp[2]=1.f; // .qs tags: resolvable, unused+ g[e]=sp; u[e]=sp+(size_t)I*rbG; d[e]=sp+(size_t)I*rbG*2;+ gs[e]=fp; us[e]=fp+1; ds[e]=fp+2;+ coli_metal_register(slab[e],wlen); coli_metal_register(fslab[e],flen);+ }+ std::vector<float> xg((size_t)R*D); for(auto&v:xg) v=((rand()%2000)-1000)/1000.f;+ std::vector<int> rows(R); std::vector<float> rw(R);+ for(int gr=0;gr<R;gr++){ rows[gr]=0; rw[gr]=0.1f+(rand()%100)/100.f; }+ int S=1;+ // CPU reference from the unrotated input, engine semantics+ std::vector<float> refout((size_t)S*D,0.f), xrot(D), gg(I),uu(I),hh(D);+ for(int e=0;e<nb;e++) for(int r=0;r<nr[e];r++){ int gr=xoff[e]+r;+ memcpy(xrot.data(), &xg[(size_t)gr*D], D*sizeof(float)); e8_rot_rows(xrot.data(),1,D);+ matmul_e8(gg.data(),xrot.data(),(const uint8_t*)g[e],NULL,1,D,I);+ matmul_e8(uu.data(),xrot.data(),(const uint8_t*)u[e],NULL,1,D,I);+ for(int o=0;o<I;o++){ float v=gg[o]; gg[o]=(v/(1.f+expf(-v)))*uu[o]; }+ e8_rot_rows(gg.data(),1,I);+ matmul_e8(hh.data(),gg.data(),(const uint8_t*)d[e],NULL,1,I,D);+ float* os=&refout[(size_t)rows[gr]*D]; for(int o=0;o<D;o++) os[o]+=rw[gr]*hh[o];+ }+ // GPU input: pre-rotated, as colibri.c stages it+ std::vector<float> xg_gpu(xg);+ for(int gr=0;gr<R;gr++) e8_rot_rows(&xg_gpu[(size_t)gr*D],1,D);+ std::vector<float> gout((size_t)S*D,0.f);+ int ok = coli_metal_moe_block(nb,D,I,fmt,g.data(),u.data(),d.data(),gs.data(),us.data(),ds.data(),+ xg_gpu.data(),xoff.data(),nr.data(),rows.data(),rw.data(),gout.data(),S);+ double maxabs=0,ymax=0; for(size_t i=0;i<gout.size();i++){ maxabs=fmax(maxabs,fabs(gout[i]-refout[i])); ymax=fmax(ymax,fabs(refout[i])); }+ double nerr=maxabs/(ymax+1e-9); int pass = ok && nerr<1e-4;+ printf(" %-22s R=%d nerr=%.2e %s\n", name, R, nerr, pass?"ok":"*** MISMATCH");+ for(int e=0;e<nb;e++){ coli_metal_unregister(slab[e]); coli_metal_unregister(fslab[e]); free(slab[e]); free(fslab[e]); }+ return pass?0:1;+}+// ---- fused decode attention vs a CPU reference replicating glm.c's exact math ----// GLM-5.2 dims (hardcoded in the backend): hidden=6144 H=64 q_lora=2048 kv_lora=512// nope=192 rope=64 vh=256; theta=10000 ascale=1/16 eps=1e-5.@@ -466,6 +520,9 @@ int main(void) {printf("Metal batched moe_block tests:\n");fail |= run_moe({1,1,1,1,1,1,1,1}, "moe decode nb=8");fail |= run_moe({3,1,4,2,1,5}, "moe ragged nb=6");+ printf("Metal fmt=6 (E8/IQ3) moe_block tests:\n");+ fail |= run_moe_e8({1,1,1,1,1,1,1,1}, "e8 decode nb=8");+ fail |= run_moe_e8({3,1,4,2,1,5}, "e8 ragged nb=6");printf("Metal large-batch gemm test:\n");{ // registered int4 weights, S=64: coli_metal_gemm vs cpu_refsrand(77); int O=2048,I=6144,S=64,rb=(I+1)/2;diff --git a/c/tools/convert_fp8_to_int4.py b/c/tools/convert_fp8_to_int4.pyindex 5bc78dee..c64b294e 100644--- a/c/tools/convert_fp8_to_int4.py+++ b/c/tools/convert_fp8_to_int4.py@@ -280,9 +280,17 @@ def _rowwise(fn, w, *args):qs.append(q); ss.append(s)return np.concatenate(qs), np.concatenate(ss)+E8_JOBS = 1 # --jobs: parallel e8 encodes per shard++def _e8_job(item):+ name, w = item+ q, s = quant_e8(w)+ return name, q, s+def convert_shard(path, out_dict, n_layers, ebits, io_bits, xbits,keep_mtp=False, keep_idx=False, group_size=0, bits_map=None):from safetensors import safe_open+ e8_jobs = [] # deferred: encoded in a pool after the scanwith safe_open(path, framework="pt") as f:keys = set(f.keys())for name in f.keys():@@ -309,8 +317,11 @@ def convert_shard(path, out_dict, n_layers, ebits, io_bits, xbits,out_dict[name] = w.astype(np.float32); continueif bits == E8:# fmt=6 E8/IQ3 β routed-expert projections only, enforced in main().- # Already row-blocked inside iq3_pack.encode.- q, s = quant_e8(w)+ # Already row-blocked inside iq3_pack.encode. The python codec is+ # slow (~5.5s per expert matrix), so encodes are deferred and run+ # across a process pool after the shard scan (--jobs).+ e8_jobs.append((name, w))+ continueelif bits == 3:# int3-g64 (fmt=5): inherently group-64, distinct from grouped-int4.q, s = _rowwise(quant_int3_g64, w)@@ -321,6 +332,16 @@ def convert_shard(path, out_dict, n_layers, ebits, io_bits, xbits,quant_int4 if bits <= 4 else quant_int8, w, bits)out_dict[name] = qout_dict[name + ".qs"] = s+ if e8_jobs:+ if E8_JOBS > 1:+ from multiprocessing import get_context+ with get_context("spawn").Pool(E8_JOBS) as pool: # spawn: safe after BLAS threads+ for name, q, s in pool.imap(_e8_job, e8_jobs, chunksize=1):+ out_dict[name] = q; out_dict[name + ".qs"] = s+ else:+ for item in e8_jobs:+ name, q, s = _e8_job(item)+ out_dict[name] = q; out_dict[name + ".qs"] = sdef free_gb(p): return shutil.disk_usage(p).free / 1e9@@ -419,6 +440,11 @@ def main():"concurrently; the main process still writes and checkpoints in ""shard order, so output and out-NNNNN numbering are identical. ""No effect on the --repo disk-safe path.")+ ap.add_argument("--jobs", type=int, default=1,+ help="parallel worker processes for the e8 encode WITHIN a shard (the python codec "+ "is ~5.5s per expert matrix single-threaded; other quant modes are fast and "+ "stay serial). Works on both --indir and the --repo disk-safe path. "+ "Untested in combination with --workers>1; use one or the other.")ap.add_argument("--selftest", action="store_true")ap.add_argument("--selftest-nvfp4", action="store_true",help="unit-test del dequant NVFP4 (LUT e2m1 + round-trip), nessun download / no network")@@ -430,6 +456,8 @@ def main():"repository (~756 GB of traffic) to retain only a few GB. Resumable per shard. ""Recommended: --ebits 8.")a = ap.parse_args()+ global E8_JOBS+ E8_JOBS = max(1, a.jobs)if a.ebits is None:# testa MTP a int4 = acceptance ~0-4% (misurato, issue #8): il draft sbaglia sempre# e la speculazione non parte mai. A int8: 39-59%, 2.2-2.8 token/forward.