Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(metal): run int4-g64 (fmt=4) models on the GPU — MoE experts + fused attention
Andrey Aristova9f31f169c2c
diff --git a/c/backend_metal.h b/c/backend_metal.hindex 2c09ba76..8ec6867f 100644--- a/c/backend_metal.h+++ b/c/backend_metal.h@@ -80,7 +80,7 @@ int coli_metal_layer_decode(float *x,const void *qa_w, const float *qa_s, int qa_fmt, int qa_gs, const float *qa_ln,const void *qb_w, const float *qb_s, int qb_fmt, int qb_gs,const void *kva_w, const float *kva_s, int kva_fmt, int kva_gs, const float *kva_ln,- const void *kvb_w, const float *kvb_s, int kvb_fmt,+ const void *kvb_w, const float *kvb_s, int kvb_fmt, int kvb_gs,const void *o_w, const float *o_s, int o_fmt, int o_gs,const void *shg_w, const float *shg_s, int shg_fmt, int shg_gs,const void *shu_w, const float *shu_s, int shu_fmt, int shu_gs,@@ -116,7 +116,7 @@ int coli_metal_attn_decode(const float *x,const void *qa_w, const float *qa_s, int qa_fmt, int qa_gs, const float *qa_ln,const void *qb_w, const float *qb_s, int qb_fmt, int qb_gs,const void *kva_w, const float *kva_s, int kva_fmt, int kva_gs, const float *kva_ln,- const void *kvb_w, const float *kvb_s, int kvb_fmt,+ const void *kvb_w, const float *kvb_s, int kvb_fmt, int kvb_gs,const void *o_w, const float *o_s, int o_fmt, int o_gs,float *Lc, float *Rc, int S, int pos_base, int st0, float eps, float theta, float ascale, float *out);@@ -137,18 +137,20 @@ int coli_metal_resset_stats(double *flush_s);** D = hidden size, Iinter = moe intermediate size* g/u/d[e] = pointers to expert e's gate/up/down quantized weights (in RAM slabs)- * gs/us/ds[e] = pointers to expert e's per-row scales- * fmt = quant format (shared across experts). NOTE: fmt=4 (grouped int4) is- * NOT yet supported here -- gates to {1,2} and returns 0 (CPU fallback)- * for fmt=4 experts, same as before this stage. Grouped-int4 gained GPU- * support in mm_gemv (coli_metal_matmul/coli_metal_gemm/bind_gemv) only;- * extending the batched routed-expert path is future work (see PR_BODY.md).+ * gs/us/ds[e] = pointers to expert e's scales: per-row [O] for fmt 1/2, per-group+ * [O,ceil(K/gs4)] for fmt=4 (K = the matmul's input dim)+ * fmt = quant format (shared across experts): {1,2,4,6}; anything else+ * returns 0 (CPU fallback)+ * gs4 = fmt=4 group size along the input dim (shared across experts and the+ * three matrices). Must be even and >=2; pass <=0 to force the CPU+ * fallback (the caller poisons it on mixed per-expert group sizes).+ * Ignored for other fmts.* xg = packed activations [total_rows, D]; xoff[e] = row offset of expert e* nr[e] = rows for expert e; rows[]/rw[] map packed rows back to out positions* out = [S, D] accumulate target* Returns 1 on success, 0 to signal the caller to fall back to the CPU path.*/-int coli_metal_moe_block(int nb, int D, int Iinter, int fmt,+int coli_metal_moe_block(int nb, int D, int Iinter, int fmt, int gs4,const void *const *g, const void *const *u, const void *const *d,const float *const *gs, const float *const *us, const float *const *ds,const float *xg, const int *xoff, const int *nr,@@ -163,7 +165,7 @@ int coli_metal_moe_block(int nb, int D, int Iinter, int fmt,* end returns 0 on GPU fault (caller redoes those experts on CPU).*/typedef struct ColiMetalMoeHandle ColiMetalMoeHandle;-ColiMetalMoeHandle* coli_metal_moe_block_begin(int nb, int D, int Iinter, int fmt,+ColiMetalMoeHandle* coli_metal_moe_block_begin(int nb, int D, int Iinter, int fmt, int gs4,const void *const *g, const void *const *u, const void *const *d,const float *const *gs, const float *const *us, const float *const *ds,const float *xg, const int *xoff, const int *nr,diff --git a/c/backend_metal.mm b/c/backend_metal.mmindex 484eb001..7fed3a44 100644--- a/c/backend_metal.mm+++ b/c/backend_metal.mm@@ -163,6 +163,7 @@ kernel void moe_gemv(device const ulong* waddr [[buffer(0)]], device const ulongconstant int& O [[buffer(5)]], constant int& K [[buffer(6)]],constant int& Kin [[buffer(7)]], constant int& fmt [[buffer(8)]],constant int& NT [[buffer(9)]],+ constant int& gsz [[buffer(10)]], // fmt==4 group size (ignored otherwise)uint tg [[threadgroup_position_in_grid]],uint slane [[thread_index_in_simdgroup]],uint sgid [[simdgroup_index_in_threadgroup]]) {@@ -179,6 +180,18 @@ kernel void moe_gemv(device const ulong* waddr [[buffer(0)]], device const ulongfloat4 w1=float4(float(int(b.z&0xF)-8),float(int(b.z>>4)-8),float(int(b.w&0xF)-8),float(int(b.w>>4)-8));acc+=dot(w0,x4[2*c])+dot(w1,x4[2*c+1]); }for(int i=K8*8+slane;i<K;i+=32){ uchar b=w[i>>1]; int v=(i&1)?(b>>4):(b&0xF); acc+=float(v-8)*xr[i]; }+ } else if (fmt == 4) { // int4 GROUPED: fmt=2's nibble packing, one f32+ // scale per gsz inputs (mm_gemv fmt==4 twin --+ // lane owns one packed byte per 64-lane stride,+ // a group never splits a byte: gsz is even)+ int rb=(K+1)/2, ng=(K+gsz-1)/gsz;+ device const uchar* w=(device const uchar*)(waddr[e])+(long)o*rb;+ device const float* scl=sc+(long)o*ng;+ for(int i=slane*2;i<K;i+=64){ uchar b=w[i>>1];+ int g0=i/gsz; float sc0=scl[g0];+ acc+=float(int(b&0xF)-8)*xr[i]*sc0;+ if(i+1<K){ int g1=(i+1)/gsz; acc+=float(int(b>>4)-8)*xr[i+1]*((g1==g0)?sc0:scl[g1]); }+ }} else if (fmt == 6) { // E8/IQ3: 98B per 256 weights, scales in-blocklong rb=((long)(K+255)/256)*98; // host guards K%256==0 (GLM dims are)device const uchar* w=(device const uchar*)(waddr[e])+(long)o*rb;@@ -206,8 +219,8 @@ kernel void moe_gemv(device const ulong* waddr [[buffer(0)]], device const ulongfor(int i=K8*8+slane;i<K;i+=32) acc+=float(w[i])*xr[i];}acc=simd_sum(acc);- if(slane==0) yout[row] = (fmt==6) ? acc : acc*sc[o]; // fmt=6: scales live in-block-}+ if(slane==0) yout[row] = (fmt==6||fmt==4) ? acc : acc*sc[o]; // fmt=6: scales live in-block;+} // fmt=4: folded per-group above// fmt=6 activation rotation for the GPU-resident down-projection input: one FWHT// tile per dispatch (block-diagonal tiling and sign stream match quant.h e8_rot_rows;@@ -267,14 +280,19 @@ kernel void a_copy(device const float* src [[buffer(0)]], constant int& off [[bu// ---- absorption core (S query rows, per-row causal). q:[S,H*QH]; qabs/clat:[S*H,KVL];// sc:[S*H,T]; ctx:[S*H,VH]. Query row s (abs pos PB+s) attends keys [0, PB+s]. ----constant int A_QHH=A_H*A_QH;-inline float a_deqrow(device const uchar* base, int row, int i, device const float* sc){- device const uchar* w=base+(long)row*((A_KVL+1)/2); uchar b=w[i>>1]; int val=(i&1)?(b>>4):(b&0xF); return float(val-8)*sc[row]; }+// kfmt 2: per-row scale sc[row]. kfmt 4 (grouped int4): sc[row*ceil(KVL/kgs) + i/kgs],+// same nibble packing either way (the weight bytes are identical between the two fmts).+inline float a_deqrow(device const uchar* base, int row, int i, device const float* sc, int kfmt, int kgs){+ device const uchar* w=base+(long)row*((A_KVL+1)/2); uchar b=w[i>>1]; int val=(i&1)?(b>>4):(b&0xF);+ float s = (kfmt==4) ? sc[(long)row*((A_KVL+kgs-1)/kgs) + i/kgs] : sc[row];+ return float(val-8)*s; }kernel void a_qabs(device const uchar* kvb [[buffer(0)]], device const float* sc [[buffer(1)]],device const float* q [[buffer(2)]], device float* qabs [[buffer(3)]],+ constant int& kfmt [[buffer(4)]], constant int& kgs [[buffer(5)]],uint gid [[thread_position_in_grid]]) {int s=gid/(A_H*A_KVL), r=gid%(A_H*A_KVL), h=r/A_KVL, i=r%A_KVL; int rbase=h*A_ROWSH;device const float* qp=q+(long)s*A_QHH+(long)h*A_QH;- float a=0; for(int d=0;d<A_NOPE;d++) a+=qp[d]*a_deqrow(kvb,rbase+d,i,sc); qabs[(long)(s*A_H+h)*A_KVL+i]=a;+ float a=0; for(int d=0;d<A_NOPE;d++) a+=qp[d]*a_deqrow(kvb,rbase+d,i,sc,kfmt,kgs); qabs[(long)(s*A_H+h)*A_KVL+i]=a;}kernel void a_score(device const float* qabs [[buffer(0)]], device const float* Lc [[buffer(1)]],device const float* Rc [[buffer(2)]], device const float* q [[buffer(3)]],@@ -302,9 +320,11 @@ kernel void a_clat(device const float* sc [[buffer(0)]], device const float* Lcfor(int t=0;t<T;t++) a+=s[t]*Lc[(long)t*A_KVL+i]; clat[(long)sh*A_KVL+i]=a;}kernel void a_ctx(device const uchar* kvb [[buffer(0)]], device const float* sc [[buffer(1)]],- device const float* clat [[buffer(2)]], device float* ctx [[buffer(3)]], uint gid [[thread_position_in_grid]]) {+ device const float* clat [[buffer(2)]], device float* ctx [[buffer(3)]],+ constant int& kfmt [[buffer(4)]], constant int& kgs [[buffer(5)]],+ uint gid [[thread_position_in_grid]]) {int sh=gid/A_VH, j=gid%A_VH, h=sh%A_H; int row=h*A_ROWSH+A_NOPE+j; device const float* cl=clat+(long)sh*A_KVL;- float a=0; for(int i=0;i<A_KVL;i++) a+=cl[i]*a_deqrow(kvb,row,i,sc); ctx[(long)sh*A_VH+j]=a;+ float a=0; for(int i=0;i<A_KVL;i++) a+=cl[i]*a_deqrow(kvb,row,i,sc,kfmt,kgs); ctx[(long)sh*A_VH+j]=a;}// ===== full-layer tail kernels =====@@ -831,14 +851,14 @@ static bool bind_gemv(id<MTLComputeCommandEncoder> e, const void* w, const float// Weight-pointer bundle for one layer's attention (+optional layer tail). All pointers// must be inside registered allocations. *_gs: fmt=4 group size for the corresponding-// weight (0 if that weight isn't grouped). kv_b has none: it never flows through bind_gemv-// -- a_qabs/a_ctx dequantize it inline with a PER-ROW-only helper (a_deqrow), so a fmt=4-// kv_b is out of scope for this stage (see PR_BODY.md UNCERTAINTIES).+// weight (0 if that weight isn't grouped). kv_b never flows through bind_gemv --+// a_qabs/a_ctx dequantize it inline via a_deqrow, which handles per-row (fmt=2) and+// per-group (fmt=4, kvb_gs) scales; anything else falls back to CPU in encode_attention.typedef struct {const void *qa_w; const float *qa_s; int qa_fmt; int qa_gs; const float *qa_ln;const void *qb_w; const float *qb_s; int qb_fmt; int qb_gs;const void *kva_w; const float *kva_s; int kva_fmt; int kva_gs; const float *kva_ln;- const void *kvb_w; const float *kvb_s; int kvb_fmt;+ const void *kvb_w; const float *kvb_s; int kvb_fmt; int kvb_gs;const void *o_w; const float *o_s; int o_fmt; int o_gs;} AttnW;@@ -848,6 +868,9 @@ static bool encode_attention(id<MTLComputeCommandEncoder> e, const AttnW *W,id<MTLBuffer> Lb, size_t loff, id<MTLBuffer> Rb, size_t roff,id<MTLBuffer> kvbW, size_t kvbwoff, id<MTLBuffer> kvbS, size_t kvbsoff,int S, int pos_base, float eps, float theta, float ascale) {+ /* a_deqrow's contract: kv_b is int4-packed with per-row (fmt=2) or per-group+ * (fmt=4, even positive gs) scales. Anything else -> CPU fallback. */+ if(W->kvb_fmt!=2 && !(W->kvb_fmt==4 && W->kvb_gs>=2 && !(W->kvb_gs&1))) return false;int T=pos_base+S;memcpy([aqaln_ contents],W->qa_ln,AQLORA*4); memcpy([akvaln_ contents],W->kva_ln,AKVL*4);size_t Loff=loff+(size_t)pos_base*AKVL*4, Roff=roff+(size_t)pos_base*AROPE*4;@@ -868,6 +891,7 @@ static bool encode_attention(id<MTLComputeCommandEncoder> e, const AttnW *W,bind_gemv(e,W->qb_w,W->qb_s,W->qb_fmt,W->qb_gs,AQLORA,AHQH,aqr_,aqf_,S); rms(Lb,Loff,akvaln_,AKVL,S); rope(Rb,Roff,0,AROPE,0,1); BAR();rope(aqf_,0,ANOPE,AHQH,AQH,AHEADS); BAR();[e setComputePipelineState:g_a_qabs]; [e setBuffer:kvbW offset:kvbwoff atIndex:0]; [e setBuffer:kvbS offset:kvbsoff atIndex:1]; [e setBuffer:aqf_ offset:0 atIndex:2]; [e setBuffer:aqabs_ offset:0 atIndex:3];+ [e setBytes:&W->kvb_fmt length:4 atIndex:4]; [e setBytes:&W->kvb_gs length:4 atIndex:5];[e dispatchThreads:MTLSizeMake((size_t)S*AHEADS*AKVL,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; BAR();[e setComputePipelineState:g_a_score]; [e setBuffer:aqabs_ offset:0 atIndex:0]; [e setBuffer:Lb offset:loff atIndex:1]; [e setBuffer:Rb offset:roff atIndex:2]; [e setBuffer:aqf_ offset:0 atIndex:3]; [e setBuffer:ascore_ offset:0 atIndex:4];[e setBytes:&T length:4 atIndex:5]; [e setBytes:&ascale length:4 atIndex:6]; [e setBytes:&pos_base length:4 atIndex:7];@@ -877,6 +901,7 @@ static bool encode_attention(id<MTLComputeCommandEncoder> e, const AttnW *W,[e setComputePipelineState:g_a_clat]; [e setBuffer:ascore_ offset:0 atIndex:0]; [e setBuffer:Lb offset:loff atIndex:1]; [e setBuffer:aclat_ offset:0 atIndex:2]; [e setBytes:&T length:4 atIndex:3];[e dispatchThreads:MTLSizeMake((size_t)S*AHEADS*AKVL,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; BAR();[e setComputePipelineState:g_a_ctx]; [e setBuffer:kvbW offset:kvbwoff atIndex:0]; [e setBuffer:kvbS offset:kvbsoff atIndex:1]; [e setBuffer:aclat_ offset:0 atIndex:2]; [e setBuffer:actx_ offset:0 atIndex:3];+ [e setBytes:&W->kvb_fmt length:4 atIndex:4]; [e setBytes:&W->kvb_gs length:4 atIndex:5];[e dispatchThreads:MTLSizeMake((size_t)S*AHEADS*AVH,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; BAR();bind_gemv(e,W->o_w,W->o_s,W->o_fmt,W->o_gs,AHVH,AH,actx_,aout_,S);return true;@@ -899,7 +924,7 @@ extern "C" int coli_metal_attn_decode(const float* x,const void* qa_w,const float* qa_s,int qa_fmt,int qa_gs,const float* qa_ln,const void* qb_w,const float* qb_s,int qb_fmt,int qb_gs,const void* kva_w,const float* kva_s,int kva_fmt,int kva_gs,const float* kva_ln,- const void* kvb_w,const float* kvb_s,int kvb_fmt,+ const void* kvb_w,const float* kvb_s,int kvb_fmt,int kvb_gs,const void* o_w,const float* o_s,int o_fmt,int o_gs,float* Lc,float* Rc,int S,int pos_base,int st0,float eps,float theta,float ascale,float* out){if(!g_dev) return 0;@@ -907,7 +932,7 @@ extern "C" int coli_metal_attn_decode(const float* x,int T=pos_base+S;@autoreleasepool {attn_scratch_init();- AttnW W={qa_w,qa_s,qa_fmt,qa_gs,qa_ln,qb_w,qb_s,qb_fmt,qb_gs,kva_w,kva_s,kva_fmt,kva_gs,kva_ln,kvb_w,kvb_s,kvb_fmt,o_w,o_s,o_fmt,o_gs};+ AttnW W={qa_w,qa_s,qa_fmt,qa_gs,qa_ln,qb_w,qb_s,qb_fmt,qb_gs,kva_w,kva_s,kva_fmt,kva_gs,kva_ln,kvb_w,kvb_s,kvb_fmt,kvb_gs,o_w,o_s,o_fmt,o_gs};id<MTLBuffer> Lb,Rb,kvbW,kvbS; size_t loff,roff,kvbwoff,kvbsoff;if(!resolve_attn(&W,Lc,Rc,&Lb,&loff,&Rb,&roff,&kvbW,&kvbwoff,&kvbS,&kvbsoff)) return 0;ascore_=ensure(ascore_,&ascore_cap,(size_t)S*AHEADS*T*4);@@ -937,7 +962,7 @@ extern "C" int coli_metal_layer_decode(float *x,const void* qa_w,const float* qa_s,int qa_fmt,int qa_gs,const float* qa_ln,const void* qb_w,const float* qb_s,int qb_fmt,int qb_gs,const void* kva_w,const float* kva_s,int kva_fmt,int kva_gs,const float* kva_ln,- const void* kvb_w,const float* kvb_s,int kvb_fmt,+ const void* kvb_w,const float* kvb_s,int kvb_fmt,int kvb_gs,const void* o_w,const float* o_s,int o_fmt,int o_gs,const void* shg_w,const float* shg_s,int shg_fmt,int shg_gs,const void* shu_w,const float* shu_s,int shu_fmt,int shu_gs,@@ -952,7 +977,7 @@ extern "C" int coli_metal_layer_decode(float *x,int T=pos_base+S; const int SI=2048;@autoreleasepool {attn_scratch_init();- AttnW W={qa_w,qa_s,qa_fmt,qa_gs,qa_ln,qb_w,qb_s,qb_fmt,qb_gs,kva_w,kva_s,kva_fmt,kva_gs,kva_ln,kvb_w,kvb_s,kvb_fmt,o_w,o_s,o_fmt,o_gs};+ AttnW W={qa_w,qa_s,qa_fmt,qa_gs,qa_ln,qb_w,qb_s,qb_fmt,qb_gs,kva_w,kva_s,kva_fmt,kva_gs,kva_ln,kvb_w,kvb_s,kvb_fmt,kvb_gs,o_w,o_s,o_fmt,o_gs};id<MTLBuffer> Lb,Rb,kvbW,kvbS; size_t loff,roff,kvbwoff,kvbsoff;if(!resolve_attn(&W,Lc,Rc,&Lb,&loff,&Rb,&roff,&kvbW,&kvbwoff,&kvbS,&kvbsoff)) return 0;uint64_t ina=0,pna=0,rwa=0,rba=0,d;@@ -1138,12 +1163,16 @@ extern "C" size_t coli_metal_tensor_bytes(const ColiMetalTensor *t) { return t ?// if Metal is off or any expert pointer is not in a registered slab.// Encode + commit a MoE block (no wait). Writes hh[R,D] into hh_buf. Returns nil on// unresolved slab / bad fmt (caller falls back to CPU).-static id<MTLCommandBuffer> moe_submit(int nb, int D, int Iinter, int fmt,+static id<MTLCommandBuffer> moe_submit(int nb, int D, int Iinter, int fmt, int gs4,const void *const *g, const void *const *u, const void *const *d,const float *const *gs, const float *const *us, const float *const *ds,const float *xg, const int *xoff, const int *nr, int R,id<MTLBuffer> xg_buf, id<MTLBuffer> gg_buf, id<MTLBuffer> uu_buf, id<MTLBuffer> hh_buf) {- if (!g_dev || (fmt != 1 && fmt != 2 && fmt != 6)) return nil;+ if (!g_dev || (fmt != 1 && fmt != 2 && fmt != 4 && fmt != 6)) return nil;+ if (fmt == 4 && (gs4 < 2 || (gs4 & 1))) return nil; /* kernel needs an even, positive group+ * size (a group never splits a packed+ * byte); gs4<0 = caller-poisoned (mixed+ * per-expert gs) -> CPU fallback */if (fmt == 6) { /* e8 kernel assumes clean block tiling, and every FWHT tile of the* down input (CPU tiling rule, e8_rot_rows) must fit threadgroup mem */if ((D & 255) || (Iinter & 31)) return nil;@@ -1196,7 +1225,7 @@ static id<MTLCommandBuffer> moe_submit(int nb, int D, int Iinter, int fmt,[e setBuffer:wa offset:0 atIndex:0];[e setBuffer:sa offset:0 atIndex:1];[e setBuffer:berow offset:0 atIndex:2];[e setBuffer:xin offset:0 atIndex:3];[e setBuffer:y offset:0 atIndex:4];[e setBytes:&O length:4 atIndex:5];[e setBytes:&K length:4 atIndex:6];[e setBytes:&Kin length:4 atIndex:7];[e setBytes:&fmt length:4 atIndex:8];- [e setBytes:&NT length:4 atIndex:9];+ [e setBytes:&NT length:4 atIndex:9];[e setBytes:&gs4 length:4 atIndex:10];[e dispatchThreadgroups:MTLSizeMake(((size_t)NT+3)/4,1,1) threadsPerThreadgroup:MTLSizeMake(128,1,1)]; };gemv(bag,bsg,xg_buf,gg_buf,Iinter,D,D); // gategemv(bau,bsu,xg_buf,uu_buf,Iinter,D,D); // up@@ -1248,7 +1277,7 @@ static int moe_finish(id<MTLCommandBuffer> cb, id<MTLBuffer> hh_buf, int nb, intreturn 1;}-extern "C" int coli_metal_moe_block(int nb, int D, int Iinter, int fmt,+extern "C" int coli_metal_moe_block(int nb, int D, int Iinter, int fmt, int gs4,const void *const *g, const void *const *u, const void *const *d,const float *const *gs, const float *const *us, const float *const *ds,const float *xg, const int *xoff, const int *nr,@@ -1261,7 +1290,7 @@ extern "C" int coli_metal_moe_block(int nb, int D, int Iinter, int fmt,g_gg = ensure(g_gg,&g_gg_cap,(size_t)R*Iinter*4);g_uu = ensure(g_uu,&g_uu_cap,(size_t)R*Iinter*4);g_hh = ensure(g_hh,&g_hh_cap,(size_t)R*D*4);- id<MTLCommandBuffer> cb = moe_submit(nb,D,Iinter,fmt,g,u,d,gs,us,ds,xg,xoff,nr,R,g_xg,g_gg,g_uu,g_hh);+ id<MTLCommandBuffer> cb = moe_submit(nb,D,Iinter,fmt,gs4,g,u,d,gs,us,ds,xg,xoff,nr,R,g_xg,g_gg,g_uu,g_hh);if (!cb) return 0;return moe_finish(cb,g_hh,nb,R,D,rows,rw,out);}@@ -1274,7 +1303,7 @@ struct ColiMetalMoeHandle {std::vector<int> rows; std::vector<float> rwv;int nb, R, D;};-extern "C" ColiMetalMoeHandle* coli_metal_moe_block_begin(int nb, int D, int Iinter, int fmt,+extern "C" ColiMetalMoeHandle* coli_metal_moe_block_begin(int nb, int D, int Iinter, int fmt, int gs4,const void *const *g, const void *const *u, const void *const *d,const float *const *gs, const float *const *us, const float *const *ds,const float *xg, const int *xoff, const int *nr,@@ -1286,7 +1315,7 @@ extern "C" ColiMetalMoeHandle* coli_metal_moe_block_begin(int nb, int D, int Iinid<MTLBuffer> bgg=[g_dev newBufferWithLength:(size_t)R*Iinter*4 options:g_res_opts];id<MTLBuffer> buu=[g_dev newBufferWithLength:(size_t)R*Iinter*4 options:g_res_opts];id<MTLBuffer> bhh=[g_dev newBufferWithLength:(size_t)R*D*4 options:g_res_opts];- id<MTLCommandBuffer> cb = moe_submit(nb,D,Iinter,fmt,g,u,d,gs,us,ds,xg,xoff,nr,R,bxg,bgg,buu,bhh);+ id<MTLCommandBuffer> cb = moe_submit(nb,D,Iinter,fmt,gs4,g,u,d,gs,us,ds,xg,xoff,nr,R,bxg,bgg,buu,bhh);if (!cb) return nullptr;ColiMetalMoeHandle *h = new ColiMetalMoeHandle();h->cb=cb; h->hh=bhh; h->rows.assign(rows,rows+R); h->rwv.assign(rw,rw+R);diff --git a/c/colibri.c b/c/colibri.cindex 1f288d9b..43a0222d 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -3195,7 +3195,7 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p* flow through the shared per-fmt shader. */if(g_metal_enabled && !kvs && S<=4 && (g_absorb==1||(g_absorb<0&&S<=4)) && m->kv_start[layer]==0&& D==6144 && H==64 && c->q_lora==2048 && c->kv_lora==512 && c->qk_nope==192- && c->qk_rope==64 && vh==256 && l->kv_b.fmt==2+ && c->qk_rope==64 && vh==256 && (l->kv_b.fmt==2||l->kv_b.fmt==4)&& metal_fused_fmt_ok(l->q_a.fmt) && metal_fused_fmt_ok(l->q_b.fmt)&& metal_fused_fmt_ok(l->kv_a.fmt) && metal_fused_fmt_ok(l->o.fmt)){int sel_active = m->has_dsa && layer<c->n_layers && c->idx_type[layer] && (pos_base+S) > c->index_topk;@@ -3211,7 +3211,7 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int pWP_(l->q_a), l->q_a.s, l->q_a.fmt, l->q_a.gs, l->q_a_ln,WP_(l->q_b), l->q_b.s, l->q_b.fmt, l->q_b.gs,WP_(l->kv_a), l->kv_a.s, l->kv_a.fmt, l->kv_a.gs, l->kv_a_ln,- WP_(l->kv_b), l->kv_b.s, l->kv_b.fmt,+ WP_(l->kv_b), l->kv_b.s, l->kv_b.fmt, l->kv_b.gs,WP_(l->o), l->o.s, l->o.fmt, l->o.gs,m->Lc[layer], m->Rc[layer], S, pos_base, m->kv_start[layer], c->eps, c->theta, c->attn_scale, out);#undef WP_@@ -4170,19 +4170,20 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int* batch's rows all share s, so this is one FWHT per token in practice. The down* input is rotated on-GPU by moe_fwht inside moe_submit. */int is_miss[64]={0}; ColiMetalMoeHandle *mh=NULL;- int cpu_res=1, cpu_miss=1, mh_shared=0, nbb=0, Rtot=0, mfmt=-1, sh_in=0;+ int cpu_res=1, cpu_miss=1, mh_shared=0, nbb=0, Rtot=0, mfmt=-1, mgs=0, sh_in=0;const void *MG[65],*MU[65],*MD[65]; const float *MGS[65],*MUS[65],*MDS[65];int xoffb[65],nrb[65];float *mxg=NULL; int *mrows=NULL; float *mrw=NULL;/* subset builder: experts with is_miss==WANTMISS (+ shared expert when TRY_SH) */#define MB_BUILD(WANTMISS, TRY_SH) do{ \- nbb=0; Rtot=0; mfmt=-1; sh_in=0; \+ nbb=0; Rtot=0; mfmt=-1; mgs=0; sh_in=0; \for(int j=0;j<nb;j++){ if(is_miss[j]!=(WANTMISS)) continue; \int eid=uniq[base+j]; ESlot *e=use[j]; int cnt=0; \for(int s=0;s<S;s++) for(int kk=0;kk<keff[s];kk++) \if(idxs[(int64_t)s*K+kk]==eid){ cnt++; break; } \if(!cnt) continue; \- if(mfmt<0) mfmt=e->g.fmt; \+ if(mfmt<0){ mfmt=e->g.fmt; mgs=e->g.gs; } \+ if(mfmt==4 && (e->g.gs!=mgs||e->u.gs!=mgs||e->d.gs!=mgs)) mgs=-1; /* poison -> moe_submit refuses, whole subset stays CPU */ \MG[nbb]=e->g.fmt==1?(const void*)e->g.q8:(const void*)e->g.q4; \MU[nbb]=e->u.fmt==1?(const void*)e->u.q8:(const void*)e->u.q4; \MD[nbb]=e->d.fmt==1?(const void*)e->d.q8:(const void*)e->d.q4; \@@ -4190,8 +4191,10 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intxoffb[nbb]=Rtot; nrb[nbb]=cnt; Rtot+=cnt; nbb++; \} \if(TRY_SH){ int shf = mfmt<0 ? l->sh_gate.fmt : mfmt; \- if(c->n_shared==1 && sI==I && l->sh_gate.fmt==shf && l->sh_up.fmt==shf && l->sh_down.fmt==shf){ \- if(mfmt<0) mfmt=shf; \+ int shg = mfmt<0 ? l->sh_gate.gs : mgs; \+ if(c->n_shared==1 && sI==I && l->sh_gate.fmt==shf && l->sh_up.fmt==shf && l->sh_down.fmt==shf \+ && (shf!=4 || (shg>0 && l->sh_gate.gs==shg && l->sh_up.gs==shg && l->sh_down.gs==shg))){ \+ if(mfmt<0){ mfmt=shf; mgs=shg; } \MG[nbb]=shf==1?(const void*)l->sh_gate.q8:(const void*)l->sh_gate.q4; \MU[nbb]=shf==1?(const void*)l->sh_up.q8 :(const void*)l->sh_up.q4; \MD[nbb]=shf==1?(const void*)l->sh_down.q8:(const void*)l->sh_down.q4; \@@ -4215,7 +4218,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intif(nbb>0){double t0=now_s();if(mfmt==6) metal_stage_rot_e8(mxg,mrows,Rtot,D);- mh=coli_metal_moe_block_begin(nbb,D,I,mfmt,MG,MU,MD,MGS,MUS,MDS,mxg,xoffb,nrb,mrows,mrw);+ mh=coli_metal_moe_block_begin(nbb,D,I,mfmt,mgs,MG,MU,MD,MGS,MUS,MDS,mxg,xoffb,nrb,mrows,mrw);m->t_emm += now_s()-t0;if(mh){ cpu_res=0; mh_shared=sh_in; }} else cpu_res=0;@@ -4278,7 +4281,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intif(nbb>0){double t0=now_s();if(mfmt==6) metal_stage_rot_e8(mxg,mrows,Rtot,D);- if(coli_metal_moe_block(nbb,D,I,mfmt,MG,MU,MD,MGS,MUS,MDS,mxg,xoffb,nrb,mrows,mrw,out,S)) cpu_miss=0;+ if(coli_metal_moe_block(nbb,D,I,mfmt,mgs,MG,MU,MD,MGS,MUS,MDS,mxg,xoffb,nrb,mrows,mrw,out,S)) cpu_miss=0;m->t_emm += now_s()-t0;} else cpu_miss=0;if(mh){ double t0=now_s();@@ -5463,7 +5466,7 @@ static void layer_forward_rows(Model *m, Layer *l, int li, float *x, int S, intif(g_metal_enabled && !kvs && S<=4 && li<c->n_layers && l->sparse&& (g_absorb==1||(g_absorb<0&&S<=4)) && m->kv_start[li]==0&& D==6144 && c->n_heads==64 && c->q_lora==2048 && c->kv_lora==512- && c->qk_nope==192 && c->qk_rope==64 && c->v_head==256 && l->kv_b.fmt==2+ && c->qk_nope==192 && c->qk_rope==64 && c->v_head==256 && (l->kv_b.fmt==2||l->kv_b.fmt==4)&& c->n_experts==256 && c->topk==8 && c->n_shared==1 && c->moe_inter==2048&& metal_fused_fmt_ok(l->q_a.fmt) && metal_fused_fmt_ok(l->q_b.fmt)&& metal_fused_fmt_ok(l->kv_a.fmt) && metal_fused_fmt_ok(l->o.fmt)@@ -5482,7 +5485,7 @@ static void layer_forward_rows(Model *m, Layer *l, int li, float *x, int S, intWP_(l->q_a), l->q_a.s, l->q_a.fmt, l->q_a.gs, l->q_a_ln,WP_(l->q_b), l->q_b.s, l->q_b.fmt, l->q_b.gs,WP_(l->kv_a), l->kv_a.s, l->kv_a.fmt, l->kv_a.gs, l->kv_a_ln,- WP_(l->kv_b), l->kv_b.s, l->kv_b.fmt,+ WP_(l->kv_b), l->kv_b.s, l->kv_b.fmt, l->kv_b.gs,WP_(l->o), l->o.s, l->o.fmt, l->o.gs,WP_(l->sh_gate), l->sh_gate.s, l->sh_gate.fmt, l->sh_gate.gs,WP_(l->sh_up), l->sh_up.s, l->sh_up.fmt, l->sh_up.gs,@@ -7392,6 +7395,7 @@ static void run_serve(Model *m, const char *snap){_setmode(_fileno(stdout), _O_BINARY);setvbuf(stdout, NULL, _IONBF, 0);#endif+ double t_serve0=now_s(); /* PROF: wall base for the exit-time profile_print */char tkp[2048]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",snap);Tok T; tok_load(&T,tkp);int eos=tok_id_of(&T,"<|endoftext|>");@@ -7544,6 +7548,12 @@ static void run_serve(Model *m, const char *snap){}free(line); free(buf);usage_save(m);+ /* PROF=1 only: the cumulative backend counters (METAL:/METAL-ATTN:/MIRROR:) were+ * unreachable in serve mode — profile_print only ran on the oracle/generate exit+ * paths, so a served session could never show GPU-vs-fallback truth. stdin has hit+ * EOF here: the last END/STAT frame is already out, so these stdout lines can no+ * longer interleave with protocol a client is parsing. */+ if(g_prof) profile_print(m, now_s()-t_serve0);#undef hist#undef len#undef firstdiff --git a/c/tests/test_backend_metal.mm b/c/tests/test_backend_metal.mmindex 07b82613..d0a9c077 100644--- a/c/tests/test_backend_metal.mm+++ b/c/tests/test_backend_metal.mm@@ -96,11 +96,13 @@ static float ref_e4m3(uint8_t b) {: (1.0f+(float)mant*(1.0f/8.0f))*powf(2.0f,(float)exp-7.0f);return sign ? -val : val;}-static int fp8_nblk(int n){ return (n+127)/128; }+// Local to this section's self-contained reference; deliberately NOT quant.h's+// fp8_nblk (which this TU also pulls in further down, with a different return type).+static int ref_fp8_nblk(int n){ return (n+127)/128; }static void cpu_ref_fp8(const uint8_t *q8, const float *bscale, const float *x,double *y, double *mag, int S, int I, int O) {- int nblkI = fp8_nblk(I);+ int nblkI = ref_fp8_nblk(I);for (int o=0;o<O;o++){const uint8_t *w = q8 + (size_t)o*I;const float *scl = bscale + (size_t)(o/128)*nblkI;@@ -172,7 +174,7 @@ static int run_grouped(int O, int I, int gs, int S, int outlier, const char *nam// the block-scale accumulation, avoiding cancellation-noise false positives on// near-zero results).static int run_fp8(int O, int I, int S, const char *name) {- int nblkO=fp8_nblk(O), nblkI=fp8_nblk(I), nblk=nblkO*nblkI;+ int nblkO=ref_fp8_nblk(O), nblkI=ref_fp8_nblk(I), nblk=nblkO*nblkI;std::vector<uint8_t> W((size_t)O*I);std::vector<float> scale((size_t)nblk), x((size_t)S*I), yg((size_t)S*O);std::vector<double> yr((size_t)S*O), mag((size_t)S*O);@@ -213,7 +215,7 @@ static int run_fp8(int O, int I, int S, const char *name) {static int run_fp8_lut(const char *name) {enum { O=256, I=1 };std::vector<uint8_t> W(O*I); for (int b=0;b<O;b++) W[b]=(uint8_t)b;- std::vector<float> scale(fp8_nblk(O)*fp8_nblk(I), 1.0f); // nblkO=2,nblkI=1 -> both blocks scale=1+ std::vector<float> scale(ref_fp8_nblk(O)*ref_fp8_nblk(I), 1.0f); // nblkO=2,nblkI=1 -> both blocks scale=1std::vector<float> x(I, 1.0f), yg(O);ColiMetalTensor *t=nullptr;if (!coli_metal_matmul(&t, yg.data(), x.data(), W.data(), scale.data(), FP8, 1, I, O, 0)) {@@ -250,9 +252,9 @@ static int run_fp8_gemm_gate(const char *name) {// up with no routed expert already fixing `mfmt`, it would submit the WRONG pointer// (q4, NULL/stale for an fmt=8 tensor whose weights live in q8) tagged as fmt=8.// This is safe ANYWAY, but only incidentally: moe_submit() (backend_metal.mm) gates-// `fmt != 1 && fmt != 2` as its very FIRST statement, before any of g/u/d/gs/us/ds is-// dereferenced or even resolve()'d -- so an fmt=8 submission is refused before the-// bad pointer would ever be read, no matter what garbage MB_BUILD packed into it. This+// on its fmt allowlist ({1,2,4,6}) as its very FIRST statement, before any of+// g/u/d/gs/us/ds is dereferenced or even resolve()'d -- so an fmt=8 submission is+// refused before the bad pointer would ever be read, no matter what garbage MB_BUILD packed into it. This// test uses deliberately-invalid weight/scale pointers (never dereferenced if the gate// holds) to prove the fence BY TEST rather than leaving it an artifact of moe_submit's// fmt allowlist happening not to include 8 (yet) -- same discipline@@ -263,7 +265,7 @@ static int run_fp8_moe_gate(const char *name) {const float *gs[1] = {(const float*)bad}, *us[1] = {(const float*)bad}, *ds[1] = {(const float*)bad};float xg[8]={0}, out[8]={0}, rw[1]={1.0f};int xoff[1]={0}, nr[1]={1}, rows[1]={0};- int rc = coli_metal_moe_block(1, 8, 8, FP8, g, u, d, gs, us, ds, xg, xoff, nr, rows, rw, out, 1);+ int rc = coli_metal_moe_block(1, 8, 8, FP8, 0, g, u, d, gs, us, ds, xg, xoff, nr, rows, rw, out, 1);int ok = (rc == 0);printf(" %-42s rc=%d (expect 0/CPU-fallback) %s\n", name, rc, ok?"ok":"*** MISMATCH (should have refused)");return ok?0:1;@@ -304,7 +306,58 @@ static int run_moe(const std::vector<int>& nrv, const char* name) {float* os=&refout[(size_t)rows[gr]*D]; for(int o=0;o<D;o++) os[o]+=rw[gr]*hh[o];}std::vector<float> gout((size_t)S*D,0.f);- int ok = coli_metal_moe_block(nb,D,I,fmt,g.data(),u.data(),d.data(),gs.data(),us.data(),ds.data(),+ int ok = coli_metal_moe_block(nb,D,I,fmt,0,g.data(),u.data(),d.data(),gs.data(),us.data(),ds.data(),+ xg.data(),xoff.data(),nr.data(),rows.data(),rw.data(),gout.data(),S);+ double maxabs=0,ymax=0; for(size_t i=0;i<gout.size();i++){ maxabs=fmax(maxabs,fabs(gout[i]-refout[i])); ymax=fmax(ymax,fabs(refout[i])); }+ double nerr=maxabs/(ymax+1e-9); int pass = ok && nerr<1e-4;+ printf(" %-22s R=%d nerr=%.2e %s\n", name, R, nerr, pass?"ok":"*** MISMATCH");+ for(int e=0;e<nb;e++){ coli_metal_unregister(slab[e]); coli_metal_unregister(fslab[e]); free(slab[e]); free(fslab[e]); }+ return pass?0:1;+}++// ---- fmt=4 (grouped int4) moe_block: run_moe's harness with per-group scales -------+// Same shapes/flow as run_moe, but the scale slab holds [O,ceil(K/gs)] floats per matrix+// (gate/up: K=D groups along hidden; down: K=I) and the reference folds scl[i/gs] into+// the MAC instead of a post-scale -- the same semantics as matmul_i4_grouped and the+// mm_gemv fmt==4 branch this kernel path mirrors. gs=64 matches the i4-g64 snapshots+// (GLM-5.2-i4) that hit this path in the engine.+static int run_moe_g4(const std::vector<int>& nrv, const char* name) {+ const int D=6144, I=2048, fmt=4, GSZ=64;+ int rbG=(D+1)/2, rbD=(I+1)/2, ngG=(D+GSZ-1)/GSZ, ngD=(I+GSZ-1)/GSZ, nb=(int)nrv.size();+ int R=0; std::vector<int> xoff(nb),nr(nrv); for(int e=0;e<nb;e++){ xoff[e]=R; R+=nrv[e]; }+ srand(4096+nb);+ std::vector<void*> slab(nb), fslab(nb);+ std::vector<const void*> g(nb),u(nb),d(nb); std::vector<const float*> gs(nb),us(nb),ds(nb);+ size_t wlen=roundpg((size_t)I*rbG*2 + (size_t)D*rbD);+ size_t nsc=(size_t)I*ngG*2 + (size_t)D*ngD, flen=roundpg(nsc*sizeof(float));+ for(int e=0;e<nb;e++){+ posix_memalign(&slab[e],16384,wlen); posix_memalign(&fslab[e],16384,flen);+ uint8_t* sp=(uint8_t*)slab[e]; for(size_t i=0;i<(size_t)I*rbG*2+(size_t)D*rbD;i++) sp[i]=(uint8_t)(rand()&0xFF);+ float* fp=(float*)fslab[e];+ for(size_t i=0;i<nsc;i++) fp[i]=(0.001f+(rand()%500)/50000.f)*((rand()&1)?1.f:-1.f);+ g[e]=sp; u[e]=sp+(size_t)I*rbG; d[e]=sp+(size_t)I*rbG*2;+ gs[e]=fp; us[e]=fp+(size_t)I*ngG; ds[e]=fp+(size_t)I*ngG*2;+ coli_metal_register(slab[e],wlen); coli_metal_register(fslab[e],flen);+ }+ std::vector<float> xg((size_t)R*D); for(auto&v:xg) v=((rand()%2000)-1000)/1000.f;+ std::vector<int> rows(R); std::vector<float> rw(R);+ for(int gr=0;gr<R;gr++){ rows[gr]=0; rw[gr]=0.1f+(rand()%100)/100.f; } // decode: all -> position 0+ int S=1;+ // CPU reference: per-group scale folded into the MAC, no post-scale+ std::vector<float> refout((size_t)S*D,0.f), gg(I),uu(I),hh(D);+ for(int e=0;e<nb;e++) for(int r=0;r<nr[e];r++){ int gr=xoff[e]+r; const float* xr=&xg[(size_t)gr*D];+ const uint8_t* wg=(const uint8_t*)g[e]; const uint8_t* wu=(const uint8_t*)u[e]; const uint8_t* wd=(const uint8_t*)d[e];+ for(int o=0;o<I;o++){ const float* scl=gs[e]+(size_t)o*ngG; float a=0;+ for(int k=0;k<D;k++) a+=deq4(wg+(size_t)o*rbG,k)*xr[k]*scl[k/GSZ]; gg[o]=a; }+ for(int o=0;o<I;o++){ const float* scl=us[e]+(size_t)o*ngG; float a=0;+ for(int k=0;k<D;k++) a+=deq4(wu+(size_t)o*rbG,k)*xr[k]*scl[k/GSZ]; uu[o]=a; }+ for(int o=0;o<I;o++){ float v=gg[o]; gg[o]=(v/(1.f+expf(-v)))*uu[o]; }+ for(int o=0;o<D;o++){ const float* scl=ds[e]+(size_t)o*ngD; float a=0;+ for(int k=0;k<I;k++) a+=deq4(wd+(size_t)o*rbD,k)*gg[k]*scl[k/GSZ]; hh[o]=a; }+ float* os=&refout[(size_t)rows[gr]*D]; for(int o=0;o<D;o++) os[o]+=rw[gr]*hh[o];+ }+ std::vector<float> gout((size_t)S*D,0.f);+ int ok = coli_metal_moe_block(nb,D,I,fmt,GSZ,g.data(),u.data(),d.data(),gs.data(),us.data(),ds.data(),xg.data(),xoff.data(),nr.data(),rows.data(),rw.data(),gout.data(),S);double maxabs=0,ymax=0; for(size_t i=0;i<gout.size();i++){ maxabs=fmax(maxabs,fabs(gout[i]-refout[i])); ymax=fmax(ymax,fabs(refout[i])); }double nerr=maxabs/(ymax+1e-9); int pass = ok && nerr<1e-4;@@ -358,7 +411,7 @@ static int run_moe_e8(const std::vector<int>& nrv, const char* name) {std::vector<float> xg_gpu(xg);for(int gr=0;gr<R;gr++) e8_rot_rows(&xg_gpu[(size_t)gr*D],1,D);std::vector<float> gout((size_t)S*D,0.f);- int ok = coli_metal_moe_block(nb,D,I,fmt,g.data(),u.data(),d.data(),gs.data(),us.data(),ds.data(),+ int ok = coli_metal_moe_block(nb,D,I,fmt,0,g.data(),u.data(),d.data(),gs.data(),us.data(),ds.data(),xg_gpu.data(),xoff.data(),nr.data(),rows.data(),rw.data(),gout.data(),S);double maxabs=0,ymax=0; for(size_t i=0;i<gout.size();i++){ maxabs=fmax(maxabs,fabs(gout[i]-refout[i])); ymax=fmax(ymax,fabs(refout[i])); }double nerr=maxabs/(ymax+1e-9); int pass = ok && nerr<1e-4;@@ -402,10 +455,14 @@ static void t_gemv4g(float*y,const float*x,const uint8_t*w,const float*sc,int O,for(int o=0;o<O;o++){ const uint8_t*r=w+(size_t)o*rb; const float* scl=sc+(size_t)o*ng; float a=0;for(int i=0;i<I;i++){ uint8_t b=r[i>>1]; int v=(i&1)?(b>>4):(b&0xF); a+=(float)(v-8)*x[i]*scl[i/gs]; }y[o]=a; } }-static int run_attn(int S, int pos_base, const char* name){+// kvb_g4=1 swaps kv_b to fmt=4 (grouped int4, gs=64 like the i4-g64 snapshots): proves+// the a_qabs/a_ctx in-kernel dequant handles per-group scales, not just per-row.+static int run_attn(int S, int pos_base, const char* name, int kvb_g4=0){const float eps=1e-5f, theta=10000.f, ascale=1.f/16.f;+ const int KGS=64, ng=(TKVL+KGS-1)/KGS;srand(4242+S+pos_base);- TW qa=t_mkw(TQL,TH), qb=t_mkw(THH*TQH,TQL), kva=t_mkw(TKVL+TROPE,TH), kvb=t_mkw(THH*TROWSH,TKVL), o=t_mkw(TH,THH*TVH);+ TW qa=t_mkw(TQL,TH), qb=t_mkw(THH*TQH,TQL), kva=t_mkw(TKVL+TROPE,TH), o=t_mkw(TH,THH*TVH);+ TW kvb = kvb_g4 ? t_mkw_g(THH*TROWSH,TKVL,KGS) : t_mkw(THH*TROWSH,TKVL);std::vector<float> qaln(TQL), kvaln(TKVL);for(auto&v:qaln) v=0.5f+(rand()%1000)/1000.f; for(auto&v:kvaln) v=0.5f+(rand()%1000)/1000.f;int T=pos_base+S; size_t lcb=(((size_t)T*TKVL*4)+16383)&~(size_t)16383, rcb=(((size_t)T*TROPE*4)+16383)&~(size_t)16383;@@ -433,22 +490,26 @@ static int run_attn(int S, int pos_base, const char* name){for(int h=0;h<THH;h++){ int rbase=h*TROWSH;const float* qp=&Q[(size_t)s*THH*TQH+(size_t)h*TQH]; const float* qro=qp+TNOPE;std::vector<float> qabs(TKVL,0);- for(int d=0;d<TNOPE;d++){ const uint8_t*r=kvb.w+(size_t)(rbase+d)*rb; float sc=kvb.s[rbase+d];- for(int i=0;i<TKVL;i++){ uint8_t b=r[i>>1]; int v=(i&1)?(b>>4):(b&0xF); qabs[i]+=qp[d]*(float)(v-8)*sc; } }+ for(int d=0;d<TNOPE;d++){ const uint8_t*r=kvb.w+(size_t)(rbase+d)*rb;+ const float* scl=kvb.s+(kvb_g4?(size_t)(rbase+d)*ng:(size_t)(rbase+d));+ for(int i=0;i<TKVL;i++){ uint8_t b=r[i>>1]; int v=(i&1)?(b>>4):(b&0xF);+ qabs[i]+=qp[d]*(float)(v-8)*(kvb_g4?scl[i/KGS]:scl[0]); } }std::vector<float> a(pos+1);for(int t=0;t<=pos;t++){ const float*Lt=&Lr[(size_t)t*TKVL]; const float*Rt=&Rr[(size_t)t*TROPE];float v=0; for(int i=0;i<TKVL;i++) v+=qabs[i]*Lt[i]; for(int d=0;d<TROPE;d++) v+=qro[d]*Rt[d]; a[t]=v*ascale; }float mx=-1e30f; for(float v:a) mx=fmaxf(mx,v); float sum=0; for(float&v:a){ v=expf(v-mx); sum+=v; } for(float&v:a) v/=sum;std::vector<float> cl(TKVL,0);for(int t=0;t<=pos;t++){ const float*Lt=&Lr[(size_t)t*TKVL]; for(int i=0;i<TKVL;i++) cl[i]+=a[t]*Lt[i]; }- for(int j=0;j<TVH;j++){ const uint8_t*r=kvb.w+(size_t)(rbase+TNOPE+j)*rb; float sc=kvb.s[rbase+TNOPE+j];- float v=0; for(int i=0;i<TKVL;i++){ uint8_t b=r[i>>1]; int vv=(i&1)?(b>>4):(b&0xF); v+=cl[i]*(float)(vv-8)*sc; }+ for(int j=0;j<TVH;j++){ const uint8_t*r=kvb.w+(size_t)(rbase+TNOPE+j)*rb;+ const float* scl=kvb.s+(kvb_g4?(size_t)(rbase+TNOPE+j)*ng:(size_t)(rbase+TNOPE+j));+ float v=0; for(int i=0;i<TKVL;i++){ uint8_t b=r[i>>1]; int vv=(i&1)?(b>>4):(b&0xF);+ v+=cl[i]*(float)(vv-8)*(kvb_g4?scl[i/KGS]:scl[0]); }ctx[(size_t)h*TVH+j]=v; } }t_gemv4(&ref[(size_t)s*TH],ctx.data(),o.w,o.s,TH,THH*TVH);}std::vector<float> got((size_t)S*TH);int ok=coli_metal_attn_decode(x.data(), qa.w,qa.s,2,0,qaln.data(), qb.w,qb.s,2,0,- kva.w,kva.s,2,0,kvaln.data(), kvb.w,kvb.s,2, o.w,o.s,2,0,+ kva.w,kva.s,2,0,kvaln.data(), kvb.w,kvb.s,kvb_g4?4:2,kvb_g4?KGS:0, o.w,o.s,2,0,Lc,Rc,S,pos_base,0,eps,theta,ascale,got.data());double ma=0,ym=0; for(size_t i=0;i<ref.size();i++){ ma=fmax(ma,fabs(got[i]-ref[i])); ym=fmax(ym,fabs(ref[i])); }// also verify the cache write-back (Lc/Rc for the new positions)@@ -610,7 +671,7 @@ static int run_attn_grouped(int S, int pos_base, int gs, const char* name){}std::vector<float> got((size_t)S*TH);int ok=coli_metal_attn_decode(x.data(), qa.w,qa.s,4,gs,qaln.data(), qb.w,qb.s,2,0, // <- qa fmt=4/gs- kva.w,kva.s,2,0,kvaln.data(), kvb.w,kvb.s,2, o.w,o.s,2,0,+ kva.w,kva.s,2,0,kvaln.data(), kvb.w,kvb.s,2,0, o.w,o.s,2,0,Lc,Rc,S,pos_base,0,eps,theta,ascale,got.data());double ma=0,ym=0; for(size_t i=0;i<ref.size();i++){ ma=fmax(ma,fabs(got[i]-ref[i])); ym=fmax(ym,fabs(ref[i])); }double mc=0; for(int s=0;s<S;s++){ int pos=pos_base+s;@@ -672,6 +733,8 @@ int main(void) {printf("Metal batched moe_block tests:\n");fail |= run_moe({1,1,1,1,1,1,1,1}, "moe decode nb=8");fail |= run_moe({3,1,4,2,1,5}, "moe ragged nb=6");+ fail |= run_moe_g4({1,1,1,1,1,1,1,1}, "moe i4-g64 decode nb=8");+ fail |= run_moe_g4({3,1,4,2,1,5}, "moe i4-g64 ragged nb=6");printf("Metal fmt=6 (E8/IQ3) moe_block tests:\n");fail |= run_moe_e8({1,1,1,1,1,1,1,1}, "e8 decode nb=8");fail |= run_moe_e8({3,1,4,2,1,5}, "e8 ragged nb=6");@@ -718,6 +781,10 @@ int main(void) {fail |= run_attn(1, 37, "attn S=1 pos=37");fail |= run_attn(4, 12, "attn S=4 pos=12 (MTP)");fail |= run_attn(3, 0, "attn S=3 pos=0");+ printf("Metal fused attention tests (fmt=4 grouped kv_b, proves a_qabs/a_ctx gs dequant):\n");+ fail |= run_attn(1, 0, "attn kvb-g64 S=1 pos=0", 1);+ fail |= run_attn(1, 37, "attn kvb-g64 S=1 pos=37", 1);+ fail |= run_attn(4, 12, "attn kvb-g64 S=4 (MTP)", 1);printf("Metal fused attention tests (fmt=4 grouped q_a, proves bind_gemv gs plumbing):\n");// pos_base=37 (not 0): at T=1 softmax is identically 1.0 regardless of q_a's output,// so an S=1 pos=0 case cannot catch ANY q_a defect (review round 1, auditor -- see