Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
metal: adapt #587's fmt=4 kv_b + moe_block coverage onto our #457 base
monotophic19d400489aef
diff --git a/c/backend_metal.h b/c/backend_metal.hindex 2c09ba76..552adf83 100644--- a/c/backend_metal.h+++ b/c/backend_metal.h@@ -80,7 +80,7 @@ int coli_metal_layer_decode(float *x,const void *qa_w, const float *qa_s, int qa_fmt, int qa_gs, const float *qa_ln,const void *qb_w, const float *qb_s, int qb_fmt, int qb_gs,const void *kva_w, const float *kva_s, int kva_fmt, int kva_gs, const float *kva_ln,- const void *kvb_w, const float *kvb_s, int kvb_fmt,+ const void *kvb_w, const float *kvb_s, int kvb_fmt, int kvb_gs,const void *o_w, const float *o_s, int o_fmt, int o_gs,const void *shg_w, const float *shg_s, int shg_fmt, int shg_gs,const void *shu_w, const float *shu_s, int shu_fmt, int shu_gs,@@ -116,7 +116,7 @@ int coli_metal_attn_decode(const float *x,const void *qa_w, const float *qa_s, int qa_fmt, int qa_gs, const float *qa_ln,const void *qb_w, const float *qb_s, int qb_fmt, int qb_gs,const void *kva_w, const float *kva_s, int kva_fmt, int kva_gs, const float *kva_ln,- const void *kvb_w, const float *kvb_s, int kvb_fmt,+ const void *kvb_w, const float *kvb_s, int kvb_fmt, int kvb_gs,const void *o_w, const float *o_s, int o_fmt, int o_gs,float *Lc, float *Rc, int S, int pos_base, int st0, float eps, float theta, float ascale, float *out);@@ -137,18 +137,16 @@ int coli_metal_resset_stats(double *flush_s);** D = hidden size, Iinter = moe intermediate size* g/u/d[e] = pointers to expert e's gate/up/down quantized weights (in RAM slabs)- * gs/us/ds[e] = pointers to expert e's per-row scales- * fmt = quant format (shared across experts). NOTE: fmt=4 (grouped int4) is- * NOT yet supported here -- gates to {1,2} and returns 0 (CPU fallback)- * for fmt=4 experts, same as before this stage. Grouped-int4 gained GPU- * support in mm_gemv (coli_metal_matmul/coli_metal_gemm/bind_gemv) only;- * extending the batched routed-expert path is future work (see PR_BODY.md).+ * gs/us/ds[e] = pointers to expert e's per-row (fmt=1/2) or per-group (fmt=4) scales+ * fmt = quant format (shared across experts): 1=int8, 2=int4 per-row, 4=int4+ * grouped. qgs is the fmt=4 group size (ignored, pass 0, for fmt!=4).+ * qgs = fmt=4 group size shared across experts in this block (0 for fmt!=4)* xg = packed activations [total_rows, D]; xoff[e] = row offset of expert e* nr[e] = rows for expert e; rows[]/rw[] map packed rows back to out positions* out = [S, D] accumulate target* Returns 1 on success, 0 to signal the caller to fall back to the CPU path.*/-int coli_metal_moe_block(int nb, int D, int Iinter, int fmt,+int coli_metal_moe_block(int nb, int D, int Iinter, int fmt, int qgs,const void *const *g, const void *const *u, const void *const *d,const float *const *gs, const float *const *us, const float *const *ds,const float *xg, const int *xoff, const int *nr,@@ -163,7 +161,7 @@ int coli_metal_moe_block(int nb, int D, int Iinter, int fmt,* end returns 0 on GPU fault (caller redoes those experts on CPU).*/typedef struct ColiMetalMoeHandle ColiMetalMoeHandle;-ColiMetalMoeHandle* coli_metal_moe_block_begin(int nb, int D, int Iinter, int fmt,+ColiMetalMoeHandle* coli_metal_moe_block_begin(int nb, int D, int Iinter, int fmt, int qgs,const void *const *g, const void *const *u, const void *const *d,const float *const *gs, const float *const *us, const float *const *ds,const float *xg, const int *xoff, const int *nr,diff --git a/c/backend_metal.mm b/c/backend_metal.mmindex 545e0350..14e1eec5 100644--- a/c/backend_metal.mm+++ b/c/backend_metal.mm@@ -154,7 +154,10 @@ kernel void mm_gemv(device const uchar* w [[buffer(0)]], // raw weight by}// Batched bindless expert GEMV: each row gr belongs to expert erow[gr], whose weight and-// scale live at gpuAddresses waddr[e]/saddr[e] (zero-copy in the RAM slab). fmt 1=i8, 2=i4.+// scale live at gpuAddresses waddr[e]/saddr[e] (zero-copy in the RAM slab). fmt 1=i8, 2=i4+// per-row, 4=i4 grouped (scale layout [O][ng], ng=ceil(K/qgs) -- same convention as mm_gemv+// fmt=4 above, but folded into the vectorized uchar4/float4 dot-product loop this kernel's+// fmt=2 branch already uses, since moe_gemv has no scalar strided branch to reuse).// One SIMDGROUP per output row, 4 rows/threadgroup, 8-value loads: measured 1.5-2.1x over// one-threadgroup-per-row with uchar2 loads (358-389 GB/s on engine-like block shapes).kernel void moe_gemv(device const ulong* waddr [[buffer(0)]], device const ulong* saddr [[buffer(1)]],@@ -162,7 +165,7 @@ kernel void moe_gemv(device const ulong* waddr [[buffer(0)]], device const ulongdevice float* yout [[buffer(4)]],constant int& O [[buffer(5)]], constant int& K [[buffer(6)]],constant int& Kin [[buffer(7)]], constant int& fmt [[buffer(8)]],- constant int& NT [[buffer(9)]],+ constant int& NT [[buffer(9)]], constant int& qgs [[buffer(10)]],uint tg [[threadgroup_position_in_grid]],uint slane [[thread_index_in_simdgroup]],uint sgid [[simdgroup_index_in_threadgroup]]) {@@ -210,13 +213,25 @@ kernel void moe_gemv(device const ulong* waddr [[buffer(0)]], device const ulongas_type<float>(uint(b.z)<<16), as_type<float>(uint(b.w)<<16));acc+=dot(w0,x4[2*c])+dot(w1,x4[2*c+1]); }for(int i=K8*8+slane;i<K;i+=32) acc += as_type<float>(uint(w[i])<<16)*xr[i];+ } else if (fmt == 4) { // grouped int4: per-expert scale [O][ng]+ int rb=(K+1)/2, ng=(K+qgs-1)/qgs; device const uchar* w=(device const uchar*)(waddr[e])+(long)o*rb;+ device const float* sr=sc+(long)o*ng; // grouped scales for this output row+ device const uchar4* w4=(device const uchar4*)w;+ for(int c=slane;c<K8;c+=32){ uchar4 b=w4[c];+ float4 w0=float4(float(int(b.x&0xF)-8),float(int(b.x>>4)-8),float(int(b.y&0xF)-8),float(int(b.y>>4)-8));+ float4 w1=float4(float(int(b.z&0xF)-8),float(int(b.z>>4)-8),float(int(b.w&0xF)-8),float(int(b.w>>4)-8));+ int g0=(8*c+0)/qgs,g1=(8*c+1)/qgs,g2=(8*c+2)/qgs,g3=(8*c+3)/qgs;+ int g4=(8*c+4)/qgs,g5=(8*c+5)/qgs,g6=(8*c+6)/qgs,g7=(8*c+7)/qgs;+ acc+=dot(w0*float4(sr[g0],sr[g1],sr[g2],sr[g3]),x4[2*c])+ +dot(w1*float4(sr[g4],sr[g5],sr[g6],sr[g7]),x4[2*c+1]); }+ for(int i=K8*8+slane;i<K;i+=32){ uchar b=w[i>>1]; int v=(i&1)?(b>>4):(b&0xF); acc+=float(v-8)*xr[i]*sr[i/qgs]; }} else { device const char* w=(device const char*)(waddr[e])+(long)o*K;device const char4* w4=(device const char4*)w;for(int c=slane;c<K8;c+=32) acc+=dot(float4(w4[2*c]),x4[2*c])+dot(float4(w4[2*c+1]),x4[2*c+1]);for(int i=K8*8+slane;i<K;i+=32) acc+=float(w[i])*xr[i];}acc=simd_sum(acc);- if(slane==0) yout[row] = (fmt==6||fmt==5) ? acc : acc*sc[o]; // fmt 6: in-block scales; fmt 5: none+ if(slane==0) yout[row] = (fmt==4 || fmt==5 || fmt==6) ? acc : acc*sc[o]; // fmt4 grouped / fmt6 in-block scales folded; fmt5 none}// fmt=6 activation rotation for the GPU-resident down-projection input: one FWHT@@ -277,14 +292,20 @@ kernel void a_copy(device const float* src [[buffer(0)]], constant int& off [[bu// ---- absorption core (S query rows, per-row causal). q:[S,H*QH]; qabs/clat:[S*H,KVL];// sc:[S*H,T]; ctx:[S*H,VH]. Query row s (abs pos PB+s) attends keys [0, PB+s]. ----constant int A_QHH=A_H*A_QH;-inline float a_deqrow(device const uchar* base, int row, int i, device const float* sc){- device const uchar* w=base+(long)row*((A_KVL+1)/2); uchar b=w[i>>1]; int val=(i&1)?(b>>4):(b&0xF); return float(val-8)*sc[row]; }+// kv_b inline dequant of column i of output row `row`. fmt=2 -> one scale per row;+// fmt=4 -> grouped int4, one scale per gs-wide group along the A_KVL input dim+// (scale layout [O][ng], ng=ceil(A_KVL/gs)), matching QT fmt=4 / mm_gemv above.+inline float a_deqrow(device const uchar* base, int row, int i, device const float* sc, int fmt, int gs){+ device const uchar* w=base+(long)row*((A_KVL+1)/2); uchar b=w[i>>1]; int val=(i&1)?(b>>4):(b&0xF);+ float s = (fmt==4) ? sc[(long)row*((A_KVL+gs-1)/gs) + i/gs] : sc[row];+ return float(val-8)*s; }kernel void a_qabs(device const uchar* kvb [[buffer(0)]], device const float* sc [[buffer(1)]],device const float* q [[buffer(2)]], device float* qabs [[buffer(3)]],+ constant int& fmt [[buffer(4)]], constant int& gs [[buffer(5)]],uint gid [[thread_position_in_grid]]) {int s=gid/(A_H*A_KVL), r=gid%(A_H*A_KVL), h=r/A_KVL, i=r%A_KVL; int rbase=h*A_ROWSH;device const float* qp=q+(long)s*A_QHH+(long)h*A_QH;- float a=0; for(int d=0;d<A_NOPE;d++) a+=qp[d]*a_deqrow(kvb,rbase+d,i,sc); qabs[(long)(s*A_H+h)*A_KVL+i]=a;+ float a=0; for(int d=0;d<A_NOPE;d++) a+=qp[d]*a_deqrow(kvb,rbase+d,i,sc,fmt,gs); qabs[(long)(s*A_H+h)*A_KVL+i]=a;}kernel void a_score(device const float* qabs [[buffer(0)]], device const float* Lc [[buffer(1)]],device const float* Rc [[buffer(2)]], device const float* q [[buffer(3)]],@@ -312,9 +333,11 @@ kernel void a_clat(device const float* sc [[buffer(0)]], device const float* Lcfor(int t=0;t<T;t++) a+=s[t]*Lc[(long)t*A_KVL+i]; clat[(long)sh*A_KVL+i]=a;}kernel void a_ctx(device const uchar* kvb [[buffer(0)]], device const float* sc [[buffer(1)]],- device const float* clat [[buffer(2)]], device float* ctx [[buffer(3)]], uint gid [[thread_position_in_grid]]) {+ device const float* clat [[buffer(2)]], device float* ctx [[buffer(3)]],+ constant int& fmt [[buffer(4)]], constant int& gs [[buffer(5)]],+ uint gid [[thread_position_in_grid]]) {int sh=gid/A_VH, j=gid%A_VH, h=sh%A_H; int row=h*A_ROWSH+A_NOPE+j; device const float* cl=clat+(long)sh*A_KVL;- float a=0; for(int i=0;i<A_KVL;i++) a+=cl[i]*a_deqrow(kvb,row,i,sc); ctx[(long)sh*A_VH+j]=a;+ float a=0; for(int i=0;i<A_KVL;i++) a+=cl[i]*a_deqrow(kvb,row,i,sc,fmt,gs); ctx[(long)sh*A_VH+j]=a;}// ===== full-layer tail kernels =====@@ -853,14 +876,14 @@ static bool bind_gemv(id<MTLComputeCommandEncoder> e, const void* w, const float// Weight-pointer bundle for one layer's attention (+optional layer tail). All pointers// must be inside registered allocations. *_gs: fmt=4 group size for the corresponding-// weight (0 if that weight isn't grouped). kv_b has none: it never flows through bind_gemv-// -- a_qabs/a_ctx dequantize it inline with a PER-ROW-only helper (a_deqrow), so a fmt=4-// kv_b is out of scope for this stage (see PR_BODY.md UNCERTAINTIES).+// weight (0 if that weight isn't grouped). kv_b never flows through bind_gemv -- a_qabs/+// a_ctx dequantize it inline via a_deqrow, which is fmt/gs-aware (fmt=2 per-row, fmt=4+// grouped along A_KVL); kvb_gs is that group size (0 for fmt=2).typedef struct {const void *qa_w; const float *qa_s; int qa_fmt; int qa_gs; const float *qa_ln;const void *qb_w; const float *qb_s; int qb_fmt; int qb_gs;const void *kva_w; const float *kva_s; int kva_fmt; int kva_gs; const float *kva_ln;- const void *kvb_w; const float *kvb_s; int kvb_fmt;+ const void *kvb_w; const float *kvb_s; int kvb_fmt; int kvb_gs;const void *o_w; const float *o_s; int o_fmt; int o_gs;} AttnW;@@ -890,6 +913,7 @@ static bool encode_attn_projections(id<MTLComputeCommandEncoder> e, const AttnWbind_gemv(e,W->qb_w,W->qb_s,W->qb_fmt,W->qb_gs,AQLORA,AHQH,aqr_,aqf_,S); rms(Lb,Loff,akvaln_,AKVL,S); rope(Rb,Roff,0,AROPE,0,1); BAR();rope(aqf_,0,ANOPE,AHQH,AQH,AHEADS); BAR();[e setComputePipelineState:g_a_qabs]; [e setBuffer:kvbW offset:kvbwoff atIndex:0]; [e setBuffer:kvbS offset:kvbsoff atIndex:1]; [e setBuffer:aqf_ offset:0 atIndex:2]; [e setBuffer:aqabs_ offset:0 atIndex:3];+ [e setBytes:&W->kvb_fmt length:4 atIndex:4]; [e setBytes:&W->kvb_gs length:4 atIndex:5];[e dispatchThreads:MTLSizeMake((size_t)S*AHEADS*AKVL,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; BAR();return true;}@@ -902,7 +926,8 @@ static bool encode_attn_projections(id<MTLComputeCommandEncoder> e, const AttnWstatic bool encode_attn_core_chunk(id<MTLComputeCommandEncoder> e,id<MTLBuffer> Lb, size_t loff, id<MTLBuffer> Rb, size_t roff,id<MTLBuffer> kvbW, size_t kvbwoff, id<MTLBuffer> kvbS, size_t kvbsoff,- int r0, int ch, int T, int pos_base, float ascale) {+ int r0, int ch, int T, int pos_base, float ascale,+ int kvb_fmt, int kvb_gs) {size_t qabs_off=(size_t)r0*AHEADS*AKVL*4, qf_off=(size_t)r0*AHQH*4, ctx_off=(size_t)r0*AHVH*4;int PB=pos_base;auto BAR=[&]{ [e memoryBarrierWithScope:MTLBarrierScopeBuffers]; };@@ -915,6 +940,7 @@ static bool encode_attn_core_chunk(id<MTLComputeCommandEncoder> e,[e setComputePipelineState:g_a_clat]; [e setBuffer:ascore_ offset:0 atIndex:0]; [e setBuffer:Lb offset:loff atIndex:1]; [e setBuffer:aclat_ offset:0 atIndex:2]; [e setBytes:&T length:4 atIndex:3];[e dispatchThreads:MTLSizeMake((size_t)ch*AHEADS*AKVL,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; BAR();[e setComputePipelineState:g_a_ctx]; [e setBuffer:kvbW offset:kvbwoff atIndex:0]; [e setBuffer:kvbS offset:kvbsoff atIndex:1]; [e setBuffer:aclat_ offset:0 atIndex:2]; [e setBuffer:actx_ offset:ctx_off atIndex:3];+ [e setBytes:&kvb_fmt length:4 atIndex:4]; [e setBytes:&kvb_gs length:4 atIndex:5];[e dispatchThreads:MTLSizeMake((size_t)ch*AHEADS*AVH,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; BAR();return true;}@@ -925,7 +951,7 @@ static bool encode_attention(id<MTLComputeCommandEncoder> e, const AttnW *W,id<MTLBuffer> kvbW, size_t kvbwoff, id<MTLBuffer> kvbS, size_t kvbsoff,int S, int T, int pos_base, float eps, float theta, float ascale) {if(!encode_attn_projections(e,W,Lb,loff,Rb,roff,kvbW,kvbwoff,kvbS,kvbsoff,S,pos_base,eps,theta)) return false;- if(!encode_attn_core_chunk(e,Lb,loff,Rb,roff,kvbW,kvbwoff,kvbS,kvbsoff,0,S,T,pos_base,ascale)) return false;+ if(!encode_attn_core_chunk(e,Lb,loff,Rb,roff,kvbW,kvbwoff,kvbS,kvbsoff,0,S,T,pos_base,ascale,W->kvb_fmt,W->kvb_gs)) return false;bind_gemv(e,W->o_w,W->o_s,W->o_fmt,W->o_gs,AHVH,AH,actx_,aout_,S);return true;}@@ -947,7 +973,7 @@ extern "C" int coli_metal_attn_decode(const float* x,const void* qa_w,const float* qa_s,int qa_fmt,int qa_gs,const float* qa_ln,const void* qb_w,const float* qb_s,int qb_fmt,int qb_gs,const void* kva_w,const float* kva_s,int kva_fmt,int kva_gs,const float* kva_ln,- const void* kvb_w,const float* kvb_s,int kvb_fmt,+ const void* kvb_w,const float* kvb_s,int kvb_fmt,int kvb_gs,const void* o_w,const float* o_s,int o_fmt,int o_gs,float* Lc,float* Rc,int S,int pos_base,int st0,float eps,float theta,float ascale,float* out){if(!g_dev) return 0;@@ -955,7 +981,7 @@ extern "C" int coli_metal_attn_decode(const float* x,int T=pos_base+S;@autoreleasepool {attn_scratch_init();- AttnW W={qa_w,qa_s,qa_fmt,qa_gs,qa_ln,qb_w,qb_s,qb_fmt,qb_gs,kva_w,kva_s,kva_fmt,kva_gs,kva_ln,kvb_w,kvb_s,kvb_fmt,o_w,o_s,o_fmt,o_gs};+ AttnW W={qa_w,qa_s,qa_fmt,qa_gs,qa_ln,qb_w,qb_s,qb_fmt,qb_gs,kva_w,kva_s,kva_fmt,kva_gs,kva_ln,kvb_w,kvb_s,kvb_fmt,kvb_gs,o_w,o_s,o_fmt,o_gs};id<MTLBuffer> Lb,Rb,kvbW,kvbS; size_t loff,roff,kvbwoff,kvbsoff;if(!resolve_attn(&W,Lc,Rc,&Lb,&loff,&Rb,&roff,&kvbW,&kvbwoff,&kvbS,&kvbsoff)) return 0;@@ -994,7 +1020,7 @@ extern "C" int coli_metal_layer_decode(float *x,const void* qa_w,const float* qa_s,int qa_fmt,int qa_gs,const float* qa_ln,const void* qb_w,const float* qb_s,int qb_fmt,int qb_gs,const void* kva_w,const float* kva_s,int kva_fmt,int kva_gs,const float* kva_ln,- const void* kvb_w,const float* kvb_s,int kvb_fmt,+ const void* kvb_w,const float* kvb_s,int kvb_fmt,int kvb_gs,const void* o_w,const float* o_s,int o_fmt,int o_gs,const void* shg_w,const float* shg_s,int shg_fmt,int shg_gs,const void* shu_w,const float* shu_s,int shu_fmt,int shu_gs,@@ -1009,7 +1035,7 @@ extern "C" int coli_metal_layer_decode(float *x,int T=pos_base+S; const int SI=2048;@autoreleasepool {attn_scratch_init();- AttnW W={qa_w,qa_s,qa_fmt,qa_gs,qa_ln,qb_w,qb_s,qb_fmt,qb_gs,kva_w,kva_s,kva_fmt,kva_gs,kva_ln,kvb_w,kvb_s,kvb_fmt,o_w,o_s,o_fmt,o_gs};+ AttnW W={qa_w,qa_s,qa_fmt,qa_gs,qa_ln,qb_w,qb_s,qb_fmt,qb_gs,kva_w,kva_s,kva_fmt,kva_gs,kva_ln,kvb_w,kvb_s,kvb_fmt,kvb_gs,o_w,o_s,o_fmt,o_gs};id<MTLBuffer> Lb,Rb,kvbW,kvbS; size_t loff,roff,kvbwoff,kvbsoff;if(!resolve_attn(&W,Lc,Rc,&Lb,&loff,&Rb,&roff,&kvbW,&kvbwoff,&kvbS,&kvbsoff)) return 0;uint64_t ina=0,pna=0,rwa=0,rba=0,d;@@ -1195,12 +1221,12 @@ extern "C" size_t coli_metal_tensor_bytes(const ColiMetalTensor *t) { return t ?// if Metal is off or any expert pointer is not in a registered slab.// Encode + commit a MoE block (no wait). Writes hh[R,D] into hh_buf. Returns nil on// unresolved slab / bad fmt (caller falls back to CPU).-static id<MTLCommandBuffer> moe_submit(int nb, int D, int Iinter, int fmt,+static id<MTLCommandBuffer> moe_submit(int nb, int D, int Iinter, int fmt, int qgs,const void *const *g, const void *const *u, const void *const *d,const float *const *gs, const float *const *us, const float *const *ds,const float *xg, const int *xoff, const int *nr, int R,id<MTLBuffer> xg_buf, id<MTLBuffer> gg_buf, id<MTLBuffer> uu_buf, id<MTLBuffer> hh_buf) {- if (!g_dev || (fmt != 1 && fmt != 2 && fmt != 5 && fmt != 6)) return nil;+ if (!g_dev || (fmt != 1 && fmt != 2 && fmt != 4 && fmt != 5 && fmt != 6)) return nil;if (fmt == 6) { /* e8 kernel assumes clean block tiling, and every FWHT tile of the* down input (CPU tiling rule, e8_rot_rows) must fit threadgroup mem */if ((D & 255) || (Iinter & 31)) return nil;@@ -1253,7 +1279,7 @@ static id<MTLCommandBuffer> moe_submit(int nb, int D, int Iinter, int fmt,[e setBuffer:wa offset:0 atIndex:0];[e setBuffer:sa offset:0 atIndex:1];[e setBuffer:berow offset:0 atIndex:2];[e setBuffer:xin offset:0 atIndex:3];[e setBuffer:y offset:0 atIndex:4];[e setBytes:&O length:4 atIndex:5];[e setBytes:&K length:4 atIndex:6];[e setBytes:&Kin length:4 atIndex:7];[e setBytes:&fmt length:4 atIndex:8];- [e setBytes:&NT length:4 atIndex:9];+ [e setBytes:&NT length:4 atIndex:9];[e setBytes:&qgs length:4 atIndex:10];[e dispatchThreadgroups:MTLSizeMake(((size_t)NT+3)/4,1,1) threadsPerThreadgroup:MTLSizeMake(128,1,1)]; };gemv(bag,bsg,xg_buf,gg_buf,Iinter,D,D); // gategemv(bau,bsu,xg_buf,uu_buf,Iinter,D,D); // up@@ -1305,7 +1331,7 @@ static int moe_finish(id<MTLCommandBuffer> cb, id<MTLBuffer> hh_buf, int nb, intreturn 1;}-extern "C" int coli_metal_moe_block(int nb, int D, int Iinter, int fmt,+extern "C" int coli_metal_moe_block(int nb, int D, int Iinter, int fmt, int qgs,const void *const *g, const void *const *u, const void *const *d,const float *const *gs, const float *const *us, const float *const *ds,const float *xg, const int *xoff, const int *nr,@@ -1318,7 +1344,7 @@ extern "C" int coli_metal_moe_block(int nb, int D, int Iinter, int fmt,g_gg = ensure(g_gg,&g_gg_cap,(size_t)R*Iinter*4);g_uu = ensure(g_uu,&g_uu_cap,(size_t)R*Iinter*4);g_hh = ensure(g_hh,&g_hh_cap,(size_t)R*D*4);- id<MTLCommandBuffer> cb = moe_submit(nb,D,Iinter,fmt,g,u,d,gs,us,ds,xg,xoff,nr,R,g_xg,g_gg,g_uu,g_hh);+ id<MTLCommandBuffer> cb = moe_submit(nb,D,Iinter,fmt,qgs,g,u,d,gs,us,ds,xg,xoff,nr,R,g_xg,g_gg,g_uu,g_hh);if (!cb) return 0;return moe_finish(cb,g_hh,nb,R,D,rows,rw,out);}@@ -1331,7 +1357,7 @@ struct ColiMetalMoeHandle {std::vector<int> rows; std::vector<float> rwv;int nb, R, D;};-extern "C" ColiMetalMoeHandle* coli_metal_moe_block_begin(int nb, int D, int Iinter, int fmt,+extern "C" ColiMetalMoeHandle* coli_metal_moe_block_begin(int nb, int D, int Iinter, int fmt, int qgs,const void *const *g, const void *const *u, const void *const *d,const float *const *gs, const float *const *us, const float *const *ds,const float *xg, const int *xoff, const int *nr,@@ -1343,7 +1369,7 @@ extern "C" ColiMetalMoeHandle* coli_metal_moe_block_begin(int nb, int D, int Iinid<MTLBuffer> bgg=[g_dev newBufferWithLength:(size_t)R*Iinter*4 options:g_res_opts];id<MTLBuffer> buu=[g_dev newBufferWithLength:(size_t)R*Iinter*4 options:g_res_opts];id<MTLBuffer> bhh=[g_dev newBufferWithLength:(size_t)R*D*4 options:g_res_opts];- id<MTLCommandBuffer> cb = moe_submit(nb,D,Iinter,fmt,g,u,d,gs,us,ds,xg,xoff,nr,R,bxg,bgg,buu,bhh);+ id<MTLCommandBuffer> cb = moe_submit(nb,D,Iinter,fmt,qgs,g,u,d,gs,us,ds,xg,xoff,nr,R,bxg,bgg,buu,bhh);if (!cb) return nullptr;ColiMetalMoeHandle *h = new ColiMetalMoeHandle();h->cb=cb; h->hh=bhh; h->rows.assign(rows,rows+R); h->rwv.assign(rw,rw+R);diff --git a/c/colibri.c b/c/colibri.cindex 2723e6f9..1c402e15 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -3271,7 +3271,7 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p* flow through the shared per-fmt shader. */if(g_metal_enabled && !kvs && g_absorb!=0 && (S<=4 || g_metal_prefill) && m->kv_start[layer]==0&& D==6144 && H==64 && c->q_lora==2048 && c->kv_lora==512 && c->qk_nope==192- && c->qk_rope==64 && vh==256 && l->kv_b.fmt==2+ && c->qk_rope==64 && vh==256 && (l->kv_b.fmt==2||l->kv_b.fmt==4)&& metal_fused_fmt_ok(l->q_a.fmt) && metal_fused_fmt_ok(l->q_b.fmt)&& metal_fused_fmt_ok(l->kv_a.fmt) && metal_fused_fmt_ok(l->o.fmt)){int sel_active = m->has_dsa && layer<c->n_layers && c->idx_type[layer] && (pos_base+S) > c->index_topk;@@ -3287,7 +3287,7 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int pWP_(l->q_a), l->q_a.s, l->q_a.fmt, l->q_a.gs, l->q_a_ln,WP_(l->q_b), l->q_b.s, l->q_b.fmt, l->q_b.gs,WP_(l->kv_a), l->kv_a.s, l->kv_a.fmt, l->kv_a.gs, l->kv_a_ln,- WP_(l->kv_b), l->kv_b.s, l->kv_b.fmt,+ WP_(l->kv_b), l->kv_b.s, l->kv_b.fmt, l->kv_b.gs,WP_(l->o), l->o.s, l->o.fmt, l->o.gs,m->Lc[layer], m->Rc[layer], S, pos_base, m->kv_start[layer], c->eps, c->theta, c->attn_scale, out);#undef WP_@@ -4253,19 +4253,19 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int* batch's rows all share s, so this is one FWHT per token in practice. The down* input is rotated on-GPU by moe_fwht inside moe_submit. */int is_miss[64]={0}; ColiMetalMoeHandle *mh=NULL;- int cpu_res=1, cpu_miss=1, mh_shared=0, nbb=0, Rtot=0, mfmt=-1, sh_in=0;+ int cpu_res=1, cpu_miss=1, mh_shared=0, nbb=0, Rtot=0, mfmt=-1, mgs=0, sh_in=0;const void *MG[65],*MU[65],*MD[65]; const float *MGS[65],*MUS[65],*MDS[65];int xoffb[65],nrb[65];float *mxg=NULL; int *mrows=NULL; float *mrw=NULL;/* subset builder: experts with is_miss==WANTMISS (+ shared expert when TRY_SH) */#define MB_BUILD(WANTMISS, TRY_SH) do{ \- nbb=0; Rtot=0; mfmt=-1; sh_in=0; \+ nbb=0; Rtot=0; mfmt=-1; mgs=0; sh_in=0; \for(int j=0;j<nb;j++){ if(is_miss[j]!=(WANTMISS)) continue; \int eid=uniq[base+j]; ESlot *e=use[j]; int cnt=0; \for(int s=0;s<S;s++) for(int kk=0;kk<keff[s];kk++) \if(idxs[(int64_t)s*K+kk]==eid){ cnt++; break; } \if(!cnt) continue; \- if(mfmt<0) mfmt=e->g.fmt; \+ if(mfmt<0){ mfmt=e->g.fmt; mgs=e->g.gs; } \MG[nbb]=e->g.fmt==1?(const void*)e->g.q8:(const void*)e->g.q4; \MU[nbb]=e->u.fmt==1?(const void*)e->u.q8:(const void*)e->u.q4; \MD[nbb]=e->d.fmt==1?(const void*)e->d.q8:(const void*)e->d.q4; \@@ -4274,7 +4274,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int} \if(TRY_SH){ int shf = mfmt<0 ? l->sh_gate.fmt : mfmt; \if(c->n_shared==1 && sI==I && l->sh_gate.fmt==shf && l->sh_up.fmt==shf && l->sh_down.fmt==shf){ \- if(mfmt<0) mfmt=shf; \+ if(mfmt<0){ mfmt=shf; mgs=l->sh_gate.gs; } \MG[nbb]=shf==1?(const void*)l->sh_gate.q8:(const void*)l->sh_gate.q4; \MU[nbb]=shf==1?(const void*)l->sh_up.q8 :(const void*)l->sh_up.q4; \MD[nbb]=shf==1?(const void*)l->sh_down.q8:(const void*)l->sh_down.q4; \@@ -4298,7 +4298,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intif(nbb>0){double t0=now_s();if(mfmt==6) metal_stage_rot_e8(mxg,mrows,Rtot,D);- mh=coli_metal_moe_block_begin(nbb,D,I,mfmt,MG,MU,MD,MGS,MUS,MDS,mxg,xoffb,nrb,mrows,mrw);+ mh=coli_metal_moe_block_begin(nbb,D,I,mfmt,mgs,MG,MU,MD,MGS,MUS,MDS,mxg,xoffb,nrb,mrows,mrw);m->t_emm += now_s()-t0;if(mh){ cpu_res=0; mh_shared=sh_in; }} else cpu_res=0;@@ -4361,7 +4361,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intif(nbb>0){double t0=now_s();if(mfmt==6) metal_stage_rot_e8(mxg,mrows,Rtot,D);- if(coli_metal_moe_block(nbb,D,I,mfmt,MG,MU,MD,MGS,MUS,MDS,mxg,xoffb,nrb,mrows,mrw,out,S)) cpu_miss=0;+ if(coli_metal_moe_block(nbb,D,I,mfmt,mgs,MG,MU,MD,MGS,MUS,MDS,mxg,xoffb,nrb,mrows,mrw,out,S)) cpu_miss=0;m->t_emm += now_s()-t0;} else cpu_miss=0;if(mh){ double t0=now_s();@@ -5549,7 +5549,7 @@ static void layer_forward_rows(Model *m, Layer *l, int li, float *x, int S, intif(g_metal_enabled && !kvs && S<=4 && li<c->n_layers && l->sparse&& (g_absorb==1||(g_absorb<0&&S<=4)) && m->kv_start[li]==0&& D==6144 && c->n_heads==64 && c->q_lora==2048 && c->kv_lora==512- && c->qk_nope==192 && c->qk_rope==64 && c->v_head==256 && l->kv_b.fmt==2+ && c->qk_nope==192 && c->qk_rope==64 && c->v_head==256 && (l->kv_b.fmt==2||l->kv_b.fmt==4)&& c->n_experts==256 && c->topk==8 && c->n_shared==1 && c->moe_inter==2048&& metal_fused_fmt_ok(l->q_a.fmt) && metal_fused_fmt_ok(l->q_b.fmt)&& metal_fused_fmt_ok(l->kv_a.fmt) && metal_fused_fmt_ok(l->o.fmt)@@ -5568,7 +5568,7 @@ static void layer_forward_rows(Model *m, Layer *l, int li, float *x, int S, intWP_(l->q_a), l->q_a.s, l->q_a.fmt, l->q_a.gs, l->q_a_ln,WP_(l->q_b), l->q_b.s, l->q_b.fmt, l->q_b.gs,WP_(l->kv_a), l->kv_a.s, l->kv_a.fmt, l->kv_a.gs, l->kv_a_ln,- WP_(l->kv_b), l->kv_b.s, l->kv_b.fmt,+ WP_(l->kv_b), l->kv_b.s, l->kv_b.fmt, l->kv_b.gs,WP_(l->o), l->o.s, l->o.fmt, l->o.gs,WP_(l->sh_gate), l->sh_gate.s, l->sh_gate.fmt, l->sh_gate.gs,WP_(l->sh_up), l->sh_up.s, l->sh_up.fmt, l->sh_up.gs,diff --git a/c/tests/test_backend_metal.mm b/c/tests/test_backend_metal.mmindex 2c1b4f10..372e2bac 100644--- a/c/tests/test_backend_metal.mm+++ b/c/tests/test_backend_metal.mm@@ -279,42 +279,50 @@ static float deq4(const uint8_t* w,int i){ uint8_t b=w[i>>1]; int v=(i&1)?(b>>4)static size_t roundpg(size_t n){ size_t p=16384; return ((n+p-1)/p)*p; }// Validate coli_metal_moe_block against a CPU reference (gate/up/silu/down + weighted scatter-add).-static int run_moe(const std::vector<int>& nrv, const char* name) {- const int D=6144, I=2048, fmt=2; int rbG=(D+1)/2, rbD=(I+1)/2, nb=(int)nrv.size();+// qgs==0 -> fmt=2 (per-row scale). qgs>0 -> fmt=4 grouped int4: per-expert scale slab is+// [O][ng] (ng=ceil(K/qgs)) for each of gate/up (K=D) and down (K=Iinter).+static int run_moe(const std::vector<int>& nrv, int qgs, const char* name) {+ const int D=6144, I=2048; int fmt = qgs>0 ? 4 : 2;+ int rbG=(D+1)/2, rbD=(I+1)/2, nb=(int)nrv.size();+ int ngG = qgs>0 ? (D+qgs-1)/qgs : 1, ngD = qgs>0 ? (I+qgs-1)/qgs : 1; // scales/row for gate-up / downint R=0; std::vector<int> xoff(nb),nr(nrv); for(int e=0;e<nb;e++){ xoff[e]=R; R+=nrv[e]; }- srand(2024+nb);+ srand(2024+nb+qgs);+ // per-column scale accessor: grouped picks s[o*ng + k/qgs], per-row picks s[o].+ auto scaG=[&](const float* s,int o,int k){ return qgs>0 ? s[(size_t)o*ngG + k/qgs] : s[o]; };+ auto scaD=[&](const float* s,int o,int k){ return qgs>0 ? s[(size_t)o*ngD + k/qgs] : s[o]; };// per-expert page-aligned slab [Wg|Wu|Wd] and fslab [Sg|Su|Sd]; register both.std::vector<void*> slab(nb), fslab(nb);std::vector<const void*> g(nb),u(nb),d(nb); std::vector<const float*> gs(nb),us(nb),ds(nb);- size_t wlen=roundpg((size_t)I*rbG*2 + (size_t)D*rbD), flen=roundpg(((size_t)I*2+D)*sizeof(float));+ size_t nsc=(size_t)I*ngG*2 + (size_t)D*ngD; // gate + up + down scale counts+ size_t wlen=roundpg((size_t)I*rbG*2 + (size_t)D*rbD), flen=roundpg(nsc*sizeof(float));for(int e=0;e<nb;e++){posix_memalign(&slab[e],16384,wlen); posix_memalign(&fslab[e],16384,flen);uint8_t* sp=(uint8_t*)slab[e]; for(size_t i=0;i<(size_t)I*rbG*2+(size_t)D*rbD;i++) sp[i]=(uint8_t)(rand()&0xFF);- float* fp=(float*)fslab[e]; for(size_t i=0;i<(size_t)I*2+D;i++) fp[i]=0.01f+(rand()%50)/50000.f;+ float* fp=(float*)fslab[e]; for(size_t i=0;i<nsc;i++) fp[i]=0.01f+(rand()%50)/50000.f;g[e]=sp; u[e]=sp+(size_t)I*rbG; d[e]=sp+(size_t)I*rbG*2;- gs[e]=fp; us[e]=fp+I; ds[e]=fp+2*I;+ gs[e]=fp; us[e]=fp+(size_t)I*ngG; ds[e]=fp+(size_t)I*ngG*2;coli_metal_register(slab[e],wlen); coli_metal_register(fslab[e],flen);}std::vector<float> xg((size_t)R*D); for(auto&v:xg) v=((rand()%2000)-1000)/1000.f;std::vector<int> rows(R); std::vector<float> rw(R);for(int gr=0;gr<R;gr++){ rows[gr]=0; rw[gr]=0.1f+(rand()%100)/100.f; } // decode: all -> position 0int S=1;- // CPU reference+ // CPU reference (grouped scale folded per-term; for fmt=2 that reduces to a*s[o])std::vector<float> refout((size_t)S*D,0.f), gg(I),uu(I),hh(D);for(int e=0;e<nb;e++) for(int r=0;r<nr[e];r++){ int gr=xoff[e]+r; const float* xr=&xg[(size_t)gr*D];const uint8_t* wg=(const uint8_t*)g[e]; const uint8_t* wu=(const uint8_t*)u[e]; const uint8_t* wd=(const uint8_t*)d[e];- for(int o=0;o<I;o++){ float a=0; for(int k=0;k<D;k++) a+=deq4(wg+(size_t)o*rbG,k)*xr[k]; gg[o]=a*gs[e][o]; }- for(int o=0;o<I;o++){ float a=0; for(int k=0;k<D;k++) a+=deq4(wu+(size_t)o*rbG,k)*xr[k]; uu[o]=a*us[e][o]; }+ for(int o=0;o<I;o++){ float a=0; for(int k=0;k<D;k++) a+=deq4(wg+(size_t)o*rbG,k)*xr[k]*scaG(gs[e],o,k); gg[o]=a; }+ for(int o=0;o<I;o++){ float a=0; for(int k=0;k<D;k++) a+=deq4(wu+(size_t)o*rbG,k)*xr[k]*scaG(us[e],o,k); uu[o]=a; }for(int o=0;o<I;o++){ float v=gg[o]; gg[o]=(v/(1.f+expf(-v)))*uu[o]; }- for(int o=0;o<D;o++){ float a=0; for(int k=0;k<I;k++) a+=deq4(wd+(size_t)o*rbD,k)*gg[k]; hh[o]=a*ds[e][o]; }+ for(int o=0;o<D;o++){ float a=0; for(int k=0;k<I;k++) a+=deq4(wd+(size_t)o*rbD,k)*gg[k]*scaD(ds[e],o,k); hh[o]=a; }float* os=&refout[(size_t)rows[gr]*D]; for(int o=0;o<D;o++) os[o]+=rw[gr]*hh[o];}std::vector<float> gout((size_t)S*D,0.f);- int ok = coli_metal_moe_block(nb,D,I,fmt,g.data(),u.data(),d.data(),gs.data(),us.data(),ds.data(),+ int ok = coli_metal_moe_block(nb,D,I,fmt,qgs,g.data(),u.data(),d.data(),gs.data(),us.data(),ds.data(),xg.data(),xoff.data(),nr.data(),rows.data(),rw.data(),gout.data(),S);double maxabs=0,ymax=0; for(size_t i=0;i<gout.size();i++){ maxabs=fmax(maxabs,fabs(gout[i]-refout[i])); ymax=fmax(ymax,fabs(refout[i])); }double nerr=maxabs/(ymax+1e-9); int pass = ok && nerr<1e-4;- printf(" %-22s R=%d nerr=%.2e %s\n", name, R, nerr, pass?"ok":"*** MISMATCH");+ printf(" %-30s R=%d nerr=%.2e %s\n", name, R, nerr, pass?"ok":"*** MISMATCH");for(int e=0;e<nb;e++){ coli_metal_unregister(slab[e]); coli_metal_unregister(fslab[e]); free(slab[e]); free(fslab[e]); }return pass?0:1;}@@ -408,10 +416,17 @@ static void t_gemv4g(float*y,const float*x,const uint8_t*w,const float*sc,int O,for(int o=0;o<O;o++){ const uint8_t*r=w+(size_t)o*rb; const float* scl=sc+(size_t)o*ng; float a=0;for(int i=0;i<I;i++){ uint8_t b=r[i>>1]; int v=(i&1)?(b>>4):(b&0xF); a+=(float)(v-8)*x[i]*scl[i/gs]; }y[o]=a; } }-static int run_attn(int S, int pos_base, const char* name){+// kvb_gs==0 -> kv_b as fmt=2 (per-row scale); kvb_gs>0 -> kv_b as fmt=4 grouped int4+// (exercises a_deqrow's grouped-scale path in a_qabs/a_ctx, #587's kv_b addition).+static int run_attn(int S, int pos_base, int kvb_gs, const char* name){const float eps=1e-5f, theta=10000.f, ascale=1.f/16.f;srand(4242+S+pos_base);- TW qa=t_mkw(TQL,TH), qb=t_mkw(THH*TQH,TQL), kva=t_mkw(TKVL+TROPE,TH), kvb=t_mkw(THH*TROWSH,TKVL), o=t_mkw(TH,THH*TVH);+ int kvb_fmt = kvb_gs>0 ? 4 : 2, kvng = kvb_gs>0 ? (TKVL+kvb_gs-1)/kvb_gs : 1;+ TW qa=t_mkw(TQL,TH), qb=t_mkw(THH*TQH,TQL), kva=t_mkw(TKVL+TROPE,TH);+ TW kvb = kvb_gs>0 ? t_mkw_g(THH*TROWSH,TKVL,kvb_gs) : t_mkw(THH*TROWSH,TKVL);+ TW o=t_mkw(TH,THH*TVH);+ // per-column kv_b scale: grouped (fmt=4) picks scale[row*ng + i/gs], else per-row.+ auto kvb_sc=[&](int row,int i)->float{ return kvb_gs>0 ? kvb.s[(size_t)row*kvng + i/kvb_gs] : kvb.s[row]; };std::vector<float> qaln(TQL), kvaln(TKVL);for(auto&v:qaln) v=0.5f+(rand()%1000)/1000.f; for(auto&v:kvaln) v=0.5f+(rand()%1000)/1000.f;int T=pos_base+S; size_t lcb=(((size_t)T*TKVL*4)+16383)&~(size_t)16383, rcb=(((size_t)T*TROPE*4)+16383)&~(size_t)16383;@@ -439,22 +454,22 @@ static int run_attn(int S, int pos_base, const char* name){for(int h=0;h<THH;h++){ int rbase=h*TROWSH;const float* qp=&Q[(size_t)s*THH*TQH+(size_t)h*TQH]; const float* qro=qp+TNOPE;std::vector<float> qabs(TKVL,0);- for(int d=0;d<TNOPE;d++){ const uint8_t*r=kvb.w+(size_t)(rbase+d)*rb; float sc=kvb.s[rbase+d];- for(int i=0;i<TKVL;i++){ uint8_t b=r[i>>1]; int v=(i&1)?(b>>4):(b&0xF); qabs[i]+=qp[d]*(float)(v-8)*sc; } }+ for(int d=0;d<TNOPE;d++){ const uint8_t*r=kvb.w+(size_t)(rbase+d)*rb;+ for(int i=0;i<TKVL;i++){ uint8_t b=r[i>>1]; int v=(i&1)?(b>>4):(b&0xF); qabs[i]+=qp[d]*(float)(v-8)*kvb_sc(rbase+d,i); } }std::vector<float> a(pos+1);for(int t=0;t<=pos;t++){ const float*Lt=&Lr[(size_t)t*TKVL]; const float*Rt=&Rr[(size_t)t*TROPE];float v=0; for(int i=0;i<TKVL;i++) v+=qabs[i]*Lt[i]; for(int d=0;d<TROPE;d++) v+=qro[d]*Rt[d]; a[t]=v*ascale; }float mx=-1e30f; for(float v:a) mx=fmaxf(mx,v); float sum=0; for(float&v:a){ v=expf(v-mx); sum+=v; } for(float&v:a) v/=sum;std::vector<float> cl(TKVL,0);for(int t=0;t<=pos;t++){ const float*Lt=&Lr[(size_t)t*TKVL]; for(int i=0;i<TKVL;i++) cl[i]+=a[t]*Lt[i]; }- for(int j=0;j<TVH;j++){ const uint8_t*r=kvb.w+(size_t)(rbase+TNOPE+j)*rb; float sc=kvb.s[rbase+TNOPE+j];- float v=0; for(int i=0;i<TKVL;i++){ uint8_t b=r[i>>1]; int vv=(i&1)?(b>>4):(b&0xF); v+=cl[i]*(float)(vv-8)*sc; }+ for(int j=0;j<TVH;j++){ const uint8_t*r=kvb.w+(size_t)(rbase+TNOPE+j)*rb;+ float v=0; for(int i=0;i<TKVL;i++){ uint8_t b=r[i>>1]; int vv=(i&1)?(b>>4):(b&0xF); v+=cl[i]*(float)(vv-8)*kvb_sc(rbase+TNOPE+j,i); }ctx[(size_t)h*TVH+j]=v; } }t_gemv4(&ref[(size_t)s*TH],ctx.data(),o.w,o.s,TH,THH*TVH);}std::vector<float> got((size_t)S*TH);int ok=coli_metal_attn_decode(x.data(), qa.w,qa.s,2,0,qaln.data(), qb.w,qb.s,2,0,- kva.w,kva.s,2,0,kvaln.data(), kvb.w,kvb.s,2, o.w,o.s,2,0,+ kva.w,kva.s,2,0,kvaln.data(), kvb.w,kvb.s,kvb_fmt,kvb_gs, o.w,o.s,2,0,Lc,Rc,S,pos_base,0,eps,theta,ascale,got.data());double ma=0,ym=0; for(size_t i=0;i<ref.size();i++){ ma=fmax(ma,fabs(got[i]-ref[i])); ym=fmax(ym,fabs(ref[i])); }// also verify the cache write-back (Lc/Rc for the new positions)@@ -537,9 +552,10 @@ static int run_rtop8(int mode, int S, int E, float topp, int normk, float rscale// plumbing (qa_gs threaded through encode_attention) is wired correctly end-to-end// through the SAME fused command buffer real decode uses (attention_rows in colibri.c),// not just the standalone coli_metal_matmul entry point run_grouped() above exercises.-// kv_b is deliberately left fmt=2: it never flows through bind_gemv (a_qabs/a_ctx-// dequantize it with a per-row-only helper) -- a fmt=4 kv_b is out of scope for this-// stage (see PR_BODY.md UNCERTAINTIES).+// kv_b is deliberately left fmt=2 HERE: this test isolates the qa_gs plumbing through+// bind_gemv specifically. kv_b never flows through bind_gemv at all (a_qabs/a_ctx+// dequantize it inline via a_deqrow, which is itself fmt/gs-aware) -- its own grouped+// (fmt=4) coverage lives in run_attn's kvb_gs>0 cases below, not here.//// Two blind spots closed here (review round 1 -- see PR_BODY.md sec 10):// (a) pos_base must be >0 for any S=1 case. At pos_base=0, T=pos_base+S=1: softmax@@ -616,7 +632,7 @@ static int run_attn_grouped(int S, int pos_base, int gs, const char* name){}std::vector<float> got((size_t)S*TH);int ok=coli_metal_attn_decode(x.data(), qa.w,qa.s,4,gs,qaln.data(), qb.w,qb.s,2,0, // <- qa fmt=4/gs- kva.w,kva.s,2,0,kvaln.data(), kvb.w,kvb.s,2, o.w,o.s,2,0,+ kva.w,kva.s,2,0,kvaln.data(), kvb.w,kvb.s,2,0, o.w,o.s,2,0,Lc,Rc,S,pos_base,0,eps,theta,ascale,got.data());double ma=0,ym=0; for(size_t i=0;i<ref.size();i++){ ma=fmax(ma,fabs(got[i]-ref[i])); ym=fmax(ym,fabs(ref[i])); }double mc=0; for(int s=0;s<S;s++){ int pos=pos_base+s;@@ -676,8 +692,11 @@ int main(void) {fail |= run_fp8_gemm_gate("fp8 GEMM entry explicitly gated off (coli_metal_gemm refuses)");fail |= run_fp8_moe_gate("fp8 MB_BUILD/moe_submit entry gated off (shared-expert fmt=8 hazard)");printf("Metal batched moe_block tests:\n");- fail |= run_moe({1,1,1,1,1,1,1,1}, "moe decode nb=8");- fail |= run_moe({3,1,4,2,1,5}, "moe ragged nb=6");+ fail |= run_moe({1,1,1,1,1,1,1,1}, 0, "moe decode nb=8");+ fail |= run_moe({3,1,4,2,1,5}, 0, "moe ragged nb=6");+ fail |= run_moe({1,1,1,1,1,1,1,1}, 128, "moe decode nb=8 fmt4-g128");+ fail |= run_moe({3,1,4,2,1,5}, 128, "moe ragged nb=6 fmt4-g128");+ fail |= run_moe({3,1,4,2,1,5}, 64, "moe ragged nb=6 fmt4-g64");printf("Metal fmt=6 (E8/IQ3) moe_block tests:\n");fail |= run_moe_e8({1,1,1,1,1,1,1,1}, "e8 decode nb=8");fail |= run_moe_e8({3,1,4,2,1,5}, "e8 ragged nb=6");@@ -720,10 +739,18 @@ int main(void) {coli_metal_unregister(W); coli_metal_unregister(Sc); free(W); free(Sc);}printf("Metal fused attention tests:\n");- fail |= run_attn(1, 0, "attn S=1 pos=0");- fail |= run_attn(1, 37, "attn S=1 pos=37");- fail |= run_attn(4, 12, "attn S=4 pos=12 (MTP)");- fail |= run_attn(3, 0, "attn S=3 pos=0");+ fail |= run_attn(1, 0, 0, "attn S=1 pos=0");+ fail |= run_attn(1, 37, 0, "attn S=1 pos=37");+ fail |= run_attn(4, 12, 0, "attn S=4 pos=12 (MTP)");+ fail |= run_attn(3, 0, 0, "attn S=3 pos=0");+ // fmt=4 grouped-int4 kv_b (g128/g64): exercises a_deqrow's grouped-scale path in+ // a_qabs/a_ctx (#587's kv_b addition). S=3/S=4 batches carry rows with T>1 (non-+ // degenerate softmax), so the qabs (NOPE-side) dequant is exercised too, not just+ // the always-live a_ctx (VH-side) dequant that a T=1 row alone would cover.+ fail |= run_attn(1, 0, 128, "attn S=1 pos=0 kvb-fmt4-g128");+ fail |= run_attn(1, 37, 128, "attn S=1 pos=37 kvb-fmt4-g128");+ fail |= run_attn(4, 12, 128, "attn S=4 pos=12 kvb-fmt4-g128 (MTP)");+ fail |= run_attn(3, 0, 64, "attn S=3 pos=0 kvb-fmt4-g64");printf("Metal fused attention tests (fmt=4 grouped q_a, proves bind_gemv gs plumbing):\n");// pos_base=37 (not 0): at T=1 softmax is identically 1.0 regardless of q_a's output,// so an S=1 pos=0 case cannot catch ANY q_a defect (review round 1, auditor -- see