Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

metal: adapt #587's fmt=4 kv_b + moe_block coverage onto our #457 base

monotophic19d400489aef
diff --git a/c/backend_metal.h b/c/backend_metal.h
index 2c09ba76..552adf83 100644
--- a/c/backend_metal.h
+++ b/c/backend_metal.h
@@ -80,7 +80,7 @@ int coli_metal_layer_decode(float *x,
const void *qa_w, const float *qa_s, int qa_fmt, int qa_gs, const float *qa_ln,
const void *qb_w, const float *qb_s, int qb_fmt, int qb_gs,
const void *kva_w, const float *kva_s, int kva_fmt, int kva_gs, const float *kva_ln,
- const void *kvb_w, const float *kvb_s, int kvb_fmt,
+ const void *kvb_w, const float *kvb_s, int kvb_fmt, int kvb_gs,
const void *o_w, const float *o_s, int o_fmt, int o_gs,
const void *shg_w, const float *shg_s, int shg_fmt, int shg_gs,
const void *shu_w, const float *shu_s, int shu_fmt, int shu_gs,
@@ -116,7 +116,7 @@ int coli_metal_attn_decode(const float *x,
const void *qa_w, const float *qa_s, int qa_fmt, int qa_gs, const float *qa_ln,
const void *qb_w, const float *qb_s, int qb_fmt, int qb_gs,
const void *kva_w, const float *kva_s, int kva_fmt, int kva_gs, const float *kva_ln,
- const void *kvb_w, const float *kvb_s, int kvb_fmt,
+ const void *kvb_w, const float *kvb_s, int kvb_fmt, int kvb_gs,
const void *o_w, const float *o_s, int o_fmt, int o_gs,
float *Lc, float *Rc, int S, int pos_base, int st0, float eps, float theta, float ascale, float *out);
@@ -137,18 +137,16 @@ int coli_metal_resset_stats(double *flush_s);
*
* D = hidden size, Iinter = moe intermediate size
* g/u/d[e] = pointers to expert e's gate/up/down quantized weights (in RAM slabs)
- * gs/us/ds[e] = pointers to expert e's per-row scales
- * fmt = quant format (shared across experts). NOTE: fmt=4 (grouped int4) is
- * NOT yet supported here -- gates to {1,2} and returns 0 (CPU fallback)
- * for fmt=4 experts, same as before this stage. Grouped-int4 gained GPU
- * support in mm_gemv (coli_metal_matmul/coli_metal_gemm/bind_gemv) only;
- * extending the batched routed-expert path is future work (see PR_BODY.md).
+ * gs/us/ds[e] = pointers to expert e's per-row (fmt=1/2) or per-group (fmt=4) scales
+ * fmt = quant format (shared across experts): 1=int8, 2=int4 per-row, 4=int4
+ * grouped. qgs is the fmt=4 group size (ignored, pass 0, for fmt!=4).
+ * qgs = fmt=4 group size shared across experts in this block (0 for fmt!=4)
* xg = packed activations [total_rows, D]; xoff[e] = row offset of expert e
* nr[e] = rows for expert e; rows[]/rw[] map packed rows back to out positions
* out = [S, D] accumulate target
* Returns 1 on success, 0 to signal the caller to fall back to the CPU path.
*/
-int coli_metal_moe_block(int nb, int D, int Iinter, int fmt,
+int coli_metal_moe_block(int nb, int D, int Iinter, int fmt, int qgs,
const void *const *g, const void *const *u, const void *const *d,
const float *const *gs, const float *const *us, const float *const *ds,
const float *xg, const int *xoff, const int *nr,
@@ -163,7 +161,7 @@ int coli_metal_moe_block(int nb, int D, int Iinter, int fmt,
* end returns 0 on GPU fault (caller redoes those experts on CPU).
*/
typedef struct ColiMetalMoeHandle ColiMetalMoeHandle;
-ColiMetalMoeHandle* coli_metal_moe_block_begin(int nb, int D, int Iinter, int fmt,
+ColiMetalMoeHandle* coli_metal_moe_block_begin(int nb, int D, int Iinter, int fmt, int qgs,
const void *const *g, const void *const *u, const void *const *d,
const float *const *gs, const float *const *us, const float *const *ds,
const float *xg, const int *xoff, const int *nr,
diff --git a/c/backend_metal.mm b/c/backend_metal.mm
index 545e0350..14e1eec5 100644
--- a/c/backend_metal.mm
+++ b/c/backend_metal.mm
@@ -154,7 +154,10 @@ kernel void mm_gemv(device const uchar* w [[buffer(0)]], // raw weight by
}
// Batched bindless expert GEMV: each row gr belongs to expert erow[gr], whose weight and
-// scale live at gpuAddresses waddr[e]/saddr[e] (zero-copy in the RAM slab). fmt 1=i8, 2=i4.
+// scale live at gpuAddresses waddr[e]/saddr[e] (zero-copy in the RAM slab). fmt 1=i8, 2=i4
+// per-row, 4=i4 grouped (scale layout [O][ng], ng=ceil(K/qgs) -- same convention as mm_gemv
+// fmt=4 above, but folded into the vectorized uchar4/float4 dot-product loop this kernel's
+// fmt=2 branch already uses, since moe_gemv has no scalar strided branch to reuse).
// One SIMDGROUP per output row, 4 rows/threadgroup, 8-value loads: measured 1.5-2.1x over
// one-threadgroup-per-row with uchar2 loads (358-389 GB/s on engine-like block shapes).
kernel void moe_gemv(device const ulong* waddr [[buffer(0)]], device const ulong* saddr [[buffer(1)]],
@@ -162,7 +165,7 @@ kernel void moe_gemv(device const ulong* waddr [[buffer(0)]], device const ulong
device float* yout [[buffer(4)]],
constant int& O [[buffer(5)]], constant int& K [[buffer(6)]],
constant int& Kin [[buffer(7)]], constant int& fmt [[buffer(8)]],
- constant int& NT [[buffer(9)]],
+ constant int& NT [[buffer(9)]], constant int& qgs [[buffer(10)]],
uint tg [[threadgroup_position_in_grid]],
uint slane [[thread_index_in_simdgroup]],
uint sgid [[simdgroup_index_in_threadgroup]]) {
@@ -210,13 +213,25 @@ kernel void moe_gemv(device const ulong* waddr [[buffer(0)]], device const ulong
as_type<float>(uint(b.z)<<16), as_type<float>(uint(b.w)<<16));
acc+=dot(w0,x4[2*c])+dot(w1,x4[2*c+1]); }
for(int i=K8*8+slane;i<K;i+=32) acc += as_type<float>(uint(w[i])<<16)*xr[i];
+ } else if (fmt == 4) { // grouped int4: per-expert scale [O][ng]
+ int rb=(K+1)/2, ng=(K+qgs-1)/qgs; device const uchar* w=(device const uchar*)(waddr[e])+(long)o*rb;
+ device const float* sr=sc+(long)o*ng; // grouped scales for this output row
+ device const uchar4* w4=(device const uchar4*)w;
+ for(int c=slane;c<K8;c+=32){ uchar4 b=w4[c];
+ float4 w0=float4(float(int(b.x&0xF)-8),float(int(b.x>>4)-8),float(int(b.y&0xF)-8),float(int(b.y>>4)-8));
+ float4 w1=float4(float(int(b.z&0xF)-8),float(int(b.z>>4)-8),float(int(b.w&0xF)-8),float(int(b.w>>4)-8));
+ int g0=(8*c+0)/qgs,g1=(8*c+1)/qgs,g2=(8*c+2)/qgs,g3=(8*c+3)/qgs;
+ int g4=(8*c+4)/qgs,g5=(8*c+5)/qgs,g6=(8*c+6)/qgs,g7=(8*c+7)/qgs;
+ acc+=dot(w0*float4(sr[g0],sr[g1],sr[g2],sr[g3]),x4[2*c])
+ +dot(w1*float4(sr[g4],sr[g5],sr[g6],sr[g7]),x4[2*c+1]); }
+ for(int i=K8*8+slane;i<K;i+=32){ uchar b=w[i>>1]; int v=(i&1)?(b>>4):(b&0xF); acc+=float(v-8)*xr[i]*sr[i/qgs]; }
} else { device const char* w=(device const char*)(waddr[e])+(long)o*K;
device const char4* w4=(device const char4*)w;
for(int c=slane;c<K8;c+=32) acc+=dot(float4(w4[2*c]),x4[2*c])+dot(float4(w4[2*c+1]),x4[2*c+1]);
for(int i=K8*8+slane;i<K;i+=32) acc+=float(w[i])*xr[i];
}
acc=simd_sum(acc);
- if(slane==0) yout[row] = (fmt==6||fmt==5) ? acc : acc*sc[o]; // fmt 6: in-block scales; fmt 5: none
+ if(slane==0) yout[row] = (fmt==4 || fmt==5 || fmt==6) ? acc : acc*sc[o]; // fmt4 grouped / fmt6 in-block scales folded; fmt5 none
}
// fmt=6 activation rotation for the GPU-resident down-projection input: one FWHT
@@ -277,14 +292,20 @@ kernel void a_copy(device const float* src [[buffer(0)]], constant int& off [[bu
// ---- absorption core (S query rows, per-row causal). q:[S,H*QH]; qabs/clat:[S*H,KVL];
// sc:[S*H,T]; ctx:[S*H,VH]. Query row s (abs pos PB+s) attends keys [0, PB+s]. ----
constant int A_QHH=A_H*A_QH;
-inline float a_deqrow(device const uchar* base, int row, int i, device const float* sc){
- device const uchar* w=base+(long)row*((A_KVL+1)/2); uchar b=w[i>>1]; int val=(i&1)?(b>>4):(b&0xF); return float(val-8)*sc[row]; }
+// kv_b inline dequant of column i of output row `row`. fmt=2 -> one scale per row;
+// fmt=4 -> grouped int4, one scale per gs-wide group along the A_KVL input dim
+// (scale layout [O][ng], ng=ceil(A_KVL/gs)), matching QT fmt=4 / mm_gemv above.
+inline float a_deqrow(device const uchar* base, int row, int i, device const float* sc, int fmt, int gs){
+ device const uchar* w=base+(long)row*((A_KVL+1)/2); uchar b=w[i>>1]; int val=(i&1)?(b>>4):(b&0xF);
+ float s = (fmt==4) ? sc[(long)row*((A_KVL+gs-1)/gs) + i/gs] : sc[row];
+ return float(val-8)*s; }
kernel void a_qabs(device const uchar* kvb [[buffer(0)]], device const float* sc [[buffer(1)]],
device const float* q [[buffer(2)]], device float* qabs [[buffer(3)]],
+ constant int& fmt [[buffer(4)]], constant int& gs [[buffer(5)]],
uint gid [[thread_position_in_grid]]) {
int s=gid/(A_H*A_KVL), r=gid%(A_H*A_KVL), h=r/A_KVL, i=r%A_KVL; int rbase=h*A_ROWSH;
device const float* qp=q+(long)s*A_QHH+(long)h*A_QH;
- float a=0; for(int d=0;d<A_NOPE;d++) a+=qp[d]*a_deqrow(kvb,rbase+d,i,sc); qabs[(long)(s*A_H+h)*A_KVL+i]=a;
+ float a=0; for(int d=0;d<A_NOPE;d++) a+=qp[d]*a_deqrow(kvb,rbase+d,i,sc,fmt,gs); qabs[(long)(s*A_H+h)*A_KVL+i]=a;
}
kernel void a_score(device const float* qabs [[buffer(0)]], device const float* Lc [[buffer(1)]],
device const float* Rc [[buffer(2)]], device const float* q [[buffer(3)]],
@@ -312,9 +333,11 @@ kernel void a_clat(device const float* sc [[buffer(0)]], device const float* Lc
for(int t=0;t<T;t++) a+=s[t]*Lc[(long)t*A_KVL+i]; clat[(long)sh*A_KVL+i]=a;
}
kernel void a_ctx(device const uchar* kvb [[buffer(0)]], device const float* sc [[buffer(1)]],
- device const float* clat [[buffer(2)]], device float* ctx [[buffer(3)]], uint gid [[thread_position_in_grid]]) {
+ device const float* clat [[buffer(2)]], device float* ctx [[buffer(3)]],
+ constant int& fmt [[buffer(4)]], constant int& gs [[buffer(5)]],
+ uint gid [[thread_position_in_grid]]) {
int sh=gid/A_VH, j=gid%A_VH, h=sh%A_H; int row=h*A_ROWSH+A_NOPE+j; device const float* cl=clat+(long)sh*A_KVL;
- float a=0; for(int i=0;i<A_KVL;i++) a+=cl[i]*a_deqrow(kvb,row,i,sc); ctx[(long)sh*A_VH+j]=a;
+ float a=0; for(int i=0;i<A_KVL;i++) a+=cl[i]*a_deqrow(kvb,row,i,sc,fmt,gs); ctx[(long)sh*A_VH+j]=a;
}
// ===== full-layer tail kernels =====
@@ -853,14 +876,14 @@ static bool bind_gemv(id<MTLComputeCommandEncoder> e, const void* w, const float
// Weight-pointer bundle for one layer's attention (+optional layer tail). All pointers
// must be inside registered allocations. *_gs: fmt=4 group size for the corresponding
-// weight (0 if that weight isn't grouped). kv_b has none: it never flows through bind_gemv
-// -- a_qabs/a_ctx dequantize it inline with a PER-ROW-only helper (a_deqrow), so a fmt=4
-// kv_b is out of scope for this stage (see PR_BODY.md UNCERTAINTIES).
+// weight (0 if that weight isn't grouped). kv_b never flows through bind_gemv -- a_qabs/
+// a_ctx dequantize it inline via a_deqrow, which is fmt/gs-aware (fmt=2 per-row, fmt=4
+// grouped along A_KVL); kvb_gs is that group size (0 for fmt=2).
typedef struct {
const void *qa_w; const float *qa_s; int qa_fmt; int qa_gs; const float *qa_ln;
const void *qb_w; const float *qb_s; int qb_fmt; int qb_gs;
const void *kva_w; const float *kva_s; int kva_fmt; int kva_gs; const float *kva_ln;
- const void *kvb_w; const float *kvb_s; int kvb_fmt;
+ const void *kvb_w; const float *kvb_s; int kvb_fmt; int kvb_gs;
const void *o_w; const float *o_s; int o_fmt; int o_gs;
} AttnW;
@@ -890,6 +913,7 @@ static bool encode_attn_projections(id<MTLComputeCommandEncoder> e, const AttnW
bind_gemv(e,W->qb_w,W->qb_s,W->qb_fmt,W->qb_gs,AQLORA,AHQH,aqr_,aqf_,S); rms(Lb,Loff,akvaln_,AKVL,S); rope(Rb,Roff,0,AROPE,0,1); BAR();
rope(aqf_,0,ANOPE,AHQH,AQH,AHEADS); BAR();
[e setComputePipelineState:g_a_qabs]; [e setBuffer:kvbW offset:kvbwoff atIndex:0]; [e setBuffer:kvbS offset:kvbsoff atIndex:1]; [e setBuffer:aqf_ offset:0 atIndex:2]; [e setBuffer:aqabs_ offset:0 atIndex:3];
+ [e setBytes:&W->kvb_fmt length:4 atIndex:4]; [e setBytes:&W->kvb_gs length:4 atIndex:5];
[e dispatchThreads:MTLSizeMake((size_t)S*AHEADS*AKVL,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; BAR();
return true;
}
@@ -902,7 +926,8 @@ static bool encode_attn_projections(id<MTLComputeCommandEncoder> e, const AttnW
static bool encode_attn_core_chunk(id<MTLComputeCommandEncoder> e,
id<MTLBuffer> Lb, size_t loff, id<MTLBuffer> Rb, size_t roff,
id<MTLBuffer> kvbW, size_t kvbwoff, id<MTLBuffer> kvbS, size_t kvbsoff,
- int r0, int ch, int T, int pos_base, float ascale) {
+ int r0, int ch, int T, int pos_base, float ascale,
+ int kvb_fmt, int kvb_gs) {
size_t qabs_off=(size_t)r0*AHEADS*AKVL*4, qf_off=(size_t)r0*AHQH*4, ctx_off=(size_t)r0*AHVH*4;
int PB=pos_base;
auto BAR=[&]{ [e memoryBarrierWithScope:MTLBarrierScopeBuffers]; };
@@ -915,6 +940,7 @@ static bool encode_attn_core_chunk(id<MTLComputeCommandEncoder> e,
[e setComputePipelineState:g_a_clat]; [e setBuffer:ascore_ offset:0 atIndex:0]; [e setBuffer:Lb offset:loff atIndex:1]; [e setBuffer:aclat_ offset:0 atIndex:2]; [e setBytes:&T length:4 atIndex:3];
[e dispatchThreads:MTLSizeMake((size_t)ch*AHEADS*AKVL,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; BAR();
[e setComputePipelineState:g_a_ctx]; [e setBuffer:kvbW offset:kvbwoff atIndex:0]; [e setBuffer:kvbS offset:kvbsoff atIndex:1]; [e setBuffer:aclat_ offset:0 atIndex:2]; [e setBuffer:actx_ offset:ctx_off atIndex:3];
+ [e setBytes:&kvb_fmt length:4 atIndex:4]; [e setBytes:&kvb_gs length:4 atIndex:5];
[e dispatchThreads:MTLSizeMake((size_t)ch*AHEADS*AVH,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; BAR();
return true;
}
@@ -925,7 +951,7 @@ static bool encode_attention(id<MTLComputeCommandEncoder> e, const AttnW *W,
id<MTLBuffer> kvbW, size_t kvbwoff, id<MTLBuffer> kvbS, size_t kvbsoff,
int S, int T, int pos_base, float eps, float theta, float ascale) {
if(!encode_attn_projections(e,W,Lb,loff,Rb,roff,kvbW,kvbwoff,kvbS,kvbsoff,S,pos_base,eps,theta)) return false;
- if(!encode_attn_core_chunk(e,Lb,loff,Rb,roff,kvbW,kvbwoff,kvbS,kvbsoff,0,S,T,pos_base,ascale)) return false;
+ if(!encode_attn_core_chunk(e,Lb,loff,Rb,roff,kvbW,kvbwoff,kvbS,kvbsoff,0,S,T,pos_base,ascale,W->kvb_fmt,W->kvb_gs)) return false;
bind_gemv(e,W->o_w,W->o_s,W->o_fmt,W->o_gs,AHVH,AH,actx_,aout_,S);
return true;
}
@@ -947,7 +973,7 @@ extern "C" int coli_metal_attn_decode(const float* x,
const void* qa_w,const float* qa_s,int qa_fmt,int qa_gs,const float* qa_ln,
const void* qb_w,const float* qb_s,int qb_fmt,int qb_gs,
const void* kva_w,const float* kva_s,int kva_fmt,int kva_gs,const float* kva_ln,
- const void* kvb_w,const float* kvb_s,int kvb_fmt,
+ const void* kvb_w,const float* kvb_s,int kvb_fmt,int kvb_gs,
const void* o_w,const float* o_s,int o_fmt,int o_gs,
float* Lc,float* Rc,int S,int pos_base,int st0,float eps,float theta,float ascale,float* out){
if(!g_dev) return 0;
@@ -955,7 +981,7 @@ extern "C" int coli_metal_attn_decode(const float* x,
int T=pos_base+S;
@autoreleasepool {
attn_scratch_init();
- AttnW W={qa_w,qa_s,qa_fmt,qa_gs,qa_ln,qb_w,qb_s,qb_fmt,qb_gs,kva_w,kva_s,kva_fmt,kva_gs,kva_ln,kvb_w,kvb_s,kvb_fmt,o_w,o_s,o_fmt,o_gs};
+ AttnW W={qa_w,qa_s,qa_fmt,qa_gs,qa_ln,qb_w,qb_s,qb_fmt,qb_gs,kva_w,kva_s,kva_fmt,kva_gs,kva_ln,kvb_w,kvb_s,kvb_fmt,kvb_gs,o_w,o_s,o_fmt,o_gs};
id<MTLBuffer> Lb,Rb,kvbW,kvbS; size_t loff,roff,kvbwoff,kvbsoff;
if(!resolve_attn(&W,Lc,Rc,&Lb,&loff,&Rb,&roff,&kvbW,&kvbwoff,&kvbS,&kvbsoff)) return 0;
@@ -994,7 +1020,7 @@ extern "C" int coli_metal_layer_decode(float *x,
const void* qa_w,const float* qa_s,int qa_fmt,int qa_gs,const float* qa_ln,
const void* qb_w,const float* qb_s,int qb_fmt,int qb_gs,
const void* kva_w,const float* kva_s,int kva_fmt,int kva_gs,const float* kva_ln,
- const void* kvb_w,const float* kvb_s,int kvb_fmt,
+ const void* kvb_w,const float* kvb_s,int kvb_fmt,int kvb_gs,
const void* o_w,const float* o_s,int o_fmt,int o_gs,
const void* shg_w,const float* shg_s,int shg_fmt,int shg_gs,
const void* shu_w,const float* shu_s,int shu_fmt,int shu_gs,
@@ -1009,7 +1035,7 @@ extern "C" int coli_metal_layer_decode(float *x,
int T=pos_base+S; const int SI=2048;
@autoreleasepool {
attn_scratch_init();
- AttnW W={qa_w,qa_s,qa_fmt,qa_gs,qa_ln,qb_w,qb_s,qb_fmt,qb_gs,kva_w,kva_s,kva_fmt,kva_gs,kva_ln,kvb_w,kvb_s,kvb_fmt,o_w,o_s,o_fmt,o_gs};
+ AttnW W={qa_w,qa_s,qa_fmt,qa_gs,qa_ln,qb_w,qb_s,qb_fmt,qb_gs,kva_w,kva_s,kva_fmt,kva_gs,kva_ln,kvb_w,kvb_s,kvb_fmt,kvb_gs,o_w,o_s,o_fmt,o_gs};
id<MTLBuffer> Lb,Rb,kvbW,kvbS; size_t loff,roff,kvbwoff,kvbsoff;
if(!resolve_attn(&W,Lc,Rc,&Lb,&loff,&Rb,&roff,&kvbW,&kvbwoff,&kvbS,&kvbsoff)) return 0;
uint64_t ina=0,pna=0,rwa=0,rba=0,d;
@@ -1195,12 +1221,12 @@ extern "C" size_t coli_metal_tensor_bytes(const ColiMetalTensor *t) { return t ?
// if Metal is off or any expert pointer is not in a registered slab.
// Encode + commit a MoE block (no wait). Writes hh[R,D] into hh_buf. Returns nil on
// unresolved slab / bad fmt (caller falls back to CPU).
-static id<MTLCommandBuffer> moe_submit(int nb, int D, int Iinter, int fmt,
+static id<MTLCommandBuffer> moe_submit(int nb, int D, int Iinter, int fmt, int qgs,
const void *const *g, const void *const *u, const void *const *d,
const float *const *gs, const float *const *us, const float *const *ds,
const float *xg, const int *xoff, const int *nr, int R,
id<MTLBuffer> xg_buf, id<MTLBuffer> gg_buf, id<MTLBuffer> uu_buf, id<MTLBuffer> hh_buf) {
- if (!g_dev || (fmt != 1 && fmt != 2 && fmt != 5 && fmt != 6)) return nil;
+ if (!g_dev || (fmt != 1 && fmt != 2 && fmt != 4 && fmt != 5 && fmt != 6)) return nil;
if (fmt == 6) { /* e8 kernel assumes clean block tiling, and every FWHT tile of the
* down input (CPU tiling rule, e8_rot_rows) must fit threadgroup mem */
if ((D & 255) || (Iinter & 31)) return nil;
@@ -1253,7 +1279,7 @@ static id<MTLCommandBuffer> moe_submit(int nb, int D, int Iinter, int fmt,
[e setBuffer:wa offset:0 atIndex:0];[e setBuffer:sa offset:0 atIndex:1];[e setBuffer:berow offset:0 atIndex:2];
[e setBuffer:xin offset:0 atIndex:3];[e setBuffer:y offset:0 atIndex:4];
[e setBytes:&O length:4 atIndex:5];[e setBytes:&K length:4 atIndex:6];[e setBytes:&Kin length:4 atIndex:7];[e setBytes:&fmt length:4 atIndex:8];
- [e setBytes:&NT length:4 atIndex:9];
+ [e setBytes:&NT length:4 atIndex:9];[e setBytes:&qgs length:4 atIndex:10];
[e dispatchThreadgroups:MTLSizeMake(((size_t)NT+3)/4,1,1) threadsPerThreadgroup:MTLSizeMake(128,1,1)]; };
gemv(bag,bsg,xg_buf,gg_buf,Iinter,D,D); // gate
gemv(bau,bsu,xg_buf,uu_buf,Iinter,D,D); // up
@@ -1305,7 +1331,7 @@ static int moe_finish(id<MTLCommandBuffer> cb, id<MTLBuffer> hh_buf, int nb, int
return 1;
}
-extern "C" int coli_metal_moe_block(int nb, int D, int Iinter, int fmt,
+extern "C" int coli_metal_moe_block(int nb, int D, int Iinter, int fmt, int qgs,
const void *const *g, const void *const *u, const void *const *d,
const float *const *gs, const float *const *us, const float *const *ds,
const float *xg, const int *xoff, const int *nr,
@@ -1318,7 +1344,7 @@ extern "C" int coli_metal_moe_block(int nb, int D, int Iinter, int fmt,
g_gg = ensure(g_gg,&g_gg_cap,(size_t)R*Iinter*4);
g_uu = ensure(g_uu,&g_uu_cap,(size_t)R*Iinter*4);
g_hh = ensure(g_hh,&g_hh_cap,(size_t)R*D*4);
- id<MTLCommandBuffer> cb = moe_submit(nb,D,Iinter,fmt,g,u,d,gs,us,ds,xg,xoff,nr,R,g_xg,g_gg,g_uu,g_hh);
+ id<MTLCommandBuffer> cb = moe_submit(nb,D,Iinter,fmt,qgs,g,u,d,gs,us,ds,xg,xoff,nr,R,g_xg,g_gg,g_uu,g_hh);
if (!cb) return 0;
return moe_finish(cb,g_hh,nb,R,D,rows,rw,out);
}
@@ -1331,7 +1357,7 @@ struct ColiMetalMoeHandle {
std::vector<int> rows; std::vector<float> rwv;
int nb, R, D;
};
-extern "C" ColiMetalMoeHandle* coli_metal_moe_block_begin(int nb, int D, int Iinter, int fmt,
+extern "C" ColiMetalMoeHandle* coli_metal_moe_block_begin(int nb, int D, int Iinter, int fmt, int qgs,
const void *const *g, const void *const *u, const void *const *d,
const float *const *gs, const float *const *us, const float *const *ds,
const float *xg, const int *xoff, const int *nr,
@@ -1343,7 +1369,7 @@ extern "C" ColiMetalMoeHandle* coli_metal_moe_block_begin(int nb, int D, int Iin
id<MTLBuffer> bgg=[g_dev newBufferWithLength:(size_t)R*Iinter*4 options:g_res_opts];
id<MTLBuffer> buu=[g_dev newBufferWithLength:(size_t)R*Iinter*4 options:g_res_opts];
id<MTLBuffer> bhh=[g_dev newBufferWithLength:(size_t)R*D*4 options:g_res_opts];
- id<MTLCommandBuffer> cb = moe_submit(nb,D,Iinter,fmt,g,u,d,gs,us,ds,xg,xoff,nr,R,bxg,bgg,buu,bhh);
+ id<MTLCommandBuffer> cb = moe_submit(nb,D,Iinter,fmt,qgs,g,u,d,gs,us,ds,xg,xoff,nr,R,bxg,bgg,buu,bhh);
if (!cb) return nullptr;
ColiMetalMoeHandle *h = new ColiMetalMoeHandle();
h->cb=cb; h->hh=bhh; h->rows.assign(rows,rows+R); h->rwv.assign(rw,rw+R);
diff --git a/c/colibri.c b/c/colibri.c
index 2723e6f9..1c402e15 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -3271,7 +3271,7 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p
* flow through the shared per-fmt shader. */
if(g_metal_enabled && !kvs && g_absorb!=0 && (S<=4 || g_metal_prefill) && m->kv_start[layer]==0
&& D==6144 && H==64 && c->q_lora==2048 && c->kv_lora==512 && c->qk_nope==192
- && c->qk_rope==64 && vh==256 && l->kv_b.fmt==2
+ && c->qk_rope==64 && vh==256 && (l->kv_b.fmt==2||l->kv_b.fmt==4)
&& metal_fused_fmt_ok(l->q_a.fmt) && metal_fused_fmt_ok(l->q_b.fmt)
&& metal_fused_fmt_ok(l->kv_a.fmt) && metal_fused_fmt_ok(l->o.fmt)){
int sel_active = m->has_dsa && layer<c->n_layers && c->idx_type[layer] && (pos_base+S) > c->index_topk;
@@ -3287,7 +3287,7 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p
WP_(l->q_a), l->q_a.s, l->q_a.fmt, l->q_a.gs, l->q_a_ln,
WP_(l->q_b), l->q_b.s, l->q_b.fmt, l->q_b.gs,
WP_(l->kv_a), l->kv_a.s, l->kv_a.fmt, l->kv_a.gs, l->kv_a_ln,
- WP_(l->kv_b), l->kv_b.s, l->kv_b.fmt,
+ WP_(l->kv_b), l->kv_b.s, l->kv_b.fmt, l->kv_b.gs,
WP_(l->o), l->o.s, l->o.fmt, l->o.gs,
m->Lc[layer], m->Rc[layer], S, pos_base, m->kv_start[layer], c->eps, c->theta, c->attn_scale, out);
#undef WP_
@@ -4253,19 +4253,19 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
* batch's rows all share s, so this is one FWHT per token in practice. The down
* input is rotated on-GPU by moe_fwht inside moe_submit. */
int is_miss[64]={0}; ColiMetalMoeHandle *mh=NULL;
- int cpu_res=1, cpu_miss=1, mh_shared=0, nbb=0, Rtot=0, mfmt=-1, sh_in=0;
+ int cpu_res=1, cpu_miss=1, mh_shared=0, nbb=0, Rtot=0, mfmt=-1, mgs=0, sh_in=0;
const void *MG[65],*MU[65],*MD[65]; const float *MGS[65],*MUS[65],*MDS[65];
int xoffb[65],nrb[65];
float *mxg=NULL; int *mrows=NULL; float *mrw=NULL;
/* subset builder: experts with is_miss==WANTMISS (+ shared expert when TRY_SH) */
#define MB_BUILD(WANTMISS, TRY_SH) do{ \
- nbb=0; Rtot=0; mfmt=-1; sh_in=0; \
+ nbb=0; Rtot=0; mfmt=-1; mgs=0; sh_in=0; \
for(int j=0;j<nb;j++){ if(is_miss[j]!=(WANTMISS)) continue; \
int eid=uniq[base+j]; ESlot *e=use[j]; int cnt=0; \
for(int s=0;s<S;s++) for(int kk=0;kk<keff[s];kk++) \
if(idxs[(int64_t)s*K+kk]==eid){ cnt++; break; } \
if(!cnt) continue; \
- if(mfmt<0) mfmt=e->g.fmt; \
+ if(mfmt<0){ mfmt=e->g.fmt; mgs=e->g.gs; } \
MG[nbb]=e->g.fmt==1?(const void*)e->g.q8:(const void*)e->g.q4; \
MU[nbb]=e->u.fmt==1?(const void*)e->u.q8:(const void*)e->u.q4; \
MD[nbb]=e->d.fmt==1?(const void*)e->d.q8:(const void*)e->d.q4; \
@@ -4274,7 +4274,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
} \
if(TRY_SH){ int shf = mfmt<0 ? l->sh_gate.fmt : mfmt; \
if(c->n_shared==1 && sI==I && l->sh_gate.fmt==shf && l->sh_up.fmt==shf && l->sh_down.fmt==shf){ \
- if(mfmt<0) mfmt=shf; \
+ if(mfmt<0){ mfmt=shf; mgs=l->sh_gate.gs; } \
MG[nbb]=shf==1?(const void*)l->sh_gate.q8:(const void*)l->sh_gate.q4; \
MU[nbb]=shf==1?(const void*)l->sh_up.q8 :(const void*)l->sh_up.q4; \
MD[nbb]=shf==1?(const void*)l->sh_down.q8:(const void*)l->sh_down.q4; \
@@ -4298,7 +4298,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
if(nbb>0){
double t0=now_s();
if(mfmt==6) metal_stage_rot_e8(mxg,mrows,Rtot,D);
- mh=coli_metal_moe_block_begin(nbb,D,I,mfmt,MG,MU,MD,MGS,MUS,MDS,mxg,xoffb,nrb,mrows,mrw);
+ mh=coli_metal_moe_block_begin(nbb,D,I,mfmt,mgs,MG,MU,MD,MGS,MUS,MDS,mxg,xoffb,nrb,mrows,mrw);
m->t_emm += now_s()-t0;
if(mh){ cpu_res=0; mh_shared=sh_in; }
} else cpu_res=0;
@@ -4361,7 +4361,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
if(nbb>0){
double t0=now_s();
if(mfmt==6) metal_stage_rot_e8(mxg,mrows,Rtot,D);
- if(coli_metal_moe_block(nbb,D,I,mfmt,MG,MU,MD,MGS,MUS,MDS,mxg,xoffb,nrb,mrows,mrw,out,S)) cpu_miss=0;
+ if(coli_metal_moe_block(nbb,D,I,mfmt,mgs,MG,MU,MD,MGS,MUS,MDS,mxg,xoffb,nrb,mrows,mrw,out,S)) cpu_miss=0;
m->t_emm += now_s()-t0;
} else cpu_miss=0;
if(mh){ double t0=now_s();
@@ -5549,7 +5549,7 @@ static void layer_forward_rows(Model *m, Layer *l, int li, float *x, int S, int
if(g_metal_enabled && !kvs && S<=4 && li<c->n_layers && l->sparse
&& (g_absorb==1||(g_absorb<0&&S<=4)) && m->kv_start[li]==0
&& D==6144 && c->n_heads==64 && c->q_lora==2048 && c->kv_lora==512
- && c->qk_nope==192 && c->qk_rope==64 && c->v_head==256 && l->kv_b.fmt==2
+ && c->qk_nope==192 && c->qk_rope==64 && c->v_head==256 && (l->kv_b.fmt==2||l->kv_b.fmt==4)
&& c->n_experts==256 && c->topk==8 && c->n_shared==1 && c->moe_inter==2048
&& metal_fused_fmt_ok(l->q_a.fmt) && metal_fused_fmt_ok(l->q_b.fmt)
&& metal_fused_fmt_ok(l->kv_a.fmt) && metal_fused_fmt_ok(l->o.fmt)
@@ -5568,7 +5568,7 @@ static void layer_forward_rows(Model *m, Layer *l, int li, float *x, int S, int
WP_(l->q_a), l->q_a.s, l->q_a.fmt, l->q_a.gs, l->q_a_ln,
WP_(l->q_b), l->q_b.s, l->q_b.fmt, l->q_b.gs,
WP_(l->kv_a), l->kv_a.s, l->kv_a.fmt, l->kv_a.gs, l->kv_a_ln,
- WP_(l->kv_b), l->kv_b.s, l->kv_b.fmt,
+ WP_(l->kv_b), l->kv_b.s, l->kv_b.fmt, l->kv_b.gs,
WP_(l->o), l->o.s, l->o.fmt, l->o.gs,
WP_(l->sh_gate), l->sh_gate.s, l->sh_gate.fmt, l->sh_gate.gs,
WP_(l->sh_up), l->sh_up.s, l->sh_up.fmt, l->sh_up.gs,
diff --git a/c/tests/test_backend_metal.mm b/c/tests/test_backend_metal.mm
index 2c1b4f10..372e2bac 100644
--- a/c/tests/test_backend_metal.mm
+++ b/c/tests/test_backend_metal.mm
@@ -279,42 +279,50 @@ static float deq4(const uint8_t* w,int i){ uint8_t b=w[i>>1]; int v=(i&1)?(b>>4)
static size_t roundpg(size_t n){ size_t p=16384; return ((n+p-1)/p)*p; }
// Validate coli_metal_moe_block against a CPU reference (gate/up/silu/down + weighted scatter-add).
-static int run_moe(const std::vector<int>& nrv, const char* name) {
- const int D=6144, I=2048, fmt=2; int rbG=(D+1)/2, rbD=(I+1)/2, nb=(int)nrv.size();
+// qgs==0 -> fmt=2 (per-row scale). qgs>0 -> fmt=4 grouped int4: per-expert scale slab is
+// [O][ng] (ng=ceil(K/qgs)) for each of gate/up (K=D) and down (K=Iinter).
+static int run_moe(const std::vector<int>& nrv, int qgs, const char* name) {
+ const int D=6144, I=2048; int fmt = qgs>0 ? 4 : 2;
+ int rbG=(D+1)/2, rbD=(I+1)/2, nb=(int)nrv.size();
+ int ngG = qgs>0 ? (D+qgs-1)/qgs : 1, ngD = qgs>0 ? (I+qgs-1)/qgs : 1; // scales/row for gate-up / down
int R=0; std::vector<int> xoff(nb),nr(nrv); for(int e=0;e<nb;e++){ xoff[e]=R; R+=nrv[e]; }
- srand(2024+nb);
+ srand(2024+nb+qgs);
+ // per-column scale accessor: grouped picks s[o*ng + k/qgs], per-row picks s[o].
+ auto scaG=[&](const float* s,int o,int k){ return qgs>0 ? s[(size_t)o*ngG + k/qgs] : s[o]; };
+ auto scaD=[&](const float* s,int o,int k){ return qgs>0 ? s[(size_t)o*ngD + k/qgs] : s[o]; };
// per-expert page-aligned slab [Wg|Wu|Wd] and fslab [Sg|Su|Sd]; register both.
std::vector<void*> slab(nb), fslab(nb);
std::vector<const void*> g(nb),u(nb),d(nb); std::vector<const float*> gs(nb),us(nb),ds(nb);
- size_t wlen=roundpg((size_t)I*rbG*2 + (size_t)D*rbD), flen=roundpg(((size_t)I*2+D)*sizeof(float));
+ size_t nsc=(size_t)I*ngG*2 + (size_t)D*ngD; // gate + up + down scale counts
+ size_t wlen=roundpg((size_t)I*rbG*2 + (size_t)D*rbD), flen=roundpg(nsc*sizeof(float));
for(int e=0;e<nb;e++){
posix_memalign(&slab[e],16384,wlen); posix_memalign(&fslab[e],16384,flen);
uint8_t* sp=(uint8_t*)slab[e]; for(size_t i=0;i<(size_t)I*rbG*2+(size_t)D*rbD;i++) sp[i]=(uint8_t)(rand()&0xFF);
- float* fp=(float*)fslab[e]; for(size_t i=0;i<(size_t)I*2+D;i++) fp[i]=0.01f+(rand()%50)/50000.f;
+ float* fp=(float*)fslab[e]; for(size_t i=0;i<nsc;i++) fp[i]=0.01f+(rand()%50)/50000.f;
g[e]=sp; u[e]=sp+(size_t)I*rbG; d[e]=sp+(size_t)I*rbG*2;
- gs[e]=fp; us[e]=fp+I; ds[e]=fp+2*I;
+ gs[e]=fp; us[e]=fp+(size_t)I*ngG; ds[e]=fp+(size_t)I*ngG*2;
coli_metal_register(slab[e],wlen); coli_metal_register(fslab[e],flen);
}
std::vector<float> xg((size_t)R*D); for(auto&v:xg) v=((rand()%2000)-1000)/1000.f;
std::vector<int> rows(R); std::vector<float> rw(R);
for(int gr=0;gr<R;gr++){ rows[gr]=0; rw[gr]=0.1f+(rand()%100)/100.f; } // decode: all -> position 0
int S=1;
- // CPU reference
+ // CPU reference (grouped scale folded per-term; for fmt=2 that reduces to a*s[o])
std::vector<float> refout((size_t)S*D,0.f), gg(I),uu(I),hh(D);
for(int e=0;e<nb;e++) for(int r=0;r<nr[e];r++){ int gr=xoff[e]+r; const float* xr=&xg[(size_t)gr*D];
const uint8_t* wg=(const uint8_t*)g[e]; const uint8_t* wu=(const uint8_t*)u[e]; const uint8_t* wd=(const uint8_t*)d[e];
- for(int o=0;o<I;o++){ float a=0; for(int k=0;k<D;k++) a+=deq4(wg+(size_t)o*rbG,k)*xr[k]; gg[o]=a*gs[e][o]; }
- for(int o=0;o<I;o++){ float a=0; for(int k=0;k<D;k++) a+=deq4(wu+(size_t)o*rbG,k)*xr[k]; uu[o]=a*us[e][o]; }
+ for(int o=0;o<I;o++){ float a=0; for(int k=0;k<D;k++) a+=deq4(wg+(size_t)o*rbG,k)*xr[k]*scaG(gs[e],o,k); gg[o]=a; }
+ for(int o=0;o<I;o++){ float a=0; for(int k=0;k<D;k++) a+=deq4(wu+(size_t)o*rbG,k)*xr[k]*scaG(us[e],o,k); uu[o]=a; }
for(int o=0;o<I;o++){ float v=gg[o]; gg[o]=(v/(1.f+expf(-v)))*uu[o]; }
- for(int o=0;o<D;o++){ float a=0; for(int k=0;k<I;k++) a+=deq4(wd+(size_t)o*rbD,k)*gg[k]; hh[o]=a*ds[e][o]; }
+ for(int o=0;o<D;o++){ float a=0; for(int k=0;k<I;k++) a+=deq4(wd+(size_t)o*rbD,k)*gg[k]*scaD(ds[e],o,k); hh[o]=a; }
float* os=&refout[(size_t)rows[gr]*D]; for(int o=0;o<D;o++) os[o]+=rw[gr]*hh[o];
}
std::vector<float> gout((size_t)S*D,0.f);
- int ok = coli_metal_moe_block(nb,D,I,fmt,g.data(),u.data(),d.data(),gs.data(),us.data(),ds.data(),
+ int ok = coli_metal_moe_block(nb,D,I,fmt,qgs,g.data(),u.data(),d.data(),gs.data(),us.data(),ds.data(),
xg.data(),xoff.data(),nr.data(),rows.data(),rw.data(),gout.data(),S);
double maxabs=0,ymax=0; for(size_t i=0;i<gout.size();i++){ maxabs=fmax(maxabs,fabs(gout[i]-refout[i])); ymax=fmax(ymax,fabs(refout[i])); }
double nerr=maxabs/(ymax+1e-9); int pass = ok && nerr<1e-4;
- printf(" %-22s R=%d nerr=%.2e %s\n", name, R, nerr, pass?"ok":"*** MISMATCH");
+ printf(" %-30s R=%d nerr=%.2e %s\n", name, R, nerr, pass?"ok":"*** MISMATCH");
for(int e=0;e<nb;e++){ coli_metal_unregister(slab[e]); coli_metal_unregister(fslab[e]); free(slab[e]); free(fslab[e]); }
return pass?0:1;
}
@@ -408,10 +416,17 @@ static void t_gemv4g(float*y,const float*x,const uint8_t*w,const float*sc,int O,
for(int o=0;o<O;o++){ const uint8_t*r=w+(size_t)o*rb; const float* scl=sc+(size_t)o*ng; float a=0;
for(int i=0;i<I;i++){ uint8_t b=r[i>>1]; int v=(i&1)?(b>>4):(b&0xF); a+=(float)(v-8)*x[i]*scl[i/gs]; }
y[o]=a; } }
-static int run_attn(int S, int pos_base, const char* name){
+// kvb_gs==0 -> kv_b as fmt=2 (per-row scale); kvb_gs>0 -> kv_b as fmt=4 grouped int4
+// (exercises a_deqrow's grouped-scale path in a_qabs/a_ctx, #587's kv_b addition).
+static int run_attn(int S, int pos_base, int kvb_gs, const char* name){
const float eps=1e-5f, theta=10000.f, ascale=1.f/16.f;
srand(4242+S+pos_base);
- TW qa=t_mkw(TQL,TH), qb=t_mkw(THH*TQH,TQL), kva=t_mkw(TKVL+TROPE,TH), kvb=t_mkw(THH*TROWSH,TKVL), o=t_mkw(TH,THH*TVH);
+ int kvb_fmt = kvb_gs>0 ? 4 : 2, kvng = kvb_gs>0 ? (TKVL+kvb_gs-1)/kvb_gs : 1;
+ TW qa=t_mkw(TQL,TH), qb=t_mkw(THH*TQH,TQL), kva=t_mkw(TKVL+TROPE,TH);
+ TW kvb = kvb_gs>0 ? t_mkw_g(THH*TROWSH,TKVL,kvb_gs) : t_mkw(THH*TROWSH,TKVL);
+ TW o=t_mkw(TH,THH*TVH);
+ // per-column kv_b scale: grouped (fmt=4) picks scale[row*ng + i/gs], else per-row.
+ auto kvb_sc=[&](int row,int i)->float{ return kvb_gs>0 ? kvb.s[(size_t)row*kvng + i/kvb_gs] : kvb.s[row]; };
std::vector<float> qaln(TQL), kvaln(TKVL);
for(auto&v:qaln) v=0.5f+(rand()%1000)/1000.f; for(auto&v:kvaln) v=0.5f+(rand()%1000)/1000.f;
int T=pos_base+S; size_t lcb=(((size_t)T*TKVL*4)+16383)&~(size_t)16383, rcb=(((size_t)T*TROPE*4)+16383)&~(size_t)16383;
@@ -439,22 +454,22 @@ static int run_attn(int S, int pos_base, const char* name){
for(int h=0;h<THH;h++){ int rbase=h*TROWSH;
const float* qp=&Q[(size_t)s*THH*TQH+(size_t)h*TQH]; const float* qro=qp+TNOPE;
std::vector<float> qabs(TKVL,0);
- for(int d=0;d<TNOPE;d++){ const uint8_t*r=kvb.w+(size_t)(rbase+d)*rb; float sc=kvb.s[rbase+d];
- for(int i=0;i<TKVL;i++){ uint8_t b=r[i>>1]; int v=(i&1)?(b>>4):(b&0xF); qabs[i]+=qp[d]*(float)(v-8)*sc; } }
+ for(int d=0;d<TNOPE;d++){ const uint8_t*r=kvb.w+(size_t)(rbase+d)*rb;
+ for(int i=0;i<TKVL;i++){ uint8_t b=r[i>>1]; int v=(i&1)?(b>>4):(b&0xF); qabs[i]+=qp[d]*(float)(v-8)*kvb_sc(rbase+d,i); } }
std::vector<float> a(pos+1);
for(int t=0;t<=pos;t++){ const float*Lt=&Lr[(size_t)t*TKVL]; const float*Rt=&Rr[(size_t)t*TROPE];
float v=0; for(int i=0;i<TKVL;i++) v+=qabs[i]*Lt[i]; for(int d=0;d<TROPE;d++) v+=qro[d]*Rt[d]; a[t]=v*ascale; }
float mx=-1e30f; for(float v:a) mx=fmaxf(mx,v); float sum=0; for(float&v:a){ v=expf(v-mx); sum+=v; } for(float&v:a) v/=sum;
std::vector<float> cl(TKVL,0);
for(int t=0;t<=pos;t++){ const float*Lt=&Lr[(size_t)t*TKVL]; for(int i=0;i<TKVL;i++) cl[i]+=a[t]*Lt[i]; }
- for(int j=0;j<TVH;j++){ const uint8_t*r=kvb.w+(size_t)(rbase+TNOPE+j)*rb; float sc=kvb.s[rbase+TNOPE+j];
- float v=0; for(int i=0;i<TKVL;i++){ uint8_t b=r[i>>1]; int vv=(i&1)?(b>>4):(b&0xF); v+=cl[i]*(float)(vv-8)*sc; }
+ for(int j=0;j<TVH;j++){ const uint8_t*r=kvb.w+(size_t)(rbase+TNOPE+j)*rb;
+ float v=0; for(int i=0;i<TKVL;i++){ uint8_t b=r[i>>1]; int vv=(i&1)?(b>>4):(b&0xF); v+=cl[i]*(float)(vv-8)*kvb_sc(rbase+TNOPE+j,i); }
ctx[(size_t)h*TVH+j]=v; } }
t_gemv4(&ref[(size_t)s*TH],ctx.data(),o.w,o.s,TH,THH*TVH);
}
std::vector<float> got((size_t)S*TH);
int ok=coli_metal_attn_decode(x.data(), qa.w,qa.s,2,0,qaln.data(), qb.w,qb.s,2,0,
- kva.w,kva.s,2,0,kvaln.data(), kvb.w,kvb.s,2, o.w,o.s,2,0,
+ kva.w,kva.s,2,0,kvaln.data(), kvb.w,kvb.s,kvb_fmt,kvb_gs, o.w,o.s,2,0,
Lc,Rc,S,pos_base,0,eps,theta,ascale,got.data());
double ma=0,ym=0; for(size_t i=0;i<ref.size();i++){ ma=fmax(ma,fabs(got[i]-ref[i])); ym=fmax(ym,fabs(ref[i])); }
// also verify the cache write-back (Lc/Rc for the new positions)
@@ -537,9 +552,10 @@ static int run_rtop8(int mode, int S, int E, float topp, int normk, float rscale
// plumbing (qa_gs threaded through encode_attention) is wired correctly end-to-end
// through the SAME fused command buffer real decode uses (attention_rows in colibri.c),
// not just the standalone coli_metal_matmul entry point run_grouped() above exercises.
-// kv_b is deliberately left fmt=2: it never flows through bind_gemv (a_qabs/a_ctx
-// dequantize it with a per-row-only helper) -- a fmt=4 kv_b is out of scope for this
-// stage (see PR_BODY.md UNCERTAINTIES).
+// kv_b is deliberately left fmt=2 HERE: this test isolates the qa_gs plumbing through
+// bind_gemv specifically. kv_b never flows through bind_gemv at all (a_qabs/a_ctx
+// dequantize it inline via a_deqrow, which is itself fmt/gs-aware) -- its own grouped
+// (fmt=4) coverage lives in run_attn's kvb_gs>0 cases below, not here.
//
// Two blind spots closed here (review round 1 -- see PR_BODY.md sec 10):
// (a) pos_base must be >0 for any S=1 case. At pos_base=0, T=pos_base+S=1: softmax
@@ -616,7 +632,7 @@ static int run_attn_grouped(int S, int pos_base, int gs, const char* name){
}
std::vector<float> got((size_t)S*TH);
int ok=coli_metal_attn_decode(x.data(), qa.w,qa.s,4,gs,qaln.data(), qb.w,qb.s,2,0, // <- qa fmt=4/gs
- kva.w,kva.s,2,0,kvaln.data(), kvb.w,kvb.s,2, o.w,o.s,2,0,
+ kva.w,kva.s,2,0,kvaln.data(), kvb.w,kvb.s,2,0, o.w,o.s,2,0,
Lc,Rc,S,pos_base,0,eps,theta,ascale,got.data());
double ma=0,ym=0; for(size_t i=0;i<ref.size();i++){ ma=fmax(ma,fabs(got[i]-ref[i])); ym=fmax(ym,fabs(ref[i])); }
double mc=0; for(int s=0;s<S;s++){ int pos=pos_base+s;
@@ -676,8 +692,11 @@ int main(void) {
fail |= run_fp8_gemm_gate("fp8 GEMM entry explicitly gated off (coli_metal_gemm refuses)");
fail |= run_fp8_moe_gate("fp8 MB_BUILD/moe_submit entry gated off (shared-expert fmt=8 hazard)");
printf("Metal batched moe_block tests:\n");
- fail |= run_moe({1,1,1,1,1,1,1,1}, "moe decode nb=8");
- fail |= run_moe({3,1,4,2,1,5}, "moe ragged nb=6");
+ fail |= run_moe({1,1,1,1,1,1,1,1}, 0, "moe decode nb=8");
+ fail |= run_moe({3,1,4,2,1,5}, 0, "moe ragged nb=6");
+ fail |= run_moe({1,1,1,1,1,1,1,1}, 128, "moe decode nb=8 fmt4-g128");
+ fail |= run_moe({3,1,4,2,1,5}, 128, "moe ragged nb=6 fmt4-g128");
+ fail |= run_moe({3,1,4,2,1,5}, 64, "moe ragged nb=6 fmt4-g64");
printf("Metal fmt=6 (E8/IQ3) moe_block tests:\n");
fail |= run_moe_e8({1,1,1,1,1,1,1,1}, "e8 decode nb=8");
fail |= run_moe_e8({3,1,4,2,1,5}, "e8 ragged nb=6");
@@ -720,10 +739,18 @@ int main(void) {
coli_metal_unregister(W); coli_metal_unregister(Sc); free(W); free(Sc);
}
printf("Metal fused attention tests:\n");
- fail |= run_attn(1, 0, "attn S=1 pos=0");
- fail |= run_attn(1, 37, "attn S=1 pos=37");
- fail |= run_attn(4, 12, "attn S=4 pos=12 (MTP)");
- fail |= run_attn(3, 0, "attn S=3 pos=0");
+ fail |= run_attn(1, 0, 0, "attn S=1 pos=0");
+ fail |= run_attn(1, 37, 0, "attn S=1 pos=37");
+ fail |= run_attn(4, 12, 0, "attn S=4 pos=12 (MTP)");
+ fail |= run_attn(3, 0, 0, "attn S=3 pos=0");
+ // fmt=4 grouped-int4 kv_b (g128/g64): exercises a_deqrow's grouped-scale path in
+ // a_qabs/a_ctx (#587's kv_b addition). S=3/S=4 batches carry rows with T>1 (non-
+ // degenerate softmax), so the qabs (NOPE-side) dequant is exercised too, not just
+ // the always-live a_ctx (VH-side) dequant that a T=1 row alone would cover.
+ fail |= run_attn(1, 0, 128, "attn S=1 pos=0 kvb-fmt4-g128");
+ fail |= run_attn(1, 37, 128, "attn S=1 pos=37 kvb-fmt4-g128");
+ fail |= run_attn(4, 12, 128, "attn S=4 pos=12 kvb-fmt4-g128 (MTP)");
+ fail |= run_attn(3, 0, 64, "attn S=3 pos=0 kvb-fmt4-g64");
printf("Metal fused attention tests (fmt=4 grouped q_a, proves bind_gemv gs plumbing):\n");
// pos_base=37 (not 0): at T=1 softmax is identically 1.0 regardless of q_a's output,
// so an S=1 pos=0 case cannot catch ANY q_a defect (review round 1, auditor -- see