Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(metal): run int4-g64 (fmt=4) models on the GPU — MoE experts + fused attention

Andrey Aristova9f31f169c2c
diff --git a/c/backend_metal.h b/c/backend_metal.h
index 2c09ba76..8ec6867f 100644
--- a/c/backend_metal.h
+++ b/c/backend_metal.h
@@ -80,7 +80,7 @@ int coli_metal_layer_decode(float *x,
const void *qa_w, const float *qa_s, int qa_fmt, int qa_gs, const float *qa_ln,
const void *qb_w, const float *qb_s, int qb_fmt, int qb_gs,
const void *kva_w, const float *kva_s, int kva_fmt, int kva_gs, const float *kva_ln,
- const void *kvb_w, const float *kvb_s, int kvb_fmt,
+ const void *kvb_w, const float *kvb_s, int kvb_fmt, int kvb_gs,
const void *o_w, const float *o_s, int o_fmt, int o_gs,
const void *shg_w, const float *shg_s, int shg_fmt, int shg_gs,
const void *shu_w, const float *shu_s, int shu_fmt, int shu_gs,
@@ -116,7 +116,7 @@ int coli_metal_attn_decode(const float *x,
const void *qa_w, const float *qa_s, int qa_fmt, int qa_gs, const float *qa_ln,
const void *qb_w, const float *qb_s, int qb_fmt, int qb_gs,
const void *kva_w, const float *kva_s, int kva_fmt, int kva_gs, const float *kva_ln,
- const void *kvb_w, const float *kvb_s, int kvb_fmt,
+ const void *kvb_w, const float *kvb_s, int kvb_fmt, int kvb_gs,
const void *o_w, const float *o_s, int o_fmt, int o_gs,
float *Lc, float *Rc, int S, int pos_base, int st0, float eps, float theta, float ascale, float *out);
@@ -137,18 +137,20 @@ int coli_metal_resset_stats(double *flush_s);
*
* D = hidden size, Iinter = moe intermediate size
* g/u/d[e] = pointers to expert e's gate/up/down quantized weights (in RAM slabs)
- * gs/us/ds[e] = pointers to expert e's per-row scales
- * fmt = quant format (shared across experts). NOTE: fmt=4 (grouped int4) is
- * NOT yet supported here -- gates to {1,2} and returns 0 (CPU fallback)
- * for fmt=4 experts, same as before this stage. Grouped-int4 gained GPU
- * support in mm_gemv (coli_metal_matmul/coli_metal_gemm/bind_gemv) only;
- * extending the batched routed-expert path is future work (see PR_BODY.md).
+ * gs/us/ds[e] = pointers to expert e's scales: per-row [O] for fmt 1/2, per-group
+ * [O,ceil(K/gs4)] for fmt=4 (K = the matmul's input dim)
+ * fmt = quant format (shared across experts): {1,2,4,6}; anything else
+ * returns 0 (CPU fallback)
+ * gs4 = fmt=4 group size along the input dim (shared across experts and the
+ * three matrices). Must be even and >=2; pass <=0 to force the CPU
+ * fallback (the caller poisons it on mixed per-expert group sizes).
+ * Ignored for other fmts.
* xg = packed activations [total_rows, D]; xoff[e] = row offset of expert e
* nr[e] = rows for expert e; rows[]/rw[] map packed rows back to out positions
* out = [S, D] accumulate target
* Returns 1 on success, 0 to signal the caller to fall back to the CPU path.
*/
-int coli_metal_moe_block(int nb, int D, int Iinter, int fmt,
+int coli_metal_moe_block(int nb, int D, int Iinter, int fmt, int gs4,
const void *const *g, const void *const *u, const void *const *d,
const float *const *gs, const float *const *us, const float *const *ds,
const float *xg, const int *xoff, const int *nr,
@@ -163,7 +165,7 @@ int coli_metal_moe_block(int nb, int D, int Iinter, int fmt,
* end returns 0 on GPU fault (caller redoes those experts on CPU).
*/
typedef struct ColiMetalMoeHandle ColiMetalMoeHandle;
-ColiMetalMoeHandle* coli_metal_moe_block_begin(int nb, int D, int Iinter, int fmt,
+ColiMetalMoeHandle* coli_metal_moe_block_begin(int nb, int D, int Iinter, int fmt, int gs4,
const void *const *g, const void *const *u, const void *const *d,
const float *const *gs, const float *const *us, const float *const *ds,
const float *xg, const int *xoff, const int *nr,
diff --git a/c/backend_metal.mm b/c/backend_metal.mm
index 484eb001..7fed3a44 100644
--- a/c/backend_metal.mm
+++ b/c/backend_metal.mm
@@ -163,6 +163,7 @@ kernel void moe_gemv(device const ulong* waddr [[buffer(0)]], device const ulong
constant int& O [[buffer(5)]], constant int& K [[buffer(6)]],
constant int& Kin [[buffer(7)]], constant int& fmt [[buffer(8)]],
constant int& NT [[buffer(9)]],
+ constant int& gsz [[buffer(10)]], // fmt==4 group size (ignored otherwise)
uint tg [[threadgroup_position_in_grid]],
uint slane [[thread_index_in_simdgroup]],
uint sgid [[simdgroup_index_in_threadgroup]]) {
@@ -179,6 +180,18 @@ kernel void moe_gemv(device const ulong* waddr [[buffer(0)]], device const ulong
float4 w1=float4(float(int(b.z&0xF)-8),float(int(b.z>>4)-8),float(int(b.w&0xF)-8),float(int(b.w>>4)-8));
acc+=dot(w0,x4[2*c])+dot(w1,x4[2*c+1]); }
for(int i=K8*8+slane;i<K;i+=32){ uchar b=w[i>>1]; int v=(i&1)?(b>>4):(b&0xF); acc+=float(v-8)*xr[i]; }
+ } else if (fmt == 4) { // int4 GROUPED: fmt=2's nibble packing, one f32
+ // scale per gsz inputs (mm_gemv fmt==4 twin --
+ // lane owns one packed byte per 64-lane stride,
+ // a group never splits a byte: gsz is even)
+ int rb=(K+1)/2, ng=(K+gsz-1)/gsz;
+ device const uchar* w=(device const uchar*)(waddr[e])+(long)o*rb;
+ device const float* scl=sc+(long)o*ng;
+ for(int i=slane*2;i<K;i+=64){ uchar b=w[i>>1];
+ int g0=i/gsz; float sc0=scl[g0];
+ acc+=float(int(b&0xF)-8)*xr[i]*sc0;
+ if(i+1<K){ int g1=(i+1)/gsz; acc+=float(int(b>>4)-8)*xr[i+1]*((g1==g0)?sc0:scl[g1]); }
+ }
} else if (fmt == 6) { // E8/IQ3: 98B per 256 weights, scales in-block
long rb=((long)(K+255)/256)*98; // host guards K%256==0 (GLM dims are)
device const uchar* w=(device const uchar*)(waddr[e])+(long)o*rb;
@@ -206,8 +219,8 @@ kernel void moe_gemv(device const ulong* waddr [[buffer(0)]], device const ulong
for(int i=K8*8+slane;i<K;i+=32) acc+=float(w[i])*xr[i];
}
acc=simd_sum(acc);
- if(slane==0) yout[row] = (fmt==6) ? acc : acc*sc[o]; // fmt=6: scales live in-block
-}
+ if(slane==0) yout[row] = (fmt==6||fmt==4) ? acc : acc*sc[o]; // fmt=6: scales live in-block;
+} // fmt=4: folded per-group above
// fmt=6 activation rotation for the GPU-resident down-projection input: one FWHT
// tile per dispatch (block-diagonal tiling and sign stream match quant.h e8_rot_rows;
@@ -267,14 +280,19 @@ kernel void a_copy(device const float* src [[buffer(0)]], constant int& off [[bu
// ---- absorption core (S query rows, per-row causal). q:[S,H*QH]; qabs/clat:[S*H,KVL];
// sc:[S*H,T]; ctx:[S*H,VH]. Query row s (abs pos PB+s) attends keys [0, PB+s]. ----
constant int A_QHH=A_H*A_QH;
-inline float a_deqrow(device const uchar* base, int row, int i, device const float* sc){
- device const uchar* w=base+(long)row*((A_KVL+1)/2); uchar b=w[i>>1]; int val=(i&1)?(b>>4):(b&0xF); return float(val-8)*sc[row]; }
+// kfmt 2: per-row scale sc[row]. kfmt 4 (grouped int4): sc[row*ceil(KVL/kgs) + i/kgs],
+// same nibble packing either way (the weight bytes are identical between the two fmts).
+inline float a_deqrow(device const uchar* base, int row, int i, device const float* sc, int kfmt, int kgs){
+ device const uchar* w=base+(long)row*((A_KVL+1)/2); uchar b=w[i>>1]; int val=(i&1)?(b>>4):(b&0xF);
+ float s = (kfmt==4) ? sc[(long)row*((A_KVL+kgs-1)/kgs) + i/kgs] : sc[row];
+ return float(val-8)*s; }
kernel void a_qabs(device const uchar* kvb [[buffer(0)]], device const float* sc [[buffer(1)]],
device const float* q [[buffer(2)]], device float* qabs [[buffer(3)]],
+ constant int& kfmt [[buffer(4)]], constant int& kgs [[buffer(5)]],
uint gid [[thread_position_in_grid]]) {
int s=gid/(A_H*A_KVL), r=gid%(A_H*A_KVL), h=r/A_KVL, i=r%A_KVL; int rbase=h*A_ROWSH;
device const float* qp=q+(long)s*A_QHH+(long)h*A_QH;
- float a=0; for(int d=0;d<A_NOPE;d++) a+=qp[d]*a_deqrow(kvb,rbase+d,i,sc); qabs[(long)(s*A_H+h)*A_KVL+i]=a;
+ float a=0; for(int d=0;d<A_NOPE;d++) a+=qp[d]*a_deqrow(kvb,rbase+d,i,sc,kfmt,kgs); qabs[(long)(s*A_H+h)*A_KVL+i]=a;
}
kernel void a_score(device const float* qabs [[buffer(0)]], device const float* Lc [[buffer(1)]],
device const float* Rc [[buffer(2)]], device const float* q [[buffer(3)]],
@@ -302,9 +320,11 @@ kernel void a_clat(device const float* sc [[buffer(0)]], device const float* Lc
for(int t=0;t<T;t++) a+=s[t]*Lc[(long)t*A_KVL+i]; clat[(long)sh*A_KVL+i]=a;
}
kernel void a_ctx(device const uchar* kvb [[buffer(0)]], device const float* sc [[buffer(1)]],
- device const float* clat [[buffer(2)]], device float* ctx [[buffer(3)]], uint gid [[thread_position_in_grid]]) {
+ device const float* clat [[buffer(2)]], device float* ctx [[buffer(3)]],
+ constant int& kfmt [[buffer(4)]], constant int& kgs [[buffer(5)]],
+ uint gid [[thread_position_in_grid]]) {
int sh=gid/A_VH, j=gid%A_VH, h=sh%A_H; int row=h*A_ROWSH+A_NOPE+j; device const float* cl=clat+(long)sh*A_KVL;
- float a=0; for(int i=0;i<A_KVL;i++) a+=cl[i]*a_deqrow(kvb,row,i,sc); ctx[(long)sh*A_VH+j]=a;
+ float a=0; for(int i=0;i<A_KVL;i++) a+=cl[i]*a_deqrow(kvb,row,i,sc,kfmt,kgs); ctx[(long)sh*A_VH+j]=a;
}
// ===== full-layer tail kernels =====
@@ -831,14 +851,14 @@ static bool bind_gemv(id<MTLComputeCommandEncoder> e, const void* w, const float
// Weight-pointer bundle for one layer's attention (+optional layer tail). All pointers
// must be inside registered allocations. *_gs: fmt=4 group size for the corresponding
-// weight (0 if that weight isn't grouped). kv_b has none: it never flows through bind_gemv
-// -- a_qabs/a_ctx dequantize it inline with a PER-ROW-only helper (a_deqrow), so a fmt=4
-// kv_b is out of scope for this stage (see PR_BODY.md UNCERTAINTIES).
+// weight (0 if that weight isn't grouped). kv_b never flows through bind_gemv --
+// a_qabs/a_ctx dequantize it inline via a_deqrow, which handles per-row (fmt=2) and
+// per-group (fmt=4, kvb_gs) scales; anything else falls back to CPU in encode_attention.
typedef struct {
const void *qa_w; const float *qa_s; int qa_fmt; int qa_gs; const float *qa_ln;
const void *qb_w; const float *qb_s; int qb_fmt; int qb_gs;
const void *kva_w; const float *kva_s; int kva_fmt; int kva_gs; const float *kva_ln;
- const void *kvb_w; const float *kvb_s; int kvb_fmt;
+ const void *kvb_w; const float *kvb_s; int kvb_fmt; int kvb_gs;
const void *o_w; const float *o_s; int o_fmt; int o_gs;
} AttnW;
@@ -848,6 +868,9 @@ static bool encode_attention(id<MTLComputeCommandEncoder> e, const AttnW *W,
id<MTLBuffer> Lb, size_t loff, id<MTLBuffer> Rb, size_t roff,
id<MTLBuffer> kvbW, size_t kvbwoff, id<MTLBuffer> kvbS, size_t kvbsoff,
int S, int pos_base, float eps, float theta, float ascale) {
+ /* a_deqrow's contract: kv_b is int4-packed with per-row (fmt=2) or per-group
+ * (fmt=4, even positive gs) scales. Anything else -> CPU fallback. */
+ if(W->kvb_fmt!=2 && !(W->kvb_fmt==4 && W->kvb_gs>=2 && !(W->kvb_gs&1))) return false;
int T=pos_base+S;
memcpy([aqaln_ contents],W->qa_ln,AQLORA*4); memcpy([akvaln_ contents],W->kva_ln,AKVL*4);
size_t Loff=loff+(size_t)pos_base*AKVL*4, Roff=roff+(size_t)pos_base*AROPE*4;
@@ -868,6 +891,7 @@ static bool encode_attention(id<MTLComputeCommandEncoder> e, const AttnW *W,
bind_gemv(e,W->qb_w,W->qb_s,W->qb_fmt,W->qb_gs,AQLORA,AHQH,aqr_,aqf_,S); rms(Lb,Loff,akvaln_,AKVL,S); rope(Rb,Roff,0,AROPE,0,1); BAR();
rope(aqf_,0,ANOPE,AHQH,AQH,AHEADS); BAR();
[e setComputePipelineState:g_a_qabs]; [e setBuffer:kvbW offset:kvbwoff atIndex:0]; [e setBuffer:kvbS offset:kvbsoff atIndex:1]; [e setBuffer:aqf_ offset:0 atIndex:2]; [e setBuffer:aqabs_ offset:0 atIndex:3];
+ [e setBytes:&W->kvb_fmt length:4 atIndex:4]; [e setBytes:&W->kvb_gs length:4 atIndex:5];
[e dispatchThreads:MTLSizeMake((size_t)S*AHEADS*AKVL,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; BAR();
[e setComputePipelineState:g_a_score]; [e setBuffer:aqabs_ offset:0 atIndex:0]; [e setBuffer:Lb offset:loff atIndex:1]; [e setBuffer:Rb offset:roff atIndex:2]; [e setBuffer:aqf_ offset:0 atIndex:3]; [e setBuffer:ascore_ offset:0 atIndex:4];
[e setBytes:&T length:4 atIndex:5]; [e setBytes:&ascale length:4 atIndex:6]; [e setBytes:&pos_base length:4 atIndex:7];
@@ -877,6 +901,7 @@ static bool encode_attention(id<MTLComputeCommandEncoder> e, const AttnW *W,
[e setComputePipelineState:g_a_clat]; [e setBuffer:ascore_ offset:0 atIndex:0]; [e setBuffer:Lb offset:loff atIndex:1]; [e setBuffer:aclat_ offset:0 atIndex:2]; [e setBytes:&T length:4 atIndex:3];
[e dispatchThreads:MTLSizeMake((size_t)S*AHEADS*AKVL,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; BAR();
[e setComputePipelineState:g_a_ctx]; [e setBuffer:kvbW offset:kvbwoff atIndex:0]; [e setBuffer:kvbS offset:kvbsoff atIndex:1]; [e setBuffer:aclat_ offset:0 atIndex:2]; [e setBuffer:actx_ offset:0 atIndex:3];
+ [e setBytes:&W->kvb_fmt length:4 atIndex:4]; [e setBytes:&W->kvb_gs length:4 atIndex:5];
[e dispatchThreads:MTLSizeMake((size_t)S*AHEADS*AVH,1,1) threadsPerThreadgroup:MTLSizeMake(256,1,1)]; BAR();
bind_gemv(e,W->o_w,W->o_s,W->o_fmt,W->o_gs,AHVH,AH,actx_,aout_,S);
return true;
@@ -899,7 +924,7 @@ extern "C" int coli_metal_attn_decode(const float* x,
const void* qa_w,const float* qa_s,int qa_fmt,int qa_gs,const float* qa_ln,
const void* qb_w,const float* qb_s,int qb_fmt,int qb_gs,
const void* kva_w,const float* kva_s,int kva_fmt,int kva_gs,const float* kva_ln,
- const void* kvb_w,const float* kvb_s,int kvb_fmt,
+ const void* kvb_w,const float* kvb_s,int kvb_fmt,int kvb_gs,
const void* o_w,const float* o_s,int o_fmt,int o_gs,
float* Lc,float* Rc,int S,int pos_base,int st0,float eps,float theta,float ascale,float* out){
if(!g_dev) return 0;
@@ -907,7 +932,7 @@ extern "C" int coli_metal_attn_decode(const float* x,
int T=pos_base+S;
@autoreleasepool {
attn_scratch_init();
- AttnW W={qa_w,qa_s,qa_fmt,qa_gs,qa_ln,qb_w,qb_s,qb_fmt,qb_gs,kva_w,kva_s,kva_fmt,kva_gs,kva_ln,kvb_w,kvb_s,kvb_fmt,o_w,o_s,o_fmt,o_gs};
+ AttnW W={qa_w,qa_s,qa_fmt,qa_gs,qa_ln,qb_w,qb_s,qb_fmt,qb_gs,kva_w,kva_s,kva_fmt,kva_gs,kva_ln,kvb_w,kvb_s,kvb_fmt,kvb_gs,o_w,o_s,o_fmt,o_gs};
id<MTLBuffer> Lb,Rb,kvbW,kvbS; size_t loff,roff,kvbwoff,kvbsoff;
if(!resolve_attn(&W,Lc,Rc,&Lb,&loff,&Rb,&roff,&kvbW,&kvbwoff,&kvbS,&kvbsoff)) return 0;
ascore_=ensure(ascore_,&ascore_cap,(size_t)S*AHEADS*T*4);
@@ -937,7 +962,7 @@ extern "C" int coli_metal_layer_decode(float *x,
const void* qa_w,const float* qa_s,int qa_fmt,int qa_gs,const float* qa_ln,
const void* qb_w,const float* qb_s,int qb_fmt,int qb_gs,
const void* kva_w,const float* kva_s,int kva_fmt,int kva_gs,const float* kva_ln,
- const void* kvb_w,const float* kvb_s,int kvb_fmt,
+ const void* kvb_w,const float* kvb_s,int kvb_fmt,int kvb_gs,
const void* o_w,const float* o_s,int o_fmt,int o_gs,
const void* shg_w,const float* shg_s,int shg_fmt,int shg_gs,
const void* shu_w,const float* shu_s,int shu_fmt,int shu_gs,
@@ -952,7 +977,7 @@ extern "C" int coli_metal_layer_decode(float *x,
int T=pos_base+S; const int SI=2048;
@autoreleasepool {
attn_scratch_init();
- AttnW W={qa_w,qa_s,qa_fmt,qa_gs,qa_ln,qb_w,qb_s,qb_fmt,qb_gs,kva_w,kva_s,kva_fmt,kva_gs,kva_ln,kvb_w,kvb_s,kvb_fmt,o_w,o_s,o_fmt,o_gs};
+ AttnW W={qa_w,qa_s,qa_fmt,qa_gs,qa_ln,qb_w,qb_s,qb_fmt,qb_gs,kva_w,kva_s,kva_fmt,kva_gs,kva_ln,kvb_w,kvb_s,kvb_fmt,kvb_gs,o_w,o_s,o_fmt,o_gs};
id<MTLBuffer> Lb,Rb,kvbW,kvbS; size_t loff,roff,kvbwoff,kvbsoff;
if(!resolve_attn(&W,Lc,Rc,&Lb,&loff,&Rb,&roff,&kvbW,&kvbwoff,&kvbS,&kvbsoff)) return 0;
uint64_t ina=0,pna=0,rwa=0,rba=0,d;
@@ -1138,12 +1163,16 @@ extern "C" size_t coli_metal_tensor_bytes(const ColiMetalTensor *t) { return t ?
// if Metal is off or any expert pointer is not in a registered slab.
// Encode + commit a MoE block (no wait). Writes hh[R,D] into hh_buf. Returns nil on
// unresolved slab / bad fmt (caller falls back to CPU).
-static id<MTLCommandBuffer> moe_submit(int nb, int D, int Iinter, int fmt,
+static id<MTLCommandBuffer> moe_submit(int nb, int D, int Iinter, int fmt, int gs4,
const void *const *g, const void *const *u, const void *const *d,
const float *const *gs, const float *const *us, const float *const *ds,
const float *xg, const int *xoff, const int *nr, int R,
id<MTLBuffer> xg_buf, id<MTLBuffer> gg_buf, id<MTLBuffer> uu_buf, id<MTLBuffer> hh_buf) {
- if (!g_dev || (fmt != 1 && fmt != 2 && fmt != 6)) return nil;
+ if (!g_dev || (fmt != 1 && fmt != 2 && fmt != 4 && fmt != 6)) return nil;
+ if (fmt == 4 && (gs4 < 2 || (gs4 & 1))) return nil; /* kernel needs an even, positive group
+ * size (a group never splits a packed
+ * byte); gs4<0 = caller-poisoned (mixed
+ * per-expert gs) -> CPU fallback */
if (fmt == 6) { /* e8 kernel assumes clean block tiling, and every FWHT tile of the
* down input (CPU tiling rule, e8_rot_rows) must fit threadgroup mem */
if ((D & 255) || (Iinter & 31)) return nil;
@@ -1196,7 +1225,7 @@ static id<MTLCommandBuffer> moe_submit(int nb, int D, int Iinter, int fmt,
[e setBuffer:wa offset:0 atIndex:0];[e setBuffer:sa offset:0 atIndex:1];[e setBuffer:berow offset:0 atIndex:2];
[e setBuffer:xin offset:0 atIndex:3];[e setBuffer:y offset:0 atIndex:4];
[e setBytes:&O length:4 atIndex:5];[e setBytes:&K length:4 atIndex:6];[e setBytes:&Kin length:4 atIndex:7];[e setBytes:&fmt length:4 atIndex:8];
- [e setBytes:&NT length:4 atIndex:9];
+ [e setBytes:&NT length:4 atIndex:9];[e setBytes:&gs4 length:4 atIndex:10];
[e dispatchThreadgroups:MTLSizeMake(((size_t)NT+3)/4,1,1) threadsPerThreadgroup:MTLSizeMake(128,1,1)]; };
gemv(bag,bsg,xg_buf,gg_buf,Iinter,D,D); // gate
gemv(bau,bsu,xg_buf,uu_buf,Iinter,D,D); // up
@@ -1248,7 +1277,7 @@ static int moe_finish(id<MTLCommandBuffer> cb, id<MTLBuffer> hh_buf, int nb, int
return 1;
}
-extern "C" int coli_metal_moe_block(int nb, int D, int Iinter, int fmt,
+extern "C" int coli_metal_moe_block(int nb, int D, int Iinter, int fmt, int gs4,
const void *const *g, const void *const *u, const void *const *d,
const float *const *gs, const float *const *us, const float *const *ds,
const float *xg, const int *xoff, const int *nr,
@@ -1261,7 +1290,7 @@ extern "C" int coli_metal_moe_block(int nb, int D, int Iinter, int fmt,
g_gg = ensure(g_gg,&g_gg_cap,(size_t)R*Iinter*4);
g_uu = ensure(g_uu,&g_uu_cap,(size_t)R*Iinter*4);
g_hh = ensure(g_hh,&g_hh_cap,(size_t)R*D*4);
- id<MTLCommandBuffer> cb = moe_submit(nb,D,Iinter,fmt,g,u,d,gs,us,ds,xg,xoff,nr,R,g_xg,g_gg,g_uu,g_hh);
+ id<MTLCommandBuffer> cb = moe_submit(nb,D,Iinter,fmt,gs4,g,u,d,gs,us,ds,xg,xoff,nr,R,g_xg,g_gg,g_uu,g_hh);
if (!cb) return 0;
return moe_finish(cb,g_hh,nb,R,D,rows,rw,out);
}
@@ -1274,7 +1303,7 @@ struct ColiMetalMoeHandle {
std::vector<int> rows; std::vector<float> rwv;
int nb, R, D;
};
-extern "C" ColiMetalMoeHandle* coli_metal_moe_block_begin(int nb, int D, int Iinter, int fmt,
+extern "C" ColiMetalMoeHandle* coli_metal_moe_block_begin(int nb, int D, int Iinter, int fmt, int gs4,
const void *const *g, const void *const *u, const void *const *d,
const float *const *gs, const float *const *us, const float *const *ds,
const float *xg, const int *xoff, const int *nr,
@@ -1286,7 +1315,7 @@ extern "C" ColiMetalMoeHandle* coli_metal_moe_block_begin(int nb, int D, int Iin
id<MTLBuffer> bgg=[g_dev newBufferWithLength:(size_t)R*Iinter*4 options:g_res_opts];
id<MTLBuffer> buu=[g_dev newBufferWithLength:(size_t)R*Iinter*4 options:g_res_opts];
id<MTLBuffer> bhh=[g_dev newBufferWithLength:(size_t)R*D*4 options:g_res_opts];
- id<MTLCommandBuffer> cb = moe_submit(nb,D,Iinter,fmt,g,u,d,gs,us,ds,xg,xoff,nr,R,bxg,bgg,buu,bhh);
+ id<MTLCommandBuffer> cb = moe_submit(nb,D,Iinter,fmt,gs4,g,u,d,gs,us,ds,xg,xoff,nr,R,bxg,bgg,buu,bhh);
if (!cb) return nullptr;
ColiMetalMoeHandle *h = new ColiMetalMoeHandle();
h->cb=cb; h->hh=bhh; h->rows.assign(rows,rows+R); h->rwv.assign(rw,rw+R);
diff --git a/c/colibri.c b/c/colibri.c
index 1f288d9b..43a0222d 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -3195,7 +3195,7 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p
* flow through the shared per-fmt shader. */
if(g_metal_enabled && !kvs && S<=4 && (g_absorb==1||(g_absorb<0&&S<=4)) && m->kv_start[layer]==0
&& D==6144 && H==64 && c->q_lora==2048 && c->kv_lora==512 && c->qk_nope==192
- && c->qk_rope==64 && vh==256 && l->kv_b.fmt==2
+ && c->qk_rope==64 && vh==256 && (l->kv_b.fmt==2||l->kv_b.fmt==4)
&& metal_fused_fmt_ok(l->q_a.fmt) && metal_fused_fmt_ok(l->q_b.fmt)
&& metal_fused_fmt_ok(l->kv_a.fmt) && metal_fused_fmt_ok(l->o.fmt)){
int sel_active = m->has_dsa && layer<c->n_layers && c->idx_type[layer] && (pos_base+S) > c->index_topk;
@@ -3211,7 +3211,7 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p
WP_(l->q_a), l->q_a.s, l->q_a.fmt, l->q_a.gs, l->q_a_ln,
WP_(l->q_b), l->q_b.s, l->q_b.fmt, l->q_b.gs,
WP_(l->kv_a), l->kv_a.s, l->kv_a.fmt, l->kv_a.gs, l->kv_a_ln,
- WP_(l->kv_b), l->kv_b.s, l->kv_b.fmt,
+ WP_(l->kv_b), l->kv_b.s, l->kv_b.fmt, l->kv_b.gs,
WP_(l->o), l->o.s, l->o.fmt, l->o.gs,
m->Lc[layer], m->Rc[layer], S, pos_base, m->kv_start[layer], c->eps, c->theta, c->attn_scale, out);
#undef WP_
@@ -4170,19 +4170,20 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
* batch's rows all share s, so this is one FWHT per token in practice. The down
* input is rotated on-GPU by moe_fwht inside moe_submit. */
int is_miss[64]={0}; ColiMetalMoeHandle *mh=NULL;
- int cpu_res=1, cpu_miss=1, mh_shared=0, nbb=0, Rtot=0, mfmt=-1, sh_in=0;
+ int cpu_res=1, cpu_miss=1, mh_shared=0, nbb=0, Rtot=0, mfmt=-1, mgs=0, sh_in=0;
const void *MG[65],*MU[65],*MD[65]; const float *MGS[65],*MUS[65],*MDS[65];
int xoffb[65],nrb[65];
float *mxg=NULL; int *mrows=NULL; float *mrw=NULL;
/* subset builder: experts with is_miss==WANTMISS (+ shared expert when TRY_SH) */
#define MB_BUILD(WANTMISS, TRY_SH) do{ \
- nbb=0; Rtot=0; mfmt=-1; sh_in=0; \
+ nbb=0; Rtot=0; mfmt=-1; mgs=0; sh_in=0; \
for(int j=0;j<nb;j++){ if(is_miss[j]!=(WANTMISS)) continue; \
int eid=uniq[base+j]; ESlot *e=use[j]; int cnt=0; \
for(int s=0;s<S;s++) for(int kk=0;kk<keff[s];kk++) \
if(idxs[(int64_t)s*K+kk]==eid){ cnt++; break; } \
if(!cnt) continue; \
- if(mfmt<0) mfmt=e->g.fmt; \
+ if(mfmt<0){ mfmt=e->g.fmt; mgs=e->g.gs; } \
+ if(mfmt==4 && (e->g.gs!=mgs||e->u.gs!=mgs||e->d.gs!=mgs)) mgs=-1; /* poison -> moe_submit refuses, whole subset stays CPU */ \
MG[nbb]=e->g.fmt==1?(const void*)e->g.q8:(const void*)e->g.q4; \
MU[nbb]=e->u.fmt==1?(const void*)e->u.q8:(const void*)e->u.q4; \
MD[nbb]=e->d.fmt==1?(const void*)e->d.q8:(const void*)e->d.q4; \
@@ -4190,8 +4191,10 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
xoffb[nbb]=Rtot; nrb[nbb]=cnt; Rtot+=cnt; nbb++; \
} \
if(TRY_SH){ int shf = mfmt<0 ? l->sh_gate.fmt : mfmt; \
- if(c->n_shared==1 && sI==I && l->sh_gate.fmt==shf && l->sh_up.fmt==shf && l->sh_down.fmt==shf){ \
- if(mfmt<0) mfmt=shf; \
+ int shg = mfmt<0 ? l->sh_gate.gs : mgs; \
+ if(c->n_shared==1 && sI==I && l->sh_gate.fmt==shf && l->sh_up.fmt==shf && l->sh_down.fmt==shf \
+ && (shf!=4 || (shg>0 && l->sh_gate.gs==shg && l->sh_up.gs==shg && l->sh_down.gs==shg))){ \
+ if(mfmt<0){ mfmt=shf; mgs=shg; } \
MG[nbb]=shf==1?(const void*)l->sh_gate.q8:(const void*)l->sh_gate.q4; \
MU[nbb]=shf==1?(const void*)l->sh_up.q8 :(const void*)l->sh_up.q4; \
MD[nbb]=shf==1?(const void*)l->sh_down.q8:(const void*)l->sh_down.q4; \
@@ -4215,7 +4218,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
if(nbb>0){
double t0=now_s();
if(mfmt==6) metal_stage_rot_e8(mxg,mrows,Rtot,D);
- mh=coli_metal_moe_block_begin(nbb,D,I,mfmt,MG,MU,MD,MGS,MUS,MDS,mxg,xoffb,nrb,mrows,mrw);
+ mh=coli_metal_moe_block_begin(nbb,D,I,mfmt,mgs,MG,MU,MD,MGS,MUS,MDS,mxg,xoffb,nrb,mrows,mrw);
m->t_emm += now_s()-t0;
if(mh){ cpu_res=0; mh_shared=sh_in; }
} else cpu_res=0;
@@ -4278,7 +4281,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int
if(nbb>0){
double t0=now_s();
if(mfmt==6) metal_stage_rot_e8(mxg,mrows,Rtot,D);
- if(coli_metal_moe_block(nbb,D,I,mfmt,MG,MU,MD,MGS,MUS,MDS,mxg,xoffb,nrb,mrows,mrw,out,S)) cpu_miss=0;
+ if(coli_metal_moe_block(nbb,D,I,mfmt,mgs,MG,MU,MD,MGS,MUS,MDS,mxg,xoffb,nrb,mrows,mrw,out,S)) cpu_miss=0;
m->t_emm += now_s()-t0;
} else cpu_miss=0;
if(mh){ double t0=now_s();
@@ -5463,7 +5466,7 @@ static void layer_forward_rows(Model *m, Layer *l, int li, float *x, int S, int
if(g_metal_enabled && !kvs && S<=4 && li<c->n_layers && l->sparse
&& (g_absorb==1||(g_absorb<0&&S<=4)) && m->kv_start[li]==0
&& D==6144 && c->n_heads==64 && c->q_lora==2048 && c->kv_lora==512
- && c->qk_nope==192 && c->qk_rope==64 && c->v_head==256 && l->kv_b.fmt==2
+ && c->qk_nope==192 && c->qk_rope==64 && c->v_head==256 && (l->kv_b.fmt==2||l->kv_b.fmt==4)
&& c->n_experts==256 && c->topk==8 && c->n_shared==1 && c->moe_inter==2048
&& metal_fused_fmt_ok(l->q_a.fmt) && metal_fused_fmt_ok(l->q_b.fmt)
&& metal_fused_fmt_ok(l->kv_a.fmt) && metal_fused_fmt_ok(l->o.fmt)
@@ -5482,7 +5485,7 @@ static void layer_forward_rows(Model *m, Layer *l, int li, float *x, int S, int
WP_(l->q_a), l->q_a.s, l->q_a.fmt, l->q_a.gs, l->q_a_ln,
WP_(l->q_b), l->q_b.s, l->q_b.fmt, l->q_b.gs,
WP_(l->kv_a), l->kv_a.s, l->kv_a.fmt, l->kv_a.gs, l->kv_a_ln,
- WP_(l->kv_b), l->kv_b.s, l->kv_b.fmt,
+ WP_(l->kv_b), l->kv_b.s, l->kv_b.fmt, l->kv_b.gs,
WP_(l->o), l->o.s, l->o.fmt, l->o.gs,
WP_(l->sh_gate), l->sh_gate.s, l->sh_gate.fmt, l->sh_gate.gs,
WP_(l->sh_up), l->sh_up.s, l->sh_up.fmt, l->sh_up.gs,
@@ -7392,6 +7395,7 @@ static void run_serve(Model *m, const char *snap){
_setmode(_fileno(stdout), _O_BINARY);
setvbuf(stdout, NULL, _IONBF, 0);
#endif
+ double t_serve0=now_s(); /* PROF: wall base for the exit-time profile_print */
char tkp[2048]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",snap);
Tok T; tok_load(&T,tkp);
int eos=tok_id_of(&T,"<|endoftext|>");
@@ -7544,6 +7548,12 @@ static void run_serve(Model *m, const char *snap){
}
free(line); free(buf);
usage_save(m);
+ /* PROF=1 only: the cumulative backend counters (METAL:/METAL-ATTN:/MIRROR:) were
+ * unreachable in serve mode — profile_print only ran on the oracle/generate exit
+ * paths, so a served session could never show GPU-vs-fallback truth. stdin has hit
+ * EOF here: the last END/STAT frame is already out, so these stdout lines can no
+ * longer interleave with protocol a client is parsing. */
+ if(g_prof) profile_print(m, now_s()-t_serve0);
#undef hist
#undef len
#undef first
diff --git a/c/tests/test_backend_metal.mm b/c/tests/test_backend_metal.mm
index 07b82613..d0a9c077 100644
--- a/c/tests/test_backend_metal.mm
+++ b/c/tests/test_backend_metal.mm
@@ -96,11 +96,13 @@ static float ref_e4m3(uint8_t b) {
: (1.0f+(float)mant*(1.0f/8.0f))*powf(2.0f,(float)exp-7.0f);
return sign ? -val : val;
}
-static int fp8_nblk(int n){ return (n+127)/128; }
+// Local to this section's self-contained reference; deliberately NOT quant.h's
+// fp8_nblk (which this TU also pulls in further down, with a different return type).
+static int ref_fp8_nblk(int n){ return (n+127)/128; }
static void cpu_ref_fp8(const uint8_t *q8, const float *bscale, const float *x,
double *y, double *mag, int S, int I, int O) {
- int nblkI = fp8_nblk(I);
+ int nblkI = ref_fp8_nblk(I);
for (int o=0;o<O;o++){
const uint8_t *w = q8 + (size_t)o*I;
const float *scl = bscale + (size_t)(o/128)*nblkI;
@@ -172,7 +174,7 @@ static int run_grouped(int O, int I, int gs, int S, int outlier, const char *nam
// the block-scale accumulation, avoiding cancellation-noise false positives on
// near-zero results).
static int run_fp8(int O, int I, int S, const char *name) {
- int nblkO=fp8_nblk(O), nblkI=fp8_nblk(I), nblk=nblkO*nblkI;
+ int nblkO=ref_fp8_nblk(O), nblkI=ref_fp8_nblk(I), nblk=nblkO*nblkI;
std::vector<uint8_t> W((size_t)O*I);
std::vector<float> scale((size_t)nblk), x((size_t)S*I), yg((size_t)S*O);
std::vector<double> yr((size_t)S*O), mag((size_t)S*O);
@@ -213,7 +215,7 @@ static int run_fp8(int O, int I, int S, const char *name) {
static int run_fp8_lut(const char *name) {
enum { O=256, I=1 };
std::vector<uint8_t> W(O*I); for (int b=0;b<O;b++) W[b]=(uint8_t)b;
- std::vector<float> scale(fp8_nblk(O)*fp8_nblk(I), 1.0f); // nblkO=2,nblkI=1 -> both blocks scale=1
+ std::vector<float> scale(ref_fp8_nblk(O)*ref_fp8_nblk(I), 1.0f); // nblkO=2,nblkI=1 -> both blocks scale=1
std::vector<float> x(I, 1.0f), yg(O);
ColiMetalTensor *t=nullptr;
if (!coli_metal_matmul(&t, yg.data(), x.data(), W.data(), scale.data(), FP8, 1, I, O, 0)) {
@@ -250,9 +252,9 @@ static int run_fp8_gemm_gate(const char *name) {
// up with no routed expert already fixing `mfmt`, it would submit the WRONG pointer
// (q4, NULL/stale for an fmt=8 tensor whose weights live in q8) tagged as fmt=8.
// This is safe ANYWAY, but only incidentally: moe_submit() (backend_metal.mm) gates
-// `fmt != 1 && fmt != 2` as its very FIRST statement, before any of g/u/d/gs/us/ds is
-// dereferenced or even resolve()'d -- so an fmt=8 submission is refused before the
-// bad pointer would ever be read, no matter what garbage MB_BUILD packed into it. This
+// on its fmt allowlist ({1,2,4,6}) as its very FIRST statement, before any of
+// g/u/d/gs/us/ds is dereferenced or even resolve()'d -- so an fmt=8 submission is
+// refused before the bad pointer would ever be read, no matter what garbage MB_BUILD packed into it. This
// test uses deliberately-invalid weight/scale pointers (never dereferenced if the gate
// holds) to prove the fence BY TEST rather than leaving it an artifact of moe_submit's
// fmt allowlist happening not to include 8 (yet) -- same discipline
@@ -263,7 +265,7 @@ static int run_fp8_moe_gate(const char *name) {
const float *gs[1] = {(const float*)bad}, *us[1] = {(const float*)bad}, *ds[1] = {(const float*)bad};
float xg[8]={0}, out[8]={0}, rw[1]={1.0f};
int xoff[1]={0}, nr[1]={1}, rows[1]={0};
- int rc = coli_metal_moe_block(1, 8, 8, FP8, g, u, d, gs, us, ds, xg, xoff, nr, rows, rw, out, 1);
+ int rc = coli_metal_moe_block(1, 8, 8, FP8, 0, g, u, d, gs, us, ds, xg, xoff, nr, rows, rw, out, 1);
int ok = (rc == 0);
printf(" %-42s rc=%d (expect 0/CPU-fallback) %s\n", name, rc, ok?"ok":"*** MISMATCH (should have refused)");
return ok?0:1;
@@ -304,7 +306,58 @@ static int run_moe(const std::vector<int>& nrv, const char* name) {
float* os=&refout[(size_t)rows[gr]*D]; for(int o=0;o<D;o++) os[o]+=rw[gr]*hh[o];
}
std::vector<float> gout((size_t)S*D,0.f);
- int ok = coli_metal_moe_block(nb,D,I,fmt,g.data(),u.data(),d.data(),gs.data(),us.data(),ds.data(),
+ int ok = coli_metal_moe_block(nb,D,I,fmt,0,g.data(),u.data(),d.data(),gs.data(),us.data(),ds.data(),
+ xg.data(),xoff.data(),nr.data(),rows.data(),rw.data(),gout.data(),S);
+ double maxabs=0,ymax=0; for(size_t i=0;i<gout.size();i++){ maxabs=fmax(maxabs,fabs(gout[i]-refout[i])); ymax=fmax(ymax,fabs(refout[i])); }
+ double nerr=maxabs/(ymax+1e-9); int pass = ok && nerr<1e-4;
+ printf(" %-22s R=%d nerr=%.2e %s\n", name, R, nerr, pass?"ok":"*** MISMATCH");
+ for(int e=0;e<nb;e++){ coli_metal_unregister(slab[e]); coli_metal_unregister(fslab[e]); free(slab[e]); free(fslab[e]); }
+ return pass?0:1;
+}
+
+// ---- fmt=4 (grouped int4) moe_block: run_moe's harness with per-group scales -------
+// Same shapes/flow as run_moe, but the scale slab holds [O,ceil(K/gs)] floats per matrix
+// (gate/up: K=D groups along hidden; down: K=I) and the reference folds scl[i/gs] into
+// the MAC instead of a post-scale -- the same semantics as matmul_i4_grouped and the
+// mm_gemv fmt==4 branch this kernel path mirrors. gs=64 matches the i4-g64 snapshots
+// (GLM-5.2-i4) that hit this path in the engine.
+static int run_moe_g4(const std::vector<int>& nrv, const char* name) {
+ const int D=6144, I=2048, fmt=4, GSZ=64;
+ int rbG=(D+1)/2, rbD=(I+1)/2, ngG=(D+GSZ-1)/GSZ, ngD=(I+GSZ-1)/GSZ, nb=(int)nrv.size();
+ int R=0; std::vector<int> xoff(nb),nr(nrv); for(int e=0;e<nb;e++){ xoff[e]=R; R+=nrv[e]; }
+ srand(4096+nb);
+ std::vector<void*> slab(nb), fslab(nb);
+ std::vector<const void*> g(nb),u(nb),d(nb); std::vector<const float*> gs(nb),us(nb),ds(nb);
+ size_t wlen=roundpg((size_t)I*rbG*2 + (size_t)D*rbD);
+ size_t nsc=(size_t)I*ngG*2 + (size_t)D*ngD, flen=roundpg(nsc*sizeof(float));
+ for(int e=0;e<nb;e++){
+ posix_memalign(&slab[e],16384,wlen); posix_memalign(&fslab[e],16384,flen);
+ uint8_t* sp=(uint8_t*)slab[e]; for(size_t i=0;i<(size_t)I*rbG*2+(size_t)D*rbD;i++) sp[i]=(uint8_t)(rand()&0xFF);
+ float* fp=(float*)fslab[e];
+ for(size_t i=0;i<nsc;i++) fp[i]=(0.001f+(rand()%500)/50000.f)*((rand()&1)?1.f:-1.f);
+ g[e]=sp; u[e]=sp+(size_t)I*rbG; d[e]=sp+(size_t)I*rbG*2;
+ gs[e]=fp; us[e]=fp+(size_t)I*ngG; ds[e]=fp+(size_t)I*ngG*2;
+ coli_metal_register(slab[e],wlen); coli_metal_register(fslab[e],flen);
+ }
+ std::vector<float> xg((size_t)R*D); for(auto&v:xg) v=((rand()%2000)-1000)/1000.f;
+ std::vector<int> rows(R); std::vector<float> rw(R);
+ for(int gr=0;gr<R;gr++){ rows[gr]=0; rw[gr]=0.1f+(rand()%100)/100.f; } // decode: all -> position 0
+ int S=1;
+ // CPU reference: per-group scale folded into the MAC, no post-scale
+ std::vector<float> refout((size_t)S*D,0.f), gg(I),uu(I),hh(D);
+ for(int e=0;e<nb;e++) for(int r=0;r<nr[e];r++){ int gr=xoff[e]+r; const float* xr=&xg[(size_t)gr*D];
+ const uint8_t* wg=(const uint8_t*)g[e]; const uint8_t* wu=(const uint8_t*)u[e]; const uint8_t* wd=(const uint8_t*)d[e];
+ for(int o=0;o<I;o++){ const float* scl=gs[e]+(size_t)o*ngG; float a=0;
+ for(int k=0;k<D;k++) a+=deq4(wg+(size_t)o*rbG,k)*xr[k]*scl[k/GSZ]; gg[o]=a; }
+ for(int o=0;o<I;o++){ const float* scl=us[e]+(size_t)o*ngG; float a=0;
+ for(int k=0;k<D;k++) a+=deq4(wu+(size_t)o*rbG,k)*xr[k]*scl[k/GSZ]; uu[o]=a; }
+ for(int o=0;o<I;o++){ float v=gg[o]; gg[o]=(v/(1.f+expf(-v)))*uu[o]; }
+ for(int o=0;o<D;o++){ const float* scl=ds[e]+(size_t)o*ngD; float a=0;
+ for(int k=0;k<I;k++) a+=deq4(wd+(size_t)o*rbD,k)*gg[k]*scl[k/GSZ]; hh[o]=a; }
+ float* os=&refout[(size_t)rows[gr]*D]; for(int o=0;o<D;o++) os[o]+=rw[gr]*hh[o];
+ }
+ std::vector<float> gout((size_t)S*D,0.f);
+ int ok = coli_metal_moe_block(nb,D,I,fmt,GSZ,g.data(),u.data(),d.data(),gs.data(),us.data(),ds.data(),
xg.data(),xoff.data(),nr.data(),rows.data(),rw.data(),gout.data(),S);
double maxabs=0,ymax=0; for(size_t i=0;i<gout.size();i++){ maxabs=fmax(maxabs,fabs(gout[i]-refout[i])); ymax=fmax(ymax,fabs(refout[i])); }
double nerr=maxabs/(ymax+1e-9); int pass = ok && nerr<1e-4;
@@ -358,7 +411,7 @@ static int run_moe_e8(const std::vector<int>& nrv, const char* name) {
std::vector<float> xg_gpu(xg);
for(int gr=0;gr<R;gr++) e8_rot_rows(&xg_gpu[(size_t)gr*D],1,D);
std::vector<float> gout((size_t)S*D,0.f);
- int ok = coli_metal_moe_block(nb,D,I,fmt,g.data(),u.data(),d.data(),gs.data(),us.data(),ds.data(),
+ int ok = coli_metal_moe_block(nb,D,I,fmt,0,g.data(),u.data(),d.data(),gs.data(),us.data(),ds.data(),
xg_gpu.data(),xoff.data(),nr.data(),rows.data(),rw.data(),gout.data(),S);
double maxabs=0,ymax=0; for(size_t i=0;i<gout.size();i++){ maxabs=fmax(maxabs,fabs(gout[i]-refout[i])); ymax=fmax(ymax,fabs(refout[i])); }
double nerr=maxabs/(ymax+1e-9); int pass = ok && nerr<1e-4;
@@ -402,10 +455,14 @@ static void t_gemv4g(float*y,const float*x,const uint8_t*w,const float*sc,int O,
for(int o=0;o<O;o++){ const uint8_t*r=w+(size_t)o*rb; const float* scl=sc+(size_t)o*ng; float a=0;
for(int i=0;i<I;i++){ uint8_t b=r[i>>1]; int v=(i&1)?(b>>4):(b&0xF); a+=(float)(v-8)*x[i]*scl[i/gs]; }
y[o]=a; } }
-static int run_attn(int S, int pos_base, const char* name){
+// kvb_g4=1 swaps kv_b to fmt=4 (grouped int4, gs=64 like the i4-g64 snapshots): proves
+// the a_qabs/a_ctx in-kernel dequant handles per-group scales, not just per-row.
+static int run_attn(int S, int pos_base, const char* name, int kvb_g4=0){
const float eps=1e-5f, theta=10000.f, ascale=1.f/16.f;
+ const int KGS=64, ng=(TKVL+KGS-1)/KGS;
srand(4242+S+pos_base);
- TW qa=t_mkw(TQL,TH), qb=t_mkw(THH*TQH,TQL), kva=t_mkw(TKVL+TROPE,TH), kvb=t_mkw(THH*TROWSH,TKVL), o=t_mkw(TH,THH*TVH);
+ TW qa=t_mkw(TQL,TH), qb=t_mkw(THH*TQH,TQL), kva=t_mkw(TKVL+TROPE,TH), o=t_mkw(TH,THH*TVH);
+ TW kvb = kvb_g4 ? t_mkw_g(THH*TROWSH,TKVL,KGS) : t_mkw(THH*TROWSH,TKVL);
std::vector<float> qaln(TQL), kvaln(TKVL);
for(auto&v:qaln) v=0.5f+(rand()%1000)/1000.f; for(auto&v:kvaln) v=0.5f+(rand()%1000)/1000.f;
int T=pos_base+S; size_t lcb=(((size_t)T*TKVL*4)+16383)&~(size_t)16383, rcb=(((size_t)T*TROPE*4)+16383)&~(size_t)16383;
@@ -433,22 +490,26 @@ static int run_attn(int S, int pos_base, const char* name){
for(int h=0;h<THH;h++){ int rbase=h*TROWSH;
const float* qp=&Q[(size_t)s*THH*TQH+(size_t)h*TQH]; const float* qro=qp+TNOPE;
std::vector<float> qabs(TKVL,0);
- for(int d=0;d<TNOPE;d++){ const uint8_t*r=kvb.w+(size_t)(rbase+d)*rb; float sc=kvb.s[rbase+d];
- for(int i=0;i<TKVL;i++){ uint8_t b=r[i>>1]; int v=(i&1)?(b>>4):(b&0xF); qabs[i]+=qp[d]*(float)(v-8)*sc; } }
+ for(int d=0;d<TNOPE;d++){ const uint8_t*r=kvb.w+(size_t)(rbase+d)*rb;
+ const float* scl=kvb.s+(kvb_g4?(size_t)(rbase+d)*ng:(size_t)(rbase+d));
+ for(int i=0;i<TKVL;i++){ uint8_t b=r[i>>1]; int v=(i&1)?(b>>4):(b&0xF);
+ qabs[i]+=qp[d]*(float)(v-8)*(kvb_g4?scl[i/KGS]:scl[0]); } }
std::vector<float> a(pos+1);
for(int t=0;t<=pos;t++){ const float*Lt=&Lr[(size_t)t*TKVL]; const float*Rt=&Rr[(size_t)t*TROPE];
float v=0; for(int i=0;i<TKVL;i++) v+=qabs[i]*Lt[i]; for(int d=0;d<TROPE;d++) v+=qro[d]*Rt[d]; a[t]=v*ascale; }
float mx=-1e30f; for(float v:a) mx=fmaxf(mx,v); float sum=0; for(float&v:a){ v=expf(v-mx); sum+=v; } for(float&v:a) v/=sum;
std::vector<float> cl(TKVL,0);
for(int t=0;t<=pos;t++){ const float*Lt=&Lr[(size_t)t*TKVL]; for(int i=0;i<TKVL;i++) cl[i]+=a[t]*Lt[i]; }
- for(int j=0;j<TVH;j++){ const uint8_t*r=kvb.w+(size_t)(rbase+TNOPE+j)*rb; float sc=kvb.s[rbase+TNOPE+j];
- float v=0; for(int i=0;i<TKVL;i++){ uint8_t b=r[i>>1]; int vv=(i&1)?(b>>4):(b&0xF); v+=cl[i]*(float)(vv-8)*sc; }
+ for(int j=0;j<TVH;j++){ const uint8_t*r=kvb.w+(size_t)(rbase+TNOPE+j)*rb;
+ const float* scl=kvb.s+(kvb_g4?(size_t)(rbase+TNOPE+j)*ng:(size_t)(rbase+TNOPE+j));
+ float v=0; for(int i=0;i<TKVL;i++){ uint8_t b=r[i>>1]; int vv=(i&1)?(b>>4):(b&0xF);
+ v+=cl[i]*(float)(vv-8)*(kvb_g4?scl[i/KGS]:scl[0]); }
ctx[(size_t)h*TVH+j]=v; } }
t_gemv4(&ref[(size_t)s*TH],ctx.data(),o.w,o.s,TH,THH*TVH);
}
std::vector<float> got((size_t)S*TH);
int ok=coli_metal_attn_decode(x.data(), qa.w,qa.s,2,0,qaln.data(), qb.w,qb.s,2,0,
- kva.w,kva.s,2,0,kvaln.data(), kvb.w,kvb.s,2, o.w,o.s,2,0,
+ kva.w,kva.s,2,0,kvaln.data(), kvb.w,kvb.s,kvb_g4?4:2,kvb_g4?KGS:0, o.w,o.s,2,0,
Lc,Rc,S,pos_base,0,eps,theta,ascale,got.data());
double ma=0,ym=0; for(size_t i=0;i<ref.size();i++){ ma=fmax(ma,fabs(got[i]-ref[i])); ym=fmax(ym,fabs(ref[i])); }
// also verify the cache write-back (Lc/Rc for the new positions)
@@ -610,7 +671,7 @@ static int run_attn_grouped(int S, int pos_base, int gs, const char* name){
}
std::vector<float> got((size_t)S*TH);
int ok=coli_metal_attn_decode(x.data(), qa.w,qa.s,4,gs,qaln.data(), qb.w,qb.s,2,0, // <- qa fmt=4/gs
- kva.w,kva.s,2,0,kvaln.data(), kvb.w,kvb.s,2, o.w,o.s,2,0,
+ kva.w,kva.s,2,0,kvaln.data(), kvb.w,kvb.s,2,0, o.w,o.s,2,0,
Lc,Rc,S,pos_base,0,eps,theta,ascale,got.data());
double ma=0,ym=0; for(size_t i=0;i<ref.size();i++){ ma=fmax(ma,fabs(got[i]-ref[i])); ym=fmax(ym,fabs(ref[i])); }
double mc=0; for(int s=0;s<S;s++){ int pos=pos_base+s;
@@ -672,6 +733,8 @@ int main(void) {
printf("Metal batched moe_block tests:\n");
fail |= run_moe({1,1,1,1,1,1,1,1}, "moe decode nb=8");
fail |= run_moe({3,1,4,2,1,5}, "moe ragged nb=6");
+ fail |= run_moe_g4({1,1,1,1,1,1,1,1}, "moe i4-g64 decode nb=8");
+ fail |= run_moe_g4({3,1,4,2,1,5}, "moe i4-g64 ragged nb=6");
printf("Metal fmt=6 (E8/IQ3) moe_block tests:\n");
fail |= run_moe_e8({1,1,1,1,1,1,1,1}, "e8 decode nb=8");
fail |= run_moe_e8({3,1,4,2,1,5}, "e8 ragged nb=6");
@@ -718,6 +781,10 @@ int main(void) {
fail |= run_attn(1, 37, "attn S=1 pos=37");
fail |= run_attn(4, 12, "attn S=4 pos=12 (MTP)");
fail |= run_attn(3, 0, "attn S=3 pos=0");
+ printf("Metal fused attention tests (fmt=4 grouped kv_b, proves a_qabs/a_ctx gs dequant):\n");
+ fail |= run_attn(1, 0, "attn kvb-g64 S=1 pos=0", 1);
+ fail |= run_attn(1, 37, "attn kvb-g64 S=1 pos=37", 1);
+ fail |= run_attn(4, 12, "attn kvb-g64 S=4 (MTP)", 1);
printf("Metal fused attention tests (fmt=4 grouped q_a, proves bind_gemv gs plumbing):\n");
// pos_base=37 (not 0): at T=1 softmax is identically 1.0 regardless of q_a's output,
// so an S=1 pos=0 case cannot catch ANY q_a defect (review round 1, auditor -- see