Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
perf(kimi_k3): hoist expert activation quantization to layer level (bit-identical)
JustVugga7f2f64e49d4
diff --git a/c/kimi_k3.c b/c/kimi_k3.cindex 9b969198..41ea5f3e 100644--- a/c/kimi_k3.c+++ b/c/kimi_k3.c@@ -1120,9 +1120,15 @@ static int cuda_expert_apply(Model *m, const uint8_t *w1p, const uint8_t *w1s,#endif/* u += wk * E(z) for one loaded expert slot (SiTU-GLU in the latent).- * gate/up are [moe_inter] scratch, hz is [latent] scratch. */+ * gate/up are [moe_inter] scratch, hz is [latent] scratch.+ * zq/zsc: riga di z già quantizzata a livello layer da mxfp4_qx (NULL = come+ * prima, quantizza in-chiamata). Gate e up la condividono; il down riquantizza+ * comunque il proprio input per-expert (gate), oggi però in scratch, non malloc.+ * EN: zq/zsc = layer-hoisted pre-quantized z row (NULL = quantize in-call as+ * before). Gate and up share it; down still quantizes its per-expert input. */static void expert_apply(Model *m, Slot *s, const float *z, float wk,- float *u, float *gate, float *up, float *hz){+ float *u, float *gate, float *up, float *hz,+ const int8_t *zq, const float *zsc){Cfg *c=&m->c;uint8_t *w1p=s->buf, *w1s=w1p+m->e_w1p, *w2p=w1s+m->e_w1s, *w2s=w2p+m->e_w2p,*w3p=w2s+m->e_w2s, *w3s=w3p+m->e_w1p;@@ -1133,8 +1139,13 @@ static void expert_apply(Model *m, Slot *s, const float *z, float wk,#endifvoid (*mm)(float*,const float*,const uint8_t*,const uint8_t*,int,int,int)= g_k3_idot ? matmul_mxfp4_i8 : matmul_mxfp4;- mm(gate,z,w1p,w1s,1,c->latent,c->moe_inter);- mm(up,z,w3p,w3s,1,c->latent,c->moe_inter);+ if(g_k3_idot && zq){+ matmul_mxfp4_i8_pre(gate,zq,zsc,w1p,w1s,1,c->latent,c->moe_inter);+ matmul_mxfp4_i8_pre(up, zq,zsc,w3p,w3s,1,c->latent,c->moe_inter);+ } else {+ mm(gate,z,w1p,w1s,1,c->latent,c->moe_inter);+ mm(up,z,w3p,w3s,1,c->latent,c->moe_inter);+ }for(int i=0;i<c->moe_inter;i++) gate[i]=situf_(gate[i],up[i],c->situ_b1,c->situ_b2);mm(hz,gate,w2p,w2s,1,c->moe_inter,c->latent);for(int i=0;i<c->latent;i++) u[i]+=wk*hz[i];@@ -1257,6 +1268,28 @@ static void experts_apply_union(Model *m, int li, int nu, const int *uids,const int *poslist, const float *wlist,const float *Z, int stride, float *U,float *gate, float *up, float *hz){+ /* Quantizzazione sollevata a livello layer (specchia colibri.c #1071):+ * ogni riga di Z veniva riquantizzata 2x per OGNI expert che la consuma+ * (gate+up dentro matmul_mxfp4_i8, con un malloc/free a chiamata) — con+ * topk=8 sono 16 quantizzazioni della stessa riga per layer. Una sola+ * passata mxfp4_qx qui produce le stesse righe int8 (ordine identico:+ * bit-identico), e il fallimento di allocazione degrada al vecchio path.+ * EN: layer-hoisted activation quantization, mirroring colibri.c (#1071).+ * One mxfp4_qx pass replaces 2 requantizations per (expert, position);+ * allocation failure degrades to the old in-call path. */+ int8_t *zq_all=NULL; float *zsc_all=NULL; int zng=0;+ if(g_k3_idot && stride%32==0 && nu>0){+ int maxt=-1;+ for(int j=0;j<nu;j++) for(int p2=0;p2<pcnt[j];p2++)+ if(poslist[pfirst[j]+p2]>maxt) maxt=poslist[pfirst[j]+p2];+ if(maxt>=0){+ zng=stride/32;+ zq_all=malloc((size_t)(maxt+1)*stride);+ zsc_all=malloc((size_t)(maxt+1)*zng*sizeof(float));+ if(zq_all&&zsc_all) mxfp4_qx(Z,maxt+1,stride,zq_all,zsc_all);+ else { free(zq_all); free(zsc_all); zq_all=NULL; zsc_all=NULL; }+ }+ }for(int base=0;base<nu;base+=LP_MAX){int nb=nu-base<LP_MAX?nu-base:LP_MAX;Slot *use[LP_MAX]; int missk[LP_MAX]; int qof[LP_MAX]; int nmiss=0;@@ -1303,7 +1336,9 @@ static void experts_apply_union(Model *m, int li, int nu, const int *uids,for(int p2=0;p2<pcnt[base+j];p2++){int t=poslist[f+p2];expert_apply(m,use[j],Z+(int64_t)t*stride,wlist[f+p2],- U+(int64_t)t*stride,gate,up,hz);+ U+(int64_t)t*stride,gate,up,hz,+ zq_all?zq_all+(int64_t)t*stride:NULL,+ zq_all?zsc_all+(int64_t)t*zng:NULL);}}/* promotion: swap the freshly-read slots into the layer LRU */@@ -1328,6 +1363,7 @@ static void experts_apply_union(Model *m, int li, int nu, const int *uids,dst->used=++m->clock;}}+ free(zq_all); free(zsc_all);}/* ---- AUTOPIN: seed the layer caches from the accumulated history ----------diff --git a/c/quant.h b/c/quant.hindex 92c4174c..d909da7b 100644--- a/c/quant.h+++ b/c/quant.h@@ -1453,13 +1453,18 @@ static void matmul_mxfp4(float *y, const float *x, const uint8_t *q4, const uint* y = sum_g idot_g * (2^(e8-127) * 0.5) * xscale_g* Activation-quant noise (~0.4%/group) rides on top of the e2m1 grid; gate* with K3_IDOT=0 in the K3 engine for exact-float A/B. */-static void matmul_mxfp4_i8(float *y, const float *x, const uint8_t *q4, const uint8_t *e8s,- int S, int I, int O){- if(I%32){ matmul_mxfp4(y,x,q4,e8s,S,I,O); return; }- int rb=I/2, ng=I/32;- int8_t *xq=(int8_t*)malloc((size_t)S*I);- float *xsc=(float*)malloc((size_t)S*ng*sizeof(float));- if(!xq||!xsc){ fprintf(stderr,"OOM mxfp4 idot scratch\n"); exit(1); }+/* Quantizzazione int8 per-gruppo-di-32 delle attivazioni per l'IDOT mxfp4.+ * Estratta da matmul_mxfp4_i8 (ordine dei loop IDENTICO -> stessi byte) così i+ * chiamanti possono sollevarla a livello layer: in kimi_k3 gate e up consumano+ * lo stesso z, e ogni expert del layer consuma le stesse righe — prima ogni+ * chiamata riquantizzava da capo, con un malloc/free a giro (issue del PR #1071+ * per colibri.c; qui è la stessa malattia in forma mxfp4).+ * EN: per-32-group int8 activation quantization for the mxfp4 IDOT kernel.+ * Extracted with the loop order unchanged (bit-identical) so callers can hoist+ * it to layer level: gate/up share one z, and every expert of the layer+ * consumes the same rows. */+static void mxfp4_qx(const float *x, int S, int I, int8_t *xq, float *xsc){+ int ng=I/32;for(int s=0;s<S;s++)for(int g=0;g<ng;g++){const float *xg=x+(int64_t)s*I+g*32;@@ -1472,6 +1477,14 @@ static void matmul_mxfp4_i8(float *y, const float *x, const uint8_t *q4, const uif(v>127)v=127; if(v<-127)v=-127; qg[i]=(int8_t)v;}}+}++/* Compute su attivazioni GIA' quantizzate (xq/xsc da mxfp4_qx). Contratto: I%32==0.+ * EN: compute on pre-quantized activations. Contract: I%32==0. */+static void matmul_mxfp4_i8_pre(float *y, const int8_t *xq, const float *xsc,+ const uint8_t *q4, const uint8_t *e8s,+ int S, int I, int O){+ int rb=I/2, ng=I/32;#pragma omp parallel for schedule(static)for(int o=0;o<O;o++){const uint8_t *w=q4+(int64_t)o*rb;@@ -1518,7 +1531,20 @@ static void matmul_mxfp4_i8(float *y, const float *x, const uint8_t *q4, const u}}}- free(xq); free(xsc);+}++/* Wrapper storico: quantizza in scratch _Thread_local (niente più malloc/free+ * per chiamata) e delega al kernel _pre. Stessi valori, stessi ordini.+ * EN: historical entry point — quantize into the growable thread-local scratch+ * (no more per-call malloc/free) and delegate to the _pre kernel. */+static void matmul_mxfp4_i8(float *y, const float *x, const uint8_t *q4, const uint8_t *e8s,+ int S, int I, int O){+ if(I%32){ matmul_mxfp4(y,x,q4,e8s,S,I,O); return; }+ int ng=I/32;+ int8_t *xq; float *xsc;+ quant_scratch((size_t)S*I,(size_t)S*ng,&xq,&xsc);+ mxfp4_qx(x,S,I,xq,xsc);+ matmul_mxfp4_i8_pre(y,xq,xsc,q4,e8s,S,I,O);}#ifdef __ARM_NEON