Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

perf(kimi_k3): hoist expert activation quantization to layer level (bit-identical)

JustVugga7f2f64e49d4
diff --git a/c/kimi_k3.c b/c/kimi_k3.c
index 9b969198..41ea5f3e 100644
--- a/c/kimi_k3.c
+++ b/c/kimi_k3.c
@@ -1120,9 +1120,15 @@ static int cuda_expert_apply(Model *m, const uint8_t *w1p, const uint8_t *w1s,
#endif
/* u += wk * E(z) for one loaded expert slot (SiTU-GLU in the latent).
- * gate/up are [moe_inter] scratch, hz is [latent] scratch. */
+ * gate/up are [moe_inter] scratch, hz is [latent] scratch.
+ * zq/zsc: riga di z già quantizzata a livello layer da mxfp4_qx (NULL = come
+ * prima, quantizza in-chiamata). Gate e up la condividono; il down riquantizza
+ * comunque il proprio input per-expert (gate), oggi però in scratch, non malloc.
+ * EN: zq/zsc = layer-hoisted pre-quantized z row (NULL = quantize in-call as
+ * before). Gate and up share it; down still quantizes its per-expert input. */
static void expert_apply(Model *m, Slot *s, const float *z, float wk,
- float *u, float *gate, float *up, float *hz){
+ float *u, float *gate, float *up, float *hz,
+ const int8_t *zq, const float *zsc){
Cfg *c=&m->c;
uint8_t *w1p=s->buf, *w1s=w1p+m->e_w1p, *w2p=w1s+m->e_w1s, *w2s=w2p+m->e_w2p,
*w3p=w2s+m->e_w2s, *w3s=w3p+m->e_w1p;
@@ -1133,8 +1139,13 @@ static void expert_apply(Model *m, Slot *s, const float *z, float wk,
#endif
void (*mm)(float*,const float*,const uint8_t*,const uint8_t*,int,int,int)
= g_k3_idot ? matmul_mxfp4_i8 : matmul_mxfp4;
- mm(gate,z,w1p,w1s,1,c->latent,c->moe_inter);
- mm(up,z,w3p,w3s,1,c->latent,c->moe_inter);
+ if(g_k3_idot && zq){
+ matmul_mxfp4_i8_pre(gate,zq,zsc,w1p,w1s,1,c->latent,c->moe_inter);
+ matmul_mxfp4_i8_pre(up, zq,zsc,w3p,w3s,1,c->latent,c->moe_inter);
+ } else {
+ mm(gate,z,w1p,w1s,1,c->latent,c->moe_inter);
+ mm(up,z,w3p,w3s,1,c->latent,c->moe_inter);
+ }
for(int i=0;i<c->moe_inter;i++) gate[i]=situf_(gate[i],up[i],c->situ_b1,c->situ_b2);
mm(hz,gate,w2p,w2s,1,c->moe_inter,c->latent);
for(int i=0;i<c->latent;i++) u[i]+=wk*hz[i];
@@ -1257,6 +1268,28 @@ static void experts_apply_union(Model *m, int li, int nu, const int *uids,
const int *poslist, const float *wlist,
const float *Z, int stride, float *U,
float *gate, float *up, float *hz){
+ /* Quantizzazione sollevata a livello layer (specchia colibri.c #1071):
+ * ogni riga di Z veniva riquantizzata 2x per OGNI expert che la consuma
+ * (gate+up dentro matmul_mxfp4_i8, con un malloc/free a chiamata) — con
+ * topk=8 sono 16 quantizzazioni della stessa riga per layer. Una sola
+ * passata mxfp4_qx qui produce le stesse righe int8 (ordine identico:
+ * bit-identico), e il fallimento di allocazione degrada al vecchio path.
+ * EN: layer-hoisted activation quantization, mirroring colibri.c (#1071).
+ * One mxfp4_qx pass replaces 2 requantizations per (expert, position);
+ * allocation failure degrades to the old in-call path. */
+ int8_t *zq_all=NULL; float *zsc_all=NULL; int zng=0;
+ if(g_k3_idot && stride%32==0 && nu>0){
+ int maxt=-1;
+ for(int j=0;j<nu;j++) for(int p2=0;p2<pcnt[j];p2++)
+ if(poslist[pfirst[j]+p2]>maxt) maxt=poslist[pfirst[j]+p2];
+ if(maxt>=0){
+ zng=stride/32;
+ zq_all=malloc((size_t)(maxt+1)*stride);
+ zsc_all=malloc((size_t)(maxt+1)*zng*sizeof(float));
+ if(zq_all&&zsc_all) mxfp4_qx(Z,maxt+1,stride,zq_all,zsc_all);
+ else { free(zq_all); free(zsc_all); zq_all=NULL; zsc_all=NULL; }
+ }
+ }
for(int base=0;base<nu;base+=LP_MAX){
int nb=nu-base<LP_MAX?nu-base:LP_MAX;
Slot *use[LP_MAX]; int missk[LP_MAX]; int qof[LP_MAX]; int nmiss=0;
@@ -1303,7 +1336,9 @@ static void experts_apply_union(Model *m, int li, int nu, const int *uids,
for(int p2=0;p2<pcnt[base+j];p2++){
int t=poslist[f+p2];
expert_apply(m,use[j],Z+(int64_t)t*stride,wlist[f+p2],
- U+(int64_t)t*stride,gate,up,hz);
+ U+(int64_t)t*stride,gate,up,hz,
+ zq_all?zq_all+(int64_t)t*stride:NULL,
+ zq_all?zsc_all+(int64_t)t*zng:NULL);
}
}
/* promotion: swap the freshly-read slots into the layer LRU */
@@ -1328,6 +1363,7 @@ static void experts_apply_union(Model *m, int li, int nu, const int *uids,
dst->used=++m->clock;
}
}
+ free(zq_all); free(zsc_all);
}
/* ---- AUTOPIN: seed the layer caches from the accumulated history ----------
diff --git a/c/quant.h b/c/quant.h
index 92c4174c..d909da7b 100644
--- a/c/quant.h
+++ b/c/quant.h
@@ -1453,13 +1453,18 @@ static void matmul_mxfp4(float *y, const float *x, const uint8_t *q4, const uint
* y = sum_g idot_g * (2^(e8-127) * 0.5) * xscale_g
* Activation-quant noise (~0.4%/group) rides on top of the e2m1 grid; gate
* with K3_IDOT=0 in the K3 engine for exact-float A/B. */
-static void matmul_mxfp4_i8(float *y, const float *x, const uint8_t *q4, const uint8_t *e8s,
- int S, int I, int O){
- if(I%32){ matmul_mxfp4(y,x,q4,e8s,S,I,O); return; }
- int rb=I/2, ng=I/32;
- int8_t *xq=(int8_t*)malloc((size_t)S*I);
- float *xsc=(float*)malloc((size_t)S*ng*sizeof(float));
- if(!xq||!xsc){ fprintf(stderr,"OOM mxfp4 idot scratch\n"); exit(1); }
+/* Quantizzazione int8 per-gruppo-di-32 delle attivazioni per l'IDOT mxfp4.
+ * Estratta da matmul_mxfp4_i8 (ordine dei loop IDENTICO -> stessi byte) così i
+ * chiamanti possono sollevarla a livello layer: in kimi_k3 gate e up consumano
+ * lo stesso z, e ogni expert del layer consuma le stesse righe — prima ogni
+ * chiamata riquantizzava da capo, con un malloc/free a giro (issue del PR #1071
+ * per colibri.c; qui è la stessa malattia in forma mxfp4).
+ * EN: per-32-group int8 activation quantization for the mxfp4 IDOT kernel.
+ * Extracted with the loop order unchanged (bit-identical) so callers can hoist
+ * it to layer level: gate/up share one z, and every expert of the layer
+ * consumes the same rows. */
+static void mxfp4_qx(const float *x, int S, int I, int8_t *xq, float *xsc){
+ int ng=I/32;
for(int s=0;s<S;s++)
for(int g=0;g<ng;g++){
const float *xg=x+(int64_t)s*I+g*32;
@@ -1472,6 +1477,14 @@ static void matmul_mxfp4_i8(float *y, const float *x, const uint8_t *q4, const u
if(v>127)v=127; if(v<-127)v=-127; qg[i]=(int8_t)v;
}
}
+}
+
+/* Compute su attivazioni GIA' quantizzate (xq/xsc da mxfp4_qx). Contratto: I%32==0.
+ * EN: compute on pre-quantized activations. Contract: I%32==0. */
+static void matmul_mxfp4_i8_pre(float *y, const int8_t *xq, const float *xsc,
+ const uint8_t *q4, const uint8_t *e8s,
+ int S, int I, int O){
+ int rb=I/2, ng=I/32;
#pragma omp parallel for schedule(static)
for(int o=0;o<O;o++){
const uint8_t *w=q4+(int64_t)o*rb;
@@ -1518,7 +1531,20 @@ static void matmul_mxfp4_i8(float *y, const float *x, const uint8_t *q4, const u
}
}
}
- free(xq); free(xsc);
+}
+
+/* Wrapper storico: quantizza in scratch _Thread_local (niente più malloc/free
+ * per chiamata) e delega al kernel _pre. Stessi valori, stessi ordini.
+ * EN: historical entry point — quantize into the growable thread-local scratch
+ * (no more per-call malloc/free) and delegate to the _pre kernel. */
+static void matmul_mxfp4_i8(float *y, const float *x, const uint8_t *q4, const uint8_t *e8s,
+ int S, int I, int O){
+ if(I%32){ matmul_mxfp4(y,x,q4,e8s,S,I,O); return; }
+ int ng=I/32;
+ int8_t *xq; float *xsc;
+ quant_scratch((size_t)S*I,(size_t)S*ng,&xq,&xsc);
+ mxfp4_qx(x,S,I,xq,xsc);
+ matmul_mxfp4_i8_pre(y,xq,xsc,q4,e8s,S,I,O);
}
#ifdef __ARM_NEON