Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

KV8: grouped latent scales (KV8_GS), FlashMLA's fp8 cache geometry

ZacharyZcRaa273d3c6ae9
diff --git a/c/colibri.c b/c/colibri.c
index 662d0bdf..f92ad837 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -1330,6 +1330,9 @@ static int g_disk_split=0; /* DISK_SPLIT=1: contatori che spezzano i DISK LOAD (
/* KV-cache quantization tier flags — defined here (before kv_persist.h) so the .coli_kv
* disk format can see them; the full rationale comments live at their original site below. */
static int g_kv8=0; /* KV8=1: fp8 e4m3 latent KV + per-row scale */
+static int g_kv8_gs=0; /* KV8_GS=<n>: one scale per n latent elements
+ * instead of per row (FlashMLA uses 128 on the
+ * 512-dim latent). 0 = per-row (unchanged). */
static int g_tq=0, g_tq_bits=4, g_tq_codec=1; /* KV_TQ: codec 1=rotated int4 (default), 0=PolarQuant */
#include "kv_persist.h"
#include "telemetry.h"
@@ -4203,7 +4206,9 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p
float *Ls=comp+(int64_t)s*cw, *Rs=Ls+c->kv_lora;
rmsnorm(Ls, Ls, l->kv_a_ln, c->kv_lora, c->eps); /* latente normato */
rope_interleave(Rs, pos, c); /* k_rot roped */
- ks->Lsc[layer][pos]=coli_kv8_quant_row(Ls, coli_kv_row8(ks->Lc8[layer],pos,c->kv_lora), c->kv_lora);
+ coli_kv8_quant_row_gs(Ls, coli_kv_row8(ks->Lc8[layer],pos,c->kv_lora),
+ ks->Lsc[layer]+(int64_t)pos*coli_kv8_nscale(c->kv_lora,g_kv8_gs),
+ c->kv_lora, g_kv8_gs);
ks->Rsc[layer][pos]=coli_kv8_quant_row(Rs, coli_kv_row8(ks->Rc8[layer],pos,c->qk_rope), c->qk_rope);
} else {
float *Ldst=coli_kv_row(ks->Lc[layer],pos,c->kv_lora);
@@ -4515,13 +4520,27 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p
for(int d=0;d<c->qk_rope;d++) a+=qr[d]*Rf[d];
} else if(g_kv8){
/* LUT-dequant inline nel dot; la scala per-riga esce dalla
- * somma: score = Lsc·Σ q·lut[b] + Rsc·Σ qr·lut[b] */
+ * somma: score = Lsc·Σ q·lut[b] + Rsc·Σ qr·lut[b].
+ * KV8_GS: per-group scales — partial dot per group, each
+ * scaled before the sum (same math, tighter grid). */
const uint8_t *Lt=coli_kv_row8(ks->Lc8[layer],t,kvl);
const uint8_t *kr=coli_kv_row8(ks->Rc8[layer],t,c->qk_rope);
- float al=0, ar=0;
- for(int i=0;i<kvl;i++) al+=qabs[i]*coli_fp8_lut[Lt[i]];
+ int nsl=coli_kv8_nscale(kvl,g_kv8_gs);
+ const float *Ls_t=ks->Lsc[layer]+(int64_t)t*nsl;
+ float ar=0; a=0;
+ if(g_kv8_gs){
+ for(int g=0,k2=0;g<kvl;g+=g_kv8_gs,k2++){
+ int m=kvl-g<g_kv8_gs?kvl-g:g_kv8_gs; float al=0;
+ for(int i=0;i<m;i++) al+=qabs[g+i]*coli_fp8_lut[Lt[g+i]];
+ a+=al*Ls_t[k2];
+ }
+ } else {
+ float al=0;
+ for(int i=0;i<kvl;i++) al+=qabs[i]*coli_fp8_lut[Lt[i]];
+ a=al*Ls_t[0];
+ }
for(int d=0;d<c->qk_rope;d++) ar+=qr[d]*coli_fp8_lut[kr[d]];
- a=al*ks->Lsc[layer][t]+ar*ks->Rsc[layer][t];
+ a+=ar*ks->Rsc[layer][t];
} else {
const float *Lt=coli_kv_row(ks->Lc[layer],t,kvl);
const float *kr=coli_kv_row(ks->Rc[layer],t,c->qk_rope);
@@ -4565,8 +4584,18 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p
for(int i=0;i<kvl;i++) clat[i]+=a*Lf[i];
} else if(g_kv8){
const uint8_t *Lt=coli_kv_row8(ks->Lc8[layer],t,kvl);
- float a=sc[jj]*ks->Lsc[layer][t]; /* la scala si fonde nel peso */
- for(int i=0;i<kvl;i++) clat[i]+=a*coli_fp8_lut[Lt[i]];
+ int nsl=coli_kv8_nscale(kvl,g_kv8_gs);
+ const float *Ls_t=ks->Lsc[layer]+(int64_t)t*nsl;
+ if(g_kv8_gs){
+ for(int g=0,k2=0;g<kvl;g+=g_kv8_gs,k2++){
+ int m=kvl-g<g_kv8_gs?kvl-g:g_kv8_gs;
+ float a=sc[jj]*Ls_t[k2];
+ for(int i=0;i<m;i++) clat[g+i]+=a*coli_fp8_lut[Lt[g+i]];
+ }
+ } else {
+ float a=sc[jj]*Ls_t[0]; /* la scala si fonde nel peso */
+ for(int i=0;i<kvl;i++) clat[i]+=a*coli_fp8_lut[Lt[i]];
+ }
} else {
const float *Lt=coli_kv_row(ks->Lc[layer],t,kvl);
/* MLA-absorb value mix: clat += sc[jj] * Lt (AXPY over kvl).
@@ -4750,9 +4779,11 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p
/* staging f32 del latente dequantizzato: kv_b vuole righe float. Il buffer
* [Tk-stL,kvl] e' rumore rispetto a kvb_all [Tk,H*(nope+vh)] gia' allocato. */
float *Lf=falloc((int64_t)(Tk-stL)*c->kv_lora);
- for(int t=stL;t<Tk;t++)
- coli_kv8_dequant_row(coli_kv_row8(m->Lc8[layer],t,c->kv_lora), m->Lsc[layer][t],
- Lf+(int64_t)(t-stL)*c->kv_lora, c->kv_lora);
+ { int nsl=coli_kv8_nscale(c->kv_lora,g_kv8_gs);
+ for(int t=stL;t<Tk;t++)
+ coli_kv8_dequant_row_gs(coli_kv_row8(m->Lc8[layer],t,c->kv_lora),
+ m->Lsc[layer]+(int64_t)t*nsl,
+ Lf+(int64_t)(t-stL)*c->kv_lora, c->kv_lora, g_kv8_gs); }
matmul_qt(kvb_all+(int64_t)stL*kvb_dim, Lf, &l->kv_b, Tk-stL);
free(Lf);
} else
@@ -6872,7 +6903,8 @@ static void kv_alloc(Model *m, int max_t){
for(int i=0;i<NR;i++){
k->Lc8[i]=malloc((size_t)max_t*lb);
k->Rc8[i]=malloc((size_t)max_t*rb);
- k->Lsc[i]=falloc(max_t); k->Rsc[i]=falloc(max_t);
+ k->Lsc[i]=falloc((int64_t)max_t*(g_kv8?coli_kv8_nscale(c->kv_lora,g_kv8_gs):1));
+ k->Rsc[i]=falloc(max_t);
if(!k->Lc8[i]||!k->Rc8[i]){fprintf(stderr,"OOM kv8\n");exit(1);}
}
} else
@@ -10775,7 +10807,13 @@ int main(int argc, char **argv){
}
#endif
coli_fp8_lut_init();
- fprintf(stderr,"[KV8] latent KV cache in fp8 e4m3 + per-row scale (~3.9x less KV RAM)\n");
+ { const char *gsv=getenv("KV8_GS"); g_kv8_gs = gsv?atoi(gsv):0; if(g_kv8_gs<0) g_kv8_gs=0; }
+ fprintf(stderr,"[KV8] latent KV cache in fp8 e4m3 + %s scale (~3.9x less KV RAM)\n",
+ g_kv8_gs?"per-group":"per-row");
+ if(g_kv8_gs && (!getenv("KVSAVE")||atoi(getenv("KVSAVE")))){
+ fprintf(stderr,"[KV8] KV8_GS has no .coli_kv format yet: KV persistence disabled for this run\n");
+ setenv("KVSAVE","0",1);
+ }
}
/* KV_TQ=3|4: tier TurboQuant/PolarQuant (mutuamente esclusivo con KV8). CPU-only,
* come KV8: si spegne dove i percorsi leggono righe f32. */
diff --git a/c/kv_fp8.h b/c/kv_fp8.h
index 480c601f..87ac512e 100644
--- a/c/kv_fp8.h
+++ b/c/kv_fp8.h
@@ -64,4 +64,24 @@ static inline void coli_kv8_dequant_row(const uint8_t *src, float scale, float *
for(int i=0;i<n;i++) dst[i]=coli_fp8_lut[src[i]]*scale;
}
+/* KV8_GS: grouped-scale variants (FlashMLA keeps one f32 scale per 128 latent
+ * elements; one amax per 512-dim row lets a single outlier dilate the grid for
+ * the whole row). gs==0 -> the per-row functions above. The row's scales live
+ * consecutively: ceil(n/gs) floats per row. */
+static inline int coli_kv8_nscale(int n, int gs){ return gs>0 ? (n+gs-1)/gs : 1; }
+static inline void coli_kv8_quant_row_gs(const float *src, uint8_t *dst, float *sc, int n, int gs){
+ if(gs<=0){ sc[0]=coli_kv8_quant_row(src,dst,n); return; }
+ for(int g=0,k=0;g<n;g+=gs,k++){
+ int m=n-g<gs?n-g:gs;
+ sc[k]=coli_kv8_quant_row(src+g,dst+g,m);
+ }
+}
+static inline void coli_kv8_dequant_row_gs(const uint8_t *src, const float *sc, float *dst, int n, int gs){
+ if(gs<=0){ coli_kv8_dequant_row(src,sc[0],dst,n); return; }
+ for(int g=0,k=0;g<n;g+=gs,k++){
+ int m=n-g<gs?n-g:gs;
+ coli_kv8_dequant_row(src+g,sc[k],dst+g,m);
+ }
+}
+
#endif
diff --git a/c/tests/test_kv_fp8.c b/c/tests/test_kv_fp8.c
index c4ed0218..9af16549 100644
--- a/c/tests/test_kv_fp8.c
+++ b/c/tests/test_kv_fp8.c
@@ -82,6 +82,41 @@ int main(void){
CHECK(coli_kv_row8(buf,4,7)==buf+28, "coli_kv_row8 arithmetic");
CHECK(coli_kv_row8(buf,0,7)==buf, "coli_kv_row8 row 0");
+ /* KV8_GS grouped scales: gs=0 is byte-identical to the per-row pair; a
+ * grouped row round-trips per group with each group's own amax bound —
+ * an outlier in one group must not dilate another group's grid */
+ {
+ enum { GN=32, GS=8 };
+ float grow[GN]; uint8_t gq[GN], gq0[GN]; float gsc[GN/GS], sc0;
+ for(int i=0;i<GN;i++) grow[i]=0.01f*(float)(i-15);
+ grow[3]=100.0f; /* outlier confined to group 0 */
+ coli_kv8_quant_row_gs(grow,gq,gsc,GN,GS);
+ sc0=coli_kv8_quant_row(grow,gq0,GN); (void)sc0;
+ CHECK(coli_kv8_nscale(GN,GS)==4, "nscale(32,8)==4");
+ CHECK(coli_kv8_nscale(GN,0)==1, "nscale(_,0)==1");
+ float gdeq[GN];
+ coli_kv8_dequant_row_gs(gq,gsc,gdeq,GN,GS);
+ for(int g=1;g<4;g++) /* groups without the outlier: tight grid */
+ for(int i=g*GS;i<(g+1)*GS;i++){
+ float err=fabsf(gdeq[i]-grow[i]);
+ CHECK(err<=fabsf(grow[i])/16.f+gsc[g]*0x1p-10f+1e-7f,
+ "gs elem %d: err %g too large", i, err);
+ }
+ /* the per-row version smears the outlier's scale across everything:
+ * grouped must beat it on the non-outlier groups */
+ float rdeq[GN]; coli_kv8_dequant_row(gq0,sc0,rdeq,GN);
+ double ge=0, re=0;
+ for(int i=GS;i<GN;i++){ ge+=fabs(gdeq[i]-grow[i]); re+=fabs(rdeq[i]-grow[i]); }
+ CHECK(ge<re, "grouped scales beat per-row off the outlier (%.3g vs %.3g)", ge, re);
+ /* gs=0 through the _gs entry points == the per-row functions, bit for bit */
+ uint8_t q0[GN]; float s0[1], d0[GN], d1[GN];
+ coli_kv8_quant_row_gs(grow,q0,s0,GN,0);
+ CHECK(memcmp(q0,gq0,GN)==0 && s0[0]==sc0, "gs=0 quant == per-row quant");
+ coli_kv8_dequant_row_gs(q0,s0,d0,GN,0);
+ coli_kv8_dequant_row(gq0,sc0,d1,GN);
+ CHECK(memcmp(d0,d1,sizeof d0)==0, "gs=0 dequant == per-row dequant");
+ }
+
if(fails){ fprintf(stderr,"%d failure(s)\n",fails); return 1; }
printf("OK kv_fp8 e4m3 (exhaustive roundtrip + RNE + row quant)\n");
return 0;