Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
KV8: grouped latent scales (KV8_GS), FlashMLA's fp8 cache geometry
ZacharyZcRaa273d3c6ae9
diff --git a/c/colibri.c b/c/colibri.cindex 662d0bdf..f92ad837 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -1330,6 +1330,9 @@ static int g_disk_split=0; /* DISK_SPLIT=1: contatori che spezzano i DISK LOAD (/* KV-cache quantization tier flags — defined here (before kv_persist.h) so the .coli_kv* disk format can see them; the full rationale comments live at their original site below. */static int g_kv8=0; /* KV8=1: fp8 e4m3 latent KV + per-row scale */+static int g_kv8_gs=0; /* KV8_GS=<n>: one scale per n latent elements+ * instead of per row (FlashMLA uses 128 on the+ * 512-dim latent). 0 = per-row (unchanged). */static int g_tq=0, g_tq_bits=4, g_tq_codec=1; /* KV_TQ: codec 1=rotated int4 (default), 0=PolarQuant */#include "kv_persist.h"#include "telemetry.h"@@ -4203,7 +4206,9 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int pfloat *Ls=comp+(int64_t)s*cw, *Rs=Ls+c->kv_lora;rmsnorm(Ls, Ls, l->kv_a_ln, c->kv_lora, c->eps); /* latente normato */rope_interleave(Rs, pos, c); /* k_rot roped */- ks->Lsc[layer][pos]=coli_kv8_quant_row(Ls, coli_kv_row8(ks->Lc8[layer],pos,c->kv_lora), c->kv_lora);+ coli_kv8_quant_row_gs(Ls, coli_kv_row8(ks->Lc8[layer],pos,c->kv_lora),+ ks->Lsc[layer]+(int64_t)pos*coli_kv8_nscale(c->kv_lora,g_kv8_gs),+ c->kv_lora, g_kv8_gs);ks->Rsc[layer][pos]=coli_kv8_quant_row(Rs, coli_kv_row8(ks->Rc8[layer],pos,c->qk_rope), c->qk_rope);} else {float *Ldst=coli_kv_row(ks->Lc[layer],pos,c->kv_lora);@@ -4515,13 +4520,27 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int pfor(int d=0;d<c->qk_rope;d++) a+=qr[d]*Rf[d];} else if(g_kv8){/* LUT-dequant inline nel dot; la scala per-riga esce dalla- * somma: score = Lsc·Σ q·lut[b] + Rsc·Σ qr·lut[b] */+ * somma: score = Lsc·Σ q·lut[b] + Rsc·Σ qr·lut[b].+ * KV8_GS: per-group scales — partial dot per group, each+ * scaled before the sum (same math, tighter grid). */const uint8_t *Lt=coli_kv_row8(ks->Lc8[layer],t,kvl);const uint8_t *kr=coli_kv_row8(ks->Rc8[layer],t,c->qk_rope);- float al=0, ar=0;- for(int i=0;i<kvl;i++) al+=qabs[i]*coli_fp8_lut[Lt[i]];+ int nsl=coli_kv8_nscale(kvl,g_kv8_gs);+ const float *Ls_t=ks->Lsc[layer]+(int64_t)t*nsl;+ float ar=0; a=0;+ if(g_kv8_gs){+ for(int g=0,k2=0;g<kvl;g+=g_kv8_gs,k2++){+ int m=kvl-g<g_kv8_gs?kvl-g:g_kv8_gs; float al=0;+ for(int i=0;i<m;i++) al+=qabs[g+i]*coli_fp8_lut[Lt[g+i]];+ a+=al*Ls_t[k2];+ }+ } else {+ float al=0;+ for(int i=0;i<kvl;i++) al+=qabs[i]*coli_fp8_lut[Lt[i]];+ a=al*Ls_t[0];+ }for(int d=0;d<c->qk_rope;d++) ar+=qr[d]*coli_fp8_lut[kr[d]];- a=al*ks->Lsc[layer][t]+ar*ks->Rsc[layer][t];+ a+=ar*ks->Rsc[layer][t];} else {const float *Lt=coli_kv_row(ks->Lc[layer],t,kvl);const float *kr=coli_kv_row(ks->Rc[layer],t,c->qk_rope);@@ -4565,8 +4584,18 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int pfor(int i=0;i<kvl;i++) clat[i]+=a*Lf[i];} else if(g_kv8){const uint8_t *Lt=coli_kv_row8(ks->Lc8[layer],t,kvl);- float a=sc[jj]*ks->Lsc[layer][t]; /* la scala si fonde nel peso */- for(int i=0;i<kvl;i++) clat[i]+=a*coli_fp8_lut[Lt[i]];+ int nsl=coli_kv8_nscale(kvl,g_kv8_gs);+ const float *Ls_t=ks->Lsc[layer]+(int64_t)t*nsl;+ if(g_kv8_gs){+ for(int g=0,k2=0;g<kvl;g+=g_kv8_gs,k2++){+ int m=kvl-g<g_kv8_gs?kvl-g:g_kv8_gs;+ float a=sc[jj]*Ls_t[k2];+ for(int i=0;i<m;i++) clat[g+i]+=a*coli_fp8_lut[Lt[g+i]];+ }+ } else {+ float a=sc[jj]*Ls_t[0]; /* la scala si fonde nel peso */+ for(int i=0;i<kvl;i++) clat[i]+=a*coli_fp8_lut[Lt[i]];+ }} else {const float *Lt=coli_kv_row(ks->Lc[layer],t,kvl);/* MLA-absorb value mix: clat += sc[jj] * Lt (AXPY over kvl).@@ -4750,9 +4779,11 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p/* staging f32 del latente dequantizzato: kv_b vuole righe float. Il buffer* [Tk-stL,kvl] e' rumore rispetto a kvb_all [Tk,H*(nope+vh)] gia' allocato. */float *Lf=falloc((int64_t)(Tk-stL)*c->kv_lora);- for(int t=stL;t<Tk;t++)- coli_kv8_dequant_row(coli_kv_row8(m->Lc8[layer],t,c->kv_lora), m->Lsc[layer][t],- Lf+(int64_t)(t-stL)*c->kv_lora, c->kv_lora);+ { int nsl=coli_kv8_nscale(c->kv_lora,g_kv8_gs);+ for(int t=stL;t<Tk;t++)+ coli_kv8_dequant_row_gs(coli_kv_row8(m->Lc8[layer],t,c->kv_lora),+ m->Lsc[layer]+(int64_t)t*nsl,+ Lf+(int64_t)(t-stL)*c->kv_lora, c->kv_lora, g_kv8_gs); }matmul_qt(kvb_all+(int64_t)stL*kvb_dim, Lf, &l->kv_b, Tk-stL);free(Lf);} else@@ -6872,7 +6903,8 @@ static void kv_alloc(Model *m, int max_t){for(int i=0;i<NR;i++){k->Lc8[i]=malloc((size_t)max_t*lb);k->Rc8[i]=malloc((size_t)max_t*rb);- k->Lsc[i]=falloc(max_t); k->Rsc[i]=falloc(max_t);+ k->Lsc[i]=falloc((int64_t)max_t*(g_kv8?coli_kv8_nscale(c->kv_lora,g_kv8_gs):1));+ k->Rsc[i]=falloc(max_t);if(!k->Lc8[i]||!k->Rc8[i]){fprintf(stderr,"OOM kv8\n");exit(1);}}} else@@ -10775,7 +10807,13 @@ int main(int argc, char **argv){}#endifcoli_fp8_lut_init();- fprintf(stderr,"[KV8] latent KV cache in fp8 e4m3 + per-row scale (~3.9x less KV RAM)\n");+ { const char *gsv=getenv("KV8_GS"); g_kv8_gs = gsv?atoi(gsv):0; if(g_kv8_gs<0) g_kv8_gs=0; }+ fprintf(stderr,"[KV8] latent KV cache in fp8 e4m3 + %s scale (~3.9x less KV RAM)\n",+ g_kv8_gs?"per-group":"per-row");+ if(g_kv8_gs && (!getenv("KVSAVE")||atoi(getenv("KVSAVE")))){+ fprintf(stderr,"[KV8] KV8_GS has no .coli_kv format yet: KV persistence disabled for this run\n");+ setenv("KVSAVE","0",1);+ }}/* KV_TQ=3|4: tier TurboQuant/PolarQuant (mutuamente esclusivo con KV8). CPU-only,* come KV8: si spegne dove i percorsi leggono righe f32. */diff --git a/c/kv_fp8.h b/c/kv_fp8.hindex 480c601f..87ac512e 100644--- a/c/kv_fp8.h+++ b/c/kv_fp8.h@@ -64,4 +64,24 @@ static inline void coli_kv8_dequant_row(const uint8_t *src, float scale, float *for(int i=0;i<n;i++) dst[i]=coli_fp8_lut[src[i]]*scale;}+/* KV8_GS: grouped-scale variants (FlashMLA keeps one f32 scale per 128 latent+ * elements; one amax per 512-dim row lets a single outlier dilate the grid for+ * the whole row). gs==0 -> the per-row functions above. The row's scales live+ * consecutively: ceil(n/gs) floats per row. */+static inline int coli_kv8_nscale(int n, int gs){ return gs>0 ? (n+gs-1)/gs : 1; }+static inline void coli_kv8_quant_row_gs(const float *src, uint8_t *dst, float *sc, int n, int gs){+ if(gs<=0){ sc[0]=coli_kv8_quant_row(src,dst,n); return; }+ for(int g=0,k=0;g<n;g+=gs,k++){+ int m=n-g<gs?n-g:gs;+ sc[k]=coli_kv8_quant_row(src+g,dst+g,m);+ }+}+static inline void coli_kv8_dequant_row_gs(const uint8_t *src, const float *sc, float *dst, int n, int gs){+ if(gs<=0){ coli_kv8_dequant_row(src,sc[0],dst,n); return; }+ for(int g=0,k=0;g<n;g+=gs,k++){+ int m=n-g<gs?n-g:gs;+ coli_kv8_dequant_row(src+g,sc[k],dst+g,m);+ }+}+#endifdiff --git a/c/tests/test_kv_fp8.c b/c/tests/test_kv_fp8.cindex c4ed0218..9af16549 100644--- a/c/tests/test_kv_fp8.c+++ b/c/tests/test_kv_fp8.c@@ -82,6 +82,41 @@ int main(void){CHECK(coli_kv_row8(buf,4,7)==buf+28, "coli_kv_row8 arithmetic");CHECK(coli_kv_row8(buf,0,7)==buf, "coli_kv_row8 row 0");+ /* KV8_GS grouped scales: gs=0 is byte-identical to the per-row pair; a+ * grouped row round-trips per group with each group's own amax bound —+ * an outlier in one group must not dilate another group's grid */+ {+ enum { GN=32, GS=8 };+ float grow[GN]; uint8_t gq[GN], gq0[GN]; float gsc[GN/GS], sc0;+ for(int i=0;i<GN;i++) grow[i]=0.01f*(float)(i-15);+ grow[3]=100.0f; /* outlier confined to group 0 */+ coli_kv8_quant_row_gs(grow,gq,gsc,GN,GS);+ sc0=coli_kv8_quant_row(grow,gq0,GN); (void)sc0;+ CHECK(coli_kv8_nscale(GN,GS)==4, "nscale(32,8)==4");+ CHECK(coli_kv8_nscale(GN,0)==1, "nscale(_,0)==1");+ float gdeq[GN];+ coli_kv8_dequant_row_gs(gq,gsc,gdeq,GN,GS);+ for(int g=1;g<4;g++) /* groups without the outlier: tight grid */+ for(int i=g*GS;i<(g+1)*GS;i++){+ float err=fabsf(gdeq[i]-grow[i]);+ CHECK(err<=fabsf(grow[i])/16.f+gsc[g]*0x1p-10f+1e-7f,+ "gs elem %d: err %g too large", i, err);+ }+ /* the per-row version smears the outlier's scale across everything:+ * grouped must beat it on the non-outlier groups */+ float rdeq[GN]; coli_kv8_dequant_row(gq0,sc0,rdeq,GN);+ double ge=0, re=0;+ for(int i=GS;i<GN;i++){ ge+=fabs(gdeq[i]-grow[i]); re+=fabs(rdeq[i]-grow[i]); }+ CHECK(ge<re, "grouped scales beat per-row off the outlier (%.3g vs %.3g)", ge, re);+ /* gs=0 through the _gs entry points == the per-row functions, bit for bit */+ uint8_t q0[GN]; float s0[1], d0[GN], d1[GN];+ coli_kv8_quant_row_gs(grow,q0,s0,GN,0);+ CHECK(memcmp(q0,gq0,GN)==0 && s0[0]==sc0, "gs=0 quant == per-row quant");+ coli_kv8_dequant_row_gs(q0,s0,d0,GN,0);+ coli_kv8_dequant_row(gq0,sc0,d1,GN);+ CHECK(memcmp(d0,d1,sizeof d0)==0, "gs=0 dequant == per-row dequant");+ }+if(fails){ fprintf(stderr,"%d failure(s)\n",fails); return 1; }printf("OK kv_fp8 e4m3 (exhaustive roundtrip + RNE + row quant)\n");return 0;