Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

attn: tile the prefill k/v reconstruction past a ceiling (#768 cause 2)

ZacharyZcR43a9ea58d5b9
diff --git a/c/colibri.c b/c/colibri.c
index 1e3748ce..b36fb493 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -4406,6 +4406,87 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p
/* 2) ricostruzione di k_nope+value per TUTTI i token 0..Tk-1 (un solo matmul su kv_b) */
m->t_aproj+=now_s()-ta0; double tk0=now_s();
int stL=m->kv_start[layer];
+ /* #768 (cause 2): the one-shot kvb_all buffer is Tk*H*(qk_nope+v_head) floats β€”
+ * 30.1 GB at ctx 262144 β€” and cap_for_ram reserves it PERMANENTLY even though it
+ * only exists during prefill; that reservation is what starves the expert cache
+ * at long context (5.46 -> 0.86 tok/s). Above KVB_FLASH_MB the reconstruction is
+ * tiled instead: rebuild kv_b for KVB_TILE_MB worth of tokens at a time and fold
+ * scores/values through an online (flash-style) softmax. Same rebuild total (one
+ * matmul pass over the context), same t-order for scores and values; only the
+ * softmax normalisation is applied incrementally, so the output can differ from
+ * the one-shot path by rounding β€” the same kernel-family divergence class as the
+ * CUDA/Metal attention arms (#510). Peak transient drops from Tk*kvb_dim*4 to the
+ * tile plus 2 floats per (row,head). DSA rows (dnsel) keep the one-shot path: the
+ * top-keep list is scanned in its two-band order and is not tileable by t-range.
+ * KVB_FLASH_MB=0 disables tiling entirely; KVB_FLASH=1 forces it at any size. */
+ int dsa_any=0; if(dnsel) for(int s=0;s<S && !dsa_any;s++) if(dnsel[s]>0) dsa_any=1;
+ int64_t kvb_rows=(int64_t)Tk-stL, kvb_need=kvb_rows*kvb_dim*4;
+ int64_t flash_mb=getenv("KVB_FLASH_MB")?atoll(getenv("KVB_FLASH_MB")):2048;
+ int use_flash=!dsa_any && flash_mb>0 && kvb_need>flash_mb*1048576;
+ if(getenv("KVB_FLASH")) use_flash=!dsa_any && atoi(getenv("KVB_FLASH"))!=0;
+ if(use_flash){
+ int64_t tile_mb=getenv("KVB_TILE_MB")?atoll(getenv("KVB_TILE_MB")):512;
+ int64_t tile=tile_mb*1048576/((int64_t)kvb_dim*4);
+ if(tile<256) tile=256; if(tile>kvb_rows) tile=kvb_rows;
+ static int said; if(!said){ said=1;
+ fprintf(stderr,"[ATTN] kvb reconstruction tiled (#768): %.1f GB one-shot buffer "
+ "-> %.0f MB tiles, online softmax (KVB_FLASH_MB=0 restores one-shot)\n",
+ kvb_need/1e9, tile*(double)kvb_dim*4/1048576.0);
+ }
+ m->t_kvb += now_s()-tk0;
+ float *kvb_tile=falloc(tile*kvb_dim);
+ float *ml=falloc((int64_t)S*H*2); /* running (max, sum) per (row, head) */
+ double tac=now_s(), kvb_acc=0;
+ #pragma omp parallel for collapse(2) schedule(static)
+ for(int s=0;s<S;s++) for(int h=0;h<H;h++){
+ ml[((int64_t)s*H+h)*2]=-1e30f; ml[((int64_t)s*H+h)*2+1]=0.f;
+ float *cx=ctx+((int64_t)s*H+h)*vh; for(int d=0;d<vh;d++) cx[d]=0;
+ }
+ for(int64_t t0=stL; t0<Tk; t0+=tile){
+ int64_t tn=Tk-t0<tile?Tk-t0:tile;
+ double tk1=now_s();
+ matmul_qt(kvb_tile, m->Lc[layer]+t0*c->kv_lora, &l->kv_b, (int)tn);
+ kvb_acc+=now_s()-tk1;
+ #pragma omp parallel for collapse(2) schedule(static,1)
+ for(int s=0;s<S;s++) for(int h=0;h<H;h++){
+ int pos=pos_base+s;
+ int64_t nt=(int64_t)pos+1-stL; /* causal length of this row */
+ if(t0-stL>=nt) continue;
+ int64_t jn=nt-(t0-stL); if(jn>tn) jn=tn;
+ const float *qp=Q+(int64_t)s*H*qh+(int64_t)h*qh, *qr=qp+c->qk_nope;
+ float *st=ml+((int64_t)s*H+h)*2, *cx=ctx+((int64_t)s*H+h)*vh;
+ float mrun=st[0], lrun=st[1];
+ for(int64_t jj=0;jj<jn;jj++){
+ const float *kn=kvb_tile+jj*kvb_dim+(int64_t)h*(c->qk_nope+vh);
+ const float *kr=m->Rc[layer]+(t0+jj)*c->qk_rope;
+ float a=0; for(int d=0;d<c->qk_nope;d++) a+=qp[d]*kn[d];
+ for(int d=0;d<c->qk_rope;d++) a+=qr[d]*kr[d];
+ a*=c->attn_scale;
+ const float *vv=kvb_tile+jj*kvb_dim+(int64_t)h*(c->qk_nope+vh)+c->qk_nope;
+ if(a>mrun){
+ float rs=expf(mrun-a);
+ lrun*=rs; for(int d=0;d<vh;d++) cx[d]*=rs;
+ mrun=a;
+ }
+ float w=expf(a-mrun);
+ lrun+=w; for(int d=0;d<vh;d++) cx[d]+=w*vv[d];
+ }
+ st[0]=mrun; st[1]=lrun;
+ }
+ }
+ #pragma omp parallel for collapse(2) schedule(static)
+ for(int s=0;s<S;s++) for(int h=0;h<H;h++){
+ float lrun=ml[((int64_t)s*H+h)*2+1];
+ float *cx=ctx+((int64_t)s*H+h)*vh;
+ if(lrun>0) for(int d=0;d<vh;d++) cx[d]/=lrun;
+ }
+ free(kvb_tile); free(ml);
+ m->t_kvb+=kvb_acc; m->t_acore+=now_s()-tac-kvb_acc; double tao=now_s();
+ matmul_qt(out, ctx, &l->o, S); m->t_aout+=now_s()-tao;
+ free(ctx); free(Q); free(QR); free(comp);
+ m->t_attn += now_s()-ta0;
+ return;
+ }
float *kvb_all=falloc((int64_t)Tk*kvb_dim);
matmul_qt(kvb_all+(int64_t)stL*kvb_dim, m->Lc[layer]+(int64_t)stL*c->kv_lora, &l->kv_b, Tk-stL);
m->t_kvb += now_s()-tk0;
@@ -9259,6 +9340,15 @@ static void cap_for_ram(Model *m, double ram_gb, int ebits, int max_ctx){
if(g_expert_budget>0 && g_expert_budget<64) ws_b = (double)(g_expert_budget+4) * (double)eb;
double kv_b = kv_pool_bytes(m,max_ctx);
double kvb_b = (double)max_ctx*c->n_heads*(c->qk_nope+c->v_head)*4.0;
+ /* #768: above KVB_FLASH_MB attention_rows tiles the reconstruction instead of
+ * materialising kvb_all, so the transient to reserve is the flash ceiling plus
+ * one tile and the per-(row,head) running state β€” not 30 GB at ctx 262144.
+ * The reserve mirrors the trigger in attention_rows: same env, same default. */
+ { int64_t flash_mb=getenv("KVB_FLASH_MB")?atoll(getenv("KVB_FLASH_MB")):2048;
+ int64_t tile_mb =getenv("KVB_TILE_MB") ?atoll(getenv("KVB_TILE_MB")) :512;
+ double capped = (double)flash_mb*1048576.0 + (double)tile_mb*1048576.0
+ + (double)max_ctx*c->n_heads*2*4.0;
+ if(flash_mb>0 && kvb_b>capped) kvb_b = capped; }
/* RISERVA PAGE-CACHE (misurato 2026-07-06 su Linux): strangolarla fa crollare
* le pread buffered da ~800 a ~180 MB/s β€” gli ultimi GB di LRU rendono MENO di
* quanto costino in banda disco persa. 2.5 GB restano SEMPRE al kernel.
diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md
index 3cceffb4..dfb5f1f9 100644
--- a/docs/ENVIRONMENT.md
+++ b/docs/ENVIRONMENT.md
@@ -73,6 +73,9 @@ Format: `VAR` β€” default β€” effect.
| `RSS_GUARD_GB` | the resolved RAM budget | Resident-set ceiling (GB) checked every 16 emitted tokens; the cache is trimmed when it is crossed. Set explicitly to guard tighter or looser than the RAM budget. |
| `XEXP` | `0` (off) | `=1` runs ONE OpenMP region across all experts of a batch-union block instead of ~2 fork/joins per expert. Engages only at S=1 with an all-resident int4 block, off the speculation window, and with the int4-IDOT S=1 family (`I4S<=1`); output is byte-identical to that family. Measured +11.6% on a 2-socket 48-core Ice Lake, but neutral-to-negative on a 24-core box β€” hence opt-in. Measure on your host. |
| `COLI_KV_SHARE` | `0` (off) | `=1` lets a new serve slot adopt an existing slot's KV prefix instead of re-prefilling it. Measured on 6x5090 with a 675-token shared prefix: slot TTFT 50.1s β†’ 1.7s, generated tokens identical. |
+| `KVB_FLASH_MB` | `2048` | Ceiling (MB) for the one-shot `kvb_all` k/v reconstruction buffer in prefill attention (#768 β€” 30.1 GB at ctx 262144, and `cap_for_ram` reserved it permanently). Above the ceiling the reconstruction is tiled with an online (flash-style) softmax: same rebuild total, ~tile-sized transient, output may differ from one-shot by rounding (same divergence class as the CUDA/Metal attention arms). `=0` disables tiling (always one-shot). DSA-selected rows always take the one-shot path. |
+| `KVB_TILE_MB` | `512` | Tile size (MB) for the tiled reconstruction above. |
+| `KVB_FLASH` | unset | `=1` forces the tiled path at any size, `=0` forces one-shot β€” overrides the `KVB_FLASH_MB` trigger (A/B switch). |
| `COLI_GROUP_ASYNC` | `0` (off) | `=1` issues and collects CUDA expert groups asynchronously so CPU and GPU overlap at decode (S≀4). |
| `COLI_DISKCLASS_WINDOW` | see source | Recency window (in ticks) for the DISK-CLASS heat statistic. |
| `URING` | `0` (off) | Linux-only queued expert I/O. `URING=1` implies `PIPE=1`, forces cold reads through io-wq (`IOSQE_ASYNC`), replaces blocking loader pthreads and spin waits with batched SQEs/CQEs, and batches `PILOT_REAL` loads on a separate ring. Use `DIRECT=1` for cold NVMe to avoid page-cache copy/readahead limits. Fails clearly if the kernel denies io_uring; incompatible with `COLI_MMAP=1`. |