Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
attn: tile the prefill k/v reconstruction past a ceiling (#768 cause 2)
ZacharyZcR43a9ea58d5b9
diff --git a/c/colibri.c b/c/colibri.cindex 1e3748ce..b36fb493 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -4406,6 +4406,87 @@ static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int p/* 2) ricostruzione di k_nope+value per TUTTI i token 0..Tk-1 (un solo matmul su kv_b) */m->t_aproj+=now_s()-ta0; double tk0=now_s();int stL=m->kv_start[layer];+ /* #768 (cause 2): the one-shot kvb_all buffer is Tk*H*(qk_nope+v_head) floats β+ * 30.1 GB at ctx 262144 β and cap_for_ram reserves it PERMANENTLY even though it+ * only exists during prefill; that reservation is what starves the expert cache+ * at long context (5.46 -> 0.86 tok/s). Above KVB_FLASH_MB the reconstruction is+ * tiled instead: rebuild kv_b for KVB_TILE_MB worth of tokens at a time and fold+ * scores/values through an online (flash-style) softmax. Same rebuild total (one+ * matmul pass over the context), same t-order for scores and values; only the+ * softmax normalisation is applied incrementally, so the output can differ from+ * the one-shot path by rounding β the same kernel-family divergence class as the+ * CUDA/Metal attention arms (#510). Peak transient drops from Tk*kvb_dim*4 to the+ * tile plus 2 floats per (row,head). DSA rows (dnsel) keep the one-shot path: the+ * top-keep list is scanned in its two-band order and is not tileable by t-range.+ * KVB_FLASH_MB=0 disables tiling entirely; KVB_FLASH=1 forces it at any size. */+ int dsa_any=0; if(dnsel) for(int s=0;s<S && !dsa_any;s++) if(dnsel[s]>0) dsa_any=1;+ int64_t kvb_rows=(int64_t)Tk-stL, kvb_need=kvb_rows*kvb_dim*4;+ int64_t flash_mb=getenv("KVB_FLASH_MB")?atoll(getenv("KVB_FLASH_MB")):2048;+ int use_flash=!dsa_any && flash_mb>0 && kvb_need>flash_mb*1048576;+ if(getenv("KVB_FLASH")) use_flash=!dsa_any && atoi(getenv("KVB_FLASH"))!=0;+ if(use_flash){+ int64_t tile_mb=getenv("KVB_TILE_MB")?atoll(getenv("KVB_TILE_MB")):512;+ int64_t tile=tile_mb*1048576/((int64_t)kvb_dim*4);+ if(tile<256) tile=256; if(tile>kvb_rows) tile=kvb_rows;+ static int said; if(!said){ said=1;+ fprintf(stderr,"[ATTN] kvb reconstruction tiled (#768): %.1f GB one-shot buffer "+ "-> %.0f MB tiles, online softmax (KVB_FLASH_MB=0 restores one-shot)\n",+ kvb_need/1e9, tile*(double)kvb_dim*4/1048576.0);+ }+ m->t_kvb += now_s()-tk0;+ float *kvb_tile=falloc(tile*kvb_dim);+ float *ml=falloc((int64_t)S*H*2); /* running (max, sum) per (row, head) */+ double tac=now_s(), kvb_acc=0;+ #pragma omp parallel for collapse(2) schedule(static)+ for(int s=0;s<S;s++) for(int h=0;h<H;h++){+ ml[((int64_t)s*H+h)*2]=-1e30f; ml[((int64_t)s*H+h)*2+1]=0.f;+ float *cx=ctx+((int64_t)s*H+h)*vh; for(int d=0;d<vh;d++) cx[d]=0;+ }+ for(int64_t t0=stL; t0<Tk; t0+=tile){+ int64_t tn=Tk-t0<tile?Tk-t0:tile;+ double tk1=now_s();+ matmul_qt(kvb_tile, m->Lc[layer]+t0*c->kv_lora, &l->kv_b, (int)tn);+ kvb_acc+=now_s()-tk1;+ #pragma omp parallel for collapse(2) schedule(static,1)+ for(int s=0;s<S;s++) for(int h=0;h<H;h++){+ int pos=pos_base+s;+ int64_t nt=(int64_t)pos+1-stL; /* causal length of this row */+ if(t0-stL>=nt) continue;+ int64_t jn=nt-(t0-stL); if(jn>tn) jn=tn;+ const float *qp=Q+(int64_t)s*H*qh+(int64_t)h*qh, *qr=qp+c->qk_nope;+ float *st=ml+((int64_t)s*H+h)*2, *cx=ctx+((int64_t)s*H+h)*vh;+ float mrun=st[0], lrun=st[1];+ for(int64_t jj=0;jj<jn;jj++){+ const float *kn=kvb_tile+jj*kvb_dim+(int64_t)h*(c->qk_nope+vh);+ const float *kr=m->Rc[layer]+(t0+jj)*c->qk_rope;+ float a=0; for(int d=0;d<c->qk_nope;d++) a+=qp[d]*kn[d];+ for(int d=0;d<c->qk_rope;d++) a+=qr[d]*kr[d];+ a*=c->attn_scale;+ const float *vv=kvb_tile+jj*kvb_dim+(int64_t)h*(c->qk_nope+vh)+c->qk_nope;+ if(a>mrun){+ float rs=expf(mrun-a);+ lrun*=rs; for(int d=0;d<vh;d++) cx[d]*=rs;+ mrun=a;+ }+ float w=expf(a-mrun);+ lrun+=w; for(int d=0;d<vh;d++) cx[d]+=w*vv[d];+ }+ st[0]=mrun; st[1]=lrun;+ }+ }+ #pragma omp parallel for collapse(2) schedule(static)+ for(int s=0;s<S;s++) for(int h=0;h<H;h++){+ float lrun=ml[((int64_t)s*H+h)*2+1];+ float *cx=ctx+((int64_t)s*H+h)*vh;+ if(lrun>0) for(int d=0;d<vh;d++) cx[d]/=lrun;+ }+ free(kvb_tile); free(ml);+ m->t_kvb+=kvb_acc; m->t_acore+=now_s()-tac-kvb_acc; double tao=now_s();+ matmul_qt(out, ctx, &l->o, S); m->t_aout+=now_s()-tao;+ free(ctx); free(Q); free(QR); free(comp);+ m->t_attn += now_s()-ta0;+ return;+ }float *kvb_all=falloc((int64_t)Tk*kvb_dim);matmul_qt(kvb_all+(int64_t)stL*kvb_dim, m->Lc[layer]+(int64_t)stL*c->kv_lora, &l->kv_b, Tk-stL);m->t_kvb += now_s()-tk0;@@ -9259,6 +9340,15 @@ static void cap_for_ram(Model *m, double ram_gb, int ebits, int max_ctx){if(g_expert_budget>0 && g_expert_budget<64) ws_b = (double)(g_expert_budget+4) * (double)eb;double kv_b = kv_pool_bytes(m,max_ctx);double kvb_b = (double)max_ctx*c->n_heads*(c->qk_nope+c->v_head)*4.0;+ /* #768: above KVB_FLASH_MB attention_rows tiles the reconstruction instead of+ * materialising kvb_all, so the transient to reserve is the flash ceiling plus+ * one tile and the per-(row,head) running state β not 30 GB at ctx 262144.+ * The reserve mirrors the trigger in attention_rows: same env, same default. */+ { int64_t flash_mb=getenv("KVB_FLASH_MB")?atoll(getenv("KVB_FLASH_MB")):2048;+ int64_t tile_mb =getenv("KVB_TILE_MB") ?atoll(getenv("KVB_TILE_MB")) :512;+ double capped = (double)flash_mb*1048576.0 + (double)tile_mb*1048576.0+ + (double)max_ctx*c->n_heads*2*4.0;+ if(flash_mb>0 && kvb_b>capped) kvb_b = capped; }/* RISERVA PAGE-CACHE (misurato 2026-07-06 su Linux): strangolarla fa crollare* le pread buffered da ~800 a ~180 MB/s β gli ultimi GB di LRU rendono MENO di* quanto costino in banda disco persa. 2.5 GB restano SEMPRE al kernel.diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.mdindex 3cceffb4..dfb5f1f9 100644--- a/docs/ENVIRONMENT.md+++ b/docs/ENVIRONMENT.md@@ -73,6 +73,9 @@ Format: `VAR` β default β effect.| `RSS_GUARD_GB` | the resolved RAM budget | Resident-set ceiling (GB) checked every 16 emitted tokens; the cache is trimmed when it is crossed. Set explicitly to guard tighter or looser than the RAM budget. || `XEXP` | `0` (off) | `=1` runs ONE OpenMP region across all experts of a batch-union block instead of ~2 fork/joins per expert. Engages only at S=1 with an all-resident int4 block, off the speculation window, and with the int4-IDOT S=1 family (`I4S<=1`); output is byte-identical to that family. Measured +11.6% on a 2-socket 48-core Ice Lake, but neutral-to-negative on a 24-core box β hence opt-in. Measure on your host. || `COLI_KV_SHARE` | `0` (off) | `=1` lets a new serve slot adopt an existing slot's KV prefix instead of re-prefilling it. Measured on 6x5090 with a 675-token shared prefix: slot TTFT 50.1s β 1.7s, generated tokens identical. |+| `KVB_FLASH_MB` | `2048` | Ceiling (MB) for the one-shot `kvb_all` k/v reconstruction buffer in prefill attention (#768 β 30.1 GB at ctx 262144, and `cap_for_ram` reserved it permanently). Above the ceiling the reconstruction is tiled with an online (flash-style) softmax: same rebuild total, ~tile-sized transient, output may differ from one-shot by rounding (same divergence class as the CUDA/Metal attention arms). `=0` disables tiling (always one-shot). DSA-selected rows always take the one-shot path. |+| `KVB_TILE_MB` | `512` | Tile size (MB) for the tiled reconstruction above. |+| `KVB_FLASH` | unset | `=1` forces the tiled path at any size, `=0` forces one-shot β overrides the `KVB_FLASH_MB` trigger (A/B switch). || `COLI_GROUP_ASYNC` | `0` (off) | `=1` issues and collects CUDA expert groups asynchronously so CPU and GPU overlap at decode (Sβ€4). || `COLI_DISKCLASS_WINDOW` | see source | Recency window (in ticks) for the DISK-CLASS heat statistic. || `URING` | `0` (off) | Linux-only queued expert I/O. `URING=1` implies `PIPE=1`, forces cold reads through io-wq (`IOSQE_ASYNC`), replaces blocking loader pthreads and spin waits with batched SQEs/CQEs, and batches `PILOT_REAL` loads on a separate ring. Use `DIRECT=1` for cold NVMe to avoid page-cache copy/readahead limits. Fails clearly if the kernel denies io_uring; incompatible with `COLI_MMAP=1`. |