Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

LOOKA: routing predictability counters; PILOT: router-lookahead disk prefetch (async I/O thread); cap auto-raise to RAM budget (#12)

JustVugg077894210e1a
diff --git a/README.md b/README.md
index 53fa86d3..ea523564 100644
--- a/README.md
+++ b/README.md
@@ -91,7 +91,7 @@ The engine at runtime is pure C — python is only used by the one-time converte
### Experimental resident CUDA backend
-This fork includes an opt-in CUDA backend for model-resident tensors. Streaming
+colibrì includes an opt-in CUDA backend for model-resident tensors. Streaming
experts deliberately remain on the original CPU path for now: copying an expert
from NVMe to the GPU on every use would only replace the disk bottleneck with a
PCIe bottleneck. Resident quantized tensors are uploaded lazily once and reused.
@@ -153,7 +153,11 @@ The fixture has random weights and is not a language model. It exists only to
preserve the real MLA/MoE/streaming shapes and compare CPU streaming, dense-only
CUDA, CPU hot-store, and CUDA hot-expert execution with identical replay tokens.
-Useful knobs (env or flags): `--temp T` token sampling temperature (default 0.7 + nucleus 0.90 — tuned for int4; 0 = greedy), `--topp 0.7` adaptive expert top-p (30–40% less disk), `--ngen N` max tokens per answer (`:piu` in chat continues a truncated one), `AUTOPIN=0` disable the learning cache's auto-pin, `THINK=1` enable GLM-5.2's reasoning block, `DRAFT=n` MTP draft depth, `TF=1` teacher-forcing validation.
+Useful knobs (env or flags): `--temp T` token sampling temperature (default 0.7 + nucleus 0.90 — tuned for int4; 0 = greedy), `--topp 0.7` adaptive expert top-p (30–40% less disk), `--ngen N` max tokens per answer (`:piu` in chat continues a truncated one), `AUTOPIN=0` disable the learning cache's auto-pin, `THINK=1` enable GLM-5.2's reasoning block, `DRAFT=n` MTP draft depth, `TF=1` teacher-forcing validation, `PILOT=1` router-lookahead disk prefetch (experimental — see below), `CAP_RAISE=0` don't auto-grow the expert cache.
+
+**The expert cache auto-sizes to your RAM** (since 2026-07-10): the engine now *raises* the LRU cap to fill your `--ram` budget instead of only lowering it. Before this fix a 128 GB machine ran with the same 8-experts/layer cache as a 16 GB one (issue #12) — **if you benchmarked colibrì before this date, rerun: your numbers were capped.**
+
+**Router-lookahead prefetch** (`PILOT=1`, experimental): GLM-5.2's expert routing is measurably predictable *ahead of time* — applying layer L+1's router to layer L's post-attention state recalls **71.6%** of the true top-8 (vs 41.3% for "same experts as last token"). `PILOT=1` uses this to issue next-layer expert readahead from a dedicated I/O thread while the current layer computes. On our dev box the disk is already ~80% saturated, so it measures neutral; on machines where compute and disk are balanced (like the Ryzen AI 9 in issue #12: 43% disk / 46% matmul) it should overlap real work — measurements welcome.
**The learning cache**: the engine records which experts your usage actually routes to (`.coli_usage` next to the model, updated every turn) and at startup automatically pins the hottest ones in spare RAM. colibrì literally gets faster the more you use it.
diff --git a/c/glm.c b/c/glm.c
index c4ef8636..3eefe362 100644
--- a/c/glm.c
+++ b/c/glm.c
@@ -23,6 +23,8 @@
#include <math.h>
#include <time.h>
#include <limits.h>
+#include <pthread.h> /* thread I/O del PILOTA */
+#include <unistd.h>
#include <sys/resource.h>
#if defined(__APPLE__) || defined(__linux__)
#include <sys/mman.h> /* mlock: inchioda le pagine in RAM / wire pages into RAM */
@@ -537,6 +539,18 @@ static int g_draft=0; /* metodo E: DRAFT=n token auto-speculati per forward v
* misurato sul run reale (2026-07-03) acceptance ~5% -> ogni draft
* rifiutato paga comunque i suoi expert dal disco = ~3x piu' lento.
* Opt-in (DRAFT=4) per testi ripetitivi dove l'acceptance e' alta. */
+static int g_looka=0; /* LOOKA=1: misura (solo contatori, zero effetti) quanto il routing MoE
+ * e' predicibile IN ANTICIPO — la domanda che decide se un prefetch
+ * pilotato dal router puo' riempire i tempi morti del disco.
+ * [0] token precedente, stesso layer (cio' che usa gia' SPEC/PREFETCH)
+ * [1] ingresso del layer -> routing dello STESSO layer (salta l'attention)
+ * [2] post-attention del layer L -> routing di L+1 (un residuo MoE e
+ * un'attention di anticipo: il punto dove il prefetch avrebbe
+ * un intero giro di disco per lavorare in ombra). */
+static int64_t la_hit[3], la_tot[3];
+static int la_pred[2][130][16]; static signed char la_val[2][130];
+static int g_pilot=0; /* PILOT=1: prefetch pilotato dal router (vedi pilot_prefetch) */
+static int g_pilot_k=8; /* PILOT_K=k: prefetcha solo le prime k predizioni per posizione */
/* sceglie il formato da `bits`: >=16 f32, 5..8 int8, <=4 int4-packed */
static void qt_alloc(QT *t, int O, int I, int bits){
t->O=O; t->I=I; t->qf=NULL; t->q8=NULL; t->q4=NULL; t->s=NULL;
@@ -1130,6 +1144,19 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out){
for(int kk=0;kk<Ke;kk++) w[kk]*=c->routed_scale;
for(int d=0;d<D;d++) out[(int64_t)s*D+d]=0;
}
+ if(g_looka && S==1 && layer<c->n_layers){
+ int Ke=keff[0];
+ if(m->enr[layer]>0){ /* [0] vs routing del token precedente */
+ for(int kk=0;kk<Ke;kk++) for(int z=0;z<m->enr[layer];z++)
+ if(m->eroute[layer][z]==idxs[kk]){ la_hit[0]++; break; }
+ la_tot[0]+=Ke;
+ }
+ for(int kind=0;kind<2;kind++) if(la_val[kind][layer]){ /* [1]/[2] vs predizioni */
+ for(int kk=0;kk<Ke;kk++) for(int z=0;z<K;z++)
+ if(la_pred[kind][layer][z]==idxs[kk]){ la_hit[1+kind]++; break; }
+ la_tot[1+kind]+=Ke; la_val[kind][layer]=0;
+ }
+ }
m->enr[layer]=keff[S-1]; for(int kk=0;kk<keff[S-1];kk++) m->eroute[layer][kk]=idxs[(int64_t)(S-1)*K+kk];
/* ---- FASE B: union degli expert del batch ---- */
int *uniq=malloc((size_t)E*sizeof(int)); int nu=0;
@@ -1212,14 +1239,89 @@ static void dense_mlp(Layer *l, float *x, int S, int D, int I, float *out){
free(g); free(u);
}
+/* LOOKA: predice il top-K del router del layer `target` dallo stato h (residual stream),
+ * usando la STESSA pipeline del routing vero (post_ln -> router -> sigmoid+bias, top-K).
+ * kind 0 = stesso layer saltando l'attention, kind 1 = layer successivo. */
+static void la_predict(Model *m, int target, const float *h, int kind){
+ Cfg *c=&m->c; Layer *l=&m->L[target]; int D=c->hidden, E=c->n_experts, K=c->topk;
+ float *nrm=falloc(D), *ch=falloc(E);
+ rmsnorm(nrm,h,l->post_ln,D,c->eps);
+ matmul(ch,nrm,l->router,1,D,E);
+ for(int e=0;e<E;e++) ch[e]=sigmoidf(ch[e])+l->router_bias[e];
+ int *pred=la_pred[kind][target];
+ for(int kk=0;kk<K;kk++){ int best=-1; float bv=-1e30f;
+ for(int e=0;e<E;e++){ int tk=0; for(int j=0;j<kk;j++) if(pred[j]==e){tk=1;break;}
+ if(!tk && ch[e]>bv){bv=ch[e];best=e;} }
+ pred[kk]=best; }
+ la_val[kind][target]=1;
+ free(nrm); free(ch);
+}
+
+/* PILOTA: prefetch guidato dal router. Predice il top-K del layer L+1 dallo stato
+ * post-attention di L (recall misurato 71.6% su GLM-5.2, vs 41.3% del token precedente)
+ * e lancia il WILLNEED degli expert mancanti MENTRE il MoE di L legge i suoi: il disco
+ * lavora nei tempi morti del calcolo invece di aspettare il routing vero. Con MTP attiva
+ * predice per TUTTE le posizioni del draft: la speculazione pilota anche l'I/O.
+ * PILOT_K limita alle prime k predizioni (la testa del ranking e' piu' affidabile
+ * della coda: meno banda sprecata sulle predizioni sbagliate).
+ *
+ * I WILLNEED partono da un THREAD I/O dedicato: con la coda disco satura la submit
+ * del fadvise BLOCCA (~0.5ms x 169k chiamate = +92s/48 token, misurato) — inline
+ * il pilota costava piu' di quanto rendesse. Ring lock-free 1P/1C; pieno = scarta
+ * (un hint perso non e' un errore). */
+static struct { int l,e; } pilot_q[4096];
+static volatile unsigned pilot_w=0, pilot_r=0;
+static Model *pilot_m=NULL;
+static void *pilot_worker(void *arg){
+ (void)arg;
+ for(;;){
+ unsigned r=__atomic_load_n(&pilot_r,__ATOMIC_ACQUIRE);
+ unsigned w=__atomic_load_n(&pilot_w,__ATOMIC_ACQUIRE);
+ if(r==w){ usleep(200); continue; }
+ expert_prefetch(pilot_m, pilot_q[r&4095].l, pilot_q[r&4095].e);
+ __atomic_store_n(&pilot_r,r+1,__ATOMIC_RELEASE);
+ }
+ return NULL;
+}
+static void pilot_prefetch(Model *m, int lnext, const float *x, int S){
+ Cfg *c=&m->c; Layer *l=&m->L[lnext]; int D=c->hidden, E=c->n_experts;
+ int K = g_pilot_k<c->topk ? g_pilot_k : c->topk;
+ if(!pilot_m){ pilot_m=m; pthread_t t; pthread_create(&t,NULL,pilot_worker,NULL); }
+ float *nrm=falloc(D), *ch=falloc(E);
+ for(int s=0;s<S;s++){
+ rmsnorm(nrm, x+(int64_t)s*D, l->post_ln, D, c->eps);
+ matmul(ch, nrm, l->router, 1, D, E);
+ for(int e=0;e<E;e++) ch[e]=sigmoidf(ch[e])+l->router_bias[e];
+ for(int kk=0;kk<K;kk++){
+ int best=0; for(int e=1;e<E;e++) if(ch[e]>ch[best]) best=e;
+ ch[best]=-2e30f;
+ int found=0; ESlot *P=m->pin[lnext];
+ for(int z=0;z<m->npin[lnext] && !found;z++) if(P[z].eid==best) found=1;
+ ESlot *Sl=m->ecache[lnext];
+ for(int z=0;z<m->ecn[lnext] && !found;z++) if(Sl[z].eid==best) found=1;
+ if(!found){
+ unsigned w=__atomic_load_n(&pilot_w,__ATOMIC_RELAXED);
+ if(w-__atomic_load_n(&pilot_r,__ATOMIC_ACQUIRE)<4096){
+ pilot_q[w&4095].l=lnext; pilot_q[w&4095].e=best;
+ __atomic_store_n(&pilot_w,w+1,__ATOMIC_RELEASE);
+ }
+ }
+ }
+ }
+ free(nrm); free(ch);
+}
+
/* forward di UN layer (usato dai 78 principali e dal layer MTP) */
static void layer_forward(Model *m, Layer *l, int li, float *x, int S, int pos_base, float *nrm, float *tmp){
Cfg *c=&m->c; int D=c->hidden;
if(g_spec && g_prefetch && l->sparse && m->enr[li]>0)
for(int z=0;z<m->enr[li];z++) expert_prefetch(m,li,m->eroute[li][z]);
+ if(g_looka && S==1 && li<c->n_layers && l->sparse) la_predict(m,li,x,0);
for(int s=0;s<S;s++) rmsnorm(nrm+(int64_t)s*D, x+(int64_t)s*D, l->in_ln, D, c->eps);
attention(m,l,li,nrm,S,pos_base,tmp);
for(int64_t j=0;j<(int64_t)S*D;j++) x[j]+=tmp[j];
+ if(g_pilot && S<=8 && li+1<c->n_layers && m->L[li+1].sparse) pilot_prefetch(m,li+1,x,S);
+ if(g_looka && S==1 && li+1<c->n_layers && m->L[li+1].sparse) la_predict(m,li+1,x,1);
for(int s=0;s<S;s++) rmsnorm(nrm+(int64_t)s*D, x+(int64_t)s*D, l->post_ln, D, c->eps);
if(l->sparse) moe(m,l,li,nrm,S,tmp); else dense_mlp(l,nrm,S,D,c->dense_inter,tmp);
for(int64_t j=0;j<(int64_t)S*D;j++) x[j]+=tmp[j];
@@ -1628,6 +1730,12 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){
if(g_cuda_enabled) cuda_stats_print();
#endif
profile_print(m,dt);
+ if(g_looka){
+ const char *nm[3]={"token precedente (=SPEC prefetch)","ingresso layer, salto attention","layer successivo (1 giro di anticipo)"};
+ printf("LOOKAHEAD routing — recall degli expert veri nel top-8 predetto:\n");
+ for(int i=0;i<3;i++) printf(" %-38s %5.1f%% (%lld/%lld)\n", nm[i],
+ la_tot[i]?100.0*la_hit[i]/la_tot[i]:0.0, (long long)la_hit[i], (long long)la_tot[i]);
+ }
free(pids); free(all);
usage_save(m);
}
@@ -2012,8 +2120,26 @@ static void cap_for_ram(Model *m, double ram_gb, int ebits, int max_ctx){
(m->resident_bytes + (double)capmax*nsp*eb + slack)/1e9);
m->ecap=capmax;
} else {
- fprintf(stderr,"[RAM_GB=%.1f%s] cap=%d ok (proiezione picco %.1f GB)\n", ram_gb, auto_b?" auto":"", m->ecap,
- (m->resident_bytes + (double)m->ecap*nsp*eb + slack)/1e9);
+ /* AUTO-RAISE (issue #12): il budget consente PIU' cache di quella chiesta.
+ * Senza questo, una macchina da 128 GB girava con la LRU di una da 16
+ * (cap=8 di default in coli): hit 23-28% con decine di GB inutilizzati.
+ * Tetto a n_experts: oltre, ogni layer avrebbe slot che non puo' riempire.
+ * CAP_RAISE=0 ripristina il comportamento fisso. */
+ int raise_on = getenv("CAP_RAISE")?atoi(getenv("CAP_RAISE")):1;
+ int newcap = capmax>c->n_experts ? c->n_experts : capmax;
+ if(raise_on && newcap>m->ecap){
+ for(int i=0;i<=c->n_layers;i++) if(m->ecache[i]){
+ m->ecache[i]=realloc(m->ecache[i],(size_t)newcap*sizeof(ESlot));
+ memset(m->ecache[i]+m->ecap,0,(size_t)(newcap-m->ecap)*sizeof(ESlot));
+ }
+ fprintf(stderr,"[RAM_GB=%.1f%s] cap ALZATO %d->%d: il budget lo consente "
+ "(proiezione picco %.1f GB; CAP_RAISE=0 per disattivare)\n",
+ ram_gb, auto_b?" auto":"", m->ecap, newcap,
+ (m->resident_bytes + (double)newcap*nsp*eb + slack)/1e9);
+ m->ecap=newcap;
+ } else
+ fprintf(stderr,"[RAM_GB=%.1f%s] cap=%d ok (proiezione picco %.1f GB)\n", ram_gb, auto_b?" auto":"", m->ecap,
+ (m->resident_bytes + (double)m->ecap*nsp*eb + slack)/1e9);
}
}
@@ -2029,6 +2155,10 @@ int main(int argc, char **argv){
g_mlock = getenv("MLOCK")?atoi(getenv("MLOCK")):-1; /* -1 auto (ON macOS), 0 off, 1 force / auto (ON macOS), 0 off, 1 force */
g_spec = getenv("SPEC")?atoi(getenv("SPEC")):1;
g_draft = getenv("DRAFT")?atoi(getenv("DRAFT")):-1; /* -1 = auto: 3 se MTP, 0 senza */
+ g_looka = getenv("LOOKA")?atoi(getenv("LOOKA")):0; /* 1 = misura predicibilita' routing */
+ g_pilot = getenv("PILOT")?atoi(getenv("PILOT")):0; /* 1 = prefetch pilotato dal router */
+ g_pilot_k = getenv("PILOT_K")?atoi(getenv("PILOT_K")):8;
+ if(g_pilot_k<1) g_pilot_k=1;
g_direct = getenv("DIRECT")?atoi(getenv("DIRECT")):0;
g_idot = getenv("IDOT")?atoi(getenv("IDOT")):1; /* 0 = kernel f32 esatti (A/B) */
g_repin = getenv("REPIN")?atoi(getenv("REPIN")):0; /* RFC: re-pin ogni n token emessi (0=off) / live re-pin every n emitted tokens (0=off) */
@@ -2165,6 +2295,12 @@ int main(int argc, char **argv){
m.gpu_expert_count,m.gpu_expert_bytes/1e9,(unsigned long long)m.gpu_expert_calls);
if(g_cuda_enabled) cuda_stats_print();
#endif
+ if(g_looka){
+ const char *nm[3]={"token precedente (=SPEC prefetch)","ingresso layer, salto attention","layer successivo (1 giro di anticipo)"};
+ printf("LOOKAHEAD routing — recall degli expert veri nel top-8 predetto:\n");
+ for(int i=0;i<3;i++) printf(" %-38s %5.1f%% (%lld/%lld)\n", nm[i],
+ la_tot[i]?100.0*la_hit[i]/la_tot[i]:0.0, (long long)la_hit[i], (long long)la_tot[i]);
+ }
if(stats) stats_dump(&m,stats);
return 0;
}