Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
REPIN follows live session heat (decaying heat map; .coli_usage stays the persistent signal) + disk→RAM→VRAM promotion (#26)
ZacharyZcR1453dab7aeeb
diff --git a/README.md b/README.mdindex 2bf9eb5b..5d2237c1 100644--- a/README.md+++ b/README.md@@ -211,7 +211,7 @@ works against the colibrì OpenAI-compatible server (in review, #21) or any othecompatible endpoint. Nothing leaves the endpoint you configure. The terminal`coli chat` remains the first-class interface.-Useful knobs (env or flags): `--temp T` token sampling temperature (default 0.7 + nucleus 0.90 — tuned for int4; 0 = greedy), `--topp 0.7` adaptive expert top-p (30–40% less disk), `--ngen N` max tokens per answer (`:piu` in chat continues a truncated one), `AUTOPIN=0` disable the learning cache's auto-pin, `THINK=1` enable GLM-5.2's reasoning block, `DRAFT=n` MTP draft depth, `TF=1` teacher-forcing validation, `PILOT=1` router-lookahead disk prefetch (experimental — see below), `CAP_RAISE=0` don't auto-grow the expert cache.+Useful knobs (env or flags): `--temp T` token sampling temperature (default 0.7 + nucleus 0.90 — tuned for int4; 0 = greedy), `--topp 0.7` adaptive expert top-p (30–40% less disk), `--ngen N` max tokens per answer (`:piu` in chat continues a truncated one), `--repin N` adapt RAM/VRAM hot experts every N emitted tokens, `AUTOPIN=0` disable the learning cache's auto-pin, `THINK=1` enable GLM-5.2's reasoning block, `DRAFT=n` MTP draft depth, `TF=1` teacher-forcing validation, `PILOT=1` router-lookahead disk prefetch (experimental — see below), `CAP_RAISE=0` don't auto-grow the expert cache.**The expert cache auto-sizes to your RAM** (since 2026-07-10): the engine now *raises* the LRU cap to fill your `--ram` budget instead of only lowering it. Before this fix a 128 GB machine ran with the same 8-experts/layer cache as a 16 GB one (issue #12) — **if you benchmarked colibrì before this date, rerun: your numbers were capped.**@@ -219,6 +219,12 @@ Useful knobs (env or flags): `--temp T` token sampling temperature (default 0.7**The learning cache**: the engine records which experts your usage actually routes to (`.coli_usage` next to the model, updated every turn) and at startup automatically pins the hottest ones in spare RAM. colibrì literally gets faster the more you use it.+**Live tier adaptation** (`--repin N`, opt-in): at safe turn boundaries, a decaying+session heat map replaces cold pinned experts with hotter streamed experts. Replacement+loads the expert from disk into the existing RAM slot; GPU-backed slots immediately+refresh the same VRAM tier budget. A 25% hysteresis and a four-swap limit prevent tier+thrashing. Persistent `.coli_usage` remains the long-term signal and is not decayed.+**Conversations reopen warm** (`.coli_kv`, since 2026-07-10): `coli chat` persists the compressed MLA KV-cache to disk after every turn (~182 KB/token, appended incrementally, crash-safe). Close the chat, reopen it tomorrow — the model still remembers the whole conversation and **zero re-prefill happens**: validated byte-identical to an uninterrupted session. `:reset` clears it, `KVSAVE=0` disables it.## Got a better machine? Try it — here's what to expectdiff --git a/c/Makefile b/c/Makefileindex 5921c5cf..6fe3ab47 100644--- a/c/Makefile+++ b/c/Makefile@@ -37,7 +37,7 @@ CUDA_ARCH ?= nativeNVCCFLAGS ?= -O3 -std=c++17 -arch=$(CUDA_ARCH) -Xcompiler=-Wall,-WextraPYTHON ?= python3CUDA_OBJ =-TEST_BINS = tests/test_json tests/test_st+TEST_BINS = tests/test_json tests/test_st tests/test_tierifeq ($(CUDA),1)ifeq ($(UNAME_S),Darwin)$(error CUDA=1 is supported only on Linux)@@ -74,6 +74,9 @@ tests/test_json: tests/test_json.c json.htests/test_st: tests/test_st.c st.h json.h compat.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+tests/test_tier: tests/test_tier.c tier.h+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+test-c: $(TEST_BINS)@for test in $(TEST_BINS); do ./$$test || exit 1; donediff --git a/c/coli b/c/coliindex 052d8570..b2f2a031 100755--- a/c/coli+++ b/c/coli@@ -14,6 +14,7 @@ CLI per far girare GLM-5.2 (744B) in locale, su CPU, in ~15-26 GB di RAM.Config via env o flag (validi anche dopo il sottocomando):COLI_MODEL=<dir> modello (default /home/vincenzo/glm52_i4)--ram N budget RAM in GB (auto-cap cache expert)+ --repin N adatta gli expert RAM/VRAM ogni N token--topp P top-p adattivo sugli expert --topk N top-k fisso--ngen N token massimi per risposta --cap N slot cache/layer"""@@ -104,6 +105,7 @@ def env_for(a):if a.topp: e["TOPP"]=str(a.topp)if a.topk: e["TOPK"]=str(a.topk)if a.temp is not None: e["TEMP"]=str(a.temp) # 0 = greedy; default motore: 1.0 + nucleus 0.95+ if a.repin: e["REPIN"]=str(a.repin)return e# ---------- rendering markdown in STREAMING per il terminale ----------@@ -426,6 +428,7 @@ def cmd_convert(a):def main():common=argparse.ArgumentParser(add_help=False)common.add_argument("--model", default=DEF_MODEL); common.add_argument("--ram", type=int, default=0) # 0 = auto (il motore usa l'88% della RAM disponibile)+ common.add_argument("--repin", type=int, default=0, help="adatta gli expert RAM/VRAM ogni N token")common.add_argument("--cap", type=int, default=8); common.add_argument("--ngen", type=int, default=1024) # rete di sicurezza: la fine vera la decidono gli stop tokencommon.add_argument("--topp", type=float, default=0); common.add_argument("--topk", type=int, default=0)common.add_argument("--temp", type=float, default=None) # temperatura token (0=greedy, default 1.0+nucleus .95)diff --git a/c/glm.c b/c/glm.cindex 95f35319..b690861e 100644--- a/c/glm.c+++ b/c/glm.c@@ -31,6 +31,7 @@#endif#include "st.h"#include "tok.h"+#include "tier.h"#ifdef COLI_CUDA#include <omp.h>#include "backend_cuda.h"@@ -112,7 +113,8 @@ typedef struct {ESlot **ecache; int *ecn; int ecap; /* LRU expert per-layer */ESlot ws[64]; /* working set del layer corrente (load paralleli) */ESlot **pin; int *npin; /* HOT-STORE: expert pinnati in RAM (mai evicted) */- uint32_t **eusage; /* contatori uso expert per layer (per STATS/PIN) */+ uint32_t **eusage; /* contatori persistenti (per STATS/PIN) */+ uint32_t **eheat; /* calore recente per promotion/demotion live *//* DSA lightning indexer (attivo solo se i pesi out-idx-* sono presenti) */int has_dsa;QT *ix_wq, *ix_wk, *ix_wp; /* per layer FULL: wq_b, wk, weights_proj */@@ -708,7 +710,7 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbitsm->ecap=cap; m->ecache=calloc(NR,sizeof(ESlot*)); m->ecn=calloc(NR,sizeof(int));m->eroute=calloc(NR,sizeof(int*)); m->enr=calloc(NR,sizeof(int));m->pin=calloc(NR,sizeof(ESlot*)); m->npin=calloc(NR,sizeof(int));- m->eusage=calloc(NR,sizeof(uint32_t*));+ m->eusage=calloc(NR,sizeof(uint32_t*)); m->eheat=calloc(NR,sizeof(uint32_t*));m->kv_start=calloc(NR,sizeof(int));for(int i=0;i<c->n_layers;i++){Layer *l=&m->L[i];@@ -737,6 +739,7 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbitsm->ecache[i]=calloc(cap,sizeof(ESlot));m->eroute[i]=calloc(c->topk,sizeof(int)); /* metodo C: ultimo routing del layer */m->eusage[i]=calloc(c->n_experts,sizeof(uint32_t));+ m->eheat[i]=calloc(c->n_experts,sizeof(uint32_t));}#undef P}@@ -781,6 +784,7 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbitsm->ecache[i]=calloc(cap,sizeof(ESlot));m->eroute[i]=calloc(c->topk,sizeof(int));m->eusage[i]=calloc(c->n_experts,sizeof(uint32_t));+ m->eheat[i]=calloc(c->n_experts,sizeof(uint32_t));m->kv_start[i]=-1; /* KV MTP: parte dalla prima posizione di decode */#undef PM}@@ -1157,7 +1161,10 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out){float cum=0; for(int kk=0;kk<Ksel;kk++){ cum+=w[kk]; if(cum>=g_topp*tot){ Ke=kk+1; break; } }}keff[s]=Ke; m->ereq+=Ke;- for(int kk=0;kk<Ke;kk++) m->eusage[layer][idx[kk]]++;+ for(int kk=0;kk<Ke;kk++){+ m->eusage[layer][idx[kk]]++;+ if(m->eheat[layer][idx[kk]]<UINT32_MAX) m->eheat[layer][idx[kk]]++;+ }if(c->norm_topk){ float sm=0; for(int kk=0;kk<Ke;kk++) sm+=w[kk]; sm+=1e-20f; for(int kk=0;kk<Ke;kk++) w[kk]/=sm; }for(int kk=0;kk<Ke;kk++) w[kk]*=c->routed_scale;for(int d=0;d<D;d++) out[(int64_t)s*D+d]=0;@@ -1768,31 +1775,24 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){* Upstream fa AUTOPIN allo START (dalla storia .coli_usage). Questo aggiunge un re-pin* TRA I TURNI: nel punto sicuro dopo la risposta scambia i pin peggiori con i non-pinnati* piu' caldi, cosi' l'hot-store insegue il carico VIVO senza un profilo a parte. Isteresi- * 25% (+4) contro il ping-pong; max 4 scambi/passata (~20 MB di disco l'uno). Usa eusage- * cumulativo (l'aging LFU del mio fork non e' incluso qui: vedi PR.md).+ * 25% (+4) contro il ping-pong; max 4 scambi/passata (~20 MB di disco l'uno). Una heat+ * map separata decade a ogni passata: la storia persistente .coli_usage resta intatta.* EN: upstream AUTOPINs at START (from .coli_usage). This adds a between-turns re-pin: at* the safe point after the reply, swap the worst pins for the hottest unpinned, so the* hot-store tracks the LIVE workload without a separate profile. 25% (+4) hysteresis vs- * ping-pong; max 4 swaps/pass (~20 MB disk each). Uses cumulative eusage (my fork's LFU- * aging is NOT included here: see PR.md). */+ * ping-pong; max 4 swaps/pass (~20 MB disk each). A separate decaying heat map keeps+ * persistent .coli_usage intact while adapting to the current workload. */static int g_repin=0;static uint64_t g_last_repin=0;typedef struct { long gain; int l, slot, eid; } RepinCand;static int repin_pick(Model *m, RepinCand *out, int maxc){Cfg *c=&m->c; int nb=0;for(int l=0;l<c->n_layers;l++){- if(!m->npin || m->npin[l]<1 || !m->eusage[l]) continue;- uint32_t *u=m->eusage[l]; ESlot *P=m->pin[l];- int zp=0; for(int z=1;z<m->npin[l];z++) if(u[P[z].eid]<u[P[zp].eid]) zp=z; /* pin piu' freddo / coldest pin */- int eu=-1; uint32_t fu=0;- for(int e=0;e<c->n_experts;e++){- int pinned=0; for(int z=0;z<m->npin[l];z++) if(P[z].eid==e){pinned=1;break;}- if(!pinned && u[e]>fu){ fu=u[e]; eu=e; } /* non-pin piu' caldo / hottest unpinned */- }- if(eu<0) continue;- uint32_t fp=u[P[zp].eid];- if(fu <= fp + (fp>>2) + 4) continue; /* isteresi 25% / hysteresis */- long g=(long)fu-(long)fp;+ if(!m->npin || m->npin[l]<1 || !m->eheat[l]) continue;+ ESlot *P=m->pin[l]; int ids[4096], zp, eu; long g;+ int np=m->npin[l]; if(np>4096) np=4096;+ for(int z=0;z<np;z++) ids[z]=P[z].eid;+ if(!tier_pick_swap(m->eheat[l],c->n_experts,ids,np,&zp,&eu,&g)) continue;if(nb<maxc){ out[nb].gain=g; out[nb].l=l; out[nb].slot=zp; out[nb].eid=eu; nb++; }else { int w=0; for(int b=1;b<maxc;b++) if(out[b].gain<out[w].gain) w=b;if(g>out[w].gain){ out[w].gain=g; out[w].l=l; out[w].slot=zp; out[w].eid=eu; } }@@ -1805,13 +1805,37 @@ static void repin_pass(Model *m){g_last_repin = m->n_emit;RepinCand cd[4]; int nb=repin_pick(m,cd,4);for(int b=0;b<nb;b++){- int old=m->pin[cd[b].l][cd[b].slot].eid;+ ESlot *s=&m->pin[cd[b].l][cd[b].slot];+ int old=s->eid;+ uint32_t old_heat=m->eheat[cd[b].l][old], new_heat=m->eheat[cd[b].l][cd[b].eid];+#ifdef COLI_CUDA+ int gpu=s->g.cuda_eligible;+ int64_t old_gpu=gpu ? (int64_t)coli_cuda_tensor_bytes(s->g.cuda)+ +(int64_t)coli_cuda_tensor_bytes(s->u.cuda)+ +(int64_t)coli_cuda_tensor_bytes(s->d.cuda) : 0;+#endifdouble t0=now_s();- expert_load(m, cd[b].l, cd[b].eid, &m->pin[cd[b].l][cd[b].slot]);- fprintf(stderr,"[REPIN] layer %d: esce/out %d (f=%u) <- entra/in %d (f=%u) in %.0f ms\n",- cd[b].l, old, m->eusage[cd[b].l][old], cd[b].eid, m->eusage[cd[b].l][cd[b].eid],- (now_s()-t0)*1e3);+ expert_load(m,cd[b].l,cd[b].eid,s); /* disk -> RAM, same resident slot */+ const char *tier="RAM";+#ifdef COLI_CUDA+ if(gpu){ /* refresh the same VRAM slot now, not lazily */+ if(qt_cuda_upload(&s->g) && qt_cuda_upload(&s->u) && qt_cuda_upload(&s->d)){+ int64_t now_gpu=(int64_t)coli_cuda_tensor_bytes(s->g.cuda)+ +(int64_t)coli_cuda_tensor_bytes(s->u.cuda)+ +(int64_t)coli_cuda_tensor_bytes(s->d.cuda);+ m->gpu_expert_bytes+=now_gpu-old_gpu; tier="VRAM";+ } else {+ qt_cuda_reset(&s->g); qt_cuda_reset(&s->u); qt_cuda_reset(&s->d);+ s->g.cuda_eligible=s->u.cuda_eligible=s->d.cuda_eligible=0;+ m->gpu_expert_count--; m->gpu_expert_bytes-=old_gpu;+ fprintf(stderr,"[REPIN] upload VRAM fallito; slot degradato a RAM\n");+ }+ }+#endif+ fprintf(stderr,"[REPIN] %s layer %d: esce/out %d (heat=%u) <- entra/in %d (heat=%u) in %.0f ms\n",+ tier,cd[b].l,old,old_heat,cd[b].eid,new_heat,(now_s()-t0)*1e3);}+ for(int l=0;l<m->c.n_layers;l++) if(m->eheat[l]) tier_decay(m->eheat[l],m->c.n_experts);}/* ---- KV SU DISCO: la conversazione si riapre CALDA (KVSAVE=0 disattiva) ----* Il re-prefill di una chat riaperta costa ore su questo disco; la KV compressa MLAdiff --git a/c/tests/test_tier.c b/c/tests/test_tier.cnew file mode 100644index 00000000..9b689810--- /dev/null+++ b/c/tests/test_tier.c@@ -0,0 +1,22 @@+#include <stdio.h>+#include "../tier.h"++static int fail(const char *message){+ fprintf(stderr,"tier test failed: %s\n",message);+ return 1;+}++int main(void){+ uint32_t heat[6]={20,2,8,3,30,1};+ int pinned[2]={0,1}, slot=-1, eid=-1; long gain=0;+ if(!tier_pick_swap(heat,6,pinned,2,&slot,&eid,&gain)) return fail("hot expert not promoted");+ if(slot!=1 || eid!=4 || gain!=28) return fail("wrong promotion candidate");++ uint32_t stable[4]={20,18,24,4}; int resident[2]={0,1};+ if(tier_pick_swap(stable,4,resident,2,&slot,&eid,&gain)) return fail("hysteresis did not block churn");++ tier_decay(heat,6);+ if(heat[0]!=10 || heat[1]!=1 || heat[4]!=15) return fail("heat decay");+ puts("tier tests: ok");+ return 0;+}diff --git a/c/tier.h b/c/tier.hnew file mode 100644index 00000000..8f66cdd4--- /dev/null+++ b/c/tier.h@@ -0,0 +1,31 @@+#ifndef COLIBRI_TIER_H+#define COLIBRI_TIER_H++#include <stdint.h>++/* Pick one RAM/VRAM hot-store slot to replace from recent routing heat.+ * The fixed margin handles tiny samples; the 25% margin prevents ping-pong. */+static int tier_pick_swap(const uint32_t *heat, int nexpert,+ const int *pinned, int npin,+ int *slot, int *eid, long *gain){+ if(!heat || !pinned || npin<1 || nexpert<1) return 0;+ int cold=0;+ for(int z=1;z<npin;z++) if(heat[pinned[z]]<heat[pinned[cold]]) cold=z;+ int hot=-1; uint32_t fh=0;+ for(int e=0;e<nexpert;e++){+ int resident=0;+ for(int z=0;z<npin;z++) if(pinned[z]==e){ resident=1; break; }+ if(!resident && heat[e]>fh){ fh=heat[e]; hot=e; }+ }+ if(hot<0) return 0;+ uint32_t fc=heat[pinned[cold]];+ if(fh<=fc+(fc>>2)+4) return 0;+ *slot=cold; *eid=hot; *gain=(long)fh-(long)fc;+ return 1;+}++static void tier_decay(uint32_t *heat, int nexpert){+ for(int e=0;e<nexpert;e++) heat[e]>>=1;+}++#endif