Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
pilot: LFRU-aware eviction guard β never evict a warm demand expert for a speculation (#441)
cdhdteaa68f4005fb
diff --git a/c/colibri.c b/c/colibri.cindex 4524c105..97ae7214 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -733,6 +733,11 @@ static int g_pilot_real=0;/* PILOT_REAL=1: il pilota fa LOAD VERI cross-layer destatic int g_pilot_two=0; /* PILOT_TWO=1: two-step prefetch β before running L+1's router,* approximate MoE(L) using only the shared expert (resident, no disk)* and add it to the state. Trades 3 small matmuls for +2.3% recall. */+static int g_pilot_evict_guard=1;/* PILOT_EVICT_GUARD=0 -> old behavior (a speculation evicts the plain LRU).+ * Default ON: a speculative pilot load may evict a RESIDENT expert only if the+ * predicted expert is historically HOTTER than the victim (same LFRU hysteresis as+ * tier_pick_lfru); otherwise it drops the speculation rather than thrash a warm+ * demand-loaded expert. Cache placement only -> output byte-identical. (#441) *//* Handshake main<->pilota per il load-vero cross-layer. Invariante di sicurezza in DUE parti:* 1) Percorso MATMUL (moe): il pilota scrive SOLO ecache[layer] con layer > g_cur_moe_layer;* il matmul in moe() legge SOLO ecache[layer]==g_cur_moe_layer, e la barriera a inizio moe()@@ -3427,7 +3432,16 @@ static void pilot_realload(Model *m, int layer, int eid){for(int z=0;z<nn;z++) if(Sl[z].eid==eid){ pthread_mutex_unlock(&g_pilot_mx); return; }int slot,isnew; /* cresci se c'e' posto, altrimenti LRU */if(nn<m->ecap){ slot=nn; isnew=1; }- else { int lru=0; for(int z=1;z<nn;z++) if(Sl[z].used<Sl[lru].used) lru=z; slot=lru; isnew=0; }+ else { int lru=0; for(int z=1;z<nn;z++) if(Sl[z].used<Sl[lru].used) lru=z; slot=lru; isnew=0;+ /* LFRU eviction guard (#441): a speculation must not drop a WARM demand-loaded expert.+ * Evict only if the predicted expert's history is hotter than the victim by tier_pick_lfru's+ * hysteresis (25% + 4 freq); else drop the speculation. Cache placement only -> output unchanged. */+ if(g_pilot_evict_guard && m->eheat && m->elast && Sl[lru].eid>=0){+ int vid=Sl[lru].eid;+ uint64_t vs=tier_lfru_score(m->eheat[layer][vid],m->elast[layer][vid],m->eaccess_clock);+ uint64_t cs=tier_lfru_score(m->eheat[layer][eid],m->elast[layer][eid],m->eaccess_clock);+ if(cs<=vs+(vs>>2)+(4u<<8)){ atomic_fetch_add_explicit(&g_pilot_drops,1,memory_order_relaxed);+ pthread_mutex_unlock(&g_pilot_mx); return; } } }ESlot *dst=&Sl[slot];dst->eid=-1; /* nascondi dagli scan-hint mentre carica */g_pilot_inflight[layer]++;@@ -3478,6 +3492,14 @@ static void pilot_uring_batch(Model *m){if(Sl[z].eid< -1) continue; /* URING reservation in flight */if(slot<0 || Sl[z].used<Sl[slot].used) slot=z;}+ /* LFRU eviction guard (#441): don't drop a warm resident for a speculation (see pilot_realload) */+ if(slot>=0 && Sl[slot].eid>=0 && g_pilot_evict_guard && m->eheat && m->elast){+ int vid=Sl[slot].eid;+ uint64_t vs=tier_lfru_score(m->eheat[layer][vid],m->elast[layer][vid],m->eaccess_clock);+ uint64_t cs=tier_lfru_score(m->eheat[layer][eid],m->elast[layer][eid],m->eaccess_clock);+ if(cs<=vs+(vs>>2)+(4u<<8)){ atomic_fetch_add_explicit(&g_pilot_drops,1,memory_order_relaxed);+ pthread_mutex_unlock(&g_pilot_mx); continue; }+ }}if(slot<0){ atomic_fetch_add_explicit(&g_pilot_drops,1,memory_order_relaxed); pthread_mutex_unlock(&g_pilot_mx); continue; }ESlot *dst=&Sl[slot];@@ -6105,6 +6127,7 @@ int main(int argc, char **argv){if(g_pilot_real) g_pilot=1; /* PILOT_REAL implica il pilota attivo */g_pilot_two = getenv("PILOT_TWO")?atoi(getenv("PILOT_TWO")):0; /* 1 = two-step: shared-expert-corrected router prediction (+2.3% recall, 3 extra matmuls) */if(g_pilot_two) g_pilot=1; /* PILOT_TWO implies PILOT active */+ g_pilot_evict_guard = getenv("PILOT_EVICT_GUARD")?atoi(getenv("PILOT_EVICT_GUARD")):1; /* 0 = old LRU eviction (A/B) *//* Default K: hint-only PILOT keeps 8 (WILLNEED hints are free, no eviction).* Under PILOT_REAL the speculative loads are REAL and create LRU eviction* pressure, so at ~28% mispredict a large K thrashes the cache β default to 6