Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

pilot: LFRU-aware eviction guard β€” never evict a warm demand expert for a speculation (#441)

cdhdteaa68f4005fb
diff --git a/c/colibri.c b/c/colibri.c
index 4524c105..97ae7214 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -733,6 +733,11 @@ static int g_pilot_real=0;/* PILOT_REAL=1: il pilota fa LOAD VERI cross-layer de
static int g_pilot_two=0; /* PILOT_TWO=1: two-step prefetch β€” before running L+1's router,
* approximate MoE(L) using only the shared expert (resident, no disk)
* and add it to the state. Trades 3 small matmuls for +2.3% recall. */
+static int g_pilot_evict_guard=1;/* PILOT_EVICT_GUARD=0 -> old behavior (a speculation evicts the plain LRU).
+ * Default ON: a speculative pilot load may evict a RESIDENT expert only if the
+ * predicted expert is historically HOTTER than the victim (same LFRU hysteresis as
+ * tier_pick_lfru); otherwise it drops the speculation rather than thrash a warm
+ * demand-loaded expert. Cache placement only -> output byte-identical. (#441) */
/* Handshake main<->pilota per il load-vero cross-layer. Invariante di sicurezza in DUE parti:
* 1) Percorso MATMUL (moe): il pilota scrive SOLO ecache[layer] con layer > g_cur_moe_layer;
* il matmul in moe() legge SOLO ecache[layer]==g_cur_moe_layer, e la barriera a inizio moe()
@@ -3427,7 +3432,16 @@ static void pilot_realload(Model *m, int layer, int eid){
for(int z=0;z<nn;z++) if(Sl[z].eid==eid){ pthread_mutex_unlock(&g_pilot_mx); return; }
int slot,isnew; /* cresci se c'e' posto, altrimenti LRU */
if(nn<m->ecap){ slot=nn; isnew=1; }
- else { int lru=0; for(int z=1;z<nn;z++) if(Sl[z].used<Sl[lru].used) lru=z; slot=lru; isnew=0; }
+ else { int lru=0; for(int z=1;z<nn;z++) if(Sl[z].used<Sl[lru].used) lru=z; slot=lru; isnew=0;
+ /* LFRU eviction guard (#441): a speculation must not drop a WARM demand-loaded expert.
+ * Evict only if the predicted expert's history is hotter than the victim by tier_pick_lfru's
+ * hysteresis (25% + 4 freq); else drop the speculation. Cache placement only -> output unchanged. */
+ if(g_pilot_evict_guard && m->eheat && m->elast && Sl[lru].eid>=0){
+ int vid=Sl[lru].eid;
+ uint64_t vs=tier_lfru_score(m->eheat[layer][vid],m->elast[layer][vid],m->eaccess_clock);
+ uint64_t cs=tier_lfru_score(m->eheat[layer][eid],m->elast[layer][eid],m->eaccess_clock);
+ if(cs<=vs+(vs>>2)+(4u<<8)){ atomic_fetch_add_explicit(&g_pilot_drops,1,memory_order_relaxed);
+ pthread_mutex_unlock(&g_pilot_mx); return; } } }
ESlot *dst=&Sl[slot];
dst->eid=-1; /* nascondi dagli scan-hint mentre carica */
g_pilot_inflight[layer]++;
@@ -3478,6 +3492,14 @@ static void pilot_uring_batch(Model *m){
if(Sl[z].eid< -1) continue; /* URING reservation in flight */
if(slot<0 || Sl[z].used<Sl[slot].used) slot=z;
}
+ /* LFRU eviction guard (#441): don't drop a warm resident for a speculation (see pilot_realload) */
+ if(slot>=0 && Sl[slot].eid>=0 && g_pilot_evict_guard && m->eheat && m->elast){
+ int vid=Sl[slot].eid;
+ uint64_t vs=tier_lfru_score(m->eheat[layer][vid],m->elast[layer][vid],m->eaccess_clock);
+ uint64_t cs=tier_lfru_score(m->eheat[layer][eid],m->elast[layer][eid],m->eaccess_clock);
+ if(cs<=vs+(vs>>2)+(4u<<8)){ atomic_fetch_add_explicit(&g_pilot_drops,1,memory_order_relaxed);
+ pthread_mutex_unlock(&g_pilot_mx); continue; }
+ }
}
if(slot<0){ atomic_fetch_add_explicit(&g_pilot_drops,1,memory_order_relaxed); pthread_mutex_unlock(&g_pilot_mx); continue; }
ESlot *dst=&Sl[slot];
@@ -6105,6 +6127,7 @@ int main(int argc, char **argv){
if(g_pilot_real) g_pilot=1; /* PILOT_REAL implica il pilota attivo */
g_pilot_two = getenv("PILOT_TWO")?atoi(getenv("PILOT_TWO")):0; /* 1 = two-step: shared-expert-corrected router prediction (+2.3% recall, 3 extra matmuls) */
if(g_pilot_two) g_pilot=1; /* PILOT_TWO implies PILOT active */
+ g_pilot_evict_guard = getenv("PILOT_EVICT_GUARD")?atoi(getenv("PILOT_EVICT_GUARD")):1; /* 0 = old LRU eviction (A/B) */
/* Default K: hint-only PILOT keeps 8 (WILLNEED hints are free, no eviction).
* Under PILOT_REAL the speculative loads are REAL and create LRU eviction
* pressure, so at ~28% mispredict a large K thrashes the cache β€” default to 6