Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat(olmoe): port LFRU prefetch eviction guard to OLMoE testbed
Egon Ruiter7e707c7b5127
diff --git a/c/olmoe.c b/c/olmoe.cindex d9460d40..74df155b 100644--- a/c/olmoe.c+++ b/c/olmoe.c@@ -82,6 +82,7 @@ typedef struct {uint8_t *is_pinned; /* [n_layers * n_experts], 1 if expert is globally pinned */uint8_t *is_queued; /* [n_layers * n_experts], 1 if expert is currently in the prefetch queue */float pilot_conf_limit; /* CONF_LIMIT env: cumulative gate probability threshold (e.g. 0.92) */+ uint64_t *last_access; /* [n_layers * n_experts], clock time when expert was last accessed */} Model;static pthread_mutex_t g_pilot_mx = PTHREAD_MUTEX_INITIALIZER;@@ -90,6 +91,13 @@ static volatile unsigned pilot_r = 0, pilot_w = 0;static Model *pilot_m = NULL;static int g_pilot = 0;static int g_wide = 1; /* IMPROVEMENT 4: top-K * g_wide candidates prefetched */+static int g_pilot_evict_guard = 1; /* PILOT_EVICT_GUARD=0 to disable LFRU prefetch eviction guard */++static uint64_t lfru_score(uint32_t heat, uint64_t last, uint64_t clock) {+ uint64_t age = (clock > last) ? (clock - last) : 0;+ uint64_t recent = (age < 255) ? (255 - age) : 0;+ return ((uint64_t)heat << 8) | recent;+}static void pilot_prefetch(Model *m, int lnext, const float *x, int S);static void *pilot_worker(void *arg);@@ -182,25 +190,6 @@ static void matmul_q(float *y, const float *x, const int8_t *q, const float *sca}}-/* quantizza un weight f32 [O,I] -> int8 q[O,I] + scala[O], simmetrica per riga.- * Replica quant_dequant() del Python: scale = amax(|w|, riga)/qmax, q = round(w/scale). */-static void quantize_rows(const float *w, int8_t *q, float *scale, int O, int I, int bits) {- int qmax = (1 << (bits - 1)) - 1; /* 8->127, 4->7, 2->1 */- #pragma omp parallel for schedule(static)- for (int o = 0; o < O; o++) {- const float *wr = w + (int64_t)o * I;- float amax = 0.f; for (int i = 0; i < I; i++) { float a = fabsf(wr[i]); if (a > amax) amax = a; }- float s = amax / qmax; if (s < 1e-8f) s = 1e-8f;- scale[o] = s;- int8_t *qr = q + (int64_t)o * I;- for (int i = 0; i < I; i++) {- int v = (int)lrintf(wr[i] / s);- if (v > qmax) v = qmax;- if (v < -qmax-1) v = -qmax-1;- qr[i] = (int8_t)v;- }- }-}/* rmsnorm su una riga di lunghezza D, in-place su out (out puo' essere == x) */static void rmsnorm_row(float *out, const float *x, const float *w, int D, float eps) {@@ -302,6 +291,7 @@ static void model_init(Model *m, const char *snap, int cap, int bits) {m->pilot_smooth = sv;m->is_pinned = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint8_t));m->is_queued = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint8_t));+ m->last_access = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint64_t));float cl = getenv("CONF_LIMIT") ? (float)atof(getenv("CONF_LIMIT")) : 0.92f;if (cl < 0.1f) cl = 0.1f; if (cl > 1.0f) cl = 1.0f;m->pilot_conf_limit = cl;@@ -402,6 +392,7 @@ static void expert_get(Model *m, int layer, int eid, Slot **out) {pthread_mutex_lock(&g_pilot_mx);for (int i = 0; i < lc->n; i++) if (lc->slots[i].eid == eid) {m->hits++; lc->slots[i].used = ++m->clock; *out = &lc->slots[i];+ if (m->last_access) m->last_access[layer * m->c.n_experts + eid] = m->clock;pthread_mutex_unlock(&g_pilot_mx);return;}@@ -440,6 +431,7 @@ static void expert_get(Model *m, int layer, int eid, Slot **out) {s->eid = eid;s->pinned = m->is_pinned[layer * c->n_experts + eid];s->used = ++m->clock;+ if (m->last_access) m->last_access[layer * c->n_experts + eid] = m->clock;*out = s;pthread_mutex_unlock(&g_pilot_mx);}@@ -719,6 +711,19 @@ static void pilot_realload(Model *m, int layer, int eid) {pthread_mutex_unlock(&g_pilot_mx);return; /* all pinned/in-flight, skip */}++ /* LFRU eviction guard: don't displace a warm resident expert with a speculation */+ if (g_pilot_evict_guard && m->freq && m->last_access && lc->slots[lru].eid >= 0) {+ int vid = lc->slots[lru].eid;+ uint64_t vs = lfru_score(m->freq[layer * c->n_experts + vid], m->last_access[layer * c->n_experts + vid], m->clock);+ uint64_t cs = lfru_score(m->freq[layer * c->n_experts + eid], m->last_access[layer * c->n_experts + eid], m->clock);+ if (cs <= vs + (vs >> 2) + (4u << 8)) {+ m->is_queued[layer * c->n_experts + eid] = 0;+ pthread_mutex_unlock(&g_pilot_mx);+ return; /* drop speculation */+ }+ }+s = &lc->slots[lru]; s->pinned = 0;}s->eid = -1; s->used = ++m->clock;@@ -730,6 +735,7 @@ static void pilot_realload(Model *m, int layer, int eid) {s->eid = eid;s->pinned = m->is_pinned[layer * c->n_experts + eid];s->used = ++m->clock;+ if (m->last_access) m->last_access[layer * c->n_experts + eid] = m->clock;m->is_queued[layer * c->n_experts + eid] = 0;pthread_mutex_unlock(&g_pilot_mx);}@@ -939,6 +945,7 @@ int main(int argc, char **argv) {if (!snap) { fprintf(stderr, "set SNAP=<snapshot directory>\n"); return 1; }g_pilot = getenv("PILOT") ? atoi(getenv("PILOT")) : 0;g_wide = getenv("WIDE") ? atoi(getenv("WIDE")) : 1;+ g_pilot_evict_guard = getenv("PILOT_EVICT_GUARD") ? atoi(getenv("PILOT_EVICT_GUARD")) : 1;if (g_wide < 1) g_wide = 1;if (g_wide > 4) g_wide = 4;int hot_n = getenv("HOT") ? atoi(getenv("HOT")) : 0;@@ -953,8 +960,8 @@ int main(int argc, char **argv) {float smooth = getenv("SMOOTH") ? (float)atof(getenv("SMOOTH")) : 0.3f;float conf = getenv("CONF_LIMIT") ? (float)atof(getenv("CONF_LIMIT")) : 0.92f;- printf("== Streaming C engine v2.2 | cache=%d/layer bits=%d pilot=%d wide=%d hot=%d smooth=%.2f conf=%.2f ==\n",- cap, bits, g_pilot, g_wide, hot_n, smooth, conf);+ printf("== Streaming C engine v2.2 | cache=%d/layer bits=%d pilot=%d wide=%d guard=%d hot=%d smooth=%.2f conf=%.2f ==\n",+ cap, bits, g_pilot, g_wide, g_pilot_evict_guard, hot_n, smooth, conf);FILE *f = fopen(refpath, "rb"); if (!f) { perror(refpath); return 1; }fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET);