Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(olmoe): port LFRU prefetch eviction guard to OLMoE testbed

Egon Ruiter7e707c7b5127
diff --git a/c/olmoe.c b/c/olmoe.c
index d9460d40..74df155b 100644
--- a/c/olmoe.c
+++ b/c/olmoe.c
@@ -82,6 +82,7 @@ typedef struct {
uint8_t *is_pinned; /* [n_layers * n_experts], 1 if expert is globally pinned */
uint8_t *is_queued; /* [n_layers * n_experts], 1 if expert is currently in the prefetch queue */
float pilot_conf_limit; /* CONF_LIMIT env: cumulative gate probability threshold (e.g. 0.92) */
+ uint64_t *last_access; /* [n_layers * n_experts], clock time when expert was last accessed */
} Model;
static pthread_mutex_t g_pilot_mx = PTHREAD_MUTEX_INITIALIZER;
@@ -90,6 +91,13 @@ static volatile unsigned pilot_r = 0, pilot_w = 0;
static Model *pilot_m = NULL;
static int g_pilot = 0;
static int g_wide = 1; /* IMPROVEMENT 4: top-K * g_wide candidates prefetched */
+static int g_pilot_evict_guard = 1; /* PILOT_EVICT_GUARD=0 to disable LFRU prefetch eviction guard */
+
+static uint64_t lfru_score(uint32_t heat, uint64_t last, uint64_t clock) {
+ uint64_t age = (clock > last) ? (clock - last) : 0;
+ uint64_t recent = (age < 255) ? (255 - age) : 0;
+ return ((uint64_t)heat << 8) | recent;
+}
static void pilot_prefetch(Model *m, int lnext, const float *x, int S);
static void *pilot_worker(void *arg);
@@ -182,25 +190,6 @@ static void matmul_q(float *y, const float *x, const int8_t *q, const float *sca
}
}
-/* quantizza un weight f32 [O,I] -> int8 q[O,I] + scala[O], simmetrica per riga.
- * Replica quant_dequant() del Python: scale = amax(|w|, riga)/qmax, q = round(w/scale). */
-static void quantize_rows(const float *w, int8_t *q, float *scale, int O, int I, int bits) {
- int qmax = (1 << (bits - 1)) - 1; /* 8->127, 4->7, 2->1 */
- #pragma omp parallel for schedule(static)
- for (int o = 0; o < O; o++) {
- const float *wr = w + (int64_t)o * I;
- float amax = 0.f; for (int i = 0; i < I; i++) { float a = fabsf(wr[i]); if (a > amax) amax = a; }
- float s = amax / qmax; if (s < 1e-8f) s = 1e-8f;
- scale[o] = s;
- int8_t *qr = q + (int64_t)o * I;
- for (int i = 0; i < I; i++) {
- int v = (int)lrintf(wr[i] / s);
- if (v > qmax) v = qmax;
- if (v < -qmax-1) v = -qmax-1;
- qr[i] = (int8_t)v;
- }
- }
-}
/* rmsnorm su una riga di lunghezza D, in-place su out (out puo' essere == x) */
static void rmsnorm_row(float *out, const float *x, const float *w, int D, float eps) {
@@ -302,6 +291,7 @@ static void model_init(Model *m, const char *snap, int cap, int bits) {
m->pilot_smooth = sv;
m->is_pinned = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint8_t));
m->is_queued = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint8_t));
+ m->last_access = calloc((size_t)c->n_layers * c->n_experts, sizeof(uint64_t));
float cl = getenv("CONF_LIMIT") ? (float)atof(getenv("CONF_LIMIT")) : 0.92f;
if (cl < 0.1f) cl = 0.1f; if (cl > 1.0f) cl = 1.0f;
m->pilot_conf_limit = cl;
@@ -402,6 +392,7 @@ static void expert_get(Model *m, int layer, int eid, Slot **out) {
pthread_mutex_lock(&g_pilot_mx);
for (int i = 0; i < lc->n; i++) if (lc->slots[i].eid == eid) {
m->hits++; lc->slots[i].used = ++m->clock; *out = &lc->slots[i];
+ if (m->last_access) m->last_access[layer * m->c.n_experts + eid] = m->clock;
pthread_mutex_unlock(&g_pilot_mx);
return;
}
@@ -440,6 +431,7 @@ static void expert_get(Model *m, int layer, int eid, Slot **out) {
s->eid = eid;
s->pinned = m->is_pinned[layer * c->n_experts + eid];
s->used = ++m->clock;
+ if (m->last_access) m->last_access[layer * c->n_experts + eid] = m->clock;
*out = s;
pthread_mutex_unlock(&g_pilot_mx);
}
@@ -719,6 +711,19 @@ static void pilot_realload(Model *m, int layer, int eid) {
pthread_mutex_unlock(&g_pilot_mx);
return; /* all pinned/in-flight, skip */
}
+
+ /* LFRU eviction guard: don't displace a warm resident expert with a speculation */
+ if (g_pilot_evict_guard && m->freq && m->last_access && lc->slots[lru].eid >= 0) {
+ int vid = lc->slots[lru].eid;
+ uint64_t vs = lfru_score(m->freq[layer * c->n_experts + vid], m->last_access[layer * c->n_experts + vid], m->clock);
+ uint64_t cs = lfru_score(m->freq[layer * c->n_experts + eid], m->last_access[layer * c->n_experts + eid], m->clock);
+ if (cs <= vs + (vs >> 2) + (4u << 8)) {
+ m->is_queued[layer * c->n_experts + eid] = 0;
+ pthread_mutex_unlock(&g_pilot_mx);
+ return; /* drop speculation */
+ }
+ }
+
s = &lc->slots[lru]; s->pinned = 0;
}
s->eid = -1; s->used = ++m->clock;
@@ -730,6 +735,7 @@ static void pilot_realload(Model *m, int layer, int eid) {
s->eid = eid;
s->pinned = m->is_pinned[layer * c->n_experts + eid];
s->used = ++m->clock;
+ if (m->last_access) m->last_access[layer * c->n_experts + eid] = m->clock;
m->is_queued[layer * c->n_experts + eid] = 0;
pthread_mutex_unlock(&g_pilot_mx);
}
@@ -939,6 +945,7 @@ int main(int argc, char **argv) {
if (!snap) { fprintf(stderr, "set SNAP=<snapshot directory>\n"); return 1; }
g_pilot = getenv("PILOT") ? atoi(getenv("PILOT")) : 0;
g_wide = getenv("WIDE") ? atoi(getenv("WIDE")) : 1;
+ g_pilot_evict_guard = getenv("PILOT_EVICT_GUARD") ? atoi(getenv("PILOT_EVICT_GUARD")) : 1;
if (g_wide < 1) g_wide = 1;
if (g_wide > 4) g_wide = 4;
int hot_n = getenv("HOT") ? atoi(getenv("HOT")) : 0;
@@ -953,8 +960,8 @@ int main(int argc, char **argv) {
float smooth = getenv("SMOOTH") ? (float)atof(getenv("SMOOTH")) : 0.3f;
float conf = getenv("CONF_LIMIT") ? (float)atof(getenv("CONF_LIMIT")) : 0.92f;
- printf("== Streaming C engine v2.2 | cache=%d/layer bits=%d pilot=%d wide=%d hot=%d smooth=%.2f conf=%.2f ==\n",
- cap, bits, g_pilot, g_wide, hot_n, smooth, conf);
+ printf("== Streaming C engine v2.2 | cache=%d/layer bits=%d pilot=%d wide=%d guard=%d hot=%d smooth=%.2f conf=%.2f ==\n",
+ cap, bits, g_pilot, g_wide, g_pilot_evict_guard, hot_n, smooth, conf);
FILE *f = fopen(refpath, "rb"); if (!f) { perror(refpath); return 1; }
fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET);