Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(olmoe): wait for an expert already being read instead of reading it twice
Marcello Costagliola2020aecd0b02
diff --git a/c/olmoe.c b/c/olmoe.cindex 92028443..944566c5 100644--- a/c/olmoe.c+++ b/c/olmoe.c@@ -132,6 +132,7 @@ typedef struct {} Model;static pthread_mutex_t g_pilot_mx = PTHREAD_MUTEX_INITIALIZER;+static pthread_cond_t g_pilot_cv = PTHREAD_COND_INITIALIZER; /* broadcast on every publish */static struct { int l, e; } pilot_q[4096];static volatile unsigned pilot_r = 0, pilot_w = 0;static Model *pilot_m = NULL;@@ -195,6 +196,28 @@ static void cache_publish(Model *m, int layer, Slot *s, int eid) {lc->slot_by_expert[eid] = (int)(s - lc->slots);}+/* A slot being read keeps the index entry of the expert it is loading, marked+ * -(eid+2) as in colibri.c's ecache_reserve: lookups still miss and eviction+ * still skips it (eid < 0), but a second loader of the same expert can see the+ * read in flight instead of starting another one into another slot. */+static void cache_reserve(Model *m, int layer, Slot *s, int eid) {+ LCache *lc = &m->cache[layer];+ cache_hide(m, layer, s);+ s->eid = -(eid + 2);+ if (lc->slot_by_expert && eid >= 0 && eid < m->c.n_experts)+ lc->slot_by_expert[eid] = (int)(s - lc->slots);+}++/* Caller holds g_pilot_mx. */+static int slot_in_flight(Model *m, int layer, int eid) {+ if (layer < 0 || layer >= m->c.n_layers || eid < 0 ||+ eid >= m->c.n_experts) return 0;+ LCache *lc = &m->cache[layer];+ if (!lc->slot_by_expert) return 0;+ int i = lc->slot_by_expert[eid];+ return i >= 0 && i < lc->n && lc->slots[i].eid == -(eid + 2);+}+static void ensure_pilot_worker_started(Model *m) {if (!pilot_m) {pilot_m = m;@@ -662,7 +685,16 @@ static void slot_ensure_allocated(Model *m, Slot *s) {s->pinned = 0;}+#ifdef COLI_CACHE_INDEX_TEST+/* Model-free tests stand in for the disk read, so they can hold a load open+ * and count how many times each expert is read. */+static void (*g_test_expert_load)(Model *m, int layer, int eid, Slot *s);+#endif+static void load_expert_merged(Model *m, int layer, int eid, Slot *s) {+#ifdef COLI_CACHE_INDEX_TEST+ if (g_test_expert_load) { g_test_expert_load(m, layer, eid, s); return; }+#endifchar nm[256], qsnm[256];snprintf(nm, sizeof(nm), "model.layers.%d.mlp.experts.%d.merged_weight", layer, eid);snprintf(qsnm, sizeof(qsnm), "model.layers.%d.mlp.experts.%d.qs", layer, eid);@@ -709,6 +741,13 @@ static void expert_get(Model *m, int layer, int eid, Slot **out) {pthread_mutex_lock(&g_pilot_mx);ehit_mark(m, layer, eid); /* under the lock: the routing loop is parallel */Slot *hit = slot_indexed(m, layer, eid);+ /* The prefetcher usually reads the next layer's experts while this one+ * computes, so a routed expert is often already on its way: wait for that+ * read to publish rather than read the same bytes again into another slot. */+ while (!hit && slot_in_flight(m, layer, eid)) {+ pthread_cond_wait(&g_pilot_cv, &g_pilot_mx);+ hit = slot_indexed(m, layer, eid);+ }if (hit) {m->hits++; hit->used = ++m->clock; *out = hit;if (m->last_access) m->last_access[layer * m->c.n_experts + eid] = m->clock;@@ -755,7 +794,7 @@ static void expert_get(Model *m, int layer, int eid, Slot **out) {s = &lc->slots[lru];s->pinned = 0;}- cache_hide(m, layer, s);+ cache_reserve(m, layer, s, eid);s->used = ++m->clock;pthread_mutex_unlock(&g_pilot_mx);@@ -767,6 +806,7 @@ static void expert_get(Model *m, int layer, int eid, Slot **out) {s->used = ++m->clock;if (m->last_access) m->last_access[layer * c->n_experts + eid] = m->clock;*out = s;+ pthread_cond_broadcast(&g_pilot_cv);pthread_mutex_unlock(&g_pilot_mx);}@@ -1126,7 +1166,7 @@ static void pilot_realload(Model *m, int layer, int eid) {pthread_mutex_unlock(&g_pilot_mx);return;}- if (slot_indexed(m, layer, eid)) {+ if (slot_indexed(m, layer, eid) || slot_in_flight(m, layer, eid)) {m->is_queued[layer * c->n_experts + eid] = 0;pthread_mutex_unlock(&g_pilot_mx);return;@@ -1163,7 +1203,7 @@ static void pilot_realload(Model *m, int layer, int eid) {s = &lc->slots[lru]; s->pinned = 0;}- cache_hide(m, layer, s); s->used = ++m->clock;+ cache_reserve(m, layer, s, eid); s->used = ++m->clock;pthread_mutex_unlock(&g_pilot_mx);load_expert_merged(m, layer, eid, s);@@ -1174,6 +1214,7 @@ static void pilot_realload(Model *m, int layer, int eid) {s->used = ++m->clock;if (m->last_access) m->last_access[layer * c->n_experts + eid] = m->clock;m->is_queued[layer * c->n_experts + eid] = 0;+ pthread_cond_broadcast(&g_pilot_cv);pthread_mutex_unlock(&g_pilot_mx);}diff --git a/c/tests/test_olmoe_cache_index.c b/c/tests/test_olmoe_cache_index.cindex 6556e2d1..8a227173 100644--- a/c/tests/test_olmoe_cache_index.c+++ b/c/tests/test_olmoe_cache_index.c@@ -40,6 +40,150 @@ static void check_lookup_scaling(int cap) {free_cache(&m);}+/* The first read of a scenario stays open until the test releases it; every+ * later read returns at once. g_io_loads counts them all. */+static pthread_mutex_t g_io_mx = PTHREAD_MUTEX_INITIALIZER;+static pthread_cond_t g_io_cv = PTHREAD_COND_INITIALIZER;+static int g_io_loads, g_io_open, g_io_release;++static void held_load(Model *m, int layer, int eid, Slot *s) {+ (void)m; (void)layer; (void)eid; (void)s;+ pthread_mutex_lock(&g_io_mx);+ if (g_io_loads++ == 0) {+ g_io_open = 1;+ pthread_cond_broadcast(&g_io_cv);+ while (!g_io_release) pthread_cond_wait(&g_io_cv, &g_io_mx);+ }+ pthread_mutex_unlock(&g_io_mx);+}++static int io_get(int *v) {+ pthread_mutex_lock(&g_io_mx); int x = *v; pthread_mutex_unlock(&g_io_mx);+ return x;+}++static void io_release(void) {+ pthread_mutex_lock(&g_io_mx);+ g_io_release = 1;+ pthread_cond_broadcast(&g_io_cv);+ pthread_mutex_unlock(&g_io_mx);+}++/* expert_get marks the expert routed under g_pilot_mx and keeps the lock until+ * it either waits or starts its own read: seeing the mark means it got there. */+static int routed(Model *m, int eid) {+ pthread_mutex_lock(&g_pilot_mx);+ int r = m->ehit && m->ehit[0][eid];+ pthread_mutex_unlock(&g_pilot_mx);+ return r;+}++typedef struct { Model *m; int eid, done; Slot *got; } Req;++static void *demand_thread(void *p) {+ Req *r = p; expert_get(r->m, 0, r->eid, &r->got);+ __atomic_store_n(&r->done, 1, __ATOMIC_RELEASE); return NULL;+}++static void *prefetch_thread(void *p) {+ Req *r = p; pilot_realload(r->m, 0, r->eid);+ __atomic_store_n(&r->done, 1, __ATOMIC_RELEASE); return NULL;+}++/* Bounded wait: a lost wakeup must fail the test, not hang the suite. */+static void wait_done(Req *r, pthread_t t, const char *who) {+ for (int ms = 0; ms < 5000 && !__atomic_load_n(&r->done, __ATOMIC_ACQUIRE); ms++) sleep_ms(1);+ if (!__atomic_load_n(&r->done, __ATOMIC_ACQUIRE)) {+ fprintf(stderr, "FAIL %s:%d: %s never returned\n", __FILE__, __LINE__, who);+ exit(1);+ }+ pthread_join(t, NULL);+}++static int copies(Model *m, int eid) {+ LCache *lc = &m->cache[0]; int n = 0;+ for (int i = 0; i < lc->n; i++) n += lc->slots[i].eid == eid;+ return n;+}++static void init_loader(Model *m) {+ init_cache(m, 8, 4);+ m->c.inter = 4; m->c.hidden = 4;+ m->is_queued = calloc(8, 1); m->is_pinned = calloc(8, 1);+ g_io_loads = g_io_open = g_io_release = 0;+ g_test_expert_load = held_load;+}++static void free_loader(Model *m) {+ LCache *lc = &m->cache[0];+ for (int i = 0; i < lc->n; i++) { free(lc->slots[i].g); free(lc->slots[i].gs); }+ if (m->ehit) { free(m->ehit[0]); free(m->ehit); }+ free(m->is_queued); free(m->is_pinned);+ free_cache(m);+ g_test_expert_load = NULL;+}++/* The prefetcher is reading expert 3 when the forward pass routes to it: the+ * forward pass must take the prefetched copy, not read it a second time. */+static void check_demand_joins_prefetch(void) {+ Model m; init_loader(&m);+ m.is_queued[3] = 1;+ Req pf = { &m, 3, 0, NULL }, dm = { &m, 3, 0, NULL };+ pthread_t tp, td;+ pthread_create(&tp, NULL, prefetch_thread, &pf);+ for (int ms = 0; ms < 5000 && !io_get(&g_io_open); ms++) sleep_ms(1);+ CHECK(io_get(&g_io_open), "prefetch never started its read");+ pthread_create(&td, NULL, demand_thread, &dm);+ for (int ms = 0; ms < 5000 && !routed(&m, 3); ms++) sleep_ms(1);+ io_release();+ wait_done(&pf, tp, "pilot_realload");+ wait_done(&dm, td, "expert_get");+ CHECK(g_io_loads == 1, "expert 3 read %d times, prefetch already had it in flight", g_io_loads);+ CHECK(copies(&m, 3) == 1, "expert 3 resident in %d slots", copies(&m, 3));+ CHECK(dm.got && dm.got == slot_indexed(&m, 0, 3), "forward pass got a slot the index does not name");+ free_loader(&m);+}++/* The forward pass is reading expert 4 when a prefetch for it comes up: the+ * prefetch must drop the request at once, without a read or a wait. */+static void check_prefetch_skips_demand(void) {+ Model m; init_loader(&m);+ Req dm = { &m, 4, 0, NULL };+ pthread_t td;+ pthread_create(&td, NULL, demand_thread, &dm);+ for (int ms = 0; ms < 5000 && !io_get(&g_io_open); ms++) sleep_ms(1);+ CHECK(io_get(&g_io_open), "forward pass never started its read");+ pthread_mutex_lock(&g_pilot_mx); m.is_queued[4] = 1; pthread_mutex_unlock(&g_pilot_mx);+ pilot_realload(&m, 0, 4);+ int loads = io_get(&g_io_loads);+ CHECK(loads == 1, "expert 4 read %d times, forward pass already had it in flight", loads);+ CHECK(m.is_queued[4] == 0, "dropped prefetch stayed queued");+ io_release();+ wait_done(&dm, td, "expert_get");+ CHECK(copies(&m, 4) == 1, "expert 4 resident in %d slots", copies(&m, 4));+ free_loader(&m);+}++/* Two callers route to expert 5 at once: the second waits for the first read+ * and must be woken by its publish. */+static void check_demand_joins_demand(void) {+ Model m; init_loader(&m);+ Req a = { &m, 5, 0, NULL }, b = { &m, 5, 0, NULL };+ pthread_t ta, tb;+ pthread_create(&ta, NULL, demand_thread, &a);+ for (int ms = 0; ms < 5000 && !io_get(&g_io_open); ms++) sleep_ms(1);+ CHECK(io_get(&g_io_open), "first caller never started its read");+ pthread_mutex_lock(&g_pilot_mx); m.ehit[0][5] = 0; pthread_mutex_unlock(&g_pilot_mx);+ pthread_create(&tb, NULL, demand_thread, &b);+ for (int ms = 0; ms < 5000 && !routed(&m, 5); ms++) sleep_ms(1);+ io_release();+ wait_done(&a, ta, "first expert_get");+ wait_done(&b, tb, "second expert_get");+ CHECK(g_io_loads == 1, "expert 5 read %d times by two callers", g_io_loads);+ CHECK(a.got && a.got == b.got, "two callers got different slots for expert 5");+ free_loader(&m);+}+int main(void) {Model m; init_cache(&m, 6, 2); LCache *lc = &m.cache[0]; lc->n = 2;cache_publish(&m, 0, &lc->slots[0], 1);@@ -62,6 +206,9 @@ int main(void) {free_cache(&m);check_lookup_scaling(44);check_lookup_scaling(219);+ check_demand_joins_prefetch();+ check_prefetch_skips_demand();+ check_demand_joins_demand();if (failures) { fprintf(stderr,"olmoe cache index: %d failure(s)\n", failures); return 1; }puts("olmoe cache index: ok");return 0;