Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(olmoe): wait for an expert already being read instead of reading it twice

Marcello Costagliola2020aecd0b02
diff --git a/c/olmoe.c b/c/olmoe.c
index 92028443..944566c5 100644
--- a/c/olmoe.c
+++ b/c/olmoe.c
@@ -132,6 +132,7 @@ typedef struct {
} Model;
static pthread_mutex_t g_pilot_mx = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t g_pilot_cv = PTHREAD_COND_INITIALIZER; /* broadcast on every publish */
static struct { int l, e; } pilot_q[4096];
static volatile unsigned pilot_r = 0, pilot_w = 0;
static Model *pilot_m = NULL;
@@ -195,6 +196,28 @@ static void cache_publish(Model *m, int layer, Slot *s, int eid) {
lc->slot_by_expert[eid] = (int)(s - lc->slots);
}
+/* A slot being read keeps the index entry of the expert it is loading, marked
+ * -(eid+2) as in colibri.c's ecache_reserve: lookups still miss and eviction
+ * still skips it (eid < 0), but a second loader of the same expert can see the
+ * read in flight instead of starting another one into another slot. */
+static void cache_reserve(Model *m, int layer, Slot *s, int eid) {
+ LCache *lc = &m->cache[layer];
+ cache_hide(m, layer, s);
+ s->eid = -(eid + 2);
+ if (lc->slot_by_expert && eid >= 0 && eid < m->c.n_experts)
+ lc->slot_by_expert[eid] = (int)(s - lc->slots);
+}
+
+/* Caller holds g_pilot_mx. */
+static int slot_in_flight(Model *m, int layer, int eid) {
+ if (layer < 0 || layer >= m->c.n_layers || eid < 0 ||
+ eid >= m->c.n_experts) return 0;
+ LCache *lc = &m->cache[layer];
+ if (!lc->slot_by_expert) return 0;
+ int i = lc->slot_by_expert[eid];
+ return i >= 0 && i < lc->n && lc->slots[i].eid == -(eid + 2);
+}
+
static void ensure_pilot_worker_started(Model *m) {
if (!pilot_m) {
pilot_m = m;
@@ -662,7 +685,16 @@ static void slot_ensure_allocated(Model *m, Slot *s) {
s->pinned = 0;
}
+#ifdef COLI_CACHE_INDEX_TEST
+/* Model-free tests stand in for the disk read, so they can hold a load open
+ * and count how many times each expert is read. */
+static void (*g_test_expert_load)(Model *m, int layer, int eid, Slot *s);
+#endif
+
static void load_expert_merged(Model *m, int layer, int eid, Slot *s) {
+#ifdef COLI_CACHE_INDEX_TEST
+ if (g_test_expert_load) { g_test_expert_load(m, layer, eid, s); return; }
+#endif
char nm[256], qsnm[256];
snprintf(nm, sizeof(nm), "model.layers.%d.mlp.experts.%d.merged_weight", layer, eid);
snprintf(qsnm, sizeof(qsnm), "model.layers.%d.mlp.experts.%d.qs", layer, eid);
@@ -709,6 +741,13 @@ static void expert_get(Model *m, int layer, int eid, Slot **out) {
pthread_mutex_lock(&g_pilot_mx);
ehit_mark(m, layer, eid); /* under the lock: the routing loop is parallel */
Slot *hit = slot_indexed(m, layer, eid);
+ /* The prefetcher usually reads the next layer's experts while this one
+ * computes, so a routed expert is often already on its way: wait for that
+ * read to publish rather than read the same bytes again into another slot. */
+ while (!hit && slot_in_flight(m, layer, eid)) {
+ pthread_cond_wait(&g_pilot_cv, &g_pilot_mx);
+ hit = slot_indexed(m, layer, eid);
+ }
if (hit) {
m->hits++; hit->used = ++m->clock; *out = hit;
if (m->last_access) m->last_access[layer * m->c.n_experts + eid] = m->clock;
@@ -755,7 +794,7 @@ static void expert_get(Model *m, int layer, int eid, Slot **out) {
s = &lc->slots[lru];
s->pinned = 0;
}
- cache_hide(m, layer, s);
+ cache_reserve(m, layer, s, eid);
s->used = ++m->clock;
pthread_mutex_unlock(&g_pilot_mx);
@@ -767,6 +806,7 @@ static void expert_get(Model *m, int layer, int eid, Slot **out) {
s->used = ++m->clock;
if (m->last_access) m->last_access[layer * c->n_experts + eid] = m->clock;
*out = s;
+ pthread_cond_broadcast(&g_pilot_cv);
pthread_mutex_unlock(&g_pilot_mx);
}
@@ -1126,7 +1166,7 @@ static void pilot_realload(Model *m, int layer, int eid) {
pthread_mutex_unlock(&g_pilot_mx);
return;
}
- if (slot_indexed(m, layer, eid)) {
+ if (slot_indexed(m, layer, eid) || slot_in_flight(m, layer, eid)) {
m->is_queued[layer * c->n_experts + eid] = 0;
pthread_mutex_unlock(&g_pilot_mx);
return;
@@ -1163,7 +1203,7 @@ static void pilot_realload(Model *m, int layer, int eid) {
s = &lc->slots[lru]; s->pinned = 0;
}
- cache_hide(m, layer, s); s->used = ++m->clock;
+ cache_reserve(m, layer, s, eid); s->used = ++m->clock;
pthread_mutex_unlock(&g_pilot_mx);
load_expert_merged(m, layer, eid, s);
@@ -1174,6 +1214,7 @@ static void pilot_realload(Model *m, int layer, int eid) {
s->used = ++m->clock;
if (m->last_access) m->last_access[layer * c->n_experts + eid] = m->clock;
m->is_queued[layer * c->n_experts + eid] = 0;
+ pthread_cond_broadcast(&g_pilot_cv);
pthread_mutex_unlock(&g_pilot_mx);
}
diff --git a/c/tests/test_olmoe_cache_index.c b/c/tests/test_olmoe_cache_index.c
index 6556e2d1..8a227173 100644
--- a/c/tests/test_olmoe_cache_index.c
+++ b/c/tests/test_olmoe_cache_index.c
@@ -40,6 +40,150 @@ static void check_lookup_scaling(int cap) {
free_cache(&m);
}
+/* The first read of a scenario stays open until the test releases it; every
+ * later read returns at once. g_io_loads counts them all. */
+static pthread_mutex_t g_io_mx = PTHREAD_MUTEX_INITIALIZER;
+static pthread_cond_t g_io_cv = PTHREAD_COND_INITIALIZER;
+static int g_io_loads, g_io_open, g_io_release;
+
+static void held_load(Model *m, int layer, int eid, Slot *s) {
+ (void)m; (void)layer; (void)eid; (void)s;
+ pthread_mutex_lock(&g_io_mx);
+ if (g_io_loads++ == 0) {
+ g_io_open = 1;
+ pthread_cond_broadcast(&g_io_cv);
+ while (!g_io_release) pthread_cond_wait(&g_io_cv, &g_io_mx);
+ }
+ pthread_mutex_unlock(&g_io_mx);
+}
+
+static int io_get(int *v) {
+ pthread_mutex_lock(&g_io_mx); int x = *v; pthread_mutex_unlock(&g_io_mx);
+ return x;
+}
+
+static void io_release(void) {
+ pthread_mutex_lock(&g_io_mx);
+ g_io_release = 1;
+ pthread_cond_broadcast(&g_io_cv);
+ pthread_mutex_unlock(&g_io_mx);
+}
+
+/* expert_get marks the expert routed under g_pilot_mx and keeps the lock until
+ * it either waits or starts its own read: seeing the mark means it got there. */
+static int routed(Model *m, int eid) {
+ pthread_mutex_lock(&g_pilot_mx);
+ int r = m->ehit && m->ehit[0][eid];
+ pthread_mutex_unlock(&g_pilot_mx);
+ return r;
+}
+
+typedef struct { Model *m; int eid, done; Slot *got; } Req;
+
+static void *demand_thread(void *p) {
+ Req *r = p; expert_get(r->m, 0, r->eid, &r->got);
+ __atomic_store_n(&r->done, 1, __ATOMIC_RELEASE); return NULL;
+}
+
+static void *prefetch_thread(void *p) {
+ Req *r = p; pilot_realload(r->m, 0, r->eid);
+ __atomic_store_n(&r->done, 1, __ATOMIC_RELEASE); return NULL;
+}
+
+/* Bounded wait: a lost wakeup must fail the test, not hang the suite. */
+static void wait_done(Req *r, pthread_t t, const char *who) {
+ for (int ms = 0; ms < 5000 && !__atomic_load_n(&r->done, __ATOMIC_ACQUIRE); ms++) sleep_ms(1);
+ if (!__atomic_load_n(&r->done, __ATOMIC_ACQUIRE)) {
+ fprintf(stderr, "FAIL %s:%d: %s never returned\n", __FILE__, __LINE__, who);
+ exit(1);
+ }
+ pthread_join(t, NULL);
+}
+
+static int copies(Model *m, int eid) {
+ LCache *lc = &m->cache[0]; int n = 0;
+ for (int i = 0; i < lc->n; i++) n += lc->slots[i].eid == eid;
+ return n;
+}
+
+static void init_loader(Model *m) {
+ init_cache(m, 8, 4);
+ m->c.inter = 4; m->c.hidden = 4;
+ m->is_queued = calloc(8, 1); m->is_pinned = calloc(8, 1);
+ g_io_loads = g_io_open = g_io_release = 0;
+ g_test_expert_load = held_load;
+}
+
+static void free_loader(Model *m) {
+ LCache *lc = &m->cache[0];
+ for (int i = 0; i < lc->n; i++) { free(lc->slots[i].g); free(lc->slots[i].gs); }
+ if (m->ehit) { free(m->ehit[0]); free(m->ehit); }
+ free(m->is_queued); free(m->is_pinned);
+ free_cache(m);
+ g_test_expert_load = NULL;
+}
+
+/* The prefetcher is reading expert 3 when the forward pass routes to it: the
+ * forward pass must take the prefetched copy, not read it a second time. */
+static void check_demand_joins_prefetch(void) {
+ Model m; init_loader(&m);
+ m.is_queued[3] = 1;
+ Req pf = { &m, 3, 0, NULL }, dm = { &m, 3, 0, NULL };
+ pthread_t tp, td;
+ pthread_create(&tp, NULL, prefetch_thread, &pf);
+ for (int ms = 0; ms < 5000 && !io_get(&g_io_open); ms++) sleep_ms(1);
+ CHECK(io_get(&g_io_open), "prefetch never started its read");
+ pthread_create(&td, NULL, demand_thread, &dm);
+ for (int ms = 0; ms < 5000 && !routed(&m, 3); ms++) sleep_ms(1);
+ io_release();
+ wait_done(&pf, tp, "pilot_realload");
+ wait_done(&dm, td, "expert_get");
+ CHECK(g_io_loads == 1, "expert 3 read %d times, prefetch already had it in flight", g_io_loads);
+ CHECK(copies(&m, 3) == 1, "expert 3 resident in %d slots", copies(&m, 3));
+ CHECK(dm.got && dm.got == slot_indexed(&m, 0, 3), "forward pass got a slot the index does not name");
+ free_loader(&m);
+}
+
+/* The forward pass is reading expert 4 when a prefetch for it comes up: the
+ * prefetch must drop the request at once, without a read or a wait. */
+static void check_prefetch_skips_demand(void) {
+ Model m; init_loader(&m);
+ Req dm = { &m, 4, 0, NULL };
+ pthread_t td;
+ pthread_create(&td, NULL, demand_thread, &dm);
+ for (int ms = 0; ms < 5000 && !io_get(&g_io_open); ms++) sleep_ms(1);
+ CHECK(io_get(&g_io_open), "forward pass never started its read");
+ pthread_mutex_lock(&g_pilot_mx); m.is_queued[4] = 1; pthread_mutex_unlock(&g_pilot_mx);
+ pilot_realload(&m, 0, 4);
+ int loads = io_get(&g_io_loads);
+ CHECK(loads == 1, "expert 4 read %d times, forward pass already had it in flight", loads);
+ CHECK(m.is_queued[4] == 0, "dropped prefetch stayed queued");
+ io_release();
+ wait_done(&dm, td, "expert_get");
+ CHECK(copies(&m, 4) == 1, "expert 4 resident in %d slots", copies(&m, 4));
+ free_loader(&m);
+}
+
+/* Two callers route to expert 5 at once: the second waits for the first read
+ * and must be woken by its publish. */
+static void check_demand_joins_demand(void) {
+ Model m; init_loader(&m);
+ Req a = { &m, 5, 0, NULL }, b = { &m, 5, 0, NULL };
+ pthread_t ta, tb;
+ pthread_create(&ta, NULL, demand_thread, &a);
+ for (int ms = 0; ms < 5000 && !io_get(&g_io_open); ms++) sleep_ms(1);
+ CHECK(io_get(&g_io_open), "first caller never started its read");
+ pthread_mutex_lock(&g_pilot_mx); m.ehit[0][5] = 0; pthread_mutex_unlock(&g_pilot_mx);
+ pthread_create(&tb, NULL, demand_thread, &b);
+ for (int ms = 0; ms < 5000 && !routed(&m, 5); ms++) sleep_ms(1);
+ io_release();
+ wait_done(&a, ta, "first expert_get");
+ wait_done(&b, tb, "second expert_get");
+ CHECK(g_io_loads == 1, "expert 5 read %d times by two callers", g_io_loads);
+ CHECK(a.got && a.got == b.got, "two callers got different slots for expert 5");
+ free_loader(&m);
+}
+
int main(void) {
Model m; init_cache(&m, 6, 2); LCache *lc = &m.cache[0]; lc->n = 2;
cache_publish(&m, 0, &lc->slots[0], 1);
@@ -62,6 +206,9 @@ int main(void) {
free_cache(&m);
check_lookup_scaling(44);
check_lookup_scaling(219);
+ check_demand_joins_prefetch();
+ check_prefetch_skips_demand();
+ check_demand_joins_demand();
if (failures) { fprintf(stderr,"olmoe cache index: %d failure(s)\n", failures); return 1; }
puts("olmoe cache index: ok");
return 0;