Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

Merge remote-tracking branch 'upstream/dev' into dev

DrewZfddb786a597d
diff --cc c/olmoe.c
index 9dd09110,325a8b5d..2d15ee7e
--- a/c/olmoe.c
+++ b/c/olmoe.c
@@@ -491,8 -498,9 +498,9 @@@ static void pin_hot_experts(Model *m)
int pinned_total = 0;
for (int l = 0; l < c->n_layers; l++) {
- uint32_t *freq_l = m->freq + (int64_t)l * c->n_experts;
+ uint32_t *freq_l = m->freq[l];
+ if (!freq_l) continue; /* a layer with no row cannot be ranked */
-
+
uint64_t layer_total = 0;
for (int e = 0; e < c->n_experts; e++) layer_total += freq_l[e];
if (layer_total == 0) continue;
@@@ -755,12 -763,13 +763,13 @@@ static void pilot_realload(Model *m, in
pthread_mutex_unlock(&g_pilot_mx);
return; /* all pinned/in-flight, skip */
}
-
+
/* LFRU eviction guard: don't displace a warm resident expert with a speculation */
- if (g_pilot_evict_guard && m->freq && m->last_access && lc->slots[lru].eid >= 0) {
+ if (g_pilot_evict_guard && m->freq && m->freq[layer] && m->last_access &&
+ lc->slots[lru].eid >= 0) {
int vid = lc->slots[lru].eid;
- uint64_t vs = lfru_score(m->freq[layer * c->n_experts + vid], m->last_access[layer * c->n_experts + vid], m->clock);
- uint64_t cs = lfru_score(m->freq[layer * c->n_experts + eid], m->last_access[layer * c->n_experts + eid], m->clock);
+ uint64_t vs = lfru_score(m->freq[layer][vid], m->last_access[layer * c->n_experts + vid], m->clock);
+ uint64_t cs = lfru_score(m->freq[layer][eid], m->last_access[layer * c->n_experts + eid], m->clock);
if (cs <= vs + (vs >> 2) + (4u << 8)) {
m->is_queued[layer * c->n_experts + eid] = 0;
pthread_mutex_unlock(&g_pilot_mx);