Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(olmoe): COLI_TEMP channel, and never evict an in-flight slot

Mario Gutiérrez3780c4e8f700
diff --git a/c/coli b/c/coli
index 021f6ae5..8d968f45 100755
--- a/c/coli
+++ b/c/coli
@@ -392,7 +392,10 @@ def env_for_engine(a, arch):
if arch == "olmoe":
env["CHAT"] = "1"
env["MAX_NEW"] = str(ngen_for(a))
- if a.temp is not None: env["TEMP"] = str(a.temp)
+ # olmoe reads COLI_TEMP like every other engine (line above). Setting the
+ # legacy TEMP alias here also clobbered %TEMP% with "0.7" for every child
+ # of the chat process on Windows — and olmoe never read COLI_TEMP, so the
+ # ONLY working channel was a poisoned system variable (#509's exact shape).
# --ram reaches the engines that read it. kimi_k3 gained a RAM budget in
# #855; before that `grep -c RAM_GB c/kimi_k3.c` returned 0, so `coli chat
# --ram 242` on Kimi K3 set an environment variable nobody looked at and the
diff --git a/c/olmoe.c b/c/olmoe.c
index 150fce0b..1e077762 100644
--- a/c/olmoe.c
+++ b/c/olmoe.c
@@ -468,7 +468,22 @@ static void expert_get(Model *m, int layer, int eid, Slot **out) {
if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i;
}
}
- if (lru < 0) lru = 0; /* absolute last resort: all in-flight, evict slot 0 */
+ while (lru < 0) {
+ /* EVERY slot is in flight: each buffer is owned by an unlocked pread
+ * in the pilot worker (or a demand load) that will publish into it.
+ * The old last resort (lru=0) stole such a slot mid-load — two writers
+ * racing the same slab, then whichever published last decided the
+ * expert id the resident bytes answered to. Wait for a publish instead
+ * and rescan; in-flight always drains because a load either finishes
+ * or the process is already dead in the water. */
+ pthread_mutex_unlock(&g_pilot_mx);
+ sleep_ms(1);
+ pthread_mutex_lock(&g_pilot_mx);
+ for (int i = 0; i < lc->n; i++) {
+ if (lc->slots[i].eid < 0) continue;
+ if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i;
+ }
+ }
s = &lc->slots[lru];
s->pinned = 0;
}
@@ -1323,7 +1338,15 @@ int main(int argc, char **argv) {
}
if (getenv("CHAT")) { /* interactive mode: bypasses the ref.json harness entirely */
- g_temp = getenv("TEMP") ? (float)atof(getenv("TEMP")) : g_temp;
+ /* #509 convention, same as the GLM engine: COLI_TEMP is the primary channel;
+ * TEMP stays a legacy alias ONLY when it is fully numeric — on Windows and under
+ * ROCm stacks %TEMP% names a directory, and atof("C:\...") == 0.0 would silently
+ * force greedy decoding for every olmoe chat on those hosts. */
+ if (getenv("COLI_TEMP")) g_temp = (float)atof(getenv("COLI_TEMP"));
+ else if (getenv("TEMP") && *getenv("TEMP")) {
+ char *tend; double tv = strtod(getenv("TEMP"), &tend);
+ if (tend != getenv("TEMP") && *tend == '\0') g_temp = (float)tv;
+ }
g_nuc = getenv("NUCLEUS") ? (float)atof(getenv("NUCLEUS")) : g_nuc;
int ctx_cap = getenv("CTX") ? atoi(getenv("CTX")) : 4096;
if (ctx_cap < 1 || ctx_cap > 4096) { /* attention()'s sc[4096] score buffer hard-caps this */
diff --git a/c/tests/test_v4_cli.py b/c/tests/test_v4_cli.py
index 47153272..e3d4dc65 100644
--- a/c/tests/test_v4_cli.py
+++ b/c/tests/test_v4_cli.py
@@ -70,7 +70,14 @@ class V4CliTest(unittest.TestCase):
env = self.cli.env_for_engine(args, "olmoe")
self.assertEqual(env["CHAT"], "1")
self.assertEqual(env["MAX_NEW"], "32")
- self.assertEqual(env["TEMP"], "0.25")
+ # #509 for olmoe: the engine reads COLI_TEMP like every other arch.
+ # The legacy TEMP channel double-poisoned on Windows: it overrode the
+ # child's %TEMP% directory with "0.25", while a real %TEMP% path read
+ # back as atof("C:\...") == 0.0 and silently forced greedy decoding.
+ # env starts from os.environ, so TEMP may be inherited — the launcher
+ # must simply never write the temperature there.
+ self.assertEqual(env["COLI_TEMP"], "0.25")
+ self.assertNotEqual(env.get("TEMP"), "0.25")
def test_olmoe_run_uses_its_engine_and_writes_one_prompt_line(self):
directory, root = self.make_model("olmoe")