Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(olmoe): COLI_TEMP channel, and never evict an in-flight slot
Mario Gutiérrez3780c4e8f700
diff --git a/c/coli b/c/coliindex 021f6ae5..8d968f45 100755--- a/c/coli+++ b/c/coli@@ -392,7 +392,10 @@ def env_for_engine(a, arch):if arch == "olmoe":env["CHAT"] = "1"env["MAX_NEW"] = str(ngen_for(a))- if a.temp is not None: env["TEMP"] = str(a.temp)+ # olmoe reads COLI_TEMP like every other engine (line above). Setting the+ # legacy TEMP alias here also clobbered %TEMP% with "0.7" for every child+ # of the chat process on Windows — and olmoe never read COLI_TEMP, so the+ # ONLY working channel was a poisoned system variable (#509's exact shape).# --ram reaches the engines that read it. kimi_k3 gained a RAM budget in# #855; before that `grep -c RAM_GB c/kimi_k3.c` returned 0, so `coli chat# --ram 242` on Kimi K3 set an environment variable nobody looked at and thediff --git a/c/olmoe.c b/c/olmoe.cindex 150fce0b..1e077762 100644--- a/c/olmoe.c+++ b/c/olmoe.c@@ -468,7 +468,22 @@ static void expert_get(Model *m, int layer, int eid, Slot **out) {if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i;}}- if (lru < 0) lru = 0; /* absolute last resort: all in-flight, evict slot 0 */+ while (lru < 0) {+ /* EVERY slot is in flight: each buffer is owned by an unlocked pread+ * in the pilot worker (or a demand load) that will publish into it.+ * The old last resort (lru=0) stole such a slot mid-load — two writers+ * racing the same slab, then whichever published last decided the+ * expert id the resident bytes answered to. Wait for a publish instead+ * and rescan; in-flight always drains because a load either finishes+ * or the process is already dead in the water. */+ pthread_mutex_unlock(&g_pilot_mx);+ sleep_ms(1);+ pthread_mutex_lock(&g_pilot_mx);+ for (int i = 0; i < lc->n; i++) {+ if (lc->slots[i].eid < 0) continue;+ if (lru < 0 || lc->slots[i].used < lc->slots[lru].used) lru = i;+ }+ }s = &lc->slots[lru];s->pinned = 0;}@@ -1323,7 +1338,15 @@ int main(int argc, char **argv) {}if (getenv("CHAT")) { /* interactive mode: bypasses the ref.json harness entirely */- g_temp = getenv("TEMP") ? (float)atof(getenv("TEMP")) : g_temp;+ /* #509 convention, same as the GLM engine: COLI_TEMP is the primary channel;+ * TEMP stays a legacy alias ONLY when it is fully numeric — on Windows and under+ * ROCm stacks %TEMP% names a directory, and atof("C:\...") == 0.0 would silently+ * force greedy decoding for every olmoe chat on those hosts. */+ if (getenv("COLI_TEMP")) g_temp = (float)atof(getenv("COLI_TEMP"));+ else if (getenv("TEMP") && *getenv("TEMP")) {+ char *tend; double tv = strtod(getenv("TEMP"), &tend);+ if (tend != getenv("TEMP") && *tend == '\0') g_temp = (float)tv;+ }g_nuc = getenv("NUCLEUS") ? (float)atof(getenv("NUCLEUS")) : g_nuc;int ctx_cap = getenv("CTX") ? atoi(getenv("CTX")) : 4096;if (ctx_cap < 1 || ctx_cap > 4096) { /* attention()'s sc[4096] score buffer hard-caps this */diff --git a/c/tests/test_v4_cli.py b/c/tests/test_v4_cli.pyindex 47153272..e3d4dc65 100644--- a/c/tests/test_v4_cli.py+++ b/c/tests/test_v4_cli.py@@ -70,7 +70,14 @@ class V4CliTest(unittest.TestCase):env = self.cli.env_for_engine(args, "olmoe")self.assertEqual(env["CHAT"], "1")self.assertEqual(env["MAX_NEW"], "32")- self.assertEqual(env["TEMP"], "0.25")+ # #509 for olmoe: the engine reads COLI_TEMP like every other arch.+ # The legacy TEMP channel double-poisoned on Windows: it overrode the+ # child's %TEMP% directory with "0.25", while a real %TEMP% path read+ # back as atof("C:\...") == 0.0 and silently forced greedy decoding.+ # env starts from os.environ, so TEMP may be inherited — the launcher+ # must simply never write the temperature there.+ self.assertEqual(env["COLI_TEMP"], "0.25")+ self.assertNotEqual(env.get("TEMP"), "0.25")def test_olmoe_run_uses_its_engine_and_writes_one_prompt_line(self):directory, root = self.make_model("olmoe")