Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
fix(olmoe): --ram was inert, and "no --cap" meant a constant eight slots
JustVugge57abd71dfe4
diff --git a/c/coli b/c/coliindex 749ccb69..bb862caa 100755--- a/c/coli+++ b/c/coli@@ -476,7 +476,7 @@ def env_for_engine(a, arch, plan=None):# #855; before that `grep -c RAM_GB c/kimi_k3.c` returned 0, so `coli chat# --ram 242` on Kimi K3 set an environment variable nobody looked at and the# user's session ran itself out of memory with the flag apparently set.- if arch in ("deepseek_v4", "kimi", "glm53"):+ if arch in ("deepseek_v4", "kimi", "glm53", "olmoe"):if a.ram: env["RAM_GB"] = str(a.ram)if arch == "glm53":# I densi vanno a int4 di default: sono 9,7 B parametri su 321, e indiff --git a/c/family_registry.py b/c/family_registry.pyindex 36418477..b4c981e6 100644--- a/c/family_registry.py+++ b/c/family_registry.py@@ -1090,7 +1090,14 @@ FAMILIES = (planner_unsupported_reason="",expert_inventory=_individual_expert_inventory(_GLM_EXPERT),config_section="root",- limits=FamilyLimits(4096, 4096, 1024, 1024, 1, 8, "CTX"),+ # implicit_cap 0, not 8: the engine sizes its expert cache from the RAM+ # budget once the dense weights are resident (#1443), so "nobody chose a+ # number" has to arrive as the sentinel. Eight slots per layer knows+ # nothing about the model or the machine, and on OLMoE it is five times+ # slower than the cache the same machine could hold: measured on a+ # 1204-token prefill, cap 8 gives 22.8% expert hit rate and 0.045 tok/s,+ # cap 64 gives 99.4% and 0.215 tok/s.+ limits=FamilyLimits(4096, 4096, 1024, 1024, 1, 0, "CTX"),capabilities=FamilyCapabilities(False, False, False, False),has_gateway_adapter=True,has_cli_adapter=True,diff --git a/c/olmoe.c b/c/olmoe.cindex c0140c80..38548337 100644--- a/c/olmoe.c+++ b/c/olmoe.c@@ -198,6 +198,26 @@ static double now_s(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &tstatic double rss_gb(void) { struct rusage r; getrusage(RUSAGE_SELF, &r); return r.ru_maxrss / (1024.0*1024.0*1024.0); } /* macOS: byte */#elsestatic double rss_gb(void) { struct rusage r; getrusage(RUSAGE_SELF, &r); return r.ru_maxrss / (1024.0*1024.0); } /* Linux: KB */++/* Quanta RAM il sistema offre ancora, in GB. Serve a dimensionare la cache+ * degli esperti quando nessuno ha scelto un numero: senza questa, il default+ * e' una costante che non sa nulla ne' del modello ne' della macchina. */+static double mem_available_gb(void) {+ double avail = 0.0;+#ifdef __linux__+ FILE *mi = fopen("/proc/meminfo", "r");+ if (mi) {+ char ln[256]; double v = 0;+ while (fgets(ln, sizeof(ln), mi))+ if (sscanf(ln, "MemAvailable: %lf", &v) == 1) { avail = v / 1e6; break; }+ fclose(mi);+ }+#elif defined(_SC_AVPHYS_PAGES) && defined(_SC_PAGESIZE)+ long pages = sysconf(_SC_AVPHYS_PAGES), page = sysconf(_SC_PAGESIZE);+ if (pages > 0 && page > 0) avail = (double)pages * (double)page / 1e9;+#endif+ return avail;+}#endifstatic float *falloc(int64_t n) { float *p = malloc(n*sizeof(float)); if(!p){fprintf(stderr,"OOM %ld\n",(long)n);exit(1);} return p; }@@ -430,6 +450,54 @@ static void model_init_range(Model *m, const char *snap, int cap, int bits,LD(gate, "mlp.gate.weight");#undef LD}+ /* cap <= 0 is "you decide", the sentinel the launcher sends when nobody+ * asked for a number (glm53 already reads it that way; #1443 asked for the+ * same here). Sized HERE rather than in main because the dense weights are+ * resident by this line: rss_gb() is a measurement, not a projection, which+ * is what made the same budget in kimi_k3 (#855) the simpler of the two.+ *+ * Until now "no explicit choice" arrived as a constant 8 slots per layer,+ * which knows nothing about the model or the machine. On a 16 GB box whose+ * entire expert set is 6.5 GB that constant costs a factor of five:+ * measured on a 1204-token prefill, cap 8 gives 22.8% expert hit rate and+ * 0.045 tok/s, cap 64 gives 99.4% and 0.215 tok/s. */+ if (cap <= 0) {+ double resident = rss_gb();+ double avail = mem_available_gb();+ const char *ram_env = getenv("RAM_GB");+ double ram_arg = ram_env ? atof(ram_env) : 0.0;+ /* An explicit --ram is a ceiling on the WHOLE process. Without it take+ * 88% of what the OS still offers and add what we already hold, the+ * same fraction and the same reason as the sibling engines: overshoot+ * means an OOM kill mid-generation, which is worse than a small cache. */+ double budget = ram_arg > 0.0 ? ram_arg : resident + avail * 0.88;+ /* The KV cache is allocated later, at the first request, so project it:+ * two tensors per layer of n_heads * max_t * head_dim floats. CTX caps+ * at 4096 because attention()'s score buffer does. */+ int max_t = getenv("CTX") ? atoi(getenv("CTX")) : 4096;+ if (max_t < 1 || max_t > 4096) max_t = 4096;+ double kv_gb = 2.0 * (double)c->n_layers * c->n_heads * max_t *+ c->head_dim * sizeof(float) / 1e9;+ /* One slot holds one expert: three int8 matrices plus their row scales,+ * the same arithmetic the Segment adapter uses to turn a memory limit+ * into a cap. */+ double slot_gb = ((double)c->hidden * c->inter * 3.0 ++ (double)(c->inter * 2 + c->hidden) * sizeof(float)) / 1e9;+ int layers = layer_end - layer_begin;+ if (layers < 1) layers = 1;+ double room = budget - resident - kv_gb - 0.5; /* 0.5 GB: activations */+ int derived = room > 0.0 && slot_gb > 0.0+ ? (int)(room / slot_gb / (double)layers) : 0;+ if (derived < 1) derived = 1;+ if (derived > c->n_experts) derived = c->n_experts;+ fprintf(stderr, "[cache] %d slots/layer of %d experts: %.1f GB budget "+ "(%s), %.1f GB dense resident, %.1f GB projected KV, "+ "%.0f MB per expert\n",+ derived, c->n_experts, budget,+ ram_arg > 0.0 ? "RAM_GB" : "88% of what the OS still offers",+ resident, kv_gb, slot_gb * 1000.0);+ cap = derived;+ }m->cache = calloc(c->n_layers, sizeof(LCache));for (int i = layer_begin; i < layer_end; i++) {m->cache[i].cap = cap;diff --git a/c/tests/test_olmoe_cap_budget.py b/c/tests/test_olmoe_cap_budget.pynew file mode 100644index 00000000..c0d6a567--- /dev/null+++ b/c/tests/test_olmoe_cap_budget.py@@ -0,0 +1,77 @@+"""OLMoE sizes its expert cache from the RAM budget, and only when asked to.++Before #1443 the launcher forwarded `--ram` to three engines and olmoe was not+one of them, and "no explicit --cap" reached the engine as a constant eight+slots per layer. Eight knows nothing about the model or the machine: on a box+whose whole expert set fits in RAM it costs a factor of five, and on a small box+it is no safer than a budget-derived number would be.++What is asserted here is the contract, not a speed: the sentinel (cap 0) makes+the engine choose, an explicit cap is left alone, and the choice is bounded by+the budget it was given. The engine prints the derivation, so the assertions+read the line it prints rather than guessing at behaviour.++Needs a converted tiny OLMoE container and a built engine; both are what the+OLMoE tiny oracle job already builds.+"""+import os+import re+import subprocess+import unittest+from pathlib import Path++HERE = Path(__file__).resolve().parent.parent+ENGINE = HERE / ("olmoe.exe" if os.name == "nt" else "olmoe")+FIXTURE = Path(os.environ.get("COLI_OLMOE_FIXTURE", ""))+CACHE_LINE = re.compile(r"^\[cache\] (\d+) slots/layer of (\d+) experts", re.M)+++def derived_cap(ram_gb=None, cap="0"):+ """Run the engine far enough to print its cache line, then stop it."""+ environment = {**os.environ, "SNAP": str(FIXTURE), "SERVE": "1"}+ if ram_gb is not None:+ environment["RAM_GB"] = str(ram_gb)+ else:+ environment.pop("RAM_GB", None)+ finished = subprocess.run([str(ENGINE), cap, "8"], input=b"",+ stdout=subprocess.DEVNULL, stderr=subprocess.PIPE,+ env=environment, timeout=120)+ match = CACHE_LINE.search(finished.stderr.decode("utf-8", "replace"))+ return (int(match.group(1)), int(match.group(2))) if match else (None, None)++[email protected](ENGINE.exists(), "olmoe engine is not built")[email protected](FIXTURE.name and (FIXTURE / "config.json").is_file(),+ "COLI_OLMOE_FIXTURE not set to a converted OLMoE container")+class OlmoeCapBudgetTest(unittest.TestCase):+ def test_an_explicit_cap_is_never_second_guessed(self):+ cap, _ = derived_cap(ram_gb=64, cap="4")+ self.assertIsNone(cap, "the engine re-derived a cap the caller had chosen")++ def test_a_generous_budget_holds_every_expert(self):+ cap, experts = derived_cap(ram_gb=64)+ self.assertIsNotNone(cap, "the sentinel produced no cache line")+ self.assertEqual(cap, experts,+ "a budget with room for the whole expert set should hold it")++ def test_a_budget_below_the_floor_still_runs(self):+ """Not an error: one slot per layer is slow, refusing to start is worse."""+ cap, experts = derived_cap(ram_gb=0.4)+ self.assertEqual(cap, 1)+ self.assertGreater(experts, 1)++ def test_the_budget_bounds_the_choice(self):+ small, _ = derived_cap(ram_gb=0.4)+ large, _ = derived_cap(ram_gb=64)+ self.assertLess(small, large,+ "the derived cap ignored the budget it was given")++ def test_without_a_budget_it_still_decides(self):+ cap, experts = derived_cap(ram_gb=None)+ self.assertIsNotNone(cap, "no RAM_GB must mean 'measure', not 'give up'")+ self.assertGreaterEqual(cap, 1)+ self.assertLessEqual(cap, experts)+++if __name__ == "__main__":+ unittest.main()diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pyindex 19bfc48e..dcb39ce3 100644--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@ -1181,10 +1181,14 @@ class CapSentinelShimTest(unittest.TestCase):self._spawn_argv("engine", str(model))def test_cap_for_arch_is_the_single_translation_point(self):+ # 0 is the "you decide" sentinel, and it goes to the engines that+ # actually do decide: glm resolves it platform-aware, olmoe sizes its+ # expert cache from the RAM budget once the dense weights are resident+ # (#1443). The others still get the legacy eight slots per layer.self.assertEqual(cap_for_arch("glm", None), 0)+ self.assertEqual(cap_for_arch("olmoe", None), 0)self.assertEqual(cap_for_arch("inkling", None), 8)self.assertEqual(cap_for_arch("kimi", None), 8)- self.assertEqual(cap_for_arch("olmoe", None), 8)self.assertEqual(cap_for_arch("glm", 3), 3)self.assertEqual(cap_for_arch("inkling", 3), 3)self.assertEqual(cap_for_arch("inkling", 0), 0) # explicit 0 is explicitdiff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.mdindex 66edc822..9e46a569 100644--- a/docs/ENVIRONMENT.md+++ b/docs/ENVIRONMENT.md@@ -40,7 +40,7 @@ Format: `VAR` β default β effect.| Variable | Default | Effect ||---|---|---|-| `RAM_GB` | `0` (auto β 88% of free RAM) | RAM budget in GB for the resident/streamed expert working set. Higher β more experts stay hot β higher cache hit rate. |+| `RAM_GB` | `0` (auto β 88% of free RAM) | RAM budget in GB for the resident/streamed expert working set. Higher β more experts stay hot β higher cache hit rate. Read by colibri, kimi_k3, glm53 and olmoe; on olmoe it sizes the expert cache once the dense weights are resident, and only when no `--cap` was given. || `CTX` | `4096` | Maximum context length (tokens) the KV cache is sized for. || `COLI_PREFILL_CHUNK` | `0` (off) | Run a long prompt through the layers in N-token slices instead of one pass. Every S-scaled activation buffer shrinks from prompt-sized to chunk-sized, which is the remedy when a long prompt exhausts CUDA scratch. Byte-identical output (verified at N=256). Skipped under an active MTP draft. **Cost:** a slice of 512 tokens already routes to essentially every expert of every layer (`P(miss) = (1-topk/n_experts)^N`), so each slice re-reads the whole non-resident expert set -- prefer the largest N that still fits your scratch. || `NGEN` | `256` (engine) | Max tokens to generate before stopping (stop tokens can end sooner). `coli --ngen` defaults to `1024`. |