Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(olmoe): --ram was inert, and "no --cap" meant a constant eight slots

JustVugge57abd71dfe4
diff --git a/c/coli b/c/coli
index 749ccb69..bb862caa 100755
--- a/c/coli
+++ b/c/coli
@@ -476,7 +476,7 @@ def env_for_engine(a, arch, plan=None):
# #855; before that `grep -c RAM_GB c/kimi_k3.c` returned 0, so `coli chat
# --ram 242` on Kimi K3 set an environment variable nobody looked at and the
# user's session ran itself out of memory with the flag apparently set.
- if arch in ("deepseek_v4", "kimi", "glm53"):
+ if arch in ("deepseek_v4", "kimi", "glm53", "olmoe"):
if a.ram: env["RAM_GB"] = str(a.ram)
if arch == "glm53":
# I densi vanno a int4 di default: sono 9,7 B parametri su 321, e in
diff --git a/c/family_registry.py b/c/family_registry.py
index 36418477..b4c981e6 100644
--- a/c/family_registry.py
+++ b/c/family_registry.py
@@ -1090,7 +1090,14 @@ FAMILIES = (
planner_unsupported_reason="",
expert_inventory=_individual_expert_inventory(_GLM_EXPERT),
config_section="root",
- limits=FamilyLimits(4096, 4096, 1024, 1024, 1, 8, "CTX"),
+ # implicit_cap 0, not 8: the engine sizes its expert cache from the RAM
+ # budget once the dense weights are resident (#1443), so "nobody chose a
+ # number" has to arrive as the sentinel. Eight slots per layer knows
+ # nothing about the model or the machine, and on OLMoE it is five times
+ # slower than the cache the same machine could hold: measured on a
+ # 1204-token prefill, cap 8 gives 22.8% expert hit rate and 0.045 tok/s,
+ # cap 64 gives 99.4% and 0.215 tok/s.
+ limits=FamilyLimits(4096, 4096, 1024, 1024, 1, 0, "CTX"),
capabilities=FamilyCapabilities(False, False, False, False),
has_gateway_adapter=True,
has_cli_adapter=True,
diff --git a/c/olmoe.c b/c/olmoe.c
index c0140c80..38548337 100644
--- a/c/olmoe.c
+++ b/c/olmoe.c
@@ -198,6 +198,26 @@ static double now_s(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t
static double rss_gb(void) { struct rusage r; getrusage(RUSAGE_SELF, &r); return r.ru_maxrss / (1024.0*1024.0*1024.0); } /* macOS: byte */
#else
static double rss_gb(void) { struct rusage r; getrusage(RUSAGE_SELF, &r); return r.ru_maxrss / (1024.0*1024.0); } /* Linux: KB */
+
+/* Quanta RAM il sistema offre ancora, in GB. Serve a dimensionare la cache
+ * degli esperti quando nessuno ha scelto un numero: senza questa, il default
+ * e' una costante che non sa nulla ne' del modello ne' della macchina. */
+static double mem_available_gb(void) {
+ double avail = 0.0;
+#ifdef __linux__
+ FILE *mi = fopen("/proc/meminfo", "r");
+ if (mi) {
+ char ln[256]; double v = 0;
+ while (fgets(ln, sizeof(ln), mi))
+ if (sscanf(ln, "MemAvailable: %lf", &v) == 1) { avail = v / 1e6; break; }
+ fclose(mi);
+ }
+#elif defined(_SC_AVPHYS_PAGES) && defined(_SC_PAGESIZE)
+ long pages = sysconf(_SC_AVPHYS_PAGES), page = sysconf(_SC_PAGESIZE);
+ if (pages > 0 && page > 0) avail = (double)pages * (double)page / 1e9;
+#endif
+ return avail;
+}
#endif
static float *falloc(int64_t n) { float *p = malloc(n*sizeof(float)); if(!p){fprintf(stderr,"OOM %ld\n",(long)n);exit(1);} return p; }
@@ -430,6 +450,54 @@ static void model_init_range(Model *m, const char *snap, int cap, int bits,
LD(gate, "mlp.gate.weight");
#undef LD
}
+ /* cap <= 0 is "you decide", the sentinel the launcher sends when nobody
+ * asked for a number (glm53 already reads it that way; #1443 asked for the
+ * same here). Sized HERE rather than in main because the dense weights are
+ * resident by this line: rss_gb() is a measurement, not a projection, which
+ * is what made the same budget in kimi_k3 (#855) the simpler of the two.
+ *
+ * Until now "no explicit choice" arrived as a constant 8 slots per layer,
+ * which knows nothing about the model or the machine. On a 16 GB box whose
+ * entire expert set is 6.5 GB that constant costs a factor of five:
+ * measured on a 1204-token prefill, cap 8 gives 22.8% expert hit rate and
+ * 0.045 tok/s, cap 64 gives 99.4% and 0.215 tok/s. */
+ if (cap <= 0) {
+ double resident = rss_gb();
+ double avail = mem_available_gb();
+ const char *ram_env = getenv("RAM_GB");
+ double ram_arg = ram_env ? atof(ram_env) : 0.0;
+ /* An explicit --ram is a ceiling on the WHOLE process. Without it take
+ * 88% of what the OS still offers and add what we already hold, the
+ * same fraction and the same reason as the sibling engines: overshoot
+ * means an OOM kill mid-generation, which is worse than a small cache. */
+ double budget = ram_arg > 0.0 ? ram_arg : resident + avail * 0.88;
+ /* The KV cache is allocated later, at the first request, so project it:
+ * two tensors per layer of n_heads * max_t * head_dim floats. CTX caps
+ * at 4096 because attention()'s score buffer does. */
+ int max_t = getenv("CTX") ? atoi(getenv("CTX")) : 4096;
+ if (max_t < 1 || max_t > 4096) max_t = 4096;
+ double kv_gb = 2.0 * (double)c->n_layers * c->n_heads * max_t *
+ c->head_dim * sizeof(float) / 1e9;
+ /* One slot holds one expert: three int8 matrices plus their row scales,
+ * the same arithmetic the Segment adapter uses to turn a memory limit
+ * into a cap. */
+ double slot_gb = ((double)c->hidden * c->inter * 3.0 +
+ (double)(c->inter * 2 + c->hidden) * sizeof(float)) / 1e9;
+ int layers = layer_end - layer_begin;
+ if (layers < 1) layers = 1;
+ double room = budget - resident - kv_gb - 0.5; /* 0.5 GB: activations */
+ int derived = room > 0.0 && slot_gb > 0.0
+ ? (int)(room / slot_gb / (double)layers) : 0;
+ if (derived < 1) derived = 1;
+ if (derived > c->n_experts) derived = c->n_experts;
+ fprintf(stderr, "[cache] %d slots/layer of %d experts: %.1f GB budget "
+ "(%s), %.1f GB dense resident, %.1f GB projected KV, "
+ "%.0f MB per expert\n",
+ derived, c->n_experts, budget,
+ ram_arg > 0.0 ? "RAM_GB" : "88% of what the OS still offers",
+ resident, kv_gb, slot_gb * 1000.0);
+ cap = derived;
+ }
m->cache = calloc(c->n_layers, sizeof(LCache));
for (int i = layer_begin; i < layer_end; i++) {
m->cache[i].cap = cap;
diff --git a/c/tests/test_olmoe_cap_budget.py b/c/tests/test_olmoe_cap_budget.py
new file mode 100644
index 00000000..c0d6a567
--- /dev/null
+++ b/c/tests/test_olmoe_cap_budget.py
@@ -0,0 +1,77 @@
+"""OLMoE sizes its expert cache from the RAM budget, and only when asked to.
+
+Before #1443 the launcher forwarded `--ram` to three engines and olmoe was not
+one of them, and "no explicit --cap" reached the engine as a constant eight
+slots per layer. Eight knows nothing about the model or the machine: on a box
+whose whole expert set fits in RAM it costs a factor of five, and on a small box
+it is no safer than a budget-derived number would be.
+
+What is asserted here is the contract, not a speed: the sentinel (cap 0) makes
+the engine choose, an explicit cap is left alone, and the choice is bounded by
+the budget it was given. The engine prints the derivation, so the assertions
+read the line it prints rather than guessing at behaviour.
+
+Needs a converted tiny OLMoE container and a built engine; both are what the
+OLMoE tiny oracle job already builds.
+"""
+import os
+import re
+import subprocess
+import unittest
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent.parent
+ENGINE = HERE / ("olmoe.exe" if os.name == "nt" else "olmoe")
+FIXTURE = Path(os.environ.get("COLI_OLMOE_FIXTURE", ""))
+CACHE_LINE = re.compile(r"^\[cache\] (\d+) slots/layer of (\d+) experts", re.M)
+
+
+def derived_cap(ram_gb=None, cap="0"):
+ """Run the engine far enough to print its cache line, then stop it."""
+ environment = {**os.environ, "SNAP": str(FIXTURE), "SERVE": "1"}
+ if ram_gb is not None:
+ environment["RAM_GB"] = str(ram_gb)
+ else:
+ environment.pop("RAM_GB", None)
+ finished = subprocess.run([str(ENGINE), cap, "8"], input=b"",
+ stdout=subprocess.DEVNULL, stderr=subprocess.PIPE,
+ env=environment, timeout=120)
+ match = CACHE_LINE.search(finished.stderr.decode("utf-8", "replace"))
+ return (int(match.group(1)), int(match.group(2))) if match else (None, None)
+
+
[email protected](ENGINE.exists(), "olmoe engine is not built")
[email protected](FIXTURE.name and (FIXTURE / "config.json").is_file(),
+ "COLI_OLMOE_FIXTURE not set to a converted OLMoE container")
+class OlmoeCapBudgetTest(unittest.TestCase):
+ def test_an_explicit_cap_is_never_second_guessed(self):
+ cap, _ = derived_cap(ram_gb=64, cap="4")
+ self.assertIsNone(cap, "the engine re-derived a cap the caller had chosen")
+
+ def test_a_generous_budget_holds_every_expert(self):
+ cap, experts = derived_cap(ram_gb=64)
+ self.assertIsNotNone(cap, "the sentinel produced no cache line")
+ self.assertEqual(cap, experts,
+ "a budget with room for the whole expert set should hold it")
+
+ def test_a_budget_below_the_floor_still_runs(self):
+ """Not an error: one slot per layer is slow, refusing to start is worse."""
+ cap, experts = derived_cap(ram_gb=0.4)
+ self.assertEqual(cap, 1)
+ self.assertGreater(experts, 1)
+
+ def test_the_budget_bounds_the_choice(self):
+ small, _ = derived_cap(ram_gb=0.4)
+ large, _ = derived_cap(ram_gb=64)
+ self.assertLess(small, large,
+ "the derived cap ignored the budget it was given")
+
+ def test_without_a_budget_it_still_decides(self):
+ cap, experts = derived_cap(ram_gb=None)
+ self.assertIsNotNone(cap, "no RAM_GB must mean 'measure', not 'give up'")
+ self.assertGreaterEqual(cap, 1)
+ self.assertLessEqual(cap, experts)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index 19bfc48e..dcb39ce3 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -1181,10 +1181,14 @@ class CapSentinelShimTest(unittest.TestCase):
self._spawn_argv("engine", str(model))
def test_cap_for_arch_is_the_single_translation_point(self):
+ # 0 is the "you decide" sentinel, and it goes to the engines that
+ # actually do decide: glm resolves it platform-aware, olmoe sizes its
+ # expert cache from the RAM budget once the dense weights are resident
+ # (#1443). The others still get the legacy eight slots per layer.
self.assertEqual(cap_for_arch("glm", None), 0)
+ self.assertEqual(cap_for_arch("olmoe", None), 0)
self.assertEqual(cap_for_arch("inkling", None), 8)
self.assertEqual(cap_for_arch("kimi", None), 8)
- self.assertEqual(cap_for_arch("olmoe", None), 8)
self.assertEqual(cap_for_arch("glm", 3), 3)
self.assertEqual(cap_for_arch("inkling", 3), 3)
self.assertEqual(cap_for_arch("inkling", 0), 0) # explicit 0 is explicit
diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md
index 66edc822..9e46a569 100644
--- a/docs/ENVIRONMENT.md
+++ b/docs/ENVIRONMENT.md
@@ -40,7 +40,7 @@ Format: `VAR` β€” default β€” effect.
| Variable | Default | Effect |
|---|---|---|
-| `RAM_GB` | `0` (auto β‰ˆ 88% of free RAM) | RAM budget in GB for the resident/streamed expert working set. Higher β†’ more experts stay hot β†’ higher cache hit rate. |
+| `RAM_GB` | `0` (auto β‰ˆ 88% of free RAM) | RAM budget in GB for the resident/streamed expert working set. Higher β†’ more experts stay hot β†’ higher cache hit rate. Read by colibri, kimi_k3, glm53 and olmoe; on olmoe it sizes the expert cache once the dense weights are resident, and only when no `--cap` was given. |
| `CTX` | `4096` | Maximum context length (tokens) the KV cache is sized for. |
| `COLI_PREFILL_CHUNK` | `0` (off) | Run a long prompt through the layers in N-token slices instead of one pass. Every S-scaled activation buffer shrinks from prompt-sized to chunk-sized, which is the remedy when a long prompt exhausts CUDA scratch. Byte-identical output (verified at N=256). Skipped under an active MTP draft. **Cost:** a slice of 512 tokens already routes to essentially every expert of every layer (`P(miss) = (1-topk/n_experts)^N`), so each slice re-reads the whole non-resident expert set -- prefer the largest N that still fits your scratch. |
| `NGEN` | `256` (engine) | Max tokens to generate before stopping (stop tokens can end sooner). `coli --ngen` defaults to `1024`. |