Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

kimi: a RAM budget, and make --ram mean something (#855)

JustVugg0f9742244149
diff --git a/c/Makefile b/c/Makefile
index b573d792..f2ae37e9 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -741,6 +741,9 @@ tests/test_json$(EXE): tests/test_json.c json.h
tests/test_tok_o200k$(EXE): tests/test_tok_o200k.c tok.h tok_unicode.h tok_unicode_o200k.h json.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+tests/test_k3_ram_budget$(EXE): tests/test_k3_ram_budget.c kimi_k3.c st.h tok.h quant.h compat.h omp_tune.h route_trace.h kv_prefix.h
+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+
tests/test_tok_kimi_tiny$(EXE): tests/test_tok_kimi_tiny.c tok.h tok_unicode.h tok_unicode_o200k.h json.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_st_pread$(EXE): tests/test_st_pread.c st.h json.h compat.h
diff --git a/c/coli b/c/coli
index d735ce29..23c58433 100755
--- a/c/coli
+++ b/c/coli
@@ -294,8 +294,13 @@ def env_for_engine(a, arch):
env.setdefault("OMP_PLACES", "cores")
if a.ngen: env["NGEN"] = str(a.ngen)
if a.temp is not None: env["COLI_TEMP"] = str(a.temp)
- if arch == "deepseek_v4":
+ # --ram reaches the engines that read it. kimi_k3 gained a RAM budget in
+ # #855; before that `grep -c RAM_GB c/kimi_k3.c` returned 0, so `coli chat
+ # --ram 242` on Kimi K3 set an environment variable nobody looked at and the
+ # user's session ran itself out of memory with the flag apparently set.
+ if arch in ("deepseek_v4", "kimi"):
if a.ram: env["RAM_GB"] = str(a.ram)
+ if arch == "deepseek_v4":
if a.ctx: env["CTX"] = str(a.ctx)
# Speculation stays opt-in. Real multi-turn chat measured only 1/15
# accepted prompt-lookup candidates; recurrent-state replay dominated
diff --git a/c/kimi_k3.c b/c/kimi_k3.c
index c056c908..2bdd64a9 100644
--- a/c/kimi_k3.c
+++ b/c/kimi_k3.c
@@ -77,6 +77,10 @@
#ifdef _OPENMP
#include <omp.h>
#endif
+#ifdef __APPLE__
+#include <sys/sysctl.h>
+#include <mach/mach.h>
+#endif
#include "st.h"
#include "tok.h"
#include "quant.h"
@@ -176,6 +180,47 @@ typedef struct {
} Model;
static double now_s(void){ struct timespec t; clock_gettime(CLOCK_MONOTONIC,&t); return t.tv_sec+t.tv_nsec*1e-9; }
+
+/* How many expert slots per layer fit a budget. PURE -- no globals, no model,
+ * no I/O -- so the arithmetic that #855 got wrong can be tested against the
+ * reported numbers without a 600 GB checkpoint. Returns the cap; writes the
+ * bytes left for experts through `for_experts_out` when non-NULL.
+ *
+ * reserve = page cache + activations + KV, all in GB. `cap_requested` is what
+ * K3_EXPERT_GB asked for: this only ever LOWERS it, never raises it, so an
+ * explicit small cache stays small. */
+static int k3_cap_for_ram(double budget_gb, double resident_gb, double reserve_gb,
+ double slot_gb, int nmoe, int cap_requested,
+ int n_experts, double *for_experts_out){
+ double for_experts = budget_gb - resident_gb - reserve_gb;
+ if(for_experts_out) *for_experts_out = for_experts;
+ if(nmoe < 1) nmoe = 1;
+ if(!(slot_gb > 0.0)) return cap_requested;
+ int fits = for_experts > 0.0 ? (int)(for_experts/(slot_gb*(double)nmoe)) : 0;
+ if(fits > n_experts) fits = n_experts;
+ return fits < cap_requested ? fits : cap_requested;
+}
+
+/* Memory the OS says is still reclaimable without swapping, in GB; 0 if unknown.
+ * The same quantity colibri.c's cap_for_ram() budgets against -- Linux
+ * MemAvailable, Windows ullAvailPhys, macOS free+inactive+purgeable. #855. */
+static double k3_mem_avail(void){
+#ifdef _WIN32
+ double total=0, avail=0; compat_meminfo(&total,&avail); return avail;
+#elif defined(__APPLE__)
+ int64_t pgsz=0; size_t sl=sizeof(pgsz);
+ if(sysctlbyname("hw.pagesize",&pgsz,&sl,NULL,0)||pgsz<=0) pgsz=16384;
+ vm_statistics64_data_t vs; mach_msg_type_number_t nc=HOST_VM_INFO64_COUNT;
+ if(host_statistics64(mach_host_self(),HOST_VM_INFO64,(host_info64_t)&vs,&nc)!=KERN_SUCCESS)
+ return 0;
+ return (double)(vs.free_count+vs.inactive_count+vs.purgeable_count)*(double)pgsz/1e9;
+#else
+ FILE *f=fopen("/proc/meminfo","r"); if(!f) return 0;
+ char ln[256]; double kb=0;
+ while(fgets(ln,sizeof(ln),f)) if(sscanf(ln,"MemAvailable: %lf",&kb)==1) break;
+ fclose(f); return kb/1e6;
+#endif
+}
static double rss_gb(void){ struct rusage r; getrusage(RUSAGE_SELF,&r);
#if defined(__APPLE__)
return r.ru_maxrss/(1024.0*1024.0*1024.0);
@@ -636,6 +681,94 @@ static void model_init(Model *m, const char *snap, int n_layers_env){
* regardless of K3_EXPERT_GB. */
if(cap<1) cap=1;
if(cap>c->n_experts) cap=c->n_experts;
+
+ /* ---- RAM budget (#855) --------------------------------------------------
+ * K3_EXPERT_GB used to be the whole story: cap = egb / slot / layers, with
+ * nothing subtracted for what is already resident, no MemAvailable, no
+ * reserve for KV or the page cache, and no guard during the run.
+ *
+ * Reported on a 256 GB box with K3_EXPERT_GB=220:
+ *
+ * 136 slots x 17.5 MB x 93 layers = 216.2 GB cache, once it fills
+ * + 35.2 GB already resident
+ * = 251.4 GB peak, of 256
+ *
+ * The cache is EMPTY when the init line prints and fills as the session
+ * runs, so the first request answers and a later one dies -- which is what
+ * "colibri engine exited unexpectedly" was.
+ *
+ * And --ram was inert here. `grep -c RAM_GB kimi_k3.c` returned 0 against 10
+ * in colibri.c: the one flag a user would reach for to prevent exactly this
+ * reached the environment and was never read. Same defect shape as #805 and
+ * #858 -- a mechanism that lands in one engine and not its siblings.
+ *
+ * Unlike colibri.c's cap_for_ram this runs AFTER the dense weights are in,
+ * so the resident set is MEASURED rather than projected. rss_gb() is the
+ * truth here, not a model of it, which makes this budget the simpler of the
+ * two rather than the more elaborate. */
+ {
+ double resident = rss_gb();
+ double avail_now = k3_mem_avail();
+ double ram_env = getenv("RAM_GB") ? atof(getenv("RAM_GB")) : 0.0;
+ /* Explicit --ram is a ceiling on the WHOLE process. Absent, take 88% of
+ * what the OS still offers and add what we already hold -- the same
+ * fraction colibri.c uses, and for the same reason: overshooting means
+ * an OOM kill mid-generation, which is far worse than a smaller cache. */
+ double budget = ram_env > 0 ? ram_env : resident + avail_now*0.88;
+
+ /* KV is allocated later, at the first request, so it has to be projected
+ * here. n_layers x max_t x (kv_lora + qk_rope) x 4, skipping KDA layers,
+ * with the same K3_MAXT default the serve path uses. */
+ int max_t = getenv("K3_MAXT") ? atoi(getenv("K3_MAXT")) : 8192;
+ if(max_t < 1) max_t = 8192;
+ int nkv = 0; for(int i=0;i<c->n_layers;i++) if(!m->L[i].kda) nkv++;
+ double kv_gb = (double)nkv*(double)max_t*(double)(c->kv_lora+c->qk_rope)*4.0/1e9;
+ /* 2.5 GB page cache -- measured on Linux 2026-07-06: strangling it drops
+ * buffered pread from ~800 to ~180 MB/s and the last GB of LRU costs
+ * more in lost bandwidth than it returns. 1.2 GB activations/logits. */
+ double reserve = 2.5 + 1.2 + kv_gb;
+ double for_experts = 0.0;
+
+ double slot_gb = (double)m->e_slot/1e9;
+ int cap_fit = k3_cap_for_ram(budget, resident, reserve, slot_gb,
+ nmoe, cap, c->n_experts, &for_experts);
+
+ if(cap_fit < cap){
+ /* Name every term. The user's number is not being ignored, it is
+ * being clamped, and they cannot check the clamp without the parts. */
+ fprintf(stderr,"[K3][RAM_GB=%.1f%s] resident %.1f GB + reserve %.1f GB "
+ "(page cache 2.5, activations 1.2, KV %dx%d %.1f) -> %.1f GB for experts; "
+ "cache %d->%d/layer (%.1f MB/slot, %d layers; projected peak %.1f GB)\n",
+ budget, ram_env>0?"":" auto", resident, reserve, nkv, max_t, kv_gb,
+ for_experts>0?for_experts:0.0, cap, cap_fit>0?cap_fit:1,
+ slot_gb*1000.0, nmoe,
+ resident + reserve + (double)(cap_fit>0?cap_fit:1)*slot_gb*nmoe);
+ if(getenv("K3_EXPERT_GB"))
+ fprintf(stderr,"[K3] K3_EXPERT_GB=%.0f does not fit alongside the "
+ "%.1f GB already resident. It is a request, not a reservation.\n",
+ egb, resident);
+ cap = cap_fit;
+ }
+
+ if(cap < 1){
+ /* Not even one slot per layer. Saying cap=1 and continuing turns
+ * "does not fit" into "overruns", which is the OOM kill this exists
+ * to avoid -- and the kernel kills with SIGKILL, so the engine dies
+ * with no error and no log at all. Refuse, unless told otherwise. */
+ cap = 1;
+ double peak = resident + reserve + slot_gb*nmoe;
+ fprintf(stderr,"[K3] WARNING: cap=1 is the floor and the projected peak is "
+ "%.1f GB, %.1f GB over the budget.\n", peak, peak-budget);
+ if(avail_now > 0 && peak > resident + avail_now &&
+ !(getenv("COLI_RAM_OVERCOMMIT") && atoi(getenv("COLI_RAM_OVERCOMMIT")))){
+ fprintf(stderr,"[K3] refusing to start: that peak also exceeds the %.1f GB "
+ "this machine actually has left, so the kernel would kill this run "
+ "mid-generation.\n[K3] lower K3_MAXT, raise --ram if the box really has "
+ "it, or set COLI_RAM_OVERCOMMIT=1 to override.\n", resident + avail_now);
+ exit(2);
+ }
+ }
+ }
{ int ncl=c->n_layers>0?c->n_layers:1;
m->ecache=calloc((size_t)ncl,sizeof(LCache)); }
for(int i=0;i<c->n_layers;i++) if(m->L[i].sparse){
diff --git a/c/tests/test_k3_ram_budget.c b/c/tests/test_k3_ram_budget.c
new file mode 100644
index 00000000..1af34c3b
--- /dev/null
+++ b/c/tests/test_k3_ram_budget.c
@@ -0,0 +1,98 @@
+/* test_k3_ram_budget.c -- #855: Kimi K3 sized its expert cache from
+ * K3_EXPERT_GB alone, with nothing subtracted for the resident set, no
+ * MemAvailable, no reserve, and no guard. On the reported 256 GB box that
+ * projected a 251.4 GB peak: the cache is empty when the init line prints and
+ * fills as the session runs, so the first request answers and a later one dies.
+ *
+ * k3_cap_for_ram() is pure by design -- no globals, no model, no I/O -- so the
+ * arithmetic can be checked against the reported numbers without the
+ * checkpoint. The cases below use @brad-evony's figures verbatim:
+ *
+ * [K3] init done in 2540.5s | 93 layers | expert cache 136/layer
+ * (17.5 MB/slot) | RSS 35.2 GB
+ * set K3_EXPERT_GB=220 ; python coli web --ram 242 ...
+ */
+#define main coli_k3_main_unused
+#include "../kimi_k3.c"
+#undef main
+
+#include <stdio.h>
+
+static int fails = 0;
+#define CHECK(c) do{ if(!(c)){ printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #c); fails++; } }while(0)
+
+/* The reported machine, exactly. */
+enum { NMOE = 93, N_EXPERTS = 384 };
+static const double SLOT_GB = 17.5e-3; /* 17.5 MB per slot */
+static const double RESIDENT = 35.2; /* RSS at the init line */
+static const double KV_GB = 3.0; /* projected, K3_MAXT default */
+static const double RESERVE = 2.5 + 1.2 + 3.0;
+
+/* What v1.5.0 did: K3_EXPERT_GB straight into a cap, nothing subtracted. */
+static int cap_v150(double egb){
+ int cap = (int)((egb*1e9)/(SLOT_GB*1e9*(double)NMOE));
+ if(cap < 1) cap = 1;
+ if(cap > N_EXPERTS) cap = N_EXPERTS;
+ return cap;
+}
+static double peak_gb(int cap){ return RESIDENT + RESERVE + (double)cap*SLOT_GB*NMOE; }
+
+int main(void){
+ /* ---- the reported failure, reproduced ---------------------------------- */
+ int old = cap_v150(220.0);
+ printf("v1.5.0: K3_EXPERT_GB=220 -> cap %d/layer, projected peak %.1f GB\n",
+ old, peak_gb(old));
+ CHECK(old == 135 || old == 136); /* the reported 136, modulo rounding */
+ CHECK(peak_gb(old) > 250.0); /* on a 256 GB box */
+
+ /* ---- the same request, now clamped ------------------------------------- */
+ /* --ram 242 is the whole-process ceiling the user actually passed. */
+ double left = 0;
+ int cap = k3_cap_for_ram(242.0, RESIDENT, RESERVE, SLOT_GB, NMOE,
+ old, N_EXPERTS, &left);
+ printf("fixed : --ram 242 -> %.1f GB for experts -> cap %d/layer, peak %.1f GB\n",
+ left, cap, peak_gb(cap));
+ CHECK(cap > 0);
+ CHECK(cap < old); /* the request was lowered */
+ CHECK(peak_gb(cap) <= 242.0 + 0.1); /* and the peak now fits the budget */
+ /* It must not overshoot in the other direction either: a cache so small it
+ * defeats the point would be a different bug, not a fix. */
+ CHECK(cap > old/2);
+
+ /* ---- the flag has to mean something ------------------------------------ */
+ /* Two budgets, everything else equal: a bigger --ram must buy more cache.
+ * Before this change --ram was not read at all, so both would be 136. */
+ int at180 = k3_cap_for_ram(180.0, RESIDENT, RESERVE, SLOT_GB, NMOE, old, N_EXPERTS, NULL);
+ int at242 = k3_cap_for_ram(242.0, RESIDENT, RESERVE, SLOT_GB, NMOE, old, N_EXPERTS, NULL);
+ printf(" --ram 180 -> cap %d | --ram 242 -> cap %d\n", at180, at242);
+ CHECK(at180 < at242);
+ CHECK(peak_gb(at180) <= 180.0 + 0.1);
+
+ /* ---- it must only ever LOWER ------------------------------------------- */
+ /* A user who asked for a small cache on a huge machine keeps it: this is a
+ * clamp, not an autotuner. Budget deliberately absurd. */
+ int small = k3_cap_for_ram(4096.0, RESIDENT, RESERVE, SLOT_GB, NMOE, 4, N_EXPERTS, NULL);
+ printf(" K3_EXPERT_GB small (cap 4) on a 4 TB budget -> cap %d\n", small);
+ CHECK(small == 4);
+
+ /* ---- no budget left ---------------------------------------------------- */
+ /* Resident alone over the budget: zero, so the caller can refuse rather than
+ * being handed a comfortable-looking 1 it never asked about. */
+ int none = k3_cap_for_ram(30.0, RESIDENT, RESERVE, SLOT_GB, NMOE, old, N_EXPERTS, &left);
+ printf(" --ram 30 with %.1f GB resident -> %.1f GB for experts, cap %d\n",
+ RESIDENT, left, none);
+ CHECK(none == 0);
+ CHECK(left < 0);
+
+ /* ---- the cap never exceeds the expert count ---------------------------- */
+ int huge = k3_cap_for_ram(100000.0, RESIDENT, RESERVE, SLOT_GB, NMOE,
+ N_EXPERTS*4, N_EXPERTS, NULL);
+ CHECK(huge <= N_EXPERTS);
+
+ /* ---- degenerate inputs do not produce a negative or wild cap ----------- */
+ CHECK(k3_cap_for_ram(242.0, RESIDENT, RESERVE, 0.0, NMOE, 7, N_EXPERTS, NULL) == 7);
+ CHECK(k3_cap_for_ram(242.0, RESIDENT, RESERVE, SLOT_GB, 0, 7, N_EXPERTS, NULL) >= 0);
+
+ printf("test_k3_ram_budget: %s\n", fails?"FAILED":"ok");
+ return fails?1:0;
+}
diff --git a/c/tests/test_v4_cli.py b/c/tests/test_v4_cli.py
index 2246216f..f2395d03 100644
--- a/c/tests/test_v4_cli.py
+++ b/c/tests/test_v4_cli.py
@@ -57,6 +57,31 @@ class V4CliTest(unittest.TestCase):
self.assertEqual(env["RAM_GB"], "64")
self.assertEqual(env["CTX"], "4096")
+ def test_kimi_engine_environment_forwards_ram(self):
+ """#855: `--ram` reached the environment for deepseek_v4 only, so on Kimi
+ K3 it was set and never read -- the flag a user reaches for to bound
+ memory did nothing, and the reported session ran itself out of RAM with
+ it apparently in effect. kimi_k3 reads RAM_GB now, so coli must send it.
+ """
+ args = argparse.Namespace(ngen=8, temp=0.0, ram=242, ctx=0)
+ env = self.cli.env_for_engine(args, "kimi")
+ self.assertEqual(env["RAM_GB"], "242")
+
+ def test_kimi_without_ram_stays_unset(self):
+ """Absent --ram, the engine budgets from MemAvailable itself. Sending an
+ empty or zero RAM_GB would read as an explicit ceiling of zero."""
+ args = argparse.Namespace(ngen=8, temp=0.0, ram=0, ctx=0)
+ env = self.cli.env_for_engine(args, "kimi")
+ self.assertNotIn("RAM_GB", env)
+
+ def test_kimi_does_not_get_v4_only_settings(self):
+ """The widening is RAM_GB alone; CTX and the V4 speculation defaults stay
+ where they were."""
+ args = argparse.Namespace(ngen=8, temp=0.0, ram=242, ctx=4096)
+ env = self.cli.env_for_engine(args, "kimi")
+ self.assertNotIn("CTX", env)
+ self.assertNotIn("V4_MTP", env)
+
def test_windows_v4_run_passes_chinese_prompt_as_utf8_file(self):
directory, root = self.make_model()
prompt = "请用中文解释:存储、内存和显存如何协同推理?"