Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
kimi: a RAM budget, and make --ram mean something (#855)
JustVugg0f9742244149
diff --git a/c/Makefile b/c/Makefileindex b573d792..f2ae37e9 100644--- a/c/Makefile+++ b/c/Makefile@@ -741,6 +741,9 @@ tests/test_json$(EXE): tests/test_json.c json.htests/test_tok_o200k$(EXE): tests/test_tok_o200k.c tok.h tok_unicode.h tok_unicode_o200k.h json.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+tests/test_k3_ram_budget$(EXE): tests/test_k3_ram_budget.c kimi_k3.c st.h tok.h quant.h compat.h omp_tune.h route_trace.h kv_prefix.h+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+tests/test_tok_kimi_tiny$(EXE): tests/test_tok_kimi_tiny.c tok.h tok_unicode.h tok_unicode_o200k.h json.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)tests/test_st_pread$(EXE): tests/test_st_pread.c st.h json.h compat.hdiff --git a/c/coli b/c/coliindex d735ce29..23c58433 100755--- a/c/coli+++ b/c/coli@@ -294,8 +294,13 @@ def env_for_engine(a, arch):env.setdefault("OMP_PLACES", "cores")if a.ngen: env["NGEN"] = str(a.ngen)if a.temp is not None: env["COLI_TEMP"] = str(a.temp)- if arch == "deepseek_v4":+ # --ram reaches the engines that read it. kimi_k3 gained a RAM budget in+ # #855; before that `grep -c RAM_GB c/kimi_k3.c` returned 0, so `coli chat+ # --ram 242` on Kimi K3 set an environment variable nobody looked at and the+ # user's session ran itself out of memory with the flag apparently set.+ if arch in ("deepseek_v4", "kimi"):if a.ram: env["RAM_GB"] = str(a.ram)+ if arch == "deepseek_v4":if a.ctx: env["CTX"] = str(a.ctx)# Speculation stays opt-in. Real multi-turn chat measured only 1/15# accepted prompt-lookup candidates; recurrent-state replay dominateddiff --git a/c/kimi_k3.c b/c/kimi_k3.cindex c056c908..2bdd64a9 100644--- a/c/kimi_k3.c+++ b/c/kimi_k3.c@@ -77,6 +77,10 @@#ifdef _OPENMP#include <omp.h>#endif+#ifdef __APPLE__+#include <sys/sysctl.h>+#include <mach/mach.h>+#endif#include "st.h"#include "tok.h"#include "quant.h"@@ -176,6 +180,47 @@ typedef struct {} Model;static double now_s(void){ struct timespec t; clock_gettime(CLOCK_MONOTONIC,&t); return t.tv_sec+t.tv_nsec*1e-9; }++/* How many expert slots per layer fit a budget. PURE -- no globals, no model,+ * no I/O -- so the arithmetic that #855 got wrong can be tested against the+ * reported numbers without a 600 GB checkpoint. Returns the cap; writes the+ * bytes left for experts through `for_experts_out` when non-NULL.+ *+ * reserve = page cache + activations + KV, all in GB. `cap_requested` is what+ * K3_EXPERT_GB asked for: this only ever LOWERS it, never raises it, so an+ * explicit small cache stays small. */+static int k3_cap_for_ram(double budget_gb, double resident_gb, double reserve_gb,+ double slot_gb, int nmoe, int cap_requested,+ int n_experts, double *for_experts_out){+ double for_experts = budget_gb - resident_gb - reserve_gb;+ if(for_experts_out) *for_experts_out = for_experts;+ if(nmoe < 1) nmoe = 1;+ if(!(slot_gb > 0.0)) return cap_requested;+ int fits = for_experts > 0.0 ? (int)(for_experts/(slot_gb*(double)nmoe)) : 0;+ if(fits > n_experts) fits = n_experts;+ return fits < cap_requested ? fits : cap_requested;+}++/* Memory the OS says is still reclaimable without swapping, in GB; 0 if unknown.+ * The same quantity colibri.c's cap_for_ram() budgets against -- Linux+ * MemAvailable, Windows ullAvailPhys, macOS free+inactive+purgeable. #855. */+static double k3_mem_avail(void){+#ifdef _WIN32+ double total=0, avail=0; compat_meminfo(&total,&avail); return avail;+#elif defined(__APPLE__)+ int64_t pgsz=0; size_t sl=sizeof(pgsz);+ if(sysctlbyname("hw.pagesize",&pgsz,&sl,NULL,0)||pgsz<=0) pgsz=16384;+ vm_statistics64_data_t vs; mach_msg_type_number_t nc=HOST_VM_INFO64_COUNT;+ if(host_statistics64(mach_host_self(),HOST_VM_INFO64,(host_info64_t)&vs,&nc)!=KERN_SUCCESS)+ return 0;+ return (double)(vs.free_count+vs.inactive_count+vs.purgeable_count)*(double)pgsz/1e9;+#else+ FILE *f=fopen("/proc/meminfo","r"); if(!f) return 0;+ char ln[256]; double kb=0;+ while(fgets(ln,sizeof(ln),f)) if(sscanf(ln,"MemAvailable: %lf",&kb)==1) break;+ fclose(f); return kb/1e6;+#endif+}static double rss_gb(void){ struct rusage r; getrusage(RUSAGE_SELF,&r);#if defined(__APPLE__)return r.ru_maxrss/(1024.0*1024.0*1024.0);@@ -636,6 +681,94 @@ static void model_init(Model *m, const char *snap, int n_layers_env){* regardless of K3_EXPERT_GB. */if(cap<1) cap=1;if(cap>c->n_experts) cap=c->n_experts;++ /* ---- RAM budget (#855) --------------------------------------------------+ * K3_EXPERT_GB used to be the whole story: cap = egb / slot / layers, with+ * nothing subtracted for what is already resident, no MemAvailable, no+ * reserve for KV or the page cache, and no guard during the run.+ *+ * Reported on a 256 GB box with K3_EXPERT_GB=220:+ *+ * 136 slots x 17.5 MB x 93 layers = 216.2 GB cache, once it fills+ * + 35.2 GB already resident+ * = 251.4 GB peak, of 256+ *+ * The cache is EMPTY when the init line prints and fills as the session+ * runs, so the first request answers and a later one dies -- which is what+ * "colibri engine exited unexpectedly" was.+ *+ * And --ram was inert here. `grep -c RAM_GB kimi_k3.c` returned 0 against 10+ * in colibri.c: the one flag a user would reach for to prevent exactly this+ * reached the environment and was never read. Same defect shape as #805 and+ * #858 -- a mechanism that lands in one engine and not its siblings.+ *+ * Unlike colibri.c's cap_for_ram this runs AFTER the dense weights are in,+ * so the resident set is MEASURED rather than projected. rss_gb() is the+ * truth here, not a model of it, which makes this budget the simpler of the+ * two rather than the more elaborate. */+ {+ double resident = rss_gb();+ double avail_now = k3_mem_avail();+ double ram_env = getenv("RAM_GB") ? atof(getenv("RAM_GB")) : 0.0;+ /* Explicit --ram is a ceiling on the WHOLE process. Absent, take 88% of+ * what the OS still offers and add what we already hold -- the same+ * fraction colibri.c uses, and for the same reason: overshooting means+ * an OOM kill mid-generation, which is far worse than a smaller cache. */+ double budget = ram_env > 0 ? ram_env : resident + avail_now*0.88;++ /* KV is allocated later, at the first request, so it has to be projected+ * here. n_layers x max_t x (kv_lora + qk_rope) x 4, skipping KDA layers,+ * with the same K3_MAXT default the serve path uses. */+ int max_t = getenv("K3_MAXT") ? atoi(getenv("K3_MAXT")) : 8192;+ if(max_t < 1) max_t = 8192;+ int nkv = 0; for(int i=0;i<c->n_layers;i++) if(!m->L[i].kda) nkv++;+ double kv_gb = (double)nkv*(double)max_t*(double)(c->kv_lora+c->qk_rope)*4.0/1e9;+ /* 2.5 GB page cache -- measured on Linux 2026-07-06: strangling it drops+ * buffered pread from ~800 to ~180 MB/s and the last GB of LRU costs+ * more in lost bandwidth than it returns. 1.2 GB activations/logits. */+ double reserve = 2.5 + 1.2 + kv_gb;+ double for_experts = 0.0;++ double slot_gb = (double)m->e_slot/1e9;+ int cap_fit = k3_cap_for_ram(budget, resident, reserve, slot_gb,+ nmoe, cap, c->n_experts, &for_experts);++ if(cap_fit < cap){+ /* Name every term. The user's number is not being ignored, it is+ * being clamped, and they cannot check the clamp without the parts. */+ fprintf(stderr,"[K3][RAM_GB=%.1f%s] resident %.1f GB + reserve %.1f GB "+ "(page cache 2.5, activations 1.2, KV %dx%d %.1f) -> %.1f GB for experts; "+ "cache %d->%d/layer (%.1f MB/slot, %d layers; projected peak %.1f GB)\n",+ budget, ram_env>0?"":" auto", resident, reserve, nkv, max_t, kv_gb,+ for_experts>0?for_experts:0.0, cap, cap_fit>0?cap_fit:1,+ slot_gb*1000.0, nmoe,+ resident + reserve + (double)(cap_fit>0?cap_fit:1)*slot_gb*nmoe);+ if(getenv("K3_EXPERT_GB"))+ fprintf(stderr,"[K3] K3_EXPERT_GB=%.0f does not fit alongside the "+ "%.1f GB already resident. It is a request, not a reservation.\n",+ egb, resident);+ cap = cap_fit;+ }++ if(cap < 1){+ /* Not even one slot per layer. Saying cap=1 and continuing turns+ * "does not fit" into "overruns", which is the OOM kill this exists+ * to avoid -- and the kernel kills with SIGKILL, so the engine dies+ * with no error and no log at all. Refuse, unless told otherwise. */+ cap = 1;+ double peak = resident + reserve + slot_gb*nmoe;+ fprintf(stderr,"[K3] WARNING: cap=1 is the floor and the projected peak is "+ "%.1f GB, %.1f GB over the budget.\n", peak, peak-budget);+ if(avail_now > 0 && peak > resident + avail_now &&+ !(getenv("COLI_RAM_OVERCOMMIT") && atoi(getenv("COLI_RAM_OVERCOMMIT")))){+ fprintf(stderr,"[K3] refusing to start: that peak also exceeds the %.1f GB "+ "this machine actually has left, so the kernel would kill this run "+ "mid-generation.\n[K3] lower K3_MAXT, raise --ram if the box really has "+ "it, or set COLI_RAM_OVERCOMMIT=1 to override.\n", resident + avail_now);+ exit(2);+ }+ }+ }{ int ncl=c->n_layers>0?c->n_layers:1;m->ecache=calloc((size_t)ncl,sizeof(LCache)); }for(int i=0;i<c->n_layers;i++) if(m->L[i].sparse){diff --git a/c/tests/test_k3_ram_budget.c b/c/tests/test_k3_ram_budget.cnew file mode 100644index 00000000..1af34c3b--- /dev/null+++ b/c/tests/test_k3_ram_budget.c@@ -0,0 +1,98 @@+/* test_k3_ram_budget.c -- #855: Kimi K3 sized its expert cache from+ * K3_EXPERT_GB alone, with nothing subtracted for the resident set, no+ * MemAvailable, no reserve, and no guard. On the reported 256 GB box that+ * projected a 251.4 GB peak: the cache is empty when the init line prints and+ * fills as the session runs, so the first request answers and a later one dies.+ *+ * k3_cap_for_ram() is pure by design -- no globals, no model, no I/O -- so the+ * arithmetic can be checked against the reported numbers without the+ * checkpoint. The cases below use @brad-evony's figures verbatim:+ *+ * [K3] init done in 2540.5s | 93 layers | expert cache 136/layer+ * (17.5 MB/slot) | RSS 35.2 GB+ * set K3_EXPERT_GB=220 ; python coli web --ram 242 ...+ */+#define main coli_k3_main_unused+#include "../kimi_k3.c"+#undef main++#include <stdio.h>++static int fails = 0;+#define CHECK(c) do{ if(!(c)){ printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #c); fails++; } }while(0)++/* The reported machine, exactly. */+enum { NMOE = 93, N_EXPERTS = 384 };+static const double SLOT_GB = 17.5e-3; /* 17.5 MB per slot */+static const double RESIDENT = 35.2; /* RSS at the init line */+static const double KV_GB = 3.0; /* projected, K3_MAXT default */+static const double RESERVE = 2.5 + 1.2 + 3.0;++/* What v1.5.0 did: K3_EXPERT_GB straight into a cap, nothing subtracted. */+static int cap_v150(double egb){+ int cap = (int)((egb*1e9)/(SLOT_GB*1e9*(double)NMOE));+ if(cap < 1) cap = 1;+ if(cap > N_EXPERTS) cap = N_EXPERTS;+ return cap;+}+static double peak_gb(int cap){ return RESIDENT + RESERVE + (double)cap*SLOT_GB*NMOE; }++int main(void){+ /* ---- the reported failure, reproduced ---------------------------------- */+ int old = cap_v150(220.0);+ printf("v1.5.0: K3_EXPERT_GB=220 -> cap %d/layer, projected peak %.1f GB\n",+ old, peak_gb(old));+ CHECK(old == 135 || old == 136); /* the reported 136, modulo rounding */+ CHECK(peak_gb(old) > 250.0); /* on a 256 GB box */++ /* ---- the same request, now clamped ------------------------------------- */+ /* --ram 242 is the whole-process ceiling the user actually passed. */+ double left = 0;+ int cap = k3_cap_for_ram(242.0, RESIDENT, RESERVE, SLOT_GB, NMOE,+ old, N_EXPERTS, &left);+ printf("fixed : --ram 242 -> %.1f GB for experts -> cap %d/layer, peak %.1f GB\n",+ left, cap, peak_gb(cap));+ CHECK(cap > 0);+ CHECK(cap < old); /* the request was lowered */+ CHECK(peak_gb(cap) <= 242.0 + 0.1); /* and the peak now fits the budget */+ /* It must not overshoot in the other direction either: a cache so small it+ * defeats the point would be a different bug, not a fix. */+ CHECK(cap > old/2);++ /* ---- the flag has to mean something ------------------------------------ */+ /* Two budgets, everything else equal: a bigger --ram must buy more cache.+ * Before this change --ram was not read at all, so both would be 136. */+ int at180 = k3_cap_for_ram(180.0, RESIDENT, RESERVE, SLOT_GB, NMOE, old, N_EXPERTS, NULL);+ int at242 = k3_cap_for_ram(242.0, RESIDENT, RESERVE, SLOT_GB, NMOE, old, N_EXPERTS, NULL);+ printf(" --ram 180 -> cap %d | --ram 242 -> cap %d\n", at180, at242);+ CHECK(at180 < at242);+ CHECK(peak_gb(at180) <= 180.0 + 0.1);++ /* ---- it must only ever LOWER ------------------------------------------- */+ /* A user who asked for a small cache on a huge machine keeps it: this is a+ * clamp, not an autotuner. Budget deliberately absurd. */+ int small = k3_cap_for_ram(4096.0, RESIDENT, RESERVE, SLOT_GB, NMOE, 4, N_EXPERTS, NULL);+ printf(" K3_EXPERT_GB small (cap 4) on a 4 TB budget -> cap %d\n", small);+ CHECK(small == 4);++ /* ---- no budget left ---------------------------------------------------- */+ /* Resident alone over the budget: zero, so the caller can refuse rather than+ * being handed a comfortable-looking 1 it never asked about. */+ int none = k3_cap_for_ram(30.0, RESIDENT, RESERVE, SLOT_GB, NMOE, old, N_EXPERTS, &left);+ printf(" --ram 30 with %.1f GB resident -> %.1f GB for experts, cap %d\n",+ RESIDENT, left, none);+ CHECK(none == 0);+ CHECK(left < 0);++ /* ---- the cap never exceeds the expert count ---------------------------- */+ int huge = k3_cap_for_ram(100000.0, RESIDENT, RESERVE, SLOT_GB, NMOE,+ N_EXPERTS*4, N_EXPERTS, NULL);+ CHECK(huge <= N_EXPERTS);++ /* ---- degenerate inputs do not produce a negative or wild cap ----------- */+ CHECK(k3_cap_for_ram(242.0, RESIDENT, RESERVE, 0.0, NMOE, 7, N_EXPERTS, NULL) == 7);+ CHECK(k3_cap_for_ram(242.0, RESIDENT, RESERVE, SLOT_GB, 0, 7, N_EXPERTS, NULL) >= 0);++ printf("test_k3_ram_budget: %s\n", fails?"FAILED":"ok");+ return fails?1:0;+}diff --git a/c/tests/test_v4_cli.py b/c/tests/test_v4_cli.pyindex 2246216f..f2395d03 100644--- a/c/tests/test_v4_cli.py+++ b/c/tests/test_v4_cli.py@@ -57,6 +57,31 @@ class V4CliTest(unittest.TestCase):self.assertEqual(env["RAM_GB"], "64")self.assertEqual(env["CTX"], "4096")+ def test_kimi_engine_environment_forwards_ram(self):+ """#855: `--ram` reached the environment for deepseek_v4 only, so on Kimi+ K3 it was set and never read -- the flag a user reaches for to bound+ memory did nothing, and the reported session ran itself out of RAM with+ it apparently in effect. kimi_k3 reads RAM_GB now, so coli must send it.+ """+ args = argparse.Namespace(ngen=8, temp=0.0, ram=242, ctx=0)+ env = self.cli.env_for_engine(args, "kimi")+ self.assertEqual(env["RAM_GB"], "242")++ def test_kimi_without_ram_stays_unset(self):+ """Absent --ram, the engine budgets from MemAvailable itself. Sending an+ empty or zero RAM_GB would read as an explicit ceiling of zero."""+ args = argparse.Namespace(ngen=8, temp=0.0, ram=0, ctx=0)+ env = self.cli.env_for_engine(args, "kimi")+ self.assertNotIn("RAM_GB", env)++ def test_kimi_does_not_get_v4_only_settings(self):+ """The widening is RAM_GB alone; CTX and the V4 speculation defaults stay+ where they were."""+ args = argparse.Namespace(ngen=8, temp=0.0, ram=242, ctx=4096)+ env = self.cli.env_for_engine(args, "kimi")+ self.assertNotIn("CTX", env)+ self.assertNotIn("V4_MTP", env)+def test_windows_v4_run_passes_chinese_prompt_as_utf8_file(self):directory, root = self.make_model()prompt = "请用中文解释:存储、内存和显存如何协同推理?"