Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(k3): reserve checkpoint memory in RAM budget
JustVugg019eaafca16b
diff --git a/c/kimi_k3.c b/c/kimi_k3.cindex d4d2e95b..ed201947 100644--- a/c/kimi_k3.c+++ b/c/kimi_k3.c@@ -214,9 +214,9 @@ static double now_s(void){ struct timespec t; clock_gettime(CLOCK_MONOTONIC,&t);* reported numbers without a 600 GB checkpoint. Returns the cap; writes the* bytes left for experts through `for_experts_out` when non-NULL.*- * reserve = page cache + activations + KV, all in GB. `cap_requested` is what- * K3_EXPERT_GB asked for: this only ever LOWERS it, never raises it, so an- * explicit small cache stays small. */+ * reserve = page cache + activations + KV + optional recurrent checkpoints,+ * all in GB. `cap_requested` is what K3_EXPERT_GB asked for: this only ever+ * LOWERS it, never raises it, so an explicit small cache stays small. */static int k3_cap_for_ram(double budget_gb, double resident_gb, double reserve_gb,double slot_gb, int nmoe, int cap_requested,int n_experts, double *for_experts_out){@@ -479,6 +479,7 @@ static int g_k3_pipe=1; /* K3_PIPE: overlap loads with computestatic float g_k3_topp=0.f; /* K3_TOPP: routed-expert top-p pruning */enum { K3_CKPT_MAX = 8 };static size_t k3_ckpt_blob_floats(const Model *m); /* def. with the ckpt block */+static double k3_ckpt_reserve_gb(const Model *m, int slots);static int g_k3_ckpt_slots=0; /* COLI_K3_CKPT: recurrent-state checkpoint* slots. STRICTLY OPT-IN, 0 = off (default,* engine behaves exactly as before). */@@ -996,8 +997,11 @@ static void model_init(Model *m, const char *snap, int n_layers_env){double kv_gb = (double)nkv*(double)max_t*(double)(c->kv_lora+c->qk_rope)*4.0/1e9;/* 2.5 GB page cache -- measured on Linux 2026-07-06: strangling it drops* buffered pread from ~800 to ~180 MB/s and the last GB of LRU costs- * more in lost bandwidth than it returns. 1.2 GB activations/logits. */- double reserve = 2.5 + 1.2 + kv_gb;+ * more in lost bandwidth than it returns. 1.2 GB activations/logits.+ * Checkpoints allocate lazily after this plan, so reserve all enabled+ * slots now: RAM_GB is a whole-process ceiling even for opt-in memory. */+ double ckpt_gb = k3_ckpt_reserve_gb(m,g_k3_ckpt_slots);+ double reserve = 2.5 + 1.2 + kv_gb + ckpt_gb;double for_experts = 0.0;double slot_gb = (double)m->e_slot/1e9;@@ -1008,9 +1012,11 @@ static void model_init(Model *m, const char *snap, int n_layers_env){/* Name every term. The user's number is not being ignored, it is* being clamped, and they cannot check the clamp without the parts. */fprintf(stderr,"[K3][RAM_GB=%.1f%s] resident %.1f GB + reserve %.1f GB "- "(page cache 2.5, activations 1.2, KV %dx%d %.1f) -> %.1f GB for experts; "+ "(page cache 2.5, activations 1.2, KV %dx%d %.1f, checkpoints %d %.1f) "+ "-> %.1f GB for experts; ""cache %d->%d/layer (%.1f MB/slot, %d layers; projected peak %.1f GB)\n",budget, ram_env>0?"":" auto", resident, reserve, nkv, max_t, kv_gb,+ g_k3_ckpt_slots,ckpt_gb,for_experts>0?for_experts:0.0, cap, cap_fit>0?cap_fit:1,slot_gb*1000.0, nmoe,resident + reserve + (double)(cap_fit>0?cap_fit:1)*slot_gb*nmoe);@@ -2450,6 +2456,11 @@ static size_t k3_ckpt_blob_floats(const Model *m){return n;}+static double k3_ckpt_reserve_gb(const Model *m, int slots){+ if(slots<1) return 0.0;+ return (double)slots*(double)k3_ckpt_blob_floats(m)*sizeof(float)/1e9;+}+static void k3_ckpt_copy(float *blob, Model *m, int to_blob){Cfg *c=&m->c; size_t o=0;size_t ks=(size_t)c->kda_heads*c->kda_hd*c->kda_hd;diff --git a/c/tests/test_k3_ram_budget.c b/c/tests/test_k3_ram_budget.cindex 8fcfb0de..4802f4c3 100644--- a/c/tests/test_k3_ram_budget.c+++ b/c/tests/test_k3_ram_budget.c@@ -36,6 +36,7 @@ static int cap_v150(double egb){return cap;}static double peak_gb(int cap){ return RESIDENT + RESERVE + (double)cap*SLOT_GB*NMOE; }+static double peak_ckpt_gb(int cap, double ckpt){ return peak_gb(cap) + ckpt; }int main(void){/* ---- the reported failure, reproduced ---------------------------------- */@@ -59,6 +60,33 @@ int main(void){* defeats the point would be a different bug, not a fix. */CHECK(cap > old/2);+ /* ---- opt-in recurrent checkpoints are part of the same ceiling ------- */+ /* Full K3 geometry: 69 KDA rows, 24 heads x 256 recurrent state and three+ * 6144x4 conv windows. This is the ~434 MiB/photo stated by COLI_K3_CKPT.+ * The planner runs before photos are allocated, so it must reserve every+ * enabled slot or RAM_GB will be exceeded later in the conversation. */+ Model cm={0}; cm.c.n_layers=69; cm.c.kda_heads=24; cm.c.kda_hd=256;+ cm.c.kda_proj=6144; cm.c.conv_k=4;+ cm.L=calloc((size_t)cm.c.n_layers,sizeof(Layer));+ for(int i=0;i<cm.c.n_layers;i++) cm.L[i].kda=1;+ double one_ckpt=k3_ckpt_reserve_gb(&cm,1);+ double four_ckpt=k3_ckpt_reserve_gb(&cm,4);+ printf(" COLI_K3_CKPT=1 -> %.1f MiB; 4 slots -> %.2f GB reserved\n",+ one_ckpt*1e9/1048576.0,four_ckpt);+ CHECK(k3_ckpt_reserve_gb(&cm,0)==0.0);+ CHECK(one_ckpt>0.42&&one_ckpt<0.47);+ CHECK(four_ckpt>1.7&&four_ckpt<1.9);+ int with_ckpt=k3_cap_for_ram(242.0,RESIDENT,RESERVE+four_ckpt,+ SLOT_GB,NMOE,old,N_EXPERTS,&left);+ printf(" --ram 242 + 4 checkpoints -> cap %d/layer, peak %.1f GB\n",+ with_ckpt,peak_ckpt_gb(with_ckpt,four_ckpt));+ CHECK(with_ckpt<cap);+ CHECK(peak_ckpt_gb(with_ckpt,four_ckpt)<=242.0+0.1);+ double one_slot_peak=RESIDENT+RESERVE+four_ckpt+SLOT_GB*NMOE;+ CHECK(k3_cap_for_ram(one_slot_peak-0.01,RESIDENT,RESERVE+four_ckpt,+ SLOT_GB,NMOE,old,N_EXPERTS,NULL)==0);+ free(cm.L);+/* ---- the flag has to mean something ------------------------------------ *//* Two budgets, everything else equal: a bigger --ram must buy more cache.* Before this change --ram was not read at all, so both would be 136. */