Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(k3): reserve checkpoint memory in RAM budget

JustVugg019eaafca16b
diff --git a/c/kimi_k3.c b/c/kimi_k3.c
index d4d2e95b..ed201947 100644
--- a/c/kimi_k3.c
+++ b/c/kimi_k3.c
@@ -214,9 +214,9 @@ static double now_s(void){ struct timespec t; clock_gettime(CLOCK_MONOTONIC,&t);
* reported numbers without a 600 GB checkpoint. Returns the cap; writes the
* bytes left for experts through `for_experts_out` when non-NULL.
*
- * reserve = page cache + activations + KV, all in GB. `cap_requested` is what
- * K3_EXPERT_GB asked for: this only ever LOWERS it, never raises it, so an
- * explicit small cache stays small. */
+ * reserve = page cache + activations + KV + optional recurrent checkpoints,
+ * all in GB. `cap_requested` is what K3_EXPERT_GB asked for: this only ever
+ * LOWERS it, never raises it, so an explicit small cache stays small. */
static int k3_cap_for_ram(double budget_gb, double resident_gb, double reserve_gb,
double slot_gb, int nmoe, int cap_requested,
int n_experts, double *for_experts_out){
@@ -479,6 +479,7 @@ static int g_k3_pipe=1; /* K3_PIPE: overlap loads with compute
static float g_k3_topp=0.f; /* K3_TOPP: routed-expert top-p pruning */
enum { K3_CKPT_MAX = 8 };
static size_t k3_ckpt_blob_floats(const Model *m); /* def. with the ckpt block */
+static double k3_ckpt_reserve_gb(const Model *m, int slots);
static int g_k3_ckpt_slots=0; /* COLI_K3_CKPT: recurrent-state checkpoint
* slots. STRICTLY OPT-IN, 0 = off (default,
* engine behaves exactly as before). */
@@ -996,8 +997,11 @@ static void model_init(Model *m, const char *snap, int n_layers_env){
double kv_gb = (double)nkv*(double)max_t*(double)(c->kv_lora+c->qk_rope)*4.0/1e9;
/* 2.5 GB page cache -- measured on Linux 2026-07-06: strangling it drops
* buffered pread from ~800 to ~180 MB/s and the last GB of LRU costs
- * more in lost bandwidth than it returns. 1.2 GB activations/logits. */
- double reserve = 2.5 + 1.2 + kv_gb;
+ * more in lost bandwidth than it returns. 1.2 GB activations/logits.
+ * Checkpoints allocate lazily after this plan, so reserve all enabled
+ * slots now: RAM_GB is a whole-process ceiling even for opt-in memory. */
+ double ckpt_gb = k3_ckpt_reserve_gb(m,g_k3_ckpt_slots);
+ double reserve = 2.5 + 1.2 + kv_gb + ckpt_gb;
double for_experts = 0.0;
double slot_gb = (double)m->e_slot/1e9;
@@ -1008,9 +1012,11 @@ static void model_init(Model *m, const char *snap, int n_layers_env){
/* Name every term. The user's number is not being ignored, it is
* being clamped, and they cannot check the clamp without the parts. */
fprintf(stderr,"[K3][RAM_GB=%.1f%s] resident %.1f GB + reserve %.1f GB "
- "(page cache 2.5, activations 1.2, KV %dx%d %.1f) -> %.1f GB for experts; "
+ "(page cache 2.5, activations 1.2, KV %dx%d %.1f, checkpoints %d %.1f) "
+ "-> %.1f GB for experts; "
"cache %d->%d/layer (%.1f MB/slot, %d layers; projected peak %.1f GB)\n",
budget, ram_env>0?"":" auto", resident, reserve, nkv, max_t, kv_gb,
+ g_k3_ckpt_slots,ckpt_gb,
for_experts>0?for_experts:0.0, cap, cap_fit>0?cap_fit:1,
slot_gb*1000.0, nmoe,
resident + reserve + (double)(cap_fit>0?cap_fit:1)*slot_gb*nmoe);
@@ -2450,6 +2456,11 @@ static size_t k3_ckpt_blob_floats(const Model *m){
return n;
}
+static double k3_ckpt_reserve_gb(const Model *m, int slots){
+ if(slots<1) return 0.0;
+ return (double)slots*(double)k3_ckpt_blob_floats(m)*sizeof(float)/1e9;
+}
+
static void k3_ckpt_copy(float *blob, Model *m, int to_blob){
Cfg *c=&m->c; size_t o=0;
size_t ks=(size_t)c->kda_heads*c->kda_hd*c->kda_hd;
diff --git a/c/tests/test_k3_ram_budget.c b/c/tests/test_k3_ram_budget.c
index 8fcfb0de..4802f4c3 100644
--- a/c/tests/test_k3_ram_budget.c
+++ b/c/tests/test_k3_ram_budget.c
@@ -36,6 +36,7 @@ static int cap_v150(double egb){
return cap;
}
static double peak_gb(int cap){ return RESIDENT + RESERVE + (double)cap*SLOT_GB*NMOE; }
+static double peak_ckpt_gb(int cap, double ckpt){ return peak_gb(cap) + ckpt; }
int main(void){
/* ---- the reported failure, reproduced ---------------------------------- */
@@ -59,6 +60,33 @@ int main(void){
* defeats the point would be a different bug, not a fix. */
CHECK(cap > old/2);
+ /* ---- opt-in recurrent checkpoints are part of the same ceiling ------- */
+ /* Full K3 geometry: 69 KDA rows, 24 heads x 256 recurrent state and three
+ * 6144x4 conv windows. This is the ~434 MiB/photo stated by COLI_K3_CKPT.
+ * The planner runs before photos are allocated, so it must reserve every
+ * enabled slot or RAM_GB will be exceeded later in the conversation. */
+ Model cm={0}; cm.c.n_layers=69; cm.c.kda_heads=24; cm.c.kda_hd=256;
+ cm.c.kda_proj=6144; cm.c.conv_k=4;
+ cm.L=calloc((size_t)cm.c.n_layers,sizeof(Layer));
+ for(int i=0;i<cm.c.n_layers;i++) cm.L[i].kda=1;
+ double one_ckpt=k3_ckpt_reserve_gb(&cm,1);
+ double four_ckpt=k3_ckpt_reserve_gb(&cm,4);
+ printf(" COLI_K3_CKPT=1 -> %.1f MiB; 4 slots -> %.2f GB reserved\n",
+ one_ckpt*1e9/1048576.0,four_ckpt);
+ CHECK(k3_ckpt_reserve_gb(&cm,0)==0.0);
+ CHECK(one_ckpt>0.42&&one_ckpt<0.47);
+ CHECK(four_ckpt>1.7&&four_ckpt<1.9);
+ int with_ckpt=k3_cap_for_ram(242.0,RESIDENT,RESERVE+four_ckpt,
+ SLOT_GB,NMOE,old,N_EXPERTS,&left);
+ printf(" --ram 242 + 4 checkpoints -> cap %d/layer, peak %.1f GB\n",
+ with_ckpt,peak_ckpt_gb(with_ckpt,four_ckpt));
+ CHECK(with_ckpt<cap);
+ CHECK(peak_ckpt_gb(with_ckpt,four_ckpt)<=242.0+0.1);
+ double one_slot_peak=RESIDENT+RESERVE+four_ckpt+SLOT_GB*NMOE;
+ CHECK(k3_cap_for_ram(one_slot_peak-0.01,RESIDENT,RESERVE+four_ckpt,
+ SLOT_GB,NMOE,old,N_EXPERTS,NULL)==0);
+ free(cm.L);
+
/* ---- the flag has to mean something ------------------------------------ */
/* Two budgets, everything else equal: a bigger --ram must buy more cache.
* Before this change --ram was not read at all, so both would be 136. */