Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat(k3): disk-parked recurrent-state checkpoints (COLI_K3_CKPT_DIR)
JustVugg3b2f1e13fc2a
diff --git a/c/kimi_k3.c b/c/kimi_k3.cindex ed201947..d6019554 100644--- a/c/kimi_k3.c+++ b/c/kimi_k3.c@@ -71,6 +71,13 @@* token 0. Unset/0 (default) = feature off, engine* byte-identical to before. ~434 MB per photo on the* full model, which is why it is opt-in.+ * COLI_K3_CKPT_DIR=dir park the photos in files under dir instead of RAM+ * (low-RAM machines: ~0.4s of sequential I/O per+ * restore against minutes of avoided re-prefill).+ * Alone it implies COLI_K3_CKPT=2; an explicit+ * COLI_K3_CKPT (0 included) always wins. Photos never+ * outlive the process: after a restart the MLA rows+ * are gone, so no restore could be valid.* COLI_TEMP=F 0 = greedy (default), else softmax temperature*/#define _GNU_SOURCE@@ -483,6 +490,10 @@ static double k3_ckpt_reserve_gb(const Model *m, int slots);static int g_k3_ckpt_slots=0; /* COLI_K3_CKPT: recurrent-state checkpoint* slots. STRICTLY OPT-IN, 0 = off (default,* engine behaves exactly as before). */+static const char *g_k3_ckpt_dir=NULL; /* COLI_K3_CKPT_DIR: park the photos in+ * files instead of RAM (low-RAM machines:+ * ~0.4s of sequential I/O per restore+ * against minutes of avoided re-prefill). */static int k3_mmap_backend_allowed(int mmap_enabled, int vk_enabled, int cuda_enabled){return !mmap_enabled || (!vk_enabled && !cuda_enabled);@@ -792,7 +803,12 @@ static void model_init(Model *m, const char *snap, int n_layers_env){const char *e=getenv("COLI_K3_CKPT");g_k3_ckpt_slots=e?atoi(e):0;if(g_k3_ckpt_slots<0) g_k3_ckpt_slots=0;- if(g_k3_ckpt_slots>K3_CKPT_MAX) g_k3_ckpt_slots=K3_CKPT_MAX; }+ if(g_k3_ckpt_slots>K3_CKPT_MAX) g_k3_ckpt_slots=K3_CKPT_MAX;+ g_k3_ckpt_dir=getenv("COLI_K3_CKPT_DIR");+ if(g_k3_ckpt_dir&&!*g_k3_ckpt_dir) g_k3_ckpt_dir=NULL;+ /* a directory alone is a clear enough request: default to 2 slots+ * there, while an explicit COLI_K3_CKPT (0 included) always wins */+ if(g_k3_ckpt_dir&&!e) g_k3_ckpt_slots=2; }g_k3_pipe = getenv("K3_PIPE")?atoi(getenv("K3_PIPE")):1;g_k3_topp = getenv("K3_TOPP")?(float)atof(getenv("K3_TOPP")):0.f;if(g_k3_topp>0.f)@@ -1055,8 +1071,9 @@ static void model_init(Model *m, const char *snap, int n_layers_env){fprintf(stderr,"[K3] init done in %.1fs | %d layers | expert cache %d/layer (%.1f MB/slot) | RSS %.1f GB\n",now_s()-t0,c->n_layers,cap,m->e_slot/1e6,rss_gb());if(g_k3_ckpt_slots>0)- fprintf(stderr,"[K3-CKPT] enabled: %d recurrent-state slots (%.1f MB each)\n",- g_k3_ckpt_slots,k3_ckpt_blob_floats(m)*sizeof(float)/1048576.0);+ fprintf(stderr,"[K3-CKPT] enabled: %d recurrent-state slots (%.1f MB each, %s)\n",+ g_k3_ckpt_slots,k3_ckpt_blob_floats(m)*sizeof(float)/1048576.0,+ g_k3_ckpt_dir?g_k3_ckpt_dir:"RAM");/* Phase 9: layer validation init */{ const char *vl=getenv("K3_VALIDATE_LAYER");const char *vt=getenv("K3_VALIDATE_TOKEN");@@ -2458,7 +2475,11 @@ static size_t k3_ckpt_blob_floats(const Model *m){static double k3_ckpt_reserve_gb(const Model *m, int slots){if(slots<1) return 0.0;- return (double)slots*(double)k3_ckpt_blob_floats(m)*sizeof(float)/1e9;+ /* Disk-parked photos keep ONE bounce buffer in RAM regardless of slot+ * count -- that is the whole point of COLI_K3_CKPT_DIR on a low-RAM+ * machine. RAM slots each hold a full blob. */+ int resident_blobs = g_k3_ckpt_dir ? 1 : slots;+ return (double)resident_blobs*(double)k3_ckpt_blob_floats(m)*sizeof(float)/1e9;}static void k3_ckpt_copy(float *blob, Model *m, int to_blob){@@ -2476,23 +2497,82 @@ static void k3_ckpt_copy(float *blob, Model *m, int to_blob){}}+/* Disk-parked photos: same photos, same validity rules, but the blob lives+ * in a file so the RAM cost is one bounce buffer instead of N slots. Slot+ * metadata (pos, fed ids, LRU stamp) stays in memory: photos never survive+ * the process, because after a restart the per-position MLA rows are gone+ * and no restore could ever be valid. Stale files from a previous run are+ * simply overwritten. Writes are atomic (tmp + rename); any I/O failure+ * drops the photo, never the request. */+static float *g_k3_ckpt_iobuf=NULL;+#define K3_CKPT_FILE_MAGIC ((((uint64_t)0x504B4333)<<32)|1u) /* "3CKP" v1 */++static void k3_ckpt_file(char *out, size_t n, int slot){+ snprintf(out,n,"%s/k3_ckpt_%d.bin",g_k3_ckpt_dir,slot);+}++static int k3_ckpt_iobuf_ready(const Model *m){+ if(!g_k3_ckpt_iobuf)+ g_k3_ckpt_iobuf=malloc(k3_ckpt_blob_floats(m)*sizeof(float));+ return g_k3_ckpt_iobuf!=NULL;+}++static int k3_ckpt_disk_write(Model *m, int slot, int pos){+ char path[512], tmp[524];+ k3_ckpt_file(path,sizeof path,slot);+ snprintf(tmp,sizeof tmp,"%s.tmp",path);+ size_t floats=k3_ckpt_blob_floats(m);+ if(!k3_ckpt_iobuf_ready(m)) return 0;+ k3_ckpt_copy(g_k3_ckpt_iobuf,m,1);+ FILE *f=fopen(tmp,"wb"); if(!f) return 0;+ uint64_t header[3]={K3_CKPT_FILE_MAGIC,(uint64_t)pos,(uint64_t)floats};+ int ok=fwrite(header,sizeof header,1,f)==1 &&+ fwrite(g_k3_ckpt_iobuf,sizeof(float),floats,f)==floats;+ ok=!fclose(f)&&ok;+ if(ok){ remove(path); ok=!rename(tmp,path); }+ if(!ok) remove(tmp);+ return ok;+}++static int k3_ckpt_disk_read(Model *m, int slot, int pos){+ char path[512];+ k3_ckpt_file(path,sizeof path,slot);+ size_t floats=k3_ckpt_blob_floats(m);+ if(!k3_ckpt_iobuf_ready(m)) return 0;+ FILE *f=fopen(path,"rb"); if(!f) return 0;+ uint64_t header[3]={0,0,0};+ int ok=fread(header,sizeof header,1,f)==1 &&+ header[0]==K3_CKPT_FILE_MAGIC &&+ header[1]==(uint64_t)pos && header[2]==(uint64_t)floats &&+ fread(g_k3_ckpt_iobuf,sizeof(float),floats,f)==floats;+ fclose(f);+ if(ok) k3_ckpt_copy(g_k3_ckpt_iobuf,m,0);+ return ok;+}++/* A slot holds a photo: in RAM mode the blob pointer says so, in disk mode+ * the committed position does (the file is only trusted through it). */+static int k3_ckpt_slot_full(const K3Ckpt *k){+ return g_k3_ckpt_dir ? k->pos>0 : k->blob!=NULL;+}+static void k3_ckpt_save(Model *m){if(g_k3_ckpt_slots<1) return;int pos=m->kvp.len;if(pos<1||m->kvp.tainted||!m->kvp.fed) return;for(int s=0;s<g_k3_ckpt_slots;s++){ /* identical photo: refresh LRU */K3Ckpt *k=&g_k3_ckpt[s];- if(k->blob&&k->pos==pos&&+ if(k3_ckpt_slot_full(k)&&k->pos==pos&&!memcmp(k->fed,m->kvp.fed,(size_t)pos*sizeof(int))){k->used=++g_k3_ckpt_clock; return; }}int victim=0;for(int s=0;s<g_k3_ckpt_slots;s++){- if(!g_k3_ckpt[s].blob){ victim=s; break; }+ if(!k3_ckpt_slot_full(&g_k3_ckpt[s])){ victim=s; break; }if(g_k3_ckpt[s].used<g_k3_ckpt[victim].used) victim=s;}K3Ckpt *k=&g_k3_ckpt[victim];- if(!k->blob){+ if(!g_k3_ckpt_dir&&!k->blob){k->blob=malloc(k3_ckpt_blob_floats(m)*sizeof(float));if(!k->blob) return; /* optimisation, never a failure */}@@ -2500,7 +2580,9 @@ static void k3_ckpt_save(Model *m){if(!fed) return; /* slot keeps its old photo */k->fed=fed;memcpy(k->fed,m->kvp.fed,(size_t)pos*sizeof(int));- k3_ckpt_copy(k->blob,m,1);+ if(g_k3_ckpt_dir){+ if(!k3_ckpt_disk_write(m,victim,pos)){ k->pos=0; return; }+ } else k3_ckpt_copy(k->blob,m,1);k->pos=pos; k->used=++g_k3_ckpt_clock;if(getenv("K3_PREFIX_LOG"))fprintf(stderr,"[K3-CKPT] saved pos=%d slot=%d\n",pos,victim);@@ -2516,13 +2598,15 @@ static int k3_ckpt_restore_best(Model *m, const int *ids, int np){int best=-1;for(int s=0;s<g_k3_ckpt_slots;s++){K3Ckpt *k=&g_k3_ckpt[s];- if(!k->blob||k->pos<1||k->pos>live) continue;+ if(!k3_ckpt_slot_full(k)||k->pos<1||k->pos>live) continue;if(memcmp(k->fed,ids,(size_t)k->pos*sizeof(int))) continue;if(best<0||k->pos>g_k3_ckpt[best].pos) best=s;}if(best<0) return 0;K3Ckpt *k=&g_k3_ckpt[best];- k3_ckpt_copy(k->blob,m,0);+ if(g_k3_ckpt_dir){+ if(!k3_ckpt_disk_read(m,best,k->pos)){ k->pos=0; return 0; }+ } else k3_ckpt_copy(k->blob,m,0);m->kvp.len=k->pos; /* the record truncates with the state it describes */k->used=++g_k3_ckpt_clock;if(getenv("K3_PREFIX_LOG"))diff --git a/c/tests/test_k3_ram_budget.c b/c/tests/test_k3_ram_budget.cindex 4802f4c3..0484e4c8 100644--- a/c/tests/test_k3_ram_budget.c+++ b/c/tests/test_k3_ram_budget.c@@ -76,6 +76,11 @@ int main(void){CHECK(k3_ckpt_reserve_gb(&cm,0)==0.0);CHECK(one_ckpt>0.42&&one_ckpt<0.47);CHECK(four_ckpt>1.7&&four_ckpt<1.9);+ /* disk-parked photos (COLI_K3_CKPT_DIR): one bounce buffer in RAM no+ * matter how many slots -- the low-RAM machine must not be charged N. */+ g_k3_ckpt_dir="/anywhere";+ CHECK(k3_ckpt_reserve_gb(&cm,4)==one_ckpt);+ g_k3_ckpt_dir=NULL;int with_ckpt=k3_cap_for_ram(242.0,RESIDENT,RESERVE+four_ckpt,SLOT_GB,NMOE,old,N_EXPERTS,&left);printf(" --ram 242 + 4 checkpoints -> cap %d/layer, peak %.1f GB\n",diff --git a/c/tests/test_kimi_k3_ckpt.py b/c/tests/test_kimi_k3_ckpt.pyindex 06afa27a..cb18f4de 100644--- a/c/tests/test_kimi_k3_ckpt.py+++ b/c/tests/test_kimi_k3_ckpt.py@@ -128,6 +128,12 @@ def main():work / "off.stderr")on, err_on = run_session(args.binary, model, {"COLI_K3_CKPT": "4"},work / "on.stderr")+ disk_dir = work / "photos"+ disk_dir.mkdir()+ disk, err_disk = run_session(+ args.binary, model,+ {"COLI_K3_CKPT": "4", "COLI_K3_CKPT_DIR": str(disk_dir)},+ work / "disk.stderr")# opt-in invariant: the default run must not know the feature existsif "[K3-CKPT]" in err_off:@@ -178,6 +184,28 @@ def main():f"{recompute} recomputed instead of {total}")if "[K3-CKPT] saved" not in err_on:raise AssertionError("no checkpoint was ever saved:\n" + err_on)++ # disk mode: same photos parked in files. Identity and restore depth+ # must match the RAM mode exactly, and the blobs must be ON DISK.+ for index, (ram_result, disk_result) in enumerate(zip(on, disk), 1):+ if ram_result != disk_result:+ raise AssertionError(+ f"request {index}: disk mode diverged from RAM mode: "+ f"{disk_result!r} vs {ram_result!r}")+ disk_restores = re.findall(+ r"\[K3-CKPT\] restored pos=(\d+) of (\d+) prompt "+ r"tokens \(recompute (\d+)\)", err_disk)+ if [tuple(map(int, r)) for r in disk_restores] != restores:+ raise AssertionError(+ f"disk-mode restores {disk_restores} differ from RAM-mode "+ f"{restores}:\n{err_disk}")+ photos = sorted(f.name for f in disk_dir.glob("k3_ckpt_*.bin"))+ if not photos:+ raise AssertionError(+ "disk mode restored correctly but wrote no photo files: "+ "the blobs are not actually on disk")+ print(f"ok disk mode: identical outputs and restores, "+ f"{len(photos)} photo file(s) on disk")print("kimi-k3-ckpt: all checks passed")return 0finally: