Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(k3): opt-in recurrent-state checkpoints for divergent prompts

JustVugg70c948cb8a96
diff --git a/.gitignore b/.gitignore
index b46b8a44..b477374b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -140,3 +140,4 @@ c/*.obj
# vendor modeling cache for the tiny K3 oracle (maintainer tool)
c/kimi_k3_tiny/.vendor_kimi_k3/
+c/kimi_k3_tiny/.coli_usage
diff --git a/c/Makefile b/c/Makefile
index d221d37c..292ce95d 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -619,6 +619,9 @@ kimi-k3-tiny-check: kimi-k3-tiny-generate kimi_k3$(EXE)
$(PYTHON) tests/test_kimi_k3_tiny.py \
--binary ./kimi_k3$(EXE) \
--fixture ./kimi_k3_tiny
+ $(PYTHON) tests/test_kimi_k3_ckpt.py \
+ --binary ./kimi_k3$(EXE) \
+ --fixture ./kimi_k3_tiny
.PHONY: deepseek-v4 deepseek-v4-oracle deepseek-v4-tiny-generate \
deepseek-v4-tiny-check deepseek-v4-clean
diff --git a/c/kimi_k3.c b/c/kimi_k3.c
index e11cca9c..d4d2e95b 100644
--- a/c/kimi_k3.c
+++ b/c/kimi_k3.c
@@ -64,6 +64,13 @@
* K3_LOGITS=path dump f32 logits per PREFILL position (teacher-forced
* bit-width comparisons; use with --ngen 0)
* K3_MAXT=N KV/context capacity (default prompt+ngen)
+ * COLI_K3_CKPT=N OPT-IN recurrent-state checkpoints for serve: keep N
+ * photos of the KDA state at turn boundaries so a
+ * divergent prompt (agentic edit) resumes from the
+ * deepest valid photo instead of re-prefilling from
+ * token 0. Unset/0 (default) = feature off, engine
+ * byte-identical to before. ~434 MB per photo on the
+ * full model, which is why it is opt-in.
* COLI_TEMP=F 0 = greedy (default), else softmax temperature
*/
#define _GNU_SOURCE
@@ -470,6 +477,11 @@ static int g_k3_direct=-1; /* K3_DIRECT: O_DIRECT expert reads */
static int g_k3_idot=1; /* K3_IDOT: int8-activation expert matmuls */
static int g_k3_pipe=1; /* K3_PIPE: overlap loads with compute */
static float g_k3_topp=0.f; /* K3_TOPP: routed-expert top-p pruning */
+enum { K3_CKPT_MAX = 8 };
+static size_t k3_ckpt_blob_floats(const Model *m); /* def. with the ckpt block */
+static int g_k3_ckpt_slots=0; /* COLI_K3_CKPT: recurrent-state checkpoint
+ * slots. STRICTLY OPT-IN, 0 = off (default,
+ * engine behaves exactly as before). */
static int k3_mmap_backend_allowed(int mmap_enabled, int vk_enabled, int cuda_enabled){
return !mmap_enabled || (!vk_enabled && !cuda_enabled);
@@ -774,6 +786,12 @@ static void model_init(Model *m, const char *snap, int n_layers_env){
#endif
g_k3_direct = getenv("K3_DIRECT")?atoi(getenv("K3_DIRECT")):1;
g_k3_idot = getenv("K3_IDOT")?atoi(getenv("K3_IDOT")):1;
+ { /* Recurrent-state checkpoints are strictly OPT-IN: unset or 0 keeps
+ * the engine byte-identical to before the feature existed. */
+ const char *e=getenv("COLI_K3_CKPT");
+ g_k3_ckpt_slots=e?atoi(e):0;
+ if(g_k3_ckpt_slots<0) g_k3_ckpt_slots=0;
+ if(g_k3_ckpt_slots>K3_CKPT_MAX) g_k3_ckpt_slots=K3_CKPT_MAX; }
g_k3_pipe = getenv("K3_PIPE")?atoi(getenv("K3_PIPE")):1;
g_k3_topp = getenv("K3_TOPP")?(float)atof(getenv("K3_TOPP")):0.f;
if(g_k3_topp>0.f)
@@ -1030,6 +1048,9 @@ static void model_init(Model *m, const char *snap, int n_layers_env){
}
fprintf(stderr,"[K3] init done in %.1fs | %d layers | expert cache %d/layer (%.1f MB/slot) | RSS %.1f GB\n",
now_s()-t0,c->n_layers,cap,m->e_slot/1e6,rss_gb());
+ if(g_k3_ckpt_slots>0)
+ fprintf(stderr,"[K3-CKPT] enabled: %d recurrent-state slots (%.1f MB each)\n",
+ g_k3_ckpt_slots,k3_ckpt_blob_floats(m)*sizeof(float)/1048576.0);
/* Phase 9: layer validation init */
{ const char *vl=getenv("K3_VALIDATE_LAYER");
const char *vt=getenv("K3_VALIDATE_TOKEN");
@@ -2389,6 +2410,116 @@ static void model_state_reset(Model *m){
}
}
+/* ---------- recurrent-state checkpoints (OPT-IN, COLI_K3_CKPT=N) ----------
+ *
+ * kv_prefix reuse is all-or-nothing because the 69 KDA layers carry a single
+ * RECURRENT state that exists only at "now": a prompt that diverges anywhere
+ * before the fed length cannot be resumed, so today it resets everything and
+ * re-prefills from token 0 — on a streaming engine that is minutes of disk
+ * reads for an agentic edit that changed one tool result near the end.
+ *
+ * A checkpoint is a photograph of that recurrent state (kstate + the three
+ * conv windows of every KDA layer) taken at a turn boundary, together with
+ * its OWN copy of the token ids that produced it. On a divergent prompt the
+ * deepest photo still valid for the shared prefix is restored and only the
+ * tail is re-prefilled.
+ *
+ * Validity needs BOTH checks, not one: the photo's ids must prefix-match the
+ * new prompt (KDA side), and the restore position must not exceed the live
+ * common prefix of the CURRENT record (the MLA Lc/Rc rows are per-position
+ * and only valid while the record still describes them — after a reset the
+ * record is empty and no restore can happen, which is correct because
+ * model_state_reset also freed those rows).
+ *
+ * With COLI_K3_CKPT unset or 0 (the default), none of this code runs and the
+ * engine is byte-identical to before the feature existed. */
+typedef struct {
+ int pos; /* state covers ids[0..pos-1] */
+ int *fed; /* own copy of those ids */
+ float *blob; /* packed kstate + cwq/cwk/cwv of every KDA layer */
+ uint64_t used; /* LRU stamp */
+} K3Ckpt;
+static K3Ckpt g_k3_ckpt[K3_CKPT_MAX];
+static uint64_t g_k3_ckpt_clock=0;
+
+static size_t k3_ckpt_blob_floats(const Model *m){
+ const Cfg *c=&m->c;
+ size_t per=(size_t)c->kda_heads*c->kda_hd*c->kda_hd
+ +3u*(size_t)c->kda_proj*c->conv_k, n=0;
+ for(int i=0;i<c->n_layers;i++) if(m->L[i].kda) n+=per;
+ return n;
+}
+
+static void k3_ckpt_copy(float *blob, Model *m, int to_blob){
+ Cfg *c=&m->c; size_t o=0;
+ size_t ks=(size_t)c->kda_heads*c->kda_hd*c->kda_hd;
+ size_t cw=(size_t)c->kda_proj*c->conv_k;
+ for(int i=0;i<c->n_layers;i++) if(m->L[i].kda){
+ float *live[4]={m->kstate[i],m->cwq[i],m->cwk[i],m->cwv[i]};
+ size_t len[4]={ks,cw,cw,cw};
+ for(int j=0;j<4;j++){
+ if(to_blob) memcpy(blob+o,live[j],len[j]*sizeof(float));
+ else memcpy(live[j],blob+o,len[j]*sizeof(float));
+ o+=len[j];
+ }
+ }
+}
+
+static void k3_ckpt_save(Model *m){
+ if(g_k3_ckpt_slots<1) return;
+ int pos=m->kvp.len;
+ if(pos<1||m->kvp.tainted||!m->kvp.fed) return;
+ for(int s=0;s<g_k3_ckpt_slots;s++){ /* identical photo: refresh LRU */
+ K3Ckpt *k=&g_k3_ckpt[s];
+ if(k->blob&&k->pos==pos&&
+ !memcmp(k->fed,m->kvp.fed,(size_t)pos*sizeof(int))){
+ k->used=++g_k3_ckpt_clock; return; }
+ }
+ int victim=0;
+ for(int s=0;s<g_k3_ckpt_slots;s++){
+ if(!g_k3_ckpt[s].blob){ victim=s; break; }
+ if(g_k3_ckpt[s].used<g_k3_ckpt[victim].used) victim=s;
+ }
+ K3Ckpt *k=&g_k3_ckpt[victim];
+ if(!k->blob){
+ k->blob=malloc(k3_ckpt_blob_floats(m)*sizeof(float));
+ if(!k->blob) return; /* optimisation, never a failure */
+ }
+ int *fed=realloc(k->fed,(size_t)pos*sizeof(int));
+ if(!fed) return; /* slot keeps its old photo */
+ k->fed=fed;
+ memcpy(k->fed,m->kvp.fed,(size_t)pos*sizeof(int));
+ k3_ckpt_copy(k->blob,m,1);
+ k->pos=pos; k->used=++g_k3_ckpt_clock;
+ if(getenv("K3_PREFIX_LOG"))
+ fprintf(stderr,"[K3-CKPT] saved pos=%d slot=%d\n",pos,victim);
+}
+
+/* Deepest valid photo for this prompt; restores it and returns the covered
+ * positions (the caller prefills only ids[pos..np)), or 0 when none fits. */
+static int k3_ckpt_restore_best(Model *m, const int *ids, int np){
+ if(g_k3_ckpt_slots<1||m->kvp.tainted||!m->kvp.fed) return 0;
+ int live=0, lim=m->kvp.len<np?m->kvp.len:np;
+ while(live<lim&&m->kvp.fed[live]==ids[live]) live++;
+ if(live>=np) live=np-1; /* at least one new token must be prefilled */
+ int best=-1;
+ for(int s=0;s<g_k3_ckpt_slots;s++){
+ K3Ckpt *k=&g_k3_ckpt[s];
+ if(!k->blob||k->pos<1||k->pos>live) continue;
+ if(memcmp(k->fed,ids,(size_t)k->pos*sizeof(int))) continue;
+ if(best<0||k->pos>g_k3_ckpt[best].pos) best=s;
+ }
+ if(best<0) return 0;
+ K3Ckpt *k=&g_k3_ckpt[best];
+ k3_ckpt_copy(k->blob,m,0);
+ m->kvp.len=k->pos; /* the record truncates with the state it describes */
+ k->used=++g_k3_ckpt_clock;
+ if(getenv("K3_PREFIX_LOG"))
+ fprintf(stderr,"[K3-CKPT] restored pos=%d of %d prompt tokens "
+ "(recompute %d)\n",k->pos,np,np-k->pos);
+ return k->pos;
+}
+
/* Decide reuse before changing the state it describes. A miss discards the
* old recurrent/KV state first and allocates a fresh target; a hit grows the
* existing target while preserving its prefix. The old order allocated first,
@@ -2396,6 +2527,9 @@ static void model_state_reset(Model *m){
* MLA prefill immediately indexed them (#855). */
static int prepare_request_state(Model *m, const int *ids, int np, int max_t){
int reuse=kv_prefix_reuse(&m->kvp,ids,np);
+ /* Divergent prompt: before surrendering to a full reset, try the
+ * checkpoint pool (a no-op unless COLI_K3_CKPT enabled it). */
+ if(!reuse) reuse=k3_ckpt_restore_best(m,ids,np);
if(!reuse) model_state_reset(m);
kv_alloc(m,max_t);
return reuse;
@@ -2547,6 +2681,9 @@ static int serve_one(Model *m, Tok *T, ServeReq *q){
if(!poll.fatal) coli_serve_write_error(stdout,q->id,"CANCELLED");
return poll.fatal?-1:0;
}
+ /* Turn boundary: the state now covers exactly the prompt. A photo here is
+ * what an agentic edit of the NEXT request restores from. */
+ k3_ckpt_save(m);
int gen=0, limited=1, cancelled=0, xsup=0, xopen=0, xtl=0;
char buf[512], xtag[320]; /* tool-call open tags carry attributes: call tool="..." index="..." (#1143) */
double tg=now_s();
@@ -2601,6 +2738,9 @@ static int serve_one(Model *m, Tok *T, ServeReq *q){
}
if(poll.fatal){ free(lo); free(ids); return -1; }
free(lo); free(ids);
+ /* End of the reply: the next turn's transcript extends THIS state, and an
+ * edited retry of this turn restores the prompt-boundary photo above. */
+ k3_ckpt_save(m);
double dt=now_s()-t0, decode=now_s()-tg;
uint64_t hits=m->hits-hit0, misses=m->miss-miss0, total=hits+misses;
ColiServeDone done={gen,decode>0?gen/decode:0.0,
diff --git a/c/kimi_k3_tiny/config.json b/c/kimi_k3_tiny/config.json
index d42f6bce..6ae008d7 100644
--- a/c/kimi_k3_tiny/config.json
+++ b/c/kimi_k3_tiny/config.json
@@ -5,7 +5,7 @@
],
"hidden_size": 128,
"num_hidden_layers": 6,
- "vocab_size": 256,
+ "vocab_size": 320,
"first_k_dense_replace": 2,
"intermediate_size": 64,
"num_attention_heads": 4,
diff --git a/c/kimi_k3_tiny/ref.json b/c/kimi_k3_tiny/ref.json
index d2fe6d34..3d50a4b9 100644
--- a/c/kimi_k3_tiny/ref.json
+++ b/c/kimi_k3_tiny/ref.json
@@ -28,22 +28,22 @@
23
],
"teacher_forcing_ids": [
- 4,
+ 127,
6,
- 219,
- 10,
- 69,
- 201,
- 18,
- 78,
- 157,
- 198,
- 33,
- 205,
- 110,
- 109,
- 108,
- 107
+ 253,
+ 186,
+ 287,
+ 11,
+ 253,
+ 22,
+ 156,
+ 166,
+ 81,
+ 80,
+ 161,
+ 160,
+ 126,
+ 67
],
"greedy_full_ids": [
5,
@@ -54,27 +54,27 @@
17,
19,
23,
- 78,
- 157,
- 198,
- 33,
- 205,
- 110,
- 109,
- 108
+ 22,
+ 156,
+ 166,
+ 81,
+ 80,
+ 161,
+ 160,
+ 126
],
"greedy_new_ids": [
- 78,
- 157,
- 198,
- 33,
- 205,
- 110,
- 109,
- 108
+ 22,
+ 156,
+ 166,
+ 81,
+ 80,
+ 161,
+ 160,
+ 126
],
"max_new_tokens": 8,
- "min_logit_margin": 0.2905
+ "min_logit_margin": 0.0822
},
"chunk": {
"prompt_ids": [
@@ -114,56 +114,56 @@
236,
243,
250,
- 2,
- 8,
- 15,
- 22
+ 257,
+ 264,
+ 271,
+ 278
],
"teacher_forcing_ids": [
- 4,
- 111,
+ 127,
+ 11,
18,
- 246,
- 250,
- 155,
- 251,
+ 187,
+ 22,
+ 150,
+ 295,
53,
- 157,
- 67,
- 74,
- 81,
- 88,
- 194,
- 235,
+ 167,
+ 135,
+ 166,
+ 293,
+ 212,
+ 0,
+ 102,
109,
- 118,
- 68,
- 91,
- 70,
+ 200,
+ 290,
+ 130,
+ 167,
144,
- 10,
- 202,
165,
- 200,
- 0,
- 186,
- 157,
- 3,
- 10,
- 122,
- 56,
- 111,
- 4,
- 34,
- 203,
- 127,
- 135,
- 254,
- 146,
- 145,
- 206,
- 205,
- 81
+ 158,
+ 33,
+ 172,
+ 179,
+ 179,
+ 41,
+ 262,
+ 207,
+ 214,
+ 221,
+ 228,
+ 81,
+ 308,
+ 249,
+ 245,
+ 67,
+ 6,
+ 156,
+ 245,
+ 212,
+ 32,
+ 107
],
"greedy_full_ids": [
5,
@@ -202,23 +202,23 @@
236,
243,
250,
- 2,
- 8,
- 15,
- 22,
- 146,
- 145,
- 206,
- 205
+ 257,
+ 264,
+ 271,
+ 278,
+ 156,
+ 245,
+ 212,
+ 32
],
"greedy_new_ids": [
- 146,
- 145,
- 206,
- 205
+ 156,
+ 245,
+ 212,
+ 32
],
"max_new_tokens": 4,
- "min_logit_margin": 0.0274
+ "min_logit_margin": 0.0675
},
"long": {
"prompt_ids": [
@@ -245,133 +245,133 @@
225,
236,
247,
- 2,
- 13,
- 24,
- 35,
- 46,
- 57,
- 68,
- 79,
- 90,
- 101,
- 112,
- 123,
- 134,
- 145,
- 156,
- 167,
- 178,
- 189,
- 200,
- 211,
- 222,
- 233,
- 244,
- 255,
- 10,
- 21,
- 32,
- 43,
- 54,
- 65,
- 76,
- 87,
- 98,
- 109,
- 120,
- 131,
- 142,
- 153,
- 164,
- 175,
- 186,
- 197,
- 208,
- 219,
- 230,
- 241,
- 252,
- 7,
- 18
- ],
- "teacher_forcing_ids": [
+ 258,
+ 269,
+ 280,
+ 291,
+ 302,
+ 313,
4,
- 144,
- 111,
+ 15,
+ 26,
37,
- 255,
- 244,
+ 48,
+ 59,
70,
- 206,
- 202,
+ 81,
+ 92,
103,
- 171,
- 14,
- 165,
+ 114,
+ 125,
+ 136,
147,
- 118,
- 133,
+ 158,
+ 169,
180,
- 6,
- 154,
+ 191,
+ 202,
213,
- 97,
- 226,
+ 224,
+ 235,
246,
- 28,
- 184,
+ 257,
+ 268,
+ 279,
+ 290,
+ 301,
+ 312,
3,
- 34,
- 141,
- 56,
- 189,
- 97,
- 81,
- 66,
- 105,
- 116,
- 133,
- 48,
- 155,
- 166,
+ 14,
+ 25,
+ 36,
+ 47,
+ 58,
+ 69,
+ 80,
+ 91,
+ 102,
+ 113,
+ 124,
+ 135,
+ 146
+ ],
+ "teacher_forcing_ids": [
+ 127,
+ 15,
+ 26,
+ 224,
+ 233,
+ 59,
+ 104,
+ 289,
+ 92,
+ 140,
+ 173,
+ 285,
+ 76,
+ 6,
+ 158,
+ 219,
+ 180,
+ 191,
+ 287,
+ 8,
+ 224,
+ 273,
+ 246,
+ 107,
+ 268,
+ 279,
+ 290,
+ 301,
+ 140,
+ 3,
+ 188,
+ 302,
+ 120,
+ 318,
+ 168,
+ 269,
+ 80,
+ 301,
+ 102,
+ 145,
+ 153,
+ 51,
+ 111,
+ 157,
+ 168,
+ 6,
+ 135,
+ 4,
+ 176,
+ 223,
+ 234,
+ 245,
+ 22,
+ 177,
184,
- 163,
- 199,
- 210,
- 170,
- 61,
- 243,
- 210,
- 235,
- 20,
- 31,
- 186,
- 53,
- 254,
- 75,
- 86,
- 97,
- 65,
- 119,
+ 227,
+ 140,
+ 220,
+ 2,
+ 13,
+ 135,
+ 35,
+ 29,
+ 181,
+ 204,
+ 140,
+ 90,
+ 101,
+ 59,
+ 7,
+ 34,
+ 145,
27,
- 142,
- 254,
- 177,
- 202,
- 185,
- 4,
- 185,
- 218,
- 251,
- 54,
- 251,
- 80,
- 11,
- 10,
- 9,
- 202,
- 201
+ 26,
+ 38,
+ 184
],
"greedy_full_ids": [
5,
@@ -397,68 +397,68 @@
225,
236,
247,
- 2,
- 13,
- 24,
- 35,
- 46,
- 57,
- 68,
- 79,
- 90,
- 101,
- 112,
- 123,
- 134,
+ 258,
+ 269,
+ 280,
+ 291,
+ 302,
+ 313,
+ 4,
+ 15,
+ 26,
+ 37,
+ 48,
+ 59,
+ 70,
+ 81,
+ 92,
+ 103,
+ 114,
+ 125,
+ 136,
+ 147,
+ 158,
+ 169,
+ 180,
+ 191,
+ 202,
+ 213,
+ 224,
+ 235,
+ 246,
+ 257,
+ 268,
+ 279,
+ 290,
+ 301,
+ 312,
+ 3,
+ 14,
+ 25,
+ 36,
+ 47,
+ 58,
+ 69,
+ 80,
+ 91,
+ 102,
+ 113,
+ 124,
+ 135,
+ 146,
145,
- 156,
- 167,
- 178,
- 189,
- 200,
- 211,
- 222,
- 233,
- 244,
- 255,
- 10,
- 21,
- 32,
- 43,
- 54,
- 65,
- 76,
- 87,
- 98,
- 109,
- 120,
- 131,
- 142,
- 153,
- 164,
- 175,
- 186,
- 197,
- 208,
- 219,
- 230,
- 241,
- 252,
- 7,
- 18,
- 11,
- 10,
- 9,
- 202
+ 27,
+ 26,
+ 38
],
"greedy_new_ids": [
- 11,
- 10,
- 9,
- 202
+ 145,
+ 27,
+ 26,
+ 38
],
"max_new_tokens": 4,
- "min_logit_margin": 0.0253
+ "min_logit_margin": 0.0475
}
}
}
diff --git a/c/tests/test_kimi_k3_ckpt.py b/c/tests/test_kimi_k3_ckpt.py
new file mode 100644
index 00000000..06afa27a
--- /dev/null
+++ b/c/tests/test_kimi_k3_ckpt.py
@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+"""Recurrent-state checkpoints (COLI_K3_CKPT) against the tiny K3 fixture.
+
+The scenario is the agentic edit kv-prefix reuse cannot serve: turn 2 shares
+only the OPENING of turn 1's prompt and diverges after it (an edited question,
+a replaced tool result). Today that is a full reset plus a full re-prefill,
+because the 69 KDA layers carry a recurrent state that cannot be rewound.
+With checkpoints enabled the engine restores the photo taken at turn 1's
+prompt boundary and recomputes only the tail.
+
+What is asserted, all deterministic:
+ - OPT-IN INVARIANT: with COLI_K3_CKPT unset the engine emits no [K3-CKPT]
+ line and behaves as before (this is the default-off run itself).
+ - IDENTITY: every request's DATA bytes are identical with checkpoints off
+ and on. The feature may only change WORK, never tokens.
+ - RESTORE DEPTH: with checkpoints on, the divergent turns restore at
+ exactly turn 1's ACCEPT np (the prompt-boundary photo), and the engine
+ recomputes prompt_len - np tokens instead of prompt_len.
+
+The serve protocol needs a tokenizer, and the base fixture ships none on
+purpose (the oracle test parses raw-id stdout). So this test builds a derived
+fixture in a tempdir: config + weights from the base fixture plus
+tests/tok_kimi_tiny.json as tokenizer.json. The fixture's vocab (320) covers
+every id that tokenizer can emit (<= 285).
+"""
+
+import argparse
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+
+class Serve:
+ """One persistent SERVE=1 kimi_k3 process; stderr goes to a file so the
+ pipe can never fill and deadlock the engine."""
+
+ def __init__(self, binary, model, extra_env, stderr_path):
+ env = dict(os.environ, SERVE="1", SNAP=str(model), K3_PREFIX_LOG="1")
+ env.update(extra_env)
+ self.stderr_file = open(stderr_path, "wb")
+ self.process = subprocess.Popen(
+ [str(binary)], stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+ stderr=self.stderr_file, env=env)
+ self.counter = 0
+
+ def submit(self, prompt, max_tokens):
+ """Returns (generated bytes, ACCEPT np) for one request."""
+ self.counter += 1
+ request_id = f"r{self.counter}"
+ payload = prompt.encode("utf-8")
+ header = (f"SUBMIT {request_id} 0 {len(payload)} {max_tokens} "
+ f"0.0 1.0\n").encode("ascii")
+ self.process.stdin.write(header + payload + b"\n")
+ self.process.stdin.flush()
+ pieces, accept_np = [], -1
+ stream = self.process.stdout
+ while True:
+ line = stream.readline()
+ if not line:
+ raise AssertionError(
+ f"engine closed stdout during {request_id}")
+ fields = line.decode("utf-8", "replace").split()
+ if not fields:
+ continue
+ if fields[0] == "ERROR":
+ raise AssertionError(f"engine ERROR: {line!r}")
+ if fields[0] == "ACCEPT" and len(fields) >= 3:
+ accept_np = int(fields[2])
+ elif fields[0] == "DATA" and len(fields) == 3:
+ size = int(fields[2])
+ pieces.append(stream.read(size))
+ stream.read(1) # trailing newline
+ elif fields[0] == "DONE":
+ break
+ # READY / PROF and other frames are irrelevant here
+ return b"".join(pieces), accept_np
+
+ def close(self):
+ try:
+ if self.process.stdin:
+ self.process.stdin.close()
+ self.process.wait(timeout=30)
+ except Exception:
+ self.process.kill()
+ self.stderr_file.close()
+
+
+PROMPT_BASE = "Alpha beta gamma delta epsilon zeta.\n"
+TAIL_A = "Question about apples?\n"
+TAIL_B = "Question about bridges?\n"
+MAX_NEW = 6
+
+
+def run_session(binary, model, extra_env, stderr_path):
+ serve = Serve(binary, model, extra_env, stderr_path)
+ try:
+ results = [serve.submit(PROMPT_BASE, MAX_NEW),
+ serve.submit(PROMPT_BASE + TAIL_A, MAX_NEW),
+ serve.submit(PROMPT_BASE + TAIL_B, MAX_NEW)]
+ finally:
+ serve.close()
+ return results, Path(stderr_path).read_text(errors="replace")
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ parser.add_argument("--binary", required=True)
+ parser.add_argument("--fixture", required=True)
+ parser.add_argument("--tokenizer",
+ default=str(Path(__file__).parent /
+ "tok_kimi_tiny.json"))
+ args = parser.parse_args()
+
+ base = Path(args.fixture)
+ work = Path(tempfile.mkdtemp(prefix="kimi-k3-ckpt-"))
+ try:
+ model = work / "fixture"
+ model.mkdir()
+ shutil.copy(base / "config.json", model / "config.json")
+ shutil.copy(base / "model.safetensors", model / "model.safetensors")
+ shutil.copy(args.tokenizer, model / "tokenizer.json")
+
+ off, err_off = run_session(args.binary, model, {},
+ work / "off.stderr")
+ on, err_on = run_session(args.binary, model, {"COLI_K3_CKPT": "4"},
+ work / "on.stderr")
+
+ # opt-in invariant: the default run must not know the feature exists
+ if "[K3-CKPT]" in err_off:
+ raise AssertionError(
+ "COLI_K3_CKPT unset but the engine logged checkpoint "
+ "activity: the feature is not opt-in")
+ print("ok opt-in: default run has no checkpoint activity")
+
+ # identity: the feature may change work, never tokens
+ for index, (off_result, on_result) in enumerate(zip(off, on), 1):
+ if off_result[0] != on_result[0]:
+ raise AssertionError(
+ f"request {index}: output differs between checkpoint "
+ f"modes: {off_result[0]!r} vs {on_result[0]!r}")
+ if off_result[1] != on_result[1]:
+ raise AssertionError(
+ f"request {index}: prompt token count differs between "
+ f"modes ({off_result[1]} vs {on_result[1]})")
+ print("ok identity: all 3 requests byte-identical across modes")
+
+ # scenario validity: in the baseline the divergent turns really are
+ # the case kv-prefix reuse cannot serve
+ if err_off.count("[PREFIX] no reuse") < 2:
+ raise AssertionError(
+ "baseline did not hit the divergent-prompt path; the "
+ "scenario no longer tests checkpoints:\n" + err_off)
+
+ # restore depth: both divergent turns restore at exactly turn 1's
+ # prompt boundary and recompute only their tail
+ np1 = off[0][1]
+ restores = [(int(pos), int(total), int(recompute))
+ for pos, total, recompute in re.findall(
+ r"\[K3-CKPT\] restored pos=(\d+) of (\d+) prompt "
+ r"tokens \(recompute (\d+)\)", err_on)]
+ if len(restores) != 2:
+ raise AssertionError(
+ f"expected 2 checkpoint restores (turns 2 and 3), saw "
+ f"{len(restores)}:\n{err_on}")
+ for turn, (pos, total, recompute) in zip((2, 3), restores):
+ expect_total = on[turn - 1][1]
+ if pos != np1 or total != expect_total or \
+ recompute != expect_total - np1:
+ raise AssertionError(
+ f"turn {turn}: restored pos={pos} of {total} "
+ f"(recompute {recompute}), expected pos={np1} of "
+ f"{expect_total} (recompute {expect_total - np1})")
+ print(f"ok restore turn {turn}: {pos}/{total} reused, "
+ f"{recompute} recomputed instead of {total}")
+ if "[K3-CKPT] saved" not in err_on:
+ raise AssertionError("no checkpoint was ever saved:\n" + err_on)
+ print("kimi-k3-ckpt: all checks passed")
+ return 0
+ finally:
+ shutil.rmtree(work, ignore_errors=True)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/c/tools/make_kimi_k3_tiny.py b/c/tools/make_kimi_k3_tiny.py
index 68022cd3..6a7d8d0b 100644
--- a/c/tools/make_kimi_k3_tiny.py
+++ b/c/tools/make_kimi_k3_tiny.py
@@ -49,7 +49,7 @@ import numpy as np
SEED = 20260823
-VOCAB = 256
+VOCAB = 320
HIDDEN = 128
LAYERS = 6
FIRST_DENSE = 2 # layers 0..1 use the dense MLP path