Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
feat(kimi): learn from use β .coli_usage, autopin and decay, as GLM has
JustVugg9212eff2f3c2
diff --git a/c/kimi_k3.c b/c/kimi_k3.cindex d93602b3..cd1d7657 100644--- a/c/kimi_k3.c+++ b/c/kimi_k3.c@@ -135,7 +135,13 @@ typedef struct {/* ---------- routed-expert streaming (native MXFP4 from the HF shards) ---- */typedef struct { int fd[6]; int64_t off[6]; int contig; } ERef; /* w1p w1s w2p w2s w3p w3s */-typedef struct { int eid; uint8_t *buf, *base; uint64_t used; } Slot;+static char g_k3_usage[2100]; /* <snap>/.coli_usage, or COLI_USAGE */+typedef struct { int eid; uint8_t *buf, *base; uint64_t used; int pinned; } Slot;+/* pinned: seeded from .coli_usage at startup and never evicted. The LRU adapts to+ * THIS session; the pin knows the history of every session before it. Capped at+ * half the layer budget so the adaptive half always survives β an all-pinned cache+ * that guessed wrong is slower than no pin at all (measured on GLM: 0.17 vs 0.25+ * tok/s when the pins came from a single prompt). *//* base = 4K-aligned allocation (O_DIRECT target);* buf = expert data view inside it (= base + off%4K) */typedef struct { Slot *s; int n, cap; } LCache;@@ -1039,13 +1045,98 @@ static void experts_apply_union(Model *m, int li, int nu, const int *uids,for(int a=0;a<promo;a++){int q=nmiss-1-a; Slot *dst;if(lc->n<lc->cap) dst=&lc->s[lc->n++];- else { int lru=0; for(int i=1;i<lc->n;i++) if(lc->s[i].used<lc->s[lru].used) lru=i; dst=&lc->s[lru]; }+ else {+ /* LRU over the UNPINNED slots. pin_seed caps pinned at cap/2, so a+ * victim always exists; the -1 fallback is a belt-and-braces guard+ * against a future caller pinning everything and deadlocking here. */+ int lru=-1;+ for(int i=0;i<lc->n;i++){+ if(lc->s[i].pinned) continue;+ if(lru<0 || lc->s[i].used<lc->s[lru].used) lru=i;+ }+ if(lru<0) break; /* every slot pinned: keep the read */+ dst=&lc->s[lru];+ }Slot tmp=*dst; *dst=m->ws[q]; m->ws[q]=tmp;dst->used=++m->clock;}}}+/* ---- AUTOPIN: seed the layer caches from the accumulated history ----------+ *+ * K3 counted every routed selection (rt_route) and could READ a history file,+ * but only when COLI_USAGE named one by hand, and it never WROTE one β so the+ * read path was unreachable for anyone who did not already have a profile from+ * somewhere else. colibri.c has seeded itself from <snap>/.coli_usage since the+ * beginning and inkling.c does too; this brings K3 in line.+ *+ * The quota scales with CONFIDENCE in the history, exactly as colibri.c does it:+ * a handful of turns is a bad predictor and pinning on it steals slots from the+ * LRU, which at least adapts to the session in front of it. Below 5,000 recorded+ * selections nothing is pinned at all; the quota reaches its cap of half the+ * layer budget at 200,000, which is a few hours of real use.+ *+ * Cost is bounded and paid once: at most cap/2 experts per sparse layer, read+ * sequentially at startup rather than demand-faulted mid-token. */+static void k3_usage(const char *argv0){+ fprintf(stderr,+ "usage: %s <model_dir> [prompt] [options]\n"+ "\n"+ " --chat apply the K3 chat template to the prompt\n"+ " --system TEXT system message (implies --chat)\n"+ " --ngen N tokens to generate (default 32)\n"+ " --ids \"1 2 3\" raw token ids instead of a prompt\n"+ " (needed when the snapshot has no tokenizer.json;\n"+ " generate one with tools/k3_tokenizer.py)\n"+ "\n"+ " %s /path/to/kimi \"What is the capital of France?\" --ngen 64\n"+ "\n"+ "For an interactive session use the launcher instead, which starts this\n"+ "engine for you and does not need the GLM engine built:\n"+ " coli chat --model /path/to/kimi\n"+ "\n"+ "environment: SERVE=1 serve mode (SNAP=<dir>) - COLI_VULKAN=1 GPU path\n"+ " OMP_NUM_THREADS=<physical cores> - AUTOPIN=0 no pin seeding\n",+ argv0, argv0);+}++static void pin_seed(Model *m, int64_t hist){+ Cfg *c=&m->c;+ if(getenv("AUTOPIN") && atoi(getenv("AUTOPIN"))==0) return;+ if(hist<5000) return;+ double conf=(double)hist/200000.0; if(conf>1) conf=1;+ int pinned_total=0;+ for(int li=0; li<c->n_layers; li++){+ if(!m->L[li].sparse) continue;+ LCache *lc=&m->ecache[li];+ int quota=(int)(lc->cap*0.5*conf);+ if(quota<1) continue;+ if(quota>lc->cap/2) quota=lc->cap/2;+ const uint32_t *u=rt_counts(li);+ if(!u) continue;+ /* top-`quota` by recorded use: a partial selection sort, since quota is+ * small (half a cache) and E is 896 β no allocation, no qsort callback. */+ for(int slot=0; slot<quota && lc->n<lc->cap; slot++){+ int best=-1; uint32_t bestc=0;+ for(int e=0;e<c->n_experts;e++){+ if(u[e]<=bestc) continue;+ int taken=0;+ for(int i=0;i<lc->n;i++) if(lc->s[i].eid==e){ taken=1; break; }+ if(!taken){ best=e; bestc=u[e]; }+ }+ if(best<0) break; /* history is exhausted */+ Slot *d=&lc->s[lc->n];+ expert_read(m,li,best,d);+ d->eid=best; d->used=++m->clock; d->pinned=1;+ lc->n++; pinned_total++;+ }+ }+ if(pinned_total)+ fprintf(stderr,"[PIN] %d experts pinned from history (%lld selections, %.0f%% confidence)\n",+ pinned_total,(long long)hist,100.0*conf);+}+static void moe_forward(Model *m, Layer *l, int li, const float *x, int C, float *out){Cfg *c=&m->c; Moe *o=&l->moe;int E=c->n_experts, K=c->topk, LT=c->latent, MI=c->moe_inter;@@ -1639,9 +1730,15 @@ static void serve_loop(Model *m, Tok *T){int main(int argc, char **argv){coli_omp_tune_threads("kimi_k3"); /* squadra sui core fisici, niente spin-wait: vedi omp_tune.h */int serving=getenv("SERVE")&&getenv("SERVE")[0]=='1';- if(!serving&&argc<2){- fprintf(stderr,"usage: %s <model_dir> [prompt] [--ids \"1 2 3\"] [--ngen N]\n",argv[0]);- return 1;+ /* Usage was printed only when there were NO arguments, so `--help` fell+ * through as the model directory and the engine went looking for+ * "--help/config.json". Nobody should have to read the source to find the+ * argument order. Print it for the help flags too, and from every refusal+ * below, so an error says what to do instead of only what went wrong. */+ if(!serving && (argc<2 || !strcmp(argv[1],"--help") || !strcmp(argv[1],"-h")+ || !strcmp(argv[1],"help"))){+ k3_usage(argv[0]);+ return argc<2 ? 1 : 0; /* asking for help is not a failure */}const char *snap=serving?getenv("SNAP"):argv[1], *prompt=NULL, *idstr=NULL, *sysmsg=NULL, *wirepath=NULL;if(!snap||!*snap){ fprintf(stderr,"set SNAP=<Kimi K3 snapshot directory>\n"); return 1; }@@ -1679,9 +1776,22 @@ int main(int argc, char **argv){* silently absorbed and written back out. See docs/routing-telemetry.md. */for(int i=0;i<m.c.n_layers;i++) if(!m.L[i].sparse) rt_drop_row(i);rt_drop_row(m.c.n_layers);- { const char *up=getenv("COLI_USAGE"); /* optional history to seed from */- if(up&&*up){ int64_t h=rt_load(up);- if(h>0) fprintf(stderr,"[USAGE] expert history: %lld selections (%s)\n",(long long)h,up); } }+ /* LEARNED CACHE. Expert use accumulates in <snap>/.coli_usage across sessions+ * and seeds the pins at startup, the same convention colibri.c and inkling.c+ * already follow. COLI_USAGE overrides the location; USAGE_SAVE=0 makes the+ * run read-only, for benchmark loops that would otherwise skew the profile+ * they are measuring. */+ { const char *up=getenv("COLI_USAGE");+ if(up&&*up) snprintf(g_k3_usage,sizeof(g_k3_usage),"%s",up);+ else snprintf(g_k3_usage,sizeof(g_k3_usage),"%s/.coli_usage",snap);+ int64_t h=rt_load(g_k3_usage);+ if(h>0) fprintf(stderr,"[USAGE] expert history: %lld selections (%s)\n",+ (long long)h,g_k3_usage);+ /* #780: without a half-life the ranking freezes β after ~18M recorded+ * selections one more turn moves it by 0.2% and the profile stops+ * following the workload. */+ if(getenv("COLI_USAGE_DECAY")) rt_decay();+ pin_seed(&m,h); }if(getenv("K3_TRACE")){m.trace=fopen(getenv("K3_TRACE"),"wb");if(!m.trace){ perror(getenv("K3_TRACE")); return 1; }@@ -1723,10 +1833,12 @@ int main(int argc, char **argv){fprintf(stderr,"\n");}} else if(prompt){- if(!has_tok){ fprintf(stderr,"no tokenizer.json β pass --ids (generate one with tools/k3_tokenizer.py)\n"); return 1; }+ if(!has_tok){ fprintf(stderr,"no tokenizer.json in the snapshot β pass --ids instead\n\n");+ k3_usage(argv[0]); return 1; }if(m.c.bos>=0) ids[np++]=m.c.bos;np+=tok_encode(&T,prompt,(int)strlen(prompt),ids+np,65536-np);- } else { fprintf(stderr,"no prompt and no --ids\n"); return 1; }+ } else { fprintf(stderr,"no prompt and no --ids: nothing to generate from\n\n");+ k3_usage(argv[0]); return 1; }fprintf(stderr,"[K3] prompt: %d tokens | ngen %d | temp %.2f\n",np,ngen,temp);int max_t=getenv("K3_MAXT")?atoi(getenv("K3_MAXT")):np+ngen;kv_alloc(&m,max_t);@@ -1800,7 +1912,8 @@ int main(int argc, char **argv){fprintf(stderr,"[K3] time: attn %.1fs moe %.1fs (eload %.1fs) head %.1fs | RSS %.1f GB\n",m.t_attn,m.t_moe,m.t_eload,m.t_head,rss_gb());if(m.trace) fclose(m.trace);- { const char *up=getenv("COLI_USAGE");- if(up&&*up) rt_save(up,0); } /* same bytes as every other engine */+ { const char *sv=getenv("USAGE_SAVE");+ if(!(sv && atoi(sv)==0) && g_k3_usage[0])+ rt_save(g_k3_usage,0); } /* same bytes as every other engine */return 0;}diff --git a/c/tests/test_kimi_usage_cli.py b/c/tests/test_kimi_usage_cli.pynew file mode 100644index 00000000..b56e5e1f--- /dev/null+++ b/c/tests/test_kimi_usage_cli.py@@ -0,0 +1,65 @@+"""kimi_k3's command line must be discoverable, and its refusals must be useful.++`--help` used to fall through as the model directory: the engine went looking+for `--help/config.json` and a user could only learn the argument order by+reading 1,800 lines of C. Reported on #783 by someone who had already got the+build right and was stopped by the interface.++`no prompt and no --ids` is accurate and useless on its own β an error should+say what to do, not only what went wrong.+"""+import os+import subprocess+import sys+import unittest+from pathlib import Path++HERE = Path(__file__).resolve().parent.parent+ENGINE = HERE / ("kimi_k3.exe" if sys.platform == "win32" else "kimi_k3")+++def run(*args, **env):+ return subprocess.run([str(ENGINE), *args], cwd=HERE, text=True,+ capture_output=True, timeout=120,+ env=dict(os.environ, **env))++[email protected](ENGINE.exists(), "kimi_k3 is not built")+class KimiUsageTest(unittest.TestCase):+ def assertUsage(self, out, ctx):+ for fragment in ("<model_dir>", "--ngen", "--ids", "coli chat --model"):+ self.assertIn(fragment, out, f"{ctx}: usage is missing {fragment!r}")++ def test_help_flags_print_usage_and_succeed(self):+ for flag in ("--help", "-h", "help"):+ with self.subTest(flag=flag):+ r = run(flag)+ out = r.stdout + r.stderr+ self.assertUsage(out, flag)+ self.assertEqual(r.returncode, 0,+ f"{flag} must succeed; asking for help is not an error")+ self.assertNotIn("config.json", out,+ f"{flag} was treated as a model directory")++ def test_no_arguments_prints_usage_and_fails(self):+ r = run()+ self.assertUsage(r.stdout + r.stderr, "no args")+ self.assertNotEqual(r.returncode, 0, "a missing model dir is a failure")++ def test_usage_names_the_launcher(self):+ """Most people reaching for the raw engine want `coli chat`. It runs this+ engine for them and does not need the GLM binary built β which is what+ the #783 reporter had assumed was required."""+ self.assertIn("coli chat --model", run("--help").stdout + run("--help").stderr)++ def test_usage_documents_autopin_and_the_gpu_path(self):+ """Both are things a user cannot guess and will otherwise never find:+ Kimi's NVIDIA path is Vulkan, not CUDA (#783), and AUTOPIN=0 is the only+ way to opt out of pin seeding once a history exists."""+ out = run("--help").stdout + run("--help").stderr+ self.assertIn("COLI_VULKAN=1", out)+ self.assertIn("AUTOPIN=0", out)+++if __name__ == "__main__":+ unittest.main()