Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
kimi_k3: --chat β K3's XTML chat format with the preserved-thinking channel
Steve Markgrafeb3fbbc5fba8
diff --git a/c/kimi_k3.c b/c/kimi_k3.cindex 28010b99..58cbac92 100644--- a/c/kimi_k3.c+++ b/c/kimi_k3.c@@ -46,6 +46,7 @@* K3_LOAD_THREADS=N loader threads for K3_PIPE (default 4)* K3_TOPP=F keep routed experts to cumulative weight F (0 = off)* K3_CHUNK=N prefill chunk size (default 32; 1 = token-at-a-time)+ * K3_THINK=0|1 chat mode: open the think channel (default 1)* K3_LAYERS=N truncate to first N layers (validation; skips head)* K3_TRACE=path dump f32 hidden state after every layer (validation)* K3_LOGITS=path dump f32 logits per PREFILL position (teacher-forced@@ -1099,16 +1100,73 @@ static int sample_tok(const float *lo, int V, float temp){free(p); return pick;}+/* ---------- K3 XTML chat format (faithful to the shipped encoding_k3.py) --+ * Only <|open|>, <|close|>, <|sep|>, <|end_of_msg|> are special TOKENS; tag+ * names and attributes are ordinary text, encoded as the same standalone+ * segments as the reference (segment boundaries are token boundaries). A+ * turn renders as+ * <|open|>message role="user"<|sep|>TEXT<|close|>message<|sep|><|end_of_msg|>+ * and the generation prompt opens the assistant message plus its structural+ * thinking channel:+ * <|open|>message role="assistant"<|sep|><|open|>think<|sep|>+ * (K3_THINK=0 opens <response> directly = non-thinking mode). The model then+ * closes think, opens response, and finishes with <|end_of_msg|> (the eos). */+typedef struct { Tok *T; int *ids; int n, cap;+ int sp_open, sp_close, sp_sep, sp_eom; } ChatB;+static void cb_special(ChatB *b, int id){+ if(b->n>=b->cap){ fprintf(stderr,"chat prompt too long\n"); exit(1); }+ b->ids[b->n++]=id;+}+static void cb_text(ChatB *b, const char *s){+ if(!*s) return;+ b->n+=tok_encode(b->T,s,(int)strlen(s),b->ids+b->n,b->cap-b->n);+}+static void cb_open(ChatB *b, const char *tag, const char *role){+ cb_special(b,b->sp_open); cb_text(b,tag);+ if(role){ cb_text(b," role"); cb_text(b,"=\""); cb_text(b,role); cb_text(b,"\""); }+ cb_special(b,b->sp_sep);+}+static void cb_close(ChatB *b, const char *tag){+ cb_special(b,b->sp_close); cb_text(b,tag); cb_special(b,b->sp_sep);+}+static int chat_special(Tok *T, const char *s){+ int l=(int)strlen(s);+ for(int i=0;i<T->nsp;i++)+ if(T->sp[i].len==l && !memcmp(T->sp[i].str,s,l)) return T->sp[i].id;+ return -1;+}+/* returns prompt length; sp[4] = {open, close, sep, end_of_msg} ids */+static int chat_build(Tok *T, const char *sys, const char *user, int thinking,+ int *ids, int cap, int *sp){+ ChatB b={T,ids,0,cap,+ chat_special(T,"<|open|>"), chat_special(T,"<|close|>"),+ chat_special(T,"<|sep|>"), chat_special(T,"<|end_of_msg|>")};+ if(b.sp_open<0||b.sp_close<0||b.sp_sep<0||b.sp_eom<0){+ fprintf(stderr,"chat: XTML special tokens not in tokenizer.json\n"); exit(1); }+ sp[0]=b.sp_open; sp[1]=b.sp_close; sp[2]=b.sp_sep; sp[3]=b.sp_eom;+ if(sys&&*sys){+ cb_open(&b,"message","system"); cb_text(&b,sys);+ cb_close(&b,"message"); cb_special(&b,b.sp_eom);+ }+ cb_open(&b,"message","user"); cb_text(&b,user);+ cb_close(&b,"message"); cb_special(&b,b.sp_eom);+ cb_open(&b,"message","assistant");+ cb_open(&b,thinking?"think":"response",NULL);+ return b.n;+}+int main(int argc, char **argv){if(argc<2){fprintf(stderr,"usage: %s <model_dir> [prompt] [--ids \"1 2 3\"] [--ngen N]\n",argv[0]);return 1;}- const char *snap=argv[1], *prompt=NULL, *idstr=NULL;- int ngen=32;+ const char *snap=argv[1], *prompt=NULL, *idstr=NULL, *sysmsg=NULL;+ int ngen=32, chat=0;for(int i=2;i<argc;i++){if(!strcmp(argv[i],"--ngen")&&i+1<argc) ngen=atoi(argv[++i]);else if(!strcmp(argv[i],"--ids")&&i+1<argc) idstr=argv[++i];+ else if(!strcmp(argv[i],"--chat")) chat=1;+ else if(!strcmp(argv[i],"--system")&&i+1<argc) sysmsg=argv[++i];else if(!prompt) prompt=argv[i];}float temp=getenv("COLI_TEMP")?(float)atof(getenv("COLI_TEMP")):0.f;@@ -1135,9 +1193,21 @@ int main(int argc, char **argv){{ char tp[2048]; snprintf(tp,sizeof(tp),"%s/tokenizer.json",snap);FILE *f=fopen(tp,"rb"); if(f){ fclose(f); tok_load(&T,tp); has_tok=1;fprintf(stderr,"[K3] tokenizer.json loaded (family=%s)\n",T.kimi?"kimi":(T.o200k?"o200k":"cl100k")); } }+ int sp[4]={-1,-1,-1,-1};+ int think=getenv("K3_THINK")?atoi(getenv("K3_THINK")):1;if(idstr){const char *p=idstr;while(*p&&np<65536){ while(*p==' '||*p==',')p++; if(!*p)break; ids[np++]=(int)strtol(p,(char**)&p,10); }+ } else if(chat){+ if(!has_tok){ fprintf(stderr,"--chat needs tokenizer.json\n"); return 1; }+ if(!prompt){ fprintf(stderr,"--chat needs a user message\n"); return 1; }+ if(m.c.bos>=0) ids[np++]=m.c.bos;+ np+=chat_build(&T,sysmsg,prompt,think,ids+np,65536-np,sp);+ if(getenv("K3_CHAT_IDS")){+ fprintf(stderr,"[K3] chat ids:");+ for(int i=0;i<np;i++) fprintf(stderr," %d",ids[i]);+ fprintf(stderr,"\n");+ }} else if(prompt){if(!has_tok){ fprintf(stderr,"no tokenizer.json β pass --ids (generate one with tools/k3_tokenizer.py)\n"); return 1; }if(m.c.bos>=0) ids[np++]=m.c.bos;@@ -1173,12 +1243,33 @@ int main(int argc, char **argv){}double tg=now_s(); int ntok=0;char buf[512];+ /* chat print filter: hide the XTML structure, label the channels.+ * Structural runs are <|open|>/<|close|> TAGTEXT <|sep|> β suppress them+ * and print a channel banner when the response channel opens. */+ int xsup=0, xopen=0; char xtag[64]; int xtl=0;+ if(chat&&think){ printf("[think] "); fflush(stdout); }for(int s=0;s<ngen;s++){int t=sample_tok(lo,m.c.vocab,temp);free(lo); lo=NULL;int is_eos=0; for(int e=0;e<m.c.n_eos;e++) if(t==m.c.eos[e]) is_eos=1;- if(has_tok){ int n2=tok_decode(&T,&t,1,buf,sizeof(buf)-1); fwrite(buf,1,n2,stdout); fflush(stdout); }- else { printf("%d ",t); fflush(stdout); }+ int show=1;+ if(chat&&sp[0]>=0){+ if(t==sp[0]||t==sp[1]){ xsup=1; xopen=(t==sp[0]); xtl=0; show=0; }+ else if(t==sp[2]){+ if(xsup){ xsup=0; xtag[xtl]=0;+ if(xopen&&!strcmp(xtag,"response")){ printf("\n\n[response] "); fflush(stdout); }+ }+ show=0;+ } else if(xsup){+ if(has_tok){ int n2=tok_decode(&T,&t,1,buf,sizeof(buf)-1);+ if(xtl+n2<(int)sizeof(xtag)){ memcpy(xtag+xtl,buf,n2); xtl+=n2; } }+ show=0;+ } else if(t==sp[3]) show=0;+ }+ if(show){+ if(has_tok){ int n2=tok_decode(&T,&t,1,buf,sizeof(buf)-1); fwrite(buf,1,n2,stdout); fflush(stdout); }+ else { printf("%d ",t); fflush(stdout); }+ }ntok++;if(is_eos){ fprintf(stderr,"\n[K3] eos\n"); break; }if(np+ntok>=max_t){ fprintf(stderr,"\n[K3] context full\n"); break; }diff --git a/docs/kimi_k3.md b/docs/kimi_k3.mdindex 04e6e597..5e5b948d 100644--- a/docs/kimi_k3.md+++ b/docs/kimi_k3.md@@ -160,6 +160,7 @@ Judge quantization choices on real-text logits, not synthetic-vector norms.| `K3_LOAD_THREADS` | 4 | loader threads for `K3_PIPE` || `K3_TOPP` | 0 | keep routed experts to cumulative weight p (0 = off) || `K3_CHUNK` | 32 | prefill chunk size (1 = token-at-a-time; forced 1 under `K3_TRACE`) |+| `K3_THINK` | 1 | chat mode: open the structural think channel (0 = response-only) || `K3_DIRS` | β | extra shard directories (multi-drive split, no duplication) || `K3_MAXT` | prompt+ngen | context capacity || `K3_LAYERS` | all | truncate the stack (validation) |@@ -182,8 +183,33 @@ teacher-forced logit streams at C=32 vs C=1 match exactly). Measured prefill:~5.3 -> **2.0 s/token** at C=32. The KDA state-update sweeps are AVX2(scalar fallback for head dims not divisible by 8).+## Chat mode (`--chat`)++```+./kimi_k3 <model_dir> --chat "your question" [--system "..."] --ngen 300+```++K3's chat format ("XTML", from the checkpoint's own `encoding_k3.py`) uses+only four special tokens β `<|open|>`, `<|close|>`, `<|sep|>`,+`<|end_of_msg|>` β around ordinary-text tag names and attributes:++```+<|open|>message role="user"<|sep|>TEXT<|close|>message<|sep|><|end_of_msg|>+<|open|>message role="assistant"<|sep|><|open|>think<|sep|> <- generation prompt+```++The assistant's thinking is a *structural* channel (preserved across turns in+multi-turn use); `K3_THINK=0` opens `<response>` directly instead. The C+builder mirrors the reference segment-for-segment (segment boundaries are+token boundaries) and was verified **token-exact** against `encoding_k3.py` ++tiktoken for system/no-system and thinking/non-thinking prompts. At decode+the CLI hides the XTML structure and prints `[think]` / `[response]` channel+banners; generation stops at `<|end_of_msg|>` (the eos).+## Current limitations- Decode is single-token (no speculative decoding β K3 has no MTP head).+- `--chat` renders a single system+user turn (no multi-turn CLI loop, no+ tool-call rendering β the format supports both; see `encoding_k3.py`).- CPU only (no CUDA/Metal/Vulkan tier), scalar KDA inner loops.- No chat template handling; prompts are raw text after `[BOS]`.