Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

kimi_k3: Kimi K3 engine (KDA + gated NoPE MLA + AttnRes + LatentMoE)

Steve Markgraf7fbc5cf0fc95
diff --git a/.gitignore b/.gitignore
index 7ac3789d..2e765b2a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,6 +16,8 @@ c/glm
c/glm.exe
c/olmoe
c/inkling
+c/kimi_k3
+c/kimi_k3.exe
c/olmoe.exe
c/iobench
c/iobench.exe
diff --git a/c/kimi_k3.c b/c/kimi_k3.c
new file mode 100644
index 00000000..efb5a043
--- /dev/null
+++ b/c/kimi_k3.c
@@ -0,0 +1,859 @@
+/* Kimi K3 inference engine in pure C — sibling of colibri.c (GLM-5.2) / olmoe.c /
+ * inkling.c, sharing st.h / json.h / tok.h / quant.h.
+ *
+ * Architecture (2.8T total / 104B active, 93 layers, hidden 7168):
+ * - Hybrid attention: 69 KDA (Kimi Delta Attention, linear/recurrent) +
+ * 24 gated MLA layers (every 4th, plus the final layer). MLA is NoPE —
+ * no positional encoding anywhere in the model; position lives in KDA's
+ * decay/conv. Both attention types carry a full-rank sigmoid output gate.
+ * - AttnRes (Attention Residuals) REPLACE the plain residual stream: a
+ * running prefix_sum plus block snapshots (layers 0,12,24,...,84), mixed
+ * by softmax twice per layer (before attention, before MLP) and once at
+ * the end. Weights: per-layer {self_attention,mlp}_res_{norm,proj}.
+ * - Stable LatentMoE: router (sigmoid + e_score_correction_bias, top-16 of
+ * 896, renormalized raw scores), shared latent down/up projections
+ * (7168<->3584), per-expert GLU in the 3584 latent with moe_inter 3072,
+ * RMSNorm on the aggregate, plus 2 fused shared experts (inter 6144) at
+ * full width. Activation is SiTU-GLU:
+ * b1*tanh(g/b1)*sigmoid(g) * b2*tanh(u/b2), b1=4, b2=25.
+ * - Routed experts are NATIVE MXFP4 (QAT; e2m1 nibbles + ue8m0 scale per
+ * 32, compressed-tensors "mxfp4-pack-quantized") and are streamed
+ * straight from the original HF shards — never re-encoded, never
+ * converted. Everything else is BF16 in the checkpoint and quantized at
+ * LOAD TIME into RAM (int8 per-row / int4-g64 / f32, see K3_*BITS).
+ *
+ * KDA per-head recurrence (head dim 128, 96 heads; fla fused_recurrent_kda):
+ * q,k,v = SiLU(ShortConv4(W{q,k,v} x)); q,k L2-normalized (eps 1e-6
+ * inside the sqrt), q *= 128^-0.5
+ * z = W_fb(W_fa x) + dt_bias (per channel, dt_bias[12288])
+ * gk = gmin * sigmoid(exp(A_log[h]) * z), gmin=-5; alpha = exp(gk)
+ * S = (I - beta k k^T) Diag(alpha) S + beta k v^T, beta = sigmoid(W_b x)
+ * o = S^T q; out = W_o [ sigmoid(W_g x) * RMSNorm_head(o) ]
+ * (checkpoint A_log is [128] = per-head [96] zero-padded, first 96 used)
+ *
+ * Model dir = the HF snapshot (config.json + model-*-of-000096.safetensors).
+ * tokenizer.json is synthesized once by tools/k3_tokenizer.py (the HF repo
+ * ships only tiktoken.model); without it the engine still runs on raw ids.
+ *
+ * ENV:
+ * K3_BITS=4|8|32 load-time quant of KDA/latent/shared/dense (default 4)
+ * K3_MLA_BITS=8|4|32 MLA projections (default 8)
+ * K3_HEAD_BITS=8|4|32 lm_head (default 8)
+ * K3_EXPERT_GB=N routed-expert LRU cache budget (default 8)
+ * K3_LAYERS=N truncate to first N layers (validation; skips head)
+ * K3_TRACE=path dump f32 hidden state after every layer (validation)
+ * K3_MAXT=N KV/context capacity (default prompt+ngen)
+ * COLI_TEMP=F 0 = greedy (default), else softmax temperature
+ */
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <time.h>
+#if defined(__APPLE__) || defined(__linux__) || defined(__FreeBSD__)
+#include <sys/resource.h>
+#include <unistd.h>
+#endif
+#ifdef _OPENMP
+#include <omp.h>
+#endif
+#include "st.h"
+#include "tok.h"
+#include "quant.h"
+
+/* ---------- config ---------- */
+typedef struct {
+ int hidden, n_layers, vocab, first_dense, dense_inter;
+ /* MLA */
+ int n_heads, q_lora, kv_lora, qk_nope, qk_rope, qk_head, v_head;
+ float attn_scale;
+ /* KDA */
+ int kda_heads, kda_hd, kda_proj, conv_k;
+ float gate_lb;
+ /* MoE */
+ int n_experts, topk, moe_inter, latent, n_shared;
+ float situ_b1, situ_b2;
+ /* AttnRes */
+ int res_bs;
+ float eps;
+ int8_t is_kda[128];
+ int bos, eos[8], n_eos;
+} Cfg;
+
+/* ---------- RAM-resident weight, quantized at load ---------- */
+typedef struct { int fmt; float *f; int8_t *q8; uint8_t *q4; float *s; int O, I, gs; } W;
+
+typedef struct { /* KDA layer */
+ W q, k, v, o, g;
+ float *conv_q, *conv_k, *conv_v; /* [proj*4] depthwise taps, oldest first */
+ float *fa, *fb; /* decay low-rank, f32 [hd,hidden] [proj,hd] */
+ float *bp; /* beta proj f32 [heads,hidden] */
+ float *dt, *A, *onw; /* dt_bias[proj], exp(A_log)[heads], o_norm[hd] */
+} Kda;
+
+typedef struct { /* gated MLA layer */
+ W qa, qb, kva, kvb, o, g;
+ float *qa_ln, *kva_ln;
+} Mla;
+
+typedef struct { /* LatentMoE */
+ float *router, *rbias, *lat_norm; /* [E,hidden] f32, [E], [latent] */
+ W lat_down, lat_up, sh_gate, sh_up, sh_down;
+} Moe;
+
+typedef struct {
+ int kda, sparse;
+ Kda a; Mla m; Moe moe;
+ W d_gate, d_up, d_down; /* dense layer only */
+ float *in_ln, *post_ln;
+ float *attn_sw, *mlp_sw; /* AttnRes score weights: norm.w * proj.w */
+} Layer;
+
+/* ---------- routed-expert streaming (native MXFP4 from the HF shards) ---- */
+typedef struct { int fd[6]; int64_t off[6]; int contig; } ERef; /* w1p w1s w2p w2s w3p w3s */
+typedef struct { int eid; uint8_t *buf; uint64_t used; } Slot;
+typedef struct { Slot *s; int n, cap; } LCache;
+
+typedef struct {
+ Cfg c;
+ shards S;
+ char pfx[40]; /* "language_model." or "" */
+ Layer *L;
+ float *final_norm, *out_sw;
+ W lm_head;
+ int has_head;
+ /* KDA state */
+ float **kstate; /* [layer] -> [heads*hd*hd], S[k][v] */
+ float **cwq, **cwk, **cwv; /* conv windows [proj*conv_k], oldest first */
+ /* MLA cache */
+ float **Lc, **Rc; int max_t;
+ /* experts */
+ ERef *eref; /* [n_layers][n_experts] (dense rows zeroed) */
+ LCache *ecache;
+ int64_t e_w1p, e_w1s, e_w2p, e_w2s, e_slot;
+ uint64_t clock, hits, miss, ebytes;
+ double t_attn, t_moe, t_eload, t_head;
+ FILE *trace;
+} Model;
+
+static double now_s(void){ struct timespec t; clock_gettime(CLOCK_MONOTONIC,&t); return t.tv_sec+t.tv_nsec*1e-9; }
+static double rss_gb(void){ struct rusage r; getrusage(RUSAGE_SELF,&r);
+#if defined(__APPLE__)
+ return r.ru_maxrss/(1024.0*1024.0*1024.0);
+#else
+ return r.ru_maxrss/(1024.0*1024.0);
+#endif
+}
+static float *falloc(int64_t n){ float *p=malloc((size_t)n*sizeof(float)); if(!p){fprintf(stderr,"OOM %lld floats\n",(long long)n);exit(1);} return p; }
+static float *fcalloc(int64_t n){ float *p=calloc((size_t)n,sizeof(float)); if(!p){fprintf(stderr,"OOM %lld floats\n",(long long)n);exit(1);} return p; }
+static inline float sigmoidf_(float x){ return 1.f/(1.f+expf(-x)); }
+static inline float siluf_(float x){ return x/(1.f+expf(-x)); }
+static void softmax_(float *x, int n){ float m=x[0]; for(int i=1;i<n;i++) if(x[i]>m)m=x[i];
+ float s=0; for(int i=0;i<n;i++){ x[i]=expf(x[i]-m); s+=x[i]; } for(int i=0;i<n;i++) x[i]/=s; }
+static void rmsnorm_(float *out, const float *x, const float *w, int D, float eps){
+ double ms=0; for(int i=0;i<D;i++) ms+=(double)x[i]*x[i];
+ float r=1.f/sqrtf((float)(ms/D)+eps);
+ for(int i=0;i<D;i++) out[i]=x[i]*r*w[i];
+}
+
+/* ---------- W: load-time quantization + matvec ---------- */
+static void w_matmul(float *y, const float *x, const W *w, int S){
+ if(w->fmt==0) matmul(y,x,w->f,S,w->I,w->O);
+ else if(w->fmt==1) matmul_q(y,x,w->q8,w->s,S,w->I,w->O);
+ else if(w->fmt==4) matmul_i4_grouped(y,x,w->q4,w->s,S,w->I,w->O,w->gs);
+ else { fprintf(stderr,"w_matmul: bad fmt %d\n",w->fmt); exit(1); }
+}
+/* acc[0..I) += coef * row r (MLA absorb builds q_abs from kv_b rows) */
+static void w_addrow(const W *w, int r, float coef, float *acc){
+ int I=w->I;
+ if(w->fmt==0){ const float *p=w->f+(int64_t)r*I; for(int i=0;i<I;i++) acc[i]+=coef*p[i]; }
+ else if(w->fmt==1){ const int8_t *p=w->q8+(int64_t)r*I; float s=w->s[r]*coef;
+ for(int i=0;i<I;i++) acc[i]+=s*p[i]; }
+ else { int rb=(I+1)/2, ng=(I+w->gs-1)/w->gs; const uint8_t *p=w->q4+(int64_t)r*rb;
+ const float *scl=w->s+(int64_t)r*ng;
+ for(int g=0;g*w->gs<I;g++){ float s=scl[g]*coef; int e=(g+1)*w->gs; if(e>I)e=I;
+ for(int i=g*w->gs;i<e;i+=2){ uint8_t b=p[i>>1];
+ acc[i]+=s*(float)((int)(b&0xF)-8);
+ if(i+1<e) acc[i+1]+=s*(float)((int)(b>>4)-8); } } }
+}
+static float w_rowdot(const W *w, int r, const float *x){
+ int I=w->I; float a=0;
+ if(w->fmt==0){ const float *p=w->f+(int64_t)r*I; for(int i=0;i<I;i++) a+=x[i]*p[i]; return a; }
+ if(w->fmt==1){ const int8_t *p=w->q8+(int64_t)r*I; for(int i=0;i<I;i++) a+=x[i]*p[i]; return a*w->s[r]; }
+ { int rb=(I+1)/2, ng=(I+w->gs-1)/w->gs; const uint8_t *p=w->q4+(int64_t)r*rb;
+ const float *scl=w->s+(int64_t)r*ng;
+ for(int g=0;g*w->gs<I;g++){ float ga=0; int e=(g+1)*w->gs; if(e>I)e=I;
+ for(int i=g*w->gs;i<e;i+=2){ uint8_t b=p[i>>1];
+ ga+=x[i]*(float)((int)(b&0xF)-8);
+ if(i+1<e) ga+=x[i+1]*(float)((int)(b>>4)-8); }
+ a+=ga*scl[g]; } }
+ return a;
+}
+
+#define QCHUNK 1024 /* rows per load-quantize pass */
+static void w_load(Model *m, W *w, const char *name, int O, int I, int bits){
+ char nm[512]; snprintf(nm,sizeof(nm),"%s%s",m->pfx,name);
+ st_tensor *t=st_find(&m->S,nm);
+ if(!t) st_die_missing(&m->S,nm);
+ memset(w,0,sizeof(*w)); w->O=O; w->I=I;
+ if(t->dtype==3){
+ /* repacked container (tools/k3_repack.py): pre-quantized U8 + .qs f32
+ * scales — no load-time quantization, K3_BITS is ignored for these */
+ char qn[560]; snprintf(qn,sizeof(qn),"%s.qs",nm);
+ st_tensor *ts=st_find(&m->S,qn);
+ if(!ts){ fprintf(stderr,"%s: quantized (U8) but no %s scale sidecar\n",nm,qn); exit(1); }
+ if(t->nbytes==(int64_t)O*I && ts->numel==O){ /* int8 per-row */
+ w->fmt=1; w->q8=malloc((size_t)O*I);
+ if(!w->q8){fprintf(stderr,"OOM int8 %s\n",nm);exit(1);}
+ st_read_raw(&m->S,nm,w->q8,1);
+ w->s=falloc(O); st_read_f32(&m->S,qn,w->s,0);
+ } else if(I%64==0 && t->nbytes==(int64_t)O*(I/2) && ts->numel==(int64_t)O*(I/64)){
+ w->fmt=4; w->gs=64; /* int4-g64 */
+ w->q4=malloc((size_t)O*(I/2));
+ if(!w->q4){fprintf(stderr,"OOM int4 %s\n",nm);exit(1);}
+ st_read_raw(&m->S,nm,w->q4,1);
+ w->s=falloc((int64_t)O*(I/64)); st_read_f32(&m->S,qn,w->s,0);
+ } else {
+ fprintf(stderr,"%s: U8 tensor is %lld bytes / %lld scales — matches neither int8 [%d,%d] nor int4-g64, refusing (untrusted container)\n",
+ nm,(long long)t->nbytes,(long long)ts->numel,O,I); exit(1);
+ }
+ return;
+ }
+ if(t->numel!=(int64_t)O*I){ fprintf(stderr,"%s: numel %lld != %dx%d\n",nm,(long long)t->numel,O,I); exit(1); }
+ if(bits>=32){ w->fmt=0; w->f=falloc((int64_t)O*I); st_read_f32(&m->S,nm,w->f,0); return; }
+ int gs=64;
+ if(bits<=4 && I%gs){ bits=8; } /* int4-g64 wants I%64==0; fall back */
+ float *scr=falloc((int64_t)QCHUNK*I);
+ if(bits>4){ w->fmt=1; w->q8=malloc((int64_t)O*I); w->s=falloc(O);
+ if(!w->q8){fprintf(stderr,"OOM int8 %s\n",nm);exit(1);}
+ for(int r0=0;r0<O;r0+=QCHUNK){ int n=O-r0<QCHUNK?O-r0:QCHUNK;
+ st_read_slice_f32(&m->S,nm,(int64_t)r0*I,(int64_t)n*I,scr,1);
+ for(int r=0;r<n;r++){ const float *src=scr+(int64_t)r*I;
+ float am=0; for(int i=0;i<I;i++){ float a=fabsf(src[i]); if(a>am)am=a; }
+ float s=am/127.f; if(s<1e-20f)s=1e-20f; w->s[r0+r]=s; float inv=1.f/s;
+ int8_t *dst=w->q8+(int64_t)(r0+r)*I;
+ for(int i=0;i<I;i++){ int v=(int)lrintf(src[i]*inv); if(v>127)v=127; if(v<-127)v=-127; dst[i]=(int8_t)v; } } }
+ } else { w->fmt=4; w->gs=gs; int rb=I/2, ng=I/gs;
+ w->q4=malloc((int64_t)O*rb); w->s=falloc((int64_t)O*ng);
+ if(!w->q4){fprintf(stderr,"OOM int4 %s\n",nm);exit(1);}
+ for(int r0=0;r0<O;r0+=QCHUNK){ int n=O-r0<QCHUNK?O-r0:QCHUNK;
+ st_read_slice_f32(&m->S,nm,(int64_t)r0*I,(int64_t)n*I,scr,1);
+ for(int r=0;r<n;r++){ const float *src=scr+(int64_t)r*I;
+ uint8_t *dst=w->q4+(int64_t)(r0+r)*rb; float *scl=w->s+(int64_t)(r0+r)*ng;
+ for(int g=0;g<ng;g++){ const float *gp=src+g*gs;
+ float am=0; for(int i=0;i<gs;i++){ float a=fabsf(gp[i]); if(a>am)am=a; }
+ float s=am/7.f; if(s<1e-20f)s=1e-20f; scl[g]=s; float inv=1.f/s;
+ for(int i=0;i<gs;i+=2){
+ int v0=(int)lrintf(gp[i]*inv); if(v0>7)v0=7; if(v0<-8)v0=-8;
+ int v1=(int)lrintf(gp[i+1]*inv); if(v1>7)v1=7; if(v1<-8)v1=-8;
+ dst[(g*gs+i)>>1]=(uint8_t)((v0+8)|((v1+8)<<4)); } } } }
+ }
+ free(scr);
+}
+static float *f32_load(Model *m, const char *name, int64_t want){
+ char nm[512]; snprintf(nm,sizeof(nm),"%s%s",m->pfx,name);
+ st_tensor *t=st_find(&m->S,nm);
+ if(!t) st_die_missing(&m->S,nm);
+ if(want>0 && t->numel!=want){ fprintf(stderr,"%s: numel %lld != %lld\n",nm,(long long)t->numel,(long long)want); exit(1); }
+ float *p=falloc(t->numel); st_read_f32(&m->S,nm,p,0); return p;
+}
+
+/* ---------- config ---------- */
+static double req_num(jval *r, const char *k){
+ jval *v=json_get(r,k);
+ if(!v||v->t!=J_NUM){ fprintf(stderr,"config.json: missing or non-numeric \"%s\"\n",k); exit(1); }
+ return v->num;
+}
+static void load_cfg(Cfg *c, const char *snap){
+ char path[2048]; snprintf(path,sizeof(path),"%s/config.json",snap);
+ long n; char *buf;
+ { FILE *f=fopen(path,"rb"); if(!f){perror(path);exit(1);}
+ fseek(f,0,SEEK_END); n=ftell(f); fseek(f,0,SEEK_SET);
+ if(n<0||n>(64L<<20)){ fprintf(stderr,"%s: bad size\n",path); exit(1); }
+ buf=malloc((size_t)n+1); if(!buf){fprintf(stderr,"OOM cfg\n");exit(1);}
+ if(fread(buf,1,(size_t)n,f)!=(size_t)n){ fprintf(stderr,"%s: short read\n",path); exit(1); }
+ buf[n]=0; fclose(f); }
+ char *arena=NULL; jval *root=json_parse(buf,&arena);
+ jval *tc=json_get(root,"text_config"); if(!tc||tc->t!=J_OBJ) tc=root;
+ memset(c,0,sizeof(*c));
+ c->hidden =(int)req_num(tc,"hidden_size");
+ c->n_layers =(int)req_num(tc,"num_hidden_layers");
+ c->vocab =(int)req_num(tc,"vocab_size");
+ c->first_dense =(int)req_num(tc,"first_k_dense_replace");
+ c->dense_inter =(int)req_num(tc,"intermediate_size");
+ c->n_heads =(int)req_num(tc,"num_attention_heads");
+ c->q_lora =(int)req_num(tc,"q_lora_rank");
+ c->kv_lora =(int)req_num(tc,"kv_lora_rank");
+ c->qk_nope =(int)req_num(tc,"qk_nope_head_dim");
+ c->qk_rope =(int)req_num(tc,"qk_rope_head_dim");
+ c->v_head =(int)req_num(tc,"v_head_dim");
+ c->n_experts =(int)req_num(tc,"num_experts");
+ c->topk =(int)req_num(tc,"num_experts_per_token");
+ c->moe_inter =(int)req_num(tc,"moe_intermediate_size");
+ c->latent =(int)req_num(tc,"routed_expert_hidden_size");
+ c->n_shared =(int)req_num(tc,"num_shared_experts");
+ c->res_bs =(int)req_num(tc,"attn_res_block_size");
+ c->situ_b1 =(float)req_num(tc,"activation_situ_beta");
+ c->situ_b2 =(float)req_num(tc,"activation_situ_linear_beta");
+ jval *ep=json_get(tc,"rms_norm_eps"); c->eps=ep?(float)ep->num:1e-5f;
+ c->qk_head=c->qk_nope+c->qk_rope;
+ c->attn_scale=1.f/sqrtf((float)c->qk_head);
+ jval *la=json_get(tc,"linear_attn_config");
+ if(!la||la->t!=J_OBJ){ fprintf(stderr,"config.json: missing linear_attn_config\n"); exit(1); }
+ c->kda_heads=(int)req_num(la,"num_heads");
+ c->kda_hd =(int)req_num(la,"head_dim");
+ c->conv_k =(int)req_num(la,"short_conv_kernel_size");
+ jval *lb=json_get(la,"gate_lower_bound"); c->gate_lb=lb?(float)lb->num:-5.f;
+ c->kda_proj=c->kda_heads*c->kda_hd;
+ if(c->hidden<1||c->hidden>65536||c->n_layers<1||c->n_layers>128||
+ c->n_experts<1||c->n_experts>4096||c->topk<1||c->topk>64||c->topk>c->n_experts||
+ c->vocab<1||c->vocab>(1<<22)||c->kda_proj<1||c->kda_proj>(1<<20)||
+ c->conv_k<1||c->conv_k>8||c->latent<32||c->latent%32||c->moe_inter%32||
+ c->res_bs<1||c->kda_hd>512||c->kv_lora>4096){
+ fprintf(stderr,"config.json: dimension out of range\n"); exit(1); }
+ jval *kl=json_get(la,"kda_layers");
+ if(!kl||kl->t!=J_ARR){ fprintf(stderr,"config.json: missing kda_layers\n"); exit(1); }
+ for(int i=0;i<kl->len;i++){ int v=(int)kl->kids[i]->num; /* 1-indexed */
+ if(v>=1&&v<=c->n_layers) c->is_kda[v-1]=1; }
+ jval *b=json_get(root,"bos_token_id"); if(!b) b=json_get(tc,"bos_token_id");
+ c->bos = b&&b->t==J_NUM ? (int)b->num : -1;
+ jval *e=json_get(root,"eos_token_id"); if(!e) e=json_get(tc,"eos_token_id");
+ if(e&&e->t==J_NUM) c->eos[c->n_eos++]=(int)e->num;
+ else if(e&&e->t==J_ARR) for(int i=0;i<e->len&&c->n_eos<8;i++) c->eos[c->n_eos++]=(int)e->kids[i]->num;
+ free(buf); (void)arena;
+}
+
+/* ---------- init ---------- */
+static void expert_table_init(Model *m){
+ Cfg *c=&m->c;
+ m->e_w1p=(int64_t)c->moe_inter*(c->latent/2); m->e_w1s=(int64_t)c->moe_inter*(c->latent/32);
+ m->e_w2p=(int64_t)c->latent*(c->moe_inter/2); m->e_w2s=(int64_t)c->latent*(c->moe_inter/32);
+ m->e_slot=2*(m->e_w1p+m->e_w1s)+m->e_w2p+m->e_w2s;
+ m->eref=calloc((size_t)c->n_layers*c->n_experts,sizeof(ERef));
+ if(!m->eref){fprintf(stderr,"OOM expert table\n");exit(1);}
+ const char *mat[3]={"w1","w2","w3"};
+ int64_t want[6]={m->e_w1p,m->e_w1s,m->e_w2p,m->e_w2s,m->e_w1p,m->e_w1s};
+ int missing=0;
+ for(int li=0;li<c->n_layers;li++){
+ if(!m->L[li].sparse) continue;
+ for(int e2=0;e2<c->n_experts;e2++){
+ ERef *er=&m->eref[(int64_t)li*c->n_experts+e2];
+ for(int k=0;k<6;k++){
+ char nm[512];
+ snprintf(nm,sizeof(nm),"%smodel.layers.%d.block_sparse_moe.experts.%d.%s.weight_%s",
+ m->pfx,li,e2,mat[k/2],(k&1)?"scale":"packed");
+ st_tensor *t=st_find(&m->S,nm);
+ if(!t){ missing++; er->fd[k]=-1; continue; }
+ if(t->nbytes!=want[k]){ fprintf(stderr,"%s: %lld bytes, expected %lld — refusing (untrusted container)\n",
+ nm,(long long)t->nbytes,(long long)want[k]); exit(1); }
+ er->fd[k]=t->fd; er->off[k]=t->off;
+ }
+ /* HF shards store the six tensors back-to-back (measured: 0 gaps
+ * across a whole layer) — collapse the load to ONE pread when so */
+ er->contig=1;
+ for(int k=0;k<5;k++)
+ if(er->fd[k]!=er->fd[k+1]||er->off[k]+want[k]!=er->off[k+1]) er->contig=0;
+ }
+ }
+ if(missing) fprintf(stderr,"[K3] WARNING: %d expert tensors missing (incomplete download?) — touching one aborts\n",missing);
+}
+
+static void model_init(Model *m, const char *snap, int n_layers_env){
+ memset(m,0,sizeof(*m));
+ load_cfg(&m->c,snap);
+ Cfg *c=&m->c;
+ if(n_layers_env>0&&n_layers_env<c->n_layers) c->n_layers=n_layers_env;
+ st_init_multi(&m->S,snap,getenv("K3_DIRS")); /* K3_DIRS: extra shard dirs (multi-drive split) */
+ m->pfx[0]=0; /* probe a layer-0 tensor: embed/head live in one of the LAST shards */
+ if(!st_has(&m->S,"model.layers.0.input_layernorm.weight")&&
+ st_has(&m->S,"language_model.model.layers.0.input_layernorm.weight"))
+ snprintf(m->pfx,sizeof(m->pfx),"language_model.");
+ if((c->n_layers+c->res_bs-1)/c->res_bs+1>16){ fprintf(stderr,"attn_res: too many blocks\n"); exit(1); }
+ int bits = getenv("K3_BITS")?atoi(getenv("K3_BITS")):4;
+ int mbits = getenv("K3_MLA_BITS")?atoi(getenv("K3_MLA_BITS")):8;
+ int hbits = getenv("K3_HEAD_BITS")?atoi(getenv("K3_HEAD_BITS")):8;
+ double t0=now_s();
+ m->L=calloc(c->n_layers,sizeof(Layer));
+ m->kstate=calloc(c->n_layers,sizeof(float*));
+ m->cwq=calloc(c->n_layers,sizeof(float*));
+ m->cwk=calloc(c->n_layers,sizeof(float*));
+ m->cwv=calloc(c->n_layers,sizeof(float*));
+ char nm[512];
+ #define NM(...) (snprintf(nm,sizeof(nm),__VA_ARGS__),nm)
+ for(int i=0;i<c->n_layers;i++){
+ Layer *l=&m->L[i];
+ l->kda=c->is_kda[i];
+ l->sparse=(i>=c->first_dense);
+ l->in_ln =f32_load(m,NM("model.layers.%d.input_layernorm.weight",i),c->hidden);
+ l->post_ln=f32_load(m,NM("model.layers.%d.post_attention_layernorm.weight",i),c->hidden);
+ /* AttnRes score weight = res_norm.weight * res_proj.weight (elementwise) */
+ { float *rn=f32_load(m,NM("model.layers.%d.self_attention_res_norm.weight",i),c->hidden);
+ float *rp=f32_load(m,NM("model.layers.%d.self_attention_res_proj.weight",i),c->hidden);
+ l->attn_sw=falloc(c->hidden); for(int d=0;d<c->hidden;d++) l->attn_sw[d]=rn[d]*rp[d];
+ free(rn); free(rp);
+ rn=f32_load(m,NM("model.layers.%d.mlp_res_norm.weight",i),c->hidden);
+ rp=f32_load(m,NM("model.layers.%d.mlp_res_proj.weight",i),c->hidden);
+ l->mlp_sw=falloc(c->hidden); for(int d=0;d<c->hidden;d++) l->mlp_sw[d]=rn[d]*rp[d];
+ free(rn); free(rp); }
+ if(l->kda){
+ Kda *a=&l->a; int P=c->kda_proj;
+ w_load(m,&a->q,NM("model.layers.%d.self_attn.q_proj.weight",i),P,c->hidden,bits);
+ w_load(m,&a->k,NM("model.layers.%d.self_attn.k_proj.weight",i),P,c->hidden,bits);
+ w_load(m,&a->v,NM("model.layers.%d.self_attn.v_proj.weight",i),P,c->hidden,bits);
+ w_load(m,&a->g,NM("model.layers.%d.self_attn.g_proj.weight",i),P,c->hidden,bits);
+ w_load(m,&a->o,NM("model.layers.%d.self_attn.o_proj.weight",i),c->hidden,P,bits);
+ a->conv_q=f32_load(m,NM("model.layers.%d.self_attn.q_conv1d.weight",i),(int64_t)P*c->conv_k);
+ a->conv_k=f32_load(m,NM("model.layers.%d.self_attn.k_conv1d.weight",i),(int64_t)P*c->conv_k);
+ a->conv_v=f32_load(m,NM("model.layers.%d.self_attn.v_conv1d.weight",i),(int64_t)P*c->conv_k);
+ a->fa=f32_load(m,NM("model.layers.%d.self_attn.f_a_proj.weight",i),(int64_t)c->kda_hd*c->hidden);
+ a->fb=f32_load(m,NM("model.layers.%d.self_attn.f_b_proj.weight",i),(int64_t)P*c->kda_hd);
+ a->bp=f32_load(m,NM("model.layers.%d.self_attn.b_proj.weight",i),(int64_t)c->kda_heads*c->hidden);
+ a->dt=f32_load(m,NM("model.layers.%d.self_attn.dt_bias",i),P);
+ a->onw=f32_load(m,NM("model.layers.%d.self_attn.o_norm.weight",i),c->kda_hd);
+ { /* A_log in the checkpoint is [kda_hd] = per-head zero-padded */
+ char an[512]; snprintf(an,sizeof(an),"%smodel.layers.%d.self_attn.A_log",m->pfx,i);
+ st_tensor *t=st_find(&m->S,an); if(!t) st_die_missing(&m->S,an);
+ if(t->numel<c->kda_heads){ fprintf(stderr,"%s: %lld < heads\n",an,(long long)t->numel); exit(1); }
+ float *al=falloc(t->numel); st_read_f32(&m->S,an,al,0);
+ a->A=falloc(c->kda_heads);
+ for(int h=0;h<c->kda_heads;h++) a->A[h]=expf(al[h]);
+ free(al); }
+ m->kstate[i]=fcalloc((int64_t)c->kda_heads*c->kda_hd*c->kda_hd);
+ m->cwq[i]=fcalloc((int64_t)P*c->conv_k);
+ m->cwk[i]=fcalloc((int64_t)P*c->conv_k);
+ m->cwv[i]=fcalloc((int64_t)P*c->conv_k);
+ } else {
+ Mla *a=&l->m;
+ w_load(m,&a->qa,NM("model.layers.%d.self_attn.q_a_proj.weight",i),c->q_lora,c->hidden,mbits);
+ w_load(m,&a->qb,NM("model.layers.%d.self_attn.q_b_proj.weight",i),c->n_heads*c->qk_head,c->q_lora,mbits);
+ w_load(m,&a->kva,NM("model.layers.%d.self_attn.kv_a_proj_with_mqa.weight",i),c->kv_lora+c->qk_rope,c->hidden,mbits);
+ w_load(m,&a->kvb,NM("model.layers.%d.self_attn.kv_b_proj.weight",i),c->n_heads*(c->qk_nope+c->v_head),c->kv_lora,mbits);
+ w_load(m,&a->o,NM("model.layers.%d.self_attn.o_proj.weight",i),c->hidden,c->n_heads*c->v_head,mbits);
+ w_load(m,&a->g,NM("model.layers.%d.self_attn.g_proj.weight",i),c->n_heads*c->v_head,c->hidden,mbits);
+ a->qa_ln =f32_load(m,NM("model.layers.%d.self_attn.q_a_layernorm.weight",i),c->q_lora);
+ a->kva_ln=f32_load(m,NM("model.layers.%d.self_attn.kv_a_layernorm.weight",i),c->kv_lora);
+ }
+ if(l->sparse){
+ Moe *o=&l->moe;
+ o->router=f32_load(m,NM("model.layers.%d.block_sparse_moe.gate.weight",i),(int64_t)c->n_experts*c->hidden);
+ o->rbias =f32_load(m,NM("model.layers.%d.block_sparse_moe.gate.e_score_correction_bias",i),c->n_experts);
+ o->lat_norm=f32_load(m,NM("model.layers.%d.block_sparse_moe.routed_expert_norm.weight",i),c->latent);
+ w_load(m,&o->lat_down,NM("model.layers.%d.block_sparse_moe.routed_expert_down_proj.weight",i),c->latent,c->hidden,bits);
+ w_load(m,&o->lat_up,NM("model.layers.%d.block_sparse_moe.routed_expert_up_proj.weight",i),c->hidden,c->latent,bits);
+ int shi=c->moe_inter*c->n_shared;
+ w_load(m,&o->sh_gate,NM("model.layers.%d.block_sparse_moe.shared_experts.gate_proj.weight",i),shi,c->hidden,bits);
+ w_load(m,&o->sh_up,NM("model.layers.%d.block_sparse_moe.shared_experts.up_proj.weight",i),shi,c->hidden,bits);
+ w_load(m,&o->sh_down,NM("model.layers.%d.block_sparse_moe.shared_experts.down_proj.weight",i),c->hidden,shi,bits);
+ } else {
+ w_load(m,&l->d_gate,NM("model.layers.%d.mlp.gate_proj.weight",i),c->dense_inter,c->hidden,bits);
+ w_load(m,&l->d_up,NM("model.layers.%d.mlp.up_proj.weight",i),c->dense_inter,c->hidden,bits);
+ w_load(m,&l->d_down,NM("model.layers.%d.mlp.down_proj.weight",i),c->hidden,c->dense_inter,bits);
+ }
+ if(i%8==0) fprintf(stderr,"[K3] loaded layer %d/%d (%.1fs, RSS %.1f GB)\n",i+1,c->n_layers,now_s()-t0,rss_gb());
+ }
+ snprintf(nm,sizeof(nm),"%smodel.norm.weight",m->pfx);
+ m->has_head = st_has(&m->S,nm);
+ if(m->has_head){
+ m->final_norm=f32_load(m,"model.norm.weight",c->hidden);
+ { float *rn=f32_load(m,"model.output_attn_res_norm.weight",c->hidden);
+ float *rp=f32_load(m,"model.output_attn_res_proj.weight",c->hidden);
+ m->out_sw=falloc(c->hidden); for(int d=0;d<c->hidden;d++) m->out_sw[d]=rn[d]*rp[d];
+ free(rn); free(rp); }
+ w_load(m,&m->lm_head,"lm_head.weight",c->vocab,c->hidden,hbits);
+ } else fprintf(stderr,"[K3] final norm/lm_head not present — trace-only mode\n");
+ expert_table_init(m);
+ /* expert LRU cache, per-layer slots from the global budget */
+ double egb = getenv("K3_EXPERT_GB")?atof(getenv("K3_EXPERT_GB")):8.0;
+ int nmoe=0; for(int i=0;i<c->n_layers;i++) if(m->L[i].sparse) nmoe++;
+ int cap=(int)((egb*1e9)/((double)m->e_slot*(nmoe?nmoe:1)));
+ if(cap<c->topk) cap=c->topk;
+ m->ecache=calloc(c->n_layers,sizeof(LCache));
+ for(int i=0;i<c->n_layers;i++) if(m->L[i].sparse){
+ m->ecache[i].cap=cap; m->ecache[i].s=calloc(cap,sizeof(Slot));
+ for(int j2=0;j2<cap;j2++) m->ecache[i].s[j2].eid=-1;
+ }
+ fprintf(stderr,"[K3] init done in %.1fs | %d layers | expert cache %d/layer (%.1f MB/slot) | RSS %.1f GB\n",
+ now_s()-t0,c->n_layers,cap,m->e_slot/1e6,rss_gb());
+ #undef NM
+}
+
+/* ---------- AttnRes softmax mix over [bres rows..., prefix] ---------- */
+static void res_mix(float *out, const float *prefix, const float *bres, int nb, int D,
+ const float *sw, float eps){
+ const float *v[16]; float sc[16];
+ for(int e=0;e<nb;e++) v[e]=bres+(int64_t)e*D;
+ v[nb]=prefix;
+ for(int e=0;e<=nb;e++){
+ double ms=0,dot=0;
+ for(int d=0;d<D;d++){ double x=v[e][d]; ms+=x*x; dot+=x*(double)sw[d]; }
+ sc[e]=(float)(dot/sqrt(ms/D+eps));
+ }
+ softmax_(sc,nb+1);
+ for(int d=0;d<D;d++){ float a=0; for(int e=0;e<=nb;e++) a+=sc[e]*v[e][d]; out[d]=a; }
+}
+
+/* ---------- KDA layer (single token) ---------- */
+static void kda_forward(Model *m, Layer *l, int li, const float *x, float *out){
+ Cfg *c=&m->c; Kda *a=&l->a;
+ int P=c->kda_proj, H=c->kda_heads, hd=c->kda_hd, K=c->conv_k;
+ float *q=falloc(P), *k=falloc(P), *v=falloc(P), *gp=falloc(P), *on=falloc(P);
+ float *t1=falloc(c->kda_hd), *graw=falloc(P), *braw=falloc(H);
+ w_matmul(q,x,&a->q,1); w_matmul(k,x,&a->k,1); w_matmul(v,x,&a->v,1);
+ w_matmul(gp,x,&a->g,1);
+ matmul(t1,x,a->fa,1,c->hidden,c->kda_hd);
+ matmul(graw,t1,a->fb,1,c->kda_hd,P);
+ matmul(braw,x,a->bp,1,c->hidden,H);
+ /* depthwise causal conv (window: oldest..newest) + SiLU, window rolls forward */
+ float *wins[3]={m->cwq[li],m->cwk[li],m->cwv[li]};
+ float *vecs[3]={q,k,v}; float *taps[3]={a->conv_q,a->conv_k,a->conv_v};
+ for(int t=0;t<3;t++){
+ float *win=wins[t], *vec=vecs[t]; const float *cw=taps[t];
+ #pragma omp parallel for schedule(static)
+ for(int d=0;d<P;d++){
+ float *wd=win+(int64_t)d*K;
+ for(int j=0;j<K-1;j++) wd[j]=wd[j+1];
+ wd[K-1]=vec[d];
+ float acc=0; const float *cd=cw+(int64_t)d*K;
+ for(int j=0;j<K;j++) acc+=cd[j]*wd[j];
+ vec[d]=siluf_(acc);
+ }
+ }
+ float qscale=1.f/sqrtf((float)hd);
+ #pragma omp parallel for schedule(static)
+ for(int h=0;h<H;h++){
+ const float *qh=q+(int64_t)h*hd, *kh=k+(int64_t)h*hd, *vh=v+(int64_t)h*hd;
+ float qn[512], kn[512], alpha[512], kS[512], vt[512], oh[512];
+ float sq=0,sk=0;
+ for(int i=0;i<hd;i++){ sq+=qh[i]*qh[i]; sk+=kh[i]*kh[i]; }
+ sq=1.f/sqrtf(sq+1e-6f); sk=1.f/sqrtf(sk+1e-6f);
+ for(int i=0;i<hd;i++){ qn[i]=qh[i]*sq*qscale; kn[i]=kh[i]*sk; }
+ for(int i=0;i<hd;i++){
+ float z=graw[(int64_t)h*hd+i]+a->dt[(int64_t)h*hd+i];
+ alpha[i]=expf(c->gate_lb*sigmoidf_(a->A[h]*z));
+ }
+ float beta=sigmoidf_(braw[h]);
+ float *S=m->kstate[li]+(int64_t)h*hd*hd;
+ memset(kS,0,hd*sizeof(float));
+ for(int kk=0;kk<hd;kk++){
+ float *row=S+(int64_t)kk*hd; float al=alpha[kk], kv=kn[kk];
+ for(int vv=0;vv<hd;vv++){ row[vv]*=al; kS[vv]+=kv*row[vv]; }
+ }
+ for(int vv=0;vv<hd;vv++) vt[vv]=(vh[vv]-kS[vv])*beta;
+ memset(oh,0,hd*sizeof(float));
+ for(int kk=0;kk<hd;kk++){
+ float *row=S+(int64_t)kk*hd; float kv=kn[kk], qq=qn[kk];
+ for(int vv=0;vv<hd;vv++){ row[vv]+=kv*vt[vv]; oh[vv]+=qq*row[vv]; }
+ }
+ /* per-head RMSNorm * sigmoid(full-rank gate) */
+ double ms=0; for(int vv=0;vv<hd;vv++) ms+=(double)oh[vv]*oh[vv];
+ float r=1.f/sqrtf((float)(ms/hd)+c->eps);
+ float *dst=on+(int64_t)h*hd;
+ for(int vv=0;vv<hd;vv++) dst[vv]=oh[vv]*r*a->onw[vv]*sigmoidf_(gp[(int64_t)h*hd+vv]);
+ }
+ w_matmul(out,on,&a->o,1);
+ free(q);free(k);free(v);free(gp);free(on);free(t1);free(graw);free(braw);
+}
+
+/* ---------- gated MLA layer (single token, NoPE, absorb) ---------- */
+static void mla_forward(Model *m, Layer *l, int li, const float *x, int pos, float *out){
+ Cfg *c=&m->c; Mla *a=&l->m;
+ int H=c->n_heads, qh=c->qk_head, vh=c->v_head, kvl=c->kv_lora, qr=c->qk_rope;
+ float *qa=falloc(c->q_lora), *qv=falloc((int64_t)H*qh), *ckv=falloc(kvl+qr);
+ float *gv=falloc((int64_t)H*vh), *ctx=falloc((int64_t)H*vh);
+ w_matmul(qa,x,&a->qa,1);
+ rmsnorm_(qa,qa,a->qa_ln,c->q_lora,c->eps);
+ w_matmul(qv,qa,&a->qb,1);
+ w_matmul(ckv,x,&a->kva,1);
+ float *Lrow=m->Lc[li]+(int64_t)pos*kvl, *Rrow=m->Rc[li]+(int64_t)pos*qr;
+ rmsnorm_(Lrow,ckv,a->kva_ln,kvl,c->eps);
+ memcpy(Rrow,ckv+kvl,qr*sizeof(float)); /* NoPE: cached raw, no rotation */
+ w_matmul(gv,x,&a->g,1);
+ int nt=pos+1;
+ #pragma omp parallel for schedule(static)
+ for(int h=0;h<H;h++){
+ const float *qp=qv+(int64_t)h*qh, *qrp=qp+c->qk_nope;
+ int rbase=h*(c->qk_nope+vh);
+ float qabs[4096]; memset(qabs,0,kvl*sizeof(float));
+ for(int d=0;d<c->qk_nope;d++) w_addrow(&a->kvb,rbase+d,qp[d],qabs);
+ float *sc=falloc(nt);
+ for(int t=0;t<nt;t++){
+ const float *Lt=m->Lc[li]+(int64_t)t*kvl, *Rt=m->Rc[li]+(int64_t)t*qr;
+ float s2=0; for(int i=0;i<kvl;i++) s2+=qabs[i]*Lt[i];
+ for(int i=0;i<qr;i++) s2+=qrp[i]*Rt[i];
+ sc[t]=s2*c->attn_scale;
+ }
+ softmax_(sc,nt);
+ float clat[4096]; memset(clat,0,kvl*sizeof(float));
+ for(int t=0;t<nt;t++){
+ const float *Lt=m->Lc[li]+(int64_t)t*kvl; float s2=sc[t];
+ for(int i=0;i<kvl;i++) clat[i]+=s2*Lt[i];
+ }
+ free(sc);
+ float *cx=ctx+(int64_t)h*vh;
+ for(int d=0;d<vh;d++)
+ cx[d]=w_rowdot(&a->kvb,rbase+c->qk_nope+d,clat)*sigmoidf_(gv[(int64_t)h*vh+d]);
+ }
+ w_matmul(out,ctx,&a->o,1);
+ free(qa);free(qv);free(ckv);free(gv);free(ctx);
+}
+
+/* ---------- routed experts: LRU + direct pread from the HF shards ---------- */
+static Slot *expert_get(Model *m, int li, int eid){
+ LCache *lc=&m->ecache[li];
+ for(int i=0;i<lc->n;i++) if(lc->s[i].eid==eid){ m->hits++; lc->s[i].used=++m->clock; return &lc->s[i]; }
+ m->miss++;
+ Slot *s;
+ if(lc->n<lc->cap){ s=&lc->s[lc->n++]; }
+ else { int lru=0; for(int i=1;i<lc->n;i++) if(lc->s[i].used<lc->s[lru].used) lru=i; s=&lc->s[lru]; }
+ if(!s->buf){ s->buf=malloc(m->e_slot); if(!s->buf){fprintf(stderr,"OOM expert slot\n");exit(1);} }
+ ERef *er=&m->eref[(int64_t)li*m->c.n_experts+eid];
+ int64_t sizes[6]={m->e_w1p,m->e_w1s,m->e_w2p,m->e_w2s,m->e_w1p,m->e_w1s};
+ double t0=now_s();
+ if(er->fd[0]<0){ fprintf(stderr,"[K3] expert L%d E%d missing on disk\n",li,eid); exit(1); }
+ if(er->contig){
+ st_pread_full(er->fd[0],s->buf,m->e_slot,er->off[0],"pread expert");
+ } else {
+ uint8_t *dst=s->buf;
+ for(int k=0;k<6;k++){
+ if(er->fd[k]<0){ fprintf(stderr,"[K3] expert L%d E%d tensor %d missing on disk\n",li,eid,k); exit(1); }
+ st_pread_full(er->fd[k],dst,sizes[k],er->off[k],"pread expert");
+ dst+=sizes[k];
+ }
+ }
+ m->ebytes+=m->e_slot; m->t_eload+=now_s()-t0;
+ s->eid=eid; s->used=++m->clock;
+ return s;
+}
+
+static inline float situf_(float g, float u, float b1, float b2){
+ return b1*tanhf(g/b1)*sigmoidf_(g) * b2*tanhf(u/b2);
+}
+
+static void moe_forward(Model *m, Layer *l, int li, const float *x, float *out){
+ Cfg *c=&m->c; Moe *o=&l->moe;
+ int E=c->n_experts, K=c->topk, LT=c->latent, MI=c->moe_inter;
+ float *sco=falloc(E);
+ matmul(sco,x,o->router,1,c->hidden,E);
+ for(int e=0;e<E;e++) sco[e]=sigmoidf_(sco[e]);
+ int idx[64]; float wsel[64];
+ for(int kk=0;kk<K;kk++){
+ int best=-1; float bv=-1e30f;
+ for(int e=0;e<E;e++){
+ int taken=0; for(int j=0;j<kk;j++) if(idx[j]==e){taken=1;break;}
+ float sv=sco[e]+o->rbias[e];
+ if(!taken&&sv>bv){ bv=sv; best=e; }
+ }
+ idx[kk]=best; wsel[kk]=sco[best]; /* weight = RAW sigmoid score */
+ }
+ { float sm=0; for(int kk=0;kk<K;kk++) sm+=wsel[kk];
+ for(int kk=0;kk<K;kk++) wsel[kk]/=(sm+1e-20f); }
+ /* prefetch every selected expert, then load in DISK-OFFSET order (experts
+ * are NOT id-ordered inside the HF shards — measured 169/895 out-of-order) */
+ for(int kk=0;kk<K;kk++){
+ ERef *er=&m->eref[(int64_t)li*E+idx[kk]];
+ int64_t sizes[6]={m->e_w1p,m->e_w1s,m->e_w2p,m->e_w2s,m->e_w1p,m->e_w1s};
+ if(er->contig){ if(er->fd[0]>=0) posix_fadvise(er->fd[0],er->off[0],m->e_slot,POSIX_FADV_WILLNEED); }
+ else for(int k2=0;k2<6;k2++) if(er->fd[k2]>=0) posix_fadvise(er->fd[k2],er->off[k2],sizes[k2],POSIX_FADV_WILLNEED);
+ }
+ for(int a2=0;a2<K-1;a2++) for(int b2=a2+1;b2<K;b2++){
+ ERef *ea=&m->eref[(int64_t)li*E+idx[a2]], *eb=&m->eref[(int64_t)li*E+idx[b2]];
+ if(eb->fd[0]<ea->fd[0]||(eb->fd[0]==ea->fd[0]&&eb->off[0]<ea->off[0])){
+ int t=idx[a2];idx[a2]=idx[b2];idx[b2]=t; float tw=wsel[a2];wsel[a2]=wsel[b2];wsel[b2]=tw; }
+ }
+ float *z=falloc(LT), *u=falloc(LT), *gate=falloc(MI), *up=falloc(MI), *hz=falloc(LT);
+ w_matmul(z,x,&o->lat_down,1);
+ memset(u,0,LT*sizeof(float));
+ for(int kk=0;kk<K;kk++){
+ Slot *s=expert_get(m,li,idx[kk]);
+ uint8_t *w1p=s->buf, *w1s=w1p+m->e_w1p, *w2p=w1s+m->e_w1s, *w2s=w2p+m->e_w2p,
+ *w3p=w2s+m->e_w2s, *w3s=w3p+m->e_w1p;
+ matmul_mxfp4(gate,z,w1p,w1s,1,LT,MI);
+ matmul_mxfp4(up,z,w3p,w3s,1,LT,MI);
+ for(int i=0;i<MI;i++) gate[i]=situf_(gate[i],up[i],c->situ_b1,c->situ_b2);
+ matmul_mxfp4(hz,gate,w2p,w2s,1,MI,LT);
+ float wk=wsel[kk];
+ for(int i=0;i<LT;i++) u[i]+=wk*hz[i];
+ }
+ rmsnorm_(u,u,o->lat_norm,LT,c->eps);
+ w_matmul(out,u,&o->lat_up,1);
+ /* shared experts at full width */
+ int shi=MI*c->n_shared;
+ float *sg=falloc(shi), *su=falloc(shi), *sd=falloc(c->hidden);
+ w_matmul(sg,x,&o->sh_gate,1); w_matmul(su,x,&o->sh_up,1);
+ for(int i=0;i<shi;i++) sg[i]=situf_(sg[i],su[i],c->situ_b1,c->situ_b2);
+ w_matmul(sd,sg,&o->sh_down,1);
+ for(int d=0;d<c->hidden;d++) out[d]+=sd[d];
+ free(sco);free(z);free(u);free(gate);free(up);free(hz);free(sg);free(su);free(sd);
+}
+
+static void dense_forward(Model *m, Layer *l, const float *x, float *out){
+ Cfg *c=&m->c; int DI=c->dense_inter;
+ float *g=falloc(DI), *u=falloc(DI);
+ w_matmul(g,x,&l->d_gate,1); w_matmul(u,x,&l->d_up,1);
+ for(int i=0;i<DI;i++) g[i]=situf_(g[i],u[i],c->situ_b1,c->situ_b2);
+ w_matmul(out,g,&l->d_down,1);
+ free(g);free(u);
+}
+
+/* ---------- one token through the stack; returns logits (or NULL pre-head) --- */
+static float *g_x0=NULL; static int g_x0_n=0; /* K3_X0: injected inputs (validation) */
+static float *step(Model *m, int id, int pos){
+ Cfg *c=&m->c; int D=c->hidden;
+ int nbmax=(c->n_layers+c->res_bs-1)/c->res_bs;
+ float *hidden=falloc(D), *bres=falloc((int64_t)nbmax*D), *prefix=falloc(D);
+ float *nrm=falloc(D), *att=falloc(D), *mix=falloc(D), *mlp=falloc(D);
+ int nb=0;
+ if(g_x0){
+ if(pos>=g_x0_n){ fprintf(stderr,"K3_X0: pos %d beyond %d injected rows\n",pos,g_x0_n); exit(1); }
+ memcpy(hidden,g_x0+(int64_t)pos*D,D*sizeof(float));
+ } else {
+ char nm[512]; snprintf(nm,sizeof(nm),"%smodel.embed_tokens.weight",m->pfx);
+ st_read_slice_f32(&m->S,nm,(int64_t)id*D,D,hidden,0);
+ }
+ for(int i=0;i<c->n_layers;i++){
+ Layer *l=&m->L[i];
+ memcpy(prefix,hidden,D*sizeof(float)); /* prefix_sum at entry */
+ int have_prefix=1;
+ if(nb>0) res_mix(hidden,prefix,bres,nb,D,l->attn_sw,c->eps);
+ if(i%c->res_bs==0){
+ memcpy(bres+(int64_t)nb*D,prefix,D*sizeof(float)); nb++;
+ have_prefix=0;
+ }
+ double t0=now_s();
+ for(int d=0;d<D;d++) nrm[d]=hidden[d];
+ rmsnorm_(nrm,nrm,l->in_ln,D,c->eps);
+ if(l->kda) kda_forward(m,l,i,nrm,att);
+ else mla_forward(m,l,i,nrm,pos,att);
+ m->t_attn+=now_s()-t0;
+ if(have_prefix){ for(int d=0;d<D;d++) prefix[d]+=att[d]; }
+ else { memcpy(prefix,att,D*sizeof(float)); }
+ res_mix(mix,prefix,bres,nb,D,l->mlp_sw,c->eps);
+ rmsnorm_(nrm,mix,l->post_ln,D,c->eps);
+ t0=now_s();
+ if(l->sparse) moe_forward(m,l,i,nrm,mlp);
+ else dense_forward(m,l,nrm,mlp);
+ m->t_moe+=now_s()-t0;
+ for(int d=0;d<D;d++) prefix[d]+=mlp[d];
+ memcpy(hidden,prefix,D*sizeof(float));
+ if(m->trace) fwrite(hidden,sizeof(float),D,m->trace);
+ }
+ float *logits=NULL;
+ if(m->has_head){
+ double t0=now_s();
+ res_mix(mix,hidden,bres,nb,D,m->out_sw,c->eps);
+ rmsnorm_(mix,mix,m->final_norm,D,c->eps);
+ if(m->trace) fwrite(mix,sizeof(float),D,m->trace);
+ logits=falloc(c->vocab);
+ w_matmul(logits,mix,&m->lm_head,1);
+ m->t_head+=now_s()-t0;
+ }
+ free(hidden);free(bres);free(prefix);free(nrm);free(att);free(mix);free(mlp);
+ return logits;
+}
+
+static void kv_alloc(Model *m, int max_t){
+ Cfg *c=&m->c; m->max_t=max_t;
+ m->Lc=calloc(c->n_layers,sizeof(float*));
+ m->Rc=calloc(c->n_layers,sizeof(float*));
+ for(int i=0;i<c->n_layers;i++) if(!m->L[i].kda){
+ m->Lc[i]=falloc((int64_t)max_t*c->kv_lora);
+ m->Rc[i]=falloc((int64_t)max_t*c->qk_rope);
+ }
+}
+
+static int sample_tok(const float *lo, int V, float temp){
+ if(temp<=0.f){ int b=0; for(int i=1;i<V;i++) if(lo[i]>lo[b]) b=i; return b; }
+ float *p=falloc(V); float mx=lo[0];
+ for(int i=1;i<V;i++) if(lo[i]>mx) mx=lo[i];
+ double s=0; for(int i=0;i<V;i++){ p[i]=expf((lo[i]-mx)/temp); s+=p[i]; }
+ double r=((double)rand()/RAND_MAX)*s, acc=0; int pick=V-1;
+ for(int i=0;i<V;i++){ acc+=p[i]; if(acc>=r){ pick=i; break; } }
+ free(p); return pick;
+}
+
+int main(int argc, char **argv){
+ if(argc<2){
+ fprintf(stderr,"usage: %s <model_dir> [prompt] [--ids \"1 2 3\"] [--ngen N]\n",argv[0]);
+ return 1;
+ }
+ const char *snap=argv[1], *prompt=NULL, *idstr=NULL;
+ int ngen=32;
+ for(int i=2;i<argc;i++){
+ if(!strcmp(argv[i],"--ngen")&&i+1<argc) ngen=atoi(argv[++i]);
+ else if(!strcmp(argv[i],"--ids")&&i+1<argc) idstr=argv[++i];
+ else if(!prompt) prompt=argv[i];
+ }
+ float temp=getenv("COLI_TEMP")?(float)atof(getenv("COLI_TEMP")):0.f;
+ int nlayers=getenv("K3_LAYERS")?atoi(getenv("K3_LAYERS")):0;
+ Model m;
+ model_init(&m,snap,nlayers);
+ if(getenv("K3_TRACE")){
+ m.trace=fopen(getenv("K3_TRACE"),"wb");
+ if(!m.trace){ perror(getenv("K3_TRACE")); return 1; }
+ }
+ if(getenv("K3_X0")){ /* injected input rows [T,hidden] f32, bypasses embed */
+ FILE *f=fopen(getenv("K3_X0"),"rb");
+ if(!f){ perror(getenv("K3_X0")); return 1; }
+ fseek(f,0,SEEK_END); long fn=ftell(f); fseek(f,0,SEEK_SET);
+ g_x0_n=(int)(fn/((long)m.c.hidden*4));
+ g_x0=falloc((int64_t)g_x0_n*m.c.hidden);
+ if(fread(g_x0,4,(size_t)g_x0_n*m.c.hidden,f)!=(size_t)g_x0_n*m.c.hidden){ fprintf(stderr,"K3_X0 short read\n"); return 1; }
+ fclose(f);
+ fprintf(stderr,"[K3] K3_X0: %d injected input rows\n",g_x0_n);
+ }
+ /* tokenize */
+ int ids[65536], np=0;
+ Tok T; int has_tok=0;
+ { char tp[2048]; snprintf(tp,sizeof(tp),"%s/tokenizer.json",snap);
+ FILE *f=fopen(tp,"rb"); if(f){ fclose(f); tok_load(&T,tp); has_tok=1;
+ fprintf(stderr,"[K3] tokenizer.json loaded (family=%s)\n",T.kimi?"kimi":(T.o200k?"o200k":"cl100k")); } }
+ if(idstr){
+ const char *p=idstr;
+ while(*p&&np<65536){ while(*p==' '||*p==',')p++; if(!*p)break; ids[np++]=(int)strtol(p,(char**)&p,10); }
+ } else if(prompt){
+ if(!has_tok){ fprintf(stderr,"no tokenizer.json — pass --ids (generate one with tools/k3_tokenizer.py)\n"); return 1; }
+ if(m.c.bos>=0) ids[np++]=m.c.bos;
+ np+=tok_encode(&T,prompt,(int)strlen(prompt),ids+np,65536-np);
+ } else { fprintf(stderr,"no prompt and no --ids\n"); return 1; }
+ fprintf(stderr,"[K3] prompt: %d tokens | ngen %d | temp %.2f\n",np,ngen,temp);
+ int max_t=getenv("K3_MAXT")?atoi(getenv("K3_MAXT")):np+ngen;
+ kv_alloc(&m,max_t);
+ double t0=now_s(); float *lo=NULL;
+ for(int i=0;i<np;i++){
+ if(lo) free(lo);
+ lo=step(&m,ids[i],i);
+ fprintf(stderr,"\r[K3] prefill %d/%d (%.1fs)",i+1,np,now_s()-t0);
+ }
+ fprintf(stderr,"\n[K3] prefill done in %.1fs (%.2f tok/s)\n",now_s()-t0,np/(now_s()-t0));
+ if(!m.has_head||!lo){
+ fprintf(stderr,"[K3] no head — trace written, stopping after prefill\n");
+ if(m.trace) fclose(m.trace);
+ return 0;
+ }
+ double tg=now_s(); int ntok=0;
+ char buf[512];
+ for(int s=0;s<ngen;s++){
+ int t=sample_tok(lo,m.c.vocab,temp);
+ free(lo); lo=NULL;
+ int is_eos=0; for(int e=0;e<m.c.n_eos;e++) if(t==m.c.eos[e]) is_eos=1;
+ if(has_tok){ int n2=tok_decode(&T,&t,1,buf,sizeof(buf)-1); fwrite(buf,1,n2,stdout); fflush(stdout); }
+ else { printf("%d ",t); fflush(stdout); }
+ ntok++;
+ if(is_eos){ fprintf(stderr,"\n[K3] eos\n"); break; }
+ if(np+ntok>=max_t){ fprintf(stderr,"\n[K3] context full\n"); break; }
+ lo=step(&m,t,np+ntok-1);
+ double el=now_s()-tg;
+ fprintf(stderr," [tok %d: %.1fs/tok, hit %.0f%%, %.1f GB read]\n",
+ ntok,el/ntok,100.0*m.hits/(m.hits+m.miss+1e-9),m.ebytes/1e9);
+ }
+ if(lo) free(lo);
+ double dt=now_s()-tg;
+ fprintf(stderr,"\n[K3] decode %d tokens in %.1fs (%.2f tok/s) | expert hit %.1f%% (%llu/%llu) | %.1f GB streamed\n",
+ ntok,dt,ntok/dt,100.0*m.hits/(m.hits+m.miss+1e-9),
+ (unsigned long long)m.hits,(unsigned long long)(m.hits+m.miss),m.ebytes/1e9);
+ fprintf(stderr,"[K3] time: attn %.1fs moe %.1fs (eload %.1fs) head %.1fs | RSS %.1f GB\n",
+ m.t_attn,m.t_moe,m.t_eload,m.t_head,rss_gb());
+ if(m.trace) fclose(m.trace);
+ return 0;
+}
diff --git a/docs/kimi_k3.md b/docs/kimi_k3.md
new file mode 100644
index 00000000..5d11d96d
--- /dev/null
+++ b/docs/kimi_k3.md
@@ -0,0 +1,154 @@
+# Kimi K3 engine (`c/kimi_k3.c`)
+
+A sibling engine for [Kimi K3](https://huggingface.co/moonshotai/Kimi-K3)
+(2.8T parameters, 104B active, 93 layers), following the one-engine-per-family
+pattern (`colibri.c` = GLM-5.2, `olmoe.c`, `inkling.c`). It shares `st.h`,
+`json.h`, `tok.h`, `quant.h` and touches nothing in the GLM engine.
+
+```
+make kimi_k3
+./kimi_k3 <model_dir> "prompt" --ngen 64
+```
+
+`<model_dir>` is either the plain HF snapshot (config.json +
+`model-*-of-000096.safetensors`) or a repacked container (below). Text-only:
+the vision tower (shards 95–96) is never read.
+
+## Architecture notes
+
+K3 is not a DeepSeek-shaped model; four pieces are new relative to everything
+else in this repo:
+
+- **Hybrid attention, NoPE.** 69 KDA (Kimi Delta Attention) layers + 24 gated
+ MLA layers (every 4th, plus the final layer). There is no positional
+ encoding anywhere — position lives in KDA's convolution and decay. The MLA
+ block has DeepSeek dims (q_lora 1536, kv_lora 512, nope 128 + 64 uncached
+ extra dims, v 128, 96 heads) and both attention types multiply their output
+ by a full-rank sigmoid gate `σ(W_g x)` before `o_proj`.
+- **KDA** per head (dim 128, 96 heads): `q,k,v = SiLU(causal-conv4(W x))`,
+ q/k L2-normalized (ε=1e-6 inside the sqrt), q scaled by 128^-1/2; per-channel
+ decay `gk = -5·σ(exp(A_log[h])·(W_fb W_fa x + dt_bias))`; the delta-rule
+ state update `S = (I − βkkᵀ)·Diag(e^gk)·S + βkvᵀ`; output
+ `W_o[σ(W_g x) ⊙ RMSNorm_head(Sᵀq)]`. The checkpoint stores `A_log` as
+ `[128]`: that is the per-head `[96]` parameter zero-padded (verified —
+ entries 96..127 are exactly 0). Decode state: 96×128×128 f32 per layer.
+- **AttnRes replaces the residual stream.** Each layer keeps a running
+ `prefix_sum`; at layers 0, 12, 24, …, 84 it is snapshotted into a block
+ list. Twice per layer (before attention, before the MLP) and once at the
+ end, the hidden state is REPLACED by a softmax mix over
+ `[snapshots…, prefix_sum]`, scored by `(v · (res_norm.w ⊙ res_proj.w)) /
+ rms(v)` and mixing the raw (un-normalized) entries in fp32.
+- **Stable LatentMoE.** Sigmoid router `[896, 7168]` + score-correction bias,
+ top-16 selected on biased scores, weights are the raw scores renormalized.
+ The token is projected 7168→3584, the 16 experts run in that latent space
+ (GLU, inter 3072), the aggregate is RMSNorm-ed and projected back
+ 3584→7168; two fused full-width shared experts (inter 6144) are added.
+ Activation is **SiTU-GLU**: `4·tanh(g/4)·σ(g) · 25·tanh(u/25)`.
+
+### Weights
+
+Only the routed experts (`experts.N.w{1,2,3}`) are quantized in the
+checkpoint — **MXFP4** (compressed-tensors `mxfp4-pack-quantized`,
+quantization-aware trained): e2m1 nibbles packed two per byte (low nibble =
+even column, bit 3 = sign) with a ue8m0 power-of-two scale per 32 columns,
+`w = v · 2^(scale-127)`. 17.55 MB per expert, 82,432 experts ≈ 1.45 TB.
+`quant.h` gained `matmul_mxfp4` (scalar + AVX2) that computes on this layout
+directly — expert bytes are **never converted**; QAT weights are exactly the
+trained values and any re-encode only adds error.
+
+Everything else is BF16 and is quantized into RAM **at load time** (int8
+per-row / int4-g64), or read pre-quantized from a repacked container.
+
+## Streaming design
+
+Experts are streamed straight from the safetensors shards through a per-layer
+LRU (`K3_EXPERT_GB`). Measured layout facts the engine exploits: the six
+tensors of an expert are stored back-to-back (one expert = one `pread`), and
+experts are *not* id-ordered inside a shard, so loads are issued in disk-offset
+order. Note that K3's router was trained with Quantile Balancing (deliberately
+flat expert usage), so LRU hit rates are structurally lower than on models
+with skewed routing — expect the expert tier to be bandwidth-bound.
+
+## Repacked container (`tools/k3_repack.py`)
+
+```
+python3 tools/k3_repack.py <hf_src> <dst> --bits 8 [--mla-bits 8] [--head-bits 8]
+```
+
+One streaming read of the source (so a disk-to-disk copy can *be* the
+conversion): experts pass through byte-identical (id-ordered, single-pread
+layout), the big BF16 matrices are quantized with exactly the engine's
+load-time algorithm and stored as `U8` + `<name>.qs` f32 scales, the small /
+sensitive tensors (norms, router, conv taps, `dt_bias`, `A_log`, `f_a/f_b`,
+`b_proj`, embeddings) pass through, vision is dropped. Output is spec-valid
+safetensors (`model-XXXXX-of-000094.safetensors`) plus a regenerated index;
+interrupted runs resume per shard. `--verify-full` re-reads every expert byte
+and compares against the source.
+
+The engine auto-detects container tensors (dtype U8 + `.qs` sidecar) and skips
+load-time quantization: measured on two layers, init drops **30.4 s → 0.6 s**;
+on the full model this removes a 10–15 minute quantization pass. Quantized
+values are bit-identical to the load-time path (verified by hidden-state
+trace), so the two formats are numerically interchangeable.
+
+Sizes: source 1.56 TB → ≈1.50 TB (`--bits 8`) / ≈1.48 TB (`--bits 4`). The
+experts (93 % of bytes) are already at 4.25 bits/weight and cannot shrink
+losslessly; sub-4-bit expert re-encodes (fmt=5/6) would be double quantization
+of QAT weights and are deliberately not offered here.
+
+## Tokenizer
+
+The HF repo ships only a raw tiktoken vocab (`tiktoken.model`).
+`tools/k3_tokenizer.py` writes a `tokenizer.json` for it, and `tok.h` gained:
+
+- a **kimi pre-tokenizer family** (sniffed via `\p{Han}` in the Split regex):
+ the o200k case-aware rules plus a leading Han-run rule, Han excluded from
+ the letter classes, and no `/` tail in the punctuation rule;
+- a **rank-BPE mode**, used when `model.merges` is empty: merge the adjacent
+ pair whose concatenation has the lowest vocab id — which is tiktoken's own
+ algorithm, so encoding is exact by construction. (Merge-list recovery from
+ ranks is provably lossy: the standard reconstruction mis-merges e.g.
+ "newlines"; that is why the generator emits no merges.)
+
+`tools/k3_tokenizer.py --ctest tests/test_tok_kimi` cross-checks the C
+tokenizer against tiktoken; it matches on all 18 corpus cases (Chinese,
+kanji-vs-kana, Korean, CRLF, contractions, emoji, mixed-script boundaries).
+
+## Validation
+
+`tools/k3_ref.py` is an independent numpy implementation of the full layer
+stack (KDA recurrence, gated MLA, AttnRes, LatentMoE with MXFP4 dequant). With
+injected inputs (`K3_X0`, bypassing the embedding) and f32 weights
+(`K3_BITS=32`), the C engine matches it on real checkpoint weights across all
+four layer types to **rel-L2 ≤ 2.2e-6** (float32 noise).
+
+One property worth knowing: the AttnRes mix logits sit close together
+(margins ~0.02), so the mix leverages *weight* noise strongly even though it
+does not amplify *input* perturbations (a 1e-4 input perturbation stays 1e-4
+through the stack). Load-time int8 produces a hidden-state drift growing to
+~14 % over four layers on synthetic inputs — per-tensor quantization SNR is a
+uniform ~1 %, so this is architectural sensitivity, not an outlier problem.
+Judge quantization choices on real-text logits, not synthetic-vector norms.
+
+## Environment variables
+
+| var | default | meaning |
+|---|---|---|
+| `K3_BITS` | 4 | load-time bits for KDA/latent/shared/dense mats (4, 8, 32=f32) |
+| `K3_MLA_BITS` | 8 | load-time bits for MLA projections |
+| `K3_HEAD_BITS` | 8 | load-time bits for lm_head |
+| `K3_EXPERT_GB` | 8 | routed-expert LRU budget |
+| `K3_DIRS` | — | extra shard directories (multi-drive split, no duplication) |
+| `K3_MAXT` | prompt+ngen | context capacity |
+| `K3_LAYERS` | all | truncate the stack (validation) |
+| `K3_TRACE` | — | dump f32 hidden state after every layer (validation) |
+| `K3_X0` | — | inject input rows, bypass embedding (validation) |
+| `COLI_TEMP` | 0 | 0 = greedy, else softmax temperature |
+
+## Current limitations
+
+- Single-token stepping only (prefill is sequential; no chunked KDA, no
+ batched MLA prefill).
+- CPU only (no CUDA/Metal/Vulkan tier), scalar KDA inner loops.
+- No chat template handling; prompts are raw text after `[BOS]`.
+- No speculative decoding (K3 has no MTP head).