Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

kimi_k3: Vulkan tier (K3_VK) β€” shared experts resident + fill-once MXFP4 routed-expert tier

Steve Markgrafb61089a812f9
diff --git a/c/Makefile b/c/Makefile
index 0c4caac2..2f481943 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -439,8 +439,8 @@ olmoe$(EXE): olmoe.c st.h json.h compat.h
inkling$(EXE): inkling.c st.h json.h compat.h $(INK_CUDA_OBJ)
$(CC) $(CFLAGS) inkling.c $(INK_CUDA_OBJ) -o inkling$(EXE) $(LDFLAGS)
-kimi_k3$(EXE): kimi_k3.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h
- $(CC) $(CFLAGS) kimi_k3.c -o kimi_k3$(EXE) $(LDFLAGS)
+kimi_k3$(EXE): kimi_k3.c st.h json.h tok.h tok_unicode.h tok_unicode_o200k.h compat.h quant.h $(VK_OBJ) $(VK_SPV)
+ $(CC) $(CFLAGS) kimi_k3.c $(VK_OBJ) -o kimi_k3$(EXE) $(LDFLAGS)
# Use a baseline that matches the compiler target. macOS already targets a
# portable baseline when ARCH is empty; forcing the x86 value there breaks
diff --git a/c/kimi_k3.c b/c/kimi_k3.c
index 4ea1e288..24df36ef 100644
--- a/c/kimi_k3.c
+++ b/c/kimi_k3.c
@@ -40,6 +40,13 @@
* K3_MLA_BITS=8|4|32 MLA projections (default 8)
* K3_HEAD_BITS=8|4|32 lm_head (default 8)
* K3_EXPERT_GB=N routed-expert LRU cache budget (default 8)
+ * K3_VK=0|1 Vulkan tier (build with `make VK=1 kimi_k3`; default
+ * 1 when built): shared experts VRAM-resident + a
+ * fill-once native-MXFP4 routed-expert tier; resident
+ * experts skip disk AND CPU at decode. CPU fallback
+ * everywhere; output identical.
+ * K3_VK_GB=N VRAM cap for the tier (default: driver budget)
+ * K3_VK_UP=N routed-expert uploads per step (default 8)
* K3_DIRECT=0|1 O_DIRECT expert reads (default 1; buffered fallback)
* K3_IDOT=0|1 int8-activation expert matmuls (default 1; 0 = float)
* K3_PIPE=0|1 overlap expert loads with compute (default 1)
@@ -94,7 +101,8 @@ typedef struct {
} Cfg;
/* ---------- RAM-resident weight, quantized at load ---------- */
-typedef struct { int fmt; float *f; int8_t *q8; uint8_t *q4; float *s; int O, I, gs; } W;
+typedef struct { int fmt; float *f; int8_t *q8; uint8_t *q4; float *s; int O, I, gs;
+ void *vk; /* ColiVkTensor* once uploaded (K3_VK); NULL = CPU only */ } W;
typedef struct { /* KDA layer */
W q, k, v, o, g;
@@ -174,7 +182,62 @@ static void rmsnorm_(float *out, const float *x, const float *w, int D, float ep
}
/* ---------- W: load-time quantization + matvec ---------- */
+/* ---------- Vulkan tier (K3_VK, build with `make VK=1 kimi_k3`) ----------
+ * Two residency classes on the card, both with transparent CPU fallback:
+ * - dense W tensors uploaded once at init (shared experts first): computed
+ * by the existing fmt=1/4 shaders whenever S==1 (w_matmul hook below);
+ * - a fill-once routed-expert tier in fmt=7 (native MXFP4, never re-encoded):
+ * experts enter from freshly-read RAM slots (K3_VK_UP per step) until the
+ * VRAM budget (K3_VK_GB) is reached; resident experts then skip BOTH the
+ * disk read and the CPU matmuls at decode (C==1). */
+static int g_k3_vk=0; /* backend live (K3_VK=0 disables) */
+#ifdef COLI_VULKAN
+#include "backend_vulkan.h"
+typedef struct { void *w1, *w2, *w3; } VkExp; /* ColiVkTensor* triple */
+static VkExp *g_vkexp; static int64_t g_vkexp_n;
+static int g_vk_upcap=8, g_vk_up_left=0, g_vk_full=0;
+static long g_vk_hit=0, g_vk_res=0;
+static double g_vk_gb=0; /* K3_VK_GB cap (0 = driver budget) */
+static const char *k3_vk_spv(char *buf, size_t n){
+ const char *env=getenv("COLI_VK_SHADERS");
+ struct stat st;
+ if(env&&*env){
+ if(!stat(env,&st)&&S_ISDIR(st.st_mode)){ snprintf(buf,n,"%s/qmatmul.spv",env); return buf; }
+ return env;
+ }
+#ifdef __linux__
+ ssize_t k=readlink("/proc/self/exe",buf,n-1);
+ if(k>0){
+ buf[k]=0;
+ char *sl=strrchr(buf,'/');
+ if(sl&&(size_t)(sl+1-buf)+sizeof("shaders/qmatmul.spv")<=n){
+ strcpy(sl+1,"shaders/qmatmul.spv");
+ if(!stat(buf,&st)) return buf;
+ }
+ }
+#endif
+ return "shaders/qmatmul.spv";
+}
+static int vk_budget_full(void){
+ double used=0,budget=0;
+ if(!coli_vk_mem_budget(&used,&budget)) return 0;
+ double cap=budget-0.5; if(g_vk_gb>0&&g_vk_gb<cap) cap=g_vk_gb;
+ return used>=cap;
+}
+static int w_vk_upload(W *w){
+ if(w->vk) return 1;
+ int fmt = w->fmt==1?1 : w->fmt==4?4 : -1;
+ if(fmt<0) return 0;
+ return coli_vk_tensor_ensure((ColiVkTensor**)&w->vk,
+ fmt==1?(const void*)w->q8:(const void*)w->q4,w->s,fmt,w->I,w->O,w->gs);
+}
+#endif
static void w_matmul(float *y, const float *x, const W *w, int S){
+#ifdef COLI_VULKAN
+ if(g_k3_vk&&S==1&&w->vk&&
+ coli_vk_matmul((ColiVkTensor**)&((W*)w)->vk,y,x,NULL,NULL,w->fmt,1,w->I,w->O,w->gs))
+ return;
+#endif
if(w->fmt==0) matmul(y,x,w->f,S,w->I,w->O);
else if(w->fmt==1) matmul_q(y,x,w->q8,w->s,S,w->I,w->O);
else if(w->fmt==4) matmul_i4_grouped(y,x,w->q4,w->s,S,w->I,w->O,w->gs);
@@ -519,6 +582,35 @@ static void model_init(Model *m, const char *snap, int n_layers_env){
free(rn); free(rp); }
w_load(m,&m->lm_head,"lm_head.weight",c->vocab,c->hidden,hbits);
} else fprintf(stderr,"[K3] final norm/lm_head not present β€” trace-only mode\n");
+#ifdef COLI_VULKAN
+ { const char *ev=getenv("K3_VK");
+ if(!ev||atoi(ev)){
+ char sbuf[512]; const char *spv=k3_vk_spv(sbuf,sizeof(sbuf));
+ g_k3_vk=coli_vk_init(spv);
+ if(!g_k3_vk) fprintf(stderr,"[K3-VK] Vulkan unavailable (tried %s) β€” CPU only\n",spv);
+ }
+ if(g_k3_vk){
+ g_vk_gb=getenv("K3_VK_GB")?atof(getenv("K3_VK_GB")):0;
+ g_vk_upcap=getenv("K3_VK_UP")?atoi(getenv("K3_VK_UP")):8;
+ g_vkexp_n=(int64_t)c->n_layers*c->n_experts;
+ g_vkexp=calloc((size_t)g_vkexp_n,sizeof(VkExp));
+ if(!g_vkexp) g_k3_vk=0;
+ }
+ if(g_k3_vk){
+ /* dense residency: shared experts first β€” they run every token and are
+ * the biggest always-on RAM-bandwidth slice that fits VRAM */
+ int nsh=0;
+ for(int i=0;i<c->n_layers&&!vk_budget_full();i++){
+ if(!m->L[i].sparse) continue;
+ Moe *sm2=&m->L[i].moe;
+ nsh+=w_vk_upload(&sm2->sh_gate)+w_vk_upload(&sm2->sh_up)+w_vk_upload(&sm2->sh_down);
+ }
+ double used=0,budget=0; coli_vk_mem_budget(&used,&budget);
+ fprintf(stderr,"[K3-VK] resident: %d shared-expert mats (%.1f/%.1f GB); routed MXFP4 tier fills at decode (K3_VK_UP=%d/step, cap %s)\n",
+ nsh,used,budget,g_vk_upcap,g_vk_gb>0?"K3_VK_GB":"driver budget");
+ }
+ }
+#endif
expert_table_init(m);
/* expert LRU cache, per-layer slots from the global budget */
double egb = getenv("K3_EXPERT_GB")?atof(getenv("K3_EXPERT_GB")):8.0;
@@ -530,7 +622,8 @@ static void model_init(Model *m, const char *snap, int n_layers_env){
* regardless of K3_EXPERT_GB. */
if(cap<1) cap=1;
if(cap>c->n_experts) cap=c->n_experts;
- m->ecache=calloc(c->n_layers,sizeof(LCache));
+ { int ncl=c->n_layers>0?c->n_layers:1;
+ m->ecache=calloc((size_t)ncl,sizeof(LCache)); }
for(int i=0;i<c->n_layers;i++) if(m->L[i].sparse){
m->ecache[i].cap=cap; m->ecache[i].s=calloc(cap,sizeof(Slot));
for(int j2=0;j2<cap;j2++) m->ecache[i].s[j2].eid=-1;
@@ -775,6 +868,59 @@ static void expert_apply(Model *m, Slot *s, const float *z, float wk,
for(int i=0;i<c->latent;i++) u[i]+=wk*hz[i];
}
+#ifdef COLI_VULKAN
+/* GPU apply for a tier-resident expert (decode, S==1): w1/w3 in one paired
+ * submit, SiTU-GLU on CPU, w2 down. Returns 0 untouched on any failure so
+ * the caller can run the normal disk+CPU path. */
+static int vk_expert_apply(Model *m, int li, int eid, const float *z, float wk,
+ float *u, float *gate, float *up, float *hz){
+ Cfg *c=&m->c;
+ VkExp *v=&g_vkexp[(int64_t)li*c->n_experts+eid];
+ if(!v->w1||!v->w2||!v->w3) return 0;
+ if(!coli_vk_matmul_pair((ColiVkTensor**)&v->w1,gate,NULL,NULL,c->moe_inter,
+ (ColiVkTensor**)&v->w3,up,NULL,NULL,c->moe_inter,
+ 7,z,1,c->latent,32)) return 0;
+ for(int i=0;i<c->moe_inter;i++) gate[i]=situf_(gate[i],up[i],c->situ_b1,c->situ_b2);
+ if(!coli_vk_matmul((ColiVkTensor**)&v->w2,hz,gate,NULL,NULL,7,1,c->moe_inter,c->latent,32))
+ return 0;
+ for(int i=0;i<c->latent;i++) u[i]+=wk*hz[i];
+ g_vk_hit++;
+ return 1;
+}
+/* Fill the tier from a freshly-read RAM slot (main thread only). The ue8m0
+ * exponents expand to f32 group scales at upload (the shader is float-only). */
+static void vk_expert_try_upload(Model *m, int li, int eid, Slot *s){
+ if(!g_k3_vk||g_vk_full||g_vk_up_left<=0) return;
+ VkExp *v=&g_vkexp[(int64_t)li*m->c.n_experts+eid];
+ if(v->w1) return;
+ if(vk_budget_full()){ g_vk_full=1;
+ fprintf(stderr,"[K3-VK] expert tier full: %ld experts resident\n",g_vk_res);
+ return; }
+ uint8_t *w1p=s->buf, *w1s=w1p+m->e_w1p, *w2p=w1s+m->e_w1s, *w2s=w2p+m->e_w2p,
+ *w3p=w2s+m->e_w2s, *w3s=w3p+m->e_w1p;
+ int LT=m->c.latent, MI=m->c.moe_inter;
+ int64_t n1=m->e_w1s, n2=m->e_w2s; /* scale counts = scale bytes (u8) */
+ float *sc=falloc(n1>n2?n1:n2);
+ int ok=1;
+ for(int64_t i=0;i<n1;i++) sc[i]=mx4_scale(w1s[i]);
+ ok=ok&&coli_vk_tensor_ensure((ColiVkTensor**)&v->w1,w1p,sc,7,LT,MI,32);
+ if(ok){ for(int64_t i=0;i<n2;i++) sc[i]=mx4_scale(w2s[i]);
+ ok=ok&&coli_vk_tensor_ensure((ColiVkTensor**)&v->w2,w2p,sc,7,MI,LT,32); }
+ if(ok){ for(int64_t i=0;i<n1;i++) sc[i]=mx4_scale(w3s[i]);
+ ok=ok&&coli_vk_tensor_ensure((ColiVkTensor**)&v->w3,w3p,sc,7,LT,MI,32); }
+ free(sc);
+ if(!ok){
+ if(v->w1){ coli_vk_tensor_free(v->w1); v->w1=NULL; }
+ if(v->w2){ coli_vk_tensor_free(v->w2); v->w2=NULL; }
+ if(v->w3){ coli_vk_tensor_free(v->w3); v->w3=NULL; }
+ g_vk_full=1;
+ fprintf(stderr,"[K3-VK] expert tier full: %ld experts resident\n",g_vk_res);
+ return;
+ }
+ g_vk_res++; g_vk_up_left--;
+}
+#endif
+
/* ---------- async loader pool (K3_PIPE): expert preads overlap compute ----
* A batch of jobs is submitted per token+layer; the compute loop below waits
* per-expert on its ready flag, so expert j's math runs while j+1.. load.
@@ -869,6 +1015,9 @@ static void experts_apply_union(Model *m, int li, int nu, const int *uids,
usleep(50);
m->t_eload+=now_s()-t0;
}
+#ifdef COLI_VULKAN
+ if(g_k3_vk&&qof[j]>=0) vk_expert_try_upload(m,li,uids[base+j],use[j]);
+#endif
int f=pfirst[base+j];
for(int p2=0;p2<pcnt[base+j];p2++){
int t=poslist[f+p2];
@@ -976,6 +1125,24 @@ static void moe_forward(Model *m, Layer *l, int li, const float *x, int C, float
if(er->contig){ if(er->fd[0]>=0) posix_fadvise(er->fd[0],er->off[0],m->e_slot,POSIX_FADV_WILLNEED); }
else for(int k2=0;k2<6;k2++) if(er->fd[k2]>=0) posix_fadvise(er->fd[k2],er->off[k2],sizes[k2],POSIX_FADV_WILLNEED);
}
+#ifdef COLI_VULKAN
+ /* decode: tier-resident experts run on the GPU and drop out of the
+ * disk union β€” that skip is the I/O saving. C>1 prefill stays on the
+ * CPU-batched path (and still feeds the tier via the upload hook). */
+ if(g_k3_vk&&C==1){
+ int keep=0;
+ for(int j=0;j<nu;j++){
+ int handled=0;
+ if(pcnt[j]==1){
+ int t=poslist[pfirst[j]];
+ handled=vk_expert_apply(m,li,uid[j],z+(int64_t)t*LT,
+ wlist[pfirst[j]],u+(int64_t)t*LT,gate,up,hz);
+ }
+ if(!handled){ uid[keep]=uid[j]; pcnt[keep]=pcnt[j]; pfirst[keep]=pfirst[j]; keep++; }
+ }
+ nu=keep;
+ }
+#endif
experts_apply_union(m,li,nu,uid,pfirst,pcnt,poslist,wlist,z,LT,u,gate,up,hz);
free(map);free(uid);free(pcnt);free(pfirst);free(poslist);free(wlist);free(cur);
}
@@ -1012,6 +1179,9 @@ static float *g_x0=NULL; static int g_x0_n=0; /* K3_X0: injected inputs (valida
static FILE *g_lfp=NULL; /* K3_LOGITS: per-position logit dump */
static float *step_chunk(Model *m, const int *ids, int pos0, int C){
Cfg *c=&m->c; int D=c->hidden;
+#ifdef COLI_VULKAN
+ g_vk_up_left=g_vk_upcap; /* routed-tier upload budget per step */
+#endif
int nbmax=(c->n_layers+c->res_bs-1)/c->res_bs;
float *hidden=falloc((int64_t)C*D), *bres=falloc((int64_t)C*nbmax*D);
float *prefix=falloc((int64_t)C*D), *nrm=falloc((int64_t)C*D);
@@ -1372,6 +1542,10 @@ static void serve_one(Model *m, Tok *T, ServeReq *q){
printf("PROF %.3f %d %d %.3f %.3f %.3f %.3f %.3f %d\n",
dt,np,gen,disk,0.0,moe>disk?moe-disk:moe,m->t_attn-a0,m->t_head-h0,gen+1);
fflush(stdout);
+#ifdef COLI_VULKAN
+ if(g_k3_vk) fprintf(stderr,"[K3-VK] routed tier: %ld resident, %ld GPU hits so far\n",
+ g_vk_res,g_vk_hit);
+#endif
}
static void serve_loop(Model *m, Tok *T){
diff --git a/docs/kimi_k3.md b/docs/kimi_k3.md
index b9b9e831..7e127207 100644
--- a/docs/kimi_k3.md
+++ b/docs/kimi_k3.md
@@ -154,6 +154,9 @@ Judge quantization choices on real-text logits, not synthetic-vector norms.
| `K3_MLA_BITS` | 8 | load-time bits for MLA projections |
| `K3_HEAD_BITS` | 8 | load-time bits for lm_head |
| `K3_EXPERT_GB` | 8 | routed-expert LRU budget |
+| `K3_VK` | 1 | Vulkan tier when built with `make VK=1 kimi_k3` (0 = pure CPU) |
+| `K3_VK_GB` | driver budget | VRAM cap for the Vulkan tier |
+| `K3_VK_UP` | 8 | routed-expert uploads per step (fill-once tier) |
| `K3_DIRECT` | 1 | O_DIRECT expert reads (0 = buffered + WILLNEED) |
| `K3_IDOT` | 1 | int8-activation expert matmuls (0 = exact-float kernel) |
| `K3_PIPE` | 1 | overlap expert loads with compute (loader threads) |
@@ -220,12 +223,37 @@ is returned as `reasoning_content`, response text as `content`, and
`<|end_of_msg|>` remains the model-owned stop token. `STOP` and `CANCEL` are
honoured between generated tokens.
+## Vulkan tier (`make VK=1 kimi_k3`)
+
+The shared Vulkan backend (`backend_vulkan.c`) gained an **fmt=7 MXFP4**
+decode path for K3's expert format β€” e2m1 nibbles with the ue8m0 exponents
+expanded to f32 per-32-group scales at upload, so the QAT bytes are uploaded
+exactly as stored and never re-encoded (kernel vs `matmul_mxfp4`: rel_l2
+2.2e-07 on an RX 9070/RADV, 2.6e-07 on llvmpipe;
+`tests/test_vk_mxfp4.c`). The engine keeps two residency classes on the
+card, both with transparent CPU fallback and identical output:
+
+- **shared experts**, uploaded once at init (int4/int8, the existing
+ fmt-1/4 shaders): they run every token and are the largest always-on
+ dense slice that fits VRAM (7.5 GB for all 92 MoE layers at int4);
+- a **fill-once routed-expert tier** in fmt=7: experts enter from
+ freshly-read RAM slots (`K3_VK_UP` per step) until the VRAM budget
+ (`K3_VK_GB`) is reached. At decode, tier-resident experts skip **both**
+ the 17.5 MB disk read and the CPU matmuls (one paired w1/w3 submit,
+ SiTU-GLU on CPU, w2 down). Chunked prefill stays on the CPU-batched path
+ and still warms the tier.
+
+K3's Quantile-Balancing-flat routing caps what any cache tier can do β€” the
+tier's value scales with how long the server lives (fill-once) and with the
+measured short-term reuse (temporal locality), not with marginal expert
+heat. `K3_VK=0` disables the tier at runtime.
+
## Current limitations
- Decode is single-token (no speculative decoding β€” K3 has no MTP head).
- Tool declarations/calls and image content are not exposed through the shared
gateway yet; unsupported requests fail explicitly.
-- CPU only (no CUDA/Metal/Vulkan tier).
+- CPU + optional Vulkan tier (no CUDA/Metal).
- The protocol, tokenizer, gateway, TUI, and Web client paths are locally
testable without the 1.5 TB checkpoint. A release claim still requires one
full-model multi-turn TUI/Web run on a host that owns the complete snapshot.