Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

kimi_k3: the dense chain on the device (COLI_VK_CHAIN=1)

JustVuggd34c7903644d
diff --git a/c/kimi_k3.c b/c/kimi_k3.c
index d925db17..dc424221 100644
--- a/c/kimi_k3.c
+++ b/c/kimi_k3.c
@@ -47,6 +47,10 @@
* COLI_VK_TIER* variables), the shared experts where
* COLI_VK_DENSE puts the dense matrices. CPU fallback
* everywhere; the tier's experts match K3_IDOT=0.
+ * COLI_VK_CHAIN=0|1|2 with COLI_VULKAN=1: every layer's dense part on the
+ * device, one frame per layer up to the router
+ * (kimi_k3_chain.h; 2: prompts only). Default: on a
+ * discrete GPU, off on an integrated one (not measured).
* K3_VK=0|1 the old switch, an alias: 1 = COLI_VULKAN=1, 0 = the
* device stays closed. Unset: COLI_VULKAN decides.
* K3_VK_GB=N alias of COLI_VK_TIER_GB (when that is unset)
@@ -1915,6 +1919,12 @@ static void pin_seed(Model *m, int64_t hist){
* matrices. Its reports: one "[VK] kimi_k3" line of device matmuls and the tier's
* line, at the end of a run and of every serve turn. */
#define K3_VK_ROWS 64 /* positions whose pairs one device batch carries at most */
+/* the dense chain (kimi_k3_chain.h, included before step_chunk_ex) */
+static void k3c_start(Model *m, int tier_on);
+static void k3c_atexit(void);
+static void k3c_adopt_shared(Model *m);
+static void k3c_report(void);
+static int g_k3c_on;
static int k3_vk_in_ram(void *ctx, int li, int e){ return slot_indexed((Model*)ctx,li,e)!=NULL; }
static size_t k3_w_bytes(const W *w){
int fmt=k3_vk_fmt(w);
@@ -1947,6 +1957,7 @@ static void k3_vk_tier_start(Model *m){
#ifdef COLI_CUDA
cuda_on=g_k3_cuda;
#endif
+ k3c_start(m,nmoe&&vkt_wanted()&&!cuda_on); /* COLI_VK_CHAIN: the trunk on the device, before the tier sizes itself */
if(cuda_on) fprintf(stderr,"[VK] tier kimi_k3: the CUDA expert tier is on and wins; the Vulkan tier stays off\n");
else if(nmoe&&vkt_wanted()){
VktConfig vc={.engine="kimi_k3",.layers=c->n_layers,.experts=c->n_experts,
@@ -1955,7 +1966,7 @@ static void k3_vk_tier_start(Model *m){
.act=VKT_ACT_SITU,.act_a=c->situ_b1,.act_b=c->situ_b2,
.max_rows=K3_VK_ROWS*c->topk,
.ram_reserve=(size_t)(m->e_slot+8192)*(size_t)cap*(size_t)nmoe,
- .dense_bytes=coli_vk_dense()?k3_vk_dense_bytes(m):0,
+ .dense_bytes=coli_vk_dense()&&!g_k3c_on?k3_vk_dense_bytes(m):0, /* the chain's are there already */
.in_ram=k3_vk_in_ram,.ram_ctx=m};
atexit(coli_vk_shutdown); /* before vkt_init, which makes the expert batch's pipelines and can still refuse (no room): the device goes at exit either way, after the tier's teardown */
if(vkt_init(&vc,rt_counts_all())){
@@ -1983,6 +1994,7 @@ static void k3_vk_tier_start(Model *m){
}
if(!vkt_ready()&&!coli_vk_dense()) coli_vk_dense_decide("kimi_k3",0,1); /* no tier after all */
if(coli_vk_dense()){
+ k3c_adopt_shared(m); /* the chain's copies, where it made them */
int nsh=0;
for(int i=0;i<c->n_layers;i++){
if(!m->L[i].sparse) continue;
@@ -1992,11 +2004,13 @@ static void k3_vk_tier_start(Model *m){
size_t used=0, count=0; coli_vk_mem_info(&used,&count);
fprintf(stderr,"[VK] kimi_k3: %d shared-expert matrices on the device (%.2f MiB)\n",nsh,used/1048576.0);
}
+ k3c_atexit(); /* after the tier's: the chain goes before the device */
}
static void k3_vk_report(const Model *m, const char *scope){
if(!g_k3_vk) return;
fprintf(stderr,"[VK] kimi_k3: %llu matmuls on the GPU\n",coli_vk_matmul_calls());
vkt_report(scope,m->hits,m->miss);
+ k3c_report(); /* the dense chain's line, when it ran */
}
#endif
@@ -2086,7 +2100,7 @@ static float *k3_vk_step(Model *m, Moe *o, int li, int C, const float *x, const
}
int n2=k3_vk_sublist(nu,uid,pfirst,pcnt,poslist,wlist,epair,st->taken,0,u2,f2,c2,p2,w2,s2);
if(n2) experts_apply_union(m,li,n2,u2,f2,c2,p2,w2,z,LT,u,gate,up,hz,R,s2);
- float *sd=k3_shared(m,o,x,C); /* while the batch runs */
+ float *sd=x?k3_shared(m,o,x,C):NULL; /* while the batch runs (x NULL: the dense chain's) */
if(st->ndev&&!vkt_join(st->dev)){ /* the batch failed: those pairs here */
n2=k3_vk_sublist(nu,uid,pfirst,pcnt,poslist,wlist,epair,st->taken,1,u2,f2,c2,p2,w2,s2);
if(n2) experts_apply_union(m,li,n2,u2,f2,c2,p2,w2,z,LT,u,gate,up,hz,R,s2);
@@ -2109,14 +2123,12 @@ static float *k3_vk_step(Model *m, Moe *o, int li, int C, const float *x, const
}
#endif
-static void moe_forward(Model *m, Layer *l, int li, const float *x, int C, float *out){
- Cfg *c=&m->c; Moe *o=&l->moe;
- int E=c->n_experts, K=c->topk, LT=c->latent, MI=c->moe_inter;
- float *sco=falloc((int64_t)C*E);
- matmul(sco,x,o->router,C,c->hidden,E);
- int *idxs=malloc((size_t)C*K*sizeof(int)); float *wsels=falloc((int64_t)C*K);
- int *keff=malloc((size_t)C*sizeof(int));
- if(!idxs||!keff){fprintf(stderr,"OOM moe sel\n");exit(1);}
+/* The router's choice for C rows from their raw scores (sco [C][E], turned into
+ * sigmoids in place): the top-k with the correction bias, the weights renormalized,
+ * K3_TOPP; idxs and wsels [C][K], keff[C] the ranks kept. */
+static void k3_moe_route(Model *m, Moe *o, int li, float *sco, int C, int *idxs, float *wsels, int *keff){
+ Cfg *c=&m->c;
+ int E=c->n_experts, K=c->topk;
for(int t=0;t<C;t++){
float *st=sco+(int64_t)t*E;
for(int e=0;e<E;e++) st[e]=sigmoidf_(st[e]);
@@ -2154,10 +2166,17 @@ static void moe_forward(Model *m, Layer *l, int li, const float *x, int C, float
keff[t]=Kt;
rt_route(li,t,idx,wsel,Kt); /* traces and counts, one call */
}
- float *z=falloc((int64_t)C*LT), *u=falloc((int64_t)C*LT);
+}
+
+/* The routed experts of C rows in the latent: u[C][LT] (zeroed by the caller) gets
+ * every kept rank's weighted output, in the union's order. x: the rows the shared
+ * experts read while a tier batch runs (NULL: the dense chain computes them on the
+ * device); their output is returned when this step computed it, else NULL. */
+static float *k3_moe_experts(Model *m, Moe *o, int li, const float *x, const float *z, int C,
+ const int *keff, const int *idxs, const float *wsels, float *u){
+ Cfg *c=&m->c;
+ int E=c->n_experts, K=c->topk, LT=c->latent, MI=c->moe_inter;
float *gate=falloc(MI), *up=falloc(MI), *hz=falloc(LT), *sd=NULL;
- w_matmul(z,x,&o->lat_down,C);
- memset(u,0,(size_t)C*LT*sizeof(float));
#ifdef COLI_VULKAN
K3VkStep vst={0}; /* the device's pairs, issued before the union is read */
if(vkt_ready()) k3_vk_issue(m,li,C,keff,idxs,z,&vst);
@@ -2217,6 +2236,23 @@ static void moe_forward(Model *m, Layer *l, int li, const float *x, int C, float
#ifdef COLI_VULKAN
k3_vk_step_free(&vst);
#endif
+ free(gate);free(up);free(hz);
+ return sd;
+}
+
+static void moe_forward(Model *m, Layer *l, int li, const float *x, int C, float *out){
+ Cfg *c=&m->c; Moe *o=&l->moe;
+ int E=c->n_experts, K=c->topk, LT=c->latent;
+ float *sco=falloc((int64_t)C*E);
+ matmul(sco,x,o->router,C,c->hidden,E);
+ int *idxs=malloc((size_t)C*K*sizeof(int)); float *wsels=falloc((int64_t)C*K);
+ int *keff=malloc((size_t)C*sizeof(int));
+ if(!idxs||!keff){fprintf(stderr,"OOM moe sel\n");exit(1);}
+ k3_moe_route(m,o,li,sco,C,idxs,wsels,keff);
+ float *z=falloc((int64_t)C*LT), *u=falloc((int64_t)C*LT);
+ w_matmul(z,x,&o->lat_down,C);
+ memset(u,0,(size_t)C*LT*sizeof(float));
+ float *sd=k3_moe_experts(m,o,li,x,z,C,keff,idxs,wsels,u);
for(int t=0;t<C;t++)
rmsnorm_(u+(int64_t)t*LT,u+(int64_t)t*LT,o->lat_norm,LT,c->eps);
w_matmul(out,u,&o->lat_up,C);
@@ -2224,7 +2260,7 @@ static void moe_forward(Model *m, Layer *l, int li, const float *x, int C, float
if(!sd) sd=k3_shared(m,o,x,C);
for(int64_t d=0;d<(int64_t)C*c->hidden;d++) out[d]+=sd[d];
free(sco);free(idxs);free(wsels);free(keff);
- free(z);free(u);free(gate);free(up);free(hz);free(sd);
+ free(z);free(u);free(sd);
}
static void dense_forward(Model *m, Layer *l, const float *x, int C, float *out){
@@ -2350,15 +2386,9 @@ static void k3_echo(const char *id, int pos, int token, const float *lo, int V,
fputc('\n', stdout); fflush(stdout);
}
-static float *step_chunk_ex(Model *m, const int *ids, int pos0, int C,
- K3CancelPoll poll_cancel, void *cancel_context,
- int *cancelled){
- Cfg *c=&m->c; int D=c->hidden;
- if(cancelled) *cancelled=0;
- int nbmax=(c->n_layers+c->res_bs-1)/c->res_bs;
- float *hidden=falloc((int64_t)C*D), *bres=falloc((int64_t)C*nbmax*D);
- float *mix=falloc(D);
- int nb=0;
+/* The input rows of positions pos0..pos0+C-1: the embeddings of ids, or K3_X0's rows. */
+static void k3_embed(Model *m, const int *ids, int pos0, int C, float *hidden){
+ int D=m->c.hidden;
for(int t=0;t<C;t++){
if(g_x0){
if(pos0+t>=g_x0_n){ fprintf(stderr,"K3_X0: pos %d beyond %d injected rows\n",pos0+t,g_x0_n); exit(1); }
@@ -2368,24 +2398,60 @@ static float *step_chunk_ex(Model *m, const int *ids, int pos0, int C,
st_read_slice_f32(&m->S,nm,(int64_t)ids[t]*D,D,hidden+(int64_t)t*D,0);
}
}
+}
+
+#ifdef COLI_VULKAN
+#include "kimi_k3_chain.h" /* COLI_VK_CHAIN: every layer's dense chain on the device */
+#endif
+
+static float *step_chunk_ex(Model *m, const int *ids, int pos0, int C,
+ K3CancelPoll poll_cancel, void *cancel_context,
+ int *cancelled){
+ Cfg *c=&m->c; int D=c->hidden;
+ if(cancelled) *cancelled=0;
+ int nbmax=(c->n_layers+c->res_bs-1)/c->res_bs;
+ float *hidden=falloc((int64_t)C*D), *bres=falloc((int64_t)C*nbmax*D);
+ float *mix=falloc(D);
+ int nb=0;
+ k3_embed(m,ids,pos0,C,hidden);
if(pos0==0&&C<=1) fprintf(stderr,"[DBG] step_chunk pos=%d C=%d metal=%d\n", pos0, C, g_k3_metal);
+ int echo_on = g_echo_k>0 && g_echo_id;
+#ifdef COLI_VULKAN
+ /* every layer on the device (kimi_k3_chain.h): the final rows (fin, when the head
+ * reads more than the last one) and the last row's logits (dev_last) come back;
+ * 0: the CPU below, its state current first */
+ float *fin=NULL, *dev_last=NULL; int chain=0;
+ if(g_k3c_on){
+ chain=k3c_forward(m,hidden,pos0,C,g_lfp||g_k3_val_lfp||echo_on,&fin,&dev_last,
+ poll_cancel,cancel_context,cancelled);
+ if(!chain) k3c_cpu_step(m,pos0);
+ }
+ if(!chain)
+#endif
k3_layers_forward_range(m,hidden,bres,&nb,pos0,C,0,c->n_layers,
poll_cancel,cancel_context,cancelled);
float *logits=NULL;
if((!cancelled||!*cancelled)&&m->has_head){
double t0=now_s();
- int echo_on = g_echo_k>0 && g_echo_id;
for(int t=0;t<C;t++){
/* head only where needed: the chunk's last token (feeds sampling)
* and every position when K3_LOGITS dumps teacher-forced logits;
* also all positions during Phase 10 validation; and every
* position when a request asked for the logprobs channel */
if(!g_lfp && !g_k3_val_lfp && !echo_on && t<C-1) continue;
+ float *lo=falloc(c->vocab);
+#ifdef COLI_VULKAN
+ if(chain){ /* the chain's final rows, normalized; the last row's logits from the device */
+ if(t==C-1&&dev_last) memcpy(lo,dev_last,(size_t)c->vocab*sizeof(float));
+ else w_matmul(lo,fin+(int64_t)t*D,&m->lm_head,1);
+ } else
+#endif
+ {
res_mix(mix,hidden+(int64_t)t*D,bres+(int64_t)t*nbmax*D,nb,D,m->out_sw,c->eps);
rmsnorm_(mix,mix,m->final_norm,D,c->eps);
if(m->trace) fwrite(mix,sizeof(float),D,m->trace);
- float *lo=falloc(c->vocab);
w_matmul(lo,mix,&m->lm_head,1);
+ }
if(g_lfp) fwrite(lo,sizeof(float),(size_t)c->vocab,g_lfp);
if(g_k3_val_lfp) fwrite(lo,sizeof(float),(size_t)c->vocab,g_k3_val_lfp);
if(echo_on){
@@ -2404,6 +2470,9 @@ static float *step_chunk_ex(Model *m, const int *ids, int pos0, int C,
/* record what was just fed, at the positions it went to (kv_prefix.h) */
if(!cancelled||!*cancelled) kv_prefix_record(&m->kvp,ids,pos0,C);
free(hidden);free(bres);free(mix);
+#ifdef COLI_VULKAN
+ free(fin);free(dev_last);
+#endif
return logits;
}
@@ -2775,6 +2844,9 @@ static const ColiServeWireProfile kimi_wire={
static void model_state_reset(Model *m){
Cfg *c=&m->c;
kv_prefix_clear(&m->kvp); /* the record describes the state we are dropping */
+#ifdef COLI_VULKAN
+ k3c_host_wrote(m,1); /* the dense chain's copy is zeros too, its MLA rows gone */
+#endif
for(int i=0;i<c->n_layers;i++){
if(m->L[i].kda){
memset(m->kstate[i],0,(size_t)c->kda_heads*c->kda_hd*c->kda_hd*sizeof(float));
@@ -2871,6 +2943,9 @@ static double k3_ckpt_reserve_gb(const Model *m, int slots){
static void k3_ckpt_copy(float *blob, Model *m, int to_blob){
Cfg *c=&m->c; size_t o=0;
+#ifdef COLI_VULKAN
+ if(to_blob) k3c_sync_host(m); /* the dense chain may hold the newest state */
+#endif
size_t ks=(size_t)c->kda_heads*c->kda_hd*c->kda_hd;
size_t cw=(size_t)c->kda_proj*c->conv_k;
for(int i=0;i<c->n_layers;i++) if(m->L[i].kda){
@@ -3031,6 +3106,9 @@ static int k3_ckpt_restore_best(Model *m, const int *ids, int np){
if(!k3_ckpt_disk_read(m,best,k->pos)){ k->pos=0; return 0; }
} else k3_ckpt_copy(k->blob,m,0);
m->kvp.len=k->pos; /* the record truncates with the state it describes */
+#ifdef COLI_VULKAN
+ k3c_host_wrote(m,0); /* the dense chain's copy goes up again */
+#endif
g_k3_restored_logit=k->logit;
k->used=++g_k3_ckpt_clock;
if(getenv("K3_PREFIX_LOG"))
diff --git a/c/kimi_k3_chain.h b/c/kimi_k3_chain.h
new file mode 100644
index 00000000..50d57c70
--- /dev/null
+++ b/c/kimi_k3_chain.h
@@ -0,0 +1,627 @@
+/* kimi_k3_chain.h -- Kimi K3's layers as a dense chain on the Vulkan device (vk_chain.h).
+ * Included once by kimi_k3.c in a COLI_VULKAN build, before step_chunk_ex, which calls
+ * k3c_forward in place of the CPU's layers; COLI_VK_CHAIN decides (coli_vk_chain_decide,
+ * this engine's integrated-GPU default COLI_VK_CHAIN_UNMEASURED: off, not measured on a
+ * Kimi K3 checkpoint).
+ *
+ * The residual is AttnRes: per row a running prefix and the block snapshots (one every
+ * attn_res_block_size layers), mixed by a softmax before the attention and before the
+ * MLP. What runs where, per layer, for one block of rows (decode: one row):
+ * device, frame A1: the previous layer's MoE output joins the prefix (the routed sum
+ * from the host: its RMSNorm and the latent up-projection, plus the shared experts,
+ * then the add: the CPU's order); the attention site's residual mix, a block
+ * boundary's snapshot, the input RMSNorm; then the KDA layer (q, k, v, the output
+ * gate, the decay's two f32 matrices and beta's; the short convolution with its
+ * window, the delta rule with its state, the output norm and gate, o_proj) or the
+ * gated MLA layer (vkc_mla_qkv: q_a, its norm, q_b, kv_a, the latent norm, the new
+ * rows into the cache; the gate; vkc_mla_attn: the absorbed core over the cache,
+ * the values times the gate, o_proj); the prefix update; the MLP site's residual
+ * mix and post-attention RMSNorm; then a dense layer's MLP (SiTU-GLU) and its add,
+ * with no host step, or the router's logits (the f32 router) and the latent
+ * down-projection. Then the frame is waited for.
+ * host: the router's sigmoid and top-k with the correction bias, K3_TOPP; the routed
+ * experts in the latent (the expert tier's batch and the CPU's share, joined in the
+ * union's order: k3_moe_experts); the new MLA rows into the host's cache.
+ * device, frame A2 (not waited for): the shared experts (SiTU-GLU at full width).
+ * after the last layer: the output residual mix and the final RMSNorm of every row,
+ * lm_head on the last one. The normalized rows come back when the host wants the
+ * logits of every row (K3_LOGITS, K3_VAL_LOGITS, a logprobs request), and the host
+ * runs lm_head on those as before.
+ * Crossing per sparse layer: the router logits (E floats a row) and the latent (latent
+ * floats a row) down, the new MLA rows down, the routed sum (latent floats a row) up.
+ *
+ * Kimi K3's MLA is NoPE: the query's and the shared key's qk_rope parts are used as
+ * they come out of the projections. vkc_mla_qkv rotates them with a table of cos 1 and
+ * sin 0, which is the identity in float (a*1 - b*0 = a).
+ *
+ * State and who owns it (one Model, the one k3c_start set up):
+ * - the MLA caches (Lc latent, Rc rope part): the host's stay canonical (every step
+ * copies its new rows back); the device mirrors them behind a watermark per layer
+ * that a CPU forward, a reset and a grown cache (kv_alloc) lower;
+ * - the KDA state and the three convolution windows of every KDA layer: on the
+ * device while the chain runs (too large to copy per token: about 6.9 MB a layer on
+ * the full model). `where` says which side holds the newest copy; the host's is
+ * brought back before anything reads it there (a checkpoint photo, a CPU forward)
+ * and pushed up after anything writes it there (a reset: a fill with zeros on the
+ * device; a restored photo: an upload).
+ * Prompt-cache and prefix reuse need nothing more: a reused prefix is rows below the
+ * watermark and a KDA state that already sits where the next step expects it.
+ *
+ * A device lost while it holds the newest KDA state: the state is rebuilt on the CPU
+ * from the host's copy (current at host_pos) and the prefix record's ids up to where
+ * the device was (a prefill's worth of CPU work), the forward runs again on the CPU
+ * from its input rows (the chain never writes them), and the CPU runs from there.
+ * COLI_VK_CHAIN_FAULT=n fakes the loss at the n-th frame.
+ *
+ * The chain declines (the CPU path runs, the state synced first): under the CUDA expert
+ * tier, with KIMI_DSA_INDEXER=1 (its index cache is filled on the CPU), for the
+ * validation dumps that read every layer on the host (K3_TRACE, K3_VALIDATE_LAYER,
+ * K3_DEBUG_OUT), without a head (K3_LAYERS), and for a geometry past the ops' limits
+ * (a KDA head above 128 floats, kv_lora above 1024, qk_rope above 128 or odd). */
+#include "vk_chain.h"
+
+#define K3C_HOST 0
+#define K3C_DEV 1
+#define K3C_BOTH 2
+
+typedef struct { ColiVkTensor *q, *k, *v, *g, *o, *fa, *fb, *bp; } K3cKda;
+typedef struct { ColiVkTensor *router, *down, *up, *sg, *su, *sd, *dg, *du, *dd; } K3cFfn;
+
+typedef struct {
+ const Model *m;
+ int ok, failed, rows, cap, nbmax;
+ VkcBuf *prm; /* every norm, AttnRes weight, conv tap and KDA parameter */
+ size_t *o_in, *o_post, *o_asw, *o_msw, *o_qn, *o_kn, *o_conv, *o_kda, *o_latn, o_osw, o_final;
+ K3cKda *kda; K3cFfn *ffn; VkcMla *mla; ColiVkTensor **mg, *head;
+ VkcMlaCache *kv; VkcBuf **win, **st;
+ int *kv_valid, *mla_ord, n_mla;
+ int where, host_zero, dev_pos, host_pos; /* the KDA state: who holds the newest, where each side is */
+ VkcMlaScratch sc;
+ VkcBuf *x, *bres, *hm, *nrm, *att, *qkv3, *cm, *t1, *kf, *kb, *kg, *ky, *gate, *lg, *z;
+ VkcBuf *gs, *us, *hs, *ds, *un, *lu, *fin, *cs;
+ VkcBuf *lgd, *zd, *kvd, *find, *outd, *routed;
+ size_t kvd_layer;
+ float *host_u, *host_sco;
+ unsigned long long forwards;
+ double host_ms;
+} K3Chain;
+
+static K3Chain *g_k3c;
+static int g_k3c_on; /* COLI_VK_CHAIN as decided (on, or prompts only), the chain set up */
+static int g_k3c_inited; /* vkc_init ran: vkc_shutdown goes at exit */
+
+static int k3c_chunk_rows(void) {
+ const char *e = getenv("COLI_VK_CHAIN_ROWS");
+ int v = e && *e ? atoi(e) : 512;
+ return v < 1 ? 1 : v > 65535 ? 65535 : v;
+}
+static int k3c_res(VkcBuf **b, size_t floats, int kind) { return vkc_reserve(b, (floats ? floats : 1) * sizeof(float), kind); }
+
+/* A W's device copy into *t: the per-matrix path's own where it made one (the shared
+ * experts under COLI_VK_DENSE), else one of the chain's, which w_matmul never sees (so
+ * a forward the chain declines stays on the CPU, as before). */
+static ColiVkTensor *k3c_w(W *w, ColiVkTensor **t) {
+ if (!*t && w->vk) *t = (ColiVkTensor *)w->vk;
+ if (*t) return *t;
+ int fmt = k3_vk_fmt(w);
+ if (fmt < 0 || w->mapped) return NULL;
+ const void *src = fmt == 1 ? (const void *)w->q8 : fmt == 4 ? (const void *)w->q4 : (const void *)w->f;
+ return coli_vk_tensor_ensure(t, src, fmt == 10 ? NULL : w->s, fmt, w->I, w->O, fmt == 4 ? w->gs : 0) ? *t : NULL;
+}
+/* the shared experts go where the per-matrix path puts them when it puts them on the
+ * device (COLI_VK_DENSE): one copy for both */
+static ColiVkTensor *k3c_w_shared(W *w, ColiVkTensor **t) {
+ if (!*t && coli_vk_dense() && !w->vk) w_vk_upload(w);
+ return k3c_w(w, t);
+}
+static ColiVkTensor *k3c_f32(const float *w, int I, int O, ColiVkTensor **own) {
+ return coli_vk_tensor_ensure(own, w, NULL, 10, I, O, 0) ? *own : NULL;
+}
+
+/* ---- the KDA state between the host and the device ------------------------------ */
+static void k3c_recover(Model *m, int dev, int upto);
+/* The host's KDA state brought up to date (before anything reads it there). */
+static void k3c_sync_host(Model *m) {
+ K3Chain *ch = g_k3c;
+ if (!ch || !ch->ok || ch->m != m || ch->where != K3C_DEV) return;
+ const Cfg *c = &m->c;
+ size_t ns = (size_t)c->kda_heads * c->kda_hd * c->kda_hd, nw = (size_t)c->kda_proj * c->conv_k;
+ int reads = 0; /* the copies that landed: after one, the host's copy is no longer the old one */
+ for (int i = 0; i < c->n_layers; i++) {
+ if (!m->L[i].kda) continue;
+ if (!(vkc_read(ch->st[i], 0, m->kstate[i], ns * sizeof(float)) && ++reads) ||
+ !(vkc_read(ch->win[i], 0, m->cwq[i], nw * sizeof(float)) && ++reads) ||
+ !(vkc_read(ch->win[i], nw, m->cwk[i], nw * sizeof(float)) && ++reads) ||
+ !(vkc_read(ch->win[i], 2 * nw, m->cwv[i], nw * sizeof(float)) && ++reads)) {
+ /* lost halfway: the host's copy is part old, part new; it is rebuilt from zeros */
+ if (reads) { ch->host_pos = 0; ch->host_zero = 1; }
+ k3c_recover(m, 1, ch->dev_pos);
+ return;
+ }
+ }
+ ch->where = K3C_BOTH; ch->host_pos = ch->dev_pos; ch->host_zero = 0;
+}
+/* The host wrote its KDA state (zero: model_state_reset's zeros; else a restored
+ * photo). A reset also drops the MLA rows: the mirror's watermark goes to 0. */
+static void k3c_host_wrote(Model *m, int zero) {
+ K3Chain *ch = g_k3c;
+ if (!ch || !ch->ok || ch->m != m) return;
+ ch->where = K3C_HOST; ch->host_zero = zero;
+ if (zero) for (int i = 0; i < m->c.n_layers; i++) ch->kv_valid[i] = 0;
+}
+/* A CPU forward from pos0: the host's state current before it, the device's stale after
+ * (and its MLA rows from pos0 on). */
+static void k3c_cpu_step(Model *m, int pos0) {
+ K3Chain *ch = g_k3c;
+ if (!ch || !ch->ok || ch->m != m) return;
+ k3c_sync_host(m);
+ ch->where = K3C_HOST; ch->host_zero = 0;
+ for (int i = 0; i < m->c.n_layers; i++) if (ch->kv_valid[i] > pos0) ch->kv_valid[i] = pos0;
+}
+
+/* The device was lost. If it held the newest KDA state (dev), the state is rebuilt on
+ * the CPU: from the host's copy, current at host_pos, through the prefix record's ids
+ * up to `upto`. The CPU runs from here on. */
+static void k3c_recover(Model *m, int dev, int upto) {
+ K3Chain *ch = g_k3c;
+ g_k3c_on = 0;
+ if (!ch) return;
+ ch->failed = 1;
+ int from = ch->host_pos, zero = ch->host_zero;
+ int had = dev && ch->where == K3C_DEV;
+ ch->where = K3C_HOST;
+ if (!had || upto <= from) {
+ fprintf(stderr, "[VK] kimi_k3 chain: the device was lost; the host's state is current, the CPU runs from here on\n");
+ return;
+ }
+ if (m->kvp.tainted || !m->kvp.fed || m->kvp.len < upto) {
+ fprintf(stderr, "[VK] kimi_k3 chain: the device was lost with a KDA state its token ids do not describe -- stopping "
+ "(COLI_VK_CHAIN=0 keeps the state on the CPU)\n");
+ exit(1);
+ }
+ fprintf(stderr, "[VK] kimi_k3 chain: the device was lost; rebuilding the state of %d positions on the CPU, "
+ "which runs from here on\n", upto - from);
+ const Cfg *c = &m->c;
+ if (zero) for (int i = 0; i < c->n_layers; i++) { /* the host was told "zeros" and kept its old arrays */
+ if (!m->L[i].kda) continue;
+ memset(m->kstate[i], 0, (size_t)c->kda_heads * c->kda_hd * c->kda_hd * sizeof(float));
+ memset(m->cwq[i], 0, (size_t)c->kda_proj * c->conv_k * sizeof(float));
+ memset(m->cwk[i], 0, (size_t)c->kda_proj * c->conv_k * sizeof(float));
+ memset(m->cwv[i], 0, (size_t)c->kda_proj * c->conv_k * sizeof(float));
+ }
+ int D = c->hidden, nbmax = ch->nbmax, chunk = 32;
+ float *h = falloc((int64_t)chunk * D), *bres = falloc((int64_t)chunk * nbmax * D);
+ for (int p = from; p < upto; p += chunk) {
+ int n = upto - p < chunk ? upto - p : chunk, nb = 0;
+ k3_embed(m, m->kvp.fed + p, p, n, h);
+ k3_layers_forward_range(m, h, bres, &nb, p, n, 0, c->n_layers, NULL, NULL, NULL);
+ }
+ free(h); free(bres);
+}
+
+/* ---- setup ----------------------------------------------------------------------- */
+static int k3c_setup(Model *m) {
+ const Cfg *c = &m->c;
+ int L = c->n_layers, D = c->hidden, P = c->kda_proj, LT = c->latent;
+ int nbmax = (L + c->res_bs - 1) / c->res_bs, any_kda = 0, any_mla = 0;
+ for (int i = 0; i < L; i++) { any_kda |= m->L[i].kda; any_mla |= !m->L[i].kda; }
+ const char *why = NULL;
+ if (!m->has_head) why = "no head (a layer range)";
+ else if (any_kda && (c->kda_hd > 128 || c->conv_k > 8)) why = "a KDA head its shaders do not take";
+ else if (any_mla && (c->kv_lora > 1024 || c->qk_rope > 128 || (c->qk_rope & 1) || c->qk_nope > 1024 || c->q_lora < 1))
+ why = "an attention geometry its shaders do not take";
+ else if (nbmax > 15) why = "more AttnRes blocks than its shader takes";
+ if (why) { fprintf(stderr, "[VK] kimi_k3 chain: %s; the CPU runs the layers\n", why); return 0; }
+ K3Chain *ch = calloc(1, sizeof *ch);
+ if (!ch) return 0;
+ size_t **offs[] = {&ch->o_in, &ch->o_post, &ch->o_asw, &ch->o_msw, &ch->o_qn, &ch->o_kn, &ch->o_conv, &ch->o_kda, &ch->o_latn};
+ for (size_t k = 0; k < sizeof offs / sizeof *offs; k++) if (!(*offs[k] = calloc(L, sizeof(size_t)))) return 0;
+ ch->kda = calloc(L, sizeof(K3cKda)); ch->ffn = calloc(L, sizeof(K3cFfn)); ch->mla = calloc(L, sizeof(VkcMla));
+ ch->mg = calloc(L, sizeof(void *)); ch->kv = calloc(L, sizeof(VkcMlaCache));
+ ch->win = calloc(L, sizeof(void *)); ch->st = calloc(L, sizeof(void *));
+ ch->kv_valid = calloc(L, sizeof(int)); ch->mla_ord = calloc(L, sizeof(int));
+ if (!ch->kda || !ch->ffn || !ch->mla || !ch->mg || !ch->kv || !ch->win || !ch->st || !ch->kv_valid || !ch->mla_ord) return 0;
+ ch->m = m; ch->nbmax = nbmax;
+ g_k3c = ch;
+ /* the parameter arena: offsets, then one upload */
+ size_t n = 0;
+ for (int i = 0; i < L; i++) {
+ const Layer *l = &m->L[i];
+ ch->o_in[i] = n; n += D; ch->o_post[i] = n; n += D; ch->o_asw[i] = n; n += D; ch->o_msw[i] = n; n += D;
+ if (l->kda) {
+ ch->o_conv[i] = n; n += (size_t)3 * P * c->conv_k;
+ ch->o_kda[i] = n; n += (size_t)c->kda_heads + P + c->kda_hd;
+ } else {
+ ch->mla_ord[i] = ch->n_mla++;
+ ch->o_qn[i] = n; n += c->q_lora; ch->o_kn[i] = n; n += c->kv_lora;
+ }
+ if (l->sparse) { ch->o_latn[i] = n; n += LT; }
+ }
+ ch->o_osw = n; n += D; ch->o_final = n; n += D;
+ float *a = calloc(n, sizeof(float));
+ if (!a) return 0;
+ for (int i = 0; i < L; i++) {
+ const Layer *l = &m->L[i];
+ memcpy(a + ch->o_in[i], l->in_ln, D * sizeof(float));
+ memcpy(a + ch->o_post[i], l->post_ln, D * sizeof(float));
+ memcpy(a + ch->o_asw[i], l->attn_sw, D * sizeof(float));
+ memcpy(a + ch->o_msw[i], l->mlp_sw, D * sizeof(float));
+ if (l->kda) {
+ size_t ck = (size_t)P * c->conv_k;
+ memcpy(a + ch->o_conv[i], l->a.conv_q, ck * sizeof(float));
+ memcpy(a + ch->o_conv[i] + ck, l->a.conv_k, ck * sizeof(float));
+ memcpy(a + ch->o_conv[i] + 2 * ck, l->a.conv_v, ck * sizeof(float));
+ memcpy(a + ch->o_kda[i], l->a.A, c->kda_heads * sizeof(float));
+ memcpy(a + ch->o_kda[i] + c->kda_heads, l->a.dt, P * sizeof(float));
+ memcpy(a + ch->o_kda[i] + c->kda_heads + P, l->a.onw, c->kda_hd * sizeof(float));
+ } else {
+ memcpy(a + ch->o_qn[i], l->m.qa_ln, c->q_lora * sizeof(float));
+ memcpy(a + ch->o_kn[i], l->m.kva_ln, c->kv_lora * sizeof(float));
+ }
+ if (l->sparse) memcpy(a + ch->o_latn[i], l->moe.lat_norm, LT * sizeof(float));
+ }
+ memcpy(a + ch->o_osw, m->out_sw, D * sizeof(float));
+ memcpy(a + ch->o_final, m->final_norm, D * sizeof(float));
+ ch->prm = vkc_buf(n * sizeof(float), VKC_DEV);
+ int ok = ch->prm && vkc_begin() && vkc_write(ch->prm, 0, a, n * sizeof(float)) && vkc_submit(1);
+ free(a);
+ /* the tensors */
+ for (int i = 0; i < L && ok; i++) {
+ Layer *l = &m->L[i];
+ if (l->kda) {
+ Kda *k = &l->a; K3cKda *t = &ch->kda[i];
+ ok = k3c_w(&k->q, &t->q) && k3c_w(&k->k, &t->k) && k3c_w(&k->v, &t->v) && k3c_w(&k->g, &t->g) &&
+ k3c_w(&k->o, &t->o) && k3c_f32(k->fa, D, c->kda_hd, &t->fa) && k3c_f32(k->fb, c->kda_hd, P, &t->fb) &&
+ k3c_f32(k->bp, D, c->kda_heads, &t->bp);
+ if (ok) {
+ ch->win[i] = vkc_buf((size_t)3 * P * c->conv_k * sizeof(float), VKC_DEV);
+ ch->st[i] = vkc_buf((size_t)c->kda_heads * c->kda_hd * c->kda_hd * sizeof(float), VKC_DEV);
+ ok = ch->win[i] && ch->st[i];
+ }
+ } else {
+ Mla *q = &l->m; VkcMla *t = &ch->mla[i];
+ /* kv_b as it is: per head Q key rows then V value rows, the absorption reads it transposed */
+ *t = (VkcMla){c->n_heads, c->qk_nope, c->qk_rope, c->v_head, c->kv_lora, D, c->q_lora, c->eps,
+ c->attn_scale, VKC_ROPE_HALF, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+ ch->prm, ch->o_qn[i], ch->o_kn[i]};
+ ok = k3c_w(&q->qa, &t->q_a) && k3c_w(&q->qb, &t->q_b) && k3c_w(&q->kva, &t->kv_a) && k3c_w(&q->kvb, &t->kv_b) &&
+ k3c_w(&q->o, &t->o) && k3c_w(&q->g, &ch->mg[i]);
+ }
+ K3cFfn *f = &ch->ffn[i];
+ if (ok && l->sparse) {
+ Moe *o = &l->moe;
+ ok = k3c_f32(o->router, D, c->n_experts, &f->router) && k3c_w(&o->lat_down, &f->down) &&
+ k3c_w(&o->lat_up, &f->up) && k3c_w_shared(&o->sh_gate, &f->sg) && k3c_w_shared(&o->sh_up, &f->su) &&
+ k3c_w_shared(&o->sh_down, &f->sd);
+ } else if (ok)
+ ok = k3c_w(&l->d_gate, &f->dg) && k3c_w(&l->d_up, &f->du) && k3c_w(&l->d_down, &f->dd);
+ }
+ ok = ok && k3c_w(&m->lm_head, &ch->head);
+ if (!ok) { fprintf(stderr, "[VK] kimi_k3 chain: a matrix did not reach the device (or has no device form); the CPU runs the layers\n"); return 0; }
+ ch->where = K3C_HOST; ch->host_zero = 1; /* the host's KDA state is the zeros it was allocated with */
+ ch->ok = 1;
+ int nkda = 0, nsparse = 0; for (int i = 0; i < L; i++) { nkda += m->L[i].kda; nsparse += m->L[i].sparse; }
+ fprintf(stderr, "[VK] kimi_k3 chain: %d layers on the device (%d KDA, %d MLA, %d dense MLP), %d AttnRes blocks, "
+ "%.1f MiB of parameters\n", L, nkda, ch->n_mla, L - nsparse, nbmax, n * 4 / 1048576.0);
+ return 1;
+}
+
+/* scratch for `rows` rows (grows, never shrinks) */
+static int k3c_scratch(K3Chain *ch, const Model *m, int rows) {
+ const Cfg *c = &m->c;
+ int D = c->hidden, P = c->kda_proj, H = c->kda_heads, LT = c->latent, E = c->n_experts, R = c->qk_rope;
+ int SI = c->moe_inter * c->n_shared, MI = SI > c->dense_inter ? SI : c->dense_inter;
+ int mfull = -1; for (int i = 0; i < c->n_layers; i++) if (!m->L[i].kda) { mfull = i; break; }
+ size_t r = (size_t)rows;
+ ch->kvd_layer = r * (c->kv_lora + R);
+ int ok = (mfull < 0 || vkc_mla_scratch(&ch->sc, &ch->mla[mfull], rows)) &&
+ k3c_res(&ch->x, r * D, VKC_DEV) && k3c_res(&ch->bres, r * ch->nbmax * D, VKC_DEV) &&
+ k3c_res(&ch->hm, r * D, VKC_DEV) && k3c_res(&ch->nrm, r * D, VKC_DEV) && k3c_res(&ch->att, r * D, VKC_DEV) &&
+ k3c_res(&ch->qkv3, 3 * r * P, VKC_DEV) && k3c_res(&ch->cm, r * 3 * P, VKC_DEV) &&
+ k3c_res(&ch->t1, r * c->kda_hd, VKC_DEV) && k3c_res(&ch->kf, r * P, VKC_DEV) && k3c_res(&ch->kb, r * H, VKC_DEV) &&
+ k3c_res(&ch->kg, r * P, VKC_DEV) && k3c_res(&ch->ky, r * P, VKC_DEV) &&
+ k3c_res(&ch->gate, r * c->n_heads * c->v_head, VKC_DEV) && k3c_res(&ch->lg, r * E, VKC_DEV) &&
+ k3c_res(&ch->z, r * LT, VKC_DEV) && k3c_res(&ch->gs, r * MI, VKC_DEV) && k3c_res(&ch->us, r * MI, VKC_DEV) &&
+ k3c_res(&ch->hs, r * MI, VKC_DEV) && k3c_res(&ch->ds, r * D, VKC_DEV) && k3c_res(&ch->un, r * LT, VKC_DEV) &&
+ k3c_res(&ch->lu, r * D, VKC_DEV) && k3c_res(&ch->fin, r * D, VKC_DEV) &&
+ k3c_res(&ch->lgd, r * E, VKC_DOWN) && k3c_res(&ch->zd, r * LT, VKC_DOWN) &&
+ k3c_res(&ch->kvd, (size_t)(ch->n_mla ? ch->n_mla : 1) * ch->kvd_layer, VKC_DOWN) &&
+ k3c_res(&ch->find, r * D, VKC_DOWN) && k3c_res(&ch->outd, c->vocab, VKC_DOWN) &&
+ k3c_res(&ch->routed, r * LT, VKC_UP);
+ if (ok && R > 0) { /* NoPE: the rope parts pass through a rotation by angle 0 */
+ size_t had = vkc_bytes(ch->cs) / sizeof(float);
+ ok = k3c_res(&ch->cs, r * R, VKC_UP);
+ if (ok && vkc_bytes(ch->cs) / sizeof(float) != had) {
+ float *cs = (float *)vkc_ptr(ch->cs);
+ size_t nf = vkc_bytes(ch->cs) / sizeof(float);
+ for (size_t i = 0; i + 1 < nf; i += 2) { cs[i] = 1.f; cs[i + 1] = 0.f; }
+ }
+ }
+ if (!ok) return 0;
+ if (ch->rows < rows) {
+ float *hu = realloc(ch->host_u, r * LT * sizeof(float)), *hs = hu ? realloc(ch->host_sco, r * E * sizeof(float)) : NULL;
+ if (hu) ch->host_u = hu;
+ if (hs) ch->host_sco = hs;
+ if (!hu || !hs) return 0;
+ ch->rows = rows;
+ }
+ return 1;
+}
+
+/* The MLA mirror at the host's capacity (m->max_t), filled again when that grows. */
+static int k3c_mirror(K3Chain *ch, const Model *m) {
+ const Cfg *c = &m->c;
+ if (ch->cap == m->max_t) return 1;
+ for (int i = 0; i < c->n_layers; i++) {
+ if (m->L[i].kda) continue;
+ vkc_free(ch->kv[i].lat); vkc_free(ch->kv[i].rope);
+ ch->kv[i] = (VkcMlaCache){NULL, NULL, 0};
+ ch->kv_valid[i] = 0;
+ }
+ ch->cap = 0;
+ for (int i = 0; i < c->n_layers; i++) {
+ if (m->L[i].kda) continue;
+ ch->kv[i].lat = vkc_buf((size_t)m->max_t * c->kv_lora * sizeof(float), VKC_DEV);
+ if (c->qk_rope > 0) ch->kv[i].rope = vkc_buf((size_t)m->max_t * c->qk_rope * sizeof(float), VKC_DEV);
+ ch->kv[i].cap = m->max_t;
+ if (!ch->kv[i].lat || (c->qk_rope > 0 && !ch->kv[i].rope)) return 0;
+ }
+ ch->cap = m->max_t;
+ return 1;
+}
+
+/* Record the uploads that make the device the host's below pb. */
+static int k3c_push(K3Chain *ch, const Model *m, int pb) {
+ const Cfg *c = &m->c; int ok = 1, K = c->kv_lora, R = c->qk_rope;
+ if (ch->where == K3C_HOST) {
+ size_t ns = (size_t)c->kda_heads * c->kda_hd * c->kda_hd, nw = (size_t)c->kda_proj * c->conv_k;
+ for (int i = 0; i < c->n_layers && ok; i++) {
+ if (!m->L[i].kda) continue;
+ if (ch->host_zero) ok = vkc_zero(ch->st[i], 0, ns) && vkc_zero(ch->win[i], 0, 3 * nw);
+ else ok = vkc_write(ch->st[i], 0, m->kstate[i], ns * sizeof(float)) &&
+ vkc_write(ch->win[i], 0, m->cwq[i], nw * sizeof(float)) &&
+ vkc_write(ch->win[i], nw, m->cwk[i], nw * sizeof(float)) &&
+ vkc_write(ch->win[i], 2 * nw, m->cwv[i], nw * sizeof(float));
+ }
+ ch->where = K3C_BOTH; ch->host_pos = pb;
+ }
+ for (int i = 0; i < c->n_layers && ok; i++) {
+ if (m->L[i].kda || ch->kv_valid[i] >= pb) continue;
+ int t0 = ch->kv_valid[i], n = pb - t0;
+ ok = vkc_write(ch->kv[i].lat, (size_t)t0 * K, m->Lc[i] + (size_t)t0 * K, (size_t)n * K * sizeof(float)) &&
+ (R == 0 || vkc_write(ch->kv[i].rope, (size_t)t0 * R, m->Rc[i] + (size_t)t0 * R, (size_t)n * R * sizeof(float)));
+ ch->kv_valid[i] = pb;
+ }
+ return ok;
+}
+/* the new MLA rows of layers [from, to) into the host's cache */
+static void k3c_pull(K3Chain *ch, Model *m, int from, int to, int pb, int n) {
+ const Cfg *c = &m->c; int K = c->kv_lora, R = c->qk_rope;
+ for (int i = from; i < to; i++) {
+ if (m->L[i].kda) continue;
+ const float *kv = (const float *)vkc_ptr(ch->kvd) + (size_t)ch->mla_ord[i] * ch->kvd_layer;
+ memcpy(m->Lc[i] + (size_t)pb * K, kv, (size_t)n * K * sizeof(float));
+ if (R) memcpy(m->Rc[i] + (size_t)pb * R, kv + (size_t)n * K, (size_t)n * R * sizeof(float));
+ }
+}
+
+/* ---- one layer's pieces ------------------------------------------------------------ */
+static int k3c_norm(VkcBuf *x, VkcBuf *w, size_t wo, VkcBuf *y, int rows, int D, float eps) {
+ VkcNorm p = {rows, D, 1, 0, D, D, 0, D, D, (int)wo, 0, 0, eps, 1.f};
+ return vkc_norm(x, w, y, &p);
+}
+static int k3c_mix(K3Chain *ch, const Model *m, size_t wo, int n, int nb) { /* res_mix(hm, x, bres) */
+ VkcAres a = {n, m->c.hidden, nb, 0, m->c.hidden, 0, ch->nbmax * m->c.hidden, (int)wo, 0, m->c.hidden, m->c.eps};
+ return vkc_ares_mix(ch->x, ch->bres, ch->prm, ch->hm, &a);
+}
+static int k3c_kda(K3Chain *ch, const Model *m, int i, int n) {
+ const Cfg *c = &m->c; int P = c->kda_proj, H = c->kda_heads, hd = c->kda_hd, r = ch->rows;
+ const K3cKda *t = &ch->kda[i];
+ VkcKdaConv cp = {n, 3 * P, c->conv_k, P, 0, P, r * P, 0, 3 * P, (int)ch->o_conv[i], 0};
+ VkcKdaRec rp = {n, H, hd, P, 0, 3 * P, 0, P, 0, H, 0, P, 0, P, 0, (int)ch->o_kda[i], c->gate_lb, 1e-6f, c->eps};
+ return vkc_matmul(t->q, ch->nrm, 0, ch->qkv3, 0, n) && vkc_matmul(t->k, ch->nrm, 0, ch->qkv3, (size_t)r * P, n) &&
+ vkc_matmul(t->v, ch->nrm, 0, ch->qkv3, (size_t)2 * r * P, n) && vkc_matmul(t->g, ch->nrm, 0, ch->kg, 0, n) &&
+ vkc_matmul(t->fa, ch->nrm, 0, ch->t1, 0, n) && vkc_matmul(t->fb, ch->t1, 0, ch->kf, 0, n) &&
+ vkc_matmul(t->bp, ch->nrm, 0, ch->kb, 0, n) &&
+ vkc_kda_conv(ch->qkv3, ch->prm, ch->win[i], ch->cm, &cp) &&
+ vkc_kda_rec_flags(hd, ch->cm, ch->kf, ch->kb, ch->kg, ch->prm, ch->st[i], ch->ky, &rp, VKC_KDA_EXP_A | VKC_KDA_K3) &&
+ vkc_matmul(t->o, ch->ky, 0, ch->att, 0, n);
+}
+static int k3c_mla(K3Chain *ch, const Model *m, int i, int n, int pb) {
+ size_t ko = (size_t)ch->mla_ord[i] * ch->kvd_layer;
+ return vkc_mla_qkv(&ch->mla[i], &ch->sc, ch->nrm, 0, n, pb, m->c.qk_rope > 0 ? ch->cs : NULL, &ch->kv[i], ch->kvd, ko) &&
+ vkc_matmul(ch->mg[i], ch->nrm, 0, ch->gate, 0, n) &&
+ vkc_mla_attn(&ch->mla[i], &ch->sc, n, pb, 0, &ch->kv[i], NULL, 0, 0, ch->gate, 0, ch->att, 0);
+}
+/* SiTU-GLU MLP of nrm into out: gate, up, the activation, down */
+static int k3c_glu(K3Chain *ch, const Model *m, ColiVkTensor *g, ColiVkTensor *u, ColiVkTensor *d, int inter, VkcBuf *out, int n) {
+ VkcSitu sp = {n * inter, 0, 0, 0, m->c.situ_b1, m->c.situ_b2};
+ return vkc_matmul(g, ch->nrm, 0, ch->gs, 0, n) && vkc_matmul(u, ch->nrm, 0, ch->us, 0, n) &&
+ vkc_situ(ch->gs, ch->us, ch->hs, &sp) && vkc_matmul(d, ch->hs, 0, out, 0, n);
+}
+/* the MoE output joining the prefix: x += lat_up(rmsnorm(routed)) + shared, moe_forward's order */
+static int k3c_join(K3Chain *ch, const Model *m, int i, int n) {
+ const Cfg *c = &m->c; int D = c->hidden, LT = c->latent;
+ VkcNorm un = {n, LT, 1, 0, LT, LT, 0, LT, LT, (int)ch->o_latn[i], 0, 0, c->eps, 1.f};
+ VkcEw cb = {VKC_EW_COMBINE, n * D, D, 1, 1 | 2, 1, 0, 0, 0, 0, 0, 1.f};
+ return vkc_norm(ch->routed, ch->prm, ch->un, &un) && vkc_matmul(ch->ffn[i].up, ch->un, 0, ch->lu, 0, n) &&
+ vkc_ew(ch->x, ch->x, ch->lu, ch->ds, NULL, &cb);
+}
+/* the host's MoE step of layer li: the router's choice from the device's logits, the
+ * routed experts on the device's latent rows, their sum into the upload buffer */
+static void k3c_moe_host(K3Chain *ch, Model *m, int li, int n) {
+ const Cfg *c = &m->c; int E = c->n_experts, K = c->topk, LT = c->latent;
+ Moe *o = &m->L[li].moe;
+ memcpy(ch->host_sco, vkc_ptr(ch->lgd), (size_t)n * E * sizeof(float));
+ int *idxs = malloc((size_t)n * K * sizeof(int)), *keff = malloc((size_t)n * sizeof(int));
+ float *wsels = falloc((int64_t)n * K);
+ if (!idxs || !keff) { fprintf(stderr, "OOM moe sel\n"); exit(1); }
+ k3_moe_route(m, o, li, ch->host_sco, n, idxs, wsels, keff);
+ memset(ch->host_u, 0, (size_t)n * LT * sizeof(float));
+ float *sd = k3_moe_experts(m, o, li, NULL, (const float *)vkc_ptr(ch->zd), n, keff, idxs, wsels, ch->host_u);
+ free(sd); free(idxs); free(keff); free(wsels);
+ memcpy(vkc_ptr(ch->routed), ch->host_u, (size_t)n * LT * sizeof(float));
+}
+
+/* Every layer for C rows of `hidden` (positions pos0..), then the output mix and the
+ * final norm: the rows into *fin (when need_rows), the last row's logits into *last.
+ * 0: not taken (the CPU runs the layers; nothing the caller holds changed). On a
+ * cancel (poll), 1 with *cancelled set: the state is the caller's to reset. */
+static int k3c_forward(Model *m, const float *hidden, int pos0, int C, int need_rows, float **fin_out,
+ float **last_out, K3CancelPoll poll, void *pctx, int *cancelled) {
+ K3Chain *ch = g_k3c;
+ if (!g_k3c_on || !ch || !ch->ok || ch->failed || ch->m != m) return 0;
+ if (g_k3c_on == COLI_VK_CHAIN_PREFILL && C <= 2) return 0; /* prompts only: decode on the CPU */
+ if (m->trace || g_k3_val_layer >= 0 || g_k3_dfp) {
+ static int said = 0;
+ if (!said++) fprintf(stderr, "[VK] kimi_k3 chain: a validation dump reads every layer on the host; the CPU runs the layers\n");
+ return 0;
+ }
+ const Cfg *c = &m->c; int L = c->n_layers, D = c->hidden, E = c->n_experts, LT = c->latent;
+ int CH = k3c_chunk_rows(), rows = C < CH ? C : CH;
+ int dev0 = ch->where == K3C_DEV, dev_start = ch->dev_pos;
+ if (vkc_lost()) { k3c_recover(m, dev0, dev_start); return 0; }
+ if (!k3c_mirror(ch, m) || !k3c_scratch(ch, m, rows)) {
+ if (vkc_lost()) { k3c_recover(m, dev0, dev_start); return 0; } /* lost while its buffers were made */
+ fprintf(stderr, "[VK] kimi_k3 chain: device memory for %d rows at %d positions refused; the CPU runs the layers\n",
+ rows, m->max_t);
+ k3c_cpu_step(m, pos0);
+ ch->failed = 1; g_k3c_on = 0;
+ return 0;
+ }
+ float *fin = need_rows ? falloc((int64_t)C * D) : NULL, *last = falloc(c->vocab);
+ vkc_gemm_rows(-1);
+ for (int c0 = 0; c0 < C; c0 += rows) {
+ int n = C - c0 < rows ? C - c0 : rows, pb = pos0 + c0, end = c0 + n == C;
+ if (!vkc_begin() || !k3c_push(ch, m, pb) || !vkc_write(ch->x, 0, hidden + (size_t)c0 * D, (size_t)n * D * sizeof(float)))
+ goto lost;
+ int ok = 1, pending = 0, pulled = 0, nb = 0;
+ for (int i = 0; i < L && ok; i++) {
+ const Layer *l = &m->L[i];
+ if (pending) { ok = k3c_join(ch, m, i - 1, n); pending = 0; }
+ int snap = i % c->res_bs == 0;
+ double ta = now_s();
+ /* the attention site: the mix over the snapshots so far, this block's snapshot */
+ VkcBuf *src = ch->x;
+ if (ok && nb > 0) { ok = k3c_mix(ch, m, ch->o_asw[i], n, nb); src = ch->hm; }
+ if (ok && snap) {
+ VkcRegion rg[64];
+ for (int s0 = 0; s0 < n && ok; s0 += 64) {
+ int k = n - s0 < 64 ? n - s0 : 64;
+ for (int s = 0; s < k; s++)
+ rg[s] = (VkcRegion){(size_t)(s0 + s) * ch->nbmax * D + (size_t)nb * D, (size_t)(s0 + s) * D, (size_t)D};
+ ok = vkc_copy_regions(ch->bres, ch->x, rg, k);
+ }
+ }
+ ok = ok && k3c_norm(src, ch->prm, ch->o_in[i], ch->nrm, n, D, c->eps);
+ int have_prefix = !snap;
+ if (snap) nb++;
+ ok = ok && (l->kda ? k3c_kda(ch, m, i, n) : k3c_mla(ch, m, i, n, pb));
+ if (ok) {
+ VkcEw add = {VKC_EW_ADD, n * D, D, 1, 0, 1, 0, 0, 0, 0, 0, 1.f};
+ ok = have_prefix ? vkc_ew(ch->x, ch->x, ch->att, NULL, NULL, &add) : vkc_copy(ch->x, 0, ch->att, 0, (size_t)n * D);
+ }
+ /* the MLP site */
+ ok = ok && k3c_mix(ch, m, ch->o_msw[i], n, nb) && k3c_norm(ch->hm, ch->prm, ch->o_post[i], ch->nrm, n, D, c->eps);
+ if (!ok) break;
+ if (!l->sparse) { /* a dense layer: its MLP and the add, no host step */
+ VkcEw add = {VKC_EW_ADD, n * D, D, 1, 0, 1, 0, 0, 0, 0, 0, 1.f};
+ const K3cFfn *f = &ch->ffn[i];
+ ok = k3c_glu(ch, m, f->dg, f->du, f->dd, c->dense_inter, ch->att, n) &&
+ vkc_ew(ch->x, ch->x, ch->att, NULL, NULL, &add);
+ m->t_attn += now_s() - ta;
+ continue;
+ }
+ ok = vkc_matmul(ch->ffn[i].router, ch->nrm, 0, ch->lg, 0, n) && vkc_matmul(ch->ffn[i].down, ch->nrm, 0, ch->z, 0, n) &&
+ vkc_copy(ch->lgd, 0, ch->lg, 0, (size_t)n * E) && vkc_copy(ch->zd, 0, ch->z, 0, (size_t)n * LT) &&
+ vkc_submit(1); /* A1 */
+ m->t_attn += now_s() - ta;
+ if (!ok) break;
+ k3c_pull(ch, m, pulled, i + 1, pb, n); pulled = i + 1;
+ /* A2: the shared experts, while the host computes the routed ones */
+ const K3cFfn *f = &ch->ffn[i];
+ ok = vkc_begin() && k3c_glu(ch, m, f->sg, f->su, f->sd, c->moe_inter * c->n_shared, ch->ds, n) && vkc_submit(0);
+ double t1 = now_s();
+ k3c_moe_host(ch, m, i, n);
+ double dt = now_s() - t1; ch->host_ms += dt * 1e3; m->t_moe += dt;
+ if (ok && poll && poll(pctx)) { /* a cancel: what the device advanced is the caller's to reset */
+ if (cancelled) *cancelled = 1;
+ vkc_finish();
+ ch->where = K3C_DEV; ch->dev_pos = pb + n;
+ free(fin); free(last);
+ return 1;
+ }
+ ok = ok && vkc_begin();
+ pending = 1;
+ }
+ if (ok && pending) ok = k3c_join(ch, m, L - 1, n);
+ /* the output mix and the final norm of every row, lm_head on the last */
+ ok = ok && k3c_mix(ch, m, ch->o_osw, n, nb) && k3c_norm(ch->hm, ch->prm, ch->o_final, ch->fin, n, D, c->eps);
+ if (ok && need_rows) ok = vkc_copy(ch->find, 0, ch->fin, 0, (size_t)n * D);
+ if (ok && end) ok = vkc_matmul(ch->head, ch->fin, (size_t)(n - 1) * D, ch->outd, 0, 1);
+ ok = ok && vkc_submit(1);
+ if (!ok) goto lost;
+ k3c_pull(ch, m, pulled, L, pb, n);
+ if (need_rows) memcpy(fin + (size_t)c0 * D, vkc_ptr(ch->find), (size_t)n * D * sizeof(float));
+ for (int i = 0; i < L; i++) if (!m->L[i].kda) ch->kv_valid[i] = pb + n;
+ ch->where = K3C_DEV;
+ if (end) memcpy(last, vkc_ptr(ch->outd), (size_t)c->vocab * sizeof(float));
+ }
+ ch->dev_pos = pos0 + C;
+ ch->forwards++;
+ *fin_out = fin; *last_out = last;
+ return 1;
+lost: /* a frame failed: the device is gone (or would not take a command); the CPU takes over */
+ free(fin); free(last);
+ if (!vkc_lost()) { vkc_finish(); coli_vk_mark_lost(); }
+ ch->where = dev0 ? K3C_DEV : K3C_HOST; /* the state as it was when this forward began */
+ k3c_recover(m, dev0, dev_start);
+ return 0;
+}
+
+static void k3c_report(void) {
+ K3Chain *ch = g_k3c;
+ if (!ch || !ch->ok || !ch->forwards) return;
+ VkcStats st; vkc_stats(&st);
+ fprintf(stderr, "[VK] kimi_k3 chain: %llu forwards, %llu frames (%llu ops, %llu matmuls, %llu tiled GEMM), "
+ "%.1f ms waiting for the device, %.1f ms of routed experts on the host, %.1f MiB on the device\n",
+ ch->forwards, st.frames, st.ops, st.matmuls, st.gemms, st.wait_ms, ch->host_ms, st.dev_bytes / 1048576.0);
+ vkc_prof_print();
+}
+
+/* COLI_VK_CHAIN at startup, before the tier sizes itself: the decision, the pipelines,
+ * the tensors. tier_on: whether the routed-expert tier is about to start. */
+static void k3c_start(Model *m, int tier_on) {
+ if (!g_k3_vk) return;
+ int on = coli_vk_chain_decide("kimi_k3", tier_on, COLI_VK_CHAIN_UNMEASURED);
+ const char *no = NULL;
+ if (on) {
+#ifdef COLI_CUDA
+ if (g_k3_cuda) no = "the CUDA expert tier is on and keeps its priority";
+#endif
+ if (!no && k3_dsa_indexer_on()) no = "KIMI_DSA_INDEXER=1 fills its index cache on the CPU";
+ if (!no && !(g_k3c_inited = vkc_init())) no = "the chain's pipelines did not come up";
+ if (!no && !(vkc_mla_ready() && vkc_kda_ready() && vkc_ares_ready())) no = "the MLA, KDA or AttnRes shaders are missing";
+ }
+ if (no) fprintf(stderr, "[VK] kimi_k3: %s: the dense chain stays off\n", no);
+ if (!on || no) return;
+ if (!k3c_setup(m)) return;
+ g_k3c_on = on;
+}
+/* After the tier's: at exit the chain goes before the device. */
+static void k3c_atexit(void) {
+ if (g_k3c_inited) atexit(vkc_shutdown);
+}
+/* The per-matrix path is about to put the shared experts on the device (no tier after
+ * all): the chain's copies become its own. */
+static void k3c_adopt_shared(Model *m) {
+ K3Chain *ch = g_k3c;
+ if (!ch || !ch->ok || ch->m != m) return;
+ for (int i = 0; i < m->c.n_layers; i++) {
+ if (!m->L[i].sparse) continue;
+ Moe *o = &m->L[i].moe; K3cFfn *f = &ch->ffn[i];
+ if (!o->sh_gate.vk) o->sh_gate.vk = f->sg;
+ if (!o->sh_up.vk) o->sh_up.vk = f->su;
+ if (!o->sh_down.vk) o->sh_down.vk = f->sd;
+ }
+}