Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(vulkan): kimi_k3 runs a partial chain: the first N layers on the device

JustVugg177e39a08d0a
diff --git a/c/deepseek_v41_chain.h b/c/deepseek_v41_chain.h
index 06d571c0..044addbf 100644
--- a/c/deepseek_v41_chain.h
+++ b/c/deepseek_v41_chain.h
@@ -409,8 +409,11 @@ static int v41c_setup(Model *m) {
ch->ok = 1;
int nsrc = 0, nidx = 0, neng = 0;
for (int i = 0; i < N; i++) { nsrc += c->kv_source[i]; nidx += c->index_source[i] && c->compress_ratio[i] > 0; neng += m->L[i].engram_index >= 0; }
+ size_t bytes = 0, tensors = 0;
+ coli_vk_mem_info(&bytes, &tensors);
fprintf(stderr, "[VK] deepseek_v41 chain: %d layers on the device (%d compressing, %d indexing, %d engram), %d streams, "
- "%.1f MiB of parameters\n", N, nsrc, nidx, neng, H, n * 4 / 1048576.0);
+ "%.1f MiB of parameters, %zu matrices (%.1f MiB) on the device\n", N, nsrc, nidx, neng, H, n * 4 / 1048576.0,
+ tensors, bytes / 1048576.0);
vkc_fit_placed("deepseek_v41", &g_v41_fit);
return 1;
}
diff --git a/c/kimi_k3.c b/c/kimi_k3.c
index ccb1cc34..e643dc1e 100644
--- a/c/kimi_k3.c
+++ b/c/kimi_k3.c
@@ -409,7 +409,17 @@ static int dsa_score_single(int *sel, int maxsel, const float *qi, const float *
* before the shared one; k3_vk_aliases() reads them into the shared names. */
#ifdef COLI_VULKAN
#include "backend_vulkan.h"
+#include "vk_chain.h" /* vkc_fit: how many layers the dense chain places (a partial chain) */
static int g_k3_vk=0; /* the device is open */
+/* The partial chain (kimi_k3_chain.h, k3c_fit_now): the first g_k3_fit.n layers on the
+ * device, decided once before anything goes up and before the expert cache is sized.
+ * g_k3_partial: something the full chain would place stays on the CPU (layers, the head),
+ * so nothing more goes to the device for the per-matrix path. g_k3_dho_layers and
+ * g_k3_dho_head: what COLI_VK_DENSE_HOST takes off the host (the chain's layers, and the
+ * head only with every layer and the tail on the device). */
+static VkcFit g_k3_fit;
+static int g_k3_fit_done, g_k3_partial;
+static int g_k3_dho_layers=1<<30, g_k3_dho_head=1;
/* COLI_VK_DENSE_HOST: the dense matrices on the device only (k3_dho_*): every W the
* chain does not run goes to the device through w_matmul, prompts included */
static int g_k3_dho=0;
@@ -791,22 +801,23 @@ static int k3_f32_dev(void *t, float *y, const float *x, int S, int I, int O){
coli_vk_matmul((ColiVkTensor**)&t,y,x,NULL,NULL,10,S,I,O,0);
}
#define K3_F32_DEV(t,y,x,S,I,O) k3_f32_dev(t,y,x,S,I,O)
-/* Every W of the model, in a fixed order. */
+/* Every W of the model, in a fixed order (with a partial chain, its layers' and the head
+ * only when it goes to the device as well). */
static void k3_dho_each(Model *m, void (*f)(W *, size_t *, int *), size_t *b, int *n){
- for(int i=0;i<m->c.n_layers;i++){
+ for(int i=0;i<m->c.n_layers&&i<g_k3_dho_layers;i++){
Layer *l=&m->L[i];
if(l->kda){ Kda *a=&l->a; W *ws[]={&a->q,&a->k,&a->v,&a->g,&a->o}; for(size_t k=0;k<5;k++) f(ws[k],b,n); }
else { Mla *a=&l->m; W *ws[]={&a->qa,&a->qb,&a->kva,&a->kvb,&a->o,&a->g,&a->wk,&a->wq,&a->wp}; for(size_t k=0;k<9;k++) f(ws[k],b,n); }
if(l->sparse){ Moe *o=&l->moe; W *ws[]={&o->lat_down,&o->lat_up,&o->sh_gate,&o->sh_up,&o->sh_down}; for(size_t k=0;k<5;k++) f(ws[k],b,n); }
else { W *ws[]={&l->d_gate,&l->d_up,&l->d_down}; for(size_t k=0;k<3;k++) f(ws[k],b,n); }
}
- if(m->has_head) f(&m->lm_head,b,n);
+ if(m->has_head&&g_k3_dho_head) f(&m->lm_head,b,n);
}
static void k3_dho_count(W *w, size_t *b, int *n){ size_t x=k3_w_host_bytes(w); if(x&&!w->mapped&&k3_vk_fmt(w)>0){ *b+=x; (*n)++; } }
/* The f32 matrices the chain multiplies on the device: the routers, the KDA decay pair and beta. */
static size_t k3_dho_f32_bytes(const Model *m){
const Cfg *c=&m->c; size_t b=0;
- for(int i=0;i<c->n_layers;i++){
+ for(int i=0;i<c->n_layers&&i<g_k3_dho_layers;i++){
const Layer *l=&m->L[i];
if(l->sparse&&l->moe.router) b+=(size_t)c->n_experts*c->hidden*4;
if(l->kda&&l->a.fa) b+=((size_t)c->kda_hd*c->hidden+(size_t)c->kda_proj*c->kda_hd+(size_t)c->kda_heads*c->hidden)*4;
@@ -817,12 +828,20 @@ static size_t k3_dho_f32_bytes(const Model *m){
* chain's own decision asked silently (k3c_start prints it later). The cache counts the
* W matrices out of RAM; the f32 ones only leave once the chain has them, so it does not
* count on those. */
+static int k3c_fit_now(Model *m, int tier_on, int cuda_on); /* kimi_k3_chain.h */
static void k3_dho_decide(Model *m, int tier_on, int cuda_on){
if(!g_k3_vk) return;
int chain=!cuda_on&&!k3_dsa_indexer_on()&&coli_vk_chain_decide(NULL,tier_on,COLI_VK_CHAIN_UNMEASURED)!=COLI_VK_CHAIN_OFF;
+ int dense_on=coli_vk_dense()||chain;
+ if(k3c_fit_now(m,tier_on,cuda_on)){ /* a partial chain: its layers, and the head with the tail */
+ g_k3_dho_layers=g_k3_fit.n; g_k3_dho_head=g_k3_fit.n==m->c.n_layers&&g_k3_fit.tail;
+ chain=g_k3_fit.n>0; dense_on=chain||(coli_vk_dense()&&!g_k3_partial);
+ }
size_t b=0; int n=0;
k3_dho_each(m,k3_dho_count,&b,&n);
- if(!coli_vk_dense_host_decide("kimi_k3",coli_vk_dense()||chain,b+(chain?k3_dho_f32_bytes(m):0))) return;
+ int on=coli_vk_dense_host_decide("kimi_k3",dense_on,b+(chain?k3_dho_f32_bytes(m):0));
+ if(g_k3_fit.L>0) coli_vk_dense_host_layers(g_k3_fit.n,m->c.n_layers);
+ if(!on) return;
g_k3_dho=1; g_k3_dho_model=m; g_k3_dho_bytes=b;
}
#else
@@ -2131,7 +2150,7 @@ static void k3_dho_place(Model *m){
k3_dho_each(m,k3_dho_drop,&b,&n);
if(g_k3c_on){
const Cfg *c=&m->c; char nm[200];
- for(int i=0;i<c->n_layers;i++){
+ for(int i=0;i<c->n_layers&&i<g_k3_dho_layers;i++){ /* the chain's layers: it holds their f32 matrices */
Layer *l=&m->L[i];
if(l->sparse){
snprintf(nm,sizeof nm,"model.layers.%d.block_sparse_moe.gate.weight",i);
@@ -2161,8 +2180,11 @@ static void k3_vk_tier_start(Model *m){
#ifdef COLI_CUDA
cuda_on=g_k3_cuda;
#endif
- if(g_k3_dho){ size_t b=0; int n=0; k3_dho_each(m,k3_dho_upload,&b,&n); } /* COLI_VK_DENSE_HOST: every W into its own vk first */
+ /* COLI_VK_DENSE_HOST: every W into its own vk first (with the chain's fit, its setup
+ * puts each of its layers' there, layer by layer) */
+ if(g_k3_dho&&g_k3_fit.L<1){ size_t b=0; int n=0; k3_dho_each(m,k3_dho_upload,&b,&n); }
k3c_start(m,nmoe&&vkt_wanted()&&!cuda_on); /* COLI_VK_CHAIN: the trunk on the device, before the tier sizes itself */
+ if(g_k3_dho&&g_k3_fit.L>0&&!g_k3c_on){ size_t b=0; int n=0; k3_dho_each(m,k3_dho_upload,&b,&n); } /* no chain after all: the per-matrix path's */
if(g_k3_dho) k3_dho_place(m); /* then the host copies go, before the tier sizes its budget */
if(cuda_on) fprintf(stderr,"[VK] tier kimi_k3: the CUDA expert tier is on and wins; the Vulkan tier stays off\n");
else if(nmoe&&vkt_wanted()){
@@ -2172,7 +2194,7 @@ static void k3_vk_tier_start(Model *m){
.act=VKT_ACT_SITU,.act_a=c->situ_b1,.act_b=c->situ_b2,
.max_rows=K3_VK_ROWS*c->topk,
.ram_reserve=(size_t)(m->e_slot+8192)*(size_t)cap*(size_t)nmoe,
- .dense_bytes=coli_vk_dense()&&!g_k3c_on&&!g_k3_dho?k3_vk_dense_bytes(m):0, /* the chain's are there already */
+ .dense_bytes=coli_vk_dense()&&!g_k3c_on&&!g_k3_dho&&!g_k3_partial?k3_vk_dense_bytes(m):0, /* the chain's are there already; a partial chain: nothing more goes up */
.in_ram=k3_vk_in_ram,.ram_ctx=m,
.load=k3_vk_load,.release=k3_vk_unhold,.load_ctx=m,.load_batch=k3_vk_load_batch};
atexit(coli_vk_shutdown); /* before vkt_init, which makes the expert batch's pipelines and can still refuse (no room): the device goes at exit either way, after the tier's teardown */
@@ -2206,7 +2228,9 @@ static void k3_vk_tier_start(Model *m){
for(int i=0;i<c->n_layers;i++){
if(!m->L[i].sparse) continue;
Moe *o=&m->L[i].moe;
- nsh+=w_vk_upload(&o->sh_gate)+w_vk_upload(&o->sh_up)+w_vk_upload(&o->sh_down);
+ if(g_k3_partial) /* a partial chain: nothing new goes up, the CPU's layers keep theirs */
+ nsh+=(o->sh_gate.vk!=NULL)+(o->sh_up.vk!=NULL)+(o->sh_down.vk!=NULL);
+ else nsh+=w_vk_upload(&o->sh_gate)+w_vk_upload(&o->sh_up)+w_vk_upload(&o->sh_down);
}
size_t used=0, count=0; coli_vk_mem_info(&used,&count);
fprintf(stderr,"[VK] kimi_k3: %d shared-expert matrices on the device (%.2f MiB)\n",nsh,used/1048576.0);
@@ -2216,6 +2240,8 @@ static void k3_vk_tier_start(Model *m){
static void k3_vk_report(const Model *m, const char *scope){
if(!g_k3_vk) return;
fprintf(stderr,"[VK] kimi_k3: %llu matmuls on the GPU\n",coli_vk_matmul_calls());
+ { size_t bytes=0, tensors=0; coli_vk_mem_info(&bytes,&tensors);
+ fprintf(stderr,"[VK] kimi_k3: %zu resident matrices on the device, %.1f MiB\n",tensors,bytes/1048576.0); }
vkt_report(scope,m->hits,m->miss);
k3c_report(); /* the dense chain's line, when it ran */
}
@@ -2677,20 +2703,24 @@ static float *step_chunk_ex(Model *m, const int *ids, int pos0, int C,
k3_embed(m,ids,pos0,C,hidden);
if(pos0==0&&C<=1) fprintf(stderr,"[DBG] step_chunk pos=%d C=%d metal=%d\n", pos0, C, g_k3_metal);
int echo_on = g_echo_k>0 && g_echo_id;
+ int from=0; /* the first layer the CPU runs */
#ifdef COLI_VULKAN
- /* every layer on the device (kimi_k3_chain.h): the final rows (fin, when the head
- * reads more than the last one) and the last row's logits (dev_last) come back;
- * 0: the CPU below, its state current first */
+ /* the layers on the device (kimi_k3_chain.h). Every layer: the final rows (fin, when
+ * the head reads more than the last one) and the last row's logits (dev_last) come
+ * back. A partial chain's first N: the rows' AttnRes state after layer N - 1 comes
+ * back in hidden, bres and nb, and the CPU runs the rest from there. 0: the CPU runs
+ * every layer below, its state current first */
float *fin=NULL, *dev_last=NULL; int chain=0;
if(g_k3c_on){
- chain=k3c_forward(m,hidden,pos0,C,g_lfp||g_k3_val_lfp||echo_on,&fin,&dev_last,
+ chain=k3c_forward(m,hidden,bres,&nb,pos0,C,g_lfp||g_k3_val_lfp||echo_on,&fin,&dev_last,
poll_cancel,cancel_context,cancelled);
if(!chain) k3c_cpu_step(m,pos0);
}
- if(!chain)
+ from=chain;
#endif
- k3_layers_forward_range(m,hidden,bres,&nb,pos0,C,0,c->n_layers,
- poll_cancel,cancel_context,cancelled);
+ if(from<c->n_layers&&!(cancelled&&*cancelled))
+ k3_layers_forward_range(m,hidden,bres,&nb,pos0,C,from,c->n_layers,
+ poll_cancel,cancel_context,cancelled);
float *logits=NULL;
if((!cancelled||!*cancelled)&&m->has_head){
double t0=now_s();
@@ -2702,7 +2732,7 @@ static float *step_chunk_ex(Model *m, const int *ids, int pos0, int C,
if(!g_lfp && !g_k3_val_lfp && !echo_on && t<C-1) continue;
float *lo=falloc(c->vocab);
#ifdef COLI_VULKAN
- if(chain){ /* the chain's final rows, normalized; the last row's logits from the device */
+ if(chain==c->n_layers){ /* the chain's final rows, normalized; the last row's logits from the device */
if(t==C-1&&dev_last) memcpy(lo,dev_last,(size_t)c->vocab*sizeof(float));
else w_matmul(lo,fin+(int64_t)t*D,&m->lm_head,1);
} else
diff --git a/c/kimi_k3_chain.h b/c/kimi_k3_chain.h
index cd7a38fa..0a64b817 100644
--- a/c/kimi_k3_chain.h
+++ b/c/kimi_k3_chain.h
@@ -93,6 +93,8 @@ typedef struct {
float *host_u, *host_sco;
unsigned long long forwards;
double host_ms;
+ int n; /* the layers on the device: the model's first n (a partial chain) */
+ VkcBuf *xo, *bo; /* the handoff after layer n - 1: the prefix and the block snapshots, down */
} K3Chain;
static K3Chain *g_k3c;
@@ -127,7 +129,7 @@ static int k3c_chunk_rows(K3Chain *ch, const Model *m) {
if (!vkc_chunk_auto()) return vkc_chunk_rows("kimi_k3", 0);
const Cfg *c = &m->c;
size_t mla = 0;
- for (int i = 0; i < c->n_layers; i++) if (!m->L[i].kda) {
+ for (int i = 0; i < ch->n; i++) if (!m->L[i].kda) {
const VkcMla *a = &ch->mla[i];
mla = (size_t)(a->q_lora > 0 ? a->q_lora : 1) + (size_t)a->H * (a->Q + a->R) + (size_t)(a->K + a->R) +
2 * (size_t)a->H * a->K + (size_t)a->H * a->V;
@@ -147,6 +149,9 @@ static int k3c_chunk_rows(K3Chain *ch, const Model *m) {
static ColiVkTensor *k3c_w(W *w, ColiVkTensor **t) {
if (!*t && w->vk) *t = (ColiVkTensor *)w->vk;
if (*t) return *t;
+ /* the dense weights on the device only, the chain's fit: the per-matrix path's copy,
+ * made here layer by layer (k3_vk_tier_start leaves the chain's layers to it) */
+ if (g_k3_dho && g_k3_fit.L > 0 && w_vk_upload(w)) return *t = (ColiVkTensor *)w->vk;
int fmt = k3_vk_fmt(w);
if (fmt < 0 || w->mapped) return NULL;
const void *src = fmt == 1 ? (const void *)w->q8 : fmt == 4 ? (const void *)w->q4 : (const void *)w->f;
@@ -171,7 +176,7 @@ static void k3c_sync_host(Model *m) {
const Cfg *c = &m->c;
size_t ns = (size_t)c->kda_heads * c->kda_hd * c->kda_hd, nw = (size_t)c->kda_proj * c->conv_k;
int reads = 0; /* the copies that landed: after one, the host's copy is no longer the old one */
- for (int i = 0; i < c->n_layers; i++) {
+ for (int i = 0; i < ch->n; i++) { /* the chain's layers: the CPU's hold their own */
if (!m->L[i].kda) continue;
if (!(vkc_read(ch->st[i], 0, m->kstate[i], ns * sizeof(float)) && ++reads) ||
!(vkc_read(ch->win[i], 0, m->cwq[i], nw * sizeof(float)) && ++reads) ||
@@ -200,7 +205,7 @@ static void k3c_cpu_step(Model *m, int pos0) {
if (!ch || !ch->ok || ch->m != m) return;
k3c_sync_host(m);
ch->where = K3C_HOST; ch->host_zero = 0;
- for (int i = 0; i < m->c.n_layers; i++) {
+ for (int i = 0; i < ch->n; i++) {
if (ch->kv_valid[i] > pos0) ch->kv_valid[i] = pos0;
if (!m->L[i].kda) vkc_kv_lower(&ch->ks, ch->mla_ord[i], pos0);
}
@@ -229,7 +234,7 @@ static void k3c_recover(Model *m, int dev, int upto) {
fprintf(stderr, "[VK] kimi_k3 chain: the device was lost; rebuilding the state of %d positions on the CPU, "
"which runs from here on\n", upto - from);
const Cfg *c = &m->c;
- if (zero) for (int i = 0; i < c->n_layers; i++) { /* the host was told "zeros" and kept its old arrays */
+ if (zero) for (int i = 0; i < ch->n; i++) { /* the host was told "zeros" and kept its old arrays (the chain's layers) */
if (!m->L[i].kda) continue;
memset(m->kstate[i], 0, (size_t)c->kda_heads * c->kda_hd * c->kda_hd * sizeof(float));
memset(m->cwq[i], 0, (size_t)c->kda_proj * c->conv_k * sizeof(float));
@@ -241,24 +246,194 @@ static void k3c_recover(Model *m, int dev, int upto) {
for (int p = from; p < upto; p += chunk) {
int n = upto - p < chunk ? upto - p : chunk, nb = 0;
k3_embed(m, m->kvp.fed + p, p, n, h);
- k3_layers_forward_range(m, h, bres, &nb, p, n, 0, c->n_layers, NULL, NULL, NULL);
+ k3_layers_forward_range(m, h, bres, &nb, p, n, 0, ch->n, NULL, NULL, NULL); /* the chain's layers: the CPU's are current */
}
free(h); free(bres);
}
/* ---- setup ----------------------------------------------------------------------- */
+/* ---- the partial chain (vk_chain.h, vkc_fit): the first N layers on the device ----------
+ * N is decided once, before anything goes up and before the expert cache is sized
+ * (k3c_fit_now, from k3_dho_decide once the device is open): the layers that fit the
+ * device's free memory less the tier's reserve, the chain's scratch for one prompt chunk
+ * and the pools' granularity; the head is the tail. The CPU runs layers N.. and the head:
+ * each row's AttnRes state after layer N - 1 (the prefix and the block snapshots) crosses
+ * to the host once per forward (k3c_forward). The chain's layers keep their KDA state and
+ * MLA mirrors on the device as before; the CPU's layers keep theirs on the host. */
+static const char *k3c_unsupported(const Model *m) {
+ const Cfg *c = &m->c;
+ int L = c->n_layers, nbmax = (L + c->res_bs - 1) / c->res_bs, any_kda = 0, any_mla = 0;
+ for (int i = 0; i < L; i++) { any_kda |= m->L[i].kda; any_mla |= !m->L[i].kda; }
+ if (!m->has_head) return "no head (a layer range)";
+ if (any_kda && (c->kda_hd > 128 || c->conv_k > 8)) return "a KDA head its shaders do not take";
+ if (any_mla && (c->kv_lora > 1024 || c->qk_rope > 128 || (c->qk_rope & 1) || c->qk_nope > 1024 || c->q_lora < 1))
+ return "an attention geometry its shaders do not take";
+ if (nbmax > 15) return "more AttnRes blocks than its shader takes";
+ return NULL;
+}
+/* a W as k3c_w puts it on the device: its bytes there and its payload */
+static void k3c_fit_w(const W *w, size_t *b, size_t *mm) {
+ int fmt = k3_vk_fmt(w);
+ if (fmt < 0 || w->mapped || !(w->f || w->q8 || w->q4) || w->O < 1 || w->I < 1) return;
+ int gs = fmt == 4 ? w->gs : 0;
+ *b += vkc_fit_tensor(fmt, w->I, w->O, gs); *mm += coli_vk_tensor_payload(fmt, w->I, w->O, gs);
+}
+static void k3c_fit_f32(int I, int O, size_t *b, size_t *mm) {
+ *b += vkc_fit_tensor(10, I, O, 0); *mm += coli_vk_tensor_payload(10, I, O, 0);
+}
+/* What layer i puts on the device, as vkc_fit counts it: its matrices (the router and the
+ * KDA decay and beta projections as f32), its state at its first size (the KDA state and
+ * convolution windows, an MLA layer's mirror at `rows` positions and its rows of the down
+ * buffer), its part of the parameters; *mat its matrices' payload. */
+static size_t k3c_layer_bytes(const Model *m, int i, int rows, size_t *mat) {
+ const Cfg *c = &m->c; const Layer *l = &m->L[i];
+ int D = c->hidden, P = c->kda_proj;
+ size_t b = 0, mm = 0, f = (size_t)4 * D;
+ if (l->kda) {
+ const Kda *k = &l->a;
+ k3c_fit_w(&k->q, &b, &mm); k3c_fit_w(&k->k, &b, &mm); k3c_fit_w(&k->v, &b, &mm);
+ k3c_fit_w(&k->g, &b, &mm); k3c_fit_w(&k->o, &b, &mm);
+ k3c_fit_f32(D, c->kda_hd, &b, &mm); k3c_fit_f32(c->kda_hd, P, &b, &mm); k3c_fit_f32(D, c->kda_heads, &b, &mm);
+ b += vkc_fit_buf((size_t)3 * P * c->conv_k * sizeof(float)) +
+ vkc_fit_buf((size_t)c->kda_heads * c->kda_hd * c->kda_hd * sizeof(float));
+ f += (size_t)3 * P * c->conv_k + c->kda_heads + P + c->kda_hd;
+ } else {
+ const Mla *q = &l->m;
+ k3c_fit_w(&q->qa, &b, &mm); k3c_fit_w(&q->qb, &b, &mm); k3c_fit_w(&q->kva, &b, &mm);
+ k3c_fit_w(&q->kvb, &b, &mm); k3c_fit_w(&q->o, &b, &mm); k3c_fit_w(&q->g, &b, &mm);
+ b += vkc_fit_buf((size_t)rows * c->kv_lora * sizeof(float)) +
+ (c->qk_rope > 0 ? vkc_fit_buf((size_t)rows * c->qk_rope * sizeof(float)) : 0);
+ f += (size_t)c->q_lora + c->kv_lora + (size_t)rows * (c->kv_lora + c->qk_rope);
+ }
+ if (l->sparse) {
+ const Moe *o = &l->moe;
+ k3c_fit_f32(D, c->n_experts, &b, &mm);
+ k3c_fit_w(&o->lat_down, &b, &mm); k3c_fit_w(&o->lat_up, &b, &mm);
+ k3c_fit_w(&o->sh_gate, &b, &mm); k3c_fit_w(&o->sh_up, &b, &mm); k3c_fit_w(&o->sh_down, &b, &mm);
+ f += c->latent;
+ } else { k3c_fit_w(&l->d_gate, &b, &mm); k3c_fit_w(&l->d_up, &b, &mm); k3c_fit_w(&l->d_down, &b, &mm); }
+ *mat = mm;
+ return b + f * sizeof(float);
+}
+/* What the chain allocates whatever N: the scratch of one prompt chunk of `rows` rows (the
+ * counting pass of k3c_scratch, the MLA scratch as k3c_chunk_rows counts it), the
+ * parameters after the last layer (the output mix and the final norm). */
+static size_t k3c_fixed_bytes(Model *m, int rows) {
+ const Cfg *c = &m->c;
+ K3Chain t;
+ memset(&t, 0, sizeof t);
+ t.m = m; t.n = c->n_layers; t.nbmax = (c->n_layers + c->res_bs - 1) / c->res_bs;
+ g_k3c_count = 0;
+ k3c_scratch(&t, m, rows);
+ size_t b = (size_t)g_k3c_count;
+ g_k3c_count = -1;
+ for (int i = 0; i < c->n_layers; i++) if (!m->L[i].kda) {
+ b += (size_t)rows * ((size_t)c->q_lora + (size_t)c->n_heads * (c->qk_nope + c->qk_rope) + (size_t)(c->kv_lora + c->qk_rope) +
+ 2 * (size_t)c->n_heads * c->kv_lora + (size_t)c->n_heads * c->v_head) * sizeof(float);
+ break;
+ }
+ return b + (size_t)2 * c->hidden * sizeof(float);
+}
+static int k3c_fit_now(Model *m, int tier_on, int cuda_on) {
+ if (g_k3_fit_done) return g_k3_fit.L > 0;
+ g_k3_fit_done = 1;
+ if (!g_k3_vk || cuda_on || k3_dsa_indexer_on() || k3c_unsupported(m)) return 0;
+ if (coli_vk_chain_decide(NULL, tier_on, COLI_VK_CHAIN_UNMEASURED) == COLI_VK_CHAIN_OFF) return 0;
+ const Cfg *c = &m->c;
+ int L = c->n_layers, rows = vkc_fit_rows(256);
+ size_t *per = calloc((size_t)L, sizeof(size_t)), *mat = calloc((size_t)L, sizeof(size_t)), tail = 0, tm = 0;
+ if (!per || !mat) { free(per); free(mat); return 0; }
+ for (int i = 0; i < L; i++) per[i] = k3c_layer_bytes(m, i, rows, &mat[i]);
+ k3c_fit_w(&m->lm_head, &tail, &tm);
+ vkc_fit("kimi_k3", L, per, mat, k3c_fixed_bytes(m, rows), tail, &g_k3_fit);
+ g_k3_partial = vkc_fit_partial(&g_k3_fit);
+ free(per); free(mat);
+ return 1;
+}
+
+/* Layer i's tensors and state on the device. 0 with *why set. */
+static int k3c_layer_up(K3Chain *ch, Model *m, int i, const char **why) {
+ const Cfg *c = &m->c; Layer *l = &m->L[i];
+ int D = c->hidden, P = c->kda_proj;
+ int ok;
+ *why = "a matrix the device refused (or one with no device form)";
+ if (l->kda) {
+ Kda *k = &l->a; K3cKda *t = &ch->kda[i];
+ ok = k3c_w(&k->q, &t->q) && k3c_w(&k->k, &t->k) && k3c_w(&k->v, &t->v) && k3c_w(&k->g, &t->g) &&
+ k3c_w(&k->o, &t->o) && k3c_f32(k->fa, D, c->kda_hd, &t->fa) && k3c_f32(k->fb, c->kda_hd, P, &t->fb) &&
+ k3c_f32(k->bp, D, c->kda_heads, &t->bp);
+ } else {
+ Mla *q = &l->m; VkcMla *t = &ch->mla[i];
+ /* kv_b as it is: per head Q key rows then V value rows, the absorption reads it transposed */
+ *t = (VkcMla){c->n_heads, c->qk_nope, c->qk_rope, c->v_head, c->kv_lora, D, c->q_lora, c->eps,
+ c->attn_scale, VKC_ROPE_HALF, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+ ch->prm, ch->o_qn[i], ch->o_kn[i]};
+ ok = k3c_w(&q->qa, &t->q_a) && k3c_w(&q->qb, &t->q_b) && k3c_w(&q->kva, &t->kv_a) && k3c_w(&q->kvb, &t->kv_b) &&
+ k3c_w(&q->o, &t->o) && k3c_w(&q->g, &ch->mg[i]);
+ }
+ K3cFfn *f = &ch->ffn[i];
+ if (ok && l->sparse) {
+ Moe *o = &l->moe;
+ ok = k3c_f32(o->router, D, c->n_experts, &f->router) && k3c_w(&o->lat_down, &f->down) &&
+ k3c_w(&o->lat_up, &f->up) && k3c_w_shared(&o->sh_gate, &f->sg) && k3c_w_shared(&o->sh_up, &f->su) &&
+ k3c_w_shared(&o->sh_down, &f->sd);
+ } else if (ok)
+ ok = k3c_w(&l->d_gate, &f->dg) && k3c_w(&l->d_up, &f->du) && k3c_w(&l->d_down, &f->dd);
+ if (!ok) return 0;
+ if (l->kda) {
+ *why = "device memory for its state refused";
+ ch->win[i] = vkc_buf((size_t)3 * P * c->conv_k * sizeof(float), VKC_DEV);
+ ch->st[i] = vkc_buf((size_t)c->kda_heads * c->kda_hd * c->kda_hd * sizeof(float), VKC_DEV);
+ if (!ch->win[i] || !ch->st[i]) return 0;
+ }
+ return 1;
+}
+/* A device copy the chain holds for W w (*t): freed, and the W's own pointer to it with it
+ * (the per-matrix path's copy, which the chain adopts). */
+static void k3c_drop_w(W *w, ColiVkTensor **t) {
+ if (*t && w && w->vk == (void *)*t) w->vk = NULL;
+ coli_vk_tensor_free(*t); *t = NULL;
+}
+/* Everything of layer i off the device: the chain's tensors (and the W copies it adopted),
+ * its state. The CPU multiplies its matrices from their host copies, which stay. */
+static void k3c_layer_free(K3Chain *ch, Model *m, int i) {
+ Layer *l = &m->L[i];
+ if (vkc_ready()) vkc_finish();
+ K3cKda *k = &ch->kda[i]; K3cFfn *f = &ch->ffn[i]; VkcMla *t = &ch->mla[i];
+ if (l->kda) {
+ k3c_drop_w(&l->a.q, &k->q); k3c_drop_w(&l->a.k, &k->k); k3c_drop_w(&l->a.v, &k->v);
+ k3c_drop_w(&l->a.g, &k->g); k3c_drop_w(&l->a.o, &k->o);
+ k3c_drop_w(NULL, &k->fa); k3c_drop_w(NULL, &k->fb); k3c_drop_w(NULL, &k->bp);
+ } else {
+ k3c_drop_w(&l->m.qa, &t->q_a); k3c_drop_w(&l->m.qb, &t->q_b); k3c_drop_w(&l->m.kva, &t->kv_a);
+ k3c_drop_w(&l->m.kvb, &t->kv_b); k3c_drop_w(&l->m.o, &t->o); k3c_drop_w(&l->m.g, &ch->mg[i]);
+ }
+ if (l->sparse) {
+ k3c_drop_w(NULL, &f->router); k3c_drop_w(&l->moe.lat_down, &f->down); k3c_drop_w(&l->moe.lat_up, &f->up);
+ k3c_drop_w(&l->moe.sh_gate, &f->sg); k3c_drop_w(&l->moe.sh_up, &f->su); k3c_drop_w(&l->moe.sh_down, &f->sd);
+ } else { k3c_drop_w(&l->d_gate, &f->dg); k3c_drop_w(&l->d_up, &f->du); k3c_drop_w(&l->d_down, &f->dd); }
+ vkc_free(ch->win[i]); ch->win[i] = NULL;
+ vkc_free(ch->st[i]); ch->st[i] = NULL;
+}
+/* Layer i did not reach the device: it (and any layer after it) off the device, the chain
+ * cut before it; with the dense weights on the device only, the layers from i keep their
+ * host copies (nothing of theirs was dropped yet: that comes after the setup). */
+static void k3c_cut(K3Chain *ch, Model *m, int i, const char *why) {
+ for (int k = i; k < ch->n; k++) k3c_layer_free(ch, m, k);
+ if (ch->n > i) ch->n = i;
+ if (g_k3_dho_layers > i) { g_k3_dho_layers = i; g_k3_dho_head = 0; coli_vk_dense_host_layers(i, m->c.n_layers); }
+ vkc_fit_shrink("kimi_k3", &g_k3_fit, i, why);
+ g_k3_partial = 1;
+}
+
static int k3c_setup(Model *m) {
const Cfg *c = &m->c;
int L = c->n_layers, D = c->hidden, P = c->kda_proj, LT = c->latent;
- int nbmax = (L + c->res_bs - 1) / c->res_bs, any_kda = 0, any_mla = 0;
- for (int i = 0; i < L; i++) { any_kda |= m->L[i].kda; any_mla |= !m->L[i].kda; }
- const char *why = NULL;
- if (!m->has_head) why = "no head (a layer range)";
- else if (any_kda && (c->kda_hd > 128 || c->conv_k > 8)) why = "a KDA head its shaders do not take";
- else if (any_mla && (c->kv_lora > 1024 || c->qk_rope > 128 || (c->qk_rope & 1) || c->qk_nope > 1024 || c->q_lora < 1))
- why = "an attention geometry its shaders do not take";
- else if (nbmax > 15) why = "more AttnRes blocks than its shader takes";
+ int nbmax = (L + c->res_bs - 1) / c->res_bs;
+ const char *why = k3c_unsupported(m);
if (why) { fprintf(stderr, "[VK] kimi_k3 chain: %s; the CPU runs the layers\n", why); return 0; }
+ int N = g_k3_fit.L > 0 ? g_k3_fit.n : L;
+ if (N < 1) return 0;
K3Chain *ch = calloc(1, sizeof *ch);
if (!ch) return 0;
size_t **offs[] = {&ch->o_in, &ch->o_post, &ch->o_asw, &ch->o_msw, &ch->o_qn, &ch->o_kn, &ch->o_conv, &ch->o_kda, &ch->o_latn};
@@ -268,26 +443,37 @@ static int k3c_setup(Model *m) {
ch->win = calloc(L, sizeof(void *)); ch->st = calloc(L, sizeof(void *));
ch->kv_valid = calloc(L, sizeof(int)); ch->mla_ord = calloc(L, sizeof(int));
if (!ch->kda || !ch->ffn || !ch->mla || !ch->mg || !ch->kv || !ch->win || !ch->st || !ch->kv_valid || !ch->mla_ord) return 0;
- ch->m = m; ch->nbmax = nbmax;
+ ch->m = m; ch->nbmax = nbmax; ch->n = N;
g_k3c = ch;
- /* the parameter arena: offsets, then one upload */
+ /* the parameter arena: offsets for the chain's layers and the output mix, then one upload */
size_t n = 0;
- for (int i = 0; i < L; i++) {
+ for (int i = 0; i < N; i++) {
const Layer *l = &m->L[i];
ch->o_in[i] = n; n += D; ch->o_post[i] = n; n += D; ch->o_asw[i] = n; n += D; ch->o_msw[i] = n; n += D;
if (l->kda) {
ch->o_conv[i] = n; n += (size_t)3 * P * c->conv_k;
ch->o_kda[i] = n; n += (size_t)c->kda_heads + P + c->kda_hd;
} else {
- ch->mla_ord[i] = ch->n_mla++;
ch->o_qn[i] = n; n += c->q_lora; ch->o_kn[i] = n; n += c->kv_lora;
}
if (l->sparse) { ch->o_latn[i] = n; n += LT; }
}
ch->o_osw = n; n += D; ch->o_final = n; n += D;
- float *a = calloc(n, sizeof(float));
- if (!a) return 0;
- for (int i = 0; i < L; i++) {
+ if (!(ch->prm = vkc_buf(n * sizeof(float), VKC_DEV))) {
+ k3c_cut(ch, m, 0, "device memory for the parameters refused");
+ vkc_fit_placed("kimi_k3", &g_k3_fit);
+ return 0;
+ }
+ /* the layers, each whole or the chain stops before it */
+ for (int i = 0; i < N; i++) {
+ if (!k3c_layer_up(ch, m, i, &why)) { k3c_cut(ch, m, i, why); break; }
+ vkc_fit_mark(&g_k3_fit, i);
+ }
+ N = ch->n;
+ for (int i = 0; i < N; i++) if (!m->L[i].kda) ch->mla_ord[i] = ch->n_mla++;
+ float *a = N > 0 ? calloc(n, sizeof(float)) : NULL;
+ int ok = a != NULL;
+ for (int i = 0; ok && i < N; i++) {
const Layer *l = &m->L[i];
memcpy(a + ch->o_in[i], l->in_ln, D * sizeof(float));
memcpy(a + ch->o_post[i], l->post_ln, D * sizeof(float));
@@ -307,49 +493,43 @@ static int k3c_setup(Model *m) {
}
if (l->sparse) memcpy(a + ch->o_latn[i], l->moe.lat_norm, LT * sizeof(float));
}
- memcpy(a + ch->o_osw, m->out_sw, D * sizeof(float));
- memcpy(a + ch->o_final, m->final_norm, D * sizeof(float));
- ch->prm = vkc_buf(n * sizeof(float), VKC_DEV);
- int ok = ch->prm && vkc_begin() && vkc_write(ch->prm, 0, a, n * sizeof(float)) && vkc_submit(1);
+ if (ok) {
+ memcpy(a + ch->o_osw, m->out_sw, D * sizeof(float));
+ memcpy(a + ch->o_final, m->final_norm, D * sizeof(float));
+ }
+ ok = ok && vkc_begin() && vkc_write(ch->prm, 0, a, n * sizeof(float)) && vkc_submit(1);
free(a);
- /* the tensors */
- for (int i = 0; i < L && ok; i++) {
- Layer *l = &m->L[i];
- if (l->kda) {
- Kda *k = &l->a; K3cKda *t = &ch->kda[i];
- ok = k3c_w(&k->q, &t->q) && k3c_w(&k->k, &t->k) && k3c_w(&k->v, &t->v) && k3c_w(&k->g, &t->g) &&
- k3c_w(&k->o, &t->o) && k3c_f32(k->fa, D, c->kda_hd, &t->fa) && k3c_f32(k->fb, c->kda_hd, P, &t->fb) &&
- k3c_f32(k->bp, D, c->kda_heads, &t->bp);
- if (ok) {
- ch->win[i] = vkc_buf((size_t)3 * P * c->conv_k * sizeof(float), VKC_DEV);
- ch->st[i] = vkc_buf((size_t)c->kda_heads * c->kda_hd * c->kda_hd * sizeof(float), VKC_DEV);
- ok = ch->win[i] && ch->st[i];
+ if (!ok) {
+ if (N > 0) {
+ fprintf(stderr, "[VK] kimi_k3 chain: the parameters did not reach the device; the CPU runs the layers\n");
+ k3c_cut(ch, m, 0, "the parameters did not reach it");
+ }
+ vkc_free(ch->prm); ch->prm = NULL;
+ vkc_fit_placed("kimi_k3", &g_k3_fit);
+ return 0;
+ }
+ vkc_fit_placed("kimi_k3", &g_k3_fit); /* the layers' matrices only, before the head */
+ /* the head goes up with every layer and the tail (the fit's; without a fit, as before);
+ * if the device refuses it the CPU multiplies it from the chain's final rows */
+ if (N == L && (g_k3_fit.L < 1 || g_k3_fit.tail)) {
+ if (g_k3_dho && g_k3_fit.L > 0) w_vk_upload(&m->lm_head);
+ if (!k3c_w(&m->lm_head, &ch->head)) {
+ if (g_k3_fit.L < 1) {
+ fprintf(stderr, "[VK] kimi_k3 chain: a matrix did not reach the device (or has no device form); the CPU runs the layers\n");
+ return 0;
}
- } else {
- Mla *q = &l->m; VkcMla *t = &ch->mla[i];
- /* kv_b as it is: per head Q key rows then V value rows, the absorption reads it transposed */
- *t = (VkcMla){c->n_heads, c->qk_nope, c->qk_rope, c->v_head, c->kv_lora, D, c->q_lora, c->eps,
- c->attn_scale, VKC_ROPE_HALF, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
- ch->prm, ch->o_qn[i], ch->o_kn[i]};
- ok = k3c_w(&q->qa, &t->q_a) && k3c_w(&q->qb, &t->q_b) && k3c_w(&q->kva, &t->kv_a) && k3c_w(&q->kvb, &t->kv_b) &&
- k3c_w(&q->o, &t->o) && k3c_w(&q->g, &ch->mg[i]);
+ fprintf(stderr, "[VK] kimi_k3 chain: the head did not reach the device; the CPU multiplies it\n");
+ g_k3_fit.tail = 0; g_k3_partial = 1; g_k3_dho_head = 0;
}
- K3cFfn *f = &ch->ffn[i];
- if (ok && l->sparse) {
- Moe *o = &l->moe;
- ok = k3c_f32(o->router, D, c->n_experts, &f->router) && k3c_w(&o->lat_down, &f->down) &&
- k3c_w(&o->lat_up, &f->up) && k3c_w_shared(&o->sh_gate, &f->sg) && k3c_w_shared(&o->sh_up, &f->su) &&
- k3c_w_shared(&o->sh_down, &f->sd);
- } else if (ok)
- ok = k3c_w(&l->d_gate, &f->dg) && k3c_w(&l->d_up, &f->du) && k3c_w(&l->d_down, &f->dd);
- }
- ok = ok && k3c_w(&m->lm_head, &ch->head);
- if (!ok) { fprintf(stderr, "[VK] kimi_k3 chain: a matrix did not reach the device (or has no device form); the CPU runs the layers\n"); return 0; }
+ }
ch->where = K3C_HOST; ch->host_zero = 1; /* the host's KDA state is the zeros it was allocated with */
ch->ok = 1;
- int nkda = 0, nsparse = 0; for (int i = 0; i < L; i++) { nkda += m->L[i].kda; nsparse += m->L[i].sparse; }
+ int nkda = 0, nsparse = 0; for (int i = 0; i < N; i++) { nkda += m->L[i].kda; nsparse += m->L[i].sparse; }
+ size_t bytes = 0, tensors = 0;
+ coli_vk_mem_info(&bytes, &tensors);
fprintf(stderr, "[VK] kimi_k3 chain: %d layers on the device (%d KDA, %d MLA, %d dense MLP), %d AttnRes blocks, "
- "%.1f MiB of parameters\n", L, nkda, ch->n_mla, L - nsparse, nbmax, n * 4 / 1048576.0);
+ "%.1f MiB of parameters, %zu matrices (%.1f MiB) on the device\n", N, nkda, ch->n_mla, N - nsparse, nbmax,
+ n * 4 / 1048576.0, tensors, bytes / 1048576.0);
return 1;
}
@@ -358,7 +538,7 @@ static int k3c_scratch(K3Chain *ch, const Model *m, int rows) {
const Cfg *c = &m->c;
int D = c->hidden, P = c->kda_proj, H = c->kda_heads, LT = c->latent, E = c->n_experts, R = c->qk_rope;
int SI = c->moe_inter * c->n_shared, MI = SI > c->dense_inter ? SI : c->dense_inter;
- int mfull = -1; for (int i = 0; i < c->n_layers; i++) if (!m->L[i].kda) { mfull = i; break; }
+ int mfull = -1; for (int i = 0; i < ch->n; i++) if (!m->L[i].kda) { mfull = i; break; } /* the chain's layers */
size_t r = (size_t)rows;
ch->kvd_layer = r * (c->kv_lora + R);
int ok = (mfull < 0 || g_k3c_count >= 0 || vkc_mla_scratch(&ch->sc, &ch->mla[mfull], rows)) &&
@@ -415,14 +595,14 @@ static int k3c_mirror(K3Chain *ch, const Model *m, int rows) {
if (ch->n_mla && !vkc_kv_plan(&ch->ks, "kimi_k3", ch->n_mla, row, m->max_t, rows, 0, (size_t)ch->n_mla * ch->dev_rows * row))
return 0;
int dr = ch->ks.on ? ch->ks.rows : m->max_t;
- for (int i = 0; i < c->n_layers; i++) {
+ for (int i = 0; i < ch->n; i++) {
if (m->L[i].kda) continue;
vkc_free(ch->kv[i].lat); vkc_free(ch->kv[i].rope);
ch->kv[i] = (VkcMlaCache){NULL, NULL, 0};
ch->kv_valid[i] = 0;
}
ch->cap = 0; ch->dev_rows = 0;
- for (int i = 0; i < c->n_layers; i++) {
+ for (int i = 0; i < ch->n; i++) {
if (m->L[i].kda) continue;
ch->kv[i].lat = vkc_buf((size_t)dr * c->kv_lora * sizeof(float), VKC_DEV);
if (c->qk_rope > 0) ch->kv[i].rope = vkc_buf((size_t)dr * c->qk_rope * sizeof(float), VKC_DEV);
@@ -439,7 +619,7 @@ static int k3c_push(K3Chain *ch, const Model *m, int pb, int n_rows) {
const Cfg *c = &m->c; int ok = 1, K = c->kv_lora, R = c->qk_rope;
if (ch->where == K3C_HOST) {
size_t ns = (size_t)c->kda_heads * c->kda_hd * c->kda_hd, nw = (size_t)c->kda_proj * c->conv_k;
- for (int i = 0; i < c->n_layers && ok; i++) {
+ for (int i = 0; i < ch->n && ok; i++) {
if (!m->L[i].kda) continue;
if (ch->host_zero) ok = vkc_zero(ch->st[i], 0, ns) && vkc_zero(ch->win[i], 0, 3 * nw);
else ok = vkc_write(ch->st[i], 0, m->kstate[i], ns * sizeof(float)) &&
@@ -449,7 +629,7 @@ static int k3c_push(K3Chain *ch, const Model *m, int pb, int n_rows) {
}
ch->where = K3C_BOTH; ch->host_pos = pb;
}
- for (int i = 0; i < c->n_layers && ok; i++) {
+ for (int i = 0; i < ch->n && ok; i++) {
if (!m->L[i].kda && ch->ks.on) { /* the split: the window placed, its rows below pb uploaded */
VkcKvPart pt[2] = {{1, K, m->Lc[i], 0, ch->kv[i].lat, 0}, {1, R, m->Rc[i], 0, ch->kv[i].rope, 0}};
vkc_kv_place(&ch->ks, ch->mla_ord[i], pb, n_rows);
@@ -540,10 +720,6 @@ static void k3c_moe_host(K3Chain *ch, Model *m, int li, int n) {
memcpy(vkc_ptr(ch->routed), ch->host_u, (size_t)n * LT * sizeof(float));
}
-/* Every layer for C rows of `hidden` (positions pos0..), then the output mix and the
- * final norm: the rows into *fin (when need_rows), the last row's logits into *last.
- * 0: not taken (the CPU runs the layers; nothing the caller holds changed). On a
- * cancel (poll), 1 with *cancelled set: the state is the caller's to reset. */
/* The prompt block the engine hands the chain: its chunk, when the chain runs with a
* chunk from the budget (vkc_chunk_auto) and K3_CHUNK is not set; 0 = K3_CHUNK's (32). */
static int k3c_prefill_rows(const Model *m) {
@@ -554,7 +730,16 @@ static int k3c_prefill_rows(const Model *m) {
int rows = k3c_chunk_rows(ch, m);
return ch->ks.on && rows > ch->ks.chunk ? ch->ks.chunk : rows;
}
-static int k3c_forward(Model *m, const float *hidden, int pos0, int C, int need_rows, float **fin_out,
+/* The chain's layers (every layer, or a partial chain's first ch->n) for C rows of
+ * `hidden` (positions pos0..). Every layer: then the output mix and the final norm, the
+ * rows into *fin (when need_rows, or when the head stays on the CPU), the last row's
+ * logits into *last (the head on the device). A partial chain: each row's AttnRes state
+ * after its last layer (the prefix into hidden, the block snapshots into bres, their
+ * count into *nb_out), from which the caller's CPU loop runs the other layers and the
+ * head. Returns the layers it ran; 0: not taken (the CPU runs every layer; nothing the
+ * caller holds changed). On a cancel (poll), nonzero with *cancelled set: the state is
+ * the caller's to reset. */
+static int k3c_forward(Model *m, float *hidden, float *bres, int *nb_out, int pos0, int C, int need_rows, float **fin_out,
float **last_out, K3CancelPoll poll, void *pctx, int *cancelled) {
K3Chain *ch = g_k3c;
if (!g_k3c_on || !ch || !ch->ok || ch->failed || ch->m != m) return 0;
@@ -564,13 +749,16 @@ static int k3c_forward(Model *m, const float *hidden, int pos0, int C, int need_
if (!said++) fprintf(stderr, "[VK] kimi_k3 chain: a validation dump reads every layer on the host; the CPU runs the layers\n");
return 0;
}
- const Cfg *c = &m->c; int L = c->n_layers, D = c->hidden, E = c->n_experts, LT = c->latent;
+ const Cfg *c = &m->c; int L = c->n_layers, N = ch->n, D = c->hidden, E = c->n_experts, LT = c->latent;
+ int part = N < L, nbx = ch->nbmax;
+ if (!part && !ch->head) need_rows = 1; /* the head on the CPU: it reads the final rows */
int dev0 = ch->where == K3C_DEV, dev_start = ch->dev_pos;
if (vkc_lost()) { k3c_recover(m, dev0, dev_start); return 0; }
int mirror_ok = k3c_mirror(ch, m, 1);
int CH = mirror_ok ? k3c_chunk_rows(ch, m) : 1, rows = C < CH ? C : CH;
if (ch->ks.on && rows > ch->ks.chunk) rows = ch->ks.chunk; /* a step's rows fit the split's window */
- if (!mirror_ok || !k3c_scratch(ch, m, rows)) {
+ if (!mirror_ok || !k3c_scratch(ch, m, rows) ||
+ (part && !(k3c_res(&ch->xo, (size_t)rows * D, VKC_DOWN) && k3c_res(&ch->bo, (size_t)rows * nbx * D, VKC_DOWN)))) {
if (vkc_lost()) { k3c_recover(m, dev0, dev_start); return 0; } /* lost while its buffers were made */
fprintf(stderr, "[VK] kimi_k3 chain: device memory for %d rows at %d positions refused; the CPU runs the layers\n",
rows, m->max_t);
@@ -578,14 +766,18 @@ static int k3c_forward(Model *m, const float *hidden, int pos0, int C, int need_
ch->failed = 1; g_k3c_on = 0;
return 0;
}
- float *fin = need_rows ? falloc((int64_t)C * D) : NULL, *last = falloc(c->vocab);
+ float *fin = need_rows && !part ? falloc((int64_t)C * D) : NULL, *last = !part && ch->head ? falloc(c->vocab) : NULL;
+ /* a partial chain's handoff, written to the caller's buffers once every chunk is through
+ * (a lost device reruns the forward on the CPU from its untouched input) */
+ float *xh = part ? falloc((int64_t)C * D) : NULL, *bh = part ? falloc((int64_t)C * nbx * D) : NULL;
+ int nb_end = 0;
vkc_gemm_rows(-1);
for (int c0 = 0; c0 < C; c0 += rows) {
int n = C - c0 < rows ? C - c0 : rows, pb = pos0 + c0, end = c0 + n == C;
if (!vkc_begin() || !k3c_push(ch, m, pb, n) || !vkc_write(ch->x, 0, hidden + (size_t)c0 * D, (size_t)n * D * sizeof(float)))
goto lost;
int ok = 1, pending = 0, pulled = 0, nb = 0;
- for (int i = 0; i < L && ok; i++) {
+ for (int i = 0; i < N && ok; i++) {
const Layer *l = &m->L[i];
if (pending) { ok = k3c_join(ch, m, i - 1, n); pending = 0; }
int snap = i % c->res_bs == 0;
@@ -641,31 +833,46 @@ static int k3c_forward(Model *m, const float *hidden, int pos0, int C, int need_
if (cancelled) *cancelled = 1;
vkc_finish();
ch->where = K3C_DEV; ch->dev_pos = pb + n;
- free(fin); free(last);
- return 1;
+ free(fin); free(last); free(xh); free(bh);
+ return N;
}
ok = ok && vkc_begin();
pending = 1;
}
- if (ok && pending) ok = k3c_join(ch, m, L - 1, n);
- /* the output mix and the final norm of every row, lm_head on the last */
- ok = ok && k3c_mix(ch, m, ch->o_osw, n, nb) && k3c_norm(ch->hm, ch->prm, ch->o_final, ch->fin, n, D, c->eps);
- if (ok && need_rows) ok = vkc_copy(ch->find, 0, ch->fin, 0, (size_t)n * D);
- if (ok && end) ok = vkc_matmul(ch->head, ch->fin, (size_t)(n - 1) * D, ch->outd, 0, 1);
+ if (ok && pending) ok = k3c_join(ch, m, N - 1, n);
+ nb_end = nb;
+ if (part) /* the handoff: each row's prefix and its block snapshots after the chain's last layer */
+ ok = ok && vkc_copy(ch->xo, 0, ch->x, 0, (size_t)n * D) && vkc_copy(ch->bo, 0, ch->bres, 0, (size_t)n * nbx * D);
+ else {
+ /* the output mix and the final norm of every row, lm_head on the last */
+ ok = ok && k3c_mix(ch, m, ch->o_osw, n, nb) && k3c_norm(ch->hm, ch->prm, ch->o_final, ch->fin, n, D, c->eps);
+ if (ok && need_rows) ok = vkc_copy(ch->find, 0, ch->fin, 0, (size_t)n * D);
+ if (ok && end && ch->head) ok = vkc_matmul(ch->head, ch->fin, (size_t)(n - 1) * D, ch->outd, 0, 1);
+ }
ok = ok && vkc_submit(1);
if (!ok) goto lost;
- k3c_pull(ch, m, pulled, L, pb, n);
- if (need_rows) memcpy(fin + (size_t)c0 * D, vkc_ptr(ch->find), (size_t)n * D * sizeof(float));
- for (int i = 0; i < L; i++) if (!m->L[i].kda) { ch->kv_valid[i] = pb + n; vkc_kv_done(&ch->ks, ch->mla_ord[i], pb + n); }
+ k3c_pull(ch, m, pulled, N, pb, n);
+ if (part) {
+ memcpy(xh + (size_t)c0 * D, vkc_ptr(ch->xo), (size_t)n * D * sizeof(float));
+ memcpy(bh + (size_t)c0 * nbx * D, vkc_ptr(ch->bo), (size_t)n * nbx * D * sizeof(float));
+ }
+ if (need_rows && fin) memcpy(fin + (size_t)c0 * D, vkc_ptr(ch->find), (size_t)n * D * sizeof(float));
+ for (int i = 0; i < N; i++) if (!m->L[i].kda) { ch->kv_valid[i] = pb + n; vkc_kv_done(&ch->ks, ch->mla_ord[i], pb + n); }
ch->where = K3C_DEV;
- if (end) memcpy(last, vkc_ptr(ch->outd), (size_t)c->vocab * sizeof(float));
+ if (end && last) memcpy(last, vkc_ptr(ch->outd), (size_t)c->vocab * sizeof(float));
}
ch->dev_pos = pos0 + C;
ch->forwards++;
+ if (part) {
+ memcpy(hidden, xh, (size_t)C * D * sizeof(float));
+ memcpy(bres, bh, (size_t)C * nbx * D * sizeof(float));
+ *nb_out = nb_end;
+ free(xh); free(bh);
+ }
*fin_out = fin; *last_out = last;
- return 1;
+ return N;
lost: /* a frame failed: the device is gone (or would not take a command); the CPU takes over */
- free(fin); free(last);
+ free(fin); free(last); free(xh); free(bh);
if (!vkc_lost()) { vkc_finish(); coli_vk_mark_lost(); }
ch->where = dev0 ? K3C_DEV : K3C_HOST; /* the state as it was when this forward began */
k3c_recover(m, dev0, dev_start);
@@ -694,6 +901,10 @@ static void k3c_start(Model *m, int tier_on) {
if (g_k3_cuda) no = "the CUDA expert tier is on and keeps its priority";
#endif
if (!no && k3_dsa_indexer_on()) no = "KIMI_DSA_INDEXER=1 fills its index cache on the CPU";
+ if (!no && g_k3_fit.L > 0 && g_k3_fit.n < 1) { /* the fit's line said so: nothing of the chain on the device */
+ vkc_fit_placed("kimi_k3", &g_k3_fit);
+ return;
+ }
if (!no && !(g_k3c_inited = vkc_init())) no = "the chain's pipelines did not come up";
if (!no && !(vkc_mla_ready() && vkc_kda_ready() && vkc_ares_ready())) no = "the MLA, KDA or AttnRes shaders are missing";
}
@@ -711,7 +922,7 @@ static void k3c_atexit(void) {
static void k3c_adopt_shared(Model *m) {
K3Chain *ch = g_k3c;
if (!ch || !ch->ok || ch->m != m) return;
- for (int i = 0; i < m->c.n_layers; i++) {
+ for (int i = 0; i < ch->n; i++) {
if (!m->L[i].sparse) continue;
Moe *o = &m->L[i].moe; K3cFfn *f = &ch->ffn[i];
if (!o->sh_gate.vk) o->sh_gate.vk = f->sg;