Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
deepseek_v41: the dense chain on the device (COLI_VK_CHAIN=1)
JustVuggee67d91b5687
diff --git a/c/deepseek_v41.c b/c/deepseek_v41.cindex 9ea5ef14..2419451f 100644--- a/c/deepseek_v41.c+++ b/c/deepseek_v41.c@@ -2421,7 +2421,7 @@ static void moe_vk_cpu(Model *m, LCache *cache, int layer, int topk, const float}static void moe_vk_block(Model *m, Layer *l, LCache *cache, int layer, int topk, const float *xc,- int rows, float *outc, const int *chosen, const float *weights) {+ int rows, float *outc, const int *chosen, const float *weights, int with_shared) {int dim = m->c.dim, draws = rows * topk;uint8_t taken[MOE_ROW_CHUNK * MOE_TOPK_MAX], want[MOE_ROW_CHUNK * MOE_TOPK_MAX];const float *dev[MOE_ROW_CHUNK * MOE_TOPK_MAX];@@ -2431,7 +2431,7 @@ static void moe_vk_block(Model *m, Layer *l, LCache *cache, int layer, int topk,float *contrib = xmalloc((size_t)draws * dim * sizeof(float), "expert contributions");float *shared = xmalloc((size_t)rows * dim * sizeof(float), "shared expert");moe_vk_cpu(m, cache, layer, topk, xc, rows, chosen, weights, want, contrib);- shared_ffn_down(m, l, xc, rows, shared);+ if (with_shared) shared_ffn_down(m, l, xc, rows, shared);if (ndev && !vkt_join(dev)) { /* the batch failed (the tier stops): those draws here */moe_vk_cpu(m, cache, layer, topk, xc, rows, chosen, weights, taken, contrib);memset(taken, 0, (size_t)draws);@@ -2451,7 +2451,7 @@ static void moe_vk_block(Model *m, Layer *l, LCache *cache, int layer, int topk,}}const float *sh = shared + (size_t)r * dim;- for (int i = 0; i < dim; i++) o[i] += sh[i];+ if (with_shared) for (int i = 0; i < dim; i++) o[i] += sh[i];}free(shared); free(contrib);}@@ -2474,8 +2474,10 @@ static void moe_vk_block(Model *m, Layer *l, LCache *cache, int layer, int topk,** A cache too small to hold one position's experts keeps the old path: there the* slots genuinely cannot all be live at once. */+/* with_shared 0: the routed experts only (the dense chain runs the shared expert on the+ * device and adds it there, in the same place of the sum). */static void moe_run_at(Model *m, Layer *l, LCache *cache, const char *kind, int layer,- int E, int topk, const float *x, int n, float *out) {+ int E, int topk, const float *x, int n, float *out, int with_shared) {Cfg *c = &m->c;int dim = c->dim;if (topk > MOE_TOPK_MAX) {@@ -2495,7 +2497,7 @@ static void moe_run_at(Model *m, Layer *l, LCache *cache, const char *kind, intfree(scores);#ifdef COLI_VULKANif (vkt_ready() && !strcmp(kind, "layers")) {- moe_vk_block(m, l, cache, layer, topk, xc, rows, outc, chosen, weights);+ moe_vk_block(m, l, cache, layer, topk, xc, rows, outc, chosen, weights, with_shared);continue;}#endif@@ -2516,7 +2518,7 @@ static void moe_run_at(Model *m, Layer *l, LCache *cache, const char *kind, int}free(down);}- shared_ffn_rows(m, l, xc, rows, outc);+ if (with_shared) shared_ffn_rows(m, l, xc, rows, outc);continue;}@@ -2588,7 +2590,7 @@ static void moe_run_at(Model *m, Layer *l, LCache *cache, const char *kind, intfor (int i = 0; i < dim; i++) o[i] += cvec[i];}}- shared_ffn_rows(m, l, xc, rows, outc);+ if (with_shared) shared_ffn_rows(m, l, xc, rows, outc);free(down); free(gathered); free(contrib);free(next); free(count); free(tail); free(head); free(uniq);}@@ -2596,7 +2598,7 @@ static void moe_run_at(Model *m, Layer *l, LCache *cache, const char *kind, intstatic void moe_run(Model *m, int layer, const float *x, int n, float *out) {moe_run_at(m, &m->L[layer], &m->cache[layer], "layers", layer,- m->c.n_routed, m->c.n_activated, x, n, out);+ m->c.n_routed, m->c.n_activated, x, n, out, 1);}/* --------------------------------------------------------------- engram ---- */@@ -3178,7 +3180,7 @@ static int spec_step(Model *m, int token, int start_pos, int main_rows,rms_into(branch_in + (size_t)i * dim, collapsed, l->ffn_norm.w, dim, c->norm_eps);}moe_run_at(m, l, &sp->cache[stage], "mtp", stage, c->spec_routed, c->spec_activated,- branch_in, block, branch_out);+ branch_in, block, branch_out, 1);for (int i = 0; i < block; i++)coli_hc_post(h + (size_t)i * hc * dim, branch_out + (size_t)i * dim,residual + (size_t)i * hc * dim, post + (size_t)i * hc,@@ -3281,6 +3283,10 @@ static void forward_batch(Model *m, const int *ids, int n, float *logits) {forward_full(m, ids, n, logits, 1, NULL, -1, 0, 0, NULL, 0);}+#ifdef COLI_VULKAN+#include "deepseek_v41_chain.h" /* the layers as a dense chain on the device (COLI_VK_CHAIN) */+#endif+static void forward_full(Model *m, const int *ids, int n, float *logits, int spec_batch,const float *image_rows, int image_at, int image_h, int image_w,const uint8_t *image_mask, int keep_rows) {@@ -3381,7 +3387,15 @@ static void forward_full(Model *m, const int *ids, int n, float *logits, int spem->main_hidden_rows = n;}- for (int layer = 0; layer < c->n_layers; layer++) {+ int rows_out = spec_batch || keep_rows;+#ifdef COLI_VULKAN+ /* the dense chain runs every layer on the device; 0: the CPU runs them, as below */+ int chained = v41c_forward(m, h, pre_mix, n, start_pos, spec_batch, rows_out);+ if (!chained) v41c_cpu_step(m, start_pos);+#else+ int chained = 0;+#endif+ for (int layer = 0; layer < c->n_layers && !chained; layer++) {Layer *l = &m->L[layer];if (l->engram_index >= 0) engram_run(m, layer, h, n, start_pos);for (int k = 0; k < targets; k++) {@@ -3454,7 +3468,6 @@ static void forward_full(Model *m, const int *ids, int n, float *logits, int spe}/* the last block's FFN mix collapses the stream one final time */- int rows_out = spec_batch || keep_rows;for (int t = rows_out ? 0 : n - 1; t < n; t++) {const float *mix = pre_mix + (size_t)t * hc;for (int i = 0; i < dim; i++) {@@ -3468,6 +3481,13 @@ static void forward_full(Model *m, const int *ids, int n, float *logits, int spemvb(logits + (size_t)(rows_out ? t : 0) * c->vocab, &m->head, branch_in);}trace("logits", -1, logits + (size_t)(rows_out ? n - 1 : 0) * c->vocab, c->vocab);+#ifdef COLI_VULKAN+ { /* DUMP=<path>: every logits row this forward computed, for the Vulkan gates */+ static FILE *dump; static int dump_init;+ if (!dump_init) { const char *d = getenv("DUMP"); dump_init = 1; if (d && *d) dump = fopen(d, "wb"); }+ if (dump) { fwrite(logits, sizeof(float), (size_t)(rows_out ? n : 1) * c->vocab, dump); fflush(dump); }+ }+#endifm->pos += n;/* Recorded where the tokens entered the state, and only for the MAIN stream:@@ -3515,6 +3535,9 @@ static void model_reset(Model *m) {m->published_index_k = NULL;m->shared_topk_rows = m->shared_topk_width = 0;m->pos = 0;+#ifdef COLI_VULKAN+ v41c_reset(m); /* nothing on the device describes the new sequence */+#endif/* Paired with the reset on purpose: whoever drops the state must also forget* what it was built from, or the two disagree in favour of the one nobody* can check. */@@ -3614,6 +3637,9 @@ static void spec_rollback(Model *m, int start_pos, int committed, int rows) {}m->pos = start_pos + committed;m->last_rows = committed;+#ifdef COLI_VULKAN+ v41c_rollback(m, m->pos); /* the dense chain's copies of the rejected rows go stale */+#endifif (m->engram.active && m->engram.history_len > m->pos) m->engram.history_len = m->pos;/* The record follows the rollback for the same reason the engram history* does: the rejected rows are no longer in the state, and a record that@@ -4133,6 +4159,7 @@ static void serve_loop(Model *m, Tok *tokenizer, const char *snap) {m->t_engram - engram0, (unsigned long long)(m->forwards - forwards0));#ifdef COLI_VULKANif (vkt_ready()) { vkt_report("turn", turn_hits, turn_miss); vk_tier_save(); }+ v41c_report(); /* the dense chain's line, when it ran */#endifserve_hits(m);serve_emap(m);@@ -4287,7 +4314,9 @@ int main(int argc, char **argv) {* The matrices go up on their first multiply (see vk_mul), from this thread. */g_vk_thread = pthread_self();g_vk_ready = coli_vk_init_env_tier("deepseek_v41", vkt_wanted() && c->n_routed > 0);+ v41c_start(&m); /* COLI_VK_CHAIN: the trunk on the device, before the tier sizes itself */vk_tier_start(&m, snap, cap);+ v41c_atexit(); /* after the tier's: the chain goes before the device */if (g_vk_ready && !vkt_ready() && !coli_vk_dense())coli_vk_dense_decide("deepseek_v41", 0, 1); /* no tier after all: the trunk to the device */#endifdiff --git a/c/deepseek_v41_chain.h b/c/deepseek_v41_chain.hnew file mode 100644index 00000000..f9662f6c--- /dev/null+++ b/c/deepseek_v41_chain.h@@ -0,0 +1,705 @@+/* deepseek_v41_chain.h -- DeepSeek V4.1 Flash's layers as a dense chain on the Vulkan+ * device (vk_chain.h). Included once by deepseek_v41.c in a COLI_VULKAN build, before+ * forward_full, whose layer loop it stands in for; COLI_VK_CHAIN decides+ * (coli_vk_chain_decide, this engine's integrated GPU default COLI_VK_CHAIN_UNMEASURED:+ * off, not measured on a V4.1 checkpoint).+ *+ * The residual is hc_mult streams per position (mHC), and a site collapses them with the+ * mix the site BEFORE it computed (forward_full: pre_mix). What runs where, per layer, for+ * one block of rows (decode: one row; a prompt in chunks of COLI_VK_CHAIN_ROWS):+ * device, frame A1: the previous layer's FFN branch (the routed sum from the host plus+ * the shared expert) written back into the streams (mHC post); on an engram layer the+ * engram (eng_wkv over the n-gram rows the host looked up, the gate into each stream);+ * on a DSpark target layer the streams' mean for the draft head; the attention site:+ * hc_attn_fn, the split with Sinkhorn, the collapse with the previous site's mix, the+ * RMSNorm; the attention: wq_a, its norm, wq_b, wkv, its norm, RoPE on interleaved+ * pairs, the new rows into the window ring; on a kv_source layer the compressor (its+ * rolling group on the device, the pooled latent's norm), the index keys (idx_wk,+ * norm, RoPE) and the compressed rows' RoPE; on an index source the indexer (idx_wq_b+ * and its RoPE, weights_proj, the scores against the keys the CPU would read, the+ * candidate blocks, the top-k); the sparse attention with the sink over the window+ * and the selection, the inverse RoPE, the grouped wo_a and wo_b; mHC post; the FFN+ * site's split, collapse and norm. Then the frame is waited for.+ * host: the router and the routed experts (moe_run_at without the shared expert: the+ * expert tier's batch and the CPU's share).+ * device, frame A2 (not waited for): the shared expert (clamped SwiGLU).+ * The final streams and the last FFN site's mix come back to the host, which collapses+ * them and runs the final norm and the head as before.+ *+ * State, and who owns it: the host's stays canonical. Everything a forward changes there+ * (the window ring and its position map, the compressed rows and index keys, the+ * compressor's group, the published index keys) is written back once the forward's last+ * frame is through, as the CPU would have left it (a speculative verify's undo rows+ * included), so a lost device, a rollback, a reset or a CPU forward never has to rebuild+ * anything. The device mirrors it: the window ring holds window + chunk rows (so no row+ * of a chunk overwrites one an earlier row still reads) behind a watermark win_valid;+ * the compressed rows and index keys of each kv_source layer behind kv_valid[layer]; the+ * compressor's group goes up at every forward's start (two groups of head_dim floats).+ * Every host write lowers the watermarks: a CPU forward (v41c_cpu_step), a rejected+ * draft (v41c_rollback), a reset (v41c_reset). A lost device (COLI_VK_CHAIN_FAULT=n+ * fakes it): the forward runs again on the CPU from its input, which the chain leaves+ * untouched until every chunk is through, and the CPU runs from there.+ *+ * The chain declines (the CPU runs the forward, the watermarks follow) under V41_TRACE,+ * for prompts only when COLI_VK_CHAIN=2 and the forward has two rows or fewer, and for a+ * model its shaders or this file do not take: head_dim above 1024, a window plus top-k+ * above 3072 entries, an indexer above 64 heads or 4096 query floats, more than 8 hyper-+ * connection streams, a compressed layer that is not an index source and reads the list+ * of an earlier index source with another ratio (or of none), a candidate-mask consumer+ * with another ratio than the candidate source. */+#include "vk_chain.h"++typedef struct {+ int ok, failed;+ int rows; /* scratch rows */+ int Wd, K, W, rmax; /* device ring rows, top-k slots, window, largest ratio */+ VkcBuf *prm;+ size_t *o_an, *o_fn, *o_qn, *o_kn, *o_sink, *o_hca, *o_hcf, *o_cn, *o_ikn, *o_eq, *o_ek;+ ColiVkTensor **fna, **fnf; /* the mHC mix matrices (f32) */+ VkcBuf **win, **ckv, **ikey, **ring;+ int *kv_valid, *ccap;+ int win_valid;+ VkcBuf *xs, *xn, *mix, *hpa, *hpf, *col, *nrm, *qa, *qr, *q, *kv, *heads, *grp, *br, *craw, *cscr;+ VkcBuf *iq, *ihw, *isc, *mask, *list, *cs, *erows, *ekv, *gs, *us, *hs, *ds;+ VkcBuf *h2d, *mh, *xd, *hpd, *pull, *routed;+ size_t wcap; /* score columns per row */+ float *host_routed;+ unsigned long long forwards;+ double host_ms;+} V41Chain;++static V41Chain *g_v41c;+static int g_vk_chain = 0;+static int g_v41c_inited = 0;++static int v41c_rows(void) {+ const char *e = getenv("COLI_VK_CHAIN_ROWS");+ int v = e && *e ? atoi(e) : 512;+ return v < 1 ? 1 : v > 65535 ? 65535 : v;+}+static int v41c_res(VkcBuf **b, size_t floats, int kind) { return vkc_reserve(b, (floats ? floats : 1) * sizeof(float), kind); }++/* The device copies the per-matrix path keeps (vk_entry), uploaded here if it has not:+ * fp8 as fmt 12 with each 32x32 tile's scale written out for its rows, bf16 as fmt 11. */+static ColiVkTensor *v41c_w8(const W8 *w) {+ if (!w->q || w->O < 1 || w->I < 1) return NULL;+ VkEntry *e = vk_entry(w->q, 12, w->O, w->I);+ if (!e || e->refused) return NULL;+ if (e->t) return e->t;+ int groups = (w->I + FP8_TILE - 1) / FP8_TILE;+ float *scales = malloc((size_t)w->O * groups * sizeof(float));+ if (!scales) return NULL;+ for (int o = 0; o < w->O; o++)+ for (int g = 0; g < groups; g++) scales[(size_t)o * groups + g] = ue8m0(w->s[(size_t)(o / FP8_TILE) * groups + g]);+ int ok = coli_vk_tensor_ensure(&e->t, w->q, scales, 12, w->I, w->O, FP8_TILE);+ free(scales);+ return ok ? e->t : NULL;+}+static ColiVkTensor *v41c_wb(const WB *w) {+ if (!w->w || w->O < 1 || w->I < 1) return NULL;+ VkEntry *e = vk_entry(w->w, 11, w->O, w->I);+ if (!e || e->refused) return NULL;+ if (e->t) return e->t;+ return coli_vk_tensor_ensure(&e->t, w->w, NULL, 11, w->I, w->O, 0) ? e->t : NULL;+}++/* ---- the host's writes: the device's copies follow ------------------------------------- */+static void v41c_lower(Model *m, int pos) {+ V41Chain *ch = g_v41c;+ if (!ch || !ch->ok) return;+ if (pos < 0) pos = 0;+ if (ch->win_valid > pos) ch->win_valid = pos;+ for (int i = 0; i < m->c.n_layers; i++) {+ int r = m->c.compress_ratio[i];+ if (m->c.kv_source[i] && r > 0 && ch->kv_valid[i] > pos / r) ch->kv_valid[i] = pos / r;+ }+}+static void v41c_cpu_step(Model *m, int start_pos) { v41c_lower(m, start_pos); } /* a CPU forward from start_pos */+static void v41c_rollback(Model *m, int committed_end) { v41c_lower(m, committed_end); } /* rows past it rejected */+static void v41c_reset(Model *m) { v41c_lower(m, 0); }++/* ---- setup ----------------------------------------------------------------------------- */+static const char *v41c_unsupported(const Model *m) {+ const Cfg *c = &m->c;+ int L = c->n_layers, K = c->index_topk > 0 ? c->index_topk : 0;+ if (c->head_dim > 1024 || c->head_dim < 1 || c->rope_dim < 2 || (c->rope_dim & 1) || c->rope_dim > c->head_dim)+ return "an attention geometry its shaders do not take";+ if (c->window + K > 3072 || c->window < 1) return "a window and top-k its attention does not take";+ if (c->hc_mult < 1 || c->hc_mult > 8 || c->hc_iters < 1) return "hyper-connections its shaders do not take";+ if ((c->n_heads * c->head_dim) % c->o_groups) return "an output grouping it does not take";+ int any_index = 0;+ for (int i = 0; i < L; i++) any_index |= c->index_source[i] && c->compress_ratio[i] > 0;+ if (any_index && (c->index_n_heads < 1 || c->index_n_heads > 64 || c->index_n_heads * c->index_head_dim > 4096 ||+ c->index_head_dim < c->rope_dim || K < 1 || K > 4096)) return "an indexer its shaders do not take";+ int pub = -1; /* the ratio of the last index source's list */+ for (int i = 0; i < L; i++) {+ int r = c->compress_ratio[i];+ if (r <= 0) continue;+ if (c->index_owner[i] < 0) return "a compressed layer with no kv_source before it";+ if (c->kv_source[i] && r > c->max_positions) return "a compression ratio past the context";+ if (c->index_source[i]) pub = r;+ else if (pub != r) return "a compressed layer that reads another ratio's index list";+ }+ int cs = c->candidate_source;+ if (cs >= 0 && cs < L && c->index_source[cs] && c->compress_ratio[cs] > 0) {+ if (c->candidate_block_size < 1) return "candidate blocks it does not take";+ for (int i = cs + 1; i < L; i++)+ if (c->index_source[i] && c->compress_ratio[i] > 0 && c->compress_ratio[i] != c->compress_ratio[cs])+ return "a candidate mask read by a layer of another ratio";+ }+ return NULL;+}+static int v41c_cand_on(const Cfg *c) {+ int cs = c->candidate_source;+ return cs >= 0 && cs < c->n_layers && c->index_source[cs] && c->compress_ratio[cs] > 0;+}++static int v41c_setup(Model *m) {+ Cfg *c = &m->c;+ int L = c->n_layers, D = c->dim, H = c->hc_mult, nm = (2 + H) * H;+ const char *why = v41c_unsupported(m);+ if (why) { fprintf(stderr, "[VK] deepseek_v41 chain: %s; the CPU runs the layers\n", why); return 0; }+ V41Chain *ch = calloc(1, sizeof *ch);+ if (!ch) return 0;+ size_t **offs[] = {&ch->o_an, &ch->o_fn, &ch->o_qn, &ch->o_kn, &ch->o_sink, &ch->o_hca, &ch->o_hcf, &ch->o_cn, &ch->o_ikn,+ &ch->o_eq, &ch->o_ek};+ for (size_t k = 0; k < sizeof offs / sizeof *offs; k++) if (!(*offs[k] = calloc(L, sizeof(size_t)))) return 0;+ ch->fna = calloc(L, sizeof(void *)); ch->fnf = calloc(L, sizeof(void *));+ ch->win = calloc(L, sizeof(void *)); ch->ckv = calloc(L, sizeof(void *));+ ch->ikey = calloc(L, sizeof(void *)); ch->ring = calloc(L, sizeof(void *));+ ch->kv_valid = calloc(L, sizeof(int)); ch->ccap = calloc(L, sizeof(int));+ if (!ch->fna || !ch->fnf || !ch->win || !ch->ckv || !ch->ikey || !ch->ring || !ch->kv_valid || !ch->ccap) return 0;+ g_v41c = ch;+ ch->W = c->window; ch->K = c->index_topk > 0 ? c->index_topk : 0; ch->rmax = 1;+ for (int i = 0; i < L; i++) if (c->kv_source[i] && c->compress_ratio[i] > ch->rmax) ch->rmax = c->compress_ratio[i];+ ch->Wd = ch->W + v41c_rows();+ /* the parameter arena */+ size_t n = 0;+ for (int i = 0; i < L; i++) {+ ch->o_an[i] = n; n += D; ch->o_fn[i] = n; n += D;+ ch->o_qn[i] = n; n += c->q_lora; ch->o_kn[i] = n; n += c->head_dim;+ ch->o_sink[i] = n; n += c->n_heads;+ ch->o_hca[i] = n; n += 3 + nm; ch->o_hcf[i] = n; n += 3 + nm;+ if (c->kv_source[i]) { ch->o_cn[i] = n; n += c->head_dim; ch->o_ikn[i] = n; n += c->index_head_dim; }+ if (m->L[i].engram_index >= 0) { ch->o_eq[i] = n; n += (size_t)H * D; ch->o_ek[i] = n; n += (size_t)H * D; }+ }+ float *a = calloc(n ? n : 1, sizeof(float));+ if (!a) return 0;+ for (int i = 0; i < L; i++) {+ Layer *l = &m->L[i];+ memcpy(a + ch->o_an[i], l->attn_norm.w, D * sizeof(float));+ memcpy(a + ch->o_fn[i], l->ffn_norm.w, D * sizeof(float));+ memcpy(a + ch->o_qn[i], l->q_norm.w, c->q_lora * sizeof(float));+ memcpy(a + ch->o_kn[i], l->kv_norm.w, c->head_dim * sizeof(float));+ memcpy(a + ch->o_sink[i], l->attn_sink.w, c->n_heads * sizeof(float));+ memcpy(a + ch->o_hca[i], l->hc_attn_scale.w, 3 * sizeof(float));+ memcpy(a + ch->o_hca[i] + 3, l->hc_attn_base.w, nm * sizeof(float));+ memcpy(a + ch->o_hcf[i], l->hc_ffn_scale.w, 3 * sizeof(float));+ memcpy(a + ch->o_hcf[i] + 3, l->hc_ffn_base.w, nm * sizeof(float));+ if (c->kv_source[i]) {+ memcpy(a + ch->o_cn[i], l->comp_norm.w, c->head_dim * sizeof(float));+ memcpy(a + ch->o_ikn[i], l->idx_knorm.w, c->index_head_dim * sizeof(float));+ }+ if (l->engram_index >= 0) {+ memcpy(a + ch->o_eq[i], l->eng_q.w, (size_t)H * D * sizeof(float));+ memcpy(a + ch->o_ek[i], l->eng_k.w, (size_t)H * D * sizeof(float));+ }+ }+ ch->prm = vkc_buf(n * sizeof(float), VKC_DEV);+ int ok = ch->prm && vkc_begin() && vkc_write(ch->prm, 0, a, n * sizeof(float)) && vkc_submit(1);+ free(a);+ /* the tensors */+ for (int i = 0; i < L && ok; i++) {+ Layer *l = &m->L[i];+ int r = c->compress_ratio[i];+ ok = coli_vk_tensor_ensure(&ch->fna[i], l->hc_attn_fn.w, NULL, 10, H * D, nm, 0) &&+ coli_vk_tensor_ensure(&ch->fnf[i], l->hc_ffn_fn.w, NULL, 10, H * D, nm, 0) &&+ v41c_w8(&l->wq_a) && v41c_w8(&l->wq_b) && v41c_w8(&l->wkv) && v41c_w8(&l->wo_a) && v41c_w8(&l->wo_b) &&+ v41c_w8(&l->sh_w1) && v41c_w8(&l->sh_w3) && v41c_w8(&l->sh_w2);+ if (ok && c->kv_source[i]) ok = v41c_wb(&l->comp_wkv) && (r <= 1 || v41c_wb(&l->comp_wgate)) && v41c_wb(&l->idx_wk);+ if (ok && c->index_source[i] && r > 0) ok = v41c_w8(&l->idx_wq_b) && v41c_wb(&l->idx_wproj);+ if (ok && l->engram_index >= 0) ok = v41c_w8(&l->eng_wkv) != NULL;+ if (ok) ok = (ch->win[i] = vkc_buf((size_t)ch->Wd * c->head_dim * sizeof(float), VKC_DEV)) != NULL;+ if (ok && c->kv_source[i] && r > 1)+ ok = (ch->ring[i] = vkc_buf((size_t)2 * r * c->head_dim * sizeof(float), VKC_DEV)) != NULL;+ }+ if (!ok) { fprintf(stderr, "[VK] deepseek_v41 chain: a matrix or the state did not reach the device; the CPU runs the layers\n"); return 0; }+ ch->ok = 1;+ int nsrc = 0, nidx = 0, neng = 0;+ for (int i = 0; i < L; i++) { nsrc += c->kv_source[i]; nidx += c->index_source[i] && c->compress_ratio[i] > 0; neng += m->L[i].engram_index >= 0; }+ fprintf(stderr, "[VK] deepseek_v41 chain: %d layers on the device (%d compressing, %d indexing, %d engram), %d streams, "+ "%.1f MiB of parameters\n", L, nsrc, nidx, neng, H, n * 4 / 1048576.0);+ return 1;+}++/* the compressed rows and index keys of a kv_source layer: room for `rows` rows */+static int v41c_cache(V41Chain *ch, Model *m, int i, int rows) {+ Cfg *c = &m->c;+ if (ch->ccap[i] >= rows) return 1;+ int r = c->compress_ratio[i], host = c->max_positions / (r > 0 ? r : 1);+ int cap = 64; while (cap < rows) cap *= 2;+ if (cap > host) cap = host;+ if (cap < rows) return 0;+ vkc_free(ch->ckv[i]); vkc_free(ch->ikey[i]);+ ch->ckv[i] = vkc_buf((size_t)cap * c->head_dim * sizeof(float), VKC_DEV);+ ch->ikey[i] = vkc_buf((size_t)cap * c->index_head_dim * sizeof(float), VKC_DEV);+ ch->kv_valid[i] = 0; ch->ccap[i] = 0;+ if (!ch->ckv[i] || !ch->ikey[i]) return 0;+ ch->ccap[i] = cap;+ return 1;+}++static int v41c_scratch(V41Chain *ch, Model *m, int rows, int ctx) {+ Cfg *c = &m->c;+ int D = c->dim, H = c->hc_mult, HD = H * D, nm = (2 + H) * H, hr = 2 * H + H * H, nh = c->n_heads, hd = c->head_dim;+ int IH = c->index_n_heads > 0 ? c->index_n_heads : 1, ID = c->index_head_dim > 0 ? c->index_head_dim : 1;+ int ew = m->engram.active ? m->engram.cols * m->engram.head_dim : 1, T = c->n_spec_targets > 0 ? c->n_spec_targets : 1;+ size_t r = (size_t)rows;+ ch->wcap = (size_t)ctx;+ int ok = v41c_res(&ch->xs, r * HD, VKC_DEV) && v41c_res(&ch->xn, r * HD, VKC_DEV) && v41c_res(&ch->mix, r * nm, VKC_DEV) &&+ v41c_res(&ch->hpa, r * hr, VKC_DEV) && v41c_res(&ch->hpf, r * hr, VKC_DEV) && v41c_res(&ch->col, r * D, VKC_DEV) &&+ v41c_res(&ch->nrm, r * D, VKC_DEV) && v41c_res(&ch->qa, r * c->q_lora, VKC_DEV) && v41c_res(&ch->qr, r * c->q_lora, VKC_DEV) &&+ v41c_res(&ch->q, r * nh * hd, VKC_DEV) && v41c_res(&ch->kv, r * hd, VKC_DEV) && v41c_res(&ch->heads, r * nh * hd, VKC_DEV) &&+ v41c_res(&ch->grp, r * c->o_groups * c->o_lora, VKC_DEV) && v41c_res(&ch->br, r * D, VKC_DEV) &&+ v41c_res(&ch->craw, r * hd, VKC_DEV) && v41c_res(&ch->cscr, r * hd, VKC_DEV) &&+ v41c_res(&ch->iq, r * IH * ID, VKC_DEV) && v41c_res(&ch->ihw, r * IH, VKC_DEV) &&+ v41c_res(&ch->isc, r * ch->wcap, VKC_DEV) && v41c_res(&ch->mask, r * ch->wcap, VKC_DEV) &&+ v41c_res(&ch->list, r * (ch->W + ch->K), VKC_DEV) &&+ v41c_res(&ch->cs, (r + ch->rmax) * 2 * c->rope_dim, VKC_DEV) && v41c_res(&ch->erows, r * ew, VKC_DEV) &&+ v41c_res(&ch->ekv, r * (size_t)D * (H + 1), VKC_DEV) && v41c_res(&ch->gs, r * c->moe_inter, VKC_DEV) &&+ v41c_res(&ch->us, r * c->moe_inter, VKC_DEV) && v41c_res(&ch->hs, r * c->moe_inter, VKC_DEV) &&+ v41c_res(&ch->ds, r * D, VKC_DEV) && v41c_res(&ch->h2d, r * D, VKC_DOWN) && v41c_res(&ch->mh, r * T * D, VKC_DOWN) &&+ v41c_res(&ch->xd, r * HD, VKC_DOWN) && v41c_res(&ch->hpd, r * hr, VKC_DOWN) && v41c_res(&ch->routed, r * D, VKC_UP);+ if (!ok) return 0;+ if (ch->rows < rows) {+ float *hrt = realloc(ch->host_routed, r * D * sizeof(float));+ if (!hrt) return 0;+ ch->host_routed = hrt; ch->rows = rows;+ }+ return 1;+}++/* ---- one layer's pieces ----------------------------------------------------------------- */+static int v41c_norm(VkcBuf *x, size_t xo, int xrow, size_t wo, VkcBuf *y, size_t yo, int yrow, int rows, int D, float eps) {+ VkcNorm p = {rows, D, 1, (int)xo, xrow, D, (int)yo, yrow, D, (int)wo, 0, 0, eps, 1.f};+ return vkc_norm(x, g_v41c->prm, y, &p);+}+/* mHC's entry for a site: the mix, the split into hp, the collapse with `prev`'s pre, the norm into nrm */+static int v41c_pre(V41Chain *ch, Model *m, ColiVkTensor *fn, size_t hco, VkcBuf *hp, VkcBuf *prev, size_t lno, int n) {+ Cfg *c = &m->c; int H = c->hc_mult, D = c->dim, HD = H * D, nm = (2 + H) * H, hr = 2 * H + H * H;+ VkcMhc sp = {n, H, D, c->hc_iters, 0, HD, 0, nm, 0, hr, 0, D, (int)hco, 0, c->norm_eps, c->hc_eps, 0.f};+ VkcMhc co = sp;+ return vkc_matmul(fn, ch->xs, 0, ch->mix, 0, n) && vkc_mhc(VKC_MHC_SPLIT, ch->xs, ch->mix, hp, ch->prm, NULL, &sp) &&+ vkc_mhc(VKC_MHC_COLLAPSE, ch->xs, NULL, prev, NULL, ch->col, &co) &&+ v41c_norm(ch->col, 0, D, lno, ch->nrm, 0, D, n, D, c->norm_eps);+}+/* mHC's exit: xn = comb(xs) + post * br, then the two swap */+static int v41c_post(V41Chain *ch, Model *m, VkcBuf *hp, int n) {+ Cfg *c = &m->c; int H = c->hc_mult, D = c->dim, HD = H * D, hr = 2 * H + H * H;+ VkcMhc po = {n, H, D, c->hc_iters, 0, HD, 0, D, 0, hr, 0, HD, 0, 0, c->norm_eps, c->hc_eps, 0.f};+ if (!vkc_mhc(VKC_MHC_POST, ch->xs, ch->br, hp, NULL, ch->xn, &po)) return 0;+ VkcBuf *t = ch->xs; ch->xs = ch->xn; ch->xn = t;+ return 1;+}+/* RoPE on interleaved pairs from this chunk's tables (ch->cs): nseg segments, per_row a row */+static int v41c_rope(V41Chain *ch, Model *m, VkcBuf *x, int nseg, int per_row, int x_off, int x_row, int x_seg,+ int cs_off, int cs_row, int inverse) {+ VkcDsRope p = {nseg, per_row, m->c.rope_dim, x_off, x_row, x_seg, cs_off, cs_row, inverse, 0};+ return vkc_dsv4_rope(x, ch->cs, &p);+}++/* The forward being recorded: positions, its rows, and where its pieces go. */+typedef struct {+ int start, n, pb, nr, spec;+ int csB; /* the compress table's offset in ch->cs: its row 0 is position pb - rmax + 1 */+ int *keyl; /* per index source layer and row: whose index keys it scores (spec: per row) */+ size_t *praw; /* per layer: where a speculative step's raw compressor rows go in ch->pull */+} V41Fwd;++/* attention_run for nr rows on the device: the projections, the window ring, the+ * compressor, the indexer, the sparse attention and the output projection, into ch->br */+static int v41c_attention(V41Chain *ch, Model *m, int i, const V41Fwd *f) {+ Cfg *c = &m->c; Layer *l = &m->L[i];+ int hd = c->head_dim, nh = c->n_heads, rd = c->rope_dim, QL = c->q_lora, r = c->compress_ratio[i];+ int nr = f->nr, pb = f->pb, qrow = nh * hd, LR = ch->W + ch->K;+ int tcs = r > 0 ? f->csB + (ch->rmax - 1) * rd : 0; /* rope_for(layer): the row of position pb */+ int ok = vkc_matmul(v41c_w8(&l->wq_a), ch->nrm, 0, ch->qa, 0, nr) &&+ v41c_norm(ch->qa, 0, QL, ch->o_qn[i], ch->qr, 0, QL, nr, QL, c->norm_eps) &&+ vkc_matmul(v41c_w8(&l->wq_b), ch->qr, 0, ch->q, 0, nr) &&+ vkc_matmul(v41c_w8(&l->wkv), ch->nrm, 0, ch->kv, 0, nr) &&+ v41c_norm(ch->kv, 0, hd, ch->o_kn[i], ch->kv, 0, hd, nr, hd, c->norm_eps) &&+ v41c_rope(ch, m, ch->q, nr * nh, nh, hd - rd, qrow, hd, tcs, rd, 0) &&+ v41c_rope(ch, m, ch->kv, nr, 1, hd - rd, hd, 0, tcs, rd, 0);+ if (!ok) return 0;+ VkcRegion *rg = malloc((size_t)nr * sizeof *rg); /* the new rows into the window ring */+ if (!rg) return 0;+ for (int s = 0; s < nr; s++) rg[s] = (VkcRegion){(size_t)((pb + s) % ch->Wd) * hd, (size_t)s * hd, (size_t)hd};+ ok = vkc_copy_regions(ch->win[i], ch->kv, rg, nr);+ free(rg);+ if (ok && r > 0 && c->kv_source[i]) { /* the compressor, its latents, the index keys */+ int g0 = pb / r, np = (pb + nr) / r - g0, ihd = c->index_head_dim;+ ok = vkc_matmul(v41c_wb(&l->comp_wkv), ch->nrm, 0, ch->craw, 0, nr);+ if (ok && r == 1) ok = v41c_norm(ch->craw, 0, hd, ch->o_cn[i], ch->ckv[i], (size_t)g0 * hd, hd, nr, hd, c->norm_eps);+ else if (ok) {+ VkcDsComp cp = {nr, pb, r, hd, hd, 0, 0, hd, 0, hd, -1, 0, hd, 0};+ ok = vkc_matmul(v41c_wb(&l->comp_wgate), ch->nrm, 0, ch->cscr, 0, nr) &&+ vkc_dsv4_compress(ch->craw, ch->cscr, ch->ring[i], NULL, ch->ckv[i], &cp) &&+ (np == 0 || v41c_norm(ch->ckv[i], (size_t)g0 * hd, hd, ch->o_cn[i], ch->ckv[i], (size_t)g0 * hd, hd, np, hd, c->norm_eps));+ if (ok && f->spec && f->praw) /* a verify's raw rows, for its undo rows */+ ok = vkc_copy(ch->pull, f->praw[i], ch->craw, 0, (size_t)nr * hd) &&+ vkc_copy(ch->pull, f->praw[i] + (size_t)nr * hd, ch->cscr, 0, (size_t)nr * hd);+ }+ if (ok && np > 0) {+ int bc = f->csB + (g0 * r - (pb - ch->rmax + 1)) * rd; /* a latent rotates at its group's first position */+ ok = vkc_matmul(v41c_wb(&l->idx_wk), ch->ckv[i], (size_t)g0 * hd, ch->ikey[i], (size_t)g0 * ihd, np) &&+ v41c_norm(ch->ikey[i], (size_t)g0 * ihd, ihd, ch->o_ikn[i], ch->ikey[i], (size_t)g0 * ihd, ihd, np, ihd, c->norm_eps) &&+ v41c_rope(ch, m, ch->ikey[i], np, 1, g0 * ihd + ihd - rd, ihd, 0, bc, r * rd, 0) &&+ v41c_rope(ch, m, ch->ckv[i], np, 1, g0 * hd + hd - rd, hd, 0, bc, r * rd, 0);+ }+ }+ if (ok && r > 0 && c->index_source[i]) { /* the indexer: this layer's selection */+ int IH = c->index_n_heads, ID = c->index_head_dim, width = (pb + nr) / r, cs = c->candidate_source;+ int cand = v41c_cand_on(c), mrow = cand && cs < i ? (int)ch->wcap : 0;+ float wscale = (1.0f / sqrtf((float)ID)) * (1.0f / sqrtf((float)IH));+ ok = vkc_matmul(v41c_w8(&l->idx_wq_b), ch->qr, 0, ch->iq, 0, nr) &&+ v41c_rope(ch, m, ch->iq, nr * IH, IH, ID - rd, IH * ID, ID, tcs, rd, 0) &&+ vkc_matmul(v41c_wb(&l->idx_wproj), ch->nrm, 0, ch->ihw, 0, nr);+ const int *kl = f->keyl + (size_t)i * f->n + (pb - f->start);+ int same = 1;+ for (int s = 1; s < nr; s++) same &= kl[s] == kl[0];+ for (int s = 0; ok && s < nr; s += same ? nr : 1) {+ VkcDsScore sp = {same ? nr : 1, pb + s, r, IH, ID, width, s * IH * ID, IH * ID, s * IH, IH, 0, ID, mrow,+ (int)ch->wcap, wscale, s * (int)ch->wcap, s * mrow};+ ok = vkc_dsv4_score(ch->iq, ch->ihw, ch->ikey[kl[s]], ch->mask, ch->isc, &sp);+ }+ if (ok && cand && cs == i) {+ VkcDsCand cp = {nr, pb, r, width, c->candidate_block_size, c->candidate_topk_blocks, (int)ch->wcap, (int)ch->wcap};+ ok = vkc_dsv4_cand(ch->isc, ch->mask, &cp);+ }+ VkcDsTopk tp = {nr, width, ch->K, (int)ch->wcap, ch->W, LR, ch->Wd, 0};+ ok = ok && vkc_dsv4_topk(ch->isc, ch->list, &tp);+ }+ if (!ok) return 0;+ VkcDsAttn a = {nr, nh, hd, ch->W + (r > 0 ? ch->K : 0), 0, LR, ch->Wd, 0, 0, 0, qrow, 0, qrow, (int)ch->o_sink[i], 0,+ 1.0f / sqrtf((float)hd)};+ int og = c->o_groups, ol = c->o_lora, pg = qrow / og;+ VkcHgemv g = {0, nr, og, ol, ol, 0, 0, qrow, pg, 0, og * ol, ol, 0, 0, 0};+ return vkc_dsv4_attn(ch->q, ch->win[i], r > 0 ? ch->ckv[c->index_owner[i]] : NULL, ch->list, ch->prm, ch->heads, &a) &&+ v41c_rope(ch, m, ch->heads, nr * nh, nh, hd - rd, qrow, hd, tcs, rd, 1) &&+ vkc_mla_hgemv(v41c_w8(&l->wo_a), ch->heads, ch->grp, NULL, &g) &&+ vkc_matmul(v41c_w8(&l->wo_b), ch->grp, 0, ch->br, 0, nr);+}++/* the shared expert: silu(min(w1 x, lim)) * clamp(w3 x, +-lim), then w2 (swiglu_into) */+static int v41c_shared(V41Chain *ch, Model *m, Layer *l, int nr) {+ Cfg *c = &m->c; int I = c->moe_inter;+ VkcMhc sw = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, nr * I, 0.f, 0.f, c->swiglu_limit > 0.0f ? c->swiglu_limit : INFINITY};+ return vkc_matmul(v41c_w8(&l->sh_w1), ch->nrm, 0, ch->gs, 0, nr) && vkc_matmul(v41c_w8(&l->sh_w3), ch->nrm, 0, ch->us, 0, nr) &&+ vkc_mhc(VKC_SWIGLU_CLAMP, ch->gs, ch->us, NULL, NULL, ch->hs, &sw) &&+ vkc_matmul(v41c_w8(&l->sh_w2), ch->hs, 0, ch->ds, 0, nr);+}+/* engram_run on the device: the host's n-gram rows, eng_wkv, the gate into each stream */+static int v41c_engram(V41Chain *ch, Model *m, int i, int nr, int pb, float *rows) {+ Cfg *c = &m->c; Engram *e = &m->engram; Layer *l = &m->L[i];+ int table = l->engram_index, width = e->cols * e->head_dim, H = c->hc_mult, D = c->dim;+ int64_t ids[V41_MAX_NGRAM * V41_MAX_EHEADS];+ double t0 = now_s();+ for (int r = 0; r < nr; r++) {+ engram_hash(e, table, pb + r, ids);+ for (int col = 0; col < e->cols; col++)+ memcpy(rows + (size_t)r * width + (size_t)col * e->head_dim, engram_row(&e->table[table], ids[col], e->head_dim),+ (size_t)e->head_dim * sizeof(float));+ }+ m->t_engram += now_s() - t0;+ VkcDsEngram p = {nr, H, D, 0, D * (H + 1), 0, H * D, (int)ch->o_eq[i], (int)ch->o_ek[i], c->norm_eps};+ return vkc_write(ch->erows, 0, rows, (size_t)nr * width * sizeof(float)) &&+ vkc_matmul(v41c_w8(&l->eng_wkv), ch->erows, 0, ch->ekv, 0, nr) && vkc_dsv4_engram(ch->ekv, ch->prm, ch->xs, &p);+}++/* Every layer for n rows of streams h (positions start..), the final streams and the last+ * FFN site's mix back into h and pre_mix (every row when all, else the last). 0 = not taken:+ * the CPU runs the layers, h and pre_mix untouched. */+static int v41c_forward(Model *m, float *h, float *pre_mix, int n, int start, int spec, int all) {+ V41Chain *ch = g_v41c;+ if (!g_vk_chain || !ch || !ch->ok || ch->failed || g_trace) return 0;+ if (g_vk_chain == COLI_VK_CHAIN_PREFILL && n <= 2) return 0;+ if (vkc_lost()) { ch->failed = 1; g_vk_chain = 0; return 0; }+ Cfg *c = &m->c;+ int L = c->n_layers, D = c->dim, H = c->hc_mult, HD = H * D, hr = 2 * H + H * H, hd = c->head_dim, rd = c->rope_dim;+ int W = ch->W, LR = W + ch->K, T = m->spec.active ? c->n_spec_targets : 0, E = start + n;+ if (spec && n > W) return 0; /* the undo rows: one window at most */+ int CH = v41c_rows(), rows;+ { int64_t lim = ((int64_t)32 << 20) / (E > 1 ? E : 1); if (lim < CH) CH = lim < 1 ? 1 : (int)lim; }+ if (spec && n > CH) return 0; /* a verify is one chunk */+ rows = n < CH ? n : CH;+ V41Fwd f = {start, n, 0, 0, spec, rows * rd, NULL, NULL};+ /* whose index keys each index source reads, per row (indexer_run, published_owner) */+ f.keyl = malloc((size_t)L * n * sizeof(int));+ f.praw = calloc((size_t)L, sizeof(size_t));+ int *need = calloc((size_t)L, sizeof(int));+ float *outs = all ? malloc((size_t)n * HD * sizeof(float)) : malloc((size_t)HD * sizeof(float));+ float *pres = all ? malloc((size_t)n * H * sizeof(float)) : malloc((size_t)H * sizeof(float));+ int *wl = malloc((size_t)rows * LR * sizeof(int));+ float *cs = malloc((size_t)(rows + ch->rmax) * 2 * rd * sizeof(float));+ float *er = m->engram.active ? malloc((size_t)rows * m->engram.cols * m->engram.head_dim * sizeof(float)) : NULL;+ if (!f.keyl || !f.praw || !need || !outs || !pres || !wl || !cs || (m->engram.active && !er)) {+ free(f.keyl); free(f.praw); free(need); free(outs); free(pres); free(wl); free(cs); free(er);+ return 0;+ }+ static int tidy = -1;+ if (tidy < 0) tidy = getenv("V41_INDEX_OWNER") != NULL;+ /* the rows of each kv_source layer the forward reads that it does not produce first:+ * the complete groups before it, and whatever another layer's selection reaches (the+ * host's rows there, as the CPU reads them) */+ for (int i = 0; i < L; i++) if (c->kv_source[i] && c->compress_ratio[i] > 0) need[i] = start / c->compress_ratio[i];+ int pub = m->published_index_k ? m->published_index_layer : -1, last_pub = -1;+ for (int i = 0; i < L; i++) {+ int r = c->compress_ratio[i];+ if (r <= 0) continue;+ int owner = c->index_owner[i];+ if (c->kv_source[i] && E / r - start / r > 0) { pub = i; last_pub = i; }+ if (E / r > need[owner]) need[owner] = E / r;+ for (int t = 0; t < n; t++) {+ int k = owner;+ if (c->index_source[i] && !tidy) {+ if (m->pub_rows > 0) {+ int p = published_owner(m, i, start, t);+ k = p >= 0 ? p : m->pub_before ? m->pub_before_layer : owner;+ } else if (n == 1 && pub >= 0) k = pub;+ }+ f.keyl[(size_t)i * n + t] = k;+ if (c->index_source[i] && (start + t + 1) / r > need[k]) need[k] = (start + t + 1) / r;+ }+ }+ int ok = 1;+ for (int i = 0; i < L && ok; i++) {+ if (!c->kv_source[i]) continue;+ int r = c->compress_ratio[i], host = c->max_positions / r;+ if (need[i] > host) need[i] = host;+ int want = need[i] > E / r ? need[i] : E / r;+ ok = v41c_cache(ch, m, i, want > 0 ? want : 1);+ }+ /* what comes back at the end: the window rows, the compressed rows and keys, the groups,+ * a verify's raw compressor rows */+ size_t pw = (size_t)(n < W ? n : W) * hd, pn = 0, *pc = calloc((size_t)L, sizeof(size_t));+ for (int i = 0; ok && pc && i < L; i++) {+ pc[i] = pn; pn += pw;+ int r = c->compress_ratio[i];+ if (!c->kv_source[i]) continue;+ pn += (size_t)(E / r - start / r) * (hd + c->index_head_dim);+ if (r > 1) { pn += (size_t)2 * r * hd; if (spec) { f.praw[i] = pn; pn += (size_t)2 * n * hd; } }+ }+ ok = ok && pc && v41c_scratch(ch, m, rows, E) && v41c_res(&ch->pull, pn, VKC_DOWN) && (!T || m->main_hidden);+ if (!ok) {+ fprintf(stderr, "[VK] deepseek_v41 chain: device memory for %d rows at %d positions refused; the CPU runs the layers\n", rows, E);+ free(f.keyl); free(f.praw); free(need); free(outs); free(pres); free(wl); free(cs); free(er); free(pc);+ ch->failed = 1; g_vk_chain = 0;+ return 0;+ }+ vkc_gemm_rows(spec ? 0 : -1); /* a verify's rows get a decode step's bits */+ for (int c0 = 0; c0 < n; c0 += rows) {+ int nr = n - c0 < rows ? n - c0 : rows, pb = start + c0, last = c0 + nr == n;+ f.pb = pb; f.nr = nr;+ /* this chunk's RoPE rows (the CPU's own tables): window ones at 0, compress ones at+ * csB from position pb - rmax + 1 */+ for (int s = 0; s < nr; s++) memcpy(cs + (size_t)s * rd, m->rope_window + (size_t)(pb + s) * rd, rd * sizeof(float));+ for (int s = 0; s < nr + ch->rmax - 1; s++) {+ int p = pb - ch->rmax + 1 + s;+ float *dst = cs + f.csB + (size_t)s * rd;+ if (p < 0) memset(dst, 0, rd * sizeof(float));+ else memcpy(dst, m->rope_compress + (size_t)p * rd, rd * sizeof(float));+ }+ /* the window half of each row's list (window_idxs): the positions of the last+ * `window` the row reaches, oldest first, as device ring rows; -1 where the CPU's+ * ring does not hold the position (window_pos). The selection half starts empty. */+ const Layer *l0 = &m->L[0];+ for (int s = 0; s < nr; s++) {+ int p = pb + s;+ for (int k = 0; k < W; k++) {+ int q = p - W + 1 + k, e = -1;+ if (q >= 0 && (q >= start || l0->window_pos[q % W] == q)) e = q % ch->Wd;+ wl[(size_t)s * LR + k] = e;+ }+ for (int k = W; k < LR; k++) wl[(size_t)s * LR + k] = -1;+ }+ if (!vkc_begin() || !vkc_write(ch->xs, 0, h + (size_t)c0 * HD, (size_t)nr * HD * sizeof(float)) ||+ !vkc_write(ch->list, 0, wl, (size_t)nr * LR * sizeof(int)) ||+ !vkc_write(ch->cs, 0, cs, (size_t)(f.csB + (nr + ch->rmax - 1) * rd) * sizeof(float))) goto lost;+ { /* the first site collapses with [1, 0, ...] (forward_full's pre_mix) */+ float *hp0 = calloc((size_t)nr * hr, sizeof(float));+ if (!hp0) goto lost;+ for (int s = 0; s < nr; s++) hp0[(size_t)s * hr] = 1.0f;+ ok = vkc_write(ch->hpf, 0, hp0, (size_t)nr * hr * sizeof(float));+ free(hp0);+ if (!ok) goto lost;+ }+ if (c0 == 0) { /* the device's copies made the host's, as the forward needs them */+ for (int q = ch->win_valid > start - W + 1 ? ch->win_valid : start - W + 1; q < start && ok; q++) {+ if (q < 0) continue;+ for (int i = 0; i < L && ok; i++) {+ const Layer *l = &m->L[i];+ if (l->window_pos[q % W] != q) continue;+ ok = vkc_write(ch->win[i], (size_t)(q % ch->Wd) * hd, l->window + (size_t)(q % W) * hd, (size_t)hd * sizeof(float));+ }+ }+ for (int i = 0; i < L && ok; i++) {+ if (!c->kv_source[i]) continue;+ const Layer *l = &m->L[i];+ int r = c->compress_ratio[i], ihd = c->index_head_dim, t0 = ch->kv_valid[i], t1 = need[i];+ if (t1 > t0) ok = vkc_write(ch->ckv[i], (size_t)t0 * hd, l->ckv + (size_t)t0 * hd, (size_t)(t1 - t0) * hd * sizeof(float)) &&+ vkc_write(ch->ikey[i], (size_t)t0 * ihd, l->ikey + (size_t)t0 * ihd, (size_t)(t1 - t0) * ihd * sizeof(float));+ if (ok && r > 1) ok = vkc_write(ch->ring[i], 0, l->cstate_kv, (size_t)r * hd * sizeof(float)) &&+ vkc_write(ch->ring[i], (size_t)r * hd, l->cstate_score, (size_t)r * hd * sizeof(float));+ }+ if (!ok) goto lost;+ }+ int pending = 0;+ for (int i = 0; i < L && ok; i++) {+ Layer *l = &m->L[i];+ if (pending) { /* the FFN branch of the layer before */+ VkcEw add = {VKC_EW_ADD, nr * D, D, 1, 0, 1, 0, 0, 0, 0, 0, 1.f};+ ok = vkc_ew(ch->br, ch->routed, ch->ds, NULL, NULL, &add) && v41c_post(ch, m, ch->hpf, nr);+ pending = 0;+ }+ if (ok && l->engram_index >= 0) ok = v41c_engram(ch, m, i, nr, pb, er);+ for (int k = 0; ok && k < T; k++) {+ if (c->spec_targets[k] != i) continue;+ VkcMhc mean = {nr, H, D, 0, 0, HD, 0, 0, 0, 0, k * D, T * D, 0, 0, 0.f, 0.f, 0.f};+ ok = vkc_mhc(VKC_MHC_MEAN, ch->xs, NULL, NULL, NULL, ch->mh, &mean);+ }+ double ta = now_s();+ ok = ok && v41c_pre(ch, m, ch->fna[i], ch->o_hca[i], ch->hpa, ch->hpf, ch->o_an[i], nr) &&+ v41c_attention(ch, m, i, &f) && v41c_post(ch, m, ch->hpa, nr) &&+ v41c_pre(ch, m, ch->fnf[i], ch->o_hcf[i], ch->hpf, ch->hpa, ch->o_fn[i], nr) &&+ vkc_copy(ch->h2d, 0, ch->nrm, 0, (size_t)nr * D) && vkc_submit(1); /* A1 */+ m->t_attn += now_s() - ta;+ if (!ok) break;+ /* A2: the shared expert, while the host computes the routed experts */+ ok = vkc_begin() && v41c_shared(ch, m, l, nr) && vkc_submit(0);+ double t1 = now_s();+ moe_run_at(m, l, &m->cache[i], "layers", i, c->n_routed, c->n_activated, (const float *)vkc_ptr(ch->h2d), nr,+ ch->host_routed, 0);+ memcpy(vkc_ptr(ch->routed), ch->host_routed, (size_t)nr * D * sizeof(float));+ ch->host_ms += (now_s() - t1) * 1e3;+ ok = ok && vkc_begin();+ pending = 1;+ }+ if (ok && pending) {+ VkcEw add = {VKC_EW_ADD, nr * D, D, 1, 0, 1, 0, 0, 0, 0, 0, 1.f};+ ok = vkc_ew(ch->br, ch->routed, ch->ds, NULL, NULL, &add) && v41c_post(ch, m, ch->hpf, nr);+ }+ ok = ok && vkc_copy(ch->xd, 0, ch->xs, 0, (size_t)nr * HD) && vkc_copy(ch->hpd, 0, ch->hpf, 0, (size_t)nr * hr);+ if (ok && last) { /* what the host's state gets back */+ for (int i = 0; i < L && ok; i++) {+ int q0 = E - W > start ? E - W : start, nq = E - q0;+ VkcRegion *rg = malloc((size_t)nq * sizeof *rg);+ if (!rg) { ok = 0; break; }+ for (int q = q0; q < E; q++) rg[q - q0] = (VkcRegion){pc[i] + (size_t)(q - q0) * hd, (size_t)(q % ch->Wd) * hd, (size_t)hd};+ ok = vkc_copy_regions(ch->pull, ch->win[i], rg, nq);+ free(rg);+ if (!ok || !c->kv_source[i]) continue;+ int r = c->compress_ratio[i], g0 = start / r, np = E / r - g0, ihd = c->index_head_dim;+ size_t at = pc[i] + pw;+ ok = vkc_copy(ch->pull, at, ch->ckv[i], (size_t)g0 * hd, (size_t)np * hd) &&+ vkc_copy(ch->pull, at + (size_t)np * hd, ch->ikey[i], (size_t)g0 * ihd, (size_t)np * ihd) &&+ (r <= 1 || vkc_copy(ch->pull, at + (size_t)np * (hd + ihd), ch->ring[i], 0, (size_t)2 * r * hd));+ }+ }+ ok = ok && vkc_submit(1);+ if (!ok) goto lost;+ const float *xd = (const float *)vkc_ptr(ch->xd), *hpd = (const float *)vkc_ptr(ch->hpd);+ for (int s = 0; s < nr; s++) {+ int t = c0 + s;+ if (!all && t != n - 1) continue;+ memcpy(outs + (size_t)(all ? t : 0) * HD, xd + (size_t)s * HD, (size_t)HD * sizeof(float));+ memcpy(pres + (size_t)(all ? t : 0) * H, hpd + (size_t)s * hr, (size_t)H * sizeof(float));+ }+ if (T) memcpy(m->main_hidden + (size_t)c0 * T * D, vkc_ptr(ch->mh), (size_t)nr * T * D * sizeof(float));+ }+ /* every chunk is through: the host's state as the CPU would have left it */+ {+ const float *pl = (const float *)vkc_ptr(ch->pull);+ int save = spec && m->rollback_save && n > 1;+ for (int i = 0; i < L; i++) {+ Layer *l = &m->L[i];+ int q0 = E - W > start ? E - W : start;+ for (int t = 0; t < n; t++) {+ int p = start + t, slot = p % W;+ if (save) {+ memcpy(l->ring_save + (size_t)t * hd, l->window + (size_t)slot * hd, (size_t)hd * sizeof(float));+ l->ring_save_pos[t] = l->window_pos[slot];+ }+ if (p < q0) continue; /* a later row takes its slot */+ memcpy(l->window + (size_t)slot * hd, pl + pc[i] + (size_t)(p - q0) * hd, (size_t)hd * sizeof(float));+ l->window_pos[slot] = p;+ }+ if (!c->kv_source[i]) continue;+ int r = c->compress_ratio[i], g0 = start / r, np = E / r - g0, ihd = c->index_head_dim;+ const float *src = pl + pc[i] + pw;+ memcpy(l->ckv + (size_t)g0 * hd, src, (size_t)np * hd * sizeof(float));+ memcpy(l->ikey + (size_t)g0 * ihd, src + (size_t)np * hd, (size_t)np * ihd * sizeof(float));+ if (r <= 1) continue;+ if (save) { /* the groups' undo rows, as compressor_run's decode path writes them */+ const float *raw = pl + f.praw[i];+ for (int t = 0; t < n; t++) {+ int slot = (start + t) % r;+ memcpy(l->cstate_save_kv + (size_t)t * hd, l->cstate_kv + (size_t)slot * hd, (size_t)hd * sizeof(float));+ memcpy(l->cstate_save_score + (size_t)t * hd, l->cstate_score + (size_t)slot * hd, (size_t)hd * sizeof(float));+ l->cstate_save_slot[t] = slot;+ memcpy(l->cstate_kv + (size_t)slot * hd, raw + (size_t)t * hd, (size_t)hd * sizeof(float));+ memcpy(l->cstate_score + (size_t)slot * hd, raw + (size_t)(n + t) * hd, (size_t)hd * sizeof(float));+ }+ }+ const float *ring = src + (size_t)np * (hd + ihd);+ memcpy(l->cstate_kv, ring, (size_t)r * hd * sizeof(float));+ memcpy(l->cstate_score, ring + (size_t)r * hd, (size_t)r * hd * sizeof(float));+ }+ if (last_pub >= 0) { m->published_index_k = m->L[last_pub].ikey; m->published_index_layer = last_pub; }+ ch->win_valid = E;+ for (int i = 0; i < L; i++) if (c->kv_source[i]) ch->kv_valid[i] = E / c->compress_ratio[i];+ }+ if (all) { memcpy(h, outs, (size_t)n * HD * sizeof(float)); memcpy(pre_mix, pres, (size_t)n * H * sizeof(float)); }+ else { memcpy(h + (size_t)(n - 1) * HD, outs, (size_t)HD * sizeof(float)); memcpy(pre_mix + (size_t)(n - 1) * H, pres, (size_t)H * sizeof(float)); }+ free(f.keyl); free(f.praw); free(need); free(outs); free(pres); free(wl); free(cs); free(er); free(pc);+ ch->forwards++;+ return 1;+lost: /* a frame failed: the device is gone (or would not take a command); the CPU takes over */+ free(f.keyl); free(f.praw); free(need); free(outs); free(pres); free(wl); free(cs); free(er); free(pc);+ if (!vkc_lost()) { vkc_finish(); coli_vk_mark_lost(); }+ g_vk_chain = 0; ch->failed = 1;+ fprintf(stderr, "[VK] deepseek_v41 chain: the device was lost; the CPU runs this forward again and from here on "+ "(the host's state is current: there is nothing to rebuild)\n");+ return 0;+}++static void v41c_report(void) {+ V41Chain *ch = g_v41c;+ if (!ch || !ch->ok || !ch->forwards) return;+ VkcStats st; vkc_stats(&st);+ fprintf(stderr, "[VK] deepseek_v41 chain: %llu forwards, %llu frames (%llu ops, %llu matmuls, %llu tiled GEMM), "+ "%.1f ms waiting for the device, %.1f ms of routed experts on the host, %.1f MiB on the device\n",+ ch->forwards, st.frames, st.ops, st.matmuls, st.gemms, st.wait_ms, ch->host_ms, st.dev_bytes / 1048576.0);+ vkc_prof_print();+}++/* COLI_VK_CHAIN at startup, before the tier sizes itself (the trunk's device copies count+ * as used): the decision, the pipelines, the tensors. */+static void v41c_start(Model *m) {+ if (!g_vk_ready) return;+ int tier_on = vkt_wanted() && m->c.n_routed > 0;+ int on = coli_vk_chain_decide("deepseek_v41", tier_on, COLI_VK_CHAIN_UNMEASURED);+ const char *no = NULL;+ if (on && !(g_v41c_inited = vkc_init())) no = "the chain's pipelines did not come up";+ if (on && !no && !(vkc_mla_ready() && vkc_mhc_ready() && vkc_dsv4_ready()))+ no = "the MLA, mHC or DeepSeek shaders are missing (chain_hgemv, chain_mhc, chain_dsv4)";+ if (no) fprintf(stderr, "[VK] deepseek_v41: %s: the dense chain stays off\n", no);+ if (!on || no) return;+ if (!v41c_setup(m)) return;+ g_vk_chain = on;+}+/* After the tier's: at exit the report, then the chain goes, then the device. */+static void v41c_atexit(void) {+ if (!g_v41c_inited) return;+ atexit(vkc_shutdown);+ atexit(v41c_report);+}