Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat(qwen36): a partial dense chain, the first N layers on the device
JustVugg5ca215dbb492
diff --git a/c/qwen36.c b/c/qwen36.cindex 9f60e7a4..0116e959 100644--- a/c/qwen36.c+++ b/c/qwen36.c@@ -3996,29 +3996,32 @@ static float *step_ex(Model *m, const int *ids, int S, int pos_base, int nlogits}q36_embed_row(m, ids[s], pos_base + s, x + (int64_t)s*D);}+ int chain_n = 0; /* layers the chain ran (a partial chain: the CPU runs the rest) */#ifdef COLI_VULKAN/* COLI_VK_CHAIN: the layers, the final norm and lm_head on the device; x comes- * back only when the prefill read-out below needs every row */+ * back only when the prefill read-out below needs every row. A partial chain+ * brings back every row's residual after its layers: the CPU continues from there. */float *chain_logit = NULL;if (g_vk_chain) {- int echo = 0;+ int echo = 0, rows_only = 0;#ifndef QWEN36_NO_MAINecho = g_echo_k > 0 && g_echo_id && S > 1;#endifif (g_hidden_sink) echo = 1; /* the head reads every row */chain_logit = falloc((int64_t)nlogits * c->vocab);- if (!q36c_forward(m, x, S, pos_base, lf, echo, nlogits, chain_logit)) {+ chain_n = q36c_forward(m, x, S, pos_base, lf, echo, nlogits, chain_logit, &rows_only);+ if (!chain_n) {free(chain_logit); chain_logit = NULL;q36c_cpu_step(m, pos_base);/* Earlier chunks may have returned their final rows for Clef or* prompt logprobs. A failed forward replays every row on the CPU. */for (int s = 0; s < S; s++)q36_embed_row(m, ids[s], pos_base + s, x + (int64_t)s*D);- }+ } else if (rows_only) { free(chain_logit); chain_logit = NULL; }}if (!chain_logit)#endif- layers_forward_range(m, x, S, pos_base, 0, c->n_layers, 1, lf);+ layers_forward_range(m, x, S, pos_base, chain_n, c->n_layers, 1, lf);if (g_hidden_sink) {#pragma omp parallel for schedule(static)for (int s = 0; s < S; s++)@@ -5553,15 +5556,19 @@ static size_t vk_qw_bytes(const QW *w) {: w->q ? (size_t)w->O * w->I + (size_t)w->O * sizeof(float): w->w ? (size_t)w->O * w->I * sizeof(float) : 0;}+/* What still goes to the device after the tier starts: not a matrix placed already (the+ * chain's setup at start, or the dense weights on the device only) nor one that stays+ * on the CPU (a partial chain's CPU layers and head: vk_off). */+static size_t vk_qw_still(const QW *w) { return w->vk || w->vk_off ? 0 : vk_qw_bytes(w); }static size_t vk_dense_bytes(Model *m) {if (!g_vk_dense && !g_vk_chain) return 0;if (coli_vk_dense_device_only()) return 0; /* placed already (q36_dho_start): the free memory the tier reads counts them */- size_t b = vk_qw_bytes(&m->lm_head);+ size_t b = vk_qw_still(&m->lm_head);for (int i = 0; i < m->c.n_layers; i++) {Layer *L = &m->L[i];const QW *w[] = {&L->q, &L->k, &L->v, &L->o, &L->gate, &L->sh_g, &L->sh_u, &L->sh_d,&L->dn_qkv, &L->dn_z, &L->dn_out};- for (size_t k = 0; k < sizeof w / sizeof *w; k++) b += vk_qw_bytes(w[k]);+ for (size_t k = 0; k < sizeof w / sizeof *w; k++) b += vk_qw_still(w[k]);}return b;}@@ -5577,6 +5584,16 @@ static int vk_in_ram(void *ctx, int layer, int e) {* the expert tier sizes its budget: the dense matrices go up now, so the tier sees them* placed. The CUDA tier and a qpack container keep the host copies (the dense part* stays theirs or the CPU's). */+/* With the fit of a partial chain (q36c_start) only the N layers' matrices (and the head's+ * when it goes up) count: the CPU's are vk_off. The chain's setup then drops each layer's+ * host copies once the layer is on the device (q36c_place, then q36_dho_placed). */+static void q36_dho_placed(void) {+ if (coli_vk_device_integrated() && coli_vk_imported_bytes())+ fprintf(stderr, "[VK] qwen36: dense rows shared with the integrated GPU, %.1f MiB read in place; "+ "the imported pages stay alive as the only weight copy\n", coli_vk_imported_bytes() / 1048576.0);+ coli_vk_dense_host_placed("qwen36", "the embedding (its rows are gathered on the CPU), the DeltaNet's a/b rows and "+ "the shared expert's gate vector, norms, the vision tower, any imported pages");+}static void q36_dho_start(Model *m) {if (!g_vk_ready) return;size_t bytes = 0; int n = 0;@@ -5585,13 +5602,10 @@ static void q36_dho_start(Model *m) {return;g_q36_dho_model = m;g_vk_dense = 1; /* the steps the chain declines run their matrices on the device too: the CPU has none */+ if (g_q36c_fit_on && g_vk_chain) return; /* the chain's setup drops them layer by layer (q36c_place) */bytes = 0; n = 0;q36_dho_each(m, q36_dho_drop, &bytes, &n);- if (coli_vk_device_integrated() && coli_vk_imported_bytes())- fprintf(stderr, "[VK] qwen36: dense rows shared with the integrated GPU, %.1f MiB read in place; "- "the imported pages stay alive as the only weight copy\n", coli_vk_imported_bytes() / 1048576.0);- coli_vk_dense_host_placed("qwen36", "the embedding (its rows are gathered on the CPU), the DeltaNet's a/b rows and "- "the shared expert's gate vector, norms, the vision tower, any imported pages");+ q36_dho_placed();}/* The tier's streaming (a big prompt chunk's cold experts on the device): an expert's@@ -6002,7 +6016,15 @@ int main(int argc, char **argv) {}#ifdef COLI_VULKAN+ q36c_start(&m); /* COLI_VK_CHAIN: how many layers the chain places (vkc_fit), before any upload */q36_dho_start(&m); /* COLI_VK_DENSE_HOST: the dense matrices on the device only, before the tier sizes its budget */+ if (g_q36c_fit_on && g_vk_chain) { /* the chain's layers on the device now, so the tier sizes after them */+ q36c_place(&m);+ if (coli_vk_dense_device_only()) {+ coli_vk_dense_host_layers(g_q36c_fit.n, g_q36c_fit.L);+ q36_dho_placed();+ }+ }vk_tier_start(&m, snap, cap, expert_is_int4, expert_mixed); /* COLI_VULKAN=1: hot routed experts on the device */if (g_vk_ready && !vkt_ready() && !g_vk_dense) g_vk_dense = coli_vk_dense_decide("qwen36", 0, 1); /* no tier after all */if (g_vk_chain && qt_ready()) { /* the CUDA tier keeps its priority */@@ -6010,7 +6032,7 @@ int main(int argc, char **argv) {g_vk_chain = 0;}if (g_vk_chain && g_pilot) fprintf(stderr, "[VK] qwen36: PILOT prefetch reads the residual on the host: the dense chain stays off\n");- if (g_vk_chain) atexit(vkc_shutdown); /* registered after the tier's: runs before the device goes */+ if (g_vk_chain || g_q36c_fit_on) atexit(vkc_shutdown); /* registered after the tier's: runs before the device goes */#endif/* coli serve mode: speak the gateway wire protocol instead of argvdiff --git a/c/qwen36_chain.h b/c/qwen36_chain.hindex 07ddc85b..d8a895f3 100644--- a/c/qwen36_chain.h+++ b/c/qwen36_chain.h@@ -54,7 +54,24 @@* state on the CPU from the prefix record (the ids it was built from; the KV rows are* the host's already), stays on the CPU, and the step runs there. Only a state that* the ids do not describe (an image) cannot be rebuilt: that stops the engine.- * COLI_VK_CHAIN_FAULT=n (vk_chain.c) fakes the loss at the n-th frame, for tests. */+ * COLI_VK_CHAIN_FAULT=n (vk_chain.c) fakes the loss at the n-th frame, for tests.+ *+ * A partial chain (vk_chain.h, vkc_fit; docs/vulkan.md, "A partial chain"): when the+ * layers do not all fit the device, the chain takes layers 0..N-1 and the CPU the rest+ * and the head. q36c_start decides N at start, before any upload and before the tier+ * sizes its budget; q36c_setup then places the N layers there and then, one layer at a+ * time (a layer that fails is freed whole and N stops before it), and with the dense+ * weights on the device only drops each layer's host copies once the layer is there.+ * The matrices of the CPU's layers (and of the head, unless the fit puts it up) are+ * marked vk_off: the per-matrix path never uploads them. A forward runs the N layers+ * on the device chunk by chunk as above, brings the residual rows after layer N-1 back+ * to the host once per chunk, and returns; the caller runs layers N..L-1 over all the+ * rows on the CPU (layer by layer, as without the chain) and the head. The device's+ * layers keep their state on the device (KV mirrors, the DeltaNet state and its verify+ * copies), the CPU's layers on the host (their KV rows, DeltaNet state and the CPU's+ * verify snapshots); every loop over layers here runs over the N. A device lost+ * mid-forward rebuilds the N layers' recurrent state from the prefix record: the CPU's+ * layers have not run that forward yet, so theirs is already the host's. */#include "vk_chain.h"#include "vk_kvsplit.h"@@ -64,6 +81,7 @@typedef struct {int ok, failed;+ int n, head; /* layers 0..n-1 on the device; lm_head there too */int rows; /* scratch capacity in rows */int cap; /* the host's kv_cap the mirrors follow */int dev_rows; /* device KV rows a layer: cap, or the split's */@@ -84,6 +102,8 @@ typedef struct {} Q36Chain;static int g_vk_chain = 0; /* COLI_VK_CHAIN decided on, and the chain's pipelines are up */+static VkcFit g_q36c_fit; /* the partial chain's fit (q36c_start) */+static int g_q36c_fit_on; /* the fit ran: the chain sets itself up at start (q36c_place) */static void q36c_fatal(const char *what) {fprintf(stderr, "[VK] qwen36 chain: %s -- stopping (COLI_VK_CHAIN=0 keeps the state on the CPU)\n", what);@@ -91,13 +111,16 @@ static void q36c_fatal(const char *what) {}/* The device was lost with the newest recurrent state on it. The host's KV rows are* canonical, the DeltaNet state is not: rebuild it on the CPU by running the `upto`- * positions the prefix record names (a prefill on the CPU), and leave the chain off. */+ * positions the prefix record names (a prefill on the CPU), and leave the chain off.+ * A partial chain rebuilds its N layers only: the CPU's layers hold their state on the+ * host already, at `upto` (a forward the device was lost in has not reached them). */static void q36c_recover(Model *m, int upto) {Q36Chain *ch = (Q36Chain *)m->vkchain;Cfg *c = &m->c; int D = c->hidden;+ int nd = ch && ch->ok ? ch->n : c->n_layers;g_vk_chain = 0;if (ch) { ch->failed = 1; ch->dn_where = Q36C_HOST; ch->host_zero = 0; ch->snap_valid = 0; }- for (int i = 0; i < c->n_layers; i++) {+ for (int i = 0; i < nd; i++) {if (c->is_attn[i]) continue;memset(m->DN_rec[i], 0, (size_t)c->dn_vheads * c->dn_kdim * c->dn_vdim * sizeof(float));memset(m->DN_conv[i], 0, (size_t)c->dn_conv_dim * (c->dn_convk - 1) * sizeof(float));@@ -115,7 +138,7 @@ static void q36c_recover(Model *m, int upto) {/* a rebuild, not the verify being run: no copies, the batch's own matmuls */int snap = m->snap_rows, rowwise = g_q36_rowwise;m->snap_rows = 0; g_q36_rowwise = 0;- layers_forward_range(m, x, upto, 0, 0, c->n_layers, 0, NULL);+ layers_forward_range(m, x, upto, 0, 0, nd, 0, NULL);m->snap_rows = snap; g_q36_rowwise = rowwise;free(ids); free(x);}@@ -139,7 +162,77 @@ static int q36c_geometry_ok(const Cfg *c) {return 1;}-/* The model's parameters on the device, its tensors resolved; NULL = the chain cannot run. */+/* The matrices of layer i: every one load_tq may have loaded (the per-matrix path's),+ * and those the chain multiplies by, in its setup order (*chain of them first). */+static int q36c_layer_all(Model *m, int i, QW **ws) {+ Layer *l = &m->L[i];+ QW *all[] = {&l->q, &l->k, &l->v, &l->o, &l->gate, &l->sh_g, &l->sh_u, &l->sh_d, &l->dn_qkv, &l->dn_z, &l->dn_out};+ for (size_t k = 0; k < sizeof all / sizeof *all; k++) ws[k] = all[k];+ return (int)(sizeof all / sizeof *all);+}+static int q36c_layer_qws(Model *m, int i, QW **ws) {+ Cfg *c = &m->c; Layer *l = &m->L[i]; int n = 0;+ if (c->is_attn[i]) { ws[n++] = &l->q; ws[n++] = &l->k; ws[n++] = &l->v; ws[n++] = &l->o; }+ else { ws[n++] = &l->dn_qkv; ws[n++] = &l->dn_z; ws[n++] = &l->dn_out; }+ if (c->n_experts > 0) ws[n++] = &l->gate;+ if (c->shared_inter > 0) { ws[n++] = &l->sh_g; ws[n++] = &l->sh_u; ws[n++] = &l->sh_d; }+ return n;+}++/* Layer i stays on the CPU for good: what the chain placed of it is freed (after every+ * frame that may read it) and its matrices are marked so the per-matrix path never+ * uploads them. Called for a layer that did not fully reach the device, before any of+ * its host copies was dropped. */+static void q36c_layer_cpu(Q36Chain *ch, Model *m, int i) {+ if (vkc_ready()) vkc_finish();+ QW *ws[16]; int nw = q36c_layer_all(m, i, ws);+ for (int k = 0; k < nw; k++) {+ QW *w = ws[k];+ if (w->vk) {+ const void *wq; const float *sc;+ int fmt = vk_qw_fmt(w, &wq, &sc);+ unsigned *pc = &g_vk_placed[w->vk_imported ? (fmt == 1 ? 0 : 3) : fmt == 1 ? 0 : fmt == 4 ? 1 : fmt == 14 ? 3 : 2];+ if (*pc) (*pc)--;+ coli_vk_tensor_free((ColiVkTensor *)w->vk);+ w->vk = NULL; w->vk_imported = 0;+ }+ w->vk_off = 1;+ }+ if (ch && ch->t_ab) {+ ColiVkTensor **t[] = {&ch->t_ab[i], &ch->t_sg[i]};+ VkcBuf **b[] = {&ch->rec[i], &ch->ring[i]};+ vkc_layer_free(t, 2, b, 2);+ }+}++/* Layer i's matrices, its DeltaNet b|a rows and shared-expert gate row, and its state on+ * the device; 0 = something did not get there (the caller frees the layer). */+static int q36c_place_layer(Q36Chain *ch, Model *m, int i) {+ Cfg *c = &m->c; Layer *l = &m->L[i]; int D = c->hidden, vh = c->dn_vheads;+ QW *ws[16]; int nw = q36c_layer_qws(m, i, ws), ok = 1;+ for (int k = 0; k < nw && ok; k++) ok = vk_qw_tensor(ws[k]) != NULL;+ if (ok && !c->is_attn[i]) {+ float *ab = malloc((size_t)2 * vh * D * sizeof(float));+ if (ab) {+ memcpy(ab, l->dn_b, (size_t)vh * D * sizeof(float));+ memcpy(ab + (size_t)vh * D, l->dn_a, (size_t)vh * D * sizeof(float));+ ch->t_ab[i] = q36c_f32_tensor(ab, D, 2 * vh);+ free(ab);+ }+ ch->rec[i] = ch->t_ab[i] ? vkc_buf((size_t)vh * c->dn_kdim * c->dn_vdim * sizeof(float), VKC_DEV) : NULL;+ ch->ring[i] = ch->rec[i] ? vkc_buf((size_t)c->dn_conv_dim * (c->dn_convk - 1) * sizeof(float), VKC_DEV) : NULL;+ ok = ch->t_ab[i] && ch->rec[i] && ch->ring[i];+ }+ if (ok && l->sh_gate) ok = (ch->t_sg[i] = q36c_f32_tensor(l->sh_gate, D, 1)) != NULL;+ return ok;+}++/* The chain on the device: the parameter arena, then layer after layer (each whole, its+ * host copies dropped once it is there when the dense weights live on the device only),+ * then the head when the fit puts it up. With the fit (q36c_start) it runs at start,+ * before the tier sizes its budget; a layer that does not reach the device is freed+ * whole and the chain keeps the layers before it (vkc_fit_shrink). NULL = the chain+ * cannot run. */static Q36Chain *q36c_setup(Model *m) {Q36Chain *ch = (Q36Chain *)m->vkchain;if (ch) return ch->ok ? ch : NULL;@@ -152,6 +245,8 @@ static Q36Chain *q36c_setup(Model *m) {c->head_dim, c->dn_kdim, c->dn_vdim);return NULL;}+ VkcFit *fit = g_q36c_fit_on ? &g_q36c_fit : NULL;+ int n = fit ? fit->n : L;ch->o_in = calloc(L, sizeof(size_t)); ch->o_post = calloc(L, sizeof(size_t)); ch->o_qn = calloc(L, sizeof(size_t));ch->o_kn = calloc(L, sizeof(size_t)); ch->o_conv = calloc(L, sizeof(size_t)); ch->o_dn = calloc(L, sizeof(size_t));ch->t_ab = calloc(L, sizeof(void *)); ch->t_sg = calloc(L, sizeof(void *));@@ -160,24 +255,23 @@ static Q36Chain *q36c_setup(Model *m) {ch->kv_valid = calloc(L, sizeof(int)); ch->attn_ord = calloc(L, sizeof(int));if (!ch->o_in || !ch->o_post || !ch->o_qn || !ch->o_kn || !ch->o_conv || !ch->o_dn || !ch->t_ab || !ch->t_sg ||!ch->rec || !ch->ring || !ch->kc || !ch->vc || !ch->kv_valid || !ch->attn_ord) return NULL;- /* the parameter arena: offsets, then one upload */- size_t n = 0;+ /* the parameter arena of the n layers: offsets, then one upload */+ size_t np = 0;int vh = c->dn_vheads, conv_dim = c->dn_conv_dim, convk = c->dn_convk;- for (int i = 0; i < L; i++) {- ch->o_in[i] = n; n += D; ch->o_post[i] = n; n += D;+ for (int i = 0; i < n; i++) {+ ch->o_in[i] = np; np += D; ch->o_post[i] = np; np += D;if (c->is_attn[i]) {- ch->attn_ord[i] = ch->n_attn++;- if (m->L[i].qn) { ch->o_qn[i] = n; n += c->head_dim; }- if (m->L[i].kn) { ch->o_kn[i] = n; n += c->k_head_dim; }+ if (m->L[i].qn) { ch->o_qn[i] = np; np += c->head_dim; }+ if (m->L[i].kn) { ch->o_kn[i] = np; np += c->k_head_dim; }} else {- ch->o_conv[i] = n; n += (size_t)conv_dim * convk;- ch->o_dn[i] = n; n += 2 * (size_t)vh + c->dn_vdim;+ ch->o_conv[i] = np; np += (size_t)conv_dim * convk;+ ch->o_dn[i] = np; np += 2 * (size_t)vh + c->dn_vdim;}}- ch->o_final = n; n += D;- float *arena = calloc(n, sizeof(float));+ ch->o_final = np; np += D;+ float *arena = calloc(np, sizeof(float));if (!arena) return NULL;- for (int i = 0; i < L; i++) {+ for (int i = 0; i < n; i++) {Layer *l = &m->L[i];memcpy(arena + ch->o_in[i], l->in_ln, D * sizeof(float));memcpy(arena + ch->o_post[i], l->post_ln, D * sizeof(float));@@ -192,50 +286,161 @@ static Q36Chain *q36c_setup(Model *m) {}}memcpy(arena + ch->o_final, m->final_norm, D * sizeof(float));- ch->prm = vkc_buf(n * sizeof(float), VKC_DEV);- int ok = ch->prm && vkc_begin() && vkc_write(ch->prm, 0, arena, n * sizeof(float)) && vkc_submit(1);+ ch->prm = vkc_buf(np * sizeof(float), VKC_DEV);+ int ok = ch->prm && vkc_begin() && vkc_write(ch->prm, 0, arena, np * sizeof(float)) && vkc_submit(1);free(arena);- if (!ok) return NULL;- /* the tensors: the same device copies the per-matrix path uses */- for (int i = 0; i < L && ok; i++) {- Layer *l = &m->L[i];- if (c->is_attn[i]) ok = vk_qw_tensor(&l->q) && vk_qw_tensor(&l->k) && vk_qw_tensor(&l->v) && vk_qw_tensor(&l->o);- else {- ok = vk_qw_tensor(&l->dn_qkv) && vk_qw_tensor(&l->dn_z) && vk_qw_tensor(&l->dn_out);- float *ab = ok ? malloc((size_t)2 * vh * D * sizeof(float)) : NULL;- if (ab) {- memcpy(ab, l->dn_b, (size_t)vh * D * sizeof(float));- memcpy(ab + (size_t)vh * D, l->dn_a, (size_t)vh * D * sizeof(float));- ch->t_ab[i] = q36c_f32_tensor(ab, D, 2 * vh);- free(ab);- }- ok = ok && ch->t_ab[i];- if (ok) {- ch->rec[i] = vkc_buf((size_t)vh * c->dn_kdim * c->dn_vdim * sizeof(float), VKC_DEV);- ch->ring[i] = vkc_buf((size_t)conv_dim * (convk - 1) * sizeof(float), VKC_DEV);- ok = ch->rec[i] && ch->ring[i];- }+ if (!ok) {+ if (!fit) return NULL;+ vkc_free(ch->prm); ch->prm = NULL;+ for (int i = 0; i < n; i++) q36c_layer_cpu(ch, m, i);+ vkc_fit_shrink("qwen36", fit, 0, vkc_lost() ? "the device was lost" : "its parameter buffer was refused");+ m->lm_head.vk_off = 1;+ g_vk_chain = 0;+ vkc_shutdown(); /* the chain's own blocks and frames go too */+ vkc_fit_placed("qwen36", fit);+ return NULL;+ }+ /* the layers: the same device copies the per-matrix path uses; a layer that does+ * not get there whole is freed and the chain stops before it */+ int placed = 0;+ for (int i = 0; i < n; i++) {+ if (!q36c_place_layer(ch, m, i)) {+ if (!fit) { fprintf(stderr, "[VK] qwen36 chain: a matrix did not reach the device; per-matrix path\n"); return NULL; }+ for (int j = i; j < n; j++) q36c_layer_cpu(ch, m, j);+ vkc_fit_shrink("qwen36", fit, i, vkc_lost() ? "the device was lost" : "an upload was refused");+ break;}- if (ok && c->n_experts > 0) ok = vk_qw_tensor(&l->gate) != NULL;- if (ok && c->shared_inter > 0) ok = vk_qw_tensor(&l->sh_g) && vk_qw_tensor(&l->sh_u) && vk_qw_tensor(&l->sh_d);- if (ok && l->sh_gate) ok = (ch->t_sg[i] = q36c_f32_tensor(l->sh_gate, D, 1)) != NULL;+ if (coli_vk_dense_device_only()) { /* the layer is there whole: its host copies go */+ QW *ws[16]; int nw = q36c_layer_qws(m, i, ws), nd = 0; size_t b = 0;+ for (int k = 0; k < nw; k++) q36_dho_drop(ws[k], &b, &nd);+ }+ if (fit) vkc_fit_mark(fit, i);+ placed = i + 1;+ }+ ch->n = placed;+ for (int i = 0; i < placed; i++) if (c->is_attn[i]) ch->attn_ord[i] = ch->n_attn++;+ if (!placed) { /* nothing of the chain stays: its arena, blocks and frames go */+ vkc_free(ch->prm); ch->prm = NULL;+ m->lm_head.vk_off = 1;+ g_vk_chain = 0;+ vkc_shutdown();}- ok = ok && vk_qw_tensor(&m->lm_head);- if (!ok) { fprintf(stderr, "[VK] qwen36 chain: a matrix did not reach the device; per-matrix path\n"); return NULL; }+ if (fit) vkc_fit_placed("qwen36", fit);+ if (!placed) return NULL;+ /* the head: with every layer, and room for it (the fit's tail) */+ ch->head = (!fit || fit->tail) && vk_qw_tensor(&m->lm_head);+ if (!ch->head) {+ if (fit) fit->tail = 0;+ if (!m->lm_head.vk) m->lm_head.vk_off = 1;+ } else if (coli_vk_dense_device_only()) { int nd = 0; size_t b = 0; q36_dho_drop(&m->lm_head, &b, &nd); }ch->dn_where = Q36C_HOST; ch->host_zero = 0;ch->ok = 1;- fprintf(stderr, "[VK] qwen36 chain: %d layers on the device (%d attention), %.1f MiB of parameters\n",- L, ch->n_attn, n * 4 / 1048576.0);+ fprintf(stderr, "[VK] qwen36 chain: %d layers on the device (%d attention)%s, %.1f MiB of parameters\n",+ placed, ch->n_attn, ch->head ? "" : ", the head on the CPU", np * 4 / 1048576.0);return ch;}-/* q36c_res counts instead of reserving while g_q36c_count >= 0 (the chunk's sizing) */+/* q36c_res counts instead of reserving while g_q36c_count >= 0 (the chunk's sizing), and+ * adds each buffer's device bytes to *g_q36c_fitsum while it is set (the fit) */static long long g_q36c_count = -1;+static size_t *g_q36c_fitsum;static int q36c_res(VkcBuf **b, size_t floats, int kind) {+ if (g_q36c_fitsum) { *g_q36c_fitsum += vkc_fit_buf((floats ? floats : 1) * sizeof(float)); return 1; }if (g_q36c_count >= 0) { g_q36c_count += (long long)(floats ? floats : 1) * (long long)sizeof(float); return 1; }return vkc_reserve(b, (floats ? floats : 1) * sizeof(float), kind);}static int q36c_bufs(Q36Chain *ch, Model *m, int rows);++/* ---- the fit: how many layers the chain places (vk_chain.h, vkc_fit) -------------- */+/* Does vk_qw_tensor read w's rows in place (COLI_VK_IMPORT)? Then its scales alone take+ * device memory. The dense-host decision comes after the fit, so its one input that+ * changes the answer, COLI_VK_DENSE_HOST=0, is read here too. */+static int q36c_imports(const QW *w, int fmt, const void *wq) {+ const char *e = getenv("COLI_VK_DENSE_HOST");+ int only = coli_vk_dense_device_only() || (e && *e && atoi(e) == 0);+ return g_vk_import && (coli_vk_device_integrated() || !only) && fmt != 4 && w->w != wq;+}+/* w as vk_qw_tensor places it: its device bytes (dev) and what coli_vk_mem_info counts+ * for it (mat); nothing for a matrix that is not loaded or stays on the CPU */+static void q36c_fit_qw(const QW *w, size_t *dev, size_t *mat) {+ const void *wq; const float *sc;+ if (w->vk_off) return;+ int fmt = vk_qw_fmt(w, &wq, &sc), gs = fmt == 4 ? 64 : 0;+ if (!wq) return;+ if (q36c_imports(w, fmt, wq)) {+ size_t s = coli_vk_tensor_scale_count(fmt, w->I, w->O, gs) * sizeof(float), a = coli_vk_buffer_alignment();+ *dev += ((s ? s : 4) + a - 1) / a * a; *mat += s;+ return;+ }+ *dev += vkc_fit_tensor(fmt, w->I, w->O, gs);+ *mat += coli_vk_tensor_payload(fmt, w->I, w->O, gs);+}+/* Layer i on the device: its matrices, the DeltaNet b|a rows and shared-expert gate row+ * (f32 tensors), the DeltaNet state and conv ring, the K/V mirror at its starting size+ * (`kv_rows` positions: the split's floor of three blocks; past it the split decides+ * from what is free then), and its share of the parameter arena. */+static void q36c_fit_layer(Model *m, int i, int kv_rows, size_t *dev, size_t *mat) {+ Cfg *c = &m->c; Layer *l = &m->L[i]; int D = c->hidden, vh = c->dn_vheads;+ QW *ws[16]; int nw = q36c_layer_qws(m, i, ws);+ for (int k = 0; k < nw; k++) q36c_fit_qw(ws[k], dev, mat);+ size_t prm = 2 * (size_t)D;+ if (c->is_attn[i]) {+ if (l->qn) prm += c->head_dim;+ if (l->kn) prm += c->k_head_dim;+ *dev += 2 * vkc_fit_buf((size_t)c->kv_heads * kv_rows * c->k_head_dim * sizeof(float));+ } else {+ prm += (size_t)c->dn_conv_dim * c->dn_convk + 2 * (size_t)vh + c->dn_vdim;+ *dev += vkc_fit_tensor(10, D, 2 * vh, 0); *mat += coli_vk_tensor_payload(10, D, 2 * vh, 0);+ *dev += vkc_fit_buf((size_t)vh * c->dn_kdim * c->dn_vdim * sizeof(float)) ++ vkc_fit_buf((size_t)c->dn_conv_dim * (c->dn_convk - 1) * sizeof(float));+ }+ if (l->sh_gate) { *dev += vkc_fit_tensor(10, D, 1, 0); *mat += coli_vk_tensor_payload(10, D, 1, 0); }+ *dev += prm * sizeof(float);+}+/* The fit, once at start (main, before the dense-host decision and the tier): N layers+ * from layer 0 on the device, the CPU's layers and (unless the fit puts it up) the head+ * marked vk_off so that nothing uploads them. N = 0 turns the chain off. Not run where+ * the chain cannot run at all (the CUDA tier, a qpack container, PILOT, a geometry the+ * shaders do not take): those keep today's handling. */+static void q36c_start(Model *m) {+ if (!g_vk_chain || qt_ready() || qq_active() || g_pilot || !q36c_geometry_ok(&m->c)) return;+ Cfg *c = &m->c; int L = c->n_layers, D = c->hidden;+ size_t *per = calloc((size_t)L, sizeof *per), *mat = calloc((size_t)L, sizeof *mat);+ if (!per || !mat) { free(per); free(mat); return; }+ const char *e = getenv("COLI_VK_KV_BLOCK");+ int B = e && *e ? atoi(e) : 64;+ if (B < 1) B = 1;+ for (int i = 0; i < L; i++) q36c_fit_layer(m, i, 3 * B, &per[i], &mat[i]);+ /* fixed: the arena's final norm (and one alignment of its rounding), the scratch of+ * one prompt chunk (q36c_bufs, counted with every attention layer's K/V rows) and its+ * rows' read-back */+ Q36Chain cnt; memset(&cnt, 0, sizeof cnt);+ for (int i = 0; i < L; i++) cnt.n_attn += c->is_attn[i] != 0;+ int rows = vkc_fit_rows(256);+ size_t fixed = vkc_fit_buf((size_t)D * sizeof(float)) + vkc_fit_buf(4);+ g_q36c_fitsum = &fixed;+ q36c_bufs(&cnt, m, rows);+ q36c_res(&cnt.xd, (size_t)rows * D, VKC_DOWN);+ g_q36c_fitsum = NULL;+ size_t tail = 0, tail_m = 0;+ q36c_fit_qw(&m->lm_head, &tail, &tail_m);+ int n = vkc_fit("qwen36", L, per, mat, fixed, tail, &g_q36c_fit);+ free(per); free(mat);+ g_q36c_fit_on = 1;+ for (int i = n; i < L; i++) q36c_layer_cpu(NULL, m, i);+ if (!g_q36c_fit.tail) m->lm_head.vk_off = 1;+ if (n == 0) { /* the chain off: its blocks and pipelines go, every byte is the tier's */+ g_vk_chain = 0;+ vkc_shutdown();+ vkc_fit_placed("qwen36", &g_q36c_fit);+ }+}+/* After the dense-host decision, before the tier: the chain's N layers (and the head)+ * on the device now, so the tier sizes its budget from what is left. */+static void q36c_place(Model *m) {+ if (!g_q36c_fit_on || !g_q36c_fit.n || !g_vk_chain) return;+ q36c_setup(m);+}/* Prompt rows per chunk (vkc_chunk_rows): the chain's scratch a row, counted from the* reservations themselves, and the routed experts' outputs (the tier's rows, the* CPU's contributions, and the host's sum) for it. */@@ -277,7 +482,7 @@ static int q36c_mirror(Q36Chain *ch, Model *m) {(size_t)ch->n_attn * ch->dev_rows * row) : 1;if (!plan) { ch->cap = 0; return 0; }ch->dev_rows = ch->ks.on ? ch->ks.rows : m->kv_cap;- for (int i = 0; i < c->n_layers; i++) {+ for (int i = 0; i < ch->n; i++) {if (!c->is_attn[i]) continue;vkc_free(ch->kc[i]); vkc_free(ch->vc[i]); ch->kc[i] = ch->vc[i] = NULL;ch->kv_valid[i] = 0;@@ -311,7 +516,7 @@ static void q36c_sync_host(Model *m) {if (!ch || !ch->ok || ch->dn_where != Q36C_DEV) return;Cfg *c = &m->c;size_t nr = (size_t)c->dn_vheads * c->dn_kdim * c->dn_vdim, nc = (size_t)c->dn_conv_dim * (c->dn_convk - 1);- for (int i = 0; i < c->n_layers; i++) {+ for (int i = 0; i < ch->n; i++) {if (c->is_attn[i]) continue;if (!vkc_read(ch->rec[i], 0, m->DN_rec[i], nr * sizeof(float)) ||!vkc_read(ch->ring[i], 0, m->DN_conv[i], nc * sizeof(float))) { q36c_recover(m, m->kv_len); return; }@@ -330,7 +535,7 @@ static void q36c_cpu_step(Model *m, int pos_base) {if (!ch || !ch->ok) return;q36c_sync_host(m);ch->dn_where = Q36C_HOST; ch->host_zero = 0; ch->snap_valid = 0;- for (int i = 0; i < m->c.n_layers; i++) {+ for (int i = 0; i < ch->n; i++) {if (ch->kv_valid[i] > pos_base) ch->kv_valid[i] = pos_base;if (m->c.is_attn[i]) vkc_kv_lower(&ch->ks, ch->attn_ord[i], pos_base);}@@ -341,12 +546,12 @@ static void q36c_cpu_step(Model *m, int pos_base) {static void q36c_rollback(Model *m, int slot, int len) {Q36Chain *ch = (Q36Chain *)m->vkchain;if (!ch || !ch->ok) return;- for (int i = 0; i < m->c.n_layers; i++) {+ for (int i = 0; i < ch->n; i++) {if (ch->kv_valid[i] > len) ch->kv_valid[i] = len;if (m->c.is_attn[i]) vkc_kv_lower(&ch->ks, ch->attn_ord[i], len);}if (slot < 0 || slot >= ch->snap_valid || ch->dn_where != Q36C_DEV) { ch->snap_valid = 0; return; }- for (int i = 0; i < m->c.n_layers; i++) {+ for (int i = 0; i < ch->n; i++) {if (m->c.is_attn[i]) continue;VkcBuf *t = ch->rec[i]; ch->rec[i] = ch->rec_snap[slot][i]; ch->rec_snap[slot][i] = t;t = ch->ring[i]; ch->ring[i] = ch->ring_snap[slot][i]; ch->ring_snap[slot][i] = t;@@ -362,7 +567,7 @@ static int q36c_spec_slots(Q36Chain *ch, Model *m, int rows) {for (int sl = ch->snap_slots; sl < rows; sl++) {if (!ch->rec_snap[sl] && !(ch->rec_snap[sl] = (VkcBuf **)calloc((size_t)L, sizeof(VkcBuf *)))) return 0;if (!ch->ring_snap[sl] && !(ch->ring_snap[sl] = (VkcBuf **)calloc((size_t)L, sizeof(VkcBuf *)))) return 0;- for (int i = 0; i < L; i++) {+ for (int i = 0; i < ch->n; i++) {if (c->is_attn[i]) continue;if (!ch->rec_snap[sl][i] && !(ch->rec_snap[sl][i] = vkc_buf(nr * sizeof(float), VKC_DEV))) return 0;if (!ch->ring_snap[sl][i] && !(ch->ring_snap[sl][i] = vkc_buf(nc * sizeof(float), VKC_DEV))) return 0;@@ -376,7 +581,7 @@ static int q36c_push_state(Q36Chain *ch, Model *m, int pos_base, int n_rows) {Cfg *c = &m->c; int ok = 1;if (ch->dn_where == Q36C_HOST) {size_t nr = (size_t)c->dn_vheads * c->dn_kdim * c->dn_vdim, nc = (size_t)c->dn_conv_dim * (c->dn_convk - 1);- for (int i = 0; i < c->n_layers && ok; i++) {+ for (int i = 0; i < ch->n && ok; i++) {if (c->is_attn[i]) continue;if (ch->host_zero) ok = vkc_zero(ch->rec[i], 0, nr) && vkc_zero(ch->ring[i], 0, nc);else ok = vkc_write(ch->rec[i], 0, m->DN_rec[i], nr * sizeof(float)) &&@@ -385,7 +590,7 @@ static int q36c_push_state(Q36Chain *ch, Model *m, int pos_base, int n_rows) {ch->dn_where = Q36C_BOTH;}int kvd = c->k_head_dim, KV = c->kv_heads;- for (int i = 0; i < c->n_layers && ok; i++) {+ for (int i = 0; i < ch->n && ok; i++) {if (c->is_attn[i] && ch->ks.on) { /* the split: the window placed, its rows below pos_base uploaded */VkcKvPart pt[2] = {{KV, kvd, m->K[i], (size_t)m->max_t * kvd, ch->kc[i], 0},{KV, kvd, m->V[i], (size_t)m->max_t * kvd, ch->vc[i], 0}};@@ -494,14 +699,21 @@ static int q36c_combine(Q36Chain *ch, Model *m, int i, int n) {}/* Every layer for S rows from host rows xh, the last row's logits into `logit`; xh gets- * the final rows back when want_x. 0 = not taken (nothing on the device changed). */-static int q36c_forward(Model *m, float *xh, int S, int pos_base, FILE *lf, int want_x, int nlogits, float *logit) {+ * the final rows back when want_x. 0 = not taken (nothing on the device changed).+ * Otherwise the layers it ran: with a partial chain (N layers, or the head on the CPU)+ * *rows_only is 1, xh holds every row's residual after those layers, and the caller+ * runs the rest of the layers and the head on the CPU. */+static int q36c_forward(Model *m, float *xh, int S, int pos_base, FILE *lf, int want_x, int nlogits, float *logit,+ int *rows_only) {+ *rows_only = 0;if (!g_vk_chain || qt_ready() || qq_active() || g_pilot) return 0;/* prompts only: decode and verifies on the CPU */if (g_vk_chain == COLI_VK_CHAIN_PREFILL && (S <= 2 || g_q36_rowwise)) return 0;Q36Chain *ch = q36c_setup(m);if (!ch || ch->failed) return 0;- Cfg *c = &m->c; int D = c->hidden, L = c->n_layers, E = c->n_experts;+ Cfg *c = &m->c; int D = c->hidden, L = ch->n, E = c->n_experts; /* L: the layers on the device */+ int part = L < c->n_layers || !ch->head;+ if (part) { want_x = 1; nlogits = 0; } /* the residual comes back; the CPU runs the rest and the head */int mirror_ok = q36c_mirror(ch, m);int CH = mirror_ok ? q36c_chunk_rows(ch, m) : 1, rows = S < CH ? S : CH;if (ch->ks.on && rows > ch->ks.chunk) rows = ch->ks.chunk; /* a step's rows fit the split's window */@@ -598,7 +810,7 @@ static int q36c_forward(Model *m, float *xh, int S, int pos_base, FILE *lf, intif (ok && want_x) ok = vkc_copy(ch->xd, 0, ch->x, 0, (size_t)n * D);int last = c0 + n == S;/* the final norm and lm_head on this chunk's share of the last nlogits rows */- int lo0 = S - nlogits > c0 ? S - nlogits - c0 : 0, lo_n = n - lo0;+ int lo0 = S - nlogits > c0 ? S - nlogits - c0 : 0, lo_n = nlogits ? n - lo0 : 0;if (ok && lo_n > 0) {int dst = c0 + lo0 - (S - nlogits);ok = q36c_norm(ch->x, (size_t)lo0 * D, ch->prm, ch->o_final, ch->fin, (size_t)dst * D, lo_n, D, c->eps) &&@@ -620,13 +832,14 @@ static int q36c_forward(Model *m, float *xh, int S, int pos_base, FILE *lf, intif (want_x) memcpy(xh + (size_t)c0 * D, vkc_ptr(ch->xd), (size_t)n * D * sizeof(float));for (int i = 0; i < L; i++) if (c->is_attn[i]) { ch->kv_valid[i] = pb + n; vkc_kv_done(&ch->ks, ch->attn_ord[i], pb + n); }ch->dn_where = Q36C_DEV; ch->host_zero = 0;- if (last) memcpy(logit, vkc_ptr(ch->outd), (size_t)nlogits * c->vocab * sizeof(float));+ if (last && !part) memcpy(logit, vkc_ptr(ch->outd), (size_t)nlogits * c->vocab * sizeof(float));}if (lf) fwrite(vkc_ptr(ch->lfd), sizeof(float), (size_t)L * 3 * D, lf);ch->snap_valid = snapped;vkc_gemm_rows(-1);ch->forwards++;- return 1;+ *rows_only = part;+ return L;lost: /* a frame failed: the device is gone (or would not take a command); the CPU takes over */vkc_gemm_rows(-1);if (!vkc_lost()) { vkc_finish(); coli_vk_mark_lost(); }