Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
qwen38: the chain's layers on two devices (COLI_VK_DEV2)
JustVugg2fa3e97712b5
diff --git a/c/qwen38.c b/c/qwen38.cindex 6600e24e..846e14ec 100644--- a/c/qwen38.c+++ b/c/qwen38.c@@ -2160,7 +2160,7 @@ int main(int argc, char **argv) {#ifdef COLI_VULKANq38_vk_tier_start(&m, cap); /* COLI_VULKAN=1: hot routed experts on the device (vk_tier.c) */if(g_vk_ready&&!vkt_ready()&&!g_vk_dense)g_vk_dense=coli_vk_dense_decide("qwen38",0,1); /* no tier after all */- if(g_vk_chain)atexit(vkc_shutdown); /* registered after the tier's: runs before the device goes */+ if(g_vk_chain)atexit(vkc_shutdown_all); /* registered after the tier's: runs before the device goes */#endiffprintf(stderr, "resident weights loaded in %.1fs | RSS after load: %.2f GB\n", m.dense_load_s, rss_gb());diff --git a/c/qwen38_chain.h b/c/qwen38_chain.hindex 188c2b9e..7de7630d 100644--- a/c/qwen38_chain.h+++ b/c/qwen38_chain.h@@ -60,7 +60,8 @@typedef struct {int ok, failed, rows, cap;- int nl, head; /* the layers on the device (the first N of c.layers) and whether the final+ int lo, d; /* its layers start at lo, on device d (vkc_device) */+ int nl, head; /* the layers on the device (lo..lo+nl-1 of c.layers) and whether the final* mixer and lm_head are there too; a partial chain hands the rest to the CPU */int built, placed_said; /* q38c_setup ran; the fit's placed line was printed */size_t prm_floats;@@ -86,6 +87,10 @@ typedef struct {unsigned long long forwards;double host_ms;} Q38Chain;+/* the primary device's chain (d 0) or the second device's (d 1, after it) */+static Q38Chain *q38c_of(Model *m, int d) { return (Q38Chain *)(d ? m->vkchain2 : m->vkchain); }+static const char *q38c_name(const Q38Chain *ch) { return ch && ch->d ? "qwen38 dev2" : "qwen38"; }+static int q38c_dev2_wanted(void) { const char *e = getenv("COLI_VK_CHAIN_DEV2"); return !(e && *e == '0'); }static int g_vk_chain = 0;static void q38c_fatal(const char *what) {@@ -159,7 +164,7 @@ static int q38c_chunk_rows(Q38Chain *ch, Model *m) {+ sizeof(Q38RouteAssignment) + 4 * sizeof(int) + sizeof(uint8_t) + sizeof(float *);row += (size_t)c->topk * assignment+ ((size_t)c->experts + 3 * (size_t)c->shared_inter + c->hidden + 1) * sizeof(float);- return vkc_chunk_rows("qwen38", row);+ return vkc_chunk_rows(q38c_name(ch), row);}/* Plan and allocate the KV mirror before sizing a prompt chunk. */static int q38c_mirror(Q38Chain *ch, Model *m) {@@ -167,11 +172,11 @@ static int q38c_mirror(Q38Chain *ch, Model *m) {int nbmax = m->kv_cap / (c->idx_ratio > 0 ? c->idx_ratio : 1) + 1;if (ch->cap != m->kv_cap) { /* the host cache is new: mirror it again (K/V whole, or split past the budget) */size_t row = (size_t)kvo * 2 * 4;- int plan = ch->n_attn ? vkc_kv_plan(&ch->ks, "qwen38", ch->n_attn, row, m->kv_cap, 1, 1,+ int plan = ch->n_attn ? vkc_kv_plan(&ch->ks, q38c_name(ch), ch->n_attn, row, m->kv_cap, 1, 1,(size_t)ch->n_attn * ch->dev_rows * row) : 1;if (!plan) { ch->cap = 0; return 0; }ch->dev_rows = ch->ks.on ? ch->ks.rows : m->kv_cap;- for (int i = 0; i < ch->nl; i++) {+ for (int i = ch->lo; i < ch->lo + ch->nl; i++) {if (!c->is_attn[i]) continue;vkc_free(ch->kc[i]); vkc_free(ch->vc[i]); vkc_free(ch->ik[i]); vkc_free(ch->pk[i]);ch->kv_valid[i] = ch->pk_valid[i] = 0;@@ -226,6 +231,14 @@ static int q38c_scratch(Q38Chain *ch, Model *m, int rows) {* keep their host copies and run on the CPU. */static VkcFit g_q38c_fit; /* q38c_start's decision */static int g_q38c_fitted;+/* The layers after the primary's on COLI_VK_DEV2's device (docs/vulkan.md, "Layers on two+ * devices"): a second chain from layer g_q38c_fit.n, its own fit on that device's memory,+ * placed at startup after the primary's. A forward runs the primary's layers, brings the+ * streams back and runs these; the CPU runs what is left and, unless this chain ends at+ * the last layer with room for it, the head. */+static VkcFit g_q38c_fit2;+static int g_q38c_fit2_on;+static int g_q38c_ran; /* the layers the last forward ran on the devices (q38c_layers) */#define Q38C_MATS 18#define Q38C_TAIL 32/* Layer i's matrices as the chain reads them (the PLE projections with their layer); the@@ -309,7 +322,8 @@ static int q38c_place_mats(Q38Chain *ch, Model *m, int i) {if (!q38c_t(w[k])) return 0;if (k == 9 && !ch->t_sg[i]) {ColiVkTensor *t = NULL;- if (!coli_vk_tensor_ensure(&t, m->L[i].sh_gate, NULL, 10, m->c.hidden, 1, 0)) return 0;+ if (!(ch->d ? coli_vk_tensor_ensure2(&t, m->L[i].sh_gate, NULL, 10, m->c.hidden, 1, 0)+ : coli_vk_tensor_ensure(&t, m->L[i].sh_gate, NULL, 10, m->c.hidden, 1, 0))) return 0;ch->t_sg[i] = t;}}@@ -363,27 +377,27 @@ static void q38c_unplace_tail(Model *m) {static void q38c_shrink(Q38Chain *ch, Model *m, int k, const char *why) {for (int j = k; j < m->c.layers; j++) q38c_unplace(ch, m, j);q38c_unplace_tail(m);- if (k < ch->nl) ch->nl = k;+ if (k - ch->lo < ch->nl) ch->nl = k - ch->lo;ch->head = 0;g_q38_vk_noup = 1;- if (g_q38c_fitted) vkc_fit_shrink("qwen38", &g_q38c_fit, k, why);+ if (g_q38c_fitted) vkc_fit_shrink(q38c_name(ch), ch->d ? &g_q38c_fit2 : &g_q38c_fit, k - ch->lo, why);}/* The head kept on the CPU (an upload of it refused): what of the tail went up goes. */static void q38c_head_off(Q38Chain *ch, Model *m) {q38c_unplace_tail(m);- ch->head = 0; g_q38c_fit.tail = 0; g_q38_vk_noup = 1;+ ch->head = 0; (ch->d ? &g_q38c_fit2 : &g_q38c_fit)->tail = 0; g_q38_vk_noup = 1;fprintf(stderr, "[VK] qwen38 chain: the head did not reach the device: the final mixer and lm_head run on the CPU ""(what it had placed was freed)\n");}/* The parameters the shaders read (norm weights, the DeltaNet constants, the PLE's) of* the layers on the device, in one buffer; the attention layers' order (n_attn). */static int q38c_arena(Q38Chain *ch, Model *m) {- Cfg *c = &m->c; int L = ch->nl, W = c->hc_width;+ Cfg *c = &m->c; int lo = ch->lo, hi = ch->lo + ch->nl, W = c->hc_width;int VH = c->dn_vheads, CD = c->dn_conv_dim, CK = c->dn_convk;- int ple = c->ple_layer >= 0 && c->ple_layer < L;+ int ple = c->ple_layer >= lo && c->ple_layer < hi;size_t n = 0;ch->n_attn = 0;- for (int i = 0; i < L; i++) {+ for (int i = lo; i < hi; i++) {ch->o_an[i] = n; n += W; ch->o_mn[i] = n; n += W;if (c->is_attn[i]) {ch->attn_ord[i] = ch->n_attn++;@@ -398,7 +412,7 @@ static int q38c_arena(Q38Chain *ch, Model *m) {if (ple) { ch->o_ple = n; n += 3 * (size_t)W; ch->o_pconv = n; n += (size_t)W * c->ple_convk; }float *a = calloc(n, sizeof(float));if (!a) return 0;- for (int i = 0; i < L; i++) {+ for (int i = lo; i < hi; i++) {Layer *l = &m->L[i];memcpy(a + ch->o_an[i], l->attn_gr.norm, W * sizeof(float));memcpy(a + ch->o_mn[i], l->mlp_gr.norm, W * sizeof(float));@@ -429,10 +443,11 @@ static int q38c_arena(Q38Chain *ch, Model *m) {}/* The chain's host-side state, no device memory yet. NULL: a model or geometry its* shaders do not take (the per-matrix path), or no memory. */-static Q38Chain *q38c_new(Model *m) {+static Q38Chain *q38c_new_dev(Model *m, int d) {Q38Chain *ch = (Q38Chain *)calloc(1, sizeof *ch);if (!ch) return NULL;- m->vkchain = ch;+ if (d) m->vkchain2 = ch; else m->vkchain = ch;+ ch->d = d;Cfg *c = &m->c; int L = c->layers;if (m->range_begin != 0 || m->range_end != L || !m->lm_head.rows || !q38c_geometry_ok(c)) {fprintf(stderr, "[VK] qwen38 chain: a model or geometry its shaders do not take; per-matrix path\n");@@ -447,46 +462,63 @@ static Q38Chain *q38c_new(Model *m) {Q38C_ARR(kv_valid, int); Q38C_ARR(pk_valid, int); Q38C_ARR(attn_ord, int);#undef Q38C_ARRch->snap_slots = 1;- ch->nl = g_q38c_fitted ? g_q38c_fit.n : L;- ch->head = !g_q38c_fitted || !vkc_fit_partial(&g_q38c_fit);+ if (d) { /* from the primary's last layer; the head with the last layer and room for it */+ ch->lo = g_q38c_fit.n; ch->nl = g_q38c_fit2.n;+ ch->head = ch->lo + ch->nl == L && !vkc_fit_partial(&g_q38c_fit2);+ } else {+ ch->nl = g_q38c_fitted ? g_q38c_fit.n : L;+ ch->head = !g_q38c_fitted || !vkc_fit_partial(&g_q38c_fit);+ }return ch;}+static Q38Chain *q38c_new(Model *m) { return q38c_new_dev(m, 0); }/* Every layer the fit gave the device, one at a time (its matrices, its state), then the* parameters, the placed line and the head. A partial chain runs it at startup* (q38c_start), the full chain at its first forward, as before. */-static Q38Chain *q38c_setup(Model *m) {- Q38Chain *ch = (Q38Chain *)m->vkchain;+/* d = 1: the second device's chain (the caller has made that device current). */+static Q38Chain *q38c_setup_dev(Model *m, int d) {+ Q38Chain *ch = q38c_of(m, d);if (ch && ch->built) return ch->ok ? ch : NULL;- if (!ch && !(ch = q38c_new(m))) return NULL;+ if (d && !g_q38c_fit2_on) return NULL;+ if (!ch && !(ch = q38c_new_dev(m, d))) return NULL;ch->built = 1;- VkcFit *fit = g_q38c_fitted ? &g_q38c_fit : NULL;- for (int i = 0; i < ch->nl; i++) {- if (q38c_place_mats(ch, m, i) && q38c_place_bufs(ch, m, i)) { if (fit) vkc_fit_mark(fit, i); continue; }+ const char *nm = q38c_name(ch);+ VkcFit *fit = d ? &g_q38c_fit2 : g_q38c_fitted ? &g_q38c_fit : NULL;+ g_q38_vk_dev = d;+ for (int i = ch->lo; i < ch->lo + ch->nl; i++) {+ if (q38c_place_mats(ch, m, i) && q38c_place_bufs(ch, m, i)) { if (fit) vkc_fit_mark(fit, i - ch->lo); continue; }q38c_shrink(ch, m, i, vkc_lost() ? "the device was lost" : "the device refused an upload or an allocation");break;}- if (ch->nl && !q38c_arena(ch, m)) q38c_shrink(ch, m, 0, "the device refused the chain's parameter buffer");- if (fit && !ch->placed_said) { vkc_fit_placed("qwen38", fit); ch->placed_said = 1; }+ if (ch->nl && !q38c_arena(ch, m)) q38c_shrink(ch, m, ch->lo, "the device refused the chain's parameter buffer");+ if (fit && !ch->placed_said) { vkc_fit_placed(nm, fit); ch->placed_said = 1; }if (ch->nl && ch->head) {Q38Weight *w[Q38C_TAIL]; int n = q38c_tail_mats(m, w, 1), ok = 1;for (int k = 0; k < n && ok; k++) ok = q38c_t(w[k]) != NULL;if (!ok) q38c_head_off(ch, m);}+ g_q38_vk_dev = 0;if (!ch->nl) { /* nothing of the chain on the device: its own pools go too */if (!fit) fprintf(stderr, "[VK] qwen38 chain: a matrix did not reach the device; per-matrix path\n");- vkc_shutdown(); g_vk_chain = 0;+ vkc_shutdown();+ if (!d) g_vk_chain = 0; /* the second device's failing: the primary's chain stands, the CPU runs the rest */return NULL;}ch->dn_where = Q38C_HOST;ch->ok = 1;size_t wb = 0, wn = 0;- coli_vk_mem_info(&wb, &wn);- fprintf(stderr, "[VK] qwen38 chain: %d layers on the device (%d QSA), %zu matrices resident, %.1f MiB of parameters%s\n",- ch->nl, ch->n_attn, wn, ch->prm_floats * 4 / 1048576.0,- ch->head ? "" : "; the CPU runs the other layers, the final mixer and lm_head");+ coli_vk_mem_info_dev(d, &wb, &wn);+ if (d) fprintf(stderr, "[VK] qwen38 chain: layers %d..%d on the second device (%d QSA), %zu matrices resident there, "+ "%.1f MiB of parameters%s\n", ch->lo, ch->lo + ch->nl - 1, ch->n_attn, wn,+ ch->prm_floats * 4 / 1048576.0, ch->head ? ", the final mixer and lm_head too" : "");+ else fprintf(stderr, "[VK] qwen38 chain: %d layers on the device (%d QSA), %zu matrices resident, %.1f MiB of parameters%s\n",+ ch->nl, ch->n_attn, wn, ch->prm_floats * 4 / 1048576.0,+ ch->head ? "" : "; the CPU runs the other layers, the final mixer and lm_head");return ch;}-static int q38c_layers(Model *m) { Q38Chain *ch = (Q38Chain *)m->vkchain; return ch && ch->ok ? ch->nl : 0; }+static Q38Chain *q38c_setup(Model *m) { return q38c_setup_dev(m, 0); }+/* the layers the last forward ran on the devices, from layer 0 */+static int q38c_layers(Model *m) { (void)m; return g_q38c_ran; }/* At startup, with the chain on (before anything of it is on the device, the dense-host* pass and the expert tier): N, the chain's pipelines (vkc_init), and a partial chain@@ -501,14 +533,40 @@ static void q38c_start(Model *m) {size_t *lb = (size_t *)calloc((size_t)L, sizeof *lb), *mb = (size_t *)calloc((size_t)L, sizeof *mb);if (!lb || !mb) { free(lb); free(mb); return; }for (int i = 0; i < L; i++) q38c_fit_layer(m, i, rows, &lb[i], &mb[i]);- vkc_fit("qwen38", L, lb, mb, q38c_fit_fixed(m, rows), q38c_fit_tail(m), &g_q38c_fit);- free(lb); free(mb);+ size_t fixed = q38c_fit_fixed(m, rows), tail = q38c_fit_tail(m);+ vkc_fit("qwen38", L, lb, mb, fixed, tail, &g_q38c_fit);g_q38c_fitted = 1;- int partial = vkc_fit_partial(&g_q38c_fit);- if (!g_q38c_fit.n) { g_vk_chain = 0; g_q38_vk_noup = 1; return; } /* the chain off: its pipelines never come up */- if (!vkc_init()) { g_vk_chain = 0; g_q38c_fitted = 0; return; } /* the per-matrix path, as before */+ int partial = vkc_fit_partial(&g_q38c_fit), n0 = g_q38c_fit.n;+ if (!n0) { g_vk_chain = 0; g_q38_vk_noup = 1; free(lb); free(mb); return; } /* the chain off: its pipelines never come up */+ if (!vkc_init()) { g_vk_chain = 0; g_q38c_fitted = 0; free(lb); free(mb); return; } /* the per-matrix path, as before */+ /* the layers the primary leaves, on COLI_VK_DEV2's device: a fit of its own from layer+ * n0, with that device's free memory (the tier there sizes itself after the placement) */+ if (partial && n0 < L && q38c_dev2_wanted() && getenv("COLI_VK_DEV2") && coli_vk_dev2_open_env()) {+ vkc_device(1);+ int n2 = vkc_fit("qwen38 dev2", L - n0, lb + n0, mb + n0, fixed, tail, &g_q38c_fit2);+ if (n2 > 0 && !vkc_init()) {+ fprintf(stderr, "[VK] qwen38 chain: the second device's pipelines did not come up; its layers stay on the CPU\n");+ n2 = 0;+ }+ g_q38c_fit2_on = n2 > 0;+ vkc_device(0);+ }+ free(lb); free(mb);if (!partial) return; /* every layer and the head: set up at the first forward, as before */- if (!q38c_setup(m)) { vkc_shutdown(); g_vk_chain = 0; }+ Q38Chain *ch = q38c_setup(m);+ if (!ch) { vkc_shutdown(); g_vk_chain = 0; }+ if (g_q38c_fit2_on) {+ vkc_device(1);+ if (!ch || ch->nl < n0) {+ /* the primary placed fewer layers than its fit: the second device's would not+ * follow them, so they stay on the CPU too */+ fprintf(stderr, "[VK] qwen38 chain: the primary device stopped before layer %d; layers %d..%d stay on the CPU, "+ "not on the second device\n", n0, n0, n0 + g_q38c_fit2.n - 1);+ g_q38c_fit2_on = 0;+ vkc_shutdown();+ } else if (!q38c_setup_dev(m, 1)) g_q38c_fit2_on = 0;+ vkc_device(0);+ }g_q38_vk_noup = 1; /* after the setup's own uploads */}/* The dense-host pass's bound (q38_dho_start): the N layers' matrices of a partial chain,@@ -516,7 +574,12 @@ static void q38c_start(Model *m) {* part is on the CPU. */static void q38c_dho_bound(int *layers, int *head, int *dev) {if (!g_q38c_fitted) return;- if (vkc_fit_partial(&g_q38c_fit)) { *layers = g_q38c_fit.n; *head = 0; }+ if (vkc_fit_partial(&g_q38c_fit)) {+ *layers = g_q38c_fit.n; *head = 0;+ /* the second device's layers too, where the chain runs every step (with prompts+ * only, decode reads them on the CPU); the head keeps its host copy */+ if (g_q38c_fit2_on && g_vk_chain == COLI_VK_CHAIN_ON) *layers += g_q38c_fit2.n;+ }if (!*layers) *dev = 0;}/* The dense-host pass with the full chain fitted (the matrices go up at startup in this@@ -554,7 +617,9 @@ static int q38c_dho_place(Model *m, size_t *bytes, int *n) {}/* The dense-host lines say which layers dropped their host copies. */static void q38c_dho_layers(void) {- if (g_q38c_fitted) coli_vk_dense_host_layers(g_q38c_fit.n, g_q38c_fit.L);+ if (g_q38c_fitted)+ coli_vk_dense_host_layers(g_q38c_fit.n + (g_q38c_fit2_on && g_vk_chain == COLI_VK_CHAIN_ON ? g_q38c_fit2.n : 0),+ g_q38c_fit.L);}/* ---- state between the host and the device ------------------------------------ */@@ -568,11 +633,16 @@ static void q38_embed_row(Model *m,int id,int abs_pos,float *out);* partial chain only the device's layers are rebuilt: the CPU's layers (and the PLE* layer's state and history when it is one of them) hold their own state already. */static void q38c_recover(Model *m, int upto) {- Q38Chain *ch = (Q38Chain *)m->vkchain;+ Q38Chain *ch = (Q38Chain *)m->vkchain, *ch2 = (Q38Chain *)m->vkchain2;Cfg *c = &m->c; int H = c->hidden, W = c->hc_width, C = c->hc_count;- int nl = ch ? ch->nl : c->layers, ple = c->ple_layer >= 0 && c->ple_layer < nl;+ int nl = ch ? ch->nl : c->layers;+ if (ch && ch->ok && ch2 && ch2->ok) nl += ch2->nl; /* the second device's layers ran in that forward too */+ int ple = c->ple_layer >= 0 && c->ple_layer < nl;g_vk_chain = 0;- if (ch) { ch->failed = 1; ch->dn_where = Q38C_HOST; ch->host_zero = 0; ch->snap_valid = 0; }+ for (int d = 0; d < 2; d++) {+ Q38Chain *x = q38c_of(m, d);+ if (x) { x->failed = 1; x->dn_where = Q38C_HOST; x->host_zero = 0; x->snap_valid = 0; }+ }for (int i = 0; i < nl; i++) {if (c->is_attn[i]) continue;memset(m->DN_rec[i], 0, (size_t)c->dn_vheads * c->dn_kdim * c->dn_vdim * sizeof(float));@@ -603,60 +673,70 @@ static void q38c_recover(Model *m, int upto) {m->ple_pref = pref; m->ple_pref_rows = pref_rows; m->snap_rows = snap; g_q38_rowwise = rowwise;free(ids); free(hyper); free(mixed); free(inject); free(block);}-static void q38c_sync_host(Model *m) {- Q38Chain *ch = (Q38Chain *)m->vkchain;- if (!ch || !ch->ok || ch->dn_where != Q38C_DEV) return;+/* (every function here covers both chains: the primary's layers and the second device's) */+static int q38c_sync_one(Model *m, Q38Chain *ch) {+ if (!ch || !ch->ok || ch->dn_where != Q38C_DEV) return 1;Cfg *c = &m->c;size_t nr = (size_t)c->dn_vheads * c->dn_kdim * c->dn_vdim, nc = (size_t)c->dn_conv_dim * (c->dn_convk - 1);- for (int i = 0; i < ch->nl; i++) {+ int was = vkc_device(ch->d), ok = 1;+ for (int i = ch->lo; i < ch->lo + ch->nl && ok; i++) {if (c->is_attn[i]) continue;- if (!vkc_read(ch->rec[i], 0, m->DN_rec[i], nr * 4) || !vkc_read(ch->ring[i], 0, m->DN_conv[i], nc * 4)) {- q38c_recover(m, m->kv_len); return;- }- }- if (ch->ple_ring && m->PLE_conv_state &&- !vkc_read(ch->ple_ring, (size_t)ch->ple_slot[0] * q38c_ple_cells(c), m->PLE_conv_state, q38c_ple_cells(c) * 4)) {- q38c_recover(m, m->kv_len); return;+ ok = vkc_read(ch->rec[i], 0, m->DN_rec[i], nr * 4) && vkc_read(ch->ring[i], 0, m->DN_conv[i], nc * 4);}- ch->dn_where = Q38C_BOTH;+ if (ok && ch->ple_ring && m->PLE_conv_state)+ ok = vkc_read(ch->ple_ring, (size_t)ch->ple_slot[0] * q38c_ple_cells(c), m->PLE_conv_state, q38c_ple_cells(c) * 4);+ vkc_device(was);+ if (ok) ch->dn_where = Q38C_BOTH;+ return ok;+}+static void q38c_sync_host(Model *m) {+ for (int d = 0; d < 2; d++)+ if (!q38c_sync_one(m, q38c_of(m, d))) { q38c_recover(m, m->kv_len); return; }}static void q38c_host_wrote(Model *m, int zero) {- Q38Chain *ch = (Q38Chain *)m->vkchain;- if (!ch || !ch->ok) return;- ch->dn_where = Q38C_HOST; ch->host_zero = zero; ch->snap_valid = 0;+ for (int d = 0; d < 2; d++) {+ Q38Chain *ch = q38c_of(m, d);+ if (!ch || !ch->ok) continue;+ ch->dn_where = Q38C_HOST; ch->host_zero = zero; ch->snap_valid = 0;+ }}-static void q38c_cpu_step(Model *m, int pos_base) {- Q38Chain *ch = (Q38Chain *)m->vkchain;+/* A CPU step from pos_base over one chain's layers: its host state current first. */+static void q38c_cpu_one(Model *m, Q38Chain *ch, int pos_base) {if (!ch || !ch->ok) return;- q38c_sync_host(m);+ if (!q38c_sync_one(m, ch)) { q38c_recover(m, m->kv_len); return; }ch->dn_where = Q38C_HOST; ch->host_zero = 0; ch->snap_valid = 0;- for (int i = 0; i < ch->nl; i++) {+ for (int i = ch->lo; i < ch->lo + ch->nl; i++) {if (ch->kv_valid[i] > pos_base) ch->kv_valid[i] = pos_base;if (m->c.is_attn[i]) vkc_kv_lower(&ch->ks, ch->attn_ord[i], pos_base);int R = m->c.idx_ratio > 0 ? m->c.idx_ratio : 1;if (ch->pk_valid[i] > pos_base / R) ch->pk_valid[i] = pos_base / R;}}+static void q38c_cpu_step(Model *m, int pos_base) {+ for (int d = 0; d < 2; d++) q38c_cpu_one(m, q38c_of(m, d), pos_base);+}/* A rejected draft: the state after the verify's row `slot` (its first slot+1 rows) is* the device's copy in that slot; `len` positions stand. The KV and pooled-key* watermarks come down to them: the rows past them are the rejected drafts'. */static void q38c_rollback(Model *m, int slot, int len) {- Q38Chain *ch = (Q38Chain *)m->vkchain;- if (!ch || !ch->ok) return;int R = m->c.idx_ratio > 0 ? m->c.idx_ratio : 1;- for (int i = 0; i < ch->nl; i++) {- if (ch->kv_valid[i] > len) ch->kv_valid[i] = len;- if (m->c.is_attn[i]) vkc_kv_lower(&ch->ks, ch->attn_ord[i], len);- if (ch->pk_valid[i] > len / R) ch->pk_valid[i] = len / R;- }- if (slot < 0 || slot >= ch->snap_valid || ch->dn_where != Q38C_DEV) { ch->snap_valid = 0; return; }- for (int i = 0; i < ch->nl; i++) {- if (m->c.is_attn[i]) continue;- VkcBuf *t = ch->rec[i]; ch->rec[i] = ch->rec_snap[slot][i]; ch->rec_snap[slot][i] = t;- t = ch->ring[i]; ch->ring[i] = ch->ring_snap[slot][i]; ch->ring_snap[slot][i] = t;+ for (int d = 0; d < 2; d++) {+ Q38Chain *ch = q38c_of(m, d);+ if (!ch || !ch->ok) continue;+ for (int i = ch->lo; i < ch->lo + ch->nl; i++) {+ if (ch->kv_valid[i] > len) ch->kv_valid[i] = len;+ if (m->c.is_attn[i]) vkc_kv_lower(&ch->ks, ch->attn_ord[i], len);+ if (ch->pk_valid[i] > len / R) ch->pk_valid[i] = len / R;+ }+ if (slot < 0 || slot >= ch->snap_valid || ch->dn_where != Q38C_DEV) { ch->snap_valid = 0; continue; }+ for (int i = ch->lo; i < ch->lo + ch->nl; i++) {+ if (m->c.is_attn[i]) continue;+ VkcBuf *t = ch->rec[i]; ch->rec[i] = ch->rec_snap[slot][i]; ch->rec_snap[slot][i] = t;+ t = ch->ring[i]; ch->ring[i] = ch->ring_snap[slot][i]; ch->ring_snap[slot][i] = t;+ }+ if (ch->ple_ring) { int t = ch->ple_slot[0]; ch->ple_slot[0] = ch->ple_slot[1 + slot]; ch->ple_slot[1 + slot] = t; }+ ch->snap_valid = 0;}- if (ch->ple_ring) { int t = ch->ple_slot[0]; ch->ple_slot[0] = ch->ple_slot[1 + slot]; ch->ple_slot[1 + slot] = t; }- ch->snap_valid = 0;}/* Copy slots on the device for a verify that snapshots `rows` rows: the DeltaNet states* and conv rings (slot 0 since setup), and the PLE ring's slots, all in one buffer since@@ -668,7 +748,7 @@ static int q38c_spec_slots(Q38Chain *ch, Model *m, int rows) {for (int sl = ch->snap_slots; sl < rows; sl++) {if (!ch->rec_snap[sl] && !(ch->rec_snap[sl] = (VkcBuf **)calloc((size_t)L, sizeof(VkcBuf *)))) return 0;if (!ch->ring_snap[sl] && !(ch->ring_snap[sl] = (VkcBuf **)calloc((size_t)L, sizeof(VkcBuf *)))) return 0;- for (int i = 0; i < ch->nl; i++) {+ for (int i = ch->lo; i < ch->lo + ch->nl; i++) {if (c->is_attn[i]) continue;if (!ch->rec_snap[sl][i] && !(ch->rec_snap[sl][i] = vkc_buf(nr * 4, VKC_DEV))) return 0;if (!ch->ring_snap[sl][i] && !(ch->ring_snap[sl][i] = vkc_buf(nc * 4, VKC_DEV))) return 0;@@ -693,7 +773,7 @@ static int q38c_push_state(Q38Chain *ch, Model *m, int pos_base, int n_rows) {Cfg *c = &m->c; int ok = 1;if (ch->dn_where == Q38C_HOST) {size_t nr = (size_t)c->dn_vheads * c->dn_kdim * c->dn_vdim, nc = (size_t)c->dn_conv_dim * (c->dn_convk - 1);- for (int i = 0; i < ch->nl && ok; i++) {+ for (int i = ch->lo; i < ch->lo + ch->nl && ok; i++) {if (c->is_attn[i]) continue;if (ch->host_zero) ok = vkc_zero(ch->rec[i], 0, nr) && vkc_zero(ch->ring[i], 0, nc);else ok = vkc_write(ch->rec[i], 0, m->DN_rec[i], nr * 4) && vkc_write(ch->ring[i], 0, m->DN_conv[i], nc * 4);@@ -705,7 +785,7 @@ static int q38c_push_state(Q38Chain *ch, Model *m, int pos_base, int n_rows) {ch->dn_where = Q38C_BOTH;}int D = c->head_dim, KVH = c->kv_heads, ID = c->idx_dim, R = c->idx_ratio;- for (int i = 0; i < ch->nl && ok; i++) {+ for (int i = ch->lo; i < ch->lo + ch->nl && ok; i++) {if (!c->is_attn[i]) continue;if (ch->pk_valid[i] > pos_base / R) ch->pk_valid[i] = pos_base / R; /* blocks this step rewrites */if (ch->ks.on) { /* the split: the window placed, the K/V rows below pos_base of its blocks uploaded */@@ -912,15 +992,15 @@ static int q38c_moe_apply(Q38Chain *ch, Model *m, int n) {* done. A partial chain runs its N layers, every chunk across all of them, and returns* 2: hyper_h holds every row's streams after layer N-1 (the one copy to the host per* chunk), and the caller runs layers N.. and the head on the CPU from them. */-static int q38c_forward(Model *m, const int *ids, int S, int pos_base, int nlogits, float *hyper_h,- int want_streams, float *mixed_h, float *logit) {- if (!g_vk_chain || qt_ready()) return 0;- /* prompts only: decode and verifies on the CPU */- if (g_vk_chain == COLI_VK_CHAIN_PREFILL && (S <= 2 || g_q38_rowwise)) return 0;- Q38Chain *ch = q38c_setup(m);- if (!ch || ch->failed) return 0;- Cfg *c = &m->c; int H = c->hidden, W = c->hc_width, L = ch->nl, E = c->experts, V = c->vocab;- if (!ch->head) { want_streams = 1; mixed_h = NULL; } /* a partial chain: the streams after layer N-1 */+/* One chain's layers (lo..lo+nl-1 on its device), from the streams in hyper_h. 0: not+ * taken, *lost = 1 when a frame failed (the state rebuilt on the CPU, the chains off), else+ * nothing on its device changed; 1: done with the head; 2: hyper_h holds the streams after+ * its last layer. */+static int q38c_forward_seg(Model *m, Q38Chain *ch, const int *ids, int S, int pos_base, int nlogits, float *hyper_h,+ int want_streams, float *mixed_h, float *logit, int *lost) {+ *lost = 0;+ Cfg *c = &m->c; int H = c->hidden, W = c->hc_width, L = ch->nl, lo = ch->lo, hi = lo + L, E = c->experts, V = c->vocab;+ if (!ch->head) { want_streams = 1; mixed_h = NULL; } /* a partial chain: the streams after its last layer */int mirror_ok = q38c_mirror(ch, m);int CH = mirror_ok ? q38c_chunk_rows(ch, m) : 1, rows = S < CH ? S : CH;if (ch->ks.on && rows > ch->ks.chunk) rows = ch->ks.chunk; /* a step's rows fit the split's window */@@ -953,9 +1033,9 @@ static int q38c_forward(Model *m, const int *ids, int S, int pos_base, int nlogics[(s * half + j) * 2] = cosf(ang); cs[(s * half + j) * 2 + 1] = sinf(ang);}}- if (c->ple_layer >= 0 && c->ple_layer < L) q38c_ple_rows(m, ids, c0, n, ch->host_emb);+ if (c->ple_layer >= lo && c->ple_layer < hi) q38c_ple_rows(m, ids, c0, n, ch->host_emb);int ok = 1, pending = 0;- for (int i = 0; i < L && ok; i++) {+ for (int i = lo; i < hi && ok; i++) {Layer *l = &m->L[i];if (pending) ok = q38c_moe_apply(ch, m, n);if (ok && i == c->ple_layer) ok = q38c_ple(ch, m, n, c0, ns);@@ -1010,7 +1090,7 @@ static int q38c_forward(Model *m, const int *ids, int S, int pos_base, int nlogiif (!ok) goto lost;if (want_streams) memcpy(hyper_h + (size_t)c0 * W, vkc_ptr(ch->hypd), (size_t)n * W * 4);if (mixed_h) memcpy(mixed_h + (size_t)c0 * H, vkc_ptr(ch->find), (size_t)n * H * 4);- for (int i = 0; i < L; i++) if (c->is_attn[i]) { ch->kv_valid[i] = pb + n; vkc_kv_done(&ch->ks, ch->attn_ord[i], pb + n); }+ for (int i = lo; i < hi; i++) if (c->is_attn[i]) { ch->kv_valid[i] = pb + n; vkc_kv_done(&ch->ks, ch->attn_ord[i], pb + n); }ch->dn_where = Q38C_DEV; ch->host_zero = 0;}if (ch->head) memcpy(logit, vkc_ptr(ch->outd), (size_t)nlogits * V * sizeof(float));@@ -1020,18 +1100,56 @@ static int q38c_forward(Model *m, const int *ids, int S, int pos_base, int nlogireturn ch->head ? 1 : 2;lost: /* a frame failed: the device is gone (or would not take a command); the CPU takes over */vkc_gemm_rows(-1);- if (!vkc_lost()) { vkc_finish(); coli_vk_mark_lost(); }+ if (!vkc_lost()) { vkc_finish(); coli_vk_mark_lost_dev(ch->d); }+ vkc_device(0);q38c_recover(m, pos_base);+ *lost = 1;return 0;}+/* Every layer the chains hold for S rows. hyper_h: the rows' streams in, the final ones+ * out when want_streams; mixed_h (or NULL): every row's final mixed row, for the prefill+ * read-out; logit: the last nlogits rows' logits. 0 = not taken (nothing changed, or a+ * device lost and the state rebuilt: the CPU runs the step), 1 = done. 2: hyper_h holds+ * every row's streams after the layers the chains ran (q38c_layers says how many: the+ * primary's, then the second device's), and the caller runs the rest and the head on the+ * CPU from them. */+static int q38c_forward(Model *m, const int *ids, int S, int pos_base, int nlogits, float *hyper_h,+ int want_streams, float *mixed_h, float *logit) {+ g_q38c_ran = 0;+ if (!g_vk_chain || qt_ready()) return 0;+ /* prompts only: decode and verifies on the CPU */+ if (g_vk_chain == COLI_VK_CHAIN_PREFILL && (S <= 2 || g_q38_rowwise)) return 0;+ Q38Chain *ch = q38c_setup(m), *ch2 = q38c_of(m, 1);+ if (!ch || ch->failed) return 0;+ if (ch2 && (!ch2->ok || ch2->failed || ch2->lo != ch->lo + ch->nl || ch->head)) ch2 = NULL;+ int lost = 0;+ int r = q38c_forward_seg(m, ch, ids, S, pos_base, nlogits, hyper_h, ch2 ? 1 : want_streams, ch2 ? NULL : mixed_h,+ logit, &lost);+ if (!r) return 0;+ g_q38c_ran = ch->nl;+ if (!ch2 || r == 1) return r;+ vkc_device(1);+ int r2 = q38c_forward_seg(m, ch2, ids, S, pos_base, nlogits, hyper_h, want_streams, mixed_h, logit, &lost);+ vkc_device(0);+ if (lost) { g_q38c_ran = 0; return 0; }+ if (!r2) { q38c_cpu_one(m, ch2, pos_base); return 2; } /* declined: the CPU runs its layers this step */+ g_q38c_ran += ch2->nl;+ return r2;+}+static void q38c_report(Model *m) {- Q38Chain *ch = m ? (Q38Chain *)m->vkchain : NULL;- if (!ch || !ch->ok || !ch->forwards) return;- VkcStats st; vkc_stats(&st);- fprintf(stderr, "[VK] qwen38 chain: %llu forwards, %llu frames (%llu ops, %llu matmuls, %llu tiled GEMM), "- "%.1f ms waiting for the device, %.1f ms of routed experts on the host, %.1f MiB on the device\n",- ch->forwards, st.frames, st.ops, st.matmuls, st.gemms, st.wait_ms, ch->host_ms, st.dev_bytes / 1048576.0);- vkc_kv_report(&ch->ks);- vkc_prof_print();+ for (int d = 0; d < 2 && m; d++) {+ Q38Chain *ch = q38c_of(m, d);+ if (!ch || !ch->ok || !ch->forwards) continue;+ int was = vkc_device(d);+ VkcStats st; vkc_stats(&st);+ fprintf(stderr, "[VK] %s chain: %llu forwards, %llu frames (%llu ops, %llu matmuls, %llu tiled GEMM), "+ "%.1f ms waiting for the device, %.1f ms of routed experts on the host, %.1f MiB on the device\n",+ q38c_name(ch), ch->forwards, st.frames, st.ops, st.matmuls, st.gemms, st.wait_ms, ch->host_ms,+ st.dev_bytes / 1048576.0);+ vkc_kv_report(&ch->ks);+ vkc_prof_print();+ vkc_device(was);+ }}diff --git a/c/qwen38_core.h b/c/qwen38_core.hindex f938cdcb..382d11ef 100644--- a/c/qwen38_core.h+++ b/c/qwen38_core.h@@ -236,6 +236,7 @@ typedef struct {int snap_ple_history_len[Q38_SPEC_SNAPS];#ifdef COLI_VULKANvoid *vkchain; /* the dense chain's device state (qwen38_chain.h), NULL until it runs */+ void *vkchain2; /* its layers on COLI_VK_DEV2's device, after the primary's (qwen38_chain.h) */#endif} Model;@@ -491,6 +492,8 @@ static unsigned g_q38_vk_placed[3]; /* uploads by format: int8 rows, bf16, f32* the device, the other layers and the head stay on the CPU, so nothing that is not on* the device already goes up any more (docs/vulkan.md, "A partial chain"). */static int g_q38_vk_noup;+static int g_q38_vk_dev; /* the device q38_vk_tensor uploads a new weight to: 1 for the chain's+ * layers on COLI_VK_DEV2's (qwen38_chain.h) */static void q38_dho_reload(Q38Weight *w); /* below, beside the trunk's int8 rows *//* The weight's device copy, uploaded on the first call; NULL when it is not eligible* or the upload failed (vk_off). The dense chain (qwen38_chain.h) reads the same one. */@@ -503,7 +506,8 @@ static ColiVkTensor *q38_vk_tensor(const Q38Weight *weight) {int fmt=q38_vk_fmt(w);const void *wq=w->q8?(const void*)w->q8:(const void*)w->data;const float *sc=w->q8?w->q8sc:NULL; /* fmt 10/11: no scales */- if(!coli_vk_tensor_ensure(t,wq,sc,fmt,w->cols,w->rows,0)){w->vk_off=1;return NULL;}+ if(!(g_q38_vk_dev?coli_vk_tensor_ensure2(t,wq,sc,fmt,w->cols,w->rows,0)+ :coli_vk_tensor_ensure(t,wq,sc,fmt,w->cols,w->rows,0))){w->vk_off=1;return NULL;}g_q38_vk_placed[fmt==1?0:fmt==11?1:2]++;return *t;}@@ -517,7 +521,7 @@ static int q38_vk_matmul(float *y,const float *x,const Q38Weight *weight,int S,iconst void *wq=w->q8?(const void*)w->q8:(const void*)w->data;const float *sc=w->q8?w->q8sc:NULL; /* fmt 10/11: no scales */ColiVkTensor *t=q38_vk_tensor(w);- if(!t)return 0;+ if(!t||coli_vk_tensor_dev(t))return 0; /* a weight of the second device's layers: only its chain reads it */return coli_vk_matmul(&t,y,x,wq,sc,fmt,S,I,O,0);}/* One line at the end of a run or a serve turn: how many matmuls the devicediff --git a/c/tests/vulkan_layers_dev2.sh b/c/tests/vulkan_layers_dev2.shindex 4bed60cf..8ee977fa 100644--- a/c/tests/vulkan_layers_dev2.sh+++ b/c/tests/vulkan_layers_dev2.sh@@ -80,21 +80,62 @@ ld2_qwen36() {$PY tests/vulkan_chain_serve.py ./qwen36 qwen36_tiny_c COLI_DENSE_I8=0 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=3 COLI_VK_CHAIN_LAYERS2=5}+ld2_qwen38() {+ [ -d qwen38_tiny ] || $PY tools/make_qwen38_tiny.py --out qwen38_tiny+ [ -d qwen38_tiny_fp8 ] || $PY tools/make_qwen38_tiny.py --out qwen38_tiny_fp8 --fp8-experts+ [ -d qwen38_kv_mtp ] || kv_qwen38_fixtures+ local R=qwen38_tiny/ref.json T="OMP_NUM_THREADS=2 SNAP=qwen38_tiny" f b+ for b in 0 1; do+ ld2_gate qwen38 "ld2 qwen38 2 + 2, the head on the second device, batch=$b" 2 2 $T Q38_PREFILL_BATCH=$b -- 4 8 $R+ done+ ld2_gate qwen38 "ld2 qwen38 1 + 2, the CPU the rest and the head" 1 2 $T -- 4 8 $R+ ld2_gate qwen38 "ld2 qwen38 1 + 3, prompt in chunks of 3" 1 3 $T COLI_VK_CHAIN_ROWS=3 -- 4 8 $R+ ld2_gate qwen38 "ld2 qwen38 bf16 rows" 2 2 $T Q38_NATIVE_BF16=1 -- 4 8 $R+ ld2_gate qwen38 "ld2 qwen38 fp8 experts" 2 2 OMP_NUM_THREADS=2 SNAP=qwen38_tiny_fp8 -- 2 8 qwen38_tiny_fp8/ref.json+ ld2_gate qwen38 "ld2 qwen38 experts on both devices too" 2 2 $T COLI_VK_TIER_GB=0.00002 -- 4 8 $R+ CHAINMODE=2 ld2_gate qwen38 "ld2 qwen38 prompts only" 2 2 $T -- 4 8 $R+ # MTP drafts verified across both devices (both chains' copies rolled back)+ for f in "" accept reject mixed; do+ ld2_gate qwen38 "ld2 qwen38 MTP ${f:-drafting}" 2 2 OMP_NUM_THREADS=2 Q38_MTP=1 Q38_MTP_FORCE=$f SNAP=qwen38_tiny_mtp -- 2 8 qwen38_tiny_mtp/ref.json+ done+ # the KV split on both devices' QSA layers (layers 1 and 3), with MTP+ ld2_gate qwen38 "ld2 qwen38 KV split on both" 2 2 COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 SNAP=qwen38_kv -- 4 8 qwen38_kv/ref.json+ [ "$(kv_hostparts qwen38 vk.log)" -gt 0 ] && [ "$(kv_hostparts "qwen38 dev2" vk.log)" -gt 0 ] ||+ { cat vk.log; fail "ld2 qwen38 KV split on both: a device's split ran no host part"; }+ ld2_gate qwen38 "ld2 qwen38 KV split on both, MTP mixed" 2 2 COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 Q38_MTP=1 \+ Q38_MTP_FORCE=mixed SNAP=qwen38_kv_mtp -- 4 8 qwen38_kv_mtp/ref.json+ # the second device lost: at its setup, mid-decode, and in a verify+ LD2_EXPECT=setup ld2_lost qwen38 "ld2 qwen38 second device lost at its setup" 2 2 1 $T -- 4 8 $R+ ld2_lost qwen38 "ld2 qwen38 second device lost mid-decode" 2 2 20 $T -- 4 8 $R+ ld2_lost qwen38 "ld2 qwen38 second device lost in a verify" 2 2 15 OMP_NUM_THREADS=2 Q38_MTP=1 Q38_MTP_FORCE=mixed \+ SNAP=qwen38_tiny_mtp -- 2 8 qwen38_tiny_mtp/ref.json+ # serve sessions with MTP and without+ CHAIN_SERVE_EXPECT='qwen38 dev2 chain: [1-9][0-9]* forwards' \+ $PY tests/vulkan_chain_serve.py ./qwen38 qwen38_tiny_mtp Q38_MTP=1 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=2 COLI_VK_CHAIN_LAYERS2=2+ CHAIN_SERVE_EXPECT='qwen38 dev2 chain: [1-9][0-9]* forwards' \+ $PY tests/vulkan_chain_serve.py ./qwen38 qwen38_tiny_mtp COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=1 COLI_VK_CHAIN_LAYERS2=2+}+family_layers_dev2() {export OMP_NUM_THREADS=2- make qwen36 tests/test_vk_chain VK=1+ make qwen36 qwen38 tests/test_vk_chain VK=1COLI_VK_DEV2=0 ./tests/test_vk_chain shaders/qmatmul.spv | tee vk_chain.logtail -1 vk_chain.log | grep -qx PASS || fail "the chain's ops on two devices"ld2_qwen36+ ld2_qwen38unset OMP_NUM_THREADS}family_layers_dev2_sanitize() {make clean >/dev/null 2>&1 || true- make qwen36 VK=1 EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"+ make qwen36 qwen38 VK=1 EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1[ -d qwen36_kv_c ] || kv_qwen36_fixtureskv_san qwen36 "asan ld2 qwen36 KV split on both" COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=4 COLI_VK_CHAIN_LAYERS2=4 \COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 COLI_DENSE_I8=0 SNAP=qwen36_kv_c ./qwen36 8 8 qwen36_kv/ref_full.json[ "$(ld2_forwards qwen36 san.log)" -gt 0 ] || { cat san.log; fail "asan ld2 qwen36: the second device's chain never ran"; }+ [ -d qwen38_kv_mtp ] || kv_qwen38_fixtures+ kv_san qwen38 "asan ld2 qwen38 KV split on both, MTP mixed" COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=2 COLI_VK_CHAIN_LAYERS2=2 \+ COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 Q38_MTP=1 Q38_MTP_FORCE=mixed SNAP=qwen38_kv_mtp ./qwen38 4 8 qwen38_kv_mtp/ref.json+ [ "$(ld2_forwards qwen38 san.log)" -gt 0 ] || { cat san.log; fail "asan ld2 qwen38: the second device's chain never ran"; }make clean >/dev/null 2>&1 || true}