Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

qwen36: the chain's layers on two devices (COLI_VK_DEV2)

JustVugga3277b291569
diff --git a/c/qwen36.c b/c/qwen36.c
index 504aa6e7..673e8912 100644
--- a/c/qwen36.c
+++ b/c/qwen36.c
@@ -922,6 +922,7 @@ typedef struct {
int *mpos, mpos_len, rope_delta;
#ifdef COLI_VULKAN
void *vkchain; /* the dense chain's device state (qwen36_chain.h), NULL until it runs */
+ void *vkchain2; /* its layers on COLI_VK_DEV2's device, after the primary's (qwen36_chain.h) */
#endif
/* A speculative verify (prompt lookup, q36_spec_step) copies the DeltaNet state
* after each of its first snap_rows rows, row r into slot r (0 = no copy), so a
@@ -1464,6 +1465,8 @@ static void vk_q4_planar_to_fmt4(const QW *w, uint8_t *dst) {
}
}
static unsigned g_vk_placed[4]; /* uploads by format: int8 rows, int4, f32, f16 */
+static int g_q36_vk_dev; /* the device vk_qw_tensor uploads a new matrix to: 1 for the
+ * chain's layers on COLI_VK_DEV2's (qwen36_chain.h) */
/* The format a matrix goes to the device in: the copy matmul_d reads (int4, int8 rows,
* f16 with COLI_DENSE_BITS=16, else f32), its rows and its scales. */
static int vk_qw_fmt(const QW *w, const void **wq, const float **sc) {
@@ -1495,6 +1498,15 @@ static ColiVkTensor *vk_qw_tensor(const QW *w) {
/* On an integrated device these are already one shared allocation. Retain
* it instead of moving the same physical bytes into a smaller logical
* device-local heap. q36_dho_drop keeps imported pages alive. */
+ if (g_q36_vk_dev) { /* the second device: a copy there (no import, no per-matrix path) */
+ uint8_t *packed = w->q4 ? malloc((size_t)O * (I / 2)) : NULL;
+ if (packed) vk_q4_planar_to_fmt4(w, packed);
+ ok = (!w->q4 || packed) && coli_vk_tensor_ensure2(t, w->q4 ? (const void *)packed : wq, sc, fmt, I, O, gs);
+ free(packed);
+ if (!ok) { mw->vk_off = 1; return NULL; }
+ g_vk_placed[fmt == 1 ? 0 : fmt == 4 ? 1 : fmt == 14 ? 3 : 2]++;
+ return *t;
+ }
if (g_vk_import && (coli_vk_device_integrated() || !coli_vk_dense_device_only()) &&
fmt != 4 && w->w != wq) { /* q36_walloc'd rows (f32 rows are falloc's) */
size_t rb = fmt == 1 ? (size_t)I : (size_t)I * 2;
@@ -1524,7 +1536,7 @@ static int vk_dense_matmul(float *y, const float *x, const QW *w, int S, int I,
int fmt = vk_qw_fmt(w, &wq, &sc), gs = fmt == 4 ? 64 : 0;
if (!wq && !w->vk_gone) return 0;
ColiVkTensor *t = vk_qw_tensor(w);
- if (!t) return 0;
+ if (!t || coli_vk_tensor_dev(t)) return 0; /* a matrix of the second device's layers: only its chain reads it */
/* a verify's rows (g_q36_rowwise) one at a time: a decode step's GEMV, not the batch's GEMM */
if (g_q36_rowwise && S > 1) {
for (int s = 0; s < S; s++)
@@ -6175,7 +6187,7 @@ int main(int argc, char **argv) {
g_vk_chain = 0;
}
if (g_vk_chain && g_pilot) fprintf(stderr, "[VK] qwen36: PILOT prefetch reads the residual on the host: the dense chain stays off\n");
- if (g_vk_chain || g_q36c_fit_on) atexit(vkc_shutdown); /* registered after the tier's: runs before the device goes */
+ if (g_vk_chain || g_q36c_fit_on) atexit(vkc_shutdown_all); /* registered after the tier's: runs before the device goes */
#endif
/* coli serve mode: speak the gateway wire protocol instead of argv
diff --git a/c/qwen36_chain.h b/c/qwen36_chain.h
index d900c7f8..6a90a105 100644
--- a/c/qwen36_chain.h
+++ b/c/qwen36_chain.h
@@ -81,7 +81,8 @@
typedef struct {
int ok, failed;
- int n, head; /* layers 0..n-1 on the device; lm_head there too */
+ int lo, d; /* its layers start at lo, on device d (vkc_device) */
+ int n, head; /* layers lo..lo+n-1 on the device; lm_head there too */
int rows; /* scratch capacity in rows */
int cap; /* the host's kv_cap the mirrors follow */
int dev_rows; /* device KV rows a layer: cap, or the split's */
@@ -104,6 +105,16 @@ typedef struct {
static int g_vk_chain = 0; /* COLI_VK_CHAIN decided on, and the chain's pipelines are up */
static VkcFit g_q36c_fit; /* the partial chain's fit (q36c_start) */
static int g_q36c_fit_on; /* the fit ran: the chain sets itself up at start (q36c_place) */
+/* The layers after the primary's on COLI_VK_DEV2's device (docs/vulkan.md, "Layers on two
+ * devices"): a second chain from layer g_q36c_fit.n, its own fit on that device's memory.
+ * A forward runs the primary's layers, brings the residual rows back and runs these;
+ * the CPU runs what is left. */
+static VkcFit g_q36c_fit2;
+static int g_q36c_fit2_on;
+static Q36Chain *q36c_of(Model *m, int d) { return (Q36Chain *)(d ? m->vkchain2 : m->vkchain); }
+static const char *q36c_name(const Q36Chain *ch) { return ch && ch->d ? "qwen36 dev2" : "qwen36"; }
+/* COLI_VK_CHAIN_DEV2=0: the second device keeps routed experts only */
+static int q36c_dev2_wanted(void) { const char *e = getenv("COLI_VK_CHAIN_DEV2"); return !(e && *e == '0'); }
static void q36c_fatal(const char *what) {
fprintf(stderr, "[VK] qwen36 chain: %s -- stopping (COLI_VK_CHAIN=0 keeps the state on the CPU)\n", what);
@@ -115,11 +126,15 @@ static void q36c_fatal(const char *what) {
* A partial chain rebuilds its N layers only: the CPU's layers hold their state on the
* host already, at `upto` (a forward the device was lost in has not reached them). */
static void q36c_recover(Model *m, int upto) {
- Q36Chain *ch = (Q36Chain *)m->vkchain;
+ Q36Chain *ch = (Q36Chain *)m->vkchain, *ch2 = (Q36Chain *)m->vkchain2;
Cfg *c = &m->c; int D = c->hidden;
int nd = ch && ch->ok ? ch->n : c->n_layers;
+ if (ch && ch->ok && ch2 && ch2->ok) nd += ch2->n; /* the second device's layers ran in that forward too */
g_vk_chain = 0;
- if (ch) { ch->failed = 1; ch->dn_where = Q36C_HOST; ch->host_zero = 0; ch->snap_valid = 0; }
+ for (int d = 0; d < 2; d++) {
+ Q36Chain *x = q36c_of(m, d);
+ if (x) { x->failed = 1; x->dn_where = Q36C_HOST; x->host_zero = 0; x->snap_valid = 0; }
+ }
for (int i = 0; i < nd; i++) {
if (c->is_attn[i]) continue;
memset(m->DN_rec[i], 0, (size_t)c->dn_vheads * c->dn_kdim * c->dn_vdim * sizeof(float));
@@ -144,9 +159,9 @@ static void q36c_recover(Model *m, int upto) {
}
/* f32 rows [O x I] as a resident fmt 10 tensor */
-static ColiVkTensor *q36c_f32_tensor(const float *w, int I, int O) {
+static ColiVkTensor *q36c_f32_tensor(int d, const float *w, int I, int O) {
ColiVkTensor *t = NULL;
- return coli_vk_tensor_ensure(&t, w, NULL, 10, I, O, 0) ? t : NULL;
+ return (d ? coli_vk_tensor_ensure2(&t, w, NULL, 10, I, O, 0) : coli_vk_tensor_ensure(&t, w, NULL, 10, I, O, 0)) ? t : NULL;
}
static int q36c_geometry_ok(const Cfg *c) {
@@ -210,20 +225,22 @@ static void q36c_layer_cpu(Q36Chain *ch, Model *m, int i) {
static int q36c_place_layer(Q36Chain *ch, Model *m, int i) {
Cfg *c = &m->c; Layer *l = &m->L[i]; int D = c->hidden, vh = c->dn_vheads;
QW *ws[16]; int nw = q36c_layer_qws(m, i, ws), ok = 1;
+ g_q36_vk_dev = ch->d;
for (int k = 0; k < nw && ok; k++) ok = vk_qw_tensor(ws[k]) != NULL;
+ g_q36_vk_dev = 0;
if (ok && !c->is_attn[i]) {
float *ab = malloc((size_t)2 * vh * D * sizeof(float));
if (ab) {
memcpy(ab, l->dn_b, (size_t)vh * D * sizeof(float));
memcpy(ab + (size_t)vh * D, l->dn_a, (size_t)vh * D * sizeof(float));
- ch->t_ab[i] = q36c_f32_tensor(ab, D, 2 * vh);
+ ch->t_ab[i] = q36c_f32_tensor(ch->d, ab, D, 2 * vh);
free(ab);
}
ch->rec[i] = ch->t_ab[i] ? vkc_buf((size_t)vh * c->dn_kdim * c->dn_vdim * sizeof(float), VKC_DEV) : NULL;
ch->ring[i] = ch->rec[i] ? vkc_buf((size_t)c->dn_conv_dim * (c->dn_convk - 1) * sizeof(float), VKC_DEV) : NULL;
ok = ch->t_ab[i] && ch->rec[i] && ch->ring[i];
}
- if (ok && l->sh_gate) ok = (ch->t_sg[i] = q36c_f32_tensor(l->sh_gate, D, 1)) != NULL;
+ if (ok && l->sh_gate) ok = (ch->t_sg[i] = q36c_f32_tensor(ch->d, l->sh_gate, D, 1)) != NULL;
return ok;
}
@@ -233,20 +250,27 @@ static int q36c_place_layer(Q36Chain *ch, Model *m, int i) {
* before the tier sizes its budget; a layer that does not reach the device is freed
* whole and the chain keeps the layers before it (vkc_fit_shrink). NULL = the chain
* cannot run. */
-static Q36Chain *q36c_setup(Model *m) {
- Q36Chain *ch = (Q36Chain *)m->vkchain;
+/* d = 1: the second device's chain, its layers from g_q36c_fit.n on (the caller has made
+ * device d current, vkc_device). */
+static Q36Chain *q36c_setup_dev(Model *m, int d) {
+ void **slot = d ? &m->vkchain2 : &m->vkchain;
+ Q36Chain *ch = (Q36Chain *)*slot;
if (ch) return ch->ok ? ch : NULL;
+ if (d && !g_q36c_fit2_on) return NULL;
ch = (Q36Chain *)calloc(1, sizeof *ch);
if (!ch) return NULL;
- m->vkchain = ch;
+ *slot = ch;
+ ch->d = d;
+ const char *nm = q36c_name(ch);
Cfg *c = &m->c; int L = c->n_layers, D = c->hidden;
if (!q36c_geometry_ok(c)) {
fprintf(stderr, "[VK] qwen36 chain: a geometry its shaders do not take (head dim %d, DeltaNet %dx%d); per-matrix path\n",
c->head_dim, c->dn_kdim, c->dn_vdim);
return NULL;
}
- VkcFit *fit = g_q36c_fit_on ? &g_q36c_fit : NULL;
- int n = fit ? fit->n : L;
+ VkcFit *fit = d ? &g_q36c_fit2 : g_q36c_fit_on ? &g_q36c_fit : NULL;
+ int lo = d ? g_q36c_fit.n : 0, n = fit ? fit->n : L;
+ ch->lo = lo;
ch->o_in = calloc(L, sizeof(size_t)); ch->o_post = calloc(L, sizeof(size_t)); ch->o_qn = calloc(L, sizeof(size_t));
ch->o_kn = calloc(L, sizeof(size_t)); ch->o_conv = calloc(L, sizeof(size_t)); ch->o_dn = calloc(L, sizeof(size_t));
ch->t_ab = calloc(L, sizeof(void *)); ch->t_sg = calloc(L, sizeof(void *));
@@ -258,7 +282,7 @@ static Q36Chain *q36c_setup(Model *m) {
/* the parameter arena of the n layers: offsets, then one upload */
size_t np = 0;
int vh = c->dn_vheads, conv_dim = c->dn_conv_dim, convk = c->dn_convk;
- for (int i = 0; i < n; i++) {
+ for (int i = lo; i < lo + n; i++) {
ch->o_in[i] = np; np += D; ch->o_post[i] = np; np += D;
if (c->is_attn[i]) {
if (m->L[i].qn) { ch->o_qn[i] = np; np += c->head_dim; }
@@ -271,7 +295,7 @@ static Q36Chain *q36c_setup(Model *m) {
ch->o_final = np; np += D;
float *arena = calloc(np, sizeof(float));
if (!arena) return NULL;
- for (int i = 0; i < n; i++) {
+ for (int i = lo; i < lo + n; i++) {
Layer *l = &m->L[i];
memcpy(arena + ch->o_in[i], l->in_ln, D * sizeof(float));
memcpy(arena + ch->o_post[i], l->post_ln, D * sizeof(float));
@@ -292,53 +316,62 @@ static Q36Chain *q36c_setup(Model *m) {
if (!ok) {
if (!fit) return NULL;
vkc_free(ch->prm); ch->prm = NULL;
- for (int i = 0; i < n; i++) q36c_layer_cpu(ch, m, i);
- vkc_fit_shrink("qwen36", fit, 0, vkc_lost() ? "the device was lost" : "its parameter buffer was refused");
- m->lm_head.vk_off = 1;
- g_vk_chain = 0;
+ for (int i = lo; i < lo + n; i++) q36c_layer_cpu(ch, m, i);
+ vkc_fit_shrink(nm, fit, 0, vkc_lost() ? "the device was lost" : "its parameter buffer was refused");
+ if (!d || fit->tail) m->lm_head.vk_off = 1;
+ if (!d) g_vk_chain = 0; /* the second device's failing: the primary's chain stands, the CPU runs the rest */
vkc_shutdown(); /* the chain's own blocks and frames go too */
- vkc_fit_placed("qwen36", fit);
+ vkc_fit_placed(nm, fit);
return NULL;
}
/* the layers: the same device copies the per-matrix path uses; a layer that does
* not get there whole is freed and the chain stops before it */
int placed = 0;
- for (int i = 0; i < n; i++) {
+ for (int i = lo; i < lo + n; i++) {
if (!q36c_place_layer(ch, m, i)) {
if (!fit) { fprintf(stderr, "[VK] qwen36 chain: a matrix did not reach the device; per-matrix path\n"); return NULL; }
- for (int j = i; j < n; j++) q36c_layer_cpu(ch, m, j);
- vkc_fit_shrink("qwen36", fit, i, vkc_lost() ? "the device was lost" : "an upload was refused");
+ for (int j = i; j < lo + n; j++) q36c_layer_cpu(ch, m, j);
+ vkc_fit_shrink(nm, fit, i - lo, vkc_lost() ? "the device was lost" : "an upload was refused");
break;
}
- if (coli_vk_dense_device_only()) { /* the layer is there whole: its host copies go */
+ /* the layer is there whole: its host copies go (the second device's only where the
+ * chain runs every step: with prompts only, decode reads them on the CPU) */
+ if (coli_vk_dense_device_only() && (!d || g_vk_chain == COLI_VK_CHAIN_ON)) {
QW *ws[16]; int nw = q36c_layer_qws(m, i, ws), nd = 0; size_t b = 0;
for (int k = 0; k < nw; k++) q36_dho_drop(ws[k], &b, &nd);
}
- if (fit) vkc_fit_mark(fit, i);
- placed = i + 1;
+ if (fit) vkc_fit_mark(fit, i - lo);
+ placed = i + 1 - lo;
}
ch->n = placed;
- for (int i = 0; i < placed; i++) if (c->is_attn[i]) ch->attn_ord[i] = ch->n_attn++;
+ for (int i = lo; i < lo + placed; i++) if (c->is_attn[i]) ch->attn_ord[i] = ch->n_attn++;
if (!placed) { /* nothing of the chain stays: its arena, blocks and frames go */
vkc_free(ch->prm); ch->prm = NULL;
- m->lm_head.vk_off = 1;
- g_vk_chain = 0;
+ if (!d || (fit && fit->tail)) m->lm_head.vk_off = 1;
+ if (!d) g_vk_chain = 0;
vkc_shutdown();
}
- if (fit) vkc_fit_placed("qwen36", fit);
+ if (fit) vkc_fit_placed(nm, fit);
if (!placed) return NULL;
- /* the head: with every layer, and room for it (the fit's tail) */
- ch->head = (!fit || fit->tail) && vk_qw_tensor(&m->lm_head);
+ /* the head: with every layer, and room for it (the fit's tail), on the chain that
+ * ends at the last layer */
+ g_q36_vk_dev = d;
+ ch->head = lo + placed == L && (!fit || fit->tail) && vk_qw_tensor(&m->lm_head);
+ g_q36_vk_dev = 0;
if (!ch->head) {
if (fit) fit->tail = 0;
- if (!m->lm_head.vk) m->lm_head.vk_off = 1;
- } else if (coli_vk_dense_device_only()) { int nd = 0; size_t b = 0; q36_dho_drop(&m->lm_head, &b, &nd); }
+ /* the second device's chain may still take it (it is set up after this one) */
+ if (!m->lm_head.vk && (d || !g_q36c_fit2_on || !g_q36c_fit2.tail)) m->lm_head.vk_off = 1;
+ } else if (coli_vk_dense_device_only() && !d) { int nd = 0; size_t b = 0; q36_dho_drop(&m->lm_head, &b, &nd); }
ch->dn_where = Q36C_HOST; ch->host_zero = 0;
ch->ok = 1;
- fprintf(stderr, "[VK] qwen36 chain: %d layers on the device (%d attention)%s, %.1f MiB of parameters\n",
- placed, ch->n_attn, ch->head ? "" : ", the head on the CPU", np * 4 / 1048576.0);
+ if (d) fprintf(stderr, "[VK] qwen36 chain: layers %d..%d on the second device (%d attention)%s, %.1f MiB of parameters\n",
+ lo, lo + placed - 1, ch->n_attn, ch->head ? ", the head too" : "", np * 4 / 1048576.0);
+ else fprintf(stderr, "[VK] qwen36 chain: %d layers on the device (%d attention)%s, %.1f MiB of parameters\n",
+ placed, ch->n_attn, ch->head ? "" : ", the head on the CPU", np * 4 / 1048576.0);
return ch;
}
+static Q36Chain *q36c_setup(Model *m) { return q36c_setup_dev(m, 0); }
/* q36c_res counts instead of reserving while g_q36c_count >= 0 (the chunk's sizing), and
* adds each buffer's device bytes to *g_q36c_fitsum while it is set (the fit) */
@@ -425,13 +458,32 @@ static void q36c_start(Model *m) {
size_t tail = 0, tail_m = 0;
q36c_fit_qw(&m->lm_head, &tail, &tail_m);
int n = vkc_fit("qwen36", L, per, mat, fixed, tail, &g_q36c_fit);
- free(per); free(mat);
/* the chain's pipelines now, after the fit read the free memory: nothing of it was on
* the device before (vkc_init's own buffer counts in the pools' share) */
- if (n > 0 && !vkc_init()) { g_vk_chain = 0; return; } /* no chain after all: no fit, as before */
+ if (n > 0 && !vkc_init()) { g_vk_chain = 0; free(per); free(mat); return; } /* no chain after all: no fit, as before */
g_q36c_fit_on = 1;
- for (int i = n; i < L; i++) q36c_layer_cpu(NULL, m, i);
- if (!g_q36c_fit.tail) m->lm_head.vk_off = 1;
+ /* the layers the primary leaves, on COLI_VK_DEV2's device: a fit of its own from layer
+ * n, with that device's free memory (the tier there sizes itself after the placement) */
+ int n2 = 0;
+ if (n > 0 && n < L && q36c_dev2_wanted() && getenv("COLI_VK_DEV2") && coli_vk_dev2_open_env()) {
+ vkc_device(1);
+ size_t fixed2 = vkc_fit_buf((size_t)D * sizeof(float)) + vkc_fit_buf(4);
+ g_q36c_fitsum = &fixed2;
+ q36c_bufs(&cnt, m, rows);
+ q36c_res(&cnt.xd, (size_t)rows * D, VKC_DOWN);
+ g_q36c_fitsum = NULL;
+ n2 = vkc_fit("qwen36 dev2", L - n, per + n, mat + n, fixed2, tail, &g_q36c_fit2);
+ if (n2 > 0 && !vkc_init()) {
+ fprintf(stderr, "[VK] qwen36 chain: the second device's pipelines did not come up; its layers stay on the CPU\n");
+ n2 = 0;
+ }
+ g_q36c_fit2_on = n2 > 0;
+ if (!g_q36c_fit2_on) g_q36c_fit2.tail = 0;
+ vkc_device(0);
+ }
+ free(per); free(mat);
+ for (int i = n + n2; i < L; i++) q36c_layer_cpu(NULL, m, i);
+ if (!g_q36c_fit.tail && !g_q36c_fit2.tail) m->lm_head.vk_off = 1;
if (n == 0) { /* the chain off: nothing of it on the device, every byte is the tier's */
g_vk_chain = 0;
vkc_fit_placed("qwen36", &g_q36c_fit);
@@ -441,7 +493,21 @@ static void q36c_start(Model *m) {
* on the device now, so the tier sizes its budget from what is left. */
static void q36c_place(Model *m) {
if (!g_q36c_fit_on || !g_q36c_fit.n || !g_vk_chain) return;
- q36c_setup(m);
+ Q36Chain *ch = q36c_setup(m);
+ if (!g_q36c_fit2_on) return;
+ int lo = g_q36c_fit.n, n2 = g_q36c_fit2.n;
+ vkc_device(1);
+ if (!ch || ch->n < lo) {
+ /* the primary placed fewer layers than its fit: the second device's would not follow
+ * them, so they go to the CPU too */
+ fprintf(stderr, "[VK] qwen36 chain: the primary device stopped before layer %d; layers %d..%d stay on the CPU, "
+ "not on the second device\n", lo, lo, lo + n2 - 1);
+ for (int i = lo; i < lo + n2; i++) q36c_layer_cpu(NULL, m, i);
+ if (g_q36c_fit2.tail) m->lm_head.vk_off = 1;
+ g_q36c_fit2_on = 0;
+ vkc_shutdown();
+ } else q36c_setup_dev(m, 1);
+ vkc_device(0);
}
/* Prompt rows per chunk (vkc_chunk_rows): the chain's scratch a row, counted from the
* reservations themselves, and the routed experts' outputs (the tier's rows, the
@@ -480,11 +546,11 @@ static int q36c_mirror(Q36Chain *ch, Model *m) {
Cfg *c = &m->c; int KV = c->kv_heads;
if (ch->cap != m->kv_cap) { /* the host cache grew: mirror it again (whole, or split past the budget) */
size_t row = (size_t)KV * c->k_head_dim * 2 * sizeof(float);
- int plan = ch->n_attn ? vkc_kv_plan(&ch->ks, "qwen36", ch->n_attn, row, m->kv_cap, 1, 0,
+ int plan = ch->n_attn ? vkc_kv_plan(&ch->ks, q36c_name(ch), ch->n_attn, row, m->kv_cap, 1, 0,
(size_t)ch->n_attn * ch->dev_rows * row) : 1;
if (!plan) { ch->cap = 0; return 0; }
ch->dev_rows = ch->ks.on ? ch->ks.rows : m->kv_cap;
- for (int i = 0; i < ch->n; i++) {
+ for (int i = ch->lo; i < ch->lo + ch->n; i++) {
if (!c->is_attn[i]) continue;
vkc_free(ch->kc[i]); vkc_free(ch->vc[i]); ch->kc[i] = ch->vc[i] = NULL;
ch->kv_valid[i] = 0;
@@ -513,52 +579,66 @@ static int q36c_scratch(Q36Chain *ch, Model *m, int rows) {
/* ---- state between the host and the device ----------------------------------- */
/* The host's DeltaNet state brought up to date (before anything reads it there). */
-static void q36c_sync_host(Model *m) {
- Q36Chain *ch = (Q36Chain *)m->vkchain;
- if (!ch || !ch->ok || ch->dn_where != Q36C_DEV) return;
+/* (every function here covers both chains: the primary's layers and the second device's) */
+static int q36c_sync_one(Model *m, Q36Chain *ch) {
+ if (!ch || !ch->ok || ch->dn_where != Q36C_DEV) return 1;
Cfg *c = &m->c;
size_t nr = (size_t)c->dn_vheads * c->dn_kdim * c->dn_vdim, nc = (size_t)c->dn_conv_dim * (c->dn_convk - 1);
- for (int i = 0; i < ch->n; i++) {
+ int was = vkc_device(ch->d), ok = 1;
+ for (int i = ch->lo; i < ch->lo + ch->n && ok; i++) {
if (c->is_attn[i]) continue;
- if (!vkc_read(ch->rec[i], 0, m->DN_rec[i], nr * sizeof(float)) ||
- !vkc_read(ch->ring[i], 0, m->DN_conv[i], nc * sizeof(float))) { q36c_recover(m, m->kv_len); return; }
+ ok = vkc_read(ch->rec[i], 0, m->DN_rec[i], nr * sizeof(float)) &&
+ vkc_read(ch->ring[i], 0, m->DN_conv[i], nc * sizeof(float));
}
- ch->dn_where = Q36C_BOTH;
+ vkc_device(was);
+ if (ok) ch->dn_where = Q36C_BOTH;
+ return ok;
+}
+static void q36c_sync_host(Model *m) {
+ for (int d = 0; d < 2; d++)
+ if (!q36c_sync_one(m, q36c_of(m, d))) { q36c_recover(m, m->kv_len); return; }
}
/* The host wrote its DeltaNet state (zero: reset_recurrent's zeros). */
static void q36c_host_wrote(Model *m, int zero) {
- Q36Chain *ch = (Q36Chain *)m->vkchain;
- if (!ch || !ch->ok) return;
- ch->dn_where = Q36C_HOST; ch->host_zero = zero; ch->snap_valid = 0;
+ for (int d = 0; d < 2; d++) {
+ Q36Chain *ch = q36c_of(m, d);
+ if (!ch || !ch->ok) continue;
+ ch->dn_where = Q36C_HOST; ch->host_zero = zero; ch->snap_valid = 0;
+ }
}
-/* A CPU step from pos_base: the host state current before it, the device's stale after. */
-static void q36c_cpu_step(Model *m, int pos_base) {
- Q36Chain *ch = (Q36Chain *)m->vkchain;
+/* A CPU step from pos_base over one chain's layers: its host state current before it,
+ * the device's stale after. */
+static void q36c_cpu_one(Model *m, Q36Chain *ch, int pos_base) {
if (!ch || !ch->ok) return;
- q36c_sync_host(m);
+ if (!q36c_sync_one(m, ch)) { q36c_recover(m, m->kv_len); return; }
ch->dn_where = Q36C_HOST; ch->host_zero = 0; ch->snap_valid = 0;
- for (int i = 0; i < ch->n; i++) {
+ for (int i = ch->lo; i < ch->lo + ch->n; i++) {
if (ch->kv_valid[i] > pos_base) ch->kv_valid[i] = pos_base;
if (m->c.is_attn[i]) vkc_kv_lower(&ch->ks, ch->attn_ord[i], pos_base);
}
}
+static void q36c_cpu_step(Model *m, int pos_base) {
+ for (int d = 0; d < 2; d++) q36c_cpu_one(m, q36c_of(m, d), pos_base);
+}
/* A rejected draft: the state after the verify's row `slot` (its first slot+1 rows) is
* the device's copy in that slot; `len` positions stand, and the KV watermark comes
* down to them (the rows past them are the rejected drafts'). */
static void q36c_rollback(Model *m, int slot, int len) {
- Q36Chain *ch = (Q36Chain *)m->vkchain;
- if (!ch || !ch->ok) return;
- for (int i = 0; i < ch->n; i++) {
- if (ch->kv_valid[i] > len) ch->kv_valid[i] = len;
- if (m->c.is_attn[i]) vkc_kv_lower(&ch->ks, ch->attn_ord[i], len);
- }
- if (slot < 0 || slot >= ch->snap_valid || ch->dn_where != Q36C_DEV) { ch->snap_valid = 0; return; }
- for (int i = 0; i < ch->n; i++) {
- if (m->c.is_attn[i]) continue;
- VkcBuf *t = ch->rec[i]; ch->rec[i] = ch->rec_snap[slot][i]; ch->rec_snap[slot][i] = t;
- t = ch->ring[i]; ch->ring[i] = ch->ring_snap[slot][i]; ch->ring_snap[slot][i] = t;
+ for (int d = 0; d < 2; d++) {
+ Q36Chain *ch = q36c_of(m, d);
+ if (!ch || !ch->ok) continue;
+ for (int i = ch->lo; i < ch->lo + ch->n; i++) {
+ if (ch->kv_valid[i] > len) ch->kv_valid[i] = len;
+ if (m->c.is_attn[i]) vkc_kv_lower(&ch->ks, ch->attn_ord[i], len);
+ }
+ if (slot < 0 || slot >= ch->snap_valid || ch->dn_where != Q36C_DEV) { ch->snap_valid = 0; continue; }
+ for (int i = ch->lo; i < ch->lo + ch->n; i++) {
+ if (m->c.is_attn[i]) continue;
+ VkcBuf *t = ch->rec[i]; ch->rec[i] = ch->rec_snap[slot][i]; ch->rec_snap[slot][i] = t;
+ t = ch->ring[i]; ch->ring[i] = ch->ring_snap[slot][i]; ch->ring_snap[slot][i] = t;
+ }
+ ch->snap_valid = 0;
}
- ch->snap_valid = 0;
}
/* The device's copy slots for a verify that copies `rows` rows, allocated the first
* time a verify that deep runs. */
@@ -569,7 +649,7 @@ static int q36c_spec_slots(Q36Chain *ch, Model *m, int rows) {
for (int sl = ch->snap_slots; sl < rows; sl++) {
if (!ch->rec_snap[sl] && !(ch->rec_snap[sl] = (VkcBuf **)calloc((size_t)L, sizeof(VkcBuf *)))) return 0;
if (!ch->ring_snap[sl] && !(ch->ring_snap[sl] = (VkcBuf **)calloc((size_t)L, sizeof(VkcBuf *)))) return 0;
- for (int i = 0; i < ch->n; i++) {
+ for (int i = ch->lo; i < ch->lo + ch->n; i++) {
if (c->is_attn[i]) continue;
if (!ch->rec_snap[sl][i] && !(ch->rec_snap[sl][i] = vkc_buf(nr * sizeof(float), VKC_DEV))) return 0;
if (!ch->ring_snap[sl][i] && !(ch->ring_snap[sl][i] = vkc_buf(nc * sizeof(float), VKC_DEV))) return 0;
@@ -583,7 +663,7 @@ static int q36c_push_state(Q36Chain *ch, Model *m, int pos_base, int n_rows) {
Cfg *c = &m->c; int ok = 1;
if (ch->dn_where == Q36C_HOST) {
size_t nr = (size_t)c->dn_vheads * c->dn_kdim * c->dn_vdim, nc = (size_t)c->dn_conv_dim * (c->dn_convk - 1);
- for (int i = 0; i < ch->n && ok; i++) {
+ for (int i = ch->lo; i < ch->lo + ch->n && ok; i++) {
if (c->is_attn[i]) continue;
if (ch->host_zero) ok = vkc_zero(ch->rec[i], 0, nr) && vkc_zero(ch->ring[i], 0, nc);
else ok = vkc_write(ch->rec[i], 0, m->DN_rec[i], nr * sizeof(float)) &&
@@ -592,7 +672,7 @@ static int q36c_push_state(Q36Chain *ch, Model *m, int pos_base, int n_rows) {
ch->dn_where = Q36C_BOTH;
}
int kvd = c->k_head_dim, KV = c->kv_heads;
- for (int i = 0; i < ch->n && ok; i++) {
+ for (int i = ch->lo; i < ch->lo + ch->n && ok; i++) {
if (c->is_attn[i] && ch->ks.on) { /* the split: the window placed, its rows below pos_base uploaded */
VkcKvPart pt[2] = {{KV, kvd, m->K[i], (size_t)m->max_t * kvd, ch->kc[i], 0},
{KV, kvd, m->V[i], (size_t)m->max_t * kvd, ch->vc[i], 0}};
@@ -705,16 +785,15 @@ static int q36c_combine(Q36Chain *ch, Model *m, int i, int n) {
* Otherwise the layers it ran: with a partial chain (N layers, or the head on the CPU)
* *rows_only is 1, xh holds every row's residual after those layers, and the caller
* runs the rest of the layers and the head on the CPU. */
-static int q36c_forward(Model *m, float *xh, int S, int pos_base, FILE *lf, int want_x, int nlogits, float *logit,
- int *rows_only) {
- *rows_only = 0;
- if (!g_vk_chain || qt_ready() || qq_active() || g_pilot) return 0;
- /* prompts only: decode and verifies on the CPU */
- if (g_vk_chain == COLI_VK_CHAIN_PREFILL && (S <= 2 || g_q36_rowwise)) return 0;
- Q36Chain *ch = q36c_setup(m);
- if (!ch || ch->failed) return 0;
- Cfg *c = &m->c; int D = c->hidden, L = ch->n, E = c->n_experts; /* L: the layers on the device */
- int part = L < c->n_layers || !ch->head;
+/* One chain's layers (lo..lo+n-1 on its device) over the S rows, from the residual rows in
+ * xh. Returns its layer count, or 0: *lost = 1 when a frame failed (the state was rebuilt
+ * on the CPU, every chain is off), else nothing on its device changed. tail: whether the
+ * layers after it run elsewhere (the second device's chain, the CPU). */
+static int q36c_forward_seg(Model *m, Q36Chain *ch, float *xh, int S, int pos_base, FILE *lf, int want_x, int nlogits,
+ float *logit, int *rows_only, int *lost) {
+ *rows_only = 0; *lost = 0;
+ Cfg *c = &m->c; int D = c->hidden, L = ch->n, lo = ch->lo, hi = lo + L, E = c->n_experts; /* L: the layers on the device */
+ int part = hi < c->n_layers || !ch->head;
if (part) { want_x = 1; nlogits = 0; } /* the residual comes back; the CPU runs the rest and the head */
int mirror_ok = q36c_mirror(ch, m);
int CH = mirror_ok ? q36c_chunk_rows(ch, m) : 1, rows = S < CH ? S : CH;
@@ -756,18 +835,18 @@ static int q36c_forward(Model *m, float *xh, int S, int pos_base, FILE *lf, int
}
}
int ok = 1, pending = 0;
- for (int i = 0; i < L && ok; i++) {
+ for (int i = lo; i < hi && ok; i++) {
Layer *l = &m->L[i];
if (pending) {
ok = q36c_combine(ch, m, i - 1, n);
- if (ok && lf) ok = vkc_copy(ch->lfd, ((size_t)(i - 1) * 3 + 2) * D, ch->x, (size_t)(n - 1) * D, D);
+ if (ok && lf) ok = vkc_copy(ch->lfd, ((size_t)(i - 1 - lo) * 3 + 2) * D, ch->x, (size_t)(n - 1) * D, D);
}
ok = ok && q36c_norm(ch->x, 0, ch->prm, ch->o_in[i], ch->nrm, 0, n, D, c->eps);
ok = ok && (c->is_attn[i] ? q36c_attention(ch, m, l, i, n, pb) : q36c_deltanet(ch, m, l, i, n, c0, ns));
- if (ok && lf) ok = vkc_copy(ch->lfd, (size_t)i * 3 * D, ch->tmp, (size_t)(n - 1) * D, D);
+ if (ok && lf) ok = vkc_copy(ch->lfd, (size_t)(i - lo) * 3 * D, ch->tmp, (size_t)(n - 1) * D, D);
VkcEw add = {VKC_EW_ADD, n * D, D, 1, 0, 1, 0, 0, 0, 0, 0, 1.f};
ok = ok && vkc_ew(ch->x, ch->x, ch->tmp, NULL, NULL, &add);
- if (ok && lf) ok = vkc_copy(ch->lfd, ((size_t)i * 3 + 1) * D, ch->x, (size_t)(n - 1) * D, D);
+ if (ok && lf) ok = vkc_copy(ch->lfd, ((size_t)(i - lo) * 3 + 1) * D, ch->x, (size_t)(n - 1) * D, D);
ok = ok && q36c_norm(ch->x, 0, ch->prm, ch->o_post[i], ch->h2, 0, n, D, c->eps);
if (ok && E > 0) {
ok = vkc_matmul(vk_qw_tensor(&l->gate), ch->h2, 0, ch->lg, 0, n) &&
@@ -803,11 +882,11 @@ static int q36c_forward(Model *m, float *xh, int S, int pos_base, FILE *lf, int
* Keep each layer in its own submission: a full 27B prompt in
* one submission can exceed the driver's watchdog. The next
* frame's barrier orders its residual add after this MLP. */
- if (ok && i + 1 < L) ok = vkc_submit(0) && vkc_begin();
+ if (ok && i + 1 < hi) ok = vkc_submit(0) && vkc_begin();
}
pending = 1;
}
- if (ok) ok = q36c_combine(ch, m, L - 1, n);
+ if (ok) ok = q36c_combine(ch, m, hi - 1, n);
if (ok && lf) ok = vkc_copy(ch->lfd, ((size_t)(L - 1) * 3 + 2) * D, ch->x, (size_t)(n - 1) * D, D);
if (ok && want_x) ok = vkc_copy(ch->xd, 0, ch->x, 0, (size_t)n * D);
int last = c0 + n == S;
@@ -822,7 +901,7 @@ static int q36c_forward(Model *m, float *xh, int S, int pos_base, FILE *lf, int
ok = ok && vkc_submit(1);
ch->frames++; ch->wait_ms += tm_now() - t0;
if (!ok) goto lost;
- if (E == 0) for (int i = 0; i < L; i++) { /* a dense model reads its K/V rows back here */
+ if (E == 0) for (int i = lo; i < hi; i++) { /* a dense model reads its K/V rows back here */
if (!c->is_attn[i]) continue;
const float *kv = (const float *)vkc_ptr(ch->kvd) + (size_t)ch->attn_ord[i] * 2 * ch->rows * kvo;
for (int s = 0; s < n; s++) for (int h = 0; h < c->kv_heads; h++) {
@@ -832,7 +911,7 @@ static int q36c_forward(Model *m, float *xh, int S, int pos_base, FILE *lf, int
}
}
if (want_x) memcpy(xh + (size_t)c0 * D, vkc_ptr(ch->xd), (size_t)n * D * sizeof(float));
- for (int i = 0; i < L; i++) if (c->is_attn[i]) { ch->kv_valid[i] = pb + n; vkc_kv_done(&ch->ks, ch->attn_ord[i], pb + n); }
+ for (int i = lo; i < hi; i++) if (c->is_attn[i]) { ch->kv_valid[i] = pb + n; vkc_kv_done(&ch->ks, ch->attn_ord[i], pb + n); }
ch->dn_where = Q36C_DEV; ch->host_zero = 0;
if (last && !part) memcpy(logit, vkc_ptr(ch->outd), (size_t)nlogits * c->vocab * sizeof(float));
}
@@ -844,18 +923,58 @@ static int q36c_forward(Model *m, float *xh, int S, int pos_base, FILE *lf, int
return L;
lost: /* a frame failed: the device is gone (or would not take a command); the CPU takes over */
vkc_gemm_rows(-1);
- if (!vkc_lost()) { vkc_finish(); coli_vk_mark_lost(); }
+ if (!vkc_lost()) { vkc_finish(); coli_vk_mark_lost_dev(ch->d); }
+ vkc_device(0);
q36c_recover(m, pos_base);
+ *lost = 1;
return 0;
}
+/* Every layer the chains hold for S rows from host rows xh, the last row's logits into
+ * `logit`; xh gets the final rows back when want_x. 0 = not taken (nothing on the devices
+ * changed, or a device was lost and the state rebuilt: the CPU runs the step). Otherwise
+ * the layers they ran, from layer 0: the primary's, then the second device's (from the
+ * residual rows the primary's brought back). With layers left (or the head on the CPU)
+ * *rows_only is 1, xh holds every row's residual after them, and the caller runs the
+ * rest of the layers and the head on the CPU. */
+static int q36c_forward(Model *m, float *xh, int S, int pos_base, FILE *lf, int want_x, int nlogits, float *logit,
+ int *rows_only) {
+ *rows_only = 0;
+ if (!g_vk_chain || qt_ready() || qq_active() || g_pilot) return 0;
+ /* prompts only: decode and verifies on the CPU */
+ if (g_vk_chain == COLI_VK_CHAIN_PREFILL && (S <= 2 || g_q36_rowwise)) return 0;
+ Q36Chain *ch = q36c_setup(m), *ch2 = q36c_of(m, 1);
+ if (!ch || ch->failed) return 0;
+ if (ch2 && (!ch2->ok || ch2->failed || ch2->lo != ch->lo + ch->n)) ch2 = NULL;
+ int lost = 0, ro = 0;
+ g_q36_vk_dev = 0;
+ int n = q36c_forward_seg(m, ch, xh, S, pos_base, lf, ch2 ? 1 : want_x, ch2 ? 0 : nlogits, logit, &ro, &lost);
+ if (!n || !ch2) { *rows_only = ro; return n; }
+ vkc_device(1); g_q36_vk_dev = 1;
+ int n2 = q36c_forward_seg(m, ch2, xh, S, pos_base, lf, want_x, nlogits, logit, &ro, &lost);
+ vkc_device(0); g_q36_vk_dev = 0;
+ if (lost) return 0;
+ if (!n2) { /* declined before touching its device: the CPU runs its layers this step */
+ q36c_cpu_one(m, ch2, pos_base);
+ *rows_only = 1;
+ return n;
+ }
+ *rows_only = ro;
+ return n + n2;
+}
+
static void q36c_report(Model *m) {
- Q36Chain *ch = (Q36Chain *)m->vkchain;
- if (!ch || !ch->ok || !ch->forwards) return;
- VkcStats st; vkc_stats(&st);
- fprintf(stderr, "[VK] qwen36 chain: %llu forwards, %llu frames (%llu ops, %llu matmuls, %llu tiled GEMM), "
- "%.1f ms waiting for the device, %.1f ms of routed experts on the host, %.1f MiB on the device\n",
- ch->forwards, st.frames, st.ops, st.matmuls, st.gemms, st.wait_ms, ch->host_ms, st.dev_bytes / 1048576.0);
- vkc_kv_report(&ch->ks);
- vkc_prof_print();
+ for (int d = 0; d < 2; d++) {
+ Q36Chain *ch = q36c_of(m, d);
+ if (!ch || !ch->ok || !ch->forwards) continue;
+ int was = vkc_device(d);
+ VkcStats st; vkc_stats(&st);
+ fprintf(stderr, "[VK] %s chain: %llu forwards, %llu frames (%llu ops, %llu matmuls, %llu tiled GEMM), "
+ "%.1f ms waiting for the device, %.1f ms of routed experts on the host, %.1f MiB on the device\n",
+ q36c_name(ch), ch->forwards, st.frames, st.ops, st.matmuls, st.gemms, st.wait_ms, ch->host_ms,
+ st.dev_bytes / 1048576.0);
+ vkc_kv_report(&ch->ks);
+ vkc_prof_print();
+ vkc_device(was);
+ }
}
diff --git a/c/tests/vulkan_engines.sh b/c/tests/vulkan_engines.sh
index bdb0c31f..8dde7921 100755
--- a/c/tests/vulkan_engines.sh
+++ b/c/tests/vulkan_engines.sh
@@ -3084,6 +3084,7 @@ for f in tests/vulkan_partial_*.sh; do
. "$f"
done
. tests/vulkan_dev2.sh # the expert tier on two devices
+. tests/vulkan_layers_dev2.sh # the chain's layers on two devices
# ---- big prompt chunks and expert streaming (docs/vulkan.md, "Big prompt chunks and
# expert streaming") ----
# Every engine with the chain, on a prompt longer than its usual block, against its CPU
@@ -4180,6 +4181,8 @@ case "${1:-}" in
prefill-deepseek) family_prefill_deepseek ;;
kv-split) family_kv_split ;;
kv-split-sanitize) family_kv_split_sanitize ;;
+ layers-dev2) family_layers_dev2 ;;
+ layers-dev2-sanitize) family_layers_dev2_sanitize ;;
kv-split-deepseek) family_kv_split_deepseek ;;
kv-split-deepseek-sanitize) family_kv_split_deepseek_sanitize ;;
staged) family_staged ;;
@@ -4200,5 +4203,5 @@ case "${1:-}" in
partial-*) g=${1#partial-}; fn=ptl_family_${g//-/_}
declare -F "$fn" >/dev/null || { echo "no partial-chain group ${g}" >&2; exit 2; }
"$fn" ;;
- *) echo "usage: $0 decide|decide-sanitize|staged|<family>-staged|shader|qwen|qwen-sanitize|inkling-olmoe|inkling-olmoe-sanitize|mimo-qwenimage|deepseek|deepseek-sanitize|kimi|kimi-mimo-sanitize|glm|glm-sanitize|qwen-chain|qwen-chain-sanitize|qwen-spec|qwen-spec-sanitize|mimo-chain|mimo-chain-sanitize|inkling-olmoe-chain|inkling-olmoe-chain-sanitize|glm-chain|glm-chain-sanitize|kimi-chain|kimi-chain-sanitize|deepseek-chain|deepseek-chain-sanitize|dense-only-<group>[-sanitize]|partial-<group>[-sanitize]|prefill-qwen|prefill-qwen-sanitize|prefill-inkling-olmoe|prefill-mimo-kimi|prefill-glm|prefill-deepseek|kv-split|kv-split-sanitize|kv-split-deepseek|kv-split-deepseek-sanitize|dev2|dev2-deepseek-kimi-mimo|dev2-sanitize" >&2; exit 2 ;;
+ *) echo "usage: $0 decide|decide-sanitize|staged|<family>-staged|shader|qwen|qwen-sanitize|inkling-olmoe|inkling-olmoe-sanitize|mimo-qwenimage|deepseek|deepseek-sanitize|kimi|kimi-mimo-sanitize|glm|glm-sanitize|qwen-chain|qwen-chain-sanitize|qwen-spec|qwen-spec-sanitize|mimo-chain|mimo-chain-sanitize|inkling-olmoe-chain|inkling-olmoe-chain-sanitize|glm-chain|glm-chain-sanitize|kimi-chain|kimi-chain-sanitize|deepseek-chain|deepseek-chain-sanitize|dense-only-<group>[-sanitize]|partial-<group>[-sanitize]|prefill-qwen|prefill-qwen-sanitize|prefill-inkling-olmoe|prefill-mimo-kimi|prefill-glm|prefill-deepseek|kv-split|kv-split-sanitize|layers-dev2|layers-dev2-sanitize|kv-split-deepseek|kv-split-deepseek-sanitize|dev2|dev2-deepseek-kimi-mimo|dev2-sanitize" >&2; exit 2 ;;
esac
diff --git a/c/tests/vulkan_layers_dev2.sh b/c/tests/vulkan_layers_dev2.sh
new file mode 100644
index 00000000..4bed60cf
--- /dev/null
+++ b/c/tests/vulkan_layers_dev2.sh
@@ -0,0 +1,100 @@
+# Sourced by tests/vulkan_engines.sh: the dense chain's layers on two devices (docs/vulkan.md,
+# "Layers on two devices"). Lavapipe is opened twice (COLI_VK_DEV2=0: a second logical device
+# on the same physical one, the test mode); COLI_VK_CHAIN_LAYERS puts the first layers on
+# the primary device, COLI_VK_CHAIN_LAYERS2 the next ones on the second, the CPU the rest.
+#
+# layers-dev2 every chain engine against its own CPU run
+# layers-dev2-sanitize the same paths under ASan and UBSan
+#
+# Each gate (ld2_gate) is chain_gate's (the CPU's tokens, logits within 1e-4 of the
+# largest) with the split forced, and the second device's chain must have run.
+
+# ld2_gate <engine> <tag> <n0> <n1> <env...> -- <argv...>
+ld2_gate() {
+ local eng=$1 tag=$2 n0=$3 n1=$4; shift 4
+ chain_gate "$eng" "$tag" 1 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=$n0 COLI_VK_CHAIN_LAYERS2=$n1 COLI_VK_TIER_BALANCE=0 "$@"
+ local f; f=$(ld2_forwards "$eng" vk.log)
+ [ "$f" -gt 0 ] || { cat vk.log; fail "$tag: the second device's chain never ran"; }
+ echo " $tag: the second device's chain ran $f forwards"
+}
+# ld2_forwards <engine> <log>: N from "[VK] <engine> dev2 chain: N forwards"
+ld2_forwards() {
+ local n; n=$(sed -n "s/^\[VK\] $1 dev2 chain: \([0-9]*\) forwards.*/\1/p" "$2" | tail -1)
+ echo "${n:-0}"
+}
+# ld2_lost <engine> <tag> <n0> <n1> <frame> <env...> -- <argv...>: the second device lost at
+# its given frame (COLI_VK_CHAIN_FAULT2): the CPU's tokens; at its first frame (the setup's
+# parameter upload) LD2_EXPECT=setup: its layers stay on the CPU, nothing to rebuild; past
+# it, the state rebuilt on the CPU
+ld2_lost() {
+ local eng=$1 tag=$2 n0=$3 n1=$4 k=$5; shift 5
+ local envs=(); while [ "$1" != "--" ]; do envs+=("$1"); shift; done; shift
+ rm -f chain.usage
+ env "${envs[@]}" ./"$eng" "$@" > cpu.log 2>&1 || true
+ env "${envs[@]}" COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=$n0 COLI_VK_CHAIN_LAYERS2=$n1 COLI_VK_CHAIN_FAULT2=$k \
+ COLI_USAGE=chain.usage COLI_VK_TIER_SYNC=1 COLI_VK_TIER_BALANCE=0 COLI_VULKAN=1 COLI_VK_CHAIN=1 ./"$eng" "$@" > vk.log 2>&1 || true
+ same_tokens cpu.log vk.log "$tag"
+ grep -q "COLI_VK_CHAIN_FAULT2" vk.log || { cat vk.log; fail "$tag: the second device's fault never fired"; }
+ if [ "${LD2_EXPECT:-}" = setup ]; then
+ grep -q "^\[VK\] $eng dev2 chain: 0 of [0-9]* layers on the device: the chain stays off (layer 0 did not reach the device: the device was lost" vk.log ||
+ { cat vk.log; fail "$tag: the second device's layers did not go to the CPU"; }
+ echo "OK $tag: tokens = CPU, the second device's layers on the CPU from the start"
+ return 0
+ fi
+ grep -q "rebuilding the state of [1-9]" vk.log || { cat vk.log; fail "$tag: no state was rebuilt"; }
+ echo "OK $tag: tokens = CPU, $(grep -o 'rebuilding the state of [0-9]* positions' vk.log | head -1)"
+}
+
+ld2_qwen36() {
+ [ -d qwen36_tiny_c ] || { $PY tools/make_qwen36_tiny.py --out qwen36_tiny --ref-mode full --emit-ref qwen36_tiny/ref_full.json
+ $PY tools/convert_qwen36.py --model qwen36_tiny --out qwen36_tiny_c --ebits 8; }
+ [ -d qwen36_kv_c ] || kv_qwen36_fixtures
+ local R=qwen36_tiny/ref_full.json T="SNAP=qwen36_tiny_c COLI_DENSE_I8=0"
+ ld2_gate qwen36 "ld2 qwen36 3 + 5, the head on the second device" 3 5 $T -- 8 8 $R
+ ld2_gate qwen36 "ld2 qwen36 2 + 3, the CPU the rest and the head" 2 3 $T -- 8 8 $R
+ ld2_gate qwen36 "ld2 qwen36 1 + 7, prompt in chunks of 3" 1 7 $T COLI_VK_CHAIN_ROWS=3 -- 8 8 $R
+ ld2_gate qwen36 "ld2 qwen36 cap 1" 4 4 $T -- 1 8 $R
+ CPUENV=COLI_DENSE_IDOT=0 ld2_gate qwen36 "ld2 qwen36 int8 trunk" 4 4 SNAP=qwen36_tiny_c -- 8 8 $R
+ ld2_gate qwen36 "ld2 qwen36 experts on both devices too" 3 5 $T COLI_VK_TIER_GB=0.00002 -- 8 8 $R
+ CHAINMODE=2 ld2_gate qwen36 "ld2 qwen36 prompts only" 3 5 $T -- 8 8 $R
+ ld2_gate qwen36 "ld2 qwen36 dense model" 2 4 SNAP=qwen38_27b_kv_c COLI_DENSE_I8=0 -- 8 8 qwen38_27b_kv/ref_full.json
+ # the KV cache split on both devices' attention layers
+ ld2_gate qwen36 "ld2 qwen36 KV split on both" 4 4 COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 SNAP=qwen36_kv_c COLI_DENSE_I8=0 \
+ -- 8 8 qwen36_kv/ref_full.json
+ [ "$(kv_hostparts qwen36 vk.log)" -gt 0 ] && [ "$(kv_hostparts "qwen36 dev2" vk.log)" -gt 0 ] ||
+ { cat vk.log; fail "ld2 qwen36 KV split on both: a device's split ran no host part"; }
+ # prompt-lookup drafts verified across both devices (rollbacks of both chains' copies)
+ local f
+ for f in accept mixed cycle row1 row3; do
+ ld2_gate qwen36 "ld2 qwen36 lookup $f" 3 5 COLI_DENSE_I8=0 COLI_LOOKUP=1 COLI_SPEC_GATE=0 COLI_LOOKUP_FORCE=$f \
+ SNAP=qwen36_kv_c -- 8 8 qwen36_kv/ref_full.json
+ done
+ # the second device lost: at its first frame, and mid-decode
+ LD2_EXPECT=setup ld2_lost qwen36 "ld2 qwen36 second device lost at its setup" 3 5 1 $T -- 8 8 $R
+ ld2_lost qwen36 "ld2 qwen36 second device lost mid-decode" 3 5 40 SNAP=qwen36_kv_c COLI_DENSE_I8=0 -- 8 8 qwen36_kv/ref_full.json
+ # the prefix-reuse contract with the state on both devices, and serve sessions (pins,
+ # prompt-cache extensions, a divergent prompt, logprobs)
+ QWEN36_TINY=$PWD/qwen36_tiny_c COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=3 COLI_VK_CHAIN_LAYERS2=5 \
+ COLI_USAGE=$PWD/chain.usage $PY tests/test_qwen36_prefix_serve.py
+ CHAIN_SERVE_EXPECT='qwen36 dev2 chain: [1-9][0-9]* forwards' \
+ $PY tests/vulkan_chain_serve.py ./qwen36 qwen36_tiny_c COLI_DENSE_I8=0 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=3 COLI_VK_CHAIN_LAYERS2=5
+}
+
+family_layers_dev2() {
+ export OMP_NUM_THREADS=2
+ make qwen36 tests/test_vk_chain VK=1
+ COLI_VK_DEV2=0 ./tests/test_vk_chain shaders/qmatmul.spv | tee vk_chain.log
+ tail -1 vk_chain.log | grep -qx PASS || fail "the chain's ops on two devices"
+ ld2_qwen36
+ unset OMP_NUM_THREADS
+}
+family_layers_dev2_sanitize() {
+ make clean >/dev/null 2>&1 || true
+ make qwen36 VK=1 EXTRA_CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -g"
+ export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0 UBSAN_OPTIONS=print_stacktrace=1
+ [ -d qwen36_kv_c ] || kv_qwen36_fixtures
+ kv_san qwen36 "asan ld2 qwen36 KV split on both" COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=4 COLI_VK_CHAIN_LAYERS2=4 \
+ COLI_VK_KV_DEVICE_ROWS=16 COLI_VK_KV_BLOCK=4 COLI_DENSE_I8=0 SNAP=qwen36_kv_c ./qwen36 8 8 qwen36_kv/ref_full.json
+ [ "$(ld2_forwards qwen36 san.log)" -gt 0 ] || { cat san.log; fail "asan ld2 qwen36: the second device's chain never ran"; }
+ make clean >/dev/null 2>&1 || true
+}
diff --git a/c/vk_chain.c b/c/vk_chain.c
index 5b973b3f..b59f514f 100644
--- a/c/vk_chain.c
+++ b/c/vk_chain.c
@@ -132,6 +132,13 @@ static int vkc_tensor_info(const ColiVkTensor *t, ColiVkTensorInfo *ti) {
int vkc_ready(void) { return KC.ready && !KC.lost; }
int vkc_device(int d) { int was = g_kd; g_kd = d == 1 ? 1 : 0; return was; }
int vkc_device_now(void) { return g_kd; }
+void vkc_shutdown(void);
+void vkc_shutdown_all(void) { /* both devices' contexts, the second first (at exit) */
+ int was = g_kd;
+ g_kd = 1; vkc_shutdown();
+ g_kd = 0; vkc_shutdown();
+ g_kd = was;
+}
int vkc_lost(void) { return KC.lost; }
static void lose(const char *what, VkResult r) {
@@ -488,10 +495,12 @@ int vkc_submit(int wait) {
vkEndCommandBuffer(f->cmd);
f->open = 0; KC.cur = -1;
if (KC.lost) return 0;
- { /* COLI_VK_CHAIN_FAULT=n (tests): the n-th submission fails as a lost device would */
- static long fault = -2;
- if (fault == -2) { const char *e = getenv("COLI_VK_CHAIN_FAULT"); fault = e && *e ? atol(e) : -1; }
- if (fault > 0 && (long)KC.st.frames + 1 >= fault) { lose("queue submit (COLI_VK_CHAIN_FAULT)", VK_ERROR_DEVICE_LOST); return 0; }
+ { /* COLI_VK_CHAIN_FAULT=n (tests): the primary device's n-th submission fails as a lost
+ * device would; COLI_VK_CHAIN_FAULT2 the second device's */
+ static long fault[2] = {-2, -2};
+ if (fault[g_kd] == -2) { const char *e = getenv(g_kd ? "COLI_VK_CHAIN_FAULT2" : "COLI_VK_CHAIN_FAULT"); fault[g_kd] = e && *e ? atol(e) : -1; }
+ if (fault[g_kd] > 0 && (long)KC.st.frames + 1 >= fault[g_kd])
+ { lose(g_kd ? "queue submit (COLI_VK_CHAIN_FAULT2)" : "queue submit (COLI_VK_CHAIN_FAULT)", VK_ERROR_DEVICE_LOST); return 0; }
}
vkResetFences(KC.dev, 1, &f->fence);
VkSubmitInfo si = {.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, .commandBufferCount = 1, .pCommandBuffers = &f->cmd};
@@ -1387,14 +1396,15 @@ int vkc_fit(const char *engine, int L, const size_t *layer_bytes, const size_t *
fit->fixed = fixed_bytes + fit->pools;
fit->tail_b = tail_bytes;
size_t room = fit->free_b > fit->reserve ? fit->free_b - fit->reserve : 0;
- const char *e = getenv("COLI_VK_CHAIN_LAYERS");
+ /* forced: COLI_VK_CHAIN_LAYERS on the primary, COLI_VK_CHAIN_LAYERS2 on the second device */
+ const char *fvar = g_kd ? "COLI_VK_CHAIN_LAYERS2" : "COLI_VK_CHAIN_LAYERS", *e = getenv(fvar);
char forced[48] = "";
if (e && *e && strcmp(e, "auto") != 0) {
int v = atoi(e);
fit->n = v < 0 ? 0 : v > fit->L ? fit->L : v;
fit->forced = 1;
fit->tail = fit->n == fit->L && fit->L > 0;
- snprintf(forced, sizeof forced, "COLI_VK_CHAIN_LAYERS=%s", e);
+ snprintf(forced, sizeof forced, "%s=%s", fvar, e);
} else if (fit->fixed + all <= room) {
fit->n = fit->L;
fit->tail = fit->fixed + all + tail_bytes <= room;
diff --git a/c/vk_chain.h b/c/vk_chain.h
index 6a83bf2e..cf5a202b 100644
--- a/c/vk_chain.h
+++ b/c/vk_chain.h
@@ -64,6 +64,7 @@ void vkc_shutdown(void); /* before coli_vk_shutdown (register it with atexi
* the op fails; vkc_free takes it back on its own device. Returns the previous device. */
int vkc_device(int d);
int vkc_device_now(void);
+void vkc_shutdown_all(void); /* both devices' contexts: what an engine registers with atexit */
VkcBuf *vkc_buf(size_t bytes, int kind); /* zero-filled; NULL when out of memory */
void vkc_free(VkcBuf *b); /* waits for the frames that may read it */