Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

mimo: the chain's layers on two devices (COLI_VK_DEV2)

JustVugg0e210f38e483
diff --git a/c/mimo.c b/c/mimo.c
index 8250de36..ba2fd79c 100644
--- a/c/mimo.c
+++ b/c/mimo.c
@@ -354,18 +354,21 @@ static int dw_vk_fmt(const DW *d, int *gs, const float **sc) {
return -1;
}
+static int g_mimo_vk_dev; /* the device dw_upload sends a new matrix to: 1 for the chain's layers on
+ * COLI_VK_DEV2's (mimo_chain.h) */
static int dw_upload(DW *d) {
int gs; const float *sc;
int fmt = dw_vk_fmt(d, &gs, &sc);
if (d->vk || d->vk_off || !d->w) return d->vk != NULL;
- if (fmt < 0 || !coli_vk_tensor_ensure((ColiVkTensor **)&d->vk, d->w, sc, fmt, d->I, d->O, gs)) d->vk_off = 1;
+ if (fmt < 0 || !(g_mimo_vk_dev ? coli_vk_tensor_ensure2((ColiVkTensor **)&d->vk, d->w, sc, fmt, d->I, d->O, gs)
+ : coli_vk_tensor_ensure((ColiVkTensor **)&d->vk, d->w, sc, fmt, d->I, d->O, gs))) d->vk_off = 1;
return d->vk != NULL;
}
static int dw_matmul_vk(float *y, const float *x, int S, const DW *d) {
if (!g_vk_ready || !(coli_vk_dense() || g_mimo_dho) || d->vk_off || !vk_main_thread()) return 0;
DW *dev = (DW *)d; /* the device copy is a cache inside a read-only matrix */
- if (!dw_upload(dev)) return 0;
+ if (!dw_upload(dev) || coli_vk_tensor_dev((ColiVkTensor *)dev->vk)) return 0; /* the second device's: only its chain */
int gs; const float *sc;
int fmt = dw_vk_fmt(d, &gs, &sc);
return coli_vk_matmul((ColiVkTensor **)&dev->vk, y, x, d->w, sc, fmt, S, d->I, d->O, gs);
@@ -608,6 +611,7 @@ typedef struct {
uint8_t **ehit; /* [layer][expert] routed this turn, for HITS */
#ifdef COLI_VULKAN
void *vkchain; /* the dense chain's device state (mimo_chain.h), NULL until it runs */
+ void *vkchain2; /* its layers on COLI_VK_DEV2's device, after the primary's (mimo_chain.h) */
#endif
} Model;
@@ -1460,6 +1464,23 @@ static void mc_fit_start(Model *m) {
g_mc_count = -1; g_mc_fitcount = 0;
free(g.o_kvd);
vkc_fit("mimo", L, per, mat, fixed, tail, &g_mc_fit);
+ /* the layers the primary leaves, on COLI_VK_DEV2's device: a fit of its own from layer
+ * n0 with that device's free memory, its pipelines up now (the head goes with it when it
+ * ends at the last layer with room; the tower stays on the CPU) */
+ int n0 = g_mc_fit.n;
+ if (vkc_fit_partial(&g_mc_fit) && n0 > 0 && n0 < L && mc_dev2_wanted() && getenv("COLI_VK_DEV2") &&
+ coli_vk_dev2_open_env()) {
+ vkc_device(1);
+ size_t tail2 = vkc_fit_tensor(hf, m->head.I, m->head.O, hg);
+ int n2 = vkc_fit("mimo dev2", L - n0, per + n0, mat + n0, fixed, tail2, &g_mc_fit2);
+ if (n2 > 0 && !vkc_init()) {
+ fprintf(stderr, "[VK] mimo chain: the second device's pipelines did not come up; its layers stay on the CPU\n");
+ n2 = 0;
+ }
+ g_mc_fit2_on = n2 > 0;
+ if (!g_mc_fit2_on) g_mc_fit2.tail = 0;
+ vkc_device(0);
+ }
free(per); free(mat);
}
/* Layers from..L-1, the head and the tower refused to the device: what of them is there
@@ -1474,7 +1495,8 @@ static void mc_refuse_from(Model *m, int from) {
Layer *l = &m->L[li];
mc_refuse(&l->qkv); mc_refuse(&l->o); mc_refuse(&l->gate); mc_refuse(&l->up); mc_refuse(&l->down);
}
- mc_refuse(&m->head);
+ if (!(m->head.vk && coli_vk_tensor_dev((ColiVkTensor *)m->head.vk))) /* the second device's chain keeps it */
+ mc_refuse(&m->head);
if (g_vision) {
Vision *v = g_vision;
mc_refuse(&v->embed); mc_refuse(&v->fc1); mc_refuse(&v->fc2);
@@ -1488,6 +1510,7 @@ static void dw_drop(DW *d, int *n); /* below */
* freed, it and the rest refused, vkc_fit_shrink); dropped: with COLI_VK_DENSE_HOST, each
* layer's host copies given back once all of it is on the device (counted there). */
static void mc_place(Model *m, int *dropped) {
+ int n0 = g_mc_fit.n;
for (int i = 0; i < g_mc_fit.n; i++) {
Layer *l = &m->L[i];
DW *d[5] = {&l->qkv, &l->o, &l->gate, &l->up, &l->down};
@@ -1501,7 +1524,38 @@ static void mc_place(Model *m, int *dropped) {
if (dropped) for (int k = 0; k < nd; k++) dw_drop(d[k], dropped);
vkc_fit_mark(&g_mc_fit, i);
}
- if (vkc_fit_partial(&g_mc_fit)) mc_refuse_from(m, g_mc_fit.n); /* the CPU's layers, the head, the tower */
+ int end = g_mc_fit.n;
+ if (g_mc_fit2_on && g_mc_fit.n < n0) {
+ /* the primary placed fewer layers than its fit: the second device's would not follow
+ * them, so they stay on the CPU too */
+ fprintf(stderr, "[VK] mimo chain: the primary device stopped before layer %d; layers %d..%d stay on the CPU, "
+ "not on the second device\n", n0, n0, n0 + g_mc_fit2.n - 1);
+ g_mc_fit2_on = 0;
+ vkc_device(1); vkc_shutdown(); vkc_device(0);
+ } else if (g_mc_fit2_on) { /* the second device's layers (their host copies kept), and the head with the last one */
+ vkc_device(1); g_mimo_vk_dev = 1;
+ for (int i = n0; i < n0 + g_mc_fit2.n; i++) {
+ Layer *l = &m->L[i];
+ DW *d[5] = {&l->qkv, &l->o, &l->gate, &l->up, &l->down};
+ int nd = m->c.moe[i] ? 2 : 5, ok = 1;
+ for (int k = 0; k < nd && ok; k++) ok = dw_upload(d[k]);
+ if (!ok) {
+ g_mimo_vk_dev = 0;
+ mc_refuse_from(m, i);
+ vkc_fit_shrink("mimo dev2", &g_mc_fit2, i - n0, "a dense matrix did not go up");
+ g_mimo_vk_dev = 1;
+ break;
+ }
+ vkc_fit_mark(&g_mc_fit2, i - n0);
+ }
+ end = n0 + g_mc_fit2.n;
+ if (end == m->c.n_layers && g_mc_fit2.tail && !dw_upload(&m->head)) g_mc_fit2.tail = 0;
+ g_mimo_vk_dev = 0;
+ vkc_fit_placed("mimo dev2", &g_mc_fit2);
+ if (!g_mc_fit2.n) { g_mc_fit2_on = 0; vkc_shutdown(); }
+ vkc_device(0);
+ }
+ if (vkc_fit_partial(&g_mc_fit)) mc_refuse_from(m, end); /* the CPU's layers, the head (unless the second device's), the tower */
vkc_fit_placed("mimo", &g_mc_fit);
g_mc_placed = 1;
}
@@ -2137,7 +2191,7 @@ int main(int argc, char **argv) {
if (g_vk_ready && !vkt_ready() && !coli_vk_dense()) coli_vk_dense_decide("mimo", 0, 1); /* no tier after all */
if (g_vk_chain) { /* the chain's teardown before the device's (the tier registered the device's) */
if (!tier) atexit(coli_vk_shutdown);
- atexit(vkc_shutdown);
+ atexit(vkc_shutdown_all);
}
}
if (g_vk_ready && (coli_vk_dense() || g_vk_chain) && !(g_mc_fit.L && !g_mc_fit.n)) { /* COLI_VK_DENSE=0: the trunk stays on the CPU */
diff --git a/c/mimo_chain.h b/c/mimo_chain.h
index a02ba4ff..60e16783 100644
--- a/c/mimo_chain.h
+++ b/c/mimo_chain.h
@@ -67,7 +67,8 @@
typedef struct {
int ok, failed;
- int nl, full; /* layers on the device (the first nl); the head there too */
+ int lo, d; /* its layers start at lo, on device d (vkc_device) */
+ int nl, full; /* layers on the device (lo..lo+nl-1); the head there too */
int rows; /* scratch capacity in rows */
int rd_off[2]; /* the RoPE table of each layer kind in cs */
VkcBuf *prm; /* the norms and the sink logits */
@@ -90,6 +91,15 @@ static int g_vk_chain = 0; /* COLI_VK_CHAIN decided on, and the chain's pipe
static VkcFit g_mc_fit;
static int g_mc_placed; /* vkc_fit_placed said it (mc_place, mimo.c) */
static int mc_layers(const Model *m) { return g_mc_fit.L ? g_mc_fit.n : m->c.n_layers; }
+/* The layers after the primary's on COLI_VK_DEV2's device (docs/vulkan.md, "Layers on two
+ * devices"): a second chain from layer g_mc_fit.n, its own fit (mc_fit_start), its
+ * matrices placed by mc_place. Each chunk crosses the primary's layers, comes down, crosses
+ * these, comes down, and the CPU runs what is left. */
+static VkcFit g_mc_fit2;
+static int g_mc_fit2_on;
+static MimoChain *mc_of(Model *m, int d) { return (MimoChain *)(d ? m->vkchain2 : m->vkchain); }
+static const char *mc_name(const MimoChain *ch) { return ch && ch->d ? "mimo dev2" : "mimo"; }
+static int mc_dev2_wanted(void) { const char *e = getenv("COLI_VK_CHAIN_DEV2"); return !(e && *e == '0'); }
/* the geometry of layer li's attention */
@@ -124,13 +134,17 @@ static size_t mc_kv_bytes(const Model *m) { return mc_kv_bytes_n(m, m->c.n_layer
/* The model's parameters on the device, its tensors resolved, its caches allocated;
* NULL = the chain cannot run. With a fit, the first n layers only (their matrices placed
* at start-up, mc_place), and the head when the tail went up too. */
-static MimoChain *mc_setup(Model *m) {
- MimoChain *ch = (MimoChain *)m->vkchain;
+/* d = 1: the second device's chain (its matrices placed by mc_place; device 1 current). */
+static MimoChain *mc_setup_dev(Model *m, int d) {
+ MimoChain *ch = mc_of(m, d);
if (ch) return ch->ok ? ch : NULL;
+ if (d && !g_mc_fit2_on) return NULL;
ch = (MimoChain *)calloc(1, sizeof *ch);
if (!ch) return NULL;
- m->vkchain = ch;
- Cfg *c = &m->c; int L = c->n_layers, H = c->hidden, nl = mc_layers(m);
+ if (d) m->vkchain2 = ch; else m->vkchain = ch;
+ ch->d = d;
+ Cfg *c = &m->c; int L = c->n_layers, H = c->hidden, lo = d ? g_mc_fit.n : 0, nl = d ? g_mc_fit2.n : mc_layers(m);
+ ch->lo = lo;
for (int k = 0; k < 2; k++)
if (c->head_dim[k] > 256 || c->v_dim[k] > 256) {
fprintf(stderr, "[VK] mimo chain: head dims %d/%d are past its attention shader's 256; per-matrix path\n",
@@ -144,27 +158,30 @@ static MimoChain *mc_setup(Model *m) {
if (!ch->o_ln1 || !ch->o_ln2 || !ch->o_sink || !ch->o_kvd || !ch->kv_valid || !ch->kvi || !ch->kc || !ch->vc) return NULL;
/* the matrices: the device copies the per-matrix path uploads (vk_dense_upload) */
int ok = 1;
- for (int i = 0; i < nl && ok; i++) {
+ g_mimo_vk_dev = d;
+ for (int i = lo; i < lo + nl && ok; i++) {
Layer *l = &m->L[i];
ok = mc_ctensor(&l->qkv) && mc_ctensor(&l->o) &&
(c->moe[i] || (mc_ctensor(&l->gate) && mc_ctensor(&l->up) && mc_ctensor(&l->down)));
}
- int full = nl == L && (!g_mc_fit.L || g_mc_fit.tail);
- if (ok && full && g_mc_fit.L && !mc_ctensor(&m->head)) { /* the fit's tail did not go up: the head on the CPU */
- full = 0; g_mc_fit.tail = 0;
+ VkcFit *fit = d ? &g_mc_fit2 : &g_mc_fit;
+ int full = lo + nl == L && (!fit->L || fit->tail);
+ if (ok && full && fit->L && !mc_ctensor(&m->head)) { /* the fit's tail did not go up: the head on the CPU */
+ full = 0; fit->tail = 0;
fprintf(stderr, "[VK] mimo chain: the head did not reach the device; it runs on the CPU, the layers here\n");
}
ok = ok && (!full || mc_ctensor(&m->head));
+ g_mimo_vk_dev = 0;
if (!ok) { fprintf(stderr, "[VK] mimo chain: a dense matrix did not reach the device; per-matrix path\n"); return NULL; }
size_t n = 0;
- for (int i = 0; i < nl; i++) {
+ for (int i = lo; i < lo + nl; i++) {
ch->o_ln1[i] = n; n += H; ch->o_ln2[i] = n; n += H;
if (m->L[i].sink) { ch->o_sink[i] = n; n += c->heads[c->swa[i]]; }
}
ch->o_final = n; n += H;
float *arena = calloc(n, sizeof(float));
if (!arena) return NULL;
- for (int i = 0; i < nl; i++) {
+ for (int i = lo; i < lo + nl; i++) {
Layer *l = &m->L[i];
memcpy(arena + ch->o_ln1[i], l->ln1, H * sizeof(float));
memcpy(arena + ch->o_ln2[i], l->ln2, H * sizeof(float));
@@ -181,17 +198,18 @@ static MimoChain *mc_setup(Model *m) {
size_t frow = 0;
for (int i = 0; i < L; i++) {
McGeo g = mc_geo(m, i);
- ch->kvi[i] = g.swa || i >= nl ? -1 : nfull++;
- if (!g.swa && i < nl) { frows = g.rows; frow = (size_t)(g.kd + g.vdd) * sizeof(float); }
+ int here = i >= lo && i < lo + nl;
+ ch->kvi[i] = g.swa || !here ? -1 : nfull++;
+ if (!g.swa && here) { frows = g.rows; frow = (size_t)(g.kd + g.vdd) * sizeof(float); }
}
/* KV placement is row-anchored; its legacy chunk argument is not used. The
* actual prompt chunk is sized after setup and clamped to ks.chunk. */
- if (nfull && !vkc_kv_plan(&ch->ks, "mimo", nfull, frow, frows, 1, 0, 0)) {
+ if (nfull && !vkc_kv_plan(&ch->ks, mc_name(ch), nfull, frow, frows, 1, 0, 0)) {
fprintf(stderr, "[VK] mimo chain: the KV split's tables refused; per-matrix path\n");
return NULL;
}
size_t kv = 0;
- for (int i = 0; i < nl; i++) {
+ for (int i = lo; i < lo + nl; i++) {
McGeo g = mc_geo(m, i);
if (!g.swa && ch->ks.on) g.rows = ch->ks.rows;
ch->kc[i] = vkc_buf((size_t)g.rows * g.kd * sizeof(float), VKC_DEV);
@@ -205,11 +223,14 @@ static MimoChain *mc_setup(Model *m) {
ch->nl = nl; ch->full = full;
ch->ok = 1;
int swa = 0;
- for (int i = 0; i < nl; i++) swa += c->swa[i];
- fprintf(stderr, "[VK] mimo chain: %d layers on the device (%d sliding window), %.1f MiB of KV mirrors\n",
- nl, swa, kv * 4 / 1048576.0);
+ for (int i = lo; i < lo + nl; i++) swa += c->swa[i];
+ if (d) fprintf(stderr, "[VK] mimo chain: layers %d..%d on the second device (%d sliding window)%s, %.1f MiB of KV mirrors\n",
+ lo, lo + nl - 1, swa, full ? ", the head too" : "", kv * 4 / 1048576.0);
+ else fprintf(stderr, "[VK] mimo chain: %d layers on the device (%d sliding window), %.1f MiB of KV mirrors\n",
+ nl, swa, kv * 4 / 1048576.0);
return ch;
}
+static MimoChain *mc_setup(Model *m) { return mc_setup_dev(m, 0); }
/* mc_res counts instead of reserving while g_mc_count >= 0 (the chunk's sizing; the fit's,
* g_mc_fitcount, each buffer as the pools round it) */
@@ -234,7 +255,7 @@ static int mc_chunk_rows(MimoChain *ch, Model *m) {
size_t row = (size_t)(b2 - b1) + (size_t)(2 * m->c.topk + 1) * m->c.hidden * sizeof(float);
/* moe_cpu retains a whole chunk's gathered inputs and gate/up outputs. */
row += (size_t)m->c.topk * ((size_t)m->c.hidden + 2 * (size_t)m->c.moe_inter) * sizeof(float);
- return vkc_chunk_rows("mimo", row);
+ return vkc_chunk_rows(mc_name(ch), row);
}
/* scratch for `rows` rows, `out` rows of logits */
@@ -286,18 +307,22 @@ static int mc_scratch(MimoChain *ch, Model *m, int rows, int out) {
/* ---- the KV caches between the host and the device ------------------------------- */
/* A CPU step from pos_base writes the host's rows from there on. */
static void mc_cpu_step(Model *m, int pos_base) {
- MimoChain *ch = (MimoChain *)m->vkchain;
- if (!ch || !ch->ok) return;
- for (int i = 0; i < ch->nl; i++) { /* the device's layers: the CPU's keep their caches on the host */
- if (ch->kv_valid[i] > pos_base) ch->kv_valid[i] = pos_base;
- if (ch->kvi[i] >= 0) vkc_kv_lower(&ch->ks, ch->kvi[i], pos_base);
+ for (int d = 0; d < 2; d++) {
+ MimoChain *ch = mc_of(m, d);
+ if (!ch || !ch->ok) continue;
+ for (int i = ch->lo; i < ch->lo + ch->nl; i++) { /* the devices' layers: the CPU's keep their caches on the host */
+ if (ch->kv_valid[i] > pos_base) ch->kv_valid[i] = pos_base;
+ if (ch->kvi[i] >= 0) vkc_kv_lower(&ch->ks, ch->kvi[i], pos_base);
+ }
}
}
-/* The host's rings were replaced (a photo restored): the device's no longer match. */
+/* The host's rings were replaced (a photo restored): the devices' no longer match. */
static void mc_rings_rewritten(Model *m) {
- MimoChain *ch = (MimoChain *)m->vkchain;
- if (!ch || !ch->ok) return;
- for (int i = 0; i < ch->nl; i++) if (m->c.swa[i]) ch->kv_valid[i] = 0;
+ for (int d = 0; d < 2; d++) {
+ MimoChain *ch = mc_of(m, d);
+ if (!ch || !ch->ok) continue;
+ for (int i = ch->lo; i < ch->lo + ch->nl; i++) if (m->c.swa[i]) ch->kv_valid[i] = 0;
+ }
}
/* host rows [t0, t1) of a ring into the device's ring (at most two runs of slots) */
static int mc_ring_up(VkcBuf *dst, const float *src, int R, int t0, int t1, int w) {
@@ -325,7 +350,7 @@ static int mc_ring_copy(VkcBuf *ring, VkcBuf *lin, int R, int base, int t0, int
* pos_base reads them. */
static int mc_push_state(MimoChain *ch, Model *m, int pos_base, int n_rows) {
Cfg *c = &m->c; int ok = 1;
- for (int i = 0; i < ch->nl && ok; i++) {
+ for (int i = ch->lo; i < ch->lo + ch->nl && ok; i++) {
McGeo g = mc_geo(m, i);
Layer *l = &m->L[i];
if (ch->kv_valid[i] > pos_base) ch->kv_valid[i] = pos_base;
@@ -445,7 +470,7 @@ static int mc_dense_mlp(MimoChain *ch, Model *m, int i, int n) {
/* The rows the step at pb wrote, into the host's caches (the CPU's own placement). */
static void mc_commit(MimoChain *ch, Model *m, int n, int pb) {
const float *kv = (const float *)vkc_ptr(ch->kvd);
- for (int i = 0; i < ch->nl; i++) {
+ for (int i = ch->lo; i < ch->lo + ch->nl; i++) {
McGeo g = mc_geo(m, i);
Layer *l = &m->L[i];
int from = g.swa && n > g.rows ? n - g.rows : 0, cap = g.swa && g.rows < ch->rows ? g.rows : ch->rows;
@@ -461,112 +486,154 @@ static void mc_commit(MimoChain *ch, Model *m, int n, int pb) {
}
}
+/* One chain's layers (lo..lo+nl-1, on its device) for one chunk of n rows at pb, from
+ * the residual rows in hr: their K/V rows into the host's caches, and either the rows'
+ * logits (ro of them into lc: a chain with the head) or the residual after its last layer
+ * back in hr. 0: a frame failed (the device lost). */
+static int mc_chunk(MimoChain *ch, Model *m, float *hr, int n, int pb, int ro, float *lc) {
+ Cfg *c = &m->c; int H = c->hidden, lo = ch->lo, hi = lo + ch->nl, V = c->vocab;
+ if (!vkc_begin() || !vkc_write(ch->x, 0, hr, (size_t)n * H * sizeof(float)) || !mc_push_state(ch, m, pb, n)) return 0;
+ float *cs = (float *)vkc_ptr(ch->cs); /* the CPU's angles, cosines and sines (rope()) */
+ for (int k = 0; k < 2; k++) {
+ int rd = c->rope_dim[k], half = rd / 2;
+ for (int s = 0; s < n; s++) for (int j = 0; j < half; j++) {
+ float inv = 1.0f / powf(c->theta[k], (float)(2 * j) / (float)rd);
+ float ang = inv * (float)(pb + s);
+ cs[ch->rd_off[k] + (s * half + j) * 2] = cosf(ang);
+ cs[ch->rd_off[k] + (s * half + j) * 2 + 1] = sinf(ang);
+ }
+ }
+ int ok = 1, pending = 0;
+ for (int i = lo; i < hi && ok; i++) {
+ if (pending) ok = mc_add(ch->x, ch->routed, n * H); /* the routed experts join, as h += out */
+ pending = 0;
+ ok = ok && mc_norm(ch->x, 0, ch->prm, ch->o_ln1[i], ch->nrm, 0, n, H, c->eps) &&
+ mc_attention(ch, m, i, n, pb) && mc_add(ch->x, ch->tmp, n * H) &&
+ mc_norm(ch->x, 0, ch->prm, ch->o_ln2[i], ch->nrm, 0, n, H, c->eps);
+ if (ok && !c->moe[i]) ok = mc_dense_mlp(ch, m, i, n) && mc_add(ch->x, ch->tmp, n * H);
+ else if (ok) {
+ ok = vkc_copy(ch->h2d, 0, ch->nrm, 0, (size_t)n * H);
+ double t0 = now_s();
+ /* while the frame runs, the tier loads the experts this layer will likely
+ * stream (a big prompt chunk only) */
+ ok = ok && vkc_submit(0);
+ if (ok) vkt_stream_prefetch(i, n);
+ ok = ok && vkc_finish();
+ ch->frames++; ch->wait_ms += (now_s() - t0) * 1e3;
+ if (!ok) break;
+ double t1 = now_s();
+ moe(m, i, (const float *)vkc_ptr(ch->h2d), n, ch->host_out);
+ memcpy(vkc_ptr(ch->routed), ch->host_out, (size_t)n * H * sizeof(float));
+ ch->host_ms += (now_s() - t1) * 1e3;
+ ok = vkc_begin();
+ pending = 1;
+ }
+ }
+ if (ok && pending) ok = mc_add(ch->x, ch->routed, n * H);
+ if (ok && ro) ok = mc_norm(ch->x, (size_t)(n - ro) * H, ch->prm, ch->o_final, ch->fin, 0, ro, H, c->eps) &&
+ vkc_matmul((ColiVkTensor *)m->head.vk, ch->fin, 0, ch->outd, 0, ro);
+ if (ok && !ch->full) ok = vkc_copy(ch->h2d, 0, ch->x, 0, (size_t)n * H); /* the handoff to what follows */
+ double t0 = now_s();
+ ok = ok && vkc_submit(1);
+ ch->frames++; ch->wait_ms += (now_s() - t0) * 1e3;
+ if (!ok) return 0;
+ mc_commit(ch, m, n, pb);
+ if (ro) memcpy(lc, vkc_ptr(ch->outd), (size_t)ro * V * sizeof(float));
+ if (!ch->full) memcpy(hr, vkc_ptr(ch->h2d), (size_t)n * H * sizeof(float));
+ return 1;
+}
+
/* Every layer for the S rows h (the embedded rows, pictures spliced) at pos_base, the
* logits as forward() wants them (NULL: none; all_rows: [S][V], else the last row's).
- * Returns how many rows it computed: S, or fewer when the device was lost or the chain
+ * Returns how many rows it computed: S, or fewer when a device was lost or the chain
* declined -- the host's caches then hold every position before pos_base + that many,
- * and the CPU runs the rest. A partial chain (ch->full 0) runs its first ch->nl layers
- * here for each chunk, brings the chunk's residual rows down with the frame that ends it,
- * and runs the CPU's layers (layers_cpu, from nl) and the head (head_cpu) on them before
- * the next chunk, in xn and tmp (forward()'s rows): a chunk counts as computed once both
- * sides ran it, so a device lost in a later chunk loses nothing either side holds. */
+ * and the CPU runs the rest. Each chunk crosses the primary's layers, then (with one)
+ * the second device's from the rows the primary's brought down; a partial chain's last
+ * chain brings the chunk's residual rows down with the frame that ends it, and the CPU
+ * runs the layers after it (layers_cpu) and the head (head_cpu) on them before the next
+ * chunk, in xn and tmp (forward()'s rows): a chunk counts as computed once every side ran
+ * it, so a device lost in a later chunk loses nothing any side holds. */
static int mc_forward(Model *m, float *h, int S, int pos_base, float *logits, int all_rows, float *xn, float *tmp) {
if (!g_vk_chain) return 0;
if (g_vk_chain == COLI_VK_CHAIN_PREFILL && S <= 2) return 0; /* prompts only: decode on the CPU */
- MimoChain *ch = mc_setup(m);
+ MimoChain *ch = mc_setup(m), *ch2 = NULL;
if (!ch || ch->failed) return 0;
- Cfg *c = &m->c; int H = c->hidden, L = ch->nl, V = c->vocab, full = ch->full;
+ if (g_mc_fit2_on && !ch->full && ch->nl == g_mc_fit.n) {
+ vkc_device(1);
+ ch2 = mc_setup_dev(m, 1);
+ vkc_device(0);
+ if (ch2 && (ch2->failed || ch2->lo != ch->lo + ch->nl)) ch2 = NULL;
+ }
+ MimoChain *last_ch = ch2 ? ch2 : ch;
+ Cfg *c = &m->c; int H = c->hidden, V = c->vocab, full = last_ch->full, done_layers = last_ch->lo + last_ch->nl;
int CH = mc_chunk_rows(ch, m), rows = S < CH ? S : CH;
if (ch->ks.on && rows > ch->ks.chunk) rows = ch->ks.chunk; /* a step's rows fit the split's window */
+ if (ch2) {
+ vkc_device(1);
+ int CH2 = mc_chunk_rows(ch2, m);
+ vkc_device(0);
+ if (rows > CH2) rows = CH2;
+ if (ch2->ks.on && rows > ch2->ks.chunk) rows = ch2->ks.chunk;
+ }
int out = logits && all_rows ? rows : 1;
- if (!mc_scratch(ch, m, rows, out)) {
+ int sc = mc_scratch(ch, m, rows, ch->full ? out : 1);
+ if (sc && ch2) { vkc_device(1); sc = mc_scratch(ch2, m, rows, ch2->full ? out : 1); vkc_device(0); }
+ if (!sc) {
fprintf(stderr, "[VK] mimo chain: device memory for %d rows refused; per-matrix path\n", rows);
ch->failed = 1;
return 0;
}
vkc_gemm_rows(-1);
int done = 0;
+ MimoChain *at = ch;
for (int c0 = 0; c0 < S; c0 += rows) {
int n = S - c0 < rows ? S - c0 : rows, pb = pos_base + c0, last = c0 + n == S;
- if (!vkc_begin() || !vkc_write(ch->x, 0, h + (size_t)c0 * H, (size_t)n * H * sizeof(float)) ||
- !mc_push_state(ch, m, pb, n)) goto lost;
- float *cs = (float *)vkc_ptr(ch->cs); /* the CPU's angles, cosines and sines (rope()) */
- for (int k = 0; k < 2; k++) {
- int rd = c->rope_dim[k], half = rd / 2;
- for (int s = 0; s < n; s++) for (int j = 0; j < half; j++) {
- float inv = 1.0f / powf(c->theta[k], (float)(2 * j) / (float)rd);
- float ang = inv * (float)(pb + s);
- cs[ch->rd_off[k] + (s * half + j) * 2] = cosf(ang);
- cs[ch->rd_off[k] + (s * half + j) * 2 + 1] = sinf(ang);
- }
- }
- int ok = 1, pending = 0;
- for (int i = 0; i < L && ok; i++) {
- if (pending) ok = mc_add(ch->x, ch->routed, n * H); /* the routed experts join, as h += out */
- pending = 0;
- ok = ok && mc_norm(ch->x, 0, ch->prm, ch->o_ln1[i], ch->nrm, 0, n, H, c->eps) &&
- mc_attention(ch, m, i, n, pb) && mc_add(ch->x, ch->tmp, n * H) &&
- mc_norm(ch->x, 0, ch->prm, ch->o_ln2[i], ch->nrm, 0, n, H, c->eps);
- if (ok && !c->moe[i]) ok = mc_dense_mlp(ch, m, i, n) && mc_add(ch->x, ch->tmp, n * H);
- else if (ok) {
- ok = vkc_copy(ch->h2d, 0, ch->nrm, 0, (size_t)n * H);
- double t0 = now_s();
- /* while the frame runs, the tier loads the experts this layer will likely
- * stream (a big prompt chunk only) */
- ok = ok && vkc_submit(0);
- if (ok) vkt_stream_prefetch(i, n);
- ok = ok && vkc_finish();
- ch->frames++; ch->wait_ms += (now_s() - t0) * 1e3;
- if (!ok) break;
- double t1 = now_s();
- moe(m, i, (const float *)vkc_ptr(ch->h2d), n, ch->host_out);
- memcpy(vkc_ptr(ch->routed), ch->host_out, (size_t)n * H * sizeof(float));
- ch->host_ms += (now_s() - t1) * 1e3;
- ok = vkc_begin();
- pending = 1;
- }
- }
- if (ok && pending) ok = mc_add(ch->x, ch->routed, n * H);
int ro = !full ? 0 : logits && all_rows ? n : logits && last ? 1 : 0;
- if (ok && ro) ok = mc_norm(ch->x, (size_t)(n - ro) * H, ch->prm, ch->o_final, ch->fin, 0, ro, H, c->eps) &&
- vkc_matmul((ColiVkTensor *)m->head.vk, ch->fin, 0, ch->outd, 0, ro);
- if (ok && !full) ok = vkc_copy(ch->h2d, 0, ch->x, 0, (size_t)n * H); /* the handoff to the CPU's layers */
- double t0 = now_s();
- ok = ok && vkc_submit(1);
- ch->frames++; ch->wait_ms += (now_s() - t0) * 1e3;
- if (!ok) goto lost;
- mc_commit(ch, m, n, pb);
- if (ro) memcpy(all_rows ? logits + (size_t)c0 * V : logits, vkc_ptr(ch->outd), (size_t)ro * V * sizeof(float));
+ float *hr = h + (size_t)c0 * H, *lc = all_rows ? logits + (size_t)c0 * V : logits;
+ at = ch;
+ if (!mc_chunk(ch, m, hr, n, pb, ch2 ? 0 : ro, lc)) goto lost;
+ if (ch2) {
+ at = ch2;
+ vkc_device(1);
+ int ok = mc_chunk(ch2, m, hr, n, pb, ro, lc);
+ vkc_device(0);
+ if (!ok) goto lost;
+ }
if (!full) { /* the CPU's layers and the head on the chunk's rows */
- float *hr = h + (size_t)c0 * H;
- memcpy(hr, vkc_ptr(ch->h2d), (size_t)n * H * sizeof(float));
double t1 = now_s();
- layers_cpu(m, hr, n, pb, L, xn, tmp, NULL);
- float *lc = logits && all_rows ? logits + (size_t)c0 * V : logits && last ? logits : NULL;
- if (lc) head_cpu(m, hr, n, lc, all_rows, xn);
- ch->host_ms += (now_s() - t1) * 1e3;
+ layers_cpu(m, hr, n, pb, done_layers, xn, tmp, NULL);
+ float *lcc = logits && all_rows ? logits + (size_t)c0 * V : logits && last ? logits : NULL;
+ if (lcc) head_cpu(m, hr, n, lcc, all_rows, xn);
+ last_ch->host_ms += (now_s() - t1) * 1e3;
}
done = c0 + n;
}
ch->forwards++;
+ if (ch2) ch2->forwards++;
return S;
lost: /* a frame failed: the device is gone (or would not take a command); the CPU takes over */
- if (!vkc_lost()) { vkc_finish(); coli_vk_mark_lost(); }
+ if (!vkc_lost()) { vkc_finish(); coli_vk_mark_lost_dev(at->d); }
+ vkc_device(0);
g_vk_chain = 0;
ch->failed = 1;
+ if (ch2) ch2->failed = 1;
fprintf(stderr, "[VK] mimo chain: the device was lost at position %d; the host's caches hold every "
"position before it, and the CPU runs from there on\n", pos_base + done);
return done;
}
static void mc_report(Model *m, const char *what) {
- MimoChain *ch = (MimoChain *)m->vkchain;
- if (!ch || !ch->ok || !ch->forwards) return;
- VkcStats st; vkc_stats(&st);
- fprintf(stderr, "[VK] mimo chain: %llu forwards, %llu frames (%llu ops, %llu matmuls, %llu tiled GEMM), "
- "%.1f ms waiting for the device, %.1f ms of routed experts on the host, %.1f MiB on the device (%s)\n",
- ch->forwards, st.frames, st.ops, st.matmuls, st.gemms, ch->wait_ms, ch->host_ms,
- st.dev_bytes / 1048576.0, what);
- vkc_kv_report(&ch->ks);
- vkc_prof_print();
+ for (int d = 0; d < 2; d++) {
+ MimoChain *ch = mc_of(m, d);
+ if (!ch || !ch->ok || !ch->forwards) continue;
+ int was = vkc_device(d);
+ VkcStats st; vkc_stats(&st);
+ fprintf(stderr, "[VK] %s chain: %llu forwards, %llu frames (%llu ops, %llu matmuls, %llu tiled GEMM), "
+ "%.1f ms waiting for the device, %.1f ms of routed experts on the host, %.1f MiB on the device (%s)\n",
+ mc_name(ch), ch->forwards, st.frames, st.ops, st.matmuls, st.gemms, ch->wait_ms, ch->host_ms,
+ st.dev_bytes / 1048576.0, what);
+ vkc_kv_report(&ch->ks);
+ vkc_prof_print();
+ vkc_device(was);
+ }
}
diff --git a/c/tests/vulkan_layers_dev2.sh b/c/tests/vulkan_layers_dev2.sh
index a821d66c..496d63e2 100644
--- a/c/tests/vulkan_layers_dev2.sh
+++ b/c/tests/vulkan_layers_dev2.sh
@@ -144,14 +144,50 @@ ld2_olmoe() {
$PY tests/vulkan_chain_serve.py ./olmoe olmoe_tiny_c PILOT=1 WIDE=2 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=1 COLI_VK_CHAIN_LAYERS2=2
}
+# mimo: its own gate (mimo_chain_gate: the vendor fixture's cases, tokens and logits), with
+# the split forced; the second device's chain must have run in the logits run
+ld2_mimo_gate() { # <tag> <case> <n0> <n1> <env...>
+ local tag=$1 c=$2 n0=$3 n1=$4; shift 4
+ mimo_chain_gate "$tag" "$c" COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=$n0 COLI_VK_CHAIN_LAYERS2=$n1 COLI_VK_TIER_BALANCE=0 "$@"
+ local f; f=$(ld2_forwards mimo mimo-vk.err)
+ [ "$f" -gt 0 ] || { cat mimo-vk.err; fail "$tag $c: the second device's chain never ran"; }
+ echo " $tag $c: the second device's chain ran $f forwards"
+}
+ld2_mimo() {
+ $PY tools/make_mimo_tiny.py --output ./mimo_tiny --force --vision
+ MGRID=$($PY -c "import json;i=json.load(open('mimo_tiny/ref.json'))['image'];print(i['grid_h'],i['grid_w'])")
+ local c
+ for c in $($PY -c "import json;print(' '.join(json.load(open('mimo_tiny/ref.json'))['cases']))"); do
+ ld2_mimo_gate "ld2 mimo 3 + 3, the head on the second device" $c 3 3 MIMO_DENSE_BITS=32
+ ld2_mimo_gate "ld2 mimo 1 + 2, the CPU the rest and the head" $c 1 2 MIMO_DENSE_BITS=32
+ done
+ ld2_mimo_gate "ld2 mimo 2 + 4, blocks of 3" image 2 4 MIMO_DENSE_BITS=32 MIMO_CHUNK=3
+ ld2_mimo_gate "ld2 mimo 1 + 5, the release's FP8 and BF16" image 1 5 MIMO_DENSE_BITS=0
+ ld2_mimo_gate "ld2 mimo experts on both devices too" image 3 3 MIMO_DENSE_BITS=32 MIMO_VK_EXPERTS=2 COLI_VK_TIER_GB=0.00002
+ ld2_mimo_gate "ld2 mimo KV split on both" image 3 3 MIMO_DENSE_BITS=32 COLI_VK_KV_DEVICE_ROWS=8 COLI_VK_KV_BLOCK=2
+ CHAINMODE=2 ld2_mimo_gate "ld2 mimo prompts only" image 3 3 MIMO_DENSE_BITS=32
+ # the second device lost at its third frame: the host's caches hold whole steps, the CPU
+ # runs the rest from where they end
+ local x=(--image mimo_tiny/patches.f32 --grid $MGRID) ids
+ ids=$(mimo_ids image prompt_ids)
+ COLI_TEMP=0 MIMO_DENSE_BITS=32 ./mimo mimo_tiny --ids "$ids" --ngen 6 "${x[@]}" > mimo-cpu.txt 2>/dev/null
+ COLI_TEMP=0 MIMO_DENSE_BITS=32 COLI_VULKAN=1 COLI_VK_CHAIN=1 COLI_VK_TIER_SYNC=1 COLI_VK_DEV2=0 COLI_VK_CHAIN_LAYERS=3 \
+ COLI_VK_CHAIN_LAYERS2=3 COLI_VK_CHAIN_FAULT2=3 ./mimo mimo_tiny --ids "$ids" --ngen 6 "${x[@]}" > mimo-vk.txt 2> mimo-vk.err
+ cmp -s mimo-cpu.txt mimo-vk.txt || { cat mimo-cpu.txt mimo-vk.txt mimo-vk.err; fail "ld2 mimo second device lost: tokens differ from the CPU"; }
+ grep -q "COLI_VK_CHAIN_FAULT2" mimo-vk.err && grep -q "the device was lost at position" mimo-vk.err ||
+ { cat mimo-vk.err; fail "ld2 mimo second device lost: the loss was not taken over"; }
+ echo "OK ld2 mimo second device lost: tokens = CPU, $(grep -o 'the device was lost at position [0-9]*' mimo-vk.err)"
+}
+
family_layers_dev2() {
export OMP_NUM_THREADS=2
- make qwen36 qwen38 olmoe tests/test_vk_chain VK=1
+ make qwen36 qwen38 olmoe mimo tests/test_vk_chain VK=1
COLI_VK_DEV2=0 ./tests/test_vk_chain shaders/qmatmul.spv | tee vk_chain.log
tail -1 vk_chain.log | grep -qx PASS || fail "the chain's ops on two devices"
ld2_qwen36
ld2_qwen38
ld2_olmoe
+ ld2_mimo
unset OMP_NUM_THREADS
}
family_layers_dev2_sanitize() {