Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
mimo: the dense chain on the device (COLI_VK_CHAIN=1)
JustVugg3fb90af087e1
diff --git a/c/mimo.c b/c/mimo.cindex da3b5302..9ce26fb3 100644--- a/c/mimo.c+++ b/c/mimo.c@@ -544,6 +544,9 @@ typedef struct {double t_disk, t_expert, t_attn;int idot, direct;uint8_t **ehit; /* [layer][expert] routed this turn, for HITS */+#ifdef COLI_VULKAN+ void *vkchain; /* the dense chain's device state (mimo_chain.h), NULL until it runs */+#endif} Model;static void expert_table_init(Model *m) {@@ -1109,6 +1112,10 @@ static void embed_rows(Model *m, const int *ids, int n, float *h) {* image-pad position of the WHOLE prompt (the first at `first_pad`). */typedef struct { const float *rows; int n_rows; } ImageRows;+#ifdef COLI_VULKAN+#include "mimo_chain.h" /* COLI_VK_CHAIN: every layer's dense chain on the device */+#endif+/* One block of n tokens at m->pos. logits: NULL, or [n, V] (all rows) when* all_rows, else [V] for the last row. */static void forward(Model *m, const int *ids, int n, float *logits, int all_rows,@@ -1133,24 +1140,35 @@ static void forward(Model *m, const int *ids, int n, float *logits, int all_rowsstatic int traced;if (!traced && getenv("MIMO_TRACE")) { trace = fopen(getenv("MIMO_TRACE"), "wb"); }if (trace) fwrite(h, sizeof(float), (size_t)n * H, trace);- for (int li = 0; li < c->n_layers; li++) {+ /* The rows the CPU computes: all of them, or with the dense chain on (COLI_VK_CHAIN,+ * mimo_chain.h) the ones it did not (a lost device), from where the host's caches end. */+ int done = 0;+#ifdef COLI_VULKAN+ if (g_vk_chain && !trace) {+ done = mc_forward(m, h, n, m->pos, logits, all_rows);+ if (done < n) mc_cpu_step(m, m->pos + done);+ }+#endif+ float *hc = h + (size_t)done * H, *lc = logits && all_rows ? logits + (size_t)done * c->vocab : logits;+ int nc = n - done, pc = m->pos + done;+ for (int li = 0; li < c->n_layers && nc > 0; li++) {Layer *l = &m->L[li];- for (int t = 0; t < n; t++) rmsnorm(xn + (size_t)t * H, h + (size_t)t * H, l->ln1, H, c->eps);- attention(m, li, xn, n, m->pos, tmp);- for (size_t i = 0; i < (size_t)n * H; i++) h[i] += tmp[i];- if (trace) fwrite(h, sizeof(float), (size_t)n * H, trace);- for (int t = 0; t < n; t++) rmsnorm(xn + (size_t)t * H, h + (size_t)t * H, l->ln2, H, c->eps);- if (c->moe[li]) moe(m, li, xn, n, tmp);- else dense_mlp(m, l, xn, n, tmp);- for (size_t i = 0; i < (size_t)n * H; i++) h[i] += tmp[i];- if (trace) fwrite(h, sizeof(float), (size_t)n * H, trace);+ for (int t = 0; t < nc; t++) rmsnorm(xn + (size_t)t * H, hc + (size_t)t * H, l->ln1, H, c->eps);+ attention(m, li, xn, nc, pc, tmp);+ for (size_t i = 0; i < (size_t)nc * H; i++) hc[i] += tmp[i];+ if (trace) fwrite(hc, sizeof(float), (size_t)nc * H, trace);+ for (int t = 0; t < nc; t++) rmsnorm(xn + (size_t)t * H, hc + (size_t)t * H, l->ln2, H, c->eps);+ if (c->moe[li]) moe(m, li, xn, nc, tmp);+ else dense_mlp(m, l, xn, nc, tmp);+ for (size_t i = 0; i < (size_t)nc * H; i++) hc[i] += tmp[i];+ if (trace) fwrite(hc, sizeof(float), (size_t)nc * H, trace);}if (trace) { fclose(trace); trace = NULL; }traced = 1;- if (logits) {- int from = all_rows ? 0 : n - 1, rows = n - from;- for (int t = from; t < n; t++) rmsnorm(xn + (size_t)(t - from) * H, h + (size_t)t * H, m->norm, H, c->eps);- dw_matmul(logits, xn, rows, &m->head);+ if (lc && nc > 0) {+ int from = all_rows ? 0 : nc - 1, rows = nc - from;+ for (int t = from; t < nc; t++) rmsnorm(xn + (size_t)(t - from) * H, hc + (size_t)t * H, m->norm, H, c->eps);+ dw_matmul(lc, xn, rows, &m->head);}kv_prefix_record(&m->kvp, ids, m->pos, n);m->pos += n;@@ -1234,8 +1252,9 @@ static Vision *g_vision;#ifdef COLI_VULKAN/* The dense matrices go up first, at start-up, the trunk before the tower, so* the experts take only what they leave; one that does not fit stays on the- * CPU. Returns how many went up. */-static int vk_dense_upload(Model *m) {+ * CPU. Returns how many went up. The dense chain alone (the per-matrix path off)+ * takes the trunk and leaves the tower on the CPU (vision 0). */+static int vk_dense_upload(Model *m, int vision) {int n = 0;for (int li = 0; li < m->c.n_layers; li++) {Layer *l = &m->L[li];@@ -1243,7 +1262,7 @@ static int vk_dense_upload(Model *m) {if (!m->c.moe[li]) n += dw_upload(&l->gate) + dw_upload(&l->up) + dw_upload(&l->down);}n += dw_upload(&m->head);- if (g_vision) {+ if (g_vision && vision) {Vision *v = g_vision;n += dw_upload(&v->embed) + dw_upload(&v->fc1) + dw_upload(&v->fc2);for (int i = 0; i < v->depth; i++)@@ -1265,14 +1284,14 @@ static size_t dw_bytes(const DW *d) {}return 0;}-static size_t vk_dense_bytes(const Model *m) {+static size_t vk_dense_bytes(const Model *m, int vision) {size_t b = dw_bytes(&m->head);for (int li = 0; li < m->c.n_layers; li++) {const Layer *l = &m->L[li];b += dw_bytes(&l->qkv) + dw_bytes(&l->o);if (!m->c.moe[li]) b += dw_bytes(&l->gate) + dw_bytes(&l->up) + dw_bytes(&l->down);}- if (g_vision) {+ if (g_vision && vision) {const Vision *v = g_vision;b += dw_bytes(&v->embed) + dw_bytes(&v->fc1) + dw_bytes(&v->fc2);for (int i = 0; i < v->depth; i++)@@ -1314,7 +1333,7 @@ static void vk_tier_start(Model *m) {.gate_up = f, .down = f, .act = VKT_ACT_SWIGLU,.max_rows = MIMO_VK_ROWS * c->topk,.ram_reserve = (size_t)(m->e_bytes + 8192) * (size_t)cap * (size_t)nmoe,- .dense_bytes = coli_vk_dense() ? vk_dense_bytes(m) : 0,+ .dense_bytes = coli_vk_dense() ? vk_dense_bytes(m, 1) : g_vk_chain ? vk_dense_bytes(m, 0) : 0,.in_ram = vk_in_ram, .ram_ctx = m};atexit(coli_vk_shutdown); /* before vkt_init, which makes the expert batch's pipelines and can still refuse (no room): the device goes at exit either way, after the tier's teardown */if (vkt_init(&vc, NULL)) atexit(vkt_shutdown);@@ -1442,6 +1461,9 @@ static int pin_state_load(Model *m, const PinState *st) {memcpy(l->V, st->V[li], st->vd[li] * sizeof(float));memcpy(l->ring_pos, st->ring_pos[li], (size_t)l->rows * sizeof(int));}+#ifdef COLI_VULKAN+ mc_rings_rewritten(m); /* the dense chain's copies of the rings go up again */+#endifreturn 1;}@@ -1738,6 +1760,9 @@ static void serve_loop(Model *m, Tok *tokenizer, const char *dir) {serve_hits(m);serve_emap(m);vkt_report("turn", m->hits, m->miss); /* the Vulkan tier's line, when it runs */+#ifdef COLI_VULKAN+ mc_report(m, "turn");+#endifcoli_serve_command_dispose(&command);}free(ids); free(logits); free(pending);@@ -1752,6 +1777,7 @@ static void vk_report(const Model *m) {if (!g_vk_ready) return;fprintf(stderr, "[VK] mimo: %llu matmuls on the GPU\n", coli_vk_matmul_calls());vkt_report("run", m->hits, m->miss);+ mc_report((Model *)m, "run");#else(void)m;#endif@@ -1818,11 +1844,31 @@ int main(int argc, char **argv) {const char *mx = getenv("MIMO_VK_EXPERTS");int tier = vkt_wanted() && any_moe && !(mx && *mx && atoi(mx) <= 0); /* MIMO_VK_EXPERTS=0: experts on the CPU */g_vk_ready = coli_vk_init_env_tier("mimo", tier);+ /* COLI_VK_CHAIN: every layer's dense chain on the device (mimo_chain.h), decided+ * before the tier so the tier's budget leaves the trunk room. Not measured on a+ * MiMo checkpoint: an integrated GPU keeps it opt-in (docs/vulkan.md). */+ if (g_vk_ready) {+ g_vk_chain = coli_vk_chain_decide(NULL, tier, COLI_VK_CHAIN_OFF);+ const char *e = getenv("COLI_VK_CHAIN");+ char why[160];+ if (e && *e) snprintf(why, sizeof why, "COLI_VK_CHAIN=%s", e);+ else if (coli_vk_device_integrated())+ snprintf(why, sizeof why, "an integrated GPU: not measured on a MiMo checkpoint; COLI_VK_CHAIN=1 on, 2 prompts only");+ else if (coli_vk_device_shares_ram()) snprintf(why, sizeof why, "a CPU device; COLI_VK_CHAIN=1 turns it on");+ else snprintf(why, sizeof why, "a discrete GPU%s", tier ? ", beside the expert tier" : "");+ fprintf(stderr, "[VK] mimo: dense chain %s (%s)\n", g_vk_chain == COLI_VK_CHAIN_ON ? "on"+ : g_vk_chain == COLI_VK_CHAIN_PREFILL ? "on for prompts" : "off", why);+ if (g_vk_chain && !vkc_init()) g_vk_chain = 0;+ }if (g_vk_ready && tier) vk_tier_start(m);if (g_vk_ready && !vkt_ready() && !coli_vk_dense()) coli_vk_dense_decide("mimo", 0, 1); /* no tier after all */+ if (g_vk_chain) { /* the chain's teardown before the device's (the tier registered the device's) */+ if (!tier) atexit(coli_vk_shutdown);+ atexit(vkc_shutdown);+ }}- if (g_vk_ready && coli_vk_dense()) { /* COLI_VK_DENSE=0: the trunk stays on the CPU */- int up = vk_dense_upload(m);+ if (g_vk_ready && (coli_vk_dense() || g_vk_chain)) { /* COLI_VK_DENSE=0: the trunk stays on the CPU */+ int up = vk_dense_upload(m, coli_vk_dense());size_t used = 0, count = 0;coli_vk_mem_info(&used, &count);fprintf(stderr, "[VK] mimo: %d dense matrices on the GPU (%.2f GB)\n", up, used / 1e9);diff --git a/c/mimo_chain.h b/c/mimo_chain.hnew file mode 100644index 00000000..3ec28a89--- /dev/null+++ b/c/mimo_chain.h@@ -0,0 +1,445 @@+/* mimo_chain.h -- MiMo-V2.6's layers as a dense chain on the Vulkan device (vk_chain.h).+ * Included once by mimo.c in a COLI_VULKAN build, after the CPU pieces it reuses (moe()+ * for the routed experts) and before forward(), which calls it; COLI_VK_CHAIN decides+ * (coli_vk_chain_decide, in main).+ *+ * What runs where, per layer, for one block of rows (decode: one row):+ * device: the routed MoE output of the layer before joins the residual (x += out,+ * the CPU's add), the input RMSNorm, the fused qkv projection, partial RoPE on q and+ * k (rotate-half on the first rope_dim dims, the CPU's own cosf/sinf in a host+ * table, one table per layer kind: the two kinds have their own theta), the value+ * scale (chain_ew SCALE), the new K/V rows into the device's cache, attention with+ * the layer's sliding window and sink logit (chain_attn), o_proj, the residual add,+ * the post-attention RMSNorm; on the dense layer (layer 0 on the release) the MLP+ * too (gate, up, SwiGLU, down, add), with no host step. A MoE layer's frame ends+ * with the normalized rows going down and is waited for.+ * host: moe() as the CPU runs it, on the device's normalized rows: the router+ * (sigmoid scores, the correction bias picks, top-k renormalized), the routed+ * experts (the Vulkan expert tier's batch and the CPU's share, joined in each row's+ * routing order). MiMo has no shared expert, so nothing runs beside it.+ * last frame: the final norm and lm_head on the last row (every row for a read-out).+ * Crossing per MoE layer: D floats a row down, D floats a row up; per step the new K/V+ * rows of every layer down (a few KB a row) and the logits.+ *+ * The KV caches, and who owns them. The host's stay canonical, in the layout the CPU+ * keeps (mimo.c's kv_alloc): a full-attention layer [ctx][kvh][hd] (V [ctx][kvh][vd]),+ * a sliding-window layer a RING of rows = min(window, ctx) rows, position p in row+ * p % rows. The device mirrors each one in exactly that layout (chain_attn reads+ * position-major rows, and a ring), behind a watermark kv_valid per layer: positions+ * [0, kv_valid) -- for a ring, the last `rows` of them -- equal the host's. A step at+ * pos_base lowers the watermark to pos_base (a reset, a rollback), then uploads the+ * host's rows between it and pos_base (for a ring only those the window still sees); a+ * CPU step lowers it to its pos_base; a photo restored into the host's rings+ * (pin_state_load) drops the rings' watermarks to 0, so the next step uploads them.+ *+ * A windowed layer's attention: one decode row reads the ring in place (its window is+ * exactly the ring's rows); a block of rows cannot (its later rows overwrite slots its+ * earlier rows still see), so it gathers the window's earlier rows from the ring and its+ * own new rows into one position-major scratch, attends there, and then writes its last+ * `rows` rows into the ring, as the CPU's attention() does with its window copy. The two+ * visit the positions in the same order, so a row gets the same bits either way.+ *+ * The new rows reach the host's caches only when the step's last frame has completed,+ * so the host's state always describes whole steps: a device lost in the middle of a+ * step loses nothing the host holds, and the CPU runs the step's remaining rows from+ * the position the host's caches end at (no rebuild, a turn with a picture included).+ * COLI_VK_CHAIN_FAULT=n (vk_chain.c) fakes the loss at the n-th frame, for tests.+ *+ * The chain declines (the per-matrix path or the CPU runs) when a dense matrix did not+ * reach the device, for geometry outside chain_attn (a head dim above 256) and under+ * MIMO_TRACE (the CPU's per-layer dump). */+#include "vk_chain.h"++typedef struct {+ int ok, failed;+ int rows; /* scratch capacity in rows */+ int rd_off[2]; /* the RoPE table of each layer kind in cs */+ VkcBuf *prm; /* the norms and the sink logits */+ size_t *o_ln1, *o_ln2, *o_sink, o_final;+ VkcBuf **kc, **vc; /* the device's caches, per layer */+ int *kv_valid;+ size_t *o_kvd; /* a layer's new K rows in kvd (V after cap rows) */+ VkcBuf *x, *nrm, *tmp, *qkv, *kn, *vn, *wk, *wv, *ctx, *g, *u, *fin;+ VkcBuf *h2d, *kvd, *outd, *routed, *cs;+ float *host_out;+ unsigned long long forwards, frames;+ double wait_ms, host_ms;+} MimoChain;++static int g_vk_chain = 0; /* COLI_VK_CHAIN decided on, and the chain's pipelines are up */++static int mc_chunk_rows(void) {+ const char *e = getenv("COLI_VK_CHAIN_ROWS");+ int v = e && *e ? atoi(e) : 512;+ return v < 1 ? 1 : v > 65535 ? 65535 : v;+}++/* the geometry of layer li's attention */+typedef struct { int k, nh, kvh, hd, vd, qd, kd, vdd, rw, half, rows, swa; } McGeo;+static McGeo mc_geo(const Model *m, int li) {+ const Cfg *c = &m->c;+ McGeo g;+ g.k = c->swa[li]; g.swa = g.k;+ g.nh = c->heads[g.k]; g.kvh = c->kv_heads[g.k]; g.hd = c->head_dim[g.k]; g.vd = c->v_dim[g.k];+ g.qd = g.nh * g.hd; g.kd = g.kvh * g.hd; g.vdd = g.kvh * g.vd; g.rw = g.qd + g.kd + g.vdd;+ g.half = c->rope_dim[g.k] / 2;+ g.rows = m->L[li].rows;+ return g;+}++static int mc_ctensor(DW *d) { return dw_upload(d) && d->vk; }++/* The model's parameters on the device, its tensors resolved, its caches allocated;+ * NULL = the chain cannot run. */+static MimoChain *mc_setup(Model *m) {+ MimoChain *ch = (MimoChain *)m->vkchain;+ if (ch) return ch->ok ? ch : NULL;+ ch = (MimoChain *)calloc(1, sizeof *ch);+ if (!ch) return NULL;+ m->vkchain = ch;+ Cfg *c = &m->c; int L = c->n_layers, H = c->hidden;+ for (int k = 0; k < 2; k++)+ if (c->head_dim[k] > 256 || c->v_dim[k] > 256) {+ fprintf(stderr, "[VK] mimo chain: head dims %d/%d are past its attention shader's 256; per-matrix path\n",+ c->head_dim[k], c->v_dim[k]);+ return NULL;+ }+ ch->o_ln1 = calloc(L, sizeof(size_t)); ch->o_ln2 = calloc(L, sizeof(size_t)); ch->o_sink = calloc(L, sizeof(size_t));+ ch->o_kvd = calloc(L, sizeof(size_t)); ch->kv_valid = calloc(L, sizeof(int));+ ch->kc = calloc(L, sizeof(void *)); ch->vc = calloc(L, sizeof(void *));+ if (!ch->o_ln1 || !ch->o_ln2 || !ch->o_sink || !ch->o_kvd || !ch->kv_valid || !ch->kc || !ch->vc) return NULL;+ size_t n = 0;+ for (int i = 0; i < L; i++) {+ ch->o_ln1[i] = n; n += H; ch->o_ln2[i] = n; n += H;+ if (m->L[i].sink) { ch->o_sink[i] = n; n += c->heads[c->swa[i]]; }+ }+ ch->o_final = n; n += H;+ float *arena = calloc(n, sizeof(float));+ if (!arena) return NULL;+ for (int i = 0; i < L; i++) {+ Layer *l = &m->L[i];+ memcpy(arena + ch->o_ln1[i], l->ln1, H * sizeof(float));+ memcpy(arena + ch->o_ln2[i], l->ln2, H * sizeof(float));+ if (l->sink) memcpy(arena + ch->o_sink[i], l->sink, c->heads[c->swa[i]] * sizeof(float));+ }+ memcpy(arena + ch->o_final, m->norm, H * sizeof(float));+ ch->prm = vkc_buf(n * sizeof(float), VKC_DEV);+ int ok = ch->prm && vkc_begin() && vkc_write(ch->prm, 0, arena, n * sizeof(float)) && vkc_submit(1);+ free(arena);+ if (!ok) return NULL;+ /* the matrices: the device copies the per-matrix path uploads (vk_dense_upload) */+ for (int i = 0; i < L && ok; i++) {+ Layer *l = &m->L[i];+ ok = mc_ctensor(&l->qkv) && mc_ctensor(&l->o) &&+ (c->moe[i] || (mc_ctensor(&l->gate) && mc_ctensor(&l->up) && mc_ctensor(&l->down)));+ }+ ok = ok && mc_ctensor(&m->head);+ if (!ok) { fprintf(stderr, "[VK] mimo chain: a dense matrix did not reach the device; per-matrix path\n"); return NULL; }+ /* the caches, in the host's layout and at its size */+ size_t kv = 0;+ for (int i = 0; i < L; i++) {+ McGeo g = mc_geo(m, i);+ ch->kc[i] = vkc_buf((size_t)g.rows * g.kd * sizeof(float), VKC_DEV);+ ch->vc[i] = vkc_buf((size_t)g.rows * g.vdd * sizeof(float), VKC_DEV);+ if (!ch->kc[i] || !ch->vc[i]) {+ fprintf(stderr, "[VK] mimo chain: device memory for the KV caches refused; per-matrix path\n");+ return NULL;+ }+ kv += (size_t)g.rows * (g.kd + g.vdd);+ }+ ch->ok = 1;+ int swa = 0;+ for (int i = 0; i < L; i++) swa += c->swa[i];+ fprintf(stderr, "[VK] mimo chain: %d layers on the device (%d sliding window), %.1f MiB of KV mirrors\n",+ L, swa, kv * 4 / 1048576.0);+ return ch;+}++static int mc_res(VkcBuf **b, size_t floats, int kind) { return vkc_reserve(b, (floats ? floats : 1) * sizeof(float), kind); }++/* scratch for `rows` rows, `out` rows of logits */+static int mc_scratch(MimoChain *ch, Model *m, int rows, int out) {+ Cfg *c = &m->c; int H = c->hidden, L = c->n_layers;+ size_t r = (size_t)rows, rw = 0, kd = 0, vdd = 0, ctxw = 0, wkd = 0, wvd = 0, kvd = 0;+ int dense = 0, win = 0;+ for (int i = 0; i < L; i++) {+ McGeo g = mc_geo(m, i);+ if ((size_t)g.rw > rw) rw = g.rw;+ if ((size_t)g.kd > kd) kd = g.kd;+ if ((size_t)g.vdd > vdd) vdd = g.vdd;+ if ((size_t)g.nh * g.vd > ctxw) ctxw = (size_t)g.nh * g.vd;+ if (g.swa) {+ if ((size_t)g.kd > wkd) wkd = g.kd;+ if ((size_t)g.vdd > wvd) wvd = g.vdd;+ if (g.rows - 1 > win) win = g.rows - 1;+ }+ int cap = g.swa && g.rows < rows ? g.rows : rows;+ ch->o_kvd[i] = kvd; kvd += (size_t)cap * (g.kd + g.vdd);+ dense |= !c->moe[i];+ }+ ch->rd_off[0] = 0; ch->rd_off[1] = rows * 2 * (c->rope_dim[0] / 2);+ size_t cs = r * 2 * (size_t)(c->rope_dim[0] / 2 + c->rope_dim[1] / 2);+ int ok = mc_res(&ch->x, r * H, VKC_DEV) && mc_res(&ch->nrm, r * H, VKC_DEV) && mc_res(&ch->tmp, r * H, VKC_DEV) &&+ mc_res(&ch->qkv, r * rw, VKC_DEV) && mc_res(&ch->kn, r * kd, VKC_DEV) && mc_res(&ch->vn, r * vdd, VKC_DEV) &&+ mc_res(&ch->ctx, r * ctxw, VKC_DEV) && mc_res(&ch->fin, (size_t)out * H, VKC_DEV) &&+ mc_res(&ch->h2d, r * H, VKC_DOWN) && mc_res(&ch->kvd, kvd, VKC_DOWN) &&+ mc_res(&ch->outd, (size_t)out * c->vocab, VKC_DOWN) && mc_res(&ch->routed, r * H, VKC_UP) &&+ mc_res(&ch->cs, cs, VKC_UP);+ if (ok && wkd) ok = mc_res(&ch->wk, ((size_t)win + r) * wkd, VKC_DEV) && mc_res(&ch->wv, ((size_t)win + r) * wvd, VKC_DEV);+ if (ok && dense) ok = mc_res(&ch->g, r * c->dense_inter, VKC_DEV) && mc_res(&ch->u, r * c->dense_inter, VKC_DEV);+ if (!ok) return 0;+ if (ch->rows < rows) {+ float *ho = realloc(ch->host_out, r * H * sizeof(float));+ if (!ho) return 0;+ ch->host_out = ho;+ }+ ch->rows = rows; /* o_kvd and rd_off are laid out for exactly this many */+ return 1;+}++/* ---- the KV caches between the host and the device ------------------------------- */+/* A CPU step from pos_base writes the host's rows from there on. */+static void mc_cpu_step(Model *m, int pos_base) {+ MimoChain *ch = (MimoChain *)m->vkchain;+ if (!ch || !ch->ok) return;+ for (int i = 0; i < m->c.n_layers; i++) if (ch->kv_valid[i] > pos_base) ch->kv_valid[i] = pos_base;+}+/* The host's rings were replaced (a photo restored): the device's no longer match. */+static void mc_rings_rewritten(Model *m) {+ MimoChain *ch = (MimoChain *)m->vkchain;+ if (!ch || !ch->ok) return;+ for (int i = 0; i < m->c.n_layers; i++) if (m->c.swa[i]) ch->kv_valid[i] = 0;+}+/* host rows [t0, t1) of a ring into the device's ring (at most two runs of slots) */+static int mc_ring_up(VkcBuf *dst, const float *src, int R, int t0, int t1, int w) {+ int ok = 1;+ while (ok && t0 < t1) {+ int slot = t0 % R, run = R - slot < t1 - t0 ? R - slot : t1 - t0;+ ok = vkc_write(dst, (size_t)slot * w, src + (size_t)slot * w, (size_t)run * w * sizeof(float));+ t0 += run;+ }+ return ok;+}+/* device rows of positions [t0, t1) between a ring (rows R) and a linear buffer whose+ * row 0 is position `base`; to_ring 1: linear -> ring, 0: ring -> linear */+static int mc_ring_copy(VkcBuf *ring, VkcBuf *lin, int R, int base, int t0, int t1, int w, int to_ring) {+ int ok = 1;+ while (ok && t0 < t1) {+ int slot = t0 % R, run = R - slot < t1 - t0 ? R - slot : t1 - t0;+ size_t ro = (size_t)slot * w, lo = (size_t)(t0 - base) * w, nf = (size_t)run * w;+ ok = to_ring ? vkc_copy(ring, ro, lin, lo, nf) : vkc_copy(lin, lo, ring, ro, nf);+ t0 += run;+ }+ return ok;+}+/* Record the uploads that make the device's caches the host's, as the step at pos_base+ * reads them. */+static int mc_push_state(MimoChain *ch, Model *m, int pos_base) {+ Cfg *c = &m->c; int ok = 1;+ for (int i = 0; i < c->n_layers && ok; i++) {+ McGeo g = mc_geo(m, i);+ Layer *l = &m->L[i];+ if (ch->kv_valid[i] > pos_base) ch->kv_valid[i] = pos_base;+ if (g.swa) {+ /* the positions the window still sees must be in the host's ring, as the+ * CPU's attention() demands */+ int lo = pos_base - (c->window - 1) > 0 ? pos_base - (c->window - 1) : 0;+ for (int p = lo; p < pos_base; p++)+ if (l->ring_pos[p % g.rows] != p) { fprintf(stderr, "[mimo] window ring lost position %d\n", p); exit(1); }+ int from = ch->kv_valid[i] > pos_base - g.rows ? ch->kv_valid[i] : pos_base - g.rows;+ if (from < 0) from = 0;+ ok = mc_ring_up(ch->kc[i], l->K, g.rows, from, pos_base, g.kd) &&+ mc_ring_up(ch->vc[i], l->V, g.rows, from, pos_base, g.vdd);+ } else if (ch->kv_valid[i] < pos_base) {+ int t0 = ch->kv_valid[i], n = pos_base - t0;+ ok = vkc_write(ch->kc[i], (size_t)t0 * g.kd, l->K + (size_t)t0 * g.kd, (size_t)n * g.kd * sizeof(float)) &&+ vkc_write(ch->vc[i], (size_t)t0 * g.vdd, l->V + (size_t)t0 * g.vdd, (size_t)n * g.vdd * sizeof(float));+ }+ ch->kv_valid[i] = pos_base;+ }+ return ok;+}++/* ---- one layer's pieces ------------------------------------------------------------- */+static int mc_norm(VkcBuf *x, size_t xo, VkcBuf *w, size_t wo, VkcBuf *y, size_t yo, int rows, int D, float eps) {+ VkcNorm p = {rows, D, 1, (int)xo, D, D, (int)yo, D, D, (int)wo, 0, 0, eps, 1.f};+ return vkc_norm(x, w, y, &p);+}+static int mc_add(VkcBuf *x, VkcBuf *b, int n) {+ VkcEw p = {VKC_EW_ADD, n, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1.f};+ return vkc_ew(x, x, b, NULL, NULL, &p);+}+/* Attention of layer i for n rows at pb, o_proj into tmp; the new rows into the+ * device's cache and into kvd for the host. */+static int mc_attention(MimoChain *ch, Model *m, int i, int n, int pb) {+ Cfg *c = &m->c; Layer *l = &m->L[i];+ McGeo g = mc_geo(m, i);+ int ok = vkc_matmul((ColiVkTensor *)l->qkv.vk, ch->nrm, 0, ch->qkv, 0, n);+ if (ok && g.half) { /* q's heads and k's heads are contiguous in a qkv row: one rotation */+ VkcRope p = {n * (g.nh + g.kvh), g.nh + g.kvh, 0, g.rw, g.hd, g.half, ch->rd_off[g.k], 2 * g.half};+ ok = vkc_rope(ch->qkv, ch->cs, &p);+ }+ VkcRegion *rg = ok ? malloc(sizeof *rg * (size_t)n) : NULL;+ if (!rg) return 0;+ for (int s = 0; s < n; s++) rg[s] = (VkcRegion){(size_t)s * g.kd, (size_t)s * g.rw + g.qd, (size_t)g.kd};+ ok = vkc_copy_regions(ch->kn, ch->qkv, rg, n);+ for (int s = 0; s < n; s++) rg[s] = (VkcRegion){(size_t)s * g.vdd, (size_t)s * g.rw + g.qd + g.kd, (size_t)g.vdd};+ ok = ok && vkc_copy_regions(ch->vn, ch->qkv, rg, n);+ free(rg);+ if (ok && c->v_scale != 1.0f) { /* v *= attention_value_scale, as the CPU does before caching */+ VkcEw p = {VKC_EW_SCALE, n * g.vdd, 1, 1, 0, 1, 0, 0, 0, 0, 0, c->v_scale};+ ok = vkc_ew(ch->vn, ch->vn, NULL, NULL, NULL, &p);+ }+ if (!ok) return 0;+ VkcAttnW a;+ memset(&a, 0, sizeof a);+ a.a = (VkcAttn){n, g.nh, g.kvh, g.hd, pb, g.rows, 0, g.rw, g.hd, 0, 0, 0, 0, 0, g.nh * g.vd, 0, 0,+ 1.0f / sqrtf((float)g.hd), 0, 0};+ a.vd = g.vd; a.kv_pm = 1;+ a.sink = l->sink != NULL; a.sink_off = (int)ch->o_sink[i];+ VkcBuf *kc = ch->kc[i], *vc = ch->vc[i];+ if (!g.swa) { /* full attention: the rows in at pb, then every position */+ ok = vkc_copy(kc, (size_t)pb * g.kd, ch->kn, 0, (size_t)n * g.kd) &&+ vkc_copy(vc, (size_t)pb * g.vdd, ch->vn, 0, (size_t)n * g.vdd);+ } else if (n == 1) { /* one row: its window is the ring, read in place */+ int slot = pb % g.rows;+ ok = vkc_copy(kc, (size_t)slot * g.kd, ch->kn, 0, g.kd) && vkc_copy(vc, (size_t)slot * g.vdd, ch->vn, 0, g.vdd);+ a.win = c->window; a.ring = g.rows;+ } else { /* a block: the window's earlier rows and the block's own, linear */+ int lo = pb - (c->window - 1) > 0 ? pb - (c->window - 1) : 0, nb = pb - lo;+ ok = mc_ring_copy(kc, ch->wk, g.rows, lo, lo, pb, g.kd, 0) && mc_ring_copy(vc, ch->wv, g.rows, lo, lo, pb, g.vdd, 0) &&+ vkc_copy(ch->wk, (size_t)nb * g.kd, ch->kn, 0, (size_t)n * g.kd) &&+ vkc_copy(ch->wv, (size_t)nb * g.vdd, ch->vn, 0, (size_t)n * g.vdd);+ a.a.pos_base = nb; a.win = c->window;+ kc = ch->wk; vc = ch->wv;+ }+ ok = ok && vkc_attn_w(ch->qkv, kc, vc, ch->ctx, NULL, NULL, ch->prm, &a);+ if (ok && g.swa && n > 1) { /* the ring keeps the block's last `rows` positions */+ int from = n > g.rows ? n - g.rows : 0;+ ok = mc_ring_copy(ch->kc[i], ch->kn, g.rows, pb, pb + from, pb + n, g.kd, 1) &&+ mc_ring_copy(ch->vc[i], ch->vn, g.rows, pb, pb + from, pb + n, g.vdd, 1);+ }+ /* for the host: every row of a full layer, the ring's share of a windowed one */+ int from = g.swa && n > g.rows ? n - g.rows : 0, cap = g.swa && g.rows < ch->rows ? g.rows : ch->rows;+ ok = ok && vkc_copy(ch->kvd, ch->o_kvd[i], ch->kn, (size_t)from * g.kd, (size_t)(n - from) * g.kd) &&+ vkc_copy(ch->kvd, ch->o_kvd[i] + (size_t)cap * g.kd, ch->vn, (size_t)from * g.vdd, (size_t)(n - from) * g.vdd);+ return ok && vkc_matmul((ColiVkTensor *)l->o.vk, ch->ctx, 0, ch->tmp, 0, n);+}+/* layer 0's dense MLP: tmp = down(silu(gate(nrm)) * up(nrm)) */+static int mc_dense_mlp(MimoChain *ch, Model *m, int i, int n) {+ Layer *l = &m->L[i];+ int F = l->gate.O;+ VkcEw p = {VKC_EW_SWIGLU, n * F, F, 1, 0, 1, 0, 0, 0, 0, 0, 1.f};+ return vkc_matmul((ColiVkTensor *)l->gate.vk, ch->nrm, 0, ch->g, 0, n) &&+ vkc_matmul((ColiVkTensor *)l->up.vk, ch->nrm, 0, ch->u, 0, n) &&+ vkc_ew(ch->g, ch->g, ch->u, NULL, NULL, &p) &&+ vkc_matmul((ColiVkTensor *)l->down.vk, ch->g, 0, ch->tmp, 0, n);+}++/* The rows the step at pb wrote, into the host's caches (the CPU's own placement). */+static void mc_commit(MimoChain *ch, Model *m, int n, int pb) {+ const float *kv = (const float *)vkc_ptr(ch->kvd);+ for (int i = 0; i < m->c.n_layers; i++) {+ McGeo g = mc_geo(m, i);+ Layer *l = &m->L[i];+ int from = g.swa && n > g.rows ? n - g.rows : 0, cap = g.swa && g.rows < ch->rows ? g.rows : ch->rows;+ const float *K = kv + ch->o_kvd[i], *V = K + (size_t)cap * g.kd;+ for (int t = from; t < n; t++) {+ int p = pb + t, slot = g.swa ? p % g.rows : p;+ memcpy(l->K + (size_t)slot * g.kd, K + (size_t)(t - from) * g.kd, (size_t)g.kd * sizeof(float));+ memcpy(l->V + (size_t)slot * g.vdd, V + (size_t)(t - from) * g.vdd, (size_t)g.vdd * sizeof(float));+ if (g.swa) l->ring_pos[slot] = p;+ }+ ch->kv_valid[i] = pb + n;+ }+}++/* Every layer for the S rows h (the embedded rows, pictures spliced) at pos_base, the+ * logits as forward() wants them (NULL: none; all_rows: [S][V], else the last row's).+ * Returns how many rows it computed: S, or fewer when the device was lost or the chain+ * declined -- the host's caches then hold every position before pos_base + that many,+ * and the CPU runs the rest. */+static int mc_forward(Model *m, const float *h, int S, int pos_base, float *logits, int all_rows) {+ if (!g_vk_chain) return 0;+ if (g_vk_chain == COLI_VK_CHAIN_PREFILL && S <= 2) return 0; /* prompts only: decode on the CPU */+ MimoChain *ch = mc_setup(m);+ if (!ch || ch->failed) return 0;+ Cfg *c = &m->c; int H = c->hidden, L = c->n_layers, V = c->vocab;+ int CH = mc_chunk_rows(), rows = S < CH ? S : CH;+ int out = logits && all_rows ? rows : 1;+ if (!mc_scratch(ch, m, rows, out)) {+ fprintf(stderr, "[VK] mimo chain: device memory for %d rows refused; per-matrix path\n", rows);+ ch->failed = 1;+ return 0;+ }+ vkc_gemm_rows(-1);+ int done = 0;+ for (int c0 = 0; c0 < S; c0 += rows) {+ int n = S - c0 < rows ? S - c0 : rows, pb = pos_base + c0, last = c0 + n == S;+ if (!vkc_begin() || !vkc_write(ch->x, 0, h + (size_t)c0 * H, (size_t)n * H * sizeof(float)) ||+ !mc_push_state(ch, m, pb)) goto lost;+ float *cs = (float *)vkc_ptr(ch->cs); /* the CPU's angles, cosines and sines (rope()) */+ for (int k = 0; k < 2; k++) {+ int rd = c->rope_dim[k], half = rd / 2;+ for (int s = 0; s < n; s++) for (int j = 0; j < half; j++) {+ float inv = 1.0f / powf(c->theta[k], (float)(2 * j) / (float)rd);+ float ang = inv * (float)(pb + s);+ cs[ch->rd_off[k] + (s * half + j) * 2] = cosf(ang);+ cs[ch->rd_off[k] + (s * half + j) * 2 + 1] = sinf(ang);+ }+ }+ int ok = 1, pending = 0;+ for (int i = 0; i < L && ok; i++) {+ if (pending) ok = mc_add(ch->x, ch->routed, n * H); /* the routed experts join, as h += out */+ pending = 0;+ ok = ok && mc_norm(ch->x, 0, ch->prm, ch->o_ln1[i], ch->nrm, 0, n, H, c->eps) &&+ mc_attention(ch, m, i, n, pb) && mc_add(ch->x, ch->tmp, n * H) &&+ mc_norm(ch->x, 0, ch->prm, ch->o_ln2[i], ch->nrm, 0, n, H, c->eps);+ if (ok && !c->moe[i]) ok = mc_dense_mlp(ch, m, i, n) && mc_add(ch->x, ch->tmp, n * H);+ else if (ok) {+ ok = vkc_copy(ch->h2d, 0, ch->nrm, 0, (size_t)n * H);+ double t0 = now_s();+ ok = ok && vkc_submit(1);+ ch->frames++; ch->wait_ms += (now_s() - t0) * 1e3;+ if (!ok) break;+ double t1 = now_s();+ moe(m, i, (const float *)vkc_ptr(ch->h2d), n, ch->host_out);+ memcpy(vkc_ptr(ch->routed), ch->host_out, (size_t)n * H * sizeof(float));+ ch->host_ms += (now_s() - t1) * 1e3;+ ok = vkc_begin();+ pending = 1;+ }+ }+ if (ok && pending) ok = mc_add(ch->x, ch->routed, n * H);+ int ro = logits && all_rows ? n : logits && last ? 1 : 0;+ if (ok && ro) ok = mc_norm(ch->x, (size_t)(n - ro) * H, ch->prm, ch->o_final, ch->fin, 0, ro, H, c->eps) &&+ vkc_matmul((ColiVkTensor *)m->head.vk, ch->fin, 0, ch->outd, 0, ro);+ double t0 = now_s();+ ok = ok && vkc_submit(1);+ ch->frames++; ch->wait_ms += (now_s() - t0) * 1e3;+ if (!ok) goto lost;+ mc_commit(ch, m, n, pb);+ if (ro) memcpy(all_rows ? logits + (size_t)c0 * V : logits, vkc_ptr(ch->outd), (size_t)ro * V * sizeof(float));+ done = c0 + n;+ }+ ch->forwards++;+ return S;+lost: /* a frame failed: the device is gone (or would not take a command); the CPU takes over */+ if (!vkc_lost()) { vkc_finish(); coli_vk_mark_lost(); }+ g_vk_chain = 0;+ ch->failed = 1;+ fprintf(stderr, "[VK] mimo chain: the device was lost at position %d; the host's caches hold every "+ "position before it, and the CPU runs from there on\n", pos_base + done);+ return done;+}++static void mc_report(Model *m, const char *what) {+ MimoChain *ch = (MimoChain *)m->vkchain;+ if (!ch || !ch->ok || !ch->forwards) return;+ VkcStats st; vkc_stats(&st);+ fprintf(stderr, "[VK] mimo chain: %llu forwards, %llu frames (%llu ops, %llu matmuls, %llu tiled GEMM), "+ "%.1f ms waiting for the device, %.1f ms of routed experts on the host, %.1f MiB on the device (%s)\n",+ ch->forwards, st.frames, st.ops, st.matmuls, st.gemms, ch->wait_ms, ch->host_ms,+ st.dev_bytes / 1048576.0, what);+ vkc_prof_print();+}