Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

mimo: the tier's budget leaves the dense chain's KV caches room

JustVugg61f669c91090
diff --git a/c/mimo.c b/c/mimo.c
index 9ce26fb3..4230848c 100644
--- a/c/mimo.c
+++ b/c/mimo.c
@@ -1333,7 +1333,8 @@ static void vk_tier_start(Model *m) {
.gate_up = f, .down = f, .act = VKT_ACT_SWIGLU,
.max_rows = MIMO_VK_ROWS * c->topk,
.ram_reserve = (size_t)(m->e_bytes + 8192) * (size_t)cap * (size_t)nmoe,
- .dense_bytes = coli_vk_dense() ? vk_dense_bytes(m, 1) : g_vk_chain ? vk_dense_bytes(m, 0) : 0,
+ .dense_bytes = (coli_vk_dense() ? vk_dense_bytes(m, 1) : g_vk_chain ? vk_dense_bytes(m, 0) : 0) +
+ (g_vk_chain ? mc_kv_bytes(m) : 0), /* the chain's KV caches */
.in_ram = vk_in_ram, .ram_ctx = m};
atexit(coli_vk_shutdown); /* before vkt_init, which makes the expert batch's pipelines and can still refuse (no room): the device goes at exit either way, after the tier's teardown */
if (vkt_init(&vc, NULL)) atexit(vkt_shutdown);
diff --git a/c/mimo_chain.h b/c/mimo_chain.h
index 3ec28a89..801bf034 100644
--- a/c/mimo_chain.h
+++ b/c/mimo_chain.h
@@ -89,6 +89,13 @@ static McGeo mc_geo(const Model *m, int li) {
static int mc_ctensor(DW *d) { return dw_upload(d) && d->vk; }
+/* the device's KV caches, in bytes (the tier's budget leaves them room) */
+static size_t mc_kv_bytes(const Model *m) {
+ size_t b = 0;
+ for (int i = 0; i < m->c.n_layers; i++) { McGeo g = mc_geo(m, i); b += (size_t)g.rows * (g.kd + g.vdd) * sizeof(float); }
+ return b;
+}
+
/* The model's parameters on the device, its tensors resolved, its caches allocated;
* NULL = the chain cannot run. */
static MimoChain *mc_setup(Model *m) {