Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
engines: the device goes at exit even when the tier refuses
JustVugge9bac64ecf85
diff --git a/c/colibri.c b/c/colibri.cindex 9ca04f99..60135001 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -10767,10 +10767,10 @@ static void vk_tier_start(Model *m){.dense_bytes=(size_t)dense,.in_ram=vk_in_ram, .ram_ctx=m,.max_experts=g_vk_experts>0 ? g_vk_experts : 0};+ atexit(coli_vk_shutdown); /* before vkt_init, which makes the expert batch's pipelines and can still refuse (no room): the device goes at exit either way, after the tier's teardown */if(!vkt_init(&vc,m->eusage)) return;g_vkt.on=1; g_vkt.gu_fmt=f[0]; g_vkt.gu_gs=gs[0]; g_vkt.dn_fmt=f[2]; g_vkt.dn_gs=gs[2];g_vkt.exp_bytes=vkt_expert_bytes(c->hidden,c->moe_inter,gu,dn);- atexit(coli_vk_shutdown); /* runs after the tier's teardown: the device goes before the drivers unload */atexit(vkt_shutdown);g_vk_model=m; atexit(vk_tier_report_run); /* runs first: the tier is still up *//* warm start: the history's hottest experts, read in parallel (pinned ones from RAM) */diff --git a/c/glm53.c b/c/glm53.cindex 514ab373..06a482b0 100644--- a/c/glm53.c+++ b/c/glm53.c@@ -4020,8 +4020,8 @@ static void glm53_vk_tier_start(GModel *m) {.ram_reserve = (size_t)cap * (size_t)sparse * (size_t)m->e_slot,.dense_bytes = g_vk_dense ? glm53_dense_dev_bytes(m) : 0,.in_ram = glm53_in_ram, .ram_ctx = m};+ atexit(coli_vk_shutdown); /* before vkt_init, which makes the expert batch's pipelines and can still refuse (no room): the device goes at exit either way, after the tier's teardown */if (vkt_init(&vc, rt_counts_all())) {- atexit(coli_vk_shutdown); /* runs after the tier's teardown: the device goes before the drivers unload */atexit(vkt_shutdown);const int all = c->n_layers * c->n_experts;int *pl = malloc((size_t)all * sizeof(int)), *pe = malloc((size_t)all * sizeof(int));diff --git a/c/inkling.c b/c/inkling.cindex 6f1d33a2..e92b58e5 100644--- a/c/inkling.c+++ b/c/inkling.c@@ -1749,8 +1749,8 @@ static void ink_vk_tier_start(Model *m) {.ram_reserve = (size_t)slotb * (size_t)m->cache[0].cap * (size_t)nsp,.dense_bytes = ink_vk_dense_bytes(m),.in_ram = ink_vk_in_ram, .ram_ctx = m};+ atexit(coli_vk_shutdown); /* before vkt_init, which makes the expert batch's pipelines and can still refuse (no room): the device goes at exit either way, after the tier's teardown */if (vkt_init(&vc, m->eusage)) {- atexit(coli_vk_shutdown); /* runs after the tier's teardown: the device goes before the drivers unload */atexit(vkt_shutdown);int all = c->n_layers * c->n_experts;int *pl = malloc((size_t)all * sizeof(int)), *pe = malloc((size_t)all * sizeof(int));diff --git a/c/kimi_k3.c b/c/kimi_k3.cindex d1aac923..d925db17 100644--- a/c/kimi_k3.c+++ b/c/kimi_k3.c@@ -1957,8 +1957,8 @@ static void k3_vk_tier_start(Model *m){.ram_reserve=(size_t)(m->e_slot+8192)*(size_t)cap*(size_t)nmoe,.dense_bytes=coli_vk_dense()?k3_vk_dense_bytes(m):0,.in_ram=k3_vk_in_ram,.ram_ctx=m};+ atexit(coli_vk_shutdown); /* before vkt_init, which makes the expert batch's pipelines and can still refuse (no room): the device goes at exit either way, after the tier's teardown */if(vkt_init(&vc,rt_counts_all())){- atexit(coli_vk_shutdown); /* runs after the tier's teardown: the device goes before the drivers unload */atexit(vkt_shutdown);int all=c->n_layers*c->n_experts;int *pl=malloc((size_t)all*sizeof(int)), *pe=malloc((size_t)all*sizeof(int));diff --git a/c/mimo.c b/c/mimo.cindex c9a37bba..da3b5302 100644--- a/c/mimo.c+++ b/c/mimo.c@@ -1316,10 +1316,8 @@ static void vk_tier_start(Model *m) {.ram_reserve = (size_t)(m->e_bytes + 8192) * (size_t)cap * (size_t)nmoe,.dense_bytes = coli_vk_dense() ? vk_dense_bytes(m) : 0,.in_ram = vk_in_ram, .ram_ctx = m};- if (vkt_init(&vc, NULL)) {- atexit(coli_vk_shutdown); /* runs after the tier's teardown: the device goes before the drivers unload */- atexit(vkt_shutdown);- }+ atexit(coli_vk_shutdown); /* before vkt_init, which makes the expert batch's pipelines and can still refuse (no room): the device goes at exit either way, after the tier's teardown */+ if (vkt_init(&vc, NULL)) atexit(vkt_shutdown);}#endifdiff --git a/c/olmoe.c b/c/olmoe.cindex f738e347..4b8a4732 100644--- a/c/olmoe.c+++ b/c/olmoe.c@@ -823,8 +823,8 @@ static void olmoe_vk_tier_start(Model *m) {.act = VKT_ACT_SWIGLU, .max_rows = OLMOE_VK_ROWS * c->topk,.ram_reserve = slotb * (size_t)m->cache[0].cap * (size_t)c->n_layers,.dense_bytes = dense, .in_ram = olmoe_vk_in_ram, .ram_ctx = m};+ atexit(coli_vk_shutdown); /* before vkt_init, which makes the expert batch's pipelines and can still refuse (no room): the device goes at exit either way, after the tier's teardown */if (!vkt_init(&vc, m->freq)) return;- atexit(coli_vk_shutdown); /* runs after the tier's teardown: the device goes before the drivers unload */atexit(vkt_shutdown);int all = c->n_layers * c->n_experts;int *pl = malloc((size_t)all * sizeof(int)), *pe = malloc((size_t)all * sizeof(int));diff --git a/c/qwen36.c b/c/qwen36.cindex 2d438929..8c876db4 100644--- a/c/qwen36.c+++ b/c/qwen36.c@@ -4563,9 +4563,9 @@ static void vk_tier_start(Model *m, const char *snap, int cap, int expert_is_intelse snprintf(g_vk_usage, sizeof g_vk_usage, "%s/.coli_usage", snap);int64_t h = rt_load(g_vk_usage);if (h > 0) fprintf(stderr, "[USAGE] expert history: %lld selections (%s)\n", (long long)h, g_vk_usage);+ atexit(coli_vk_shutdown); /* before vkt_init, which makes the expert batch's pipelines and can still refuse (no room): the device goes at exit either way, after the tier's teardown */if (!vkt_init(&vc, rt_counts_all())) { rt_destroy(); return; }m->vk_hist = 1;- atexit(coli_vk_shutdown); /* runs after the tier's teardown: the device goes before the drivers unload */atexit(vkt_shutdown);int all = c->n_layers * c->n_experts;int *pl = malloc((size_t)all * sizeof(int)), *pe = malloc((size_t)all * sizeof(int));diff --git a/c/qwen38.c b/c/qwen38.cindex 8613f13d..5b87d62e 100644--- a/c/qwen38.c+++ b/c/qwen38.c@@ -2487,8 +2487,8 @@ static void q38_vk_tier_start(Model *m,int cap){.in_ram=q38_vk_in_ram,.ram_ctx=m};if(vc.max_rows<64)vc.max_rows=64;uint32_t **heat=rt_counts_all();+ atexit(coli_vk_shutdown); /* before vkt_init, which makes the expert batch's pipelines and can still refuse (no room): the device goes at exit either way, after the tier's teardown */if(!vkt_init(&vc,heat))return;- atexit(coli_vk_shutdown); /* runs after the tier's teardown: the device goes before the drivers unload */atexit(vkt_shutdown);/* the warm start: the history's hottest experts, read in parallel */int all=c->layers*c->experts,*pl=(int*)malloc((size_t)all*sizeof(int)),*pe=(int*)malloc((size_t)all*sizeof(int));diff --git a/c/vk_tier.h b/c/vk_tier.hindex 2486c8eb..41043878 100644--- a/c/vk_tier.h+++ b/c/vk_tier.h@@ -43,10 +43,12 @@* .ram_reserve = bytes the RAM expert cache may still grow by,* .dense_bytes = bytes of dense weights the engine will put* on the device after this call};- * if (vkt_init(&c, rt_counts_all())) {- * atexit(coli_vk_shutdown); // runs second: the device goes before- * atexit(vkt_shutdown); // the drivers unload (a driver can still- * } // be compiling the tier's pipelines)+ * atexit(coli_vk_shutdown); // runs last: the device goes before the+ * if (vkt_init(&c, rt_counts_all())) // drivers unload (a driver can still+ * atexit(vkt_shutdown); // be compiling the tier's pipelines), and+ * // whether or not the tier starts: vkt_init+ * // makes the expert batch's pipelines+ * // before it can refuse (no room)** .in_ram (optional) lets the tier hand experts the CPU holds in RAM back to the* CPU when the device is the slower side of a step (an integrated GPU at its