Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

engines: the device goes at exit even when the tier refuses

JustVugge9bac64ecf85
diff --git a/c/colibri.c b/c/colibri.c
index 9ca04f99..60135001 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -10767,10 +10767,10 @@ static void vk_tier_start(Model *m){
.dense_bytes=(size_t)dense,
.in_ram=vk_in_ram, .ram_ctx=m,
.max_experts=g_vk_experts>0 ? g_vk_experts : 0};
+ atexit(coli_vk_shutdown); /* before vkt_init, which makes the expert batch's pipelines and can still refuse (no room): the device goes at exit either way, after the tier's teardown */
if(!vkt_init(&vc,m->eusage)) return;
g_vkt.on=1; g_vkt.gu_fmt=f[0]; g_vkt.gu_gs=gs[0]; g_vkt.dn_fmt=f[2]; g_vkt.dn_gs=gs[2];
g_vkt.exp_bytes=vkt_expert_bytes(c->hidden,c->moe_inter,gu,dn);
- atexit(coli_vk_shutdown); /* runs after the tier's teardown: the device goes before the drivers unload */
atexit(vkt_shutdown);
g_vk_model=m; atexit(vk_tier_report_run); /* runs first: the tier is still up */
/* warm start: the history's hottest experts, read in parallel (pinned ones from RAM) */
diff --git a/c/glm53.c b/c/glm53.c
index 514ab373..06a482b0 100644
--- a/c/glm53.c
+++ b/c/glm53.c
@@ -4020,8 +4020,8 @@ static void glm53_vk_tier_start(GModel *m) {
.ram_reserve = (size_t)cap * (size_t)sparse * (size_t)m->e_slot,
.dense_bytes = g_vk_dense ? glm53_dense_dev_bytes(m) : 0,
.in_ram = glm53_in_ram, .ram_ctx = m};
+ atexit(coli_vk_shutdown); /* before vkt_init, which makes the expert batch's pipelines and can still refuse (no room): the device goes at exit either way, after the tier's teardown */
if (vkt_init(&vc, rt_counts_all())) {
- atexit(coli_vk_shutdown); /* runs after the tier's teardown: the device goes before the drivers unload */
atexit(vkt_shutdown);
const int all = c->n_layers * c->n_experts;
int *pl = malloc((size_t)all * sizeof(int)), *pe = malloc((size_t)all * sizeof(int));
diff --git a/c/inkling.c b/c/inkling.c
index 6f1d33a2..e92b58e5 100644
--- a/c/inkling.c
+++ b/c/inkling.c
@@ -1749,8 +1749,8 @@ static void ink_vk_tier_start(Model *m) {
.ram_reserve = (size_t)slotb * (size_t)m->cache[0].cap * (size_t)nsp,
.dense_bytes = ink_vk_dense_bytes(m),
.in_ram = ink_vk_in_ram, .ram_ctx = m};
+ atexit(coli_vk_shutdown); /* before vkt_init, which makes the expert batch's pipelines and can still refuse (no room): the device goes at exit either way, after the tier's teardown */
if (vkt_init(&vc, m->eusage)) {
- atexit(coli_vk_shutdown); /* runs after the tier's teardown: the device goes before the drivers unload */
atexit(vkt_shutdown);
int all = c->n_layers * c->n_experts;
int *pl = malloc((size_t)all * sizeof(int)), *pe = malloc((size_t)all * sizeof(int));
diff --git a/c/kimi_k3.c b/c/kimi_k3.c
index d1aac923..d925db17 100644
--- a/c/kimi_k3.c
+++ b/c/kimi_k3.c
@@ -1957,8 +1957,8 @@ static void k3_vk_tier_start(Model *m){
.ram_reserve=(size_t)(m->e_slot+8192)*(size_t)cap*(size_t)nmoe,
.dense_bytes=coli_vk_dense()?k3_vk_dense_bytes(m):0,
.in_ram=k3_vk_in_ram,.ram_ctx=m};
+ atexit(coli_vk_shutdown); /* before vkt_init, which makes the expert batch's pipelines and can still refuse (no room): the device goes at exit either way, after the tier's teardown */
if(vkt_init(&vc,rt_counts_all())){
- atexit(coli_vk_shutdown); /* runs after the tier's teardown: the device goes before the drivers unload */
atexit(vkt_shutdown);
int all=c->n_layers*c->n_experts;
int *pl=malloc((size_t)all*sizeof(int)), *pe=malloc((size_t)all*sizeof(int));
diff --git a/c/mimo.c b/c/mimo.c
index c9a37bba..da3b5302 100644
--- a/c/mimo.c
+++ b/c/mimo.c
@@ -1316,10 +1316,8 @@ static void vk_tier_start(Model *m) {
.ram_reserve = (size_t)(m->e_bytes + 8192) * (size_t)cap * (size_t)nmoe,
.dense_bytes = coli_vk_dense() ? vk_dense_bytes(m) : 0,
.in_ram = vk_in_ram, .ram_ctx = m};
- if (vkt_init(&vc, NULL)) {
- atexit(coli_vk_shutdown); /* runs after the tier's teardown: the device goes before the drivers unload */
- atexit(vkt_shutdown);
- }
+ atexit(coli_vk_shutdown); /* before vkt_init, which makes the expert batch's pipelines and can still refuse (no room): the device goes at exit either way, after the tier's teardown */
+ if (vkt_init(&vc, NULL)) atexit(vkt_shutdown);
}
#endif
diff --git a/c/olmoe.c b/c/olmoe.c
index f738e347..4b8a4732 100644
--- a/c/olmoe.c
+++ b/c/olmoe.c
@@ -823,8 +823,8 @@ static void olmoe_vk_tier_start(Model *m) {
.act = VKT_ACT_SWIGLU, .max_rows = OLMOE_VK_ROWS * c->topk,
.ram_reserve = slotb * (size_t)m->cache[0].cap * (size_t)c->n_layers,
.dense_bytes = dense, .in_ram = olmoe_vk_in_ram, .ram_ctx = m};
+ atexit(coli_vk_shutdown); /* before vkt_init, which makes the expert batch's pipelines and can still refuse (no room): the device goes at exit either way, after the tier's teardown */
if (!vkt_init(&vc, m->freq)) return;
- atexit(coli_vk_shutdown); /* runs after the tier's teardown: the device goes before the drivers unload */
atexit(vkt_shutdown);
int all = c->n_layers * c->n_experts;
int *pl = malloc((size_t)all * sizeof(int)), *pe = malloc((size_t)all * sizeof(int));
diff --git a/c/qwen36.c b/c/qwen36.c
index 2d438929..8c876db4 100644
--- a/c/qwen36.c
+++ b/c/qwen36.c
@@ -4563,9 +4563,9 @@ static void vk_tier_start(Model *m, const char *snap, int cap, int expert_is_int
else snprintf(g_vk_usage, sizeof g_vk_usage, "%s/.coli_usage", snap);
int64_t h = rt_load(g_vk_usage);
if (h > 0) fprintf(stderr, "[USAGE] expert history: %lld selections (%s)\n", (long long)h, g_vk_usage);
+ atexit(coli_vk_shutdown); /* before vkt_init, which makes the expert batch's pipelines and can still refuse (no room): the device goes at exit either way, after the tier's teardown */
if (!vkt_init(&vc, rt_counts_all())) { rt_destroy(); return; }
m->vk_hist = 1;
- atexit(coli_vk_shutdown); /* runs after the tier's teardown: the device goes before the drivers unload */
atexit(vkt_shutdown);
int all = c->n_layers * c->n_experts;
int *pl = malloc((size_t)all * sizeof(int)), *pe = malloc((size_t)all * sizeof(int));
diff --git a/c/qwen38.c b/c/qwen38.c
index 8613f13d..5b87d62e 100644
--- a/c/qwen38.c
+++ b/c/qwen38.c
@@ -2487,8 +2487,8 @@ static void q38_vk_tier_start(Model *m,int cap){
.in_ram=q38_vk_in_ram,.ram_ctx=m};
if(vc.max_rows<64)vc.max_rows=64;
uint32_t **heat=rt_counts_all();
+ atexit(coli_vk_shutdown); /* before vkt_init, which makes the expert batch's pipelines and can still refuse (no room): the device goes at exit either way, after the tier's teardown */
if(!vkt_init(&vc,heat))return;
- atexit(coli_vk_shutdown); /* runs after the tier's teardown: the device goes before the drivers unload */
atexit(vkt_shutdown);
/* the warm start: the history's hottest experts, read in parallel */
int all=c->layers*c->experts,*pl=(int*)malloc((size_t)all*sizeof(int)),*pe=(int*)malloc((size_t)all*sizeof(int));
diff --git a/c/vk_tier.h b/c/vk_tier.h
index 2486c8eb..41043878 100644
--- a/c/vk_tier.h
+++ b/c/vk_tier.h
@@ -43,10 +43,12 @@
* .ram_reserve = bytes the RAM expert cache may still grow by,
* .dense_bytes = bytes of dense weights the engine will put
* on the device after this call};
- * if (vkt_init(&c, rt_counts_all())) {
- * atexit(coli_vk_shutdown); // runs second: the device goes before
- * atexit(vkt_shutdown); // the drivers unload (a driver can still
- * } // be compiling the tier's pipelines)
+ * atexit(coli_vk_shutdown); // runs last: the device goes before the
+ * if (vkt_init(&c, rt_counts_all())) // drivers unload (a driver can still
+ * atexit(vkt_shutdown); // be compiling the tier's pipelines), and
+ * // whether or not the tier starts: vkt_init
+ * // makes the expert batch's pipelines
+ * // before it can refuse (no room)
*
* .in_ram (optional) lets the tier hand experts the CPU holds in RAM back to the
* CPU when the device is the slower side of a step (an integrated GPU at its