Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
vk_tier: with COLI_VK_TIER_SYNC=1 an upload waits for the victim's free
JustVugg292eb7179905
diff --git a/c/backend_vulkan.c b/c/backend_vulkan.cindex f45f1c36..7071b787 100644--- a/c/backend_vulkan.c+++ b/c/backend_vulkan.c@@ -2109,6 +2109,10 @@ void coli_vk_tier_pool_limit(size_t bytes) {g_tpool.p.limit = bytes;/* blocks no larger than the budget, so a small budget is not one refused block */g_tpool.p.block_bytes = bytes && bytes < VK_WBLOCK ? ((bytes + 4095) & ~(size_t)4095) : VK_WBLOCK;+ /* the empty block the pool keeps as a spare (vka_free) goes too: a tier started+ * again after a shutdown would otherwise fill it past its new, smaller budget */+ for (int k = 0; k < g_tpool.p.nb; k++)+ if (g_tpool.p.b[k].present && !g_tpool.p.b[k].live) pool_release_block(&g_tpool, k);pthread_mutex_unlock(&g_tpool.mx);}diff --git a/c/kimi_k3.c b/c/kimi_k3.cindex ad31128e..d1aac923 100644--- a/c/kimi_k3.c+++ b/c/kimi_k3.c@@ -2024,9 +2024,9 @@ static float *k3_shared(Model *m, Moe *o, const float *x, int C){* the sum never depends on which experts were resident, and an expert computed* on the CPU keeps the CPU run's bits. The experts the CPU computed are offered* to the tier (vkt_note) after the join, those still in the RAM cache: a- * promotion that displaces a resident then frees it at once instead of at the- * next step, so its upload never meets a full pool (with COLI_VK_TIER_SYNC=1 such- * an upload would fail rather than wait, and the tier's budget would shrink). */+ * promotion that displaces a resident then frees it at once, and its upload+ * starts on a pool with room instead of waiting on the uploader thread for the+ * join's free. */typedef struct { int *idx, *left; uint8_t *taken; const float **dev; int ndev; } K3VkStep;static void k3_vk_issue(Model *m, int li, int C, const int *keff, const int *idxs,const float *z, K3VkStep *st){diff --git a/c/mimo.c b/c/mimo.cindex 052c82fb..c9a37bba 100644--- a/c/mimo.c+++ b/c/mimo.c@@ -1014,9 +1014,8 @@ static int moe_cpu(Model *m, int li, const float *xn, int n, const int *sel,* the device's and the CPU's alike: the order never depends on which experts were* resident. The experts the CPU computed are offered to the tier after the join* (those still in the RAM cache), not while the batch runs: a promotion that- * displaces a resident then frees it at once, instead of at the next step, so- * its upload never meets a full pool (with COLI_VK_TIER_SYNC=1 such an upload- * would fail rather than wait, and the tier's budget would shrink). */+ * displaces a resident then frees it at once, and its upload starts on a pool+ * with room instead of waiting on the uploader thread for the join's free. */#define MIMO_VK_ROWS 64 /* rows whose pairs one device batch carries at most */static void moe(Model *m, int li, const float *xn, int n, float *out) {Cfg *c = &m->c;diff --git a/c/tests/test_vk_tier.c b/c/tests/test_vk_tier.cindex 0ec92f4d..daff610b 100644--- a/c/tests/test_vk_tier.c+++ b/c/tests/test_vk_tier.c@@ -11,6 +11,9 @@* go to the device;* partial a step with more assignments than max_rows: the device takes* max_rows of them, the CPU the rest, the sum is still right;+ * sync COLI_VK_TIER_SYNC=1 and a budget of two experts: a promotion that+ * displaces a resident while a batch is in flight finds its room at+ * the join (no failed upload, the budget holds);* books device + CPU = routed, resident <= budget, no failed upload;* v4 DeepSeek V4's activation (route weight on the device, its bf16 and E4M3* roundings): device rows against the engine's CPU arithmetic.@@ -20,6 +23,7 @@#include <stdlib.h>#include <string.h>#include <math.h>+#include <unistd.h>#include "../backend_vulkan.h"#include "../vk_tier.h"@@ -328,6 +332,65 @@ static void partial(void) {vkt_shutdown(); model_free();}+/* vkt_report's evictions and failed uploads, read back from its stderr line */+static void report_counts(unsigned long long *evictions, unsigned long long *failed) {+ *evictions = *failed = ~0ull;+ FILE *t = tmpfile();+ if (!t) return;+ fflush(stderr);+ int saved = dup(2);+ if (saved < 0) { fclose(t); return; }+ dup2(fileno(t), 2);+ vkt_report("test", 0, 0);+ fflush(stderr);+ dup2(saved, 2); close(saved);+ rewind(t);+ char line[4096];+ while (fgets(line, sizeof line, t)) {+ fputs(line, stderr);+ const char *e = strstr(line, "evictions "), *f = strstr(line, "failed ");+ if (e) *evictions = strtoull(e + 10, NULL, 10);+ if (f) *failed = strtoull(f + 7, NULL, 10);+ }+ fclose(t);+}++/* COLI_VK_TIER_SYNC=1 and a budget of two experts. Phase A makes experts 0 and 1 of+ * layer 0 resident; in phase B every step routes 0 (resident: the step's batch is in+ * flight) with 5 and 6, which the CPU computes and notes, so the promotion of 5 that+ * displaces the cold 1 is decided while a batch runs and 1 is freed only at the+ * join. The uploader must wait for that free: it used to try at once, meet the full+ * pool and give up (sync mode does not retry), the upload counted as failed and the+ * budget shrank to the one expert left, so 5 never became resident. */+static void sync_evict(void) {+ VktFmt f = {VKT_SRC_I8_ROW, 0};+ model_make(f, f); g_act = VKT_ACT_SWIGLU; g_limit = 0;+ set_budget(2, f, f);+ setenv("COLI_VK_TIER_RATE", "64", 1);+ setenv("COLI_VK_TIER_SYNC", "1", 1);+ VktConfig vc = cfg_of(f, f, VKT_ACT_SWIGLU, 0);+ int on = vkt_init(&vc, NULL);+ unsetenv("COLI_VK_TIER_SYNC");+ CHECK(on, "sync: the tier did not start");+ if (!on) { model_free(); return; }+ Books a = {0, 0}, b = {0, 0}; double worst = 0;+ int idx_a[K] = {0, 1, 2}, idx_b[K] = {0, 5, 6}; float w[K] = {0.5f, 0.3f, 0.2f};+ for (int t = 0; t < 12; t++) { vkt_begin_forward(); double r = step(0, 1, idx_a, w, &a); if (r > worst) worst = r; }+ int res_a = vkt_resident(0, 0) + vkt_resident(0, 1);+ for (int t = 0; t < 40; t++) { vkt_begin_forward(); double r = step(0, 1, idx_b, w, &b); if (r > worst) worst = r; }+ int res = 0; for (int l = 0; l < L; l++) for (int e = 0; e < E; e++) res += vkt_resident(l, e);+ unsigned long long ev, failed;+ report_counts(&ev, &failed);+ printf(" sync: phase A resident %d of 2; phase B device %llu of %llu, resident 0 %d, 1 %d, 5 %d, all %d (budget 2), evictions %llu, failed %llu\n",+ res_a, b.dev, b.routed, vkt_resident(0, 0), vkt_resident(0, 1), vkt_resident(0, 5), res, ev, failed);+ CHECK(res_a == 2, "sync: phase A made %d of experts 0 and 1 resident", res_a);+ CHECK(ev >= 1 && failed == 0, "sync: %llu evictions, %llu failed uploads (a promotion gave up on its victim's room)", ev, failed);+ CHECK(vkt_resident(0, 5) && !vkt_resident(0, 1) && res == 2,+ "sync: expert 5 did not displace expert 1 (resident 1 %d, 5 %d, all %d of a budget of 2)", vkt_resident(0, 1), vkt_resident(0, 5), res);+ CHECK(worst < 2e-3, "sync: relative error %.3g", worst);+ vkt_shutdown(); model_free();+}+/* DeepSeek V4's activation (VKT_ACT_SWIGLU_V4) through the tier: MXFP4 ue8m0 experts,* x rounded to E4M3 per 128 here as the engine rounds it, the route weight applied on* the device (vkt_issue_w), each device row rounded to bf16 and added with no weight@@ -447,6 +510,7 @@ int main(int argc, char **argv) {printf("warm:\n"); warm();printf("adapt:\n"); adapt();printf("partial:\n"); partial();+ printf("sync:\n"); sync_evict();printf("DeepSeek V4:\n"); v4_act();ColiVkPoolStats ps; coli_vk_pool_stats(1, &ps);CHECK(ps.live == 0, "%d tier ranges still live after every shutdown", ps.live);diff --git a/c/vk_tier.c b/c/vk_tier.cindex 3550ecef..0f8580bf 100644--- a/c/vk_tier.c+++ b/c/vk_tier.c@@ -8,7 +8,8 @@* thread makes them resident at a quiescent point (no batch in flight), where it* also frees evicted experts. So the slot table, the counters and every Vulkan* call but the pool's allocation are single-threaded; the lock guards the queue,- * the done list and the room-made signal only. */+ * the done list, the room-made signal and the count of victims not freed yet+ * only. */#ifdef COLI_VULKAN#include <stdio.h>#include <stdlib.h>@@ -51,6 +52,7 @@ static struct {pthread_mutex_t mx; pthread_cond_t cv, cv_room, cv_done;int busy; /* the uploader is between taking an entry and push_done */int sync; /* COLI_VK_TIER_SYNC=1: quiescent points wait for the uploader */+ int evict_pending; /* victims chosen and not freed yet (a batch is in flight) */VQ q[VKT_QCAP]; int qh, qn;VDone *done; int ndone, cdone;unsigned long room_gen;@@ -245,6 +247,11 @@ static void *uploader(void *arg) {for (;;) {pthread_mutex_lock(&T.mx);while (!T.qn && !T.stop) pthread_cond_wait(&T.cv, &T.mx);+ /* COLI_VK_TIER_SYNC=1: a victim chosen while a batch is in flight is freed at that+ * batch's join, before the join waits for the uploads staged so far: wait for+ * that free here, so the promotion that displaced it finds its room. Uploading+ * first would meet the full pool, and a refusal is final in this mode. */+ while (T.sync && T.evict_pending && !T.stop) pthread_cond_wait(&T.cv_room, &T.mx);if (T.stop) { pthread_mutex_unlock(&T.mx); return NULL; }VQ e = T.q[T.qh]; T.qh = (T.qh + 1) % VKT_QCAP; T.qn--; T.busy = 1;pthread_mutex_unlock(&T.mx);@@ -260,7 +267,9 @@ static void *uploader(void *arg) {* minute (the engine stopped stepping), the pool is full for real. */for (int frees = 0, waited = 0; !ok; ) {ok = upload(g, u, d, gs, us, ds, t);- if (ok || frees >= 3 || waited >= 600 || T.sync) break; /* sync: the engine waits on us, not we on it */+ /* sync: every free decided so far came first (above), and the engine now waits+ * on us, not we on it: the pool is full for real */+ if (ok || frees >= 3 || waited >= 600 || T.sync) break;pthread_mutex_lock(&T.mx);unsigned long gen = T.room_gen;while (T.room_gen == gen && !T.stop && waited < 600) {@@ -282,15 +291,19 @@ static void *uploader(void *arg) {/* ---- quiescent points: nothing in flight ---------------------------------------- */static void quiesce(void) {if (T.inflight) return;- int freed = 0;+ int freed = 0, had = T.nevict;for (int i = 0; i < T.nevict; i++) {VSlot *v = &T.s[T.evict[i]];if (v->ex) { coli_vk_xb_expert_free(v->ex); v->ex = NULL; freed = 1; }v->state = VS_NONE;}T.nevict = 0;- if (freed) {- pthread_mutex_lock(&T.mx); T.room_gen++; pthread_cond_broadcast(&T.cv_room); pthread_mutex_unlock(&T.mx);+ if (had) { /* room made: an upload may be waiting for it */+ pthread_mutex_lock(&T.mx);+ if (freed) T.room_gen++;+ T.evict_pending = 0;+ pthread_cond_broadcast(&T.cv_room);+ pthread_mutex_unlock(&T.mx);}/* finished uploads become resident; COLI_VK_TIER_SYNC=1 waits for every staged one* first, so what is resident depends on the routing alone, not on thread timing@@ -400,7 +413,8 @@ void vkt_note(int layer, int eid, const VktExpertSrc *src) {T.evict = n; T.cevict = nc;}T.evict[T.nevict++] = victim;- quiesce(); /* nothing in flight: free it right away */+ pthread_mutex_lock(&T.mx); T.evict_pending = T.nevict; pthread_mutex_unlock(&T.mx);+ quiesce(); /* nothing in flight: free it right away; else at the join */}v->state = VS_QUEUED; T.queued++; T.promos++;pthread_mutex_lock(&T.mx);