Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

vk_tier: with COLI_VK_TIER_SYNC=1 an upload waits for the victim's free

JustVugg292eb7179905
diff --git a/c/backend_vulkan.c b/c/backend_vulkan.c
index f45f1c36..7071b787 100644
--- a/c/backend_vulkan.c
+++ b/c/backend_vulkan.c
@@ -2109,6 +2109,10 @@ void coli_vk_tier_pool_limit(size_t bytes) {
g_tpool.p.limit = bytes;
/* blocks no larger than the budget, so a small budget is not one refused block */
g_tpool.p.block_bytes = bytes && bytes < VK_WBLOCK ? ((bytes + 4095) & ~(size_t)4095) : VK_WBLOCK;
+ /* the empty block the pool keeps as a spare (vka_free) goes too: a tier started
+ * again after a shutdown would otherwise fill it past its new, smaller budget */
+ for (int k = 0; k < g_tpool.p.nb; k++)
+ if (g_tpool.p.b[k].present && !g_tpool.p.b[k].live) pool_release_block(&g_tpool, k);
pthread_mutex_unlock(&g_tpool.mx);
}
diff --git a/c/kimi_k3.c b/c/kimi_k3.c
index ad31128e..d1aac923 100644
--- a/c/kimi_k3.c
+++ b/c/kimi_k3.c
@@ -2024,9 +2024,9 @@ static float *k3_shared(Model *m, Moe *o, const float *x, int C){
* the sum never depends on which experts were resident, and an expert computed
* on the CPU keeps the CPU run's bits. The experts the CPU computed are offered
* to the tier (vkt_note) after the join, those still in the RAM cache: a
- * promotion that displaces a resident then frees it at once instead of at the
- * next step, so its upload never meets a full pool (with COLI_VK_TIER_SYNC=1 such
- * an upload would fail rather than wait, and the tier's budget would shrink). */
+ * promotion that displaces a resident then frees it at once, and its upload
+ * starts on a pool with room instead of waiting on the uploader thread for the
+ * join's free. */
typedef struct { int *idx, *left; uint8_t *taken; const float **dev; int ndev; } K3VkStep;
static void k3_vk_issue(Model *m, int li, int C, const int *keff, const int *idxs,
const float *z, K3VkStep *st){
diff --git a/c/mimo.c b/c/mimo.c
index 052c82fb..c9a37bba 100644
--- a/c/mimo.c
+++ b/c/mimo.c
@@ -1014,9 +1014,8 @@ static int moe_cpu(Model *m, int li, const float *xn, int n, const int *sel,
* the device's and the CPU's alike: the order never depends on which experts were
* resident. The experts the CPU computed are offered to the tier after the join
* (those still in the RAM cache), not while the batch runs: a promotion that
- * displaces a resident then frees it at once, instead of at the next step, so
- * its upload never meets a full pool (with COLI_VK_TIER_SYNC=1 such an upload
- * would fail rather than wait, and the tier's budget would shrink). */
+ * displaces a resident then frees it at once, and its upload starts on a pool
+ * with room instead of waiting on the uploader thread for the join's free. */
#define MIMO_VK_ROWS 64 /* rows whose pairs one device batch carries at most */
static void moe(Model *m, int li, const float *xn, int n, float *out) {
Cfg *c = &m->c;
diff --git a/c/tests/test_vk_tier.c b/c/tests/test_vk_tier.c
index 0ec92f4d..daff610b 100644
--- a/c/tests/test_vk_tier.c
+++ b/c/tests/test_vk_tier.c
@@ -11,6 +11,9 @@
* go to the device;
* partial a step with more assignments than max_rows: the device takes
* max_rows of them, the CPU the rest, the sum is still right;
+ * sync COLI_VK_TIER_SYNC=1 and a budget of two experts: a promotion that
+ * displaces a resident while a batch is in flight finds its room at
+ * the join (no failed upload, the budget holds);
* books device + CPU = routed, resident <= budget, no failed upload;
* v4 DeepSeek V4's activation (route weight on the device, its bf16 and E4M3
* roundings): device rows against the engine's CPU arithmetic.
@@ -20,6 +23,7 @@
#include <stdlib.h>
#include <string.h>
#include <math.h>
+#include <unistd.h>
#include "../backend_vulkan.h"
#include "../vk_tier.h"
@@ -328,6 +332,65 @@ static void partial(void) {
vkt_shutdown(); model_free();
}
+/* vkt_report's evictions and failed uploads, read back from its stderr line */
+static void report_counts(unsigned long long *evictions, unsigned long long *failed) {
+ *evictions = *failed = ~0ull;
+ FILE *t = tmpfile();
+ if (!t) return;
+ fflush(stderr);
+ int saved = dup(2);
+ if (saved < 0) { fclose(t); return; }
+ dup2(fileno(t), 2);
+ vkt_report("test", 0, 0);
+ fflush(stderr);
+ dup2(saved, 2); close(saved);
+ rewind(t);
+ char line[4096];
+ while (fgets(line, sizeof line, t)) {
+ fputs(line, stderr);
+ const char *e = strstr(line, "evictions "), *f = strstr(line, "failed ");
+ if (e) *evictions = strtoull(e + 10, NULL, 10);
+ if (f) *failed = strtoull(f + 7, NULL, 10);
+ }
+ fclose(t);
+}
+
+/* COLI_VK_TIER_SYNC=1 and a budget of two experts. Phase A makes experts 0 and 1 of
+ * layer 0 resident; in phase B every step routes 0 (resident: the step's batch is in
+ * flight) with 5 and 6, which the CPU computes and notes, so the promotion of 5 that
+ * displaces the cold 1 is decided while a batch runs and 1 is freed only at the
+ * join. The uploader must wait for that free: it used to try at once, meet the full
+ * pool and give up (sync mode does not retry), the upload counted as failed and the
+ * budget shrank to the one expert left, so 5 never became resident. */
+static void sync_evict(void) {
+ VktFmt f = {VKT_SRC_I8_ROW, 0};
+ model_make(f, f); g_act = VKT_ACT_SWIGLU; g_limit = 0;
+ set_budget(2, f, f);
+ setenv("COLI_VK_TIER_RATE", "64", 1);
+ setenv("COLI_VK_TIER_SYNC", "1", 1);
+ VktConfig vc = cfg_of(f, f, VKT_ACT_SWIGLU, 0);
+ int on = vkt_init(&vc, NULL);
+ unsetenv("COLI_VK_TIER_SYNC");
+ CHECK(on, "sync: the tier did not start");
+ if (!on) { model_free(); return; }
+ Books a = {0, 0}, b = {0, 0}; double worst = 0;
+ int idx_a[K] = {0, 1, 2}, idx_b[K] = {0, 5, 6}; float w[K] = {0.5f, 0.3f, 0.2f};
+ for (int t = 0; t < 12; t++) { vkt_begin_forward(); double r = step(0, 1, idx_a, w, &a); if (r > worst) worst = r; }
+ int res_a = vkt_resident(0, 0) + vkt_resident(0, 1);
+ for (int t = 0; t < 40; t++) { vkt_begin_forward(); double r = step(0, 1, idx_b, w, &b); if (r > worst) worst = r; }
+ int res = 0; for (int l = 0; l < L; l++) for (int e = 0; e < E; e++) res += vkt_resident(l, e);
+ unsigned long long ev, failed;
+ report_counts(&ev, &failed);
+ printf(" sync: phase A resident %d of 2; phase B device %llu of %llu, resident 0 %d, 1 %d, 5 %d, all %d (budget 2), evictions %llu, failed %llu\n",
+ res_a, b.dev, b.routed, vkt_resident(0, 0), vkt_resident(0, 1), vkt_resident(0, 5), res, ev, failed);
+ CHECK(res_a == 2, "sync: phase A made %d of experts 0 and 1 resident", res_a);
+ CHECK(ev >= 1 && failed == 0, "sync: %llu evictions, %llu failed uploads (a promotion gave up on its victim's room)", ev, failed);
+ CHECK(vkt_resident(0, 5) && !vkt_resident(0, 1) && res == 2,
+ "sync: expert 5 did not displace expert 1 (resident 1 %d, 5 %d, all %d of a budget of 2)", vkt_resident(0, 1), vkt_resident(0, 5), res);
+ CHECK(worst < 2e-3, "sync: relative error %.3g", worst);
+ vkt_shutdown(); model_free();
+}
+
/* DeepSeek V4's activation (VKT_ACT_SWIGLU_V4) through the tier: MXFP4 ue8m0 experts,
* x rounded to E4M3 per 128 here as the engine rounds it, the route weight applied on
* the device (vkt_issue_w), each device row rounded to bf16 and added with no weight
@@ -447,6 +510,7 @@ int main(int argc, char **argv) {
printf("warm:\n"); warm();
printf("adapt:\n"); adapt();
printf("partial:\n"); partial();
+ printf("sync:\n"); sync_evict();
printf("DeepSeek V4:\n"); v4_act();
ColiVkPoolStats ps; coli_vk_pool_stats(1, &ps);
CHECK(ps.live == 0, "%d tier ranges still live after every shutdown", ps.live);
diff --git a/c/vk_tier.c b/c/vk_tier.c
index 3550ecef..0f8580bf 100644
--- a/c/vk_tier.c
+++ b/c/vk_tier.c
@@ -8,7 +8,8 @@
* thread makes them resident at a quiescent point (no batch in flight), where it
* also frees evicted experts. So the slot table, the counters and every Vulkan
* call but the pool's allocation are single-threaded; the lock guards the queue,
- * the done list and the room-made signal only. */
+ * the done list, the room-made signal and the count of victims not freed yet
+ * only. */
#ifdef COLI_VULKAN
#include <stdio.h>
#include <stdlib.h>
@@ -51,6 +52,7 @@ static struct {
pthread_mutex_t mx; pthread_cond_t cv, cv_room, cv_done;
int busy; /* the uploader is between taking an entry and push_done */
int sync; /* COLI_VK_TIER_SYNC=1: quiescent points wait for the uploader */
+ int evict_pending; /* victims chosen and not freed yet (a batch is in flight) */
VQ q[VKT_QCAP]; int qh, qn;
VDone *done; int ndone, cdone;
unsigned long room_gen;
@@ -245,6 +247,11 @@ static void *uploader(void *arg) {
for (;;) {
pthread_mutex_lock(&T.mx);
while (!T.qn && !T.stop) pthread_cond_wait(&T.cv, &T.mx);
+ /* COLI_VK_TIER_SYNC=1: a victim chosen while a batch is in flight is freed at that
+ * batch's join, before the join waits for the uploads staged so far: wait for
+ * that free here, so the promotion that displaced it finds its room. Uploading
+ * first would meet the full pool, and a refusal is final in this mode. */
+ while (T.sync && T.evict_pending && !T.stop) pthread_cond_wait(&T.cv_room, &T.mx);
if (T.stop) { pthread_mutex_unlock(&T.mx); return NULL; }
VQ e = T.q[T.qh]; T.qh = (T.qh + 1) % VKT_QCAP; T.qn--; T.busy = 1;
pthread_mutex_unlock(&T.mx);
@@ -260,7 +267,9 @@ static void *uploader(void *arg) {
* minute (the engine stopped stepping), the pool is full for real. */
for (int frees = 0, waited = 0; !ok; ) {
ok = upload(g, u, d, gs, us, ds, t);
- if (ok || frees >= 3 || waited >= 600 || T.sync) break; /* sync: the engine waits on us, not we on it */
+ /* sync: every free decided so far came first (above), and the engine now waits
+ * on us, not we on it: the pool is full for real */
+ if (ok || frees >= 3 || waited >= 600 || T.sync) break;
pthread_mutex_lock(&T.mx);
unsigned long gen = T.room_gen;
while (T.room_gen == gen && !T.stop && waited < 600) {
@@ -282,15 +291,19 @@ static void *uploader(void *arg) {
/* ---- quiescent points: nothing in flight ---------------------------------------- */
static void quiesce(void) {
if (T.inflight) return;
- int freed = 0;
+ int freed = 0, had = T.nevict;
for (int i = 0; i < T.nevict; i++) {
VSlot *v = &T.s[T.evict[i]];
if (v->ex) { coli_vk_xb_expert_free(v->ex); v->ex = NULL; freed = 1; }
v->state = VS_NONE;
}
T.nevict = 0;
- if (freed) {
- pthread_mutex_lock(&T.mx); T.room_gen++; pthread_cond_broadcast(&T.cv_room); pthread_mutex_unlock(&T.mx);
+ if (had) { /* room made: an upload may be waiting for it */
+ pthread_mutex_lock(&T.mx);
+ if (freed) T.room_gen++;
+ T.evict_pending = 0;
+ pthread_cond_broadcast(&T.cv_room);
+ pthread_mutex_unlock(&T.mx);
}
/* finished uploads become resident; COLI_VK_TIER_SYNC=1 waits for every staged one
* first, so what is resident depends on the routing alone, not on thread timing
@@ -400,7 +413,8 @@ void vkt_note(int layer, int eid, const VktExpertSrc *src) {
T.evict = n; T.cevict = nc;
}
T.evict[T.nevict++] = victim;
- quiesce(); /* nothing in flight: free it right away */
+ pthread_mutex_lock(&T.mx); T.evict_pending = T.nevict; pthread_mutex_unlock(&T.mx);
+ quiesce(); /* nothing in flight: free it right away; else at the join */
}
v->state = VS_QUEUED; T.queued++; T.promos++;
pthread_mutex_lock(&T.mx);