Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

vk_tier: a forward is a step back in the layer index; the uploader waits for room

JustVuggaebd65ba6f48
diff --git a/c/vk_tier.c b/c/vk_tier.c
index 9c269126..d87b1c5e 100644
--- a/c/vk_tier.c
+++ b/c/vk_tier.c
@@ -43,8 +43,8 @@ static struct {
size_t exp_bytes, budget;
int max_resident, resident, queued, rate, uma;
VSlot *s;
- uint32_t tick, decay_at;
- int last_layer, promos;
+ uint32_t tick, decay_at; /* tokens seen (rows of the forward's first layer) */
+ int last_layer, first_layer, promos;
/* uploader */
pthread_t th; int th_on, stop;
pthread_mutex_t mx; pthread_cond_t cv, cv_room;
@@ -244,16 +244,21 @@ static void *uploader(void *arg) {
ColiVkTensor *t[3];
int ok = 0;
/* Refused: the pool is at its budget until the engine thread frees the victim
- * this promotion displaced, at its next join. Wait for that, a bounded number
- * of times: a pool that stays full is full for real (fragmentation). */
- for (int tries = 0; tries < 20 && !ok; tries++) {
+ * this promotion displaced, at its next quiescent point (a join: every layer
+ * in decode, after a chunk's CPU work in prefill). Wait for room to be made
+ * and try again; refused again after three frees, or nothing freed for a
+ * minute (the engine stopped stepping), the pool is full for real. */
+ for (int frees = 0, waited = 0; !ok; ) {
ok = upload(g, u, d, gs, us, ds, t);
- if (ok) break;
+ if (ok || frees >= 3 || waited >= 600) break;
pthread_mutex_lock(&T.mx);
unsigned long gen = T.room_gen;
- struct timespec until; clock_gettime(CLOCK_REALTIME, &until);
- until.tv_nsec += 50 * 1000000L; if (until.tv_nsec >= 1000000000L) { until.tv_sec++; until.tv_nsec -= 1000000000L; }
- while (T.room_gen == gen && !T.stop && pthread_cond_timedwait(&T.cv_room, &T.mx, &until) != ETIMEDOUT) {}
+ while (T.room_gen == gen && !T.stop && waited < 600) {
+ struct timespec until; clock_gettime(CLOCK_REALTIME, &until);
+ until.tv_nsec += 100 * 1000000L; if (until.tv_nsec >= 1000000000L) { until.tv_sec++; until.tv_nsec -= 1000000000L; }
+ if (pthread_cond_timedwait(&T.cv_room, &T.mx, &until) == ETIMEDOUT) waited++;
+ }
+ if (T.room_gen != gen) frees++;
int stop = T.stop;
pthread_mutex_unlock(&T.mx);
if (stop) break;
@@ -334,8 +339,10 @@ static int pick_victim(uint64_t hs) {
return -1;
}
-static void new_forward(int S) {
- T.tick += (uint32_t)(S > 0 ? S : 1);
+/* A forward starts when the layer index goes back (a layer that issues several
+ * blocks of rows, or rows one by one, is the same forward). The tick counts tokens:
+ * the rows of the forward's first layer, every block of them. */
+static void new_forward(void) {
T.promos = 0;
if (T.tick >= T.decay_at) {
size_t n = (size_t)T.c.layers * T.c.experts;
@@ -359,7 +366,10 @@ void vkt_note(int layer, int eid, const VktExpertSrc *src) {
victim = pick_victim(score(v));
if (victim < 0) return;
}
- if (T.qn >= VKT_QCAP) { T.qfull++; return; }
+ pthread_mutex_lock(&T.mx);
+ int full = T.qn >= VKT_QCAP;
+ pthread_mutex_unlock(&T.mx);
+ if (full) { T.qfull++; return; }
if (T.gu_scales && (!src->gs || !src->us || !src->ds)) return;
uint8_t *buf = malloc(T.stage_bytes);
if (!buf) return;
@@ -471,7 +481,8 @@ int vkt_issue(int layer, const float *x, int S, int K, const int *idx, uint8_t *
if (T.on) { fprintf(stderr, "[VK] tier %s: the device stopped answering, the experts stay on the CPU\n", T.engine); T.on = 0; }
return 0;
}
- if (layer <= T.last_layer) new_forward(S);
+ if (layer < T.last_layer) { T.first_layer = layer; new_forward(); }
+ if (layer == T.first_layer) T.tick += (uint32_t)S;
T.last_layer = layer;
quiesce();
int E = T.c.experts, H = T.c.hidden, n = S * K;
@@ -643,7 +654,7 @@ int vkt_init(const VktConfig *cfg, uint32_t *const *heat) {
if (!T.s || !T.grp) { free(T.s); free(T.grp); return 0; }
for (int e = 0; e < T.c.experts; e++) T.grp[e] = -1;
T.max_dev_rows = T.c.max_rows > 0 ? T.c.max_rows : 1 << 20;
- T.last_layer = 1 << 30; T.decay_at = VKT_DECAY_TOKENS;
+ T.last_layer = 1 << 30; T.first_layer = -1; T.decay_at = VKT_DECAY_TOKENS;
/* The history, scaled: the hottest expert of a layer starts at 32 and the rest in
* proportion, so a few minutes of a new workload can displace it (raw counts
* from a long history would hold the tier for hours). */