Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

vk_tier: look for a victim before staging a promotion, take it after

JustVugg642ff0a2adb7
diff --git a/c/vk_tier.c b/c/vk_tier.c
index 097be333..1bbed022 100644
--- a/c/vk_tier.c
+++ b/c/vk_tier.c
@@ -325,19 +325,20 @@ static void refresh_candidates(void) {
T.cand[at] = (int)i; T.cand_score[at] = sc;
}
}
-/* The resident a newcomer of score hs displaces, or -1: the coldest candidate still
- * resident, when the newcomer beats it by tier.h's LFRU margin (25% + 4 counts). */
-static int pick_victim(uint64_t hs) {
+/* The resident a newcomer of score hs would displace, or -1: the coldest candidate
+ * still resident, when the newcomer beats it by tier.h's LFRU margin (25% + 4
+ * counts). It stays first in the list until drop_victim takes it. */
+static void drop_victim(void) {
+ memmove(T.cand, T.cand + 1, (size_t)--T.ncand * sizeof(int));
+ memmove(T.cand_score, T.cand_score + 1, (size_t)T.ncand * sizeof(uint64_t));
+}
+static int peek_victim(uint64_t hs) {
while (T.ncand) {
int i = T.cand[0];
VSlot *v = &T.s[i];
- if (v->state != VS_RESIDENT) { memmove(T.cand, T.cand + 1, (size_t)--T.ncand * sizeof(int));
- memmove(T.cand_score, T.cand_score + 1, (size_t)T.ncand * sizeof(uint64_t)); continue; }
+ if (v->state != VS_RESIDENT) { drop_victim(); continue; }
uint64_t cs = score(v);
- if (hs <= cs + (cs >> 2) + (4u << 8)) return -1;
- memmove(T.cand, T.cand + 1, (size_t)--T.ncand * sizeof(int));
- memmove(T.cand_score, T.cand_score + 1, (size_t)T.ncand * sizeof(uint64_t));
- return i;
+ return hs > cs + (cs >> 2) + (4u << 8) ? i : -1;
}
return -1;
}
@@ -363,19 +364,16 @@ void vkt_note(int layer, int eid, const VktExpertSrc *src) {
VSlot *v = slot(layer, eid);
if (v->state != VS_NONE || !src->g || !src->u || !src->d) return;
if (T.promos >= T.promo_cap) { T.rated++; return; }
+ if (T.gu_scales && (!src->gs || !src->us || !src->ds)) return;
int victim = -1;
- if (T.resident + T.queued >= T.max_resident) {
- if (!v->heat) return;
- victim = pick_victim(score(v));
- if (victim < 0) return;
- }
+ if (T.resident + T.queued >= T.max_resident && (!v->heat || (victim = peek_victim(score(v))) < 0)) return;
pthread_mutex_lock(&T.mx);
- int full = T.qn >= VKT_QCAP;
+ int qfull = T.qn >= VKT_QCAP;
pthread_mutex_unlock(&T.mx);
- if (full) { T.qfull++; return; }
- if (T.gu_scales && (!src->gs || !src->us || !src->ds)) return;
+ if (qfull) { T.qfull++; return; }
uint8_t *buf = malloc(T.stage_bytes);
if (!buf) return;
+ if (victim >= 0) drop_victim();
uint8_t *p = buf;
memcpy(p, src->g, T.gu_codes); p += T.gu_codes;
memcpy(p, src->u, T.gu_codes); p += T.gu_codes;