Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(qwen36): keep int8 expert weights the tier still points at (#1341)
Courtney Andrew Richardsond937ad233b16
diff --git a/c/Makefile b/c/Makefileindex 1c61bd51..39859b64 100644--- a/c/Makefile+++ b/c/Makefile@@ -1656,6 +1656,15 @@ tests/test_qwen36_tier_multidev$(EXE): tests/test_qwen36_tier_multidev.c tests/qtests/test_qwen36_tier_shutdown$(EXE): tests/test_qwen36_tier_shutdown.c tests/qwen36_fake_cuda.h qwen36_tier.c qwen36_tier.h backend_cuda.h$(CC) $(CFLAGS) -DCOLI_CUDA $< -o $@ $(LDFLAGS)+# Same fake backend, but driving the ENGINE: the warmstart in qwen36.c hands the+# tier raw pointers into the RAM expert slots, so only a test that includes+# qwen36.c can prove the weights behind them are still there afterwards -- on an+# int8 container there is no packed copy to rebuild them from (#1341). Includes+# qwen36_tier.c in its own TU like the other tier tests, so nothing else is+# compiled in and no model file is needed.+tests/test_qwen36_tier_int8_engine$(EXE): tests/test_qwen36_tier_int8_engine.c tests/qwen36_fake_cuda.h qwen36.c qwen36_tier.c qwen36_tier.h backend_cuda.h cli_args.h st.h json.h compat.h+ $(CC) $(CFLAGS) -DCOLI_CUDA $< -o $@ $(LDFLAGS)+tests/test_serve_poll$(EXE): tests/test_serve_poll.c serve_poll.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)diff --git a/c/qwen36.c b/c/qwen36.cindex 6fe13bd9..775addb8 100644--- a/c/qwen36.c+++ b/c/qwen36.c@@ -2645,6 +2645,59 @@ static void serve_loop(Model *m){}}+/* Warmstart body, lifted out of main so a test can drive it against an+ * in-memory model without a container (the QT_NO_WARMSTART check stays at the+ * call site). expert_is_int4 is what main probed from the on-disk expert size.+ */+static void tier_warmstart(Model *m, int expert_is_int4) {+ /* Plan the set (heat order), then load+stage IN PARALLEL. The+ * load path is thread-safe: expert_get locks the layer cache+ * (g_pilot_mx), st_read_raw uses pread; entries are unique. */+ double t0 = now_s();+ int cap_total = m->c.n_layers * m->c.n_experts;+ int *wpl = malloc((size_t)cap_total*sizeof(int));+ int *wpe = malloc((size_t)cap_total*sizeof(int));+ int wn = qt_plan_fill(wpl, wpe, cap_total);+ /* Load ALL experts into RAM, not just the planned (VRAM) set:+ * otherwise the first touch of a CPU-fallback expert triggers a+ * ~12 ms container read in the middle of decode (measured: 139+ * ms/token on a single-GPU run). Planned ones also go to VRAM. */+ uint8_t *planned = calloc((size_t)cap_total, 1);+ for (int i = 0; i < wn; i++) planned[wpl[i]*m->c.n_experts + wpe[i]] = 1;+ int keep8 = getenv("COLI_KEEP_INT8") != NULL;+ #pragma omp parallel for schedule(dynamic, 16)+ for (int gi = 0; gi < cap_total; gi++) {+ int l = gi / m->c.n_experts, eidw = gi % m->c.n_experts;+ Slot *e; expert_get(m, l, eidw, &e);+ /* int4: i puntatori impacchettati; int8: i pesi stessi. Prima+ * qui si esigeva e->g4, che su un container int8 e' NULL: la+ * promozione non partiva mai e il budget restava riservato a+ * vuoto (#1331). */+ const uint8_t *wg = expert_is_int4 ? e->g4 : (const uint8_t *)e->g;+ const uint8_t *wu = expert_is_int4 ? e->u4 : (const uint8_t *)e->u;+ const uint8_t *wd = expert_is_int4 ? e->d4 : (const uint8_t *)e->d;+ if (planned[gi] && wg) {+ qt_note_planned(l, eidw, wg, wu, wd, e->gs, e->us, e->ds);+ /* The staging copy is done. On an int4 container the int8 copy+ * can go RIGHT AWAY so it never shows up in peak RSS: g4/u4/d4+ * stay as the source of truth, and slot_ensure_int8() rebuilds+ * the int8 block on an LFRU eviction with no container access.+ * An int8 container has no second copy -- e->g4 is NULL -- so+ * freeing e->g there both dangles the pointer qt_note_planned+ * just stored (any later stage() reads freed memory) and leaves+ * slot_ensure_int8() unable to bring the expert back: it returns+ * early without g4, and the CPU fallback in the decode loop then+ * dereferences NULL. Keep it (#1341). COLI_KEEP_INT8 keeps its+ * meaning for int4. */+ if (!keep8 && expert_is_int4 && e->g) { free(e->g); e->g = e->u = e->d = NULL; }+ }+ }+ qt_fill_wait();+ free(wpl); free(wpe); free(planned);+ fprintf(stderr, "[qtier] warmstart (parallel): all %d experts in RAM (int8 only for non-residents), %d in VRAM -- %.1f s\n",+ cap_total, wn, now_s()-t0);+}+int main(int argc, char **argv) {const char *snap = getenv("SNAP");if (!snap) { coli_print_launcher_help("Qwen3.6"); return 1; }@@ -2782,47 +2835,7 @@ int main(int argc, char **argv) {* when HEAT_FILE exists, natural order otherwise), loading all RAM* slots along the way. */const char *nws = getenv("QT_NO_WARMSTART");- if (!(nws && *nws=='1')) {- /* Plan the set (heat order), then load+stage IN PARALLEL. The- * load path is thread-safe: expert_get locks the layer cache- * (g_pilot_mx), st_read_raw uses pread; entries are unique. */- double t0 = now_s();- int cap_total = m.c.n_layers * m.c.n_experts;- int *wpl = malloc((size_t)cap_total*sizeof(int));- int *wpe = malloc((size_t)cap_total*sizeof(int));- int wn = qt_plan_fill(wpl, wpe, cap_total);- /* Load ALL experts into RAM, not just the planned (VRAM) set:- * otherwise the first touch of a CPU-fallback expert triggers a- * ~12 ms container read in the middle of decode (measured: 139- * ms/token on a single-GPU run). Planned ones also go to VRAM. */- uint8_t *planned = calloc((size_t)cap_total, 1);- for (int i = 0; i < wn; i++) planned[wpl[i]*m.c.n_experts + wpe[i]] = 1;- int keep8 = getenv("COLI_KEEP_INT8") != NULL;- #pragma omp parallel for schedule(dynamic, 16)- for (int gi = 0; gi < cap_total; gi++) {- int l = gi / m.c.n_experts, eidw = gi % m.c.n_experts;- Slot *e; expert_get(&m, l, eidw, &e);- /* int4: i puntatori impacchettati; int8: i pesi stessi. Prima- * qui si esigeva e->g4, che su un container int8 e' NULL: la- * promozione non partiva mai e il budget restava riservato a- * vuoto (#1331). */- const uint8_t *wg = expert_is_int4 ? e->g4 : (const uint8_t *)e->g;- const uint8_t *wu = expert_is_int4 ? e->u4 : (const uint8_t *)e->u;- const uint8_t *wd = expert_is_int4 ? e->d4 : (const uint8_t *)e->d;- if (planned[gi] && wg) {- qt_note_planned(l, eidw, wg, wu, wd, e->gs, e->us, e->ds);- /* The staging copy is done; free the int8 copy RIGHT AWAY- * so it never shows up in peak RSS. On LFRU eviction- * slot_ensure_int8() rematerializes from g4 (no container- * access). */- if (!keep8 && e->g) { free(e->g); e->g = e->u = e->d = NULL; }- }- }- qt_fill_wait();- free(wpl); free(wpe); free(planned);- fprintf(stderr, "[qtier] warmstart (parallel): all %d experts in RAM (int8 only for non-residents), %d in VRAM -- %.1f s\n",- cap_total, wn, now_s()-t0);- }+ if (!(nws && *nws=='1')) tier_warmstart(&m, expert_is_int4);}/* coli serve mode: speak the gateway wire protocol instead of argvdiff --git a/c/tests/test_qwen36_tier_int8_engine.c b/c/tests/test_qwen36_tier_int8_engine.cnew file mode 100644index 00000000..d60ae8de--- /dev/null+++ b/c/tests/test_qwen36_tier_int8_engine.c@@ -0,0 +1,243 @@+/* The warmstart must not free the int8 weights the tier still points at (#1341).+ *+ * The defect lives in the engine, not the tier: the warmstart hands the tier+ * the live RAM weights -- `wg = expert_is_int4 ? e->g4 : (const uint8_t *)e->g`+ * -- and qt_note_planned parks that pointer in the tier's slot. The very next+ * statement used to free it unconditionally:+ *+ * if (!keep8 && e->g) { free(e->g); e->g = e->u = e->d = NULL; }+ *+ * with a comment claiming an LFRU eviction "rematerializes from g4". That is+ * true only for an int4 container. On an int8 container e->g4 is NULL: there is+ * no second copy, slot_ensure_int8() returns early (`if (s->g || !s->g4)+ * return;`), and the CPU fallback in the decode loop dereferences a NULL e->g.+ * The tier's stored pointer is dangling on top of that, so any later stage()+ * reads freed memory.+ *+ * Proving that needs the ENGINE, not just the tier -- the two earlier tier+ * tests would pass with the engine freeing everything underneath them. So this+ * one includes qwen36.c the way tests/test_qwen36_ctx.c does, builds an+ * in-memory int8 model with NO container behind it (every slot pre-populated+ * and indexed, so expert_get takes its hit path and never reads a file), and+ * drives tier_warmstart() directly against the fake CUDA backend shared with+ * the other tier tests.+ *+ * Include order matters: qwen36.c first (it never references coli_cuda_*),+ * then the fake backend, then qwen36_tier.c -- which puts the tier's own+ * statics (qs(), G) in this TU, so the test can read back the exact pointer+ * the tier kept. No symbol collides between the two sources.+ */+#define main qwen36_main_unused+#include "../qwen36.c"+#undef main++/* compat.h (pulled in by qwen36.c) maps setenv/unsetenv to+ * SetEnvironmentVariableA, which updates the Win32 environment block -- but+ * getenv() reads the CRT's own copy and never sees it, so the tier would+ * silently not get COLI_CUDA here. Drop that mapping before the fake backend+ * installs its _putenv_s version, and unset the same way. Same reasoning as+ * tests/test_qwen36_ctx.c. */+#ifdef _WIN32+#undef setenv+#undef unsetenv+#define unsetenv(name) _putenv_s(name, "")+#endif++#include "qwen36_fake_cuda.h"++#include "../qwen36_tier.c"++static int fails;+static void ck(int ok, const char *what) {+ if (ok) { printf(" ok %s\n", what); return; }+ printf(" FAIL %s\n", what);+ fails++;+}++enum { NL = 1, NE = 4, EXP_D = 64, EXP_IH = 32, TOPK = 1 };+enum { NG = EXP_IH * EXP_D, ND = EXP_D * EXP_IH }; /* gate/up and down elements */++/* Byte the setup writes at element i of matrix `which` (0=g,1=u,2=d) of expert+ * eid. Kept in int4 range so the same generator can feed the packed model. */+static int8_t want_byte(int eid, int which, int64_t i) {+ return (int8_t)(((int)i + eid * 3 + which * 5) % 15 - 7);+}++/* An in-memory model with one layer of int8 experts and no container: cap ==+ * n_experts (what qt_init demands) and every slot published in the layer's+ * index, so expert_get() hits and load_expert_merged() is never reached. */+static void build_model(Model *m) {+ memset(m, 0, sizeof *m);+ m->c.n_layers = NL; m->c.n_experts = NE;+ m->c.hidden = EXP_D; m->c.inter = EXP_IH;+ m->c.topk = TOPK; m->c.expert_gs = 0;+ m->active_of = calloc(NL, sizeof(int));+ m->is_pinned = calloc((size_t)NL * NE, 1);+ m->is_queued = calloc((size_t)NL * NE, 1);+ m->cache = calloc(NL, sizeof(LCache));+ LCache *lc = &m->cache[0];+ lc->cap = NE; lc->n = NE;+ lc->slots = calloc(NE, sizeof(Slot));+ lc->slot_by_expert = malloc(NE * sizeof(int));+ for (int e = 0; e < NE; e++) {+ Slot *s = &lc->slots[e];+ slot_ensure_allocated(m, s); /* the engine's own g|u|d block */+ s->eid = e;+ s->pinned = 1; /* nothing may evict during the test */+ s->used = ++m->clock;+ lc->slot_by_expert[e] = e;+ }+}++static void fill_int8(Model *m) {+ for (int e = 0; e < NE; e++) {+ Slot *s = &m->cache[0].slots[e];+ for (int64_t i = 0; i < NG; i++) { s->g[i] = want_byte(e, 0, i); s->u[i] = want_byte(e, 1, i); }+ for (int64_t i = 0; i < ND; i++) s->d[i] = want_byte(e, 2, i);+ for (int64_t i = 0; i < EXP_IH; i++) { s->gs[i] = 1.f; s->us[i] = 1.f; }+ for (int64_t i = 0; i < EXP_D; i++) s->ds[i] = 1.f;+ }+}++/* Same weights, but stored the way an int4 container stores them: packed+ * g4|u4|d4 as the source of truth plus the unpacked int8 copy, exactly as+ * load_expert_merged() leaves a slot when the tier is running. */+static void fill_int4(Model *m) {+ int64_t want_w = NG + NG + ND;+ uint8_t *raw = malloc((size_t)(want_w / 2));+ for (int e = 0; e < NE; e++) {+ Slot *s = &m->cache[0].slots[e];+ for (int64_t i = 0; i < want_w; i += 2) {+ int which = i < NG ? 0 : (i < 2 * NG ? 1 : 2);+ int64_t off = i - (which == 0 ? 0 : (which == 1 ? NG : 2 * NG));+ uint8_t lo = (uint8_t)(want_byte(e, which, off) & 0xF);+ uint8_t hi = (uint8_t)(want_byte(e, which, off + 1) & 0xF);+ raw[i >> 1] = (uint8_t)(lo | (hi << 4));+ }+ unpack_int4_to_int8(s->g, raw, want_w);+ s->is_int4 = 1;+ int64_t gp = NG / 2, up = NG / 2, dp = ND / 2;+ s->g4 = malloc((size_t)gp); s->u4 = malloc((size_t)up); s->d4 = malloc((size_t)dp);+ memcpy(s->g4, raw, (size_t)gp);+ memcpy(s->u4, raw + gp, (size_t)up);+ memcpy(s->d4, raw + gp + up, (size_t)dp);+ for (int64_t i = 0; i < EXP_IH; i++) { s->gs[i] = 1.f; s->us[i] = 1.f; }+ for (int64_t i = 0; i < EXP_D; i++) s->ds[i] = 1.f;+ }+ free(raw);+}++static void free_model(Model *m) {+ LCache *lc = &m->cache[0];+ for (int e = 0; e < NE; e++) {+ Slot *s = &lc->slots[e];+ free(s->g); free(s->gs);+ free(s->g4); free(s->u4); free(s->d4);+ }+ free(lc->slots); free(lc->slot_by_expert);+ free(m->cache); free(m->active_of); free(m->is_pinned); free(m->is_queued);+}++/* qt_fill_wait() returns when the queue is empty, and the uploader dequeues+ * BEFORE uploading -- so wait on the observable end state instead. */+static int wait_resident(void) {+ for (int i = 0; i < 500; i++) {+ int all = 1;+ for (int e = 0; e < NE; e++) if (!qt_is_resident(0, e)) all = 0;+ if (all) return 1;+ struct timespec ts = {0, 2000000}; /* 2 ms */+ nanosleep(&ts, NULL);+ }+ return 0;+}++/* Every weight the warmstart touched must still read back as written: an int8+ * expert has no second copy to rebuild it from. */+static int bytes_intact(Slot *s, int eid) {+ for (int64_t i = 0; i < NG; i++)+ if (s->g[i] != want_byte(eid, 0, i) || s->u[i] != want_byte(eid, 1, i)) return 0;+ for (int64_t i = 0; i < ND; i++)+ if (s->d[i] != want_byte(eid, 2, i)) return 0;+ return 1;+}++/* --- an int8 container: the warmstart must leave the weights computable ---- */+static void case_int8(void) {+ printf("int8 container\n");+ Model m; build_model(&m); fill_int8(&m);++ setenv("COLI_CUDA", "1", 1);+ setenv("COLI_GPUS", "0", 1);+ unsetenv("QT_NO_WARMSTART");+ unsetenv("COLI_KEEP_INT8");+ fake_uploads = 0;++ if (!qt_init(NL, NE, EXP_D, EXP_IH, NE, TOPK, 0 /* per-row */, 0 /* int8 */)) {+ printf(" FAIL the tier refuses a per-row int8 container\n");+ fails++; free_model(&m); return;+ }+ tier_warmstart(&m, 0);+ qt_fill_wait();++ ck(wait_resident(), "every planned int8 expert reached VRAM");+ ck(fake_uploads == 3 * NE, "three uploads per planned expert (gate, up, down)");++ int computable = 1, intact = 1, tier_live = 1;+ for (int e = 0; e < NE; e++) {+ Slot *s; expert_get(&m, 0, e, &s);+ slot_ensure_int8(&m, s); /* the decode loop's CPU fallback */+ if (!s->g || !s->u || !s->d) { computable = 0; intact = 0; }+ else if (!bytes_intact(s, e)) intact = 0;+ if (qs(0, e)->g4 != (const uint8_t *)s->g) tier_live = 0;+ }+ /* outcome: planned_int8_experts_stay_computable_on_cpu */+ ck(computable, "a planned int8 expert still has weights for the CPU fallback");+ ck(intact, "those weights are the bytes the loader wrote");+ /* outcome: tier_pointer_still_targets_live_weights */+ ck(tier_live, "the pointer the tier kept still targets the live int8 block");++ qt_shutdown();+ free_model(&m);+}++/* --- an int4 container: the RSS optimisation must survive the fix ---------- */+static void case_int4(void) {+ printf("int4 container\n");+ Model m; build_model(&m); fill_int4(&m);++ setenv("COLI_CUDA", "1", 1);+ setenv("COLI_GPUS", "0", 1);+ unsetenv("QT_NO_WARMSTART");+ unsetenv("COLI_KEEP_INT8");+ fake_uploads = 0;++ if (!qt_init(NL, NE, EXP_D, EXP_IH, NE, TOPK, 0 /* per-row */, 1 /* int4 */)) {+ printf(" FAIL regression: the tier no longer starts on an int4 container\n");+ fails++; free_model(&m); return;+ }+ tier_warmstart(&m, 1);+ qt_fill_wait();+ ck(wait_resident(), "every planned int4 expert reached VRAM");++ int freed = 1, remat = 1;+ for (int e = 0; e < NE; e++) {+ Slot *s; expert_get(&m, 0, e, &s);+ if (s->g) freed = 0;+ slot_ensure_int8(&m, s);+ if (!s->g || !bytes_intact(s, e)) remat = 0;+ }+ /* outcome: int4_path_still_frees_its_second_copy */+ ck(freed, "int4 still drops the int8 copy right after staging (peak RSS)");+ ck(remat, "slot_ensure_int8 rebuilds it from the packed g4/u4/d4");++ qt_shutdown();+ free_model(&m);+}++int main(void) {+ case_int8();+ case_int4();+ if (fails) { printf("FAILED %d\n", fails); return 1; }+ printf("OK test_qwen36_tier_int8_engine\n");+ return 0;+}