Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(qwen36): keep int8 expert weights the tier still points at (#1341)

Courtney Andrew Richardsond937ad233b16
diff --git a/c/Makefile b/c/Makefile
index 1c61bd51..39859b64 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -1656,6 +1656,15 @@ tests/test_qwen36_tier_multidev$(EXE): tests/test_qwen36_tier_multidev.c tests/q
tests/test_qwen36_tier_shutdown$(EXE): tests/test_qwen36_tier_shutdown.c tests/qwen36_fake_cuda.h qwen36_tier.c qwen36_tier.h backend_cuda.h
$(CC) $(CFLAGS) -DCOLI_CUDA $< -o $@ $(LDFLAGS)
+# Same fake backend, but driving the ENGINE: the warmstart in qwen36.c hands the
+# tier raw pointers into the RAM expert slots, so only a test that includes
+# qwen36.c can prove the weights behind them are still there afterwards -- on an
+# int8 container there is no packed copy to rebuild them from (#1341). Includes
+# qwen36_tier.c in its own TU like the other tier tests, so nothing else is
+# compiled in and no model file is needed.
+tests/test_qwen36_tier_int8_engine$(EXE): tests/test_qwen36_tier_int8_engine.c tests/qwen36_fake_cuda.h qwen36.c qwen36_tier.c qwen36_tier.h backend_cuda.h cli_args.h st.h json.h compat.h
+ $(CC) $(CFLAGS) -DCOLI_CUDA $< -o $@ $(LDFLAGS)
+
tests/test_serve_poll$(EXE): tests/test_serve_poll.c serve_poll.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
diff --git a/c/qwen36.c b/c/qwen36.c
index 6fe13bd9..775addb8 100644
--- a/c/qwen36.c
+++ b/c/qwen36.c
@@ -2645,6 +2645,59 @@ static void serve_loop(Model *m){
}
}
+/* Warmstart body, lifted out of main so a test can drive it against an
+ * in-memory model without a container (the QT_NO_WARMSTART check stays at the
+ * call site). expert_is_int4 is what main probed from the on-disk expert size.
+ */
+static void tier_warmstart(Model *m, int expert_is_int4) {
+ /* Plan the set (heat order), then load+stage IN PARALLEL. The
+ * load path is thread-safe: expert_get locks the layer cache
+ * (g_pilot_mx), st_read_raw uses pread; entries are unique. */
+ double t0 = now_s();
+ int cap_total = m->c.n_layers * m->c.n_experts;
+ int *wpl = malloc((size_t)cap_total*sizeof(int));
+ int *wpe = malloc((size_t)cap_total*sizeof(int));
+ int wn = qt_plan_fill(wpl, wpe, cap_total);
+ /* Load ALL experts into RAM, not just the planned (VRAM) set:
+ * otherwise the first touch of a CPU-fallback expert triggers a
+ * ~12 ms container read in the middle of decode (measured: 139
+ * ms/token on a single-GPU run). Planned ones also go to VRAM. */
+ uint8_t *planned = calloc((size_t)cap_total, 1);
+ for (int i = 0; i < wn; i++) planned[wpl[i]*m->c.n_experts + wpe[i]] = 1;
+ int keep8 = getenv("COLI_KEEP_INT8") != NULL;
+ #pragma omp parallel for schedule(dynamic, 16)
+ for (int gi = 0; gi < cap_total; gi++) {
+ int l = gi / m->c.n_experts, eidw = gi % m->c.n_experts;
+ Slot *e; expert_get(m, l, eidw, &e);
+ /* int4: i puntatori impacchettati; int8: i pesi stessi. Prima
+ * qui si esigeva e->g4, che su un container int8 e' NULL: la
+ * promozione non partiva mai e il budget restava riservato a
+ * vuoto (#1331). */
+ const uint8_t *wg = expert_is_int4 ? e->g4 : (const uint8_t *)e->g;
+ const uint8_t *wu = expert_is_int4 ? e->u4 : (const uint8_t *)e->u;
+ const uint8_t *wd = expert_is_int4 ? e->d4 : (const uint8_t *)e->d;
+ if (planned[gi] && wg) {
+ qt_note_planned(l, eidw, wg, wu, wd, e->gs, e->us, e->ds);
+ /* The staging copy is done. On an int4 container the int8 copy
+ * can go RIGHT AWAY so it never shows up in peak RSS: g4/u4/d4
+ * stay as the source of truth, and slot_ensure_int8() rebuilds
+ * the int8 block on an LFRU eviction with no container access.
+ * An int8 container has no second copy -- e->g4 is NULL -- so
+ * freeing e->g there both dangles the pointer qt_note_planned
+ * just stored (any later stage() reads freed memory) and leaves
+ * slot_ensure_int8() unable to bring the expert back: it returns
+ * early without g4, and the CPU fallback in the decode loop then
+ * dereferences NULL. Keep it (#1341). COLI_KEEP_INT8 keeps its
+ * meaning for int4. */
+ if (!keep8 && expert_is_int4 && e->g) { free(e->g); e->g = e->u = e->d = NULL; }
+ }
+ }
+ qt_fill_wait();
+ free(wpl); free(wpe); free(planned);
+ fprintf(stderr, "[qtier] warmstart (parallel): all %d experts in RAM (int8 only for non-residents), %d in VRAM -- %.1f s\n",
+ cap_total, wn, now_s()-t0);
+}
+
int main(int argc, char **argv) {
const char *snap = getenv("SNAP");
if (!snap) { coli_print_launcher_help("Qwen3.6"); return 1; }
@@ -2782,47 +2835,7 @@ int main(int argc, char **argv) {
* when HEAT_FILE exists, natural order otherwise), loading all RAM
* slots along the way. */
const char *nws = getenv("QT_NO_WARMSTART");
- if (!(nws && *nws=='1')) {
- /* Plan the set (heat order), then load+stage IN PARALLEL. The
- * load path is thread-safe: expert_get locks the layer cache
- * (g_pilot_mx), st_read_raw uses pread; entries are unique. */
- double t0 = now_s();
- int cap_total = m.c.n_layers * m.c.n_experts;
- int *wpl = malloc((size_t)cap_total*sizeof(int));
- int *wpe = malloc((size_t)cap_total*sizeof(int));
- int wn = qt_plan_fill(wpl, wpe, cap_total);
- /* Load ALL experts into RAM, not just the planned (VRAM) set:
- * otherwise the first touch of a CPU-fallback expert triggers a
- * ~12 ms container read in the middle of decode (measured: 139
- * ms/token on a single-GPU run). Planned ones also go to VRAM. */
- uint8_t *planned = calloc((size_t)cap_total, 1);
- for (int i = 0; i < wn; i++) planned[wpl[i]*m.c.n_experts + wpe[i]] = 1;
- int keep8 = getenv("COLI_KEEP_INT8") != NULL;
- #pragma omp parallel for schedule(dynamic, 16)
- for (int gi = 0; gi < cap_total; gi++) {
- int l = gi / m.c.n_experts, eidw = gi % m.c.n_experts;
- Slot *e; expert_get(&m, l, eidw, &e);
- /* int4: i puntatori impacchettati; int8: i pesi stessi. Prima
- * qui si esigeva e->g4, che su un container int8 e' NULL: la
- * promozione non partiva mai e il budget restava riservato a
- * vuoto (#1331). */
- const uint8_t *wg = expert_is_int4 ? e->g4 : (const uint8_t *)e->g;
- const uint8_t *wu = expert_is_int4 ? e->u4 : (const uint8_t *)e->u;
- const uint8_t *wd = expert_is_int4 ? e->d4 : (const uint8_t *)e->d;
- if (planned[gi] && wg) {
- qt_note_planned(l, eidw, wg, wu, wd, e->gs, e->us, e->ds);
- /* The staging copy is done; free the int8 copy RIGHT AWAY
- * so it never shows up in peak RSS. On LFRU eviction
- * slot_ensure_int8() rematerializes from g4 (no container
- * access). */
- if (!keep8 && e->g) { free(e->g); e->g = e->u = e->d = NULL; }
- }
- }
- qt_fill_wait();
- free(wpl); free(wpe); free(planned);
- fprintf(stderr, "[qtier] warmstart (parallel): all %d experts in RAM (int8 only for non-residents), %d in VRAM -- %.1f s\n",
- cap_total, wn, now_s()-t0);
- }
+ if (!(nws && *nws=='1')) tier_warmstart(&m, expert_is_int4);
}
/* coli serve mode: speak the gateway wire protocol instead of argv
diff --git a/c/tests/test_qwen36_tier_int8_engine.c b/c/tests/test_qwen36_tier_int8_engine.c
new file mode 100644
index 00000000..d60ae8de
--- /dev/null
+++ b/c/tests/test_qwen36_tier_int8_engine.c
@@ -0,0 +1,243 @@
+/* The warmstart must not free the int8 weights the tier still points at (#1341).
+ *
+ * The defect lives in the engine, not the tier: the warmstart hands the tier
+ * the live RAM weights -- `wg = expert_is_int4 ? e->g4 : (const uint8_t *)e->g`
+ * -- and qt_note_planned parks that pointer in the tier's slot. The very next
+ * statement used to free it unconditionally:
+ *
+ * if (!keep8 && e->g) { free(e->g); e->g = e->u = e->d = NULL; }
+ *
+ * with a comment claiming an LFRU eviction "rematerializes from g4". That is
+ * true only for an int4 container. On an int8 container e->g4 is NULL: there is
+ * no second copy, slot_ensure_int8() returns early (`if (s->g || !s->g4)
+ * return;`), and the CPU fallback in the decode loop dereferences a NULL e->g.
+ * The tier's stored pointer is dangling on top of that, so any later stage()
+ * reads freed memory.
+ *
+ * Proving that needs the ENGINE, not just the tier -- the two earlier tier
+ * tests would pass with the engine freeing everything underneath them. So this
+ * one includes qwen36.c the way tests/test_qwen36_ctx.c does, builds an
+ * in-memory int8 model with NO container behind it (every slot pre-populated
+ * and indexed, so expert_get takes its hit path and never reads a file), and
+ * drives tier_warmstart() directly against the fake CUDA backend shared with
+ * the other tier tests.
+ *
+ * Include order matters: qwen36.c first (it never references coli_cuda_*),
+ * then the fake backend, then qwen36_tier.c -- which puts the tier's own
+ * statics (qs(), G) in this TU, so the test can read back the exact pointer
+ * the tier kept. No symbol collides between the two sources.
+ */
+#define main qwen36_main_unused
+#include "../qwen36.c"
+#undef main
+
+/* compat.h (pulled in by qwen36.c) maps setenv/unsetenv to
+ * SetEnvironmentVariableA, which updates the Win32 environment block -- but
+ * getenv() reads the CRT's own copy and never sees it, so the tier would
+ * silently not get COLI_CUDA here. Drop that mapping before the fake backend
+ * installs its _putenv_s version, and unset the same way. Same reasoning as
+ * tests/test_qwen36_ctx.c. */
+#ifdef _WIN32
+#undef setenv
+#undef unsetenv
+#define unsetenv(name) _putenv_s(name, "")
+#endif
+
+#include "qwen36_fake_cuda.h"
+
+#include "../qwen36_tier.c"
+
+static int fails;
+static void ck(int ok, const char *what) {
+ if (ok) { printf(" ok %s\n", what); return; }
+ printf(" FAIL %s\n", what);
+ fails++;
+}
+
+enum { NL = 1, NE = 4, EXP_D = 64, EXP_IH = 32, TOPK = 1 };
+enum { NG = EXP_IH * EXP_D, ND = EXP_D * EXP_IH }; /* gate/up and down elements */
+
+/* Byte the setup writes at element i of matrix `which` (0=g,1=u,2=d) of expert
+ * eid. Kept in int4 range so the same generator can feed the packed model. */
+static int8_t want_byte(int eid, int which, int64_t i) {
+ return (int8_t)(((int)i + eid * 3 + which * 5) % 15 - 7);
+}
+
+/* An in-memory model with one layer of int8 experts and no container: cap ==
+ * n_experts (what qt_init demands) and every slot published in the layer's
+ * index, so expert_get() hits and load_expert_merged() is never reached. */
+static void build_model(Model *m) {
+ memset(m, 0, sizeof *m);
+ m->c.n_layers = NL; m->c.n_experts = NE;
+ m->c.hidden = EXP_D; m->c.inter = EXP_IH;
+ m->c.topk = TOPK; m->c.expert_gs = 0;
+ m->active_of = calloc(NL, sizeof(int));
+ m->is_pinned = calloc((size_t)NL * NE, 1);
+ m->is_queued = calloc((size_t)NL * NE, 1);
+ m->cache = calloc(NL, sizeof(LCache));
+ LCache *lc = &m->cache[0];
+ lc->cap = NE; lc->n = NE;
+ lc->slots = calloc(NE, sizeof(Slot));
+ lc->slot_by_expert = malloc(NE * sizeof(int));
+ for (int e = 0; e < NE; e++) {
+ Slot *s = &lc->slots[e];
+ slot_ensure_allocated(m, s); /* the engine's own g|u|d block */
+ s->eid = e;
+ s->pinned = 1; /* nothing may evict during the test */
+ s->used = ++m->clock;
+ lc->slot_by_expert[e] = e;
+ }
+}
+
+static void fill_int8(Model *m) {
+ for (int e = 0; e < NE; e++) {
+ Slot *s = &m->cache[0].slots[e];
+ for (int64_t i = 0; i < NG; i++) { s->g[i] = want_byte(e, 0, i); s->u[i] = want_byte(e, 1, i); }
+ for (int64_t i = 0; i < ND; i++) s->d[i] = want_byte(e, 2, i);
+ for (int64_t i = 0; i < EXP_IH; i++) { s->gs[i] = 1.f; s->us[i] = 1.f; }
+ for (int64_t i = 0; i < EXP_D; i++) s->ds[i] = 1.f;
+ }
+}
+
+/* Same weights, but stored the way an int4 container stores them: packed
+ * g4|u4|d4 as the source of truth plus the unpacked int8 copy, exactly as
+ * load_expert_merged() leaves a slot when the tier is running. */
+static void fill_int4(Model *m) {
+ int64_t want_w = NG + NG + ND;
+ uint8_t *raw = malloc((size_t)(want_w / 2));
+ for (int e = 0; e < NE; e++) {
+ Slot *s = &m->cache[0].slots[e];
+ for (int64_t i = 0; i < want_w; i += 2) {
+ int which = i < NG ? 0 : (i < 2 * NG ? 1 : 2);
+ int64_t off = i - (which == 0 ? 0 : (which == 1 ? NG : 2 * NG));
+ uint8_t lo = (uint8_t)(want_byte(e, which, off) & 0xF);
+ uint8_t hi = (uint8_t)(want_byte(e, which, off + 1) & 0xF);
+ raw[i >> 1] = (uint8_t)(lo | (hi << 4));
+ }
+ unpack_int4_to_int8(s->g, raw, want_w);
+ s->is_int4 = 1;
+ int64_t gp = NG / 2, up = NG / 2, dp = ND / 2;
+ s->g4 = malloc((size_t)gp); s->u4 = malloc((size_t)up); s->d4 = malloc((size_t)dp);
+ memcpy(s->g4, raw, (size_t)gp);
+ memcpy(s->u4, raw + gp, (size_t)up);
+ memcpy(s->d4, raw + gp + up, (size_t)dp);
+ for (int64_t i = 0; i < EXP_IH; i++) { s->gs[i] = 1.f; s->us[i] = 1.f; }
+ for (int64_t i = 0; i < EXP_D; i++) s->ds[i] = 1.f;
+ }
+ free(raw);
+}
+
+static void free_model(Model *m) {
+ LCache *lc = &m->cache[0];
+ for (int e = 0; e < NE; e++) {
+ Slot *s = &lc->slots[e];
+ free(s->g); free(s->gs);
+ free(s->g4); free(s->u4); free(s->d4);
+ }
+ free(lc->slots); free(lc->slot_by_expert);
+ free(m->cache); free(m->active_of); free(m->is_pinned); free(m->is_queued);
+}
+
+/* qt_fill_wait() returns when the queue is empty, and the uploader dequeues
+ * BEFORE uploading -- so wait on the observable end state instead. */
+static int wait_resident(void) {
+ for (int i = 0; i < 500; i++) {
+ int all = 1;
+ for (int e = 0; e < NE; e++) if (!qt_is_resident(0, e)) all = 0;
+ if (all) return 1;
+ struct timespec ts = {0, 2000000}; /* 2 ms */
+ nanosleep(&ts, NULL);
+ }
+ return 0;
+}
+
+/* Every weight the warmstart touched must still read back as written: an int8
+ * expert has no second copy to rebuild it from. */
+static int bytes_intact(Slot *s, int eid) {
+ for (int64_t i = 0; i < NG; i++)
+ if (s->g[i] != want_byte(eid, 0, i) || s->u[i] != want_byte(eid, 1, i)) return 0;
+ for (int64_t i = 0; i < ND; i++)
+ if (s->d[i] != want_byte(eid, 2, i)) return 0;
+ return 1;
+}
+
+/* --- an int8 container: the warmstart must leave the weights computable ---- */
+static void case_int8(void) {
+ printf("int8 container\n");
+ Model m; build_model(&m); fill_int8(&m);
+
+ setenv("COLI_CUDA", "1", 1);
+ setenv("COLI_GPUS", "0", 1);
+ unsetenv("QT_NO_WARMSTART");
+ unsetenv("COLI_KEEP_INT8");
+ fake_uploads = 0;
+
+ if (!qt_init(NL, NE, EXP_D, EXP_IH, NE, TOPK, 0 /* per-row */, 0 /* int8 */)) {
+ printf(" FAIL the tier refuses a per-row int8 container\n");
+ fails++; free_model(&m); return;
+ }
+ tier_warmstart(&m, 0);
+ qt_fill_wait();
+
+ ck(wait_resident(), "every planned int8 expert reached VRAM");
+ ck(fake_uploads == 3 * NE, "three uploads per planned expert (gate, up, down)");
+
+ int computable = 1, intact = 1, tier_live = 1;
+ for (int e = 0; e < NE; e++) {
+ Slot *s; expert_get(&m, 0, e, &s);
+ slot_ensure_int8(&m, s); /* the decode loop's CPU fallback */
+ if (!s->g || !s->u || !s->d) { computable = 0; intact = 0; }
+ else if (!bytes_intact(s, e)) intact = 0;
+ if (qs(0, e)->g4 != (const uint8_t *)s->g) tier_live = 0;
+ }
+ /* outcome: planned_int8_experts_stay_computable_on_cpu */
+ ck(computable, "a planned int8 expert still has weights for the CPU fallback");
+ ck(intact, "those weights are the bytes the loader wrote");
+ /* outcome: tier_pointer_still_targets_live_weights */
+ ck(tier_live, "the pointer the tier kept still targets the live int8 block");
+
+ qt_shutdown();
+ free_model(&m);
+}
+
+/* --- an int4 container: the RSS optimisation must survive the fix ---------- */
+static void case_int4(void) {
+ printf("int4 container\n");
+ Model m; build_model(&m); fill_int4(&m);
+
+ setenv("COLI_CUDA", "1", 1);
+ setenv("COLI_GPUS", "0", 1);
+ unsetenv("QT_NO_WARMSTART");
+ unsetenv("COLI_KEEP_INT8");
+ fake_uploads = 0;
+
+ if (!qt_init(NL, NE, EXP_D, EXP_IH, NE, TOPK, 0 /* per-row */, 1 /* int4 */)) {
+ printf(" FAIL regression: the tier no longer starts on an int4 container\n");
+ fails++; free_model(&m); return;
+ }
+ tier_warmstart(&m, 1);
+ qt_fill_wait();
+ ck(wait_resident(), "every planned int4 expert reached VRAM");
+
+ int freed = 1, remat = 1;
+ for (int e = 0; e < NE; e++) {
+ Slot *s; expert_get(&m, 0, e, &s);
+ if (s->g) freed = 0;
+ slot_ensure_int8(&m, s);
+ if (!s->g || !bytes_intact(s, e)) remat = 0;
+ }
+ /* outcome: int4_path_still_frees_its_second_copy */
+ ck(freed, "int4 still drops the int8 copy right after staging (peak RSS)");
+ ck(remat, "slot_ensure_int8 rebuilds it from the packed g4/u4/d4");
+
+ qt_shutdown();
+ free_model(&m);
+}
+
+int main(void) {
+ case_int8();
+ case_int4();
+ if (fails) { printf("FAILED %d\n", fails); return 1; }
+ printf("OK test_qwen36_tier_int8_engine\n");
+ return 0;
+}