Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(qwen36 tier): offer int8 experts on the decode path (#1391)

Petteri Kosonenaa4629e546b6
diff --git a/c/Makefile b/c/Makefile
index 0c3edf8b..1835fe8d 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -1711,6 +1711,10 @@ tests/test_qwen36_tier_fp8$(EXE): tests/test_qwen36_tier_fp8.c tests/qwen36_fake
tests/test_qwen36_tier_int8_engine$(EXE): tests/test_qwen36_tier_int8_engine.c tests/qwen36_fake_cuda.h qwen36.c qwen36_tier.c qwen36_tier.h backend_cuda.h cli_args.h st.h json.h compat.h
$(CC) $(CFLAGS) -DCOLI_CUDA $< -o $@ $(LDFLAGS)
+# #1391: il decode path deve offrire gli esperti int8 al tier, non solo il warmstart
+tests/test_qwen36_tier_int8_decode$(EXE): tests/test_qwen36_tier_int8_decode.c tests/qwen36_fake_cuda.h qwen36.c qwen36_tier.c qwen36_tier.h backend_cuda.h cli_args.h st.h json.h compat.h
+ $(CC) $(CFLAGS) -DCOLI_CUDA $< -o $@ $(LDFLAGS)
+
tests/test_serve_poll$(EXE): tests/test_serve_poll.c serve_poll.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
diff --git a/c/qwen36.c b/c/qwen36.c
index c439909b..54b58736 100644
--- a/c/qwen36.c
+++ b/c/qwen36.c
@@ -955,6 +955,10 @@ static void matmul_q_batch(float *y, const float *x, const int8_t *q,
/* Group-scaled int8 GEMV: one f32 scale per `gs` input elements per row
* (gs64 expert containers). Row layout of `scale`: [O][I/gs] row-major. */
static int g_expert_gs = 0; /* set from qwen36_meta.json (expert_gs) at load */
+/* 1 = expert container packs int4 (tier fmt=4); 0 = int8 per-row (tier fmt=1).
+ * Same signal main's nbytes probe and tier_warmstart receive; the decode path
+ * needs it to offer int8 experts (#1391): on an int8 container e->g4 is NULL. */
+static int g_expert_is_int4 = 1;
static void matmul_q_gs(float *y, const float *x, const int8_t *q, const float *scale,
int I, int O, int gs) {
int ng = (I + gs - 1) / gs;
@@ -1966,7 +1970,17 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
* collect the GPU results. */
for (int kk = 0; kk < K; kk++) {
Slot *e; expert_get(m, layer, idx[kk], &e);
- if (e->g4) qt_note(layer, idx[kk], e->g4, e->u4, e->d4, e->gs, e->us, e->ds);
+ /* Offer whichever format the container actually packed. The old
+ * gate `if (e->g4)` never fired on an int8 container (g4 is
+ * NULL there), so the tier stayed at 0 uploads for the life of
+ * the process (#1391). Same pointer choice tier_warmstart
+ * makes: int4 → packed, int8 → the weights themselves. The
+ * priority on g4 first keeps mid-flight format flips harmless. */
+ if (e->g4)
+ qt_note(layer, idx[kk], e->g4, e->u4, e->d4, e->gs, e->us, e->ds);
+ else if (!g_expert_is_int4 && e->g)
+ qt_note(layer, idx[kk], (const uint8_t *)e->g, (const uint8_t *)e->u,
+ (const uint8_t *)e->d, e->gs, e->us, e->ds);
}
double _q0 = tm_now();
uint32_t qmask = qt_issue(layer, idx, K, xs);
@@ -2371,7 +2385,14 @@ static void pilot_prefetch(Model *m, int lnext, const float *x, int S) {
/* Lookahead: RAM-resident layer-L+1 candidates go to VRAM asynchronously */
if (found && fz >= 0 && qt_ready()) {
Slot *ps = &lc->slots[fz];
- if (ps->g4) qt_note(lnext, eid, ps->g4, ps->u4, ps->d4, ps->gs, ps->us, ps->ds);
+ /* Same int8 container case as the resident offer in moe():
+ * on an int8 container ps->g4 is NULL and the prefetch offer
+ * never fired (#1391). Offer the int8 weights instead. */
+ if (ps->g4)
+ qt_note(lnext, eid, ps->g4, ps->u4, ps->d4, ps->gs, ps->us, ps->ds);
+ else if (!g_expert_is_int4 && ps->g)
+ qt_note(lnext, eid, (const uint8_t *)ps->g, (const uint8_t *)ps->u,
+ (const uint8_t *)ps->d, ps->gs, ps->us, ps->ds);
}
if (!found) {
int gidx = lnext*E + eid;
@@ -2933,6 +2954,7 @@ int main(int argc, char **argv) {
* (CI sul container tiny int8, #1331) senza una scheda. */
fprintf(stderr, "[qwen36] expert format on disk: %s\n",
expert_is_int4 ? "int4 packed (tier fmt=4)" : "int8 (tier fmt=1)");
+ g_expert_is_int4 = expert_is_int4;
/* Offer the dense trunk to the placer before the tier decides its budget:
* sizes only, from the same dense-i8 entries the uploads below will use.
* No entry (dense-i8 off) means nothing to offer, and the CPU path stands. */
diff --git a/c/tests/test_qwen36_tier_int8_decode.c b/c/tests/test_qwen36_tier_int8_decode.c
new file mode 100644
index 00000000..cf344ef6
--- /dev/null
+++ b/c/tests/test_qwen36_tier_int8_decode.c
@@ -0,0 +1,233 @@
+/* The decode path must offer int8 experts to the tier, not only int4 (#1391).
+ *
+ * #1331 fixed the warmstart: `wg = expert_is_int4 ? e->g4 : (const uint8_t *)e->g`
+ * hands the tier the live RAM weights on an int8 container. The two decode-path
+ * offer sites were not given the same choice -- `if (e->g4) qt_note(...)` in
+ * moe()'s resident offer and `if (ps->g4) qt_note(...)` in the pilot-prefetch
+ * lookahead -- so with QT_NO_WARMSTART=1 on an int8 container nothing was ever
+ * enqueued: the tier reported itself active and sat at 0 uploads, 0 % hit rate
+ * for the life of the process.
+ *
+ * Same shape as the parent test: include qwen36.c (with main renamed away),
+ * the fake CUDA backend, then qwen36_tier.c, so the test can read the tier's
+ * own statics. The model is in-memory with NO container: every slot is
+ * pre-populated and indexed so expert_get() hits and never reads a file.
+ *
+ * The moe() call sites need a Model in step() state, which drags in the whole
+ * attention/DeltaNet machinery. The offer code, however, is two identical
+ * three-line blocks around one qt_note call; what the fix changes is exactly
+ * the pointer choice those sites make, and both sites now share that choice
+ * with tier_warmstart. This test drives the same offer decision the decode
+ * sites make, against the same Slot state, the same way the parent test drives
+ * tier_warmstart: call the mirror expression directly on a real Slot, assert
+ * the tier accepted an int8 pointer (fake_uploads > 0) and that the resident
+ * set grows -- the assertion from #1391 that fails on the old expression.
+ */
+#define main qwen36_main_unused
+#include "../qwen36.c"
+#undef main
+
+#ifdef _WIN32
+#undef setenv
+#undef unsetenv
+#define unsetenv(name) _putenv_s(name, "")
+#endif
+
+#include "qwen36_fake_cuda.h"
+
+#include "../qwen36_tier.c"
+
+static int fails;
+static void ck(int ok, const char *what) {
+ if (ok) { printf(" ok %s\n", what); return; }
+ printf(" FAIL %s\n", what);
+ fails++;
+}
+
+enum { NL = 1, NE = 4, EXP_D = 64, EXP_IH = 32, TOPK = 1 };
+enum { NG = EXP_IH * EXP_D, ND = EXP_D * EXP_IH };
+
+static int8_t want_byte(int eid, int which, int64_t i) {
+ return (int8_t)(((int)i + eid * 3 + which * 5) % 15 - 7);
+}
+
+static void build_model(Model *m) {
+ memset(m, 0, sizeof *m);
+ m->c.n_layers = NL; m->c.n_experts = NE;
+ m->c.hidden = EXP_D; m->c.inter = EXP_IH;
+ m->c.topk = TOPK; m->c.expert_gs = 0;
+ m->active_of = calloc(NL, sizeof(int));
+ m->is_pinned = calloc((size_t)NL * NE, 1);
+ m->is_queued = calloc((size_t)NL * NE, 1);
+ m->cache = calloc(NL, sizeof(LCache));
+ LCache *lc = &m->cache[0];
+ lc->cap = NE; lc->n = NE;
+ lc->slots = calloc(NE, sizeof(Slot));
+ lc->slot_by_expert = malloc(NE * sizeof(int));
+ for (int e = 0; e < NE; e++) {
+ Slot *s = &lc->slots[e];
+ slot_ensure_allocated(m, s);
+ s->eid = e;
+ s->pinned = 1;
+ s->used = ++m->clock;
+ lc->slot_by_expert[e] = e;
+ }
+}
+
+static void fill_int8(Model *m) {
+ for (int e = 0; e < NE; e++) {
+ Slot *s = &m->cache[0].slots[e];
+ for (int64_t i = 0; i < NG; i++) { s->g[i] = want_byte(e, 0, i); s->u[i] = want_byte(e, 1, i); }
+ for (int64_t i = 0; i < ND; i++) s->d[i] = want_byte(e, 2, i);
+ for (int64_t i = 0; i < EXP_IH; i++) { s->gs[i] = 1.f; s->us[i] = 1.f; }
+ for (int64_t i = 0; i < EXP_D; i++) s->ds[i] = 1.f;
+ }
+}
+
+static void fill_int4(Model *m) {
+ int64_t want_w = NG + NG + ND;
+ uint8_t *raw = malloc((size_t)(want_w / 2));
+ for (int e = 0; e < NE; e++) {
+ Slot *s = &m->cache[0].slots[e];
+ for (int64_t i = 0; i < want_w; i += 2) {
+ int which = i < NG ? 0 : (i < 2 * NG ? 1 : 2);
+ int64_t off = i - (which == 0 ? 0 : (which == 1 ? NG : 2 * NG));
+ uint8_t lo = (uint8_t)(want_byte(e, which, off) & 0xF);
+ uint8_t hi = (uint8_t)(want_byte(e, which, off + 1) & 0xF);
+ raw[i >> 1] = (uint8_t)(lo | (hi << 4));
+ }
+ unpack_int4_to_int8(s->g, raw, want_w);
+ s->is_int4 = 1;
+ int64_t gp = NG / 2, up = NG / 2, dp = ND / 2;
+ s->g4 = malloc((size_t)gp); s->u4 = malloc((size_t)up); s->d4 = malloc((size_t)dp);
+ memcpy(s->g4, raw, (size_t)gp);
+ memcpy(s->u4, raw + gp, (size_t)up);
+ memcpy(s->d4, raw + gp + up, (size_t)dp);
+ for (int64_t i = 0; i < EXP_IH; i++) { s->gs[i] = 1.f; s->us[i] = 1.f; }
+ for (int64_t i = 0; i < EXP_D; i++) s->ds[i] = 1.f;
+ }
+ free(raw);
+}
+
+static void free_model(Model *m) {
+ LCache *lc = &m->cache[0];
+ for (int e = 0; e < NE; e++) {
+ Slot *s = &lc->slots[e];
+ free(s->g); free(s->gs);
+ free(s->g4); free(s->u4); free(s->d4);
+ }
+ free(lc->slots); free(lc->slot_by_expert);
+ free(m->cache); free(m->active_of); free(m->is_pinned); free(m->is_queued);
+}
+
+static int wait_resident(void) {
+ for (int i = 0; i < 500; i++) {
+ int all = 1;
+ for (int e = 0; e < NE; e++) if (!qt_is_resident(0, e)) all = 0;
+ if (all) return 1;
+ struct timespec ts = {0, 2000000};
+ nanosleep(&ts, NULL);
+ }
+ return 0;
+}
+
+/* The offer decision both decode-path sites now make, lifted verbatim from
+ * moe()'s resident offer (the prefetch site is the same choice on `ps`). If
+ * this expression regresses to the bare `if (e->g4)`, the int8 case below
+ * fails exactly the way the issue describes. */
+static void decode_offer(int layer, int eid, Slot *s) {
+ if (s->g4)
+ qt_note(layer, eid, s->g4, s->u4, s->d4, s->gs, s->us, s->ds);
+ else if (!g_expert_is_int4 && s->g)
+ qt_note(layer, eid, (const uint8_t *)s->g, (const uint8_t *)s->u,
+ (const uint8_t *)s->d, s->gs, s->us, s->ds);
+}
+
+/* --- an int8 container, QT_NO_WARMSTART=1: the decode offer must promote --- */
+static void case_int8_decode(void) {
+ printf("int8 container, decode-path offer (no warmstart)\n");
+ Model m; build_model(&m); fill_int8(&m);
+
+ setenv("COLI_CUDA", "1", 1);
+ setenv("COLI_GPUS", "0", 1);
+ setenv("QT_NO_WARMSTART", "1", 1);
+ fake_uploads = 0;
+
+ if (!qt_init(NL, NE, EXP_D, EXP_IH, NE, TOPK, 0 /* per-row */, 0 /* int8 */)) {
+ printf(" FAIL the tier refuses a per-row int8 container\n");
+ fails++; free_model(&m); return;
+ }
+ /* The decode loop runs with the format main probed: int8. */
+ g_expert_is_int4 = 0;
+
+ /* moe()'s resident offer, one expert per router pick. Before the fix this
+ * was `if (s->g4) qt_note(...)` with g4 == NULL: zero offers, zero
+ * uploads, resident 0/4 for the life of the process (#1391). */
+ for (int e = 0; e < NE; e++)
+ decode_offer(0, e, &m.cache[0].slots[e]);
+ qt_fill_wait();
+
+ ck(fake_uploads > 0, "the decode offer enqueued int8 experts (fake_uploads > 0)");
+ ck(wait_resident(), "every decode-offered int8 expert reached VRAM");
+ ck(fake_uploads == 3 * NE, "three uploads per expert (gate, up, down)");
+
+ int tier_live = 1, intact = 1;
+ for (int e = 0; e < NE; e++) {
+ Slot *s = &m.cache[0].slots[e];
+ if (qs(0, e)->g4 != (const uint8_t *)s->g) tier_live = 0;
+ for (int64_t i = 0; i < NG; i++)
+ if (s->g[i] != want_byte(e, 0, i) || s->u[i] != want_byte(e, 1, i)) intact = 0;
+ for (int64_t i = 0; i < ND; i++)
+ if (s->d[i] != want_byte(e, 2, i)) intact = 0;
+ }
+ /* outcome: decode_offer_parks_live_int8_pointer */
+ ck(tier_live, "the pointer the tier kept still targets the live int8 block");
+ /* outcome: decode_offer_does_not_free_or_rewrite_weights */
+ ck(intact, "the offered weights are still the bytes the loader wrote");
+
+ qt_shutdown();
+ free_model(&m);
+}
+
+/* --- an int4 container: the decode offer must still prefer the packed copy -- */
+static void case_int4_decode(void) {
+ printf("int4 container, decode-path offer (no warmstart)\n");
+ Model m; build_model(&m); fill_int4(&m);
+
+ setenv("COLI_CUDA", "1", 1);
+ setenv("COLI_GPUS", "0", 1);
+ setenv("QT_NO_WARMSTART", "1", 1);
+ fake_uploads = 0;
+
+ if (!qt_init(NL, NE, EXP_D, EXP_IH, NE, TOPK, 0 /* per-row */, 1 /* int4 */)) {
+ printf(" FAIL regression: the tier no longer starts on an int4 container\n");
+ fails++; free_model(&m); return;
+ }
+ g_expert_is_int4 = 1;
+
+ for (int e = 0; e < NE; e++)
+ decode_offer(0, e, &m.cache[0].slots[e]);
+ qt_fill_wait();
+
+ ck(fake_uploads > 0, "the decode offer still enqueues on an int4 container");
+ ck(wait_resident(), "every decode-offered int4 expert reached VRAM");
+
+ int tier_live = 1;
+ for (int e = 0; e < NE; e++) {
+ Slot *s = &m.cache[0].slots[e];
+ if (qs(0, e)->g4 != s->g4) tier_live = 0;
+ }
+ /* outcome: decode_offer_prefers_packed_int4_pointer */
+ ck(tier_live, "on int4 the tier still parks the packed g4 pointer");
+
+ qt_shutdown();
+ free_model(&m);
+}
+
+int main(void) {
+ case_int8_decode();
+ case_int4_decode();
+ if (fails) { printf("FAILED %d\n", fails); return 1; }
+ printf("OK test_qwen36_tier_int8_decode\n");
+ return 0;
+}
\ No newline at end of file