Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(qwen36): same logits at every expert-cache capacity on FMA builds
Marcello Costagliola28e444ed8794
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 886d0182..db9aa977 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -557,13 +557,15 @@ jobs:# (int4 experts drift from f32 by design): it is that the kernel and# the unpack-to-int8 path it replaces emit the SAME ids at every cap,# including cap=1 (one expert resident: the per-pair cut) and cap=16- # (the whole prompt in one run).+ # (the whole prompt in one run). Within each path the last logits must+ # be the same bytes at every cap: an id survives a last-bit drift, and+ # the per-pair cut once rounded twice where an FMA build rounds once.python3 tools/make_qwen36_tiny.py --out qwen36_tiny64 --ref-mode full --inter 64 \--emit-ref qwen36_tiny64/ref_full.jsonpython3 tools/convert_qwen36.py --model qwen36_tiny64 --out qwen36_tiny64_c --ebits 4 --gs 64for cap in 1 2 8 16; dofor k in 0 1; do- QWEN_EXPERT_KERNEL=$k COLI_DENSE_I8=0 SNAP=qwen36_tiny64_c \+ QWEN_EXPERT_KERNEL=$k COLI_DENSE_I8=0 SNAP=qwen36_tiny64_c DUMP=logits_${cap}_$k.f32 \./qwen36 "$cap" 4 qwen36_tiny64/ref_full.json > ab_${cap}_$k.log 2>&1 || truegrep -E "^C engine" ab_${cap}_$k.log > ids_${cap}_$k.txtdone@@ -573,6 +575,13 @@ jobs:cmp ids_${cap}_0.txt ids_${cap}_1.txt || { echo "FAIL: ids differ at cap=$cap"; cat ids_${cap}_0.txt ids_${cap}_1.txt; exit 1; }echo "cap=$cap: identical ($(cat ids_${cap}_1.txt))"done+ for k in 0 1; do+ for cap in 1 2 8; do+ test -s logits_${cap}_$k.f32 || { echo "FAIL: no logits dump at cap=$cap kernel=$k"; exit 1; }+ cmp logits_${cap}_$k.f32 logits_16_$k.f32 || { echo "FAIL: kernel=$k logits at cap=$cap differ from cap=16"; exit 1; }+ done+ echo "kernel=$k: identical logits at cap 1/2/8/16"+ donemake tests/test_expert_ffn && ./tests/test_expert_ffn- name: Mixed expert layout (int4 gs64 gate/up, int8 down) loads and is cap-independentrun: |diff --git a/c/expert_ffn.h b/c/expert_ffn.hindex c1939f38..65415844 100644--- a/c/expert_ffn.h+++ b/c/expert_ffn.h@@ -310,7 +310,12 @@ static inline void xf_down_rows(float *y, const XfExpert *e, const XfAct *h, int* S tokens, K routed experts each. idx[s*K+k] is the expert id (or -1), val the* router weight, experts[s*K+k] the resident expert (NULL iff idx < 0).* out[s][H] = sum_k val[s][k] * expert_k(x[s]) accumulated in k order, as the- * per-token loops it replaces did. Callers own `scratch` (xf_moe_scratch_bytes).+ * per-token loops it replaces did. xf_moe_add adds that sum into out instead of+ * overwriting it: a caller that cuts one layer into several runs (qwen36, when+ * its cache holds fewer experts than the batch routes to) sends every cut+ * through the same statement, so the bits cannot depend on where it cut, whether+ * or not the compiler fuses that multiply-add. xf_moe_run zeroes out first.+ * Callers own `scratch` (xf_moe_scratch_bytes).** Work is split into (expert, row chunk) items twice: gate+up over F rows, then* down over H rows. A prompt of S rows routed to the same expert reads that@@ -324,7 +329,7 @@ static inline size_t xf_moe_scratch_bytes(int S, int K, int H, int F) {+ n * sizeof(int) * 4 + 4096;}-static inline void xf_moe_run(float *out, const float *x, int S, int K, int H, int F,+static inline void xf_moe_add(float *out, const float *x, int S, int K, int H, int F,const int *idx, const float *val, const XfExpert *const *experts,int mode, void *scratch) {const size_t n = (size_t)S * K;@@ -358,7 +363,7 @@ static inline void xf_moe_run(float *out, const float *x, int S, int K, int H, iif (j == nu) { uniq[nu] = idx[i]; head[nu] = i; cnt[nu] = 1; nu++; }else { int t = head[j]; while (next[t] >= 0) t = next[t]; next[t] = i; cnt[j]++; }}- if (nu == 0) { memset(out, 0, (size_t)S * H * sizeof(float)); return; }+ if (nu == 0) return;int T = 1;#ifdef _OPENMP@@ -398,11 +403,10 @@ static inline void xf_moe_run(float *out, const float *x, int S, int K, int H, ixf_down_rows(ctb + (size_t)i * H, experts[i], &a, F, r0, r1, mode);}}- /* rank-order sum per token: out[s] = sum_k val[s][k] * ctb[s][k] */+ /* rank-order sum per token: out[s] += sum_k val[s][k] * ctb[s][k] */#pragma omp parallel for schedule(static)for (int s = 0; s < S; s++) {float *os = out + (size_t)s * H;- memset(os, 0, (size_t)H * sizeof(float));for (int k = 0; k < K; k++) {int i = s * K + k; if (idx[i] < 0 || !experts[i]) continue;float w = val[i]; const float *c = ctb + (size_t)i * H;@@ -411,4 +415,11 @@ static inline void xf_moe_run(float *out, const float *x, int S, int K, int H, i}}+static inline void xf_moe_run(float *out, const float *x, int S, int K, int H, int F,+ const int *idx, const float *val, const XfExpert *const *experts,+ int mode, void *scratch) {+ memset(out, 0, (size_t)S * H * sizeof(float));+ xf_moe_add(out, x, S, K, H, F, idx, val, experts, mode, scratch);+}+#endif /* COLI_EXPERT_FFN_H */diff --git a/c/qwen36.c b/c/qwen36.cindex e7515014..ed2a55f0 100644--- a/c/qwen36.c+++ b/c/qwen36.c@@ -2232,9 +2232,11 @@ static void qwen_shared_experts_cpu(Model *m, Layer *l, const float *x, int S,* resident at once, so the batch is cut to what the layer cache can hold:* the whole prompt chunk when cap covers S*K slots, one token when it covers* K, one (token, expert) pair otherwise (cap=1 in CI evicts on every routed- * expert). A pair run adds val*expert into out exactly as the per-token loop- * did: out starts at zero and the kernel's rank-order sum is one fma per- * element, so the three cuts produce the same bits. */+ * expert). out starts at zero (moe) and every cut adds into it through the+ * one xf_moe_add call below, val*expert in rank order, so the three cuts+ * produce the same bits. A pair run used to go through a zeroed buffer and a+ * second add: two roundings where the fused multiply-add of an FMA build+ * does one, and cap=1 moved the logits in their last bits. */static void moe_xf_run(Model *m, int layer, const float *x, int S, float *out, const int *idx, const float *val) {Cfg *c = &m->c; int D = c->hidden, K = c->topk, F = c->inter;int cap = m->cache[layer].cap;@@ -2245,7 +2247,6 @@ static void moe_xf_run(Model *m, int layer, const float *x, int S, float *out, cXfExpert *ex = malloc(sizeof(XfExpert) * (size_t)n);const XfExpert **exp = malloc(sizeof(XfExpert *) * (size_t)n);int *ridx = malloc(sizeof(int) * (size_t)n); float *rval = falloc(n);- float *tmp = kper < K ? falloc(D) : NULL;void *scratch = malloc(xf_moe_scratch_bytes(per, kper, D, F));if (!ex || !exp || !ridx || !scratch) { fprintf(stderr, "OOM moe_xf_run\n"); exit(1); }int timed = tm_on() && S == 1;@@ -2262,15 +2263,11 @@ static void moe_xf_run(Model *m, int layer, const float *x, int S, float *out, cexp[dst] = &ex[dst];}double t1 = timed ? tm_now() : 0;- if (kper == K) xf_moe_run(out + (int64_t)s0 * D, x + (int64_t)s0 * D, per, K, D, F, ridx, rval, exp, xf_act_mode(), scratch);- else {- xf_moe_run(tmp, x + (int64_t)s0 * D, 1, 1, D, F, ridx, rval, exp, xf_act_mode(), scratch);- float *os = out + (int64_t)s0 * D; for (int d = 0; d < D; d++) os[d] += tmp[d];- }+ xf_moe_add(out + (int64_t)s0 * D, x + (int64_t)s0 * D, per, kper, D, F, ridx, rval, exp, xf_act_mode(), scratch);if (timed) { double t2 = tm_now(); g_xf_load += t1 - t0; g_xf_run += t2 - t1; }}}- free(ex); free(exp); free(ridx); free(rval); free(tmp); free(scratch);+ free(ex); free(exp); free(ridx); free(rval); free(scratch);}/* ---------- CACHE_ROUTE: residency-aware top-K fill (docs/CACHE_ROUTE.md) ----------diff --git a/c/tests/test_expert_ffn.c b/c/tests/test_expert_ffn.cindex 9c9f1d4b..72858e71 100644--- a/c/tests/test_expert_ffn.c+++ b/c/tests/test_expert_ffn.c@@ -109,6 +109,16 @@ static void test_layer(int S, int K, int H, int F, int NE, int mode) {int bad = 0; for (int i = 0; i < S * H; i++) if (o1[i] != o2[i]) { if (!bad) printf(" first diff at %d: %.9g vs %.9g\n", i, o2[i], o1[i]); bad++; }CHECK(bad == 0, "layer S=%d K=%d H=%d F=%d mode=%d: %d of %d outputs differ from the per-token loop", S, K, H, F, mode, bad, S * H);printf("layer S=%2d K=%d H=%4d F=%4d mode=%d: bit-identical to the per-token loop\n", S, K, H, F, mode);+ /* the same batch cut into (token, expert) pairs, each added into out: the cut+ * qwen36 makes when its cache holds fewer experts than the batch routes to.+ * Every cut goes through xf_moe_add's one statement, so the bits must match+ * the whole run whether or not the build fuses the multiply-add. */+ float *o3 = calloc((size_t)S * H, sizeof(float)); void *sc1 = malloc(xf_moe_scratch_bytes(1, 1, H, F));+ for (int i = 0; i < S * K; i++) xf_moe_add(o3 + (size_t)(i / K) * H, x + (size_t)(i / K) * H, 1, 1, H, F, idx + i, val + i, ex + i, mode, sc1);+ bad = 0; for (int i = 0; i < S * H; i++) if (o3[i] != o2[i]) { if (!bad) printf(" first diff at %d: %.9g vs %.9g\n", i, o3[i], o2[i]); bad++; }+ CHECK(bad == 0, "layer S=%d K=%d H=%d F=%d mode=%d: %d of %d outputs differ when cut into pairs", S, K, H, F, mode, bad, S * H);+ printf("layer S=%2d K=%d H=%4d F=%4d mode=%d: bit-identical when cut into pairs\n", S, K, H, F, mode);+ free(o3); free(sc1);for (int e = 0; e < NE; e++) { free((void *)E[e].g4); free((void *)E[e].u4); free((void *)E[e].d4); free((void *)E[e].gs); free((void *)E[e].us); free((void *)E[e].ds); }free(E); free(tmp); free(vals); free(x); free(idx); free(val); free(ex); free(o1); free(o2); free(scratch);}