Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(qwen36): same logits at every expert-cache capacity on FMA builds

Marcello Costagliola28e444ed8794
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 886d0182..db9aa977 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -557,13 +557,15 @@ jobs:
# (int4 experts drift from f32 by design): it is that the kernel and
# the unpack-to-int8 path it replaces emit the SAME ids at every cap,
# including cap=1 (one expert resident: the per-pair cut) and cap=16
- # (the whole prompt in one run).
+ # (the whole prompt in one run). Within each path the last logits must
+ # be the same bytes at every cap: an id survives a last-bit drift, and
+ # the per-pair cut once rounded twice where an FMA build rounds once.
python3 tools/make_qwen36_tiny.py --out qwen36_tiny64 --ref-mode full --inter 64 \
--emit-ref qwen36_tiny64/ref_full.json
python3 tools/convert_qwen36.py --model qwen36_tiny64 --out qwen36_tiny64_c --ebits 4 --gs 64
for cap in 1 2 8 16; do
for k in 0 1; do
- QWEN_EXPERT_KERNEL=$k COLI_DENSE_I8=0 SNAP=qwen36_tiny64_c \
+ QWEN_EXPERT_KERNEL=$k COLI_DENSE_I8=0 SNAP=qwen36_tiny64_c DUMP=logits_${cap}_$k.f32 \
./qwen36 "$cap" 4 qwen36_tiny64/ref_full.json > ab_${cap}_$k.log 2>&1 || true
grep -E "^C engine" ab_${cap}_$k.log > ids_${cap}_$k.txt
done
@@ -573,6 +575,13 @@ jobs:
cmp ids_${cap}_0.txt ids_${cap}_1.txt || { echo "FAIL: ids differ at cap=$cap"; cat ids_${cap}_0.txt ids_${cap}_1.txt; exit 1; }
echo "cap=$cap: identical ($(cat ids_${cap}_1.txt))"
done
+ for k in 0 1; do
+ for cap in 1 2 8; do
+ test -s logits_${cap}_$k.f32 || { echo "FAIL: no logits dump at cap=$cap kernel=$k"; exit 1; }
+ cmp logits_${cap}_$k.f32 logits_16_$k.f32 || { echo "FAIL: kernel=$k logits at cap=$cap differ from cap=16"; exit 1; }
+ done
+ echo "kernel=$k: identical logits at cap 1/2/8/16"
+ done
make tests/test_expert_ffn && ./tests/test_expert_ffn
- name: Mixed expert layout (int4 gs64 gate/up, int8 down) loads and is cap-independent
run: |
diff --git a/c/expert_ffn.h b/c/expert_ffn.h
index c1939f38..65415844 100644
--- a/c/expert_ffn.h
+++ b/c/expert_ffn.h
@@ -310,7 +310,12 @@ static inline void xf_down_rows(float *y, const XfExpert *e, const XfAct *h, int
* S tokens, K routed experts each. idx[s*K+k] is the expert id (or -1), val the
* router weight, experts[s*K+k] the resident expert (NULL iff idx < 0).
* out[s][H] = sum_k val[s][k] * expert_k(x[s]) accumulated in k order, as the
- * per-token loops it replaces did. Callers own `scratch` (xf_moe_scratch_bytes).
+ * per-token loops it replaces did. xf_moe_add adds that sum into out instead of
+ * overwriting it: a caller that cuts one layer into several runs (qwen36, when
+ * its cache holds fewer experts than the batch routes to) sends every cut
+ * through the same statement, so the bits cannot depend on where it cut, whether
+ * or not the compiler fuses that multiply-add. xf_moe_run zeroes out first.
+ * Callers own `scratch` (xf_moe_scratch_bytes).
*
* Work is split into (expert, row chunk) items twice: gate+up over F rows, then
* down over H rows. A prompt of S rows routed to the same expert reads that
@@ -324,7 +329,7 @@ static inline size_t xf_moe_scratch_bytes(int S, int K, int H, int F) {
+ n * sizeof(int) * 4 + 4096;
}
-static inline void xf_moe_run(float *out, const float *x, int S, int K, int H, int F,
+static inline void xf_moe_add(float *out, const float *x, int S, int K, int H, int F,
const int *idx, const float *val, const XfExpert *const *experts,
int mode, void *scratch) {
const size_t n = (size_t)S * K;
@@ -358,7 +363,7 @@ static inline void xf_moe_run(float *out, const float *x, int S, int K, int H, i
if (j == nu) { uniq[nu] = idx[i]; head[nu] = i; cnt[nu] = 1; nu++; }
else { int t = head[j]; while (next[t] >= 0) t = next[t]; next[t] = i; cnt[j]++; }
}
- if (nu == 0) { memset(out, 0, (size_t)S * H * sizeof(float)); return; }
+ if (nu == 0) return;
int T = 1;
#ifdef _OPENMP
@@ -398,11 +403,10 @@ static inline void xf_moe_run(float *out, const float *x, int S, int K, int H, i
xf_down_rows(ctb + (size_t)i * H, experts[i], &a, F, r0, r1, mode);
}
}
- /* rank-order sum per token: out[s] = sum_k val[s][k] * ctb[s][k] */
+ /* rank-order sum per token: out[s] += sum_k val[s][k] * ctb[s][k] */
#pragma omp parallel for schedule(static)
for (int s = 0; s < S; s++) {
float *os = out + (size_t)s * H;
- memset(os, 0, (size_t)H * sizeof(float));
for (int k = 0; k < K; k++) {
int i = s * K + k; if (idx[i] < 0 || !experts[i]) continue;
float w = val[i]; const float *c = ctb + (size_t)i * H;
@@ -411,4 +415,11 @@ static inline void xf_moe_run(float *out, const float *x, int S, int K, int H, i
}
}
+static inline void xf_moe_run(float *out, const float *x, int S, int K, int H, int F,
+ const int *idx, const float *val, const XfExpert *const *experts,
+ int mode, void *scratch) {
+ memset(out, 0, (size_t)S * H * sizeof(float));
+ xf_moe_add(out, x, S, K, H, F, idx, val, experts, mode, scratch);
+}
+
#endif /* COLI_EXPERT_FFN_H */
diff --git a/c/qwen36.c b/c/qwen36.c
index e7515014..ed2a55f0 100644
--- a/c/qwen36.c
+++ b/c/qwen36.c
@@ -2232,9 +2232,11 @@ static void qwen_shared_experts_cpu(Model *m, Layer *l, const float *x, int S,
* resident at once, so the batch is cut to what the layer cache can hold:
* the whole prompt chunk when cap covers S*K slots, one token when it covers
* K, one (token, expert) pair otherwise (cap=1 in CI evicts on every routed
- * expert). A pair run adds val*expert into out exactly as the per-token loop
- * did: out starts at zero and the kernel's rank-order sum is one fma per
- * element, so the three cuts produce the same bits. */
+ * expert). out starts at zero (moe) and every cut adds into it through the
+ * one xf_moe_add call below, val*expert in rank order, so the three cuts
+ * produce the same bits. A pair run used to go through a zeroed buffer and a
+ * second add: two roundings where the fused multiply-add of an FMA build
+ * does one, and cap=1 moved the logits in their last bits. */
static void moe_xf_run(Model *m, int layer, const float *x, int S, float *out, const int *idx, const float *val) {
Cfg *c = &m->c; int D = c->hidden, K = c->topk, F = c->inter;
int cap = m->cache[layer].cap;
@@ -2245,7 +2247,6 @@ static void moe_xf_run(Model *m, int layer, const float *x, int S, float *out, c
XfExpert *ex = malloc(sizeof(XfExpert) * (size_t)n);
const XfExpert **exp = malloc(sizeof(XfExpert *) * (size_t)n);
int *ridx = malloc(sizeof(int) * (size_t)n); float *rval = falloc(n);
- float *tmp = kper < K ? falloc(D) : NULL;
void *scratch = malloc(xf_moe_scratch_bytes(per, kper, D, F));
if (!ex || !exp || !ridx || !scratch) { fprintf(stderr, "OOM moe_xf_run\n"); exit(1); }
int timed = tm_on() && S == 1;
@@ -2262,15 +2263,11 @@ static void moe_xf_run(Model *m, int layer, const float *x, int S, float *out, c
exp[dst] = &ex[dst];
}
double t1 = timed ? tm_now() : 0;
- if (kper == K) xf_moe_run(out + (int64_t)s0 * D, x + (int64_t)s0 * D, per, K, D, F, ridx, rval, exp, xf_act_mode(), scratch);
- else {
- xf_moe_run(tmp, x + (int64_t)s0 * D, 1, 1, D, F, ridx, rval, exp, xf_act_mode(), scratch);
- float *os = out + (int64_t)s0 * D; for (int d = 0; d < D; d++) os[d] += tmp[d];
- }
+ xf_moe_add(out + (int64_t)s0 * D, x + (int64_t)s0 * D, per, kper, D, F, ridx, rval, exp, xf_act_mode(), scratch);
if (timed) { double t2 = tm_now(); g_xf_load += t1 - t0; g_xf_run += t2 - t1; }
}
}
- free(ex); free(exp); free(ridx); free(rval); free(tmp); free(scratch);
+ free(ex); free(exp); free(ridx); free(rval); free(scratch);
}
/* ---------- CACHE_ROUTE: residency-aware top-K fill (docs/CACHE_ROUTE.md) ----------
diff --git a/c/tests/test_expert_ffn.c b/c/tests/test_expert_ffn.c
index 9c9f1d4b..72858e71 100644
--- a/c/tests/test_expert_ffn.c
+++ b/c/tests/test_expert_ffn.c
@@ -109,6 +109,16 @@ static void test_layer(int S, int K, int H, int F, int NE, int mode) {
int bad = 0; for (int i = 0; i < S * H; i++) if (o1[i] != o2[i]) { if (!bad) printf(" first diff at %d: %.9g vs %.9g\n", i, o2[i], o1[i]); bad++; }
CHECK(bad == 0, "layer S=%d K=%d H=%d F=%d mode=%d: %d of %d outputs differ from the per-token loop", S, K, H, F, mode, bad, S * H);
printf("layer S=%2d K=%d H=%4d F=%4d mode=%d: bit-identical to the per-token loop\n", S, K, H, F, mode);
+ /* the same batch cut into (token, expert) pairs, each added into out: the cut
+ * qwen36 makes when its cache holds fewer experts than the batch routes to.
+ * Every cut goes through xf_moe_add's one statement, so the bits must match
+ * the whole run whether or not the build fuses the multiply-add. */
+ float *o3 = calloc((size_t)S * H, sizeof(float)); void *sc1 = malloc(xf_moe_scratch_bytes(1, 1, H, F));
+ for (int i = 0; i < S * K; i++) xf_moe_add(o3 + (size_t)(i / K) * H, x + (size_t)(i / K) * H, 1, 1, H, F, idx + i, val + i, ex + i, mode, sc1);
+ bad = 0; for (int i = 0; i < S * H; i++) if (o3[i] != o2[i]) { if (!bad) printf(" first diff at %d: %.9g vs %.9g\n", i, o3[i], o2[i]); bad++; }
+ CHECK(bad == 0, "layer S=%d K=%d H=%d F=%d mode=%d: %d of %d outputs differ when cut into pairs", S, K, H, F, mode, bad, S * H);
+ printf("layer S=%2d K=%d H=%4d F=%4d mode=%d: bit-identical when cut into pairs\n", S, K, H, F, mode);
+ free(o3); free(sc1);
for (int e = 0; e < NE; e++) { free((void *)E[e].g4); free((void *)E[e].u4); free((void *)E[e].d4); free((void *)E[e].gs); free((void *)E[e].us); free((void *)E[e].ds); }
free(E); free(tmp); free(vals); free(x); free(idx); free(val); free(ex); free(o1); free(o2); free(scratch);
}