Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
olmoe: make IDOT opt-in on x86 and NEON, and add the test that was missing
nadiaf48fe8cddee
diff --git a/c/Makefile b/c/Makefileindex 27abc4b1..9bc8bfbe 100644--- a/c/Makefile+++ b/c/Makefile@@ -809,6 +809,12 @@ tests/test_spec_decode_state$(EXE): tests/test_spec_decode_state.c colibri.c st.tests/test_decode_batch$(EXE): tests/test_decode_batch.c decode_batch.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+# olmoe's matmul_q, not colibri's: compares the IDOT path against FP32+# ACTIVATIONS rather than an integer reference. NOCUDA_* for the same reason+# the olmoe target uses it -- olmoe.c contains no COLI_CUDA code.+tests/test_olmoe_matmul_q$(EXE): tests/test_olmoe_matmul_q.c olmoe.c st.h json.h compat.h sample.h tok.h tok_unicode.h tok_unicode_o200k.h omp_tune.h route_trace.h+ $(CC) $(NOCUDA_CFLAGS) $< -o $@ $(NOCUDA_LDFLAGS)+tests/test_idot$(EXE): tests/test_idot.c colibri.c st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h sample.h kv_persist.h telemetry.h route_trace.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)diff --git a/c/olmoe.c b/c/olmoe.cindex 1e077762..0f59ea0c 100644--- a/c/olmoe.c+++ b/c/olmoe.c@@ -166,7 +166,11 @@ static void matmul(float *y, const float *x, const float *W, int S, int I, int O* W[o,i] ~= q[o,i]*scale[o] -> y[o] = scale[o] * sum_i x[i]*q[o,i].* Su ARM: attivazione quantizzata Q8_0 (scala per blocco di 16) + dot int8* NEON (sdot dove c'e' dotprod) — stessa famiglia IDOT di glm.c, IDOT=0 per- * la via scalare byte-esatta. Misurato 2.7x end-to-end su M5. */+ * la via scalare byte-esatta. Misurato 2.7x end-to-end su M5.+ *+ * NB: la quantizzazione delle ATTIVAZIONI rende questo percorso non+ * equivalente alla via scalare (issue #1044). Vale per NEON come per AVX2:+ * IDOT e' opt-in (IDOT=1), non piu' attivo di default. */#if defined(__ARM_NEON)#include <arm_neon.h>static inline int32_t dot_i8_16(const int8_t *a, const int8_t *b) {@@ -187,8 +191,14 @@ static inline int32_t dot_i8_16(const int8_t *a, const int8_t *b) {* the only fast path was ARM, so x86 boxes silently used the scalar* fallback even when AVX2 was available).* Sign-extend both int8 vectors to int16 (exact, no precision loss) then- * madd+horizontal-sum in int32: pure integer arithmetic, so this is- * bit-for-bit identical to the scalar dot product, just vectorized. */+ * madd+horizontal-sum in int32: pure integer arithmetic, so THIS DOT is+ * bit-for-bit identical to a scalar int8 dot, just vectorized.+ *+ * That exactness does NOT extend to the branch that calls it. matmul_q's IDOT+ * path quantizes the ACTIVATIONS to Q8_0 per 16-block before calling this,+ * which the scalar fallback does not do -- so the two paths differ. This+ * comment previously read as if it covered the whole path, which is how+ * issue #1044 stayed invisible. See the note at the idot default below. */static inline int32_t dot_i8_16(const int8_t *a, const int8_t *b) {__m256i va16 = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)a));__m256i vb16 = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)b));@@ -202,10 +212,36 @@ static inline int32_t dot_i8_16(const int8_t *a, const int8_t *b) {}#define HAVE_FAST_DOT_I8 1#endif+/* Test-only hook, compiled out of the shipping binary.+ *+ * matmul_q reads IDOT once into a static, so a test cannot exercise both paths+ * in one process without it. Guarded by OLMOE_TESTING so production builds have+ * neither the global nor the branch: tests/test_olmoe_matmul_q.c defines it. */+#ifdef OLMOE_TESTING+int matmul_q_idot_force = -1;+static inline void matmul_q_reset_for_test(void) { matmul_q_idot_force = -1; }+#endif+static void matmul_q(float *y, const float *x, const int8_t *q, const float *scale, int I, int O) {#if defined(HAVE_FAST_DOT_I8)+ /* IDOT is OPT-IN. It quantizes the ACTIVATIONS to Q8_0 per 16-block (below),+ * which the scalar fallback never does, so the two paths are not numerically+ * equivalent: measured 5/12 vs 12/12 matching tokens against a reference+ * (issue #1044). Before 2c4e9de x86 had no fast dot at all, so IDOT=1 fell+ * through to the exact scalar path and x86 was token-exact by accident; that+ * commit silently made every AVX2 box lossy by default. Defaulting to off+ * restores the behaviour users actually had.+ *+ * Cost of turning it on: ~+5.8e-4 relative error per dot from the activation+ * quantization (colibri.c:910-915 measures the same mechanism at ~+0.117+ * nats/token on GLM, which is why GLM keeps q/k/v off IDOT). On AVX2-only+ * hardware it is also not faster: 11.01 vs 10.67 tok/s measured on an+ * i5-9600K, n=4 interleaved. Set IDOT=1 to opt in knowingly. */static int idot = -1;- if (idot < 0) { const char *e = getenv("IDOT"); idot = !(e && *e == '0'); }+ if (idot < 0) { const char *e = getenv("IDOT"); idot = (e && *e == '1'); }+#ifdef OLMOE_TESTING+ if (matmul_q_idot_force >= 0) idot = matmul_q_idot_force;+#endifif (idot && I % 16 == 0 && I <= 4096) {int nb = I / 16; int8_t xi[4096]; float xs[256];for (int b = 0; b < nb; b++) {diff --git a/c/tests/test_olmoe_matmul_q.c b/c/tests/test_olmoe_matmul_q.cnew file mode 100644index 00000000..230d643f--- /dev/null+++ b/c/tests/test_olmoe_matmul_q.c@@ -0,0 +1,171 @@+/* olmoe matmul_q: the IDOT path must be compared against FP32 ACTIVATIONS,+ * not against an integer reference.+ *+ * This is the test gap behind issue #1044. tests/test_idot.c asserts that the+ * integer dot kernel matches a plain-C integer reference, which it does --+ * exactly, because integer arithmetic has no rounding. But matmul_q's IDOT+ * branch quantizes the ACTIVATIONS to Q8_0 per 16-element block before calling+ * that kernel, and the scalar fallback does not. Comparing int8 against int8+ * can never see that. So a lossy path shipped as the default on every AVX2+ * machine (since 2c4e9de) with a green test suite.+ *+ * What this file asserts:+ * 1. the scalar fallback is EXACT against an fp64 reference;+ * 2. the IDOT path is NOT exact, and its error sits at the magnitude the+ * activation quantization predicts (~1e-3 relative, not ~1e-7);+ * 3. IDOT is OPT-IN: with the variable unset, matmul_q takes the exact path.+ *+ * (2) is deliberately a two-sided bound. Asserting only "error is small" would+ * pass if someone silently made IDOT exact, and asserting only "error exists"+ * would pass if it got much worse. Both directions are load-bearing: the point+ * is that the two paths DIVERGE and by how much.+ *+ * Build: make -C c tests/test_olmoe_matmul_q+ */+#define OLMOE_TESTING 1+#define main coli_olmoe_main_unused+#include "../olmoe.c"+#undef main++#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <math.h>++static uint32_t rng_state = 0x9e3779b9u;+static uint32_t xr(void) {+ rng_state ^= rng_state << 13;+ rng_state ^= rng_state >> 17;+ rng_state ^= rng_state << 5;+ return rng_state;+}+static float frand(void) { return (float)((double)(xr() % 20001) / 10000.0 - 1.0); }++/* Reference in double precision: y[o] = scale[o] * sum_i x[i]*q[o,i].+ * fp64 so that fp32 rounding in the implementation is visible as error rather+ * than being hidden by an equally-rounded reference. */+static void ref_matmul(double *y, const float *x, const int8_t *q,+ const float *scale, int I, int O) {+ for (int o = 0; o < O; o++) {+ double acc = 0.0;+ const int8_t *w = q + (int64_t)o * I;+ for (int i = 0; i < I; i++) acc += (double)x[i] * (double)w[i];+ y[o] = acc * (double)scale[o];+ }+}++/* Error normalised by the MAGNITUDE OF THE SUMMANDS, not by the result.+ *+ * y[o] is a sum of I signed products that largely cancel: with random+ * activations and random int8 weights the result sits near zero while the+ * individual terms do not. Dividing by |y[o]| then reports catastrophic+ * cancellation in the reference as if it were kernel error -- an early version+ * of this test did exactly that and reported 0.86 relative error for a kernel+ * that is correct to 1e-7. Normalising by sum|x_i * w_i| measures what we+ * actually care about: error relative to the work performed. */+static double max_rel_err(const float *got, const double *want,+ const float *x, const int8_t *q, const float *scale,+ int I, int O) {+ double worst = 0.0;+ for (int o = 0; o < O; o++) {+ const int8_t *w = q + (int64_t)o * I;+ double mag = 0.0;+ for (int i = 0; i < I; i++) mag += fabs((double)x[i] * (double)w[i]);+ mag *= fabs((double)scale[o]);+ double d = fabs((double)got[o] - want[o]);+ double r = mag > 1e-12 ? d / mag : d;+ if (r > worst) worst = r;+ }+ return worst;+}++int main(void) {+ const int I = 512, O = 64; /* I % 16 == 0 and I <= 4096: the IDOT gate */+ int failures = 0;++ float *x = malloc((size_t)I * sizeof *x);+ int8_t *q = malloc((size_t)O * I);+ float *scale = malloc((size_t)O * sizeof *scale);+ float *y = malloc((size_t)O * sizeof *y);+ double *ref = malloc((size_t)O * sizeof *ref);+ if (!x || !q || !scale || !y || !ref) { fprintf(stderr, "oom\n"); return 1; }++ for (int i = 0; i < I; i++) x[i] = frand();+ for (int64_t i = 0; i < (int64_t)O * I; i++) q[i] = (int8_t)((int)(xr() % 256) - 128);+ for (int o = 0; o < O; o++) scale[o] = 0.001f + (float)(xr() % 1000) * 1e-6f;++ ref_matmul(ref, x, q, scale, I, O);++#if !defined(HAVE_FAST_DOT_I8)+ printf("SKIP: no fast int8 dot compiled in; IDOT path unreachable\n");+ printf(" (build with -mavx2 on x86 or on an ARM NEON target)\n");+ free(x); free(q); free(scale); free(y); free(ref);+ return 0;+#else+ /* --- 1. THE SHIPPING DEFAULT must be exact ---------------------------+ * Checked FIRST and with the hook untouched, because matmul_q reads IDOT+ * once into a static. Running this after any forced path would test the+ * cache, not the default. This is the assertion that would have caught+ * issue #1044 the day 2c4e9de landed. */+ unsetenv("IDOT");+ matmul_q(y, x, q, scale, I, O);+ double err_default = max_rel_err(y, ref, x, q, scale, I, O);+ if (err_default < 1e-5) {+ printf("PASS shipping default (IDOT unset) is exact (max rel %.3e)\n", err_default);+ } else {+ printf("FAIL shipping default is LOSSY (max rel %.3e).\n", err_default);+ printf(" IDOT must be opt-in: a default that silently quantizes\n");+ printf(" activations is exactly issue #1044.\n");+ failures++;+ }++ /* --- 2. the scalar fallback must be exact when forced ----------------- */+ matmul_q_idot_force = 0;+ matmul_q(y, x, q, scale, I, O);+ double err_scalar = max_rel_err(y, ref, x, q, scale, I, O);+ if (err_scalar < 1e-5) {+ printf("PASS scalar path exact vs fp64 reference (max rel %.3e)\n", err_scalar);+ } else {+ printf("FAIL scalar path should be ~exact, got max rel %.3e\n", err_scalar);+ failures++;+ }++ /* --- 3. the IDOT path must diverge, by the predicted magnitude ------- */+ matmul_q_idot_force = 1;+ matmul_q(y, x, q, scale, I, O);+ double err_idot = max_rel_err(y, ref, x, q, scale, I, O);++ /* Q8_0 over a 16-block: the activation is rounded to a 1/127 grid of the+ * block maximum, so relative error lands around 1e-3, orders of magnitude+ * above fp32 noise. Bound it on both sides. */+ if (err_idot > 20.0 * err_scalar + 1e-7 && err_idot < 1e-2) {+ printf("PASS IDOT path diverges as expected (max rel %.3e vs scalar %.3e)\n",+ err_idot, err_scalar);+ printf(" this is the activation quantization, not a kernel bug\n");+ } else if (err_idot <= 20.0 * err_scalar + 1e-7) {+ printf("FAIL IDOT path is now as exact as scalar (%.3e vs %.3e).\n",+ err_idot, err_scalar);+ printf(" If the activation quantization was removed, delete this test\n");+ printf(" and the IDOT opt-in note in matmul_q -- do not just relax it.\n");+ failures++;+ } else {+ printf("FAIL IDOT error %.3e is far larger than activation quantization\n", err_idot);+ printf(" predicts (~1e-3). Suspect a real kernel bug.\n");+ failures++;+ }++ /* --- 4. IDOT=1 must still be reachable ------------------------------- */+ matmul_q_idot_force = 1;+ matmul_q(y, x, q, scale, I, O);+ if (max_rel_err(y, ref, x, q, scale, I, O) > 20.0 * err_scalar + 1e-7) {+ printf("PASS IDOT=1 still opts in to the fast path\n");+ } else {+ printf("FAIL IDOT=1 no longer reaches the fast path\n");+ failures++;+ }++ free(x); free(q); free(scale); free(y); free(ref);+ printf("\n%s\n", failures ? "FAILED" : "ALL PASS");+ return failures ? 1 : 0;+#endif+}