Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(moe): FUSED3=1 opt-in AVX2 expert matmul β€” vectorized activation quant + gate/up pair

outtodatad2d9fd673d39
diff --git a/c/fused_simd.h b/c/fused_simd.h
new file mode 100644
index 00000000..594bcf6d
--- /dev/null
+++ b/c/fused_simd.h
@@ -0,0 +1,156 @@
+/* fused_simd.h β€” FUSED3: faster AVX2 kernels for the int8 (Q8_0-per-row)
+ * routed-expert path of olmoe.c. Three cooperating pieces:
+ *
+ * 1) dot_i8idot_avx2_v2 / matmul_q_idot_v2: re-vectorized per-row IDOT matmul
+ * (2 independent accumulator chains instead of matmul_q's 1 per row).
+ * 2) quant_x_q8_avx2: AVX2 activation quantization, BIT-IDENTICAL to the
+ * scalar loop in matmul_q (see the proof comment above it).
+ * 3) matmul_q_idot_v3 / matmul_q_idot_pair_v3: v2 + vectorized quant, plus a
+ * gate/up pair that quantizes the shared input ONCE instead of twice.
+ *
+ * Numerics: all integer dots are exact (sign-extend + madd, as dot_i8_16);
+ * activation quantization is byte-exact vs the scalar path, so v3/pair-v3
+ * outputs are bit-identical to the stock matmul_q calls β€” verified by
+ * memcmp in tests/bench_fused3.c. The flag only changes instruction
+ * scheduling, never values.
+ *
+ * Header-only, all static, no dependencies beyond immintrin β€” mirrors quant.h
+ * conventions. Guarded by __AVX2__; callers keep the stock fallback.
+ */
+#ifndef COLI_FUSED_SIMD_H
+#define COLI_FUSED_SIMD_H
+
+#include <stdint.h>
+#include <math.h>
+
+#ifdef __AVX2__
+#include <immintrin.h>
+
+/* One Q8_0-style row dot: y = sum_b xs[b] * dot16(xi_b, w_b), 16 int8 per
+ * block β€” the same activation block contract as olmoe.c matmul_q's IDOT
+ * branch. Integer dots are exact (cvtepi8_epi16 + madd_epi16), so the only
+ * numeric delta vs the stock path is fp32 summation order of the per-block
+ * fold, exactly as in matmul_q. Requires nb%2==0 for the paired fast loop
+ * (activation blocks come in pairs); tail block scalar with v1 semantics. */
+static inline float dot_i8idot_avx2_v2(const int8_t *xi, const int8_t *w,
+ const float *xs, int nb){
+ __m256 tot0=_mm256_setzero_ps(), tot1=_mm256_setzero_ps();
+ int b=0;
+ for(; b+2<=nb; b+=2){
+ __m256i pa=_mm256_madd_epi16(_mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)(xi+b*16))),
+ _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)(w +b*16))));
+ tot0=_mm256_fmadd_ps(_mm256_cvtepi32_ps(pa),_mm256_set1_ps(xs[b]),tot0);
+ __m256i pb=_mm256_madd_epi16(_mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)(xi+b*16+16))),
+ _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)(w +b*16+16))));
+ tot1=_mm256_fmadd_ps(_mm256_cvtepi32_ps(pb),_mm256_set1_ps(xs[b+1]),tot1);
+ }
+ __m256 tot=_mm256_add_ps(tot0,tot1);
+ __m128 lo4=_mm256_castps256_ps128(tot), hi4=_mm256_extractf128_ps(tot,1);
+ lo4=_mm_add_ps(lo4,hi4); __m128 sh=_mm_movehl_ps(lo4,lo4); lo4=_mm_add_ps(lo4,sh);
+ sh=_mm_shuffle_ps(lo4,lo4,1); lo4=_mm_add_ss(lo4,sh);
+ float acc=_mm_cvtss_f32(lo4);
+ for(; b<nb; b++){ /* odd trailing block: v1 semantics, scalar */
+ const int8_t *xb=xi+b*16, *wb=w+b*16; int32_t d=0;
+ for(int i=0;i<16;i++) d+=(int32_t)xb[i]*wb[i];
+ acc+=xs[b]*(float)d;
+ }
+ return acc;
+}
+
+/* Drop-in faster replacement for olmoe.c:matmul_q's IDOT branch (S==1 GEMV,
+ * I%16==0, I<=4096). Activation quantization is byte-identical to v1 (scalar,
+ * same rounding), so xq/xs match; only the row dot is re-vectorized. */
+static void matmul_q_idot_v2(float *y, const float *x, const int8_t *q,
+ const float *scale, int I, int O){
+ int nb=I/16; int8_t xi[4096]; float xs[256];
+ for(int b=0;b<nb;b++){
+ const float *xb=x+b*16;
+ float am=0.f; for(int i=0;i<16;i++){ float a=fabsf(xb[i]); if(a>am) am=a; }
+ float s=am/127.f; if(s<1e-12f) s=1e-12f;
+ xs[b]=s; float inv=1.f/s;
+ for(int i=0;i<16;i++) xi[b*16+i]=(int8_t)lrintf(xb[i]*inv);
+ }
+ #pragma omp parallel for schedule(static)
+ for(int o=0;o<O;o++){
+ const int8_t *w=q+(int64_t)o*I;
+ y[o]=dot_i8idot_avx2_v2(xi,w,xs,nb)*scale[o];
+ }
+}
+
+/* ---- FUSED3: vectorized activation quantization + gate/up pair ------------
+ * Profiling the v2 path (bench_fused3 i8 shapes) showed the SCALAR activation
+ * quantization β€” not the row dots β€” dominating: ~0.12 ms of ~0.19 ms per
+ * [1024,2048] call (2048 fabsf + 2048 lrintf, the latter a msvcrt libcall
+ * under MinGW). The IDOT row kernel already runs at ~30 GB/s, near the memory
+ * ceiling of the mobile APUs this engine targets, so the remaining win is
+ * quantizing x with AVX2 and doing it ONCE for the shared gate/up input
+ * instead of twice.
+ *
+ * quant_x_q8_avx2 is BIT-IDENTICAL to the scalar loop above, line by line:
+ * - amax: vector abs (andnot sign) + max is exact, order-free;
+ * - s = am/127.f, clamp, inv = 1.f/s: the same scalar fp ops per block;
+ * - xi = (int8_t)lrintf(x*inv): lrintf honors the current rounding mode
+ * (nearest-even by default) and so does vcvtps2dq; |x|<=am keeps every
+ * product inside [-127,127], so the packs_epi16 saturation never engages
+ * differently from the scalar (int8_t) cast;
+ * - the lane fix-ups (packs_epi32 interleaves 128-bit lanes -> permute4x64)
+ * only reorder, they do not round.
+ * Verified byte-for-byte (xi memcmp + xs bit-compare) in bench_fused3.
+ * Requires I%16==0, I<=4096 β€” the same contract as the v2 IDOT path. */
+static inline void quant_x_q8_avx2(const float *x, int8_t *xi, float *xs, int I){
+ const __m256 sgn=_mm256_castsi256_ps(_mm256_set1_epi32(0x80000000));
+ int nb=I/16;
+ for(int b=0;b<nb;b++){
+ const float *xb=x+b*16;
+ __m256 v0=_mm256_andnot_ps(sgn,_mm256_loadu_ps(xb));
+ __m256 v1=_mm256_andnot_ps(sgn,_mm256_loadu_ps(xb+8));
+ __m256 mx=_mm256_max_ps(v0,v1);
+ __m128 m4=_mm_max_ps(_mm256_castps256_ps128(mx),_mm256_extractf128_ps(mx,1));
+ m4=_mm_max_ps(m4,_mm_movehl_ps(m4,m4));
+ m4=_mm_max_ps(m4,_mm_shuffle_ps(m4,m4,1));
+ float am=_mm_cvtss_f32(m4);
+ float s=am/127.f; if(s<1e-12f) s=1e-12f;
+ xs[b]=s; float inv=1.f/s;
+ __m256 vi=_mm256_set1_ps(inv);
+ __m256i q0=_mm256_cvtps_epi32(_mm256_mul_ps(_mm256_loadu_ps(xb), vi));
+ __m256i q1=_mm256_cvtps_epi32(_mm256_mul_ps(_mm256_loadu_ps(xb+8),vi));
+ __m256i p16=_mm256_packs_epi32(q0,q1); /* [q0_0..3 q1_0..3 | q0_4..7 q1_4..7] */
+ p16=_mm256_permute4x64_epi64(p16,0xD8); /* [q0(8) | q1(8)] int16 */
+ __m128i p8=_mm_packs_epi16(_mm256_castsi256_si128(p16),
+ _mm256_extracti128_si256(p16,1));
+ _mm_storeu_si128((__m128i*)(xi+b*16),p8);
+ }
+}
+
+/* v3 single-projection: v2 with the activation quantization vectorized.
+ * Row dots unchanged (dot_i8idot_avx2_v2) -> output bit-identical to v2. */
+static void matmul_q_idot_v3(float *y, const float *x, const int8_t *q,
+ const float *scale, int I, int O){
+ int nb=I/16; int8_t xi[4096]; float xs[256];
+ quant_x_q8_avx2(x,xi,xs,I);
+ #pragma omp parallel for schedule(static)
+ for(int o=0;o<O;o++){
+ const int8_t *w=q+(int64_t)o*I;
+ y[o]=dot_i8idot_avx2_v2(xi,w,xs,nb)*scale[o];
+ }
+}
+
+/* v3 gate+up pair: one quantization of the shared input feeds both weight
+ * streams (v2 runs the scalar quant twice β€” once per matmul_q call). Each
+ * row dot is unchanged, so yg/yu are bit-identical to two v2/v3 calls. */
+static void matmul_q_idot_pair_v3(float *yg, float *yu, const float *x,
+ const int8_t *qg, const float *sg,
+ const int8_t *qu, const float *su,
+ int I, int O){
+ int nb=I/16; int8_t xi[4096]; float xs[256];
+ quant_x_q8_avx2(x,xi,xs,I);
+ #pragma omp parallel for schedule(static)
+ for(int o=0;o<O;o++){
+ const int8_t *wg=qg+(int64_t)o*I, *wu=qu+(int64_t)o*I;
+ yg[o]=dot_i8idot_avx2_v2(xi,wg,xs,nb)*sg[o];
+ yu[o]=dot_i8idot_avx2_v2(xi,wu,xs,nb)*su[o];
+ }
+}
+#endif /* __AVX2__ */
+
+#endif /* COLI_FUSED_SIMD_H */
diff --git a/c/olmoe.c b/c/olmoe.c
index 0f59ea0c..6ff808a1 100644
--- a/c/olmoe.c
+++ b/c/olmoe.c
@@ -106,6 +106,10 @@ static int g_pilot = 0;
static int g_wide = 1; /* IMPROVEMENT 4: top-K * g_wide candidates prefetched */
static int g_pilot_evict_guard = 1; /* PILOT_EVICT_GUARD=0 to disable LFRU prefetch eviction guard */
static int g_expert_drop = 0; /* EXPERT_DROP=1 restores fadvise(DONTNEED) after expert reads */
+static int g_fused3 = 0; /* FUSED3=1: AVX2 activation quant + gate/up pair matmul
+ * (fused_simd.h: quant_x_q8_avx2, matmul_q_idot_v3,
+ * matmul_q_idot_pair_v3). Exact integer arithmetic only β€”
+ * bit-identical to the stock matmul_q path; OFF by default. */
static uint64_t lfru_score(uint32_t heat, uint64_t last, uint64_t clock) {
uint64_t age = (clock > last) ? (clock - last) : 0;
@@ -222,6 +226,10 @@ int matmul_q_idot_force = -1;
static inline void matmul_q_reset_for_test(void) { matmul_q_idot_force = -1; }
#endif
+#if defined(__AVX2__)
+#include "fused_simd.h" /* FUSED3=1: quant_x_q8_avx2 + matmul_q_idot{,_pair}_v3 (bit-exact) */
+#endif
+
static void matmul_q(float *y, const float *x, const int8_t *q, const float *scale, int I, int O) {
#if defined(HAVE_FAST_DOT_I8)
/* IDOT is OPT-IN. It quantizes the ACTIVATIONS to Q8_0 per 16-block (below),
@@ -723,10 +731,25 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {
const float *xs = x + (int64_t)s*D;
for (int kk = 0; kk < K; kk++) {
Slot *e; expert_get(m, layer, idx[kk], &e);
+#if defined(__AVX2__)
+ /* FUSED3: same contract as matmul_q's IDOT fast branch (IDOT env,
+ * dims %16==0, <=4096) β€” outside it the stock calls below run
+ * unchanged. Exact integer arithmetic only: bit-identical output
+ * (verified by memcmp in tests/bench_fused3.c). OFF by default. */
+ static int idot_moe = -1;
+ if (idot_moe < 0) { const char *ie = getenv("IDOT"); idot_moe = !(ie && *ie == '0'); }
+ if (g_fused3 && idot_moe && D % 16 == 0 && D <= 4096 && I % 16 == 0 && I <= 4096) {
+ matmul_q_idot_pair_v3(g, u, xs, e->g, e->gs, e->u, e->us, D, I); /* gate+up share one quant of xs */
+ for (int i = 0; i < I; i++) { float gv = g[i]; g[i] = (gv / (1.f + expf(-gv))) * u[i]; }
+ matmul_q_idot_v3(hh, g, e->d, e->ds, I, D); /* down_proj [D,I] */
+ } else
+#endif
+ {
matmul_q(g, xs, e->g, e->gs, D, I); /* gate_proj [I,D] */
matmul_q(u, xs, e->u, e->us, D, I); /* up_proj [I,D] */
for (int i = 0; i < I; i++) { float gv = g[i]; g[i] = (gv / (1.f + expf(-gv))) * u[i]; }
matmul_q(hh, g, e->d, e->ds, I, D); /* down_proj [D,I] */
+ }
float w = val[kk];
float *os = out + (int64_t)s*D;
for (int d = 0; d < D; d++) os[d] += w * hh[d];
@@ -1337,6 +1360,7 @@ int main(int argc, char **argv) {
g_wide = getenv("WIDE") ? atoi(getenv("WIDE")) : 1;
g_pilot_evict_guard = getenv("PILOT_EVICT_GUARD") ? atoi(getenv("PILOT_EVICT_GUARD")) : 1;
g_expert_drop = getenv("EXPERT_DROP") ? atoi(getenv("EXPERT_DROP")) : 0;
+ g_fused3 = getenv("FUSED3") ? atoi(getenv("FUSED3")) : 0;
if (g_wide < 1) g_wide = 1;
if (g_wide > 4) g_wide = 4;
int hot_n = getenv("HOT") ? atoi(getenv("HOT")) : 0;
@@ -1405,8 +1429,8 @@ int main(int argc, char **argv) {
float smooth = getenv("SMOOTH") ? (float)atof(getenv("SMOOTH")) : 0.3f;
float conf = getenv("CONF_LIMIT") ? (float)atof(getenv("CONF_LIMIT")) : 0.92f;
- printf("== Streaming C engine v2.2 | cache=%d/layer bits=%d pilot=%d wide=%d guard=%d hot=%d smooth=%.2f conf=%.2f ==\n",
- cap, bits, g_pilot, g_wide, g_pilot_evict_guard, hot_n, smooth, conf);
+ printf("== Streaming C engine v2.2 | cache=%d/layer bits=%d pilot=%d wide=%d guard=%d hot=%d smooth=%.2f conf=%.2f fused3=%d ==\n",
+ cap, bits, g_pilot, g_wide, g_pilot_evict_guard, hot_n, smooth, conf, g_fused3);
FILE *f = fopen(refpath, "rb"); if (!f) { perror(refpath); return 1; }
fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET);
diff --git a/c/tests/bench_fused3.c b/c/tests/bench_fused3.c
new file mode 100644
index 00000000..28bf3c6f
--- /dev/null
+++ b/c/tests/bench_fused3.c
@@ -0,0 +1,183 @@
+/* bench_fused3.c β€” microbenchmark + correctness harness for the FUSED3
+ * kernels (c/fused_simd.h) on OLMoE expert shapes: stock matmul_q IDOT branch
+ * (v1, copied verbatim below) vs v2 (re-vectorized row dots) vs v3 (+
+ * vectorized activation quant, gate/up pair).
+ * Build: gcc -O3 -mavx2 -mfma -fopenmp -I.. bench_fused3.c -o bench_fused3.exe -lm
+ * Prints KEY=VALUE lines. Correctness gates:
+ * (a) quant_x_q8_avx2 output byte-identical to the scalar quant
+ * (xi memcmp, xs bit-compare), incl. the all-zero clamp path;
+ * (b) v3/pair-v3 row outputs bit-identical to matmul_q_idot_v2. */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <time.h>
+#include <stdint.h>
+#ifdef _OPENMP
+#include <omp.h>
+#endif
+
+#include "fused_simd.h"
+
+/* dot_i8_16: verbatim AVX2 copy from olmoe.c (static there) β€” v1 baseline dot */
+static inline int32_t dot_i8_16(const int8_t *a, const int8_t *b) {
+ __m256i va16 = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)a));
+ __m256i vb16 = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)b));
+ __m256i prod = _mm256_madd_epi16(va16, vb16);
+ __m128i sum128 = _mm_add_epi32(_mm256_castsi256_si128(prod), _mm256_extractf128_si256(prod, 1));
+ __m128i hi64 = _mm_unpackhi_epi64(sum128, sum128);
+ __m128i sum64 = _mm_add_epi32(sum128, hi64);
+ __m128i hi32 = _mm_shuffle_epi32(sum64, _MM_SHUFFLE(2, 3, 0, 1));
+ __m128i sum32 = _mm_add_epi32(sum64, hi32);
+ return _mm_cvtsi128_si32(sum32);
+}
+
+static double now_s(void){
+ struct timespec ts; clock_gettime(CLOCK_MONOTONIC,&ts);
+ return ts.tv_sec + ts.tv_nsec*1e-9;
+}
+
+static unsigned rng_s=12345;
+static unsigned rnd(void){ rng_s=rng_s*1664525u+1013904223u; return rng_s>>8; }
+
+int main(int argc, char **argv){
+ int reps = argc>1 ? atoi(argv[1]) : 200;
+ printf("# bench_fused3 (S=1 GEMV, int8 Q8_0-per-row, olmoe expert path)\n");
+#ifdef _OPENMP
+ printf("omp_threads=%d\n", omp_get_max_threads());
+#else
+ printf("omp_threads=1\n");
+#endif
+
+ /* ---- int8 per-row IDOT (olmoe.c expert path) -------------------------- */
+ /* v1 baseline: verbatim copy of olmoe.c matmul_q's HAVE_FAST_DOT_I8 branch */
+ int i8shapes[2][2] = { {1024,2048}, {2048,1024} };
+ for(int sh=0; sh<2; sh++){
+ int O=i8shapes[sh][0], I=i8shapes[sh][1];
+ int8_t *q8=malloc((int64_t)O*I);
+ float *scl=malloc((int64_t)O*sizeof(float));
+ float *x=malloc(I*sizeof(float));
+ float *y1=malloc((int64_t)O*sizeof(float));
+ float *y2=malloc((int64_t)O*sizeof(float));
+ if(!q8||!scl||!x||!y1||!y2){ fprintf(stderr,"OOM\n"); return 1; }
+ for(int64_t i=0;i<(int64_t)O*I;i++) q8[i]=(int8_t)(rnd()%255-127);
+ for(int o=0;o<O;o++) scl[o]=0.001f+((int)(rnd()%1000))/1e6f;
+ for(int i=0;i<I;i++) x[i]=((int)(rnd()%2000)-1000)/500.f;
+
+ /* v1 (copy of olmoe.c matmul_q AVX2 IDOT branch) */
+ #define MATMUL_Q_IDOT_V1(Y) do{ \
+ int nb=I/16; int8_t xi[4096]; float xs[256]; \
+ for(int b=0;b<nb;b++){ const float *xb=x+b*16; \
+ float am=0.f; for(int i=0;i<16;i++){ float a=fabsf(xb[i]); if(a>am) am=a; } \
+ float s=am/127.f; if(s<1e-12f) s=1e-12f; \
+ xs[b]=s; float inv=1.f/s; \
+ for(int i=0;i<16;i++) xi[b*16+i]=(int8_t)lrintf(xb[i]*inv); } \
+ _Pragma("omp parallel for schedule(static)") \
+ for(int o=0;o<O;o++){ const int8_t *w=q8+(int64_t)o*I; float acc=0.f; \
+ for(int b=0;b<nb;b++) acc+=xs[b]*(float)dot_i8_16(xi+b*16,w+b*16); \
+ (Y)[o]=acc*scl[o]; } }while(0)
+
+ /* correctness vs int64 reference (int dot is exact; fp fold differs) */
+ double e1=0,e2=0;
+ for(int o=0;o<O;o+=O/8+1){
+ int nb=I/16; int8_t xi[4096]; float xs[256];
+ for(int b=0;b<nb;b++){ const float *xb=x+b*16;
+ float am=0.f; for(int i=0;i<16;i++){ float a=fabsf(xb[i]); if(a>am) am=a; }
+ float s=am/127.f; if(s<1e-12f) s=1e-12f;
+ xs[b]=s; float inv=1.f/s;
+ for(int i=0;i<16;i++) xi[b*16+i]=(int8_t)lrintf(xb[i]*inv); }
+ double r=0; const int8_t *w=q8+(int64_t)o*I;
+ for(int b=0;b<nb;b++){ int32_t d=0; for(int i=0;i<16;i++) d+=(int32_t)xi[b*16+i]*w[b*16+i];
+ r+=(double)xs[b]*d; }
+ r*=scl[o];
+ MATMUL_Q_IDOT_V1(y1);
+ matmul_q_idot_v2(y2,x,q8,scl,I,O);
+ double d1=fabs(y1[o]-r)/(1+fabs(r)), d2=fabs(y2[o]-r)/(1+fabs(r));
+ if(d1>e1)e1=d1; if(d2>e2)e2=d2;
+ }
+ printf("i8_O%d_I%d: relerr_v1=%.3g relerr_v2=%.3g\n",O,I,e1,e2);
+
+ double t1=1e30,t2=1e30;
+ for(int k=0;k<3;k++){ MATMUL_Q_IDOT_V1(y1); matmul_q_idot_v2(y2,x,q8,scl,I,O); }
+ for(int r=0;r<reps;r++){
+ double a=now_s(); MATMUL_Q_IDOT_V1(y1); double b=now_s();
+ if(b-a<t1)t1=b-a;
+ a=now_s(); matmul_q_idot_v2(y2,x,q8,scl,I,O); b=now_s();
+ if(b-a<t2)t2=b-a;
+ }
+ double wbytes=(double)O*I;
+ printf("i8_O%d_I%d: v1_ms=%.4f v2_ms=%.4f speedup=%.3f v1_GBs=%.1f v2_GBs=%.1f\n",
+ O,I,t1*1e3,t2*1e3,t1/t2,wbytes/t1/1e9,wbytes/t2/1e9);
+ free(q8);free(scl);free(x);free(y1);free(y2);
+ #undef MATMUL_Q_IDOT_V1
+ }
+
+ /* ---- FUSED3: vectorized quant + gate/up pair ---------------------------
+ * Contract: (a) quant_x_q8_avx2 output byte-identical to the scalar quant
+ * (xi memcmp, xs bit-compare); (b) v3/pair-v3 row outputs bit-identical
+ * to matmul_q_idot_v2; (c) timing on the two real expert shapes, with the
+ * per-expert total = gate+up pair + down. */
+ {
+ int O=1024, I=2048, Od=2048, Id=1024; /* gate/up then down (OLMoE) */
+ int8_t *qg=malloc((int64_t)O*I), *qu=malloc((int64_t)O*I), *qd=malloc((int64_t)Od*Id);
+ float *sg=malloc(O*sizeof(float)), *su=malloc(O*sizeof(float)), *sd=malloc(Od*sizeof(float));
+ float *x=malloc(I*sizeof(float)), *xd=malloc(Id*sizeof(float));
+ float *yg=malloc(O*sizeof(float)), *yu=malloc(O*sizeof(float)), *yd=malloc(Od*sizeof(float));
+ float *yg2=malloc(O*sizeof(float)), *yu2=malloc(O*sizeof(float)), *yd2=malloc(Od*sizeof(float));
+ if(!qg||!qu||!qd||!sg||!su||!sd||!x||!xd||!yg||!yu||!yd||!yg2||!yu2||!yd2){ fprintf(stderr,"OOM\n"); return 1; }
+ for(int64_t i=0;i<(int64_t)O*I;i++){ qg[i]=(int8_t)(rnd()%255-127); qu[i]=(int8_t)(rnd()%255-127); }
+ for(int64_t i=0;i<(int64_t)Od*Id;i++) qd[i]=(int8_t)(rnd()%255-127);
+ for(int o=0;o<O;o++){ sg[o]=0.001f+((int)(rnd()%1000))/1e6f; su[o]=0.001f+((int)(rnd()%1000))/1e6f; }
+ for(int o=0;o<Od;o++) sd[o]=0.001f+((int)(rnd()%1000))/1e6f;
+ for(int i=0;i<I;i++) x[i]=((int)(rnd()%2000)-1000)/500.f;
+ for(int i=0;i<Id;i++) xd[i]=((int)(rnd()%2000)-1000)/500.f;
+ x[37]=0.f; x[38]=0.f; /* near-zero block elements: exercise the s<1e-12 clamp path partially */
+
+ /* (a) quant bit-identity vs the scalar loop */
+ int nb=I/16; static int8_t xi_s[4096], xi_v[4096]; static float xs_s[256], xs_v[256];
+ for(int b=0;b<nb;b++){ const float *xb=x+b*16;
+ float am=0.f; for(int i=0;i<16;i++){ float a=fabsf(xb[i]); if(a>am) am=a; }
+ float s=am/127.f; if(s<1e-12f) s=1e-12f;
+ xs_s[b]=s; float inv=1.f/s;
+ for(int i=0;i<16;i++) xi_s[b*16+i]=(int8_t)lrintf(xb[i]*inv); }
+ quant_x_q8_avx2(x,xi_v,xs_v,I);
+ int qbad=memcmp(xi_s,xi_v,I)!=0 || memcmp(xs_s,xs_v,nb*sizeof(float))!=0;
+ /* all-zero block: clamp path */
+ { float xz[64]={0}; int8_t xz_s[64],xz_v[64]; float xs2_s[4],xs2_v[4];
+ for(int b=0;b<4;b++){ float am=0.f; float s=am/127.f; if(s<1e-12f) s=1e-12f;
+ xs2_s[b]=s; float inv=1.f/s;
+ for(int i=0;i<16;i++) xz_s[b*16+i]=(int8_t)lrintf(xz[b*16+i]*inv); }
+ quant_x_q8_avx2(xz,xz_v,xs2_v,64);
+ if(memcmp(xz_s,xz_v,64)!=0 || memcmp(xs2_s,xs2_v,4*sizeof(float))!=0) qbad=1; }
+ printf("v3_quant_bitexact=%s\n", qbad?"FAIL":"yes");
+
+ /* (b) v3 / pair-v3 outputs bit-identical to v2 */
+ matmul_q_idot_v2(yg,x,qg,sg,I,O); matmul_q_idot_v3(yg2,x,qg,sg,I,O);
+ int bid = memcmp(yg,yg2,O*sizeof(float))==0;
+ matmul_q_idot_v2(yg,x,qg,sg,I,O); matmul_q_idot_v2(yu,x,qu,su,I,O);
+ matmul_q_idot_pair_v3(yg2,yu2,x,qg,sg,qu,su,I,O);
+ bid = bid && memcmp(yg,yg2,O*sizeof(float))==0 && memcmp(yu,yu2,O*sizeof(float))==0;
+ matmul_q_idot_v2(yd,xd,qd,sd,Id,Od); matmul_q_idot_v3(yd2,xd,qd,sd,Id,Od);
+ bid = bid && memcmp(yd,yd2,Od*sizeof(float))==0;
+ printf("v3_output_bitidentical_v2=%s\n", bid?"yes":"FAIL");
+
+ /* (c) per-expert timing: v2 (3 separate calls) vs v3 (pair + down) */
+ double t2e=1e30,t3e=1e30;
+ for(int k=0;k<3;k++){
+ matmul_q_idot_v2(yg,x,qg,sg,I,O); matmul_q_idot_v2(yu,x,qu,su,I,O); matmul_q_idot_v2(yd,xd,qd,sd,Id,Od);
+ matmul_q_idot_pair_v3(yg2,yu2,x,qg,sg,qu,su,I,O); matmul_q_idot_v3(yd,xd,qd,sd,Id,Od); }
+ for(int r=0;r<reps;r++){
+ double a=now_s();
+ matmul_q_idot_v2(yg,x,qg,sg,I,O); matmul_q_idot_v2(yu,x,qu,su,I,O); matmul_q_idot_v2(yd,xd,qd,sd,Id,Od);
+ double b=now_s(); if(b-a<t2e)t2e=b-a;
+ a=now_s();
+ matmul_q_idot_pair_v3(yg2,yu2,x,qg,sg,qu,su,I,O); matmul_q_idot_v3(yd,xd,qd,sd,Id,Od);
+ b=now_s(); if(b-a<t3e)t3e=b-a;
+ }
+ printf("v3_expert: v2_3calls_ms=%.4f v3_pair+down_ms=%.4f speedup=%.3f\n",
+ t2e*1e3,t3e*1e3,t2e/t3e);
+ free(qg);free(qu);free(qd);free(sg);free(su);free(sd);free(x);free(xd);
+ free(yg);free(yu);free(yd);free(yg2);free(yu2);free(yd2);
+ }
+ return 0;
+}