Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
feat(moe): FUSED3=1 opt-in AVX2 expert matmul β vectorized activation quant + gate/up pair
outtodatad2d9fd673d39
diff --git a/c/fused_simd.h b/c/fused_simd.hnew file mode 100644index 00000000..594bcf6d--- /dev/null+++ b/c/fused_simd.h@@ -0,0 +1,156 @@+/* fused_simd.h β FUSED3: faster AVX2 kernels for the int8 (Q8_0-per-row)+ * routed-expert path of olmoe.c. Three cooperating pieces:+ *+ * 1) dot_i8idot_avx2_v2 / matmul_q_idot_v2: re-vectorized per-row IDOT matmul+ * (2 independent accumulator chains instead of matmul_q's 1 per row).+ * 2) quant_x_q8_avx2: AVX2 activation quantization, BIT-IDENTICAL to the+ * scalar loop in matmul_q (see the proof comment above it).+ * 3) matmul_q_idot_v3 / matmul_q_idot_pair_v3: v2 + vectorized quant, plus a+ * gate/up pair that quantizes the shared input ONCE instead of twice.+ *+ * Numerics: all integer dots are exact (sign-extend + madd, as dot_i8_16);+ * activation quantization is byte-exact vs the scalar path, so v3/pair-v3+ * outputs are bit-identical to the stock matmul_q calls β verified by+ * memcmp in tests/bench_fused3.c. The flag only changes instruction+ * scheduling, never values.+ *+ * Header-only, all static, no dependencies beyond immintrin β mirrors quant.h+ * conventions. Guarded by __AVX2__; callers keep the stock fallback.+ */+#ifndef COLI_FUSED_SIMD_H+#define COLI_FUSED_SIMD_H++#include <stdint.h>+#include <math.h>++#ifdef __AVX2__+#include <immintrin.h>++/* One Q8_0-style row dot: y = sum_b xs[b] * dot16(xi_b, w_b), 16 int8 per+ * block β the same activation block contract as olmoe.c matmul_q's IDOT+ * branch. Integer dots are exact (cvtepi8_epi16 + madd_epi16), so the only+ * numeric delta vs the stock path is fp32 summation order of the per-block+ * fold, exactly as in matmul_q. Requires nb%2==0 for the paired fast loop+ * (activation blocks come in pairs); tail block scalar with v1 semantics. */+static inline float dot_i8idot_avx2_v2(const int8_t *xi, const int8_t *w,+ const float *xs, int nb){+ __m256 tot0=_mm256_setzero_ps(), tot1=_mm256_setzero_ps();+ int b=0;+ for(; b+2<=nb; b+=2){+ __m256i pa=_mm256_madd_epi16(_mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)(xi+b*16))),+ _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)(w +b*16))));+ tot0=_mm256_fmadd_ps(_mm256_cvtepi32_ps(pa),_mm256_set1_ps(xs[b]),tot0);+ __m256i pb=_mm256_madd_epi16(_mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)(xi+b*16+16))),+ _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)(w +b*16+16))));+ tot1=_mm256_fmadd_ps(_mm256_cvtepi32_ps(pb),_mm256_set1_ps(xs[b+1]),tot1);+ }+ __m256 tot=_mm256_add_ps(tot0,tot1);+ __m128 lo4=_mm256_castps256_ps128(tot), hi4=_mm256_extractf128_ps(tot,1);+ lo4=_mm_add_ps(lo4,hi4); __m128 sh=_mm_movehl_ps(lo4,lo4); lo4=_mm_add_ps(lo4,sh);+ sh=_mm_shuffle_ps(lo4,lo4,1); lo4=_mm_add_ss(lo4,sh);+ float acc=_mm_cvtss_f32(lo4);+ for(; b<nb; b++){ /* odd trailing block: v1 semantics, scalar */+ const int8_t *xb=xi+b*16, *wb=w+b*16; int32_t d=0;+ for(int i=0;i<16;i++) d+=(int32_t)xb[i]*wb[i];+ acc+=xs[b]*(float)d;+ }+ return acc;+}++/* Drop-in faster replacement for olmoe.c:matmul_q's IDOT branch (S==1 GEMV,+ * I%16==0, I<=4096). Activation quantization is byte-identical to v1 (scalar,+ * same rounding), so xq/xs match; only the row dot is re-vectorized. */+static void matmul_q_idot_v2(float *y, const float *x, const int8_t *q,+ const float *scale, int I, int O){+ int nb=I/16; int8_t xi[4096]; float xs[256];+ for(int b=0;b<nb;b++){+ const float *xb=x+b*16;+ float am=0.f; for(int i=0;i<16;i++){ float a=fabsf(xb[i]); if(a>am) am=a; }+ float s=am/127.f; if(s<1e-12f) s=1e-12f;+ xs[b]=s; float inv=1.f/s;+ for(int i=0;i<16;i++) xi[b*16+i]=(int8_t)lrintf(xb[i]*inv);+ }+ #pragma omp parallel for schedule(static)+ for(int o=0;o<O;o++){+ const int8_t *w=q+(int64_t)o*I;+ y[o]=dot_i8idot_avx2_v2(xi,w,xs,nb)*scale[o];+ }+}++/* ---- FUSED3: vectorized activation quantization + gate/up pair ------------+ * Profiling the v2 path (bench_fused3 i8 shapes) showed the SCALAR activation+ * quantization β not the row dots β dominating: ~0.12 ms of ~0.19 ms per+ * [1024,2048] call (2048 fabsf + 2048 lrintf, the latter a msvcrt libcall+ * under MinGW). The IDOT row kernel already runs at ~30 GB/s, near the memory+ * ceiling of the mobile APUs this engine targets, so the remaining win is+ * quantizing x with AVX2 and doing it ONCE for the shared gate/up input+ * instead of twice.+ *+ * quant_x_q8_avx2 is BIT-IDENTICAL to the scalar loop above, line by line:+ * - amax: vector abs (andnot sign) + max is exact, order-free;+ * - s = am/127.f, clamp, inv = 1.f/s: the same scalar fp ops per block;+ * - xi = (int8_t)lrintf(x*inv): lrintf honors the current rounding mode+ * (nearest-even by default) and so does vcvtps2dq; |x|<=am keeps every+ * product inside [-127,127], so the packs_epi16 saturation never engages+ * differently from the scalar (int8_t) cast;+ * - the lane fix-ups (packs_epi32 interleaves 128-bit lanes -> permute4x64)+ * only reorder, they do not round.+ * Verified byte-for-byte (xi memcmp + xs bit-compare) in bench_fused3.+ * Requires I%16==0, I<=4096 β the same contract as the v2 IDOT path. */+static inline void quant_x_q8_avx2(const float *x, int8_t *xi, float *xs, int I){+ const __m256 sgn=_mm256_castsi256_ps(_mm256_set1_epi32(0x80000000));+ int nb=I/16;+ for(int b=0;b<nb;b++){+ const float *xb=x+b*16;+ __m256 v0=_mm256_andnot_ps(sgn,_mm256_loadu_ps(xb));+ __m256 v1=_mm256_andnot_ps(sgn,_mm256_loadu_ps(xb+8));+ __m256 mx=_mm256_max_ps(v0,v1);+ __m128 m4=_mm_max_ps(_mm256_castps256_ps128(mx),_mm256_extractf128_ps(mx,1));+ m4=_mm_max_ps(m4,_mm_movehl_ps(m4,m4));+ m4=_mm_max_ps(m4,_mm_shuffle_ps(m4,m4,1));+ float am=_mm_cvtss_f32(m4);+ float s=am/127.f; if(s<1e-12f) s=1e-12f;+ xs[b]=s; float inv=1.f/s;+ __m256 vi=_mm256_set1_ps(inv);+ __m256i q0=_mm256_cvtps_epi32(_mm256_mul_ps(_mm256_loadu_ps(xb), vi));+ __m256i q1=_mm256_cvtps_epi32(_mm256_mul_ps(_mm256_loadu_ps(xb+8),vi));+ __m256i p16=_mm256_packs_epi32(q0,q1); /* [q0_0..3 q1_0..3 | q0_4..7 q1_4..7] */+ p16=_mm256_permute4x64_epi64(p16,0xD8); /* [q0(8) | q1(8)] int16 */+ __m128i p8=_mm_packs_epi16(_mm256_castsi256_si128(p16),+ _mm256_extracti128_si256(p16,1));+ _mm_storeu_si128((__m128i*)(xi+b*16),p8);+ }+}++/* v3 single-projection: v2 with the activation quantization vectorized.+ * Row dots unchanged (dot_i8idot_avx2_v2) -> output bit-identical to v2. */+static void matmul_q_idot_v3(float *y, const float *x, const int8_t *q,+ const float *scale, int I, int O){+ int nb=I/16; int8_t xi[4096]; float xs[256];+ quant_x_q8_avx2(x,xi,xs,I);+ #pragma omp parallel for schedule(static)+ for(int o=0;o<O;o++){+ const int8_t *w=q+(int64_t)o*I;+ y[o]=dot_i8idot_avx2_v2(xi,w,xs,nb)*scale[o];+ }+}++/* v3 gate+up pair: one quantization of the shared input feeds both weight+ * streams (v2 runs the scalar quant twice β once per matmul_q call). Each+ * row dot is unchanged, so yg/yu are bit-identical to two v2/v3 calls. */+static void matmul_q_idot_pair_v3(float *yg, float *yu, const float *x,+ const int8_t *qg, const float *sg,+ const int8_t *qu, const float *su,+ int I, int O){+ int nb=I/16; int8_t xi[4096]; float xs[256];+ quant_x_q8_avx2(x,xi,xs,I);+ #pragma omp parallel for schedule(static)+ for(int o=0;o<O;o++){+ const int8_t *wg=qg+(int64_t)o*I, *wu=qu+(int64_t)o*I;+ yg[o]=dot_i8idot_avx2_v2(xi,wg,xs,nb)*sg[o];+ yu[o]=dot_i8idot_avx2_v2(xi,wu,xs,nb)*su[o];+ }+}+#endif /* __AVX2__ */++#endif /* COLI_FUSED_SIMD_H */diff --git a/c/olmoe.c b/c/olmoe.cindex 0f59ea0c..6ff808a1 100644--- a/c/olmoe.c+++ b/c/olmoe.c@@ -106,6 +106,10 @@ static int g_pilot = 0;static int g_wide = 1; /* IMPROVEMENT 4: top-K * g_wide candidates prefetched */static int g_pilot_evict_guard = 1; /* PILOT_EVICT_GUARD=0 to disable LFRU prefetch eviction guard */static int g_expert_drop = 0; /* EXPERT_DROP=1 restores fadvise(DONTNEED) after expert reads */+static int g_fused3 = 0; /* FUSED3=1: AVX2 activation quant + gate/up pair matmul+ * (fused_simd.h: quant_x_q8_avx2, matmul_q_idot_v3,+ * matmul_q_idot_pair_v3). Exact integer arithmetic only β+ * bit-identical to the stock matmul_q path; OFF by default. */static uint64_t lfru_score(uint32_t heat, uint64_t last, uint64_t clock) {uint64_t age = (clock > last) ? (clock - last) : 0;@@ -222,6 +226,10 @@ int matmul_q_idot_force = -1;static inline void matmul_q_reset_for_test(void) { matmul_q_idot_force = -1; }#endif+#if defined(__AVX2__)+#include "fused_simd.h" /* FUSED3=1: quant_x_q8_avx2 + matmul_q_idot{,_pair}_v3 (bit-exact) */+#endif+static void matmul_q(float *y, const float *x, const int8_t *q, const float *scale, int I, int O) {#if defined(HAVE_FAST_DOT_I8)/* IDOT is OPT-IN. It quantizes the ACTIVATIONS to Q8_0 per 16-block (below),@@ -723,10 +731,25 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out) {const float *xs = x + (int64_t)s*D;for (int kk = 0; kk < K; kk++) {Slot *e; expert_get(m, layer, idx[kk], &e);+#if defined(__AVX2__)+ /* FUSED3: same contract as matmul_q's IDOT fast branch (IDOT env,+ * dims %16==0, <=4096) β outside it the stock calls below run+ * unchanged. Exact integer arithmetic only: bit-identical output+ * (verified by memcmp in tests/bench_fused3.c). OFF by default. */+ static int idot_moe = -1;+ if (idot_moe < 0) { const char *ie = getenv("IDOT"); idot_moe = !(ie && *ie == '0'); }+ if (g_fused3 && idot_moe && D % 16 == 0 && D <= 4096 && I % 16 == 0 && I <= 4096) {+ matmul_q_idot_pair_v3(g, u, xs, e->g, e->gs, e->u, e->us, D, I); /* gate+up share one quant of xs */+ for (int i = 0; i < I; i++) { float gv = g[i]; g[i] = (gv / (1.f + expf(-gv))) * u[i]; }+ matmul_q_idot_v3(hh, g, e->d, e->ds, I, D); /* down_proj [D,I] */+ } else+#endif+ {matmul_q(g, xs, e->g, e->gs, D, I); /* gate_proj [I,D] */matmul_q(u, xs, e->u, e->us, D, I); /* up_proj [I,D] */for (int i = 0; i < I; i++) { float gv = g[i]; g[i] = (gv / (1.f + expf(-gv))) * u[i]; }matmul_q(hh, g, e->d, e->ds, I, D); /* down_proj [D,I] */+ }float w = val[kk];float *os = out + (int64_t)s*D;for (int d = 0; d < D; d++) os[d] += w * hh[d];@@ -1337,6 +1360,7 @@ int main(int argc, char **argv) {g_wide = getenv("WIDE") ? atoi(getenv("WIDE")) : 1;g_pilot_evict_guard = getenv("PILOT_EVICT_GUARD") ? atoi(getenv("PILOT_EVICT_GUARD")) : 1;g_expert_drop = getenv("EXPERT_DROP") ? atoi(getenv("EXPERT_DROP")) : 0;+ g_fused3 = getenv("FUSED3") ? atoi(getenv("FUSED3")) : 0;if (g_wide < 1) g_wide = 1;if (g_wide > 4) g_wide = 4;int hot_n = getenv("HOT") ? atoi(getenv("HOT")) : 0;@@ -1405,8 +1429,8 @@ int main(int argc, char **argv) {float smooth = getenv("SMOOTH") ? (float)atof(getenv("SMOOTH")) : 0.3f;float conf = getenv("CONF_LIMIT") ? (float)atof(getenv("CONF_LIMIT")) : 0.92f;- printf("== Streaming C engine v2.2 | cache=%d/layer bits=%d pilot=%d wide=%d guard=%d hot=%d smooth=%.2f conf=%.2f ==\n",- cap, bits, g_pilot, g_wide, g_pilot_evict_guard, hot_n, smooth, conf);+ printf("== Streaming C engine v2.2 | cache=%d/layer bits=%d pilot=%d wide=%d guard=%d hot=%d smooth=%.2f conf=%.2f fused3=%d ==\n",+ cap, bits, g_pilot, g_wide, g_pilot_evict_guard, hot_n, smooth, conf, g_fused3);FILE *f = fopen(refpath, "rb"); if (!f) { perror(refpath); return 1; }fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET);diff --git a/c/tests/bench_fused3.c b/c/tests/bench_fused3.cnew file mode 100644index 00000000..28bf3c6f--- /dev/null+++ b/c/tests/bench_fused3.c@@ -0,0 +1,183 @@+/* bench_fused3.c β microbenchmark + correctness harness for the FUSED3+ * kernels (c/fused_simd.h) on OLMoE expert shapes: stock matmul_q IDOT branch+ * (v1, copied verbatim below) vs v2 (re-vectorized row dots) vs v3 (++ * vectorized activation quant, gate/up pair).+ * Build: gcc -O3 -mavx2 -mfma -fopenmp -I.. bench_fused3.c -o bench_fused3.exe -lm+ * Prints KEY=VALUE lines. Correctness gates:+ * (a) quant_x_q8_avx2 output byte-identical to the scalar quant+ * (xi memcmp, xs bit-compare), incl. the all-zero clamp path;+ * (b) v3/pair-v3 row outputs bit-identical to matmul_q_idot_v2. */+#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <math.h>+#include <time.h>+#include <stdint.h>+#ifdef _OPENMP+#include <omp.h>+#endif++#include "fused_simd.h"++/* dot_i8_16: verbatim AVX2 copy from olmoe.c (static there) β v1 baseline dot */+static inline int32_t dot_i8_16(const int8_t *a, const int8_t *b) {+ __m256i va16 = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)a));+ __m256i vb16 = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)b));+ __m256i prod = _mm256_madd_epi16(va16, vb16);+ __m128i sum128 = _mm_add_epi32(_mm256_castsi256_si128(prod), _mm256_extractf128_si256(prod, 1));+ __m128i hi64 = _mm_unpackhi_epi64(sum128, sum128);+ __m128i sum64 = _mm_add_epi32(sum128, hi64);+ __m128i hi32 = _mm_shuffle_epi32(sum64, _MM_SHUFFLE(2, 3, 0, 1));+ __m128i sum32 = _mm_add_epi32(sum64, hi32);+ return _mm_cvtsi128_si32(sum32);+}++static double now_s(void){+ struct timespec ts; clock_gettime(CLOCK_MONOTONIC,&ts);+ return ts.tv_sec + ts.tv_nsec*1e-9;+}++static unsigned rng_s=12345;+static unsigned rnd(void){ rng_s=rng_s*1664525u+1013904223u; return rng_s>>8; }++int main(int argc, char **argv){+ int reps = argc>1 ? atoi(argv[1]) : 200;+ printf("# bench_fused3 (S=1 GEMV, int8 Q8_0-per-row, olmoe expert path)\n");+#ifdef _OPENMP+ printf("omp_threads=%d\n", omp_get_max_threads());+#else+ printf("omp_threads=1\n");+#endif++ /* ---- int8 per-row IDOT (olmoe.c expert path) -------------------------- */+ /* v1 baseline: verbatim copy of olmoe.c matmul_q's HAVE_FAST_DOT_I8 branch */+ int i8shapes[2][2] = { {1024,2048}, {2048,1024} };+ for(int sh=0; sh<2; sh++){+ int O=i8shapes[sh][0], I=i8shapes[sh][1];+ int8_t *q8=malloc((int64_t)O*I);+ float *scl=malloc((int64_t)O*sizeof(float));+ float *x=malloc(I*sizeof(float));+ float *y1=malloc((int64_t)O*sizeof(float));+ float *y2=malloc((int64_t)O*sizeof(float));+ if(!q8||!scl||!x||!y1||!y2){ fprintf(stderr,"OOM\n"); return 1; }+ for(int64_t i=0;i<(int64_t)O*I;i++) q8[i]=(int8_t)(rnd()%255-127);+ for(int o=0;o<O;o++) scl[o]=0.001f+((int)(rnd()%1000))/1e6f;+ for(int i=0;i<I;i++) x[i]=((int)(rnd()%2000)-1000)/500.f;++ /* v1 (copy of olmoe.c matmul_q AVX2 IDOT branch) */+ #define MATMUL_Q_IDOT_V1(Y) do{ \+ int nb=I/16; int8_t xi[4096]; float xs[256]; \+ for(int b=0;b<nb;b++){ const float *xb=x+b*16; \+ float am=0.f; for(int i=0;i<16;i++){ float a=fabsf(xb[i]); if(a>am) am=a; } \+ float s=am/127.f; if(s<1e-12f) s=1e-12f; \+ xs[b]=s; float inv=1.f/s; \+ for(int i=0;i<16;i++) xi[b*16+i]=(int8_t)lrintf(xb[i]*inv); } \+ _Pragma("omp parallel for schedule(static)") \+ for(int o=0;o<O;o++){ const int8_t *w=q8+(int64_t)o*I; float acc=0.f; \+ for(int b=0;b<nb;b++) acc+=xs[b]*(float)dot_i8_16(xi+b*16,w+b*16); \+ (Y)[o]=acc*scl[o]; } }while(0)++ /* correctness vs int64 reference (int dot is exact; fp fold differs) */+ double e1=0,e2=0;+ for(int o=0;o<O;o+=O/8+1){+ int nb=I/16; int8_t xi[4096]; float xs[256];+ for(int b=0;b<nb;b++){ const float *xb=x+b*16;+ float am=0.f; for(int i=0;i<16;i++){ float a=fabsf(xb[i]); if(a>am) am=a; }+ float s=am/127.f; if(s<1e-12f) s=1e-12f;+ xs[b]=s; float inv=1.f/s;+ for(int i=0;i<16;i++) xi[b*16+i]=(int8_t)lrintf(xb[i]*inv); }+ double r=0; const int8_t *w=q8+(int64_t)o*I;+ for(int b=0;b<nb;b++){ int32_t d=0; for(int i=0;i<16;i++) d+=(int32_t)xi[b*16+i]*w[b*16+i];+ r+=(double)xs[b]*d; }+ r*=scl[o];+ MATMUL_Q_IDOT_V1(y1);+ matmul_q_idot_v2(y2,x,q8,scl,I,O);+ double d1=fabs(y1[o]-r)/(1+fabs(r)), d2=fabs(y2[o]-r)/(1+fabs(r));+ if(d1>e1)e1=d1; if(d2>e2)e2=d2;+ }+ printf("i8_O%d_I%d: relerr_v1=%.3g relerr_v2=%.3g\n",O,I,e1,e2);++ double t1=1e30,t2=1e30;+ for(int k=0;k<3;k++){ MATMUL_Q_IDOT_V1(y1); matmul_q_idot_v2(y2,x,q8,scl,I,O); }+ for(int r=0;r<reps;r++){+ double a=now_s(); MATMUL_Q_IDOT_V1(y1); double b=now_s();+ if(b-a<t1)t1=b-a;+ a=now_s(); matmul_q_idot_v2(y2,x,q8,scl,I,O); b=now_s();+ if(b-a<t2)t2=b-a;+ }+ double wbytes=(double)O*I;+ printf("i8_O%d_I%d: v1_ms=%.4f v2_ms=%.4f speedup=%.3f v1_GBs=%.1f v2_GBs=%.1f\n",+ O,I,t1*1e3,t2*1e3,t1/t2,wbytes/t1/1e9,wbytes/t2/1e9);+ free(q8);free(scl);free(x);free(y1);free(y2);+ #undef MATMUL_Q_IDOT_V1+ }++ /* ---- FUSED3: vectorized quant + gate/up pair ---------------------------+ * Contract: (a) quant_x_q8_avx2 output byte-identical to the scalar quant+ * (xi memcmp, xs bit-compare); (b) v3/pair-v3 row outputs bit-identical+ * to matmul_q_idot_v2; (c) timing on the two real expert shapes, with the+ * per-expert total = gate+up pair + down. */+ {+ int O=1024, I=2048, Od=2048, Id=1024; /* gate/up then down (OLMoE) */+ int8_t *qg=malloc((int64_t)O*I), *qu=malloc((int64_t)O*I), *qd=malloc((int64_t)Od*Id);+ float *sg=malloc(O*sizeof(float)), *su=malloc(O*sizeof(float)), *sd=malloc(Od*sizeof(float));+ float *x=malloc(I*sizeof(float)), *xd=malloc(Id*sizeof(float));+ float *yg=malloc(O*sizeof(float)), *yu=malloc(O*sizeof(float)), *yd=malloc(Od*sizeof(float));+ float *yg2=malloc(O*sizeof(float)), *yu2=malloc(O*sizeof(float)), *yd2=malloc(Od*sizeof(float));+ if(!qg||!qu||!qd||!sg||!su||!sd||!x||!xd||!yg||!yu||!yd||!yg2||!yu2||!yd2){ fprintf(stderr,"OOM\n"); return 1; }+ for(int64_t i=0;i<(int64_t)O*I;i++){ qg[i]=(int8_t)(rnd()%255-127); qu[i]=(int8_t)(rnd()%255-127); }+ for(int64_t i=0;i<(int64_t)Od*Id;i++) qd[i]=(int8_t)(rnd()%255-127);+ for(int o=0;o<O;o++){ sg[o]=0.001f+((int)(rnd()%1000))/1e6f; su[o]=0.001f+((int)(rnd()%1000))/1e6f; }+ for(int o=0;o<Od;o++) sd[o]=0.001f+((int)(rnd()%1000))/1e6f;+ for(int i=0;i<I;i++) x[i]=((int)(rnd()%2000)-1000)/500.f;+ for(int i=0;i<Id;i++) xd[i]=((int)(rnd()%2000)-1000)/500.f;+ x[37]=0.f; x[38]=0.f; /* near-zero block elements: exercise the s<1e-12 clamp path partially */++ /* (a) quant bit-identity vs the scalar loop */+ int nb=I/16; static int8_t xi_s[4096], xi_v[4096]; static float xs_s[256], xs_v[256];+ for(int b=0;b<nb;b++){ const float *xb=x+b*16;+ float am=0.f; for(int i=0;i<16;i++){ float a=fabsf(xb[i]); if(a>am) am=a; }+ float s=am/127.f; if(s<1e-12f) s=1e-12f;+ xs_s[b]=s; float inv=1.f/s;+ for(int i=0;i<16;i++) xi_s[b*16+i]=(int8_t)lrintf(xb[i]*inv); }+ quant_x_q8_avx2(x,xi_v,xs_v,I);+ int qbad=memcmp(xi_s,xi_v,I)!=0 || memcmp(xs_s,xs_v,nb*sizeof(float))!=0;+ /* all-zero block: clamp path */+ { float xz[64]={0}; int8_t xz_s[64],xz_v[64]; float xs2_s[4],xs2_v[4];+ for(int b=0;b<4;b++){ float am=0.f; float s=am/127.f; if(s<1e-12f) s=1e-12f;+ xs2_s[b]=s; float inv=1.f/s;+ for(int i=0;i<16;i++) xz_s[b*16+i]=(int8_t)lrintf(xz[b*16+i]*inv); }+ quant_x_q8_avx2(xz,xz_v,xs2_v,64);+ if(memcmp(xz_s,xz_v,64)!=0 || memcmp(xs2_s,xs2_v,4*sizeof(float))!=0) qbad=1; }+ printf("v3_quant_bitexact=%s\n", qbad?"FAIL":"yes");++ /* (b) v3 / pair-v3 outputs bit-identical to v2 */+ matmul_q_idot_v2(yg,x,qg,sg,I,O); matmul_q_idot_v3(yg2,x,qg,sg,I,O);+ int bid = memcmp(yg,yg2,O*sizeof(float))==0;+ matmul_q_idot_v2(yg,x,qg,sg,I,O); matmul_q_idot_v2(yu,x,qu,su,I,O);+ matmul_q_idot_pair_v3(yg2,yu2,x,qg,sg,qu,su,I,O);+ bid = bid && memcmp(yg,yg2,O*sizeof(float))==0 && memcmp(yu,yu2,O*sizeof(float))==0;+ matmul_q_idot_v2(yd,xd,qd,sd,Id,Od); matmul_q_idot_v3(yd2,xd,qd,sd,Id,Od);+ bid = bid && memcmp(yd,yd2,Od*sizeof(float))==0;+ printf("v3_output_bitidentical_v2=%s\n", bid?"yes":"FAIL");++ /* (c) per-expert timing: v2 (3 separate calls) vs v3 (pair + down) */+ double t2e=1e30,t3e=1e30;+ for(int k=0;k<3;k++){+ matmul_q_idot_v2(yg,x,qg,sg,I,O); matmul_q_idot_v2(yu,x,qu,su,I,O); matmul_q_idot_v2(yd,xd,qd,sd,Id,Od);+ matmul_q_idot_pair_v3(yg2,yu2,x,qg,sg,qu,su,I,O); matmul_q_idot_v3(yd,xd,qd,sd,Id,Od); }+ for(int r=0;r<reps;r++){+ double a=now_s();+ matmul_q_idot_v2(yg,x,qg,sg,I,O); matmul_q_idot_v2(yu,x,qu,su,I,O); matmul_q_idot_v2(yd,xd,qd,sd,Id,Od);+ double b=now_s(); if(b-a<t2e)t2e=b-a;+ a=now_s();+ matmul_q_idot_pair_v3(yg2,yu2,x,qg,sg,qu,su,I,O); matmul_q_idot_v3(yd,xd,qd,sd,Id,Od);+ b=now_s(); if(b-a<t3e)t3e=b-a;+ }+ printf("v3_expert: v2_3calls_ms=%.4f v3_pair+down_ms=%.4f speedup=%.3f\n",+ t2e*1e3,t3e*1e3,t2e/t3e);+ free(qg);free(qu);free(qd);free(sg);free(su);free(sd);free(x);free(xd);+ free(yg);free(yu);free(yd);free(yg2);free(yu2);free(yd2);+ }+ return 0;+}