Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
fix(sse41): preserve grouped-int4 scalar order
codex 20260830-005353-2548412838e09b3db9d2ec
diff --git a/c/Makefile b/c/Makefileindex d3f34064..1c9a88f2 100644--- a/c/Makefile+++ b/c/Makefile@@ -491,7 +491,7 @@ endifTEST_RULES := $(shell sed -n 's|^tests/\(test_[a-z0-9_]*\)\$$(EXE):.*|\1|p' $(firstword $(MAKEFILE_LIST)))# test_uring is Linux-only. V4 engine tests are appended below only on supported# x86-64 Linux/Windows and aarch64 Linux hosts; the V4 infrastructure tests have-# unconditional rules and therefore remain portable gates. The four forced+# unconditional rules and therefore remain portable gates. The six forced# SSE4.1 tests use -msse4.1/-mno-avx2/-mno-fma, which only exist as gcc/clang# flags on x86 -- unconditionally in TEST_BINS they take down `check` on# arm64 (e.g. macOS): "unsupported option '-msse4.1' for target arm64-...".@@ -504,14 +504,16 @@ TEST_EXCLUDE = test_uring test_deepseek_v4 test_v4_ownership test_v4_serve_framitest_edge_adapters_registration test_edge_adapters_real \test_qwen38_tier_engine \test_gsgemv_sse41 test_qgemv_sse41 test_olmoe_dot_i8_16_sse41 \- test_i4_grouped_sse41+ test_i4_grouped_sse41 test_i4_grouped_sse41_o1 \+ test_i4_grouped_sse41_no_contractTEST_BINS = $(addprefix tests/,$(addsuffix $(EXE),$(filter-out $(TEST_EXCLUDE),$(TEST_RULES))))ifneq (,$(LINUX))TEST_BINS += tests/test_uring$(EXE)endififneq (,$(X86_64))TEST_BINS += tests/test_gsgemv_sse41$(EXE) tests/test_qgemv_sse41$(EXE) \- tests/test_olmoe_dot_i8_16_sse41$(EXE) tests/test_i4_grouped_sse41$(EXE)+ tests/test_olmoe_dot_i8_16_sse41$(EXE) tests/test_i4_grouped_sse41$(EXE) \+ tests/test_i4_grouped_sse41_o1$(EXE) tests/test_i4_grouped_sse41_no_contract$(EXE)endififeq ($(COLI_V4_SUPPORTED),1)TEST_BINS += tests/test_v4_hybrid_policy$(EXE) tests/test_k3_fill_budget$(EXE) tests/test_v4_bank_pair$(EXE)@@ -1483,7 +1485,13 @@ tests/test_i4_grouped$(EXE): tests/test_i4_grouped.c colibri.c oracle.h st.h uri# Force the grouped-int4 oracle onto the SSE4.1 tier even on an AVX2 host.tests/test_i4_grouped_sse41$(EXE): tests/test_i4_grouped.c colibri.c oracle.h st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h idot.h sse41_kernels.h sample.h kv_persist.h telemetry.h route_trace.h- $(CC) $(CFLAGS) -DCOLI_HAVE_GROUPED_PAIR -msse4.1 -mno-avx2 -mno-fma $< -o $@ $(LDFLAGS)+ $(CC) $(CFLAGS) -O3 -DCOLI_HAVE_GROUPED_PAIR -DCOLI_I4_GROUPED_SCALAR_EXACT -msse4.1 -mno-avx2 -mno-fma $< -o $@ $(LDFLAGS)++tests/test_i4_grouped_sse41_o1$(EXE): tests/test_i4_grouped.c colibri.c oracle.h st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h idot.h sse41_kernels.h sample.h kv_persist.h telemetry.h route_trace.h+ $(CC) $(CFLAGS) -O1 -DCOLI_HAVE_GROUPED_PAIR -DCOLI_I4_GROUPED_SCALAR_EXACT -msse4.1 -mno-avx2 -mno-fma $< -o $@ $(LDFLAGS)++tests/test_i4_grouped_sse41_no_contract$(EXE): tests/test_i4_grouped.c colibri.c oracle.h st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h idot.h sse41_kernels.h sample.h kv_persist.h telemetry.h route_trace.h+ $(CC) $(CFLAGS) -O3 -ffp-contract=off -DCOLI_HAVE_GROUPED_PAIR -DCOLI_I4_GROUPED_SCALAR_EXACT -msse4.1 -mno-avx2 -mno-fma $< -o $@ $(LDFLAGS)tests/test_stops$(EXE): tests/test_stops.c colibri.c oracle.h st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h idot.h sample.h kv_persist.h telemetry.h route_trace.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)diff --git a/c/colibri.c b/c/colibri.cindex ead9853c..62d2c5f7 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -1010,13 +1010,62 @@ static void *xzalloc(size_t n, const char *what){/* Fused gate+up for grouped int4 (fmt=4): computes both yg[S,O] and yu[S,O] from* the same x[S,I], reading x once instead of twice β saves ~33% of expert-matmul time at decode.* The per-group scale logic matches matmul_i4_grouped exactly. */+#if defined(__SSE4_1__) && !defined(__AVX2__)+static void matmul_i4_grouped_pair_sse41_rows4(float *yg,float *yu,const float *x,+ const uint8_t *qg,const float *sg,+ const uint8_t *qu,const float *su,+ int S,int I,int O,int gs,int rb,+ int ng,int o4){+ #pragma omp parallel for schedule(static)+ for(int o=0;o<o4;o+=4){+ for(int s=0;s<S;s++){+ const float *xs=x+(int64_t)s*I;+ __m128 ag=_mm_setzero_ps(),au=_mm_setzero_ps();+ for(int g=0;g*gs<I;g++){+ int base=g*gs,end=base+gs; if(end>I) end=I;+ __m128 scg=colibri_f32_rows4(sg,ng,o,g);+ __m128 scu=colibri_f32_rows4(su,ng,o,g); int i=base;+ for(;i+1<end;i+=2){+ __m128 gl,gh,ul,uh;+ colibri_i4_rows4(qg,rb,o,i>>1,&gl,&gh);+ colibri_i4_rows4(qu,rb,o,i>>1,&ul,&uh);+ __m128 x0=_mm_set1_ps(xs[i]),x1=_mm_set1_ps(xs[i+1]);+ __m128 gp=_mm_add_ps(_mm_mul_ps(x0,gl),_mm_mul_ps(x1,gh));+ __m128 up=_mm_add_ps(_mm_mul_ps(x0,ul),_mm_mul_ps(x1,uh));+ ag=_mm_add_ps(ag,_mm_mul_ps(gp,scg));+ au=_mm_add_ps(au,_mm_mul_ps(up,scu));+ }+ if(i<end){+ __m128 gl,gh,ul,uh;+ colibri_i4_rows4(qg,rb,o,i>>1,&gl,&gh);+ colibri_i4_rows4(qu,rb,o,i>>1,&ul,&uh); (void)gh; (void)uh;+ __m128 xi=_mm_set1_ps(xs[i]);+ ag=_mm_add_ps(ag,_mm_mul_ps(_mm_mul_ps(xi,gl),scg));+ au=_mm_add_ps(au,_mm_mul_ps(_mm_mul_ps(xi,ul),scu));+ }+ }+ colibri_sse41_storeu_ps(yg+(int64_t)s*O+o,ag);+ colibri_sse41_storeu_ps(yu+(int64_t)s*O+o,au);+ }+ }+}+#endif+static void matmul_i4_grouped_pair(float *yg, float *yu, const float *x,const uint8_t *qg, const float *sg,const uint8_t *qu, const float *su,int S, int I, int O, int gs){int rb=(I+1)/2; int ng=(I+gs-1)/gs;+ int o0=0;+#if defined(__SSE4_1__) && !defined(__AVX2__)+ if(!(gs&1)){+ o0=O&~3;+ if(o0) matmul_i4_grouped_pair_sse41_rows4(yg,yu,x,qg,sg,qu,su,S,I,O,gs,rb,ng,o0);+ if(o0==O) return;+ }+#endif#pragma omp parallel for schedule(static)- for(int o=0;o<O;o++){+ for(int o=o0;o<O;o++){const uint8_t *wg=qg+(int64_t)o*rb; const uint8_t *wu2=qu+(int64_t)o*rb;const float *sgl=sg+(int64_t)o*ng; const float *sul=su+(int64_t)o*ng;for(int s=0;s<S;s++){@@ -1048,12 +1097,6 @@ static void matmul_i4_grouped_pair(float *yg, float *yu, const float *x,/* Same pinning as matmul_i4_grouped in quant.h: contraction* here is the compiler's choice, so the result was too. */ag=fmaf(hsum256(accg),scg,ag); au=fmaf(hsum256(accu),scu,au);-#elif defined(__SSE4_1__)- if(!(base&1)){- float dg,du;- dot_i4f_sse41_pair(wg+(base>>1),wu2+(base>>1),xs+base,glen,&dg,&du);- ag+=dg*scg; au+=du*scu; i=base+glen;- }#endiffor(; i+1<base+glen; i+=2){uint8_t bg=wg[i>>1], bu=wu2[i>>1];diff --git a/c/quant.h b/c/quant.hindex 251eb5e5..25c1e097 100644--- a/c/quant.h+++ b/c/quant.h@@ -19,67 +19,59 @@#if defined(__SSE4_1__)#include <immintrin.h>#include "sse41_kernels.h"+#endif-/* Exact-activation W4A32 dot for x86 CPUs below AVX2. Packed weights use- * offset nibbles: 0 means -8 and 15 means +7. */-static inline float dot_i4f_sse41(const uint8_t *w,const float *x,int I){+#if defined(__SSE4_1__) && !defined(__AVX2__)+/* Load one packed byte from each of four output rows, then unpack their low and+ * high offset nibbles into four f32 lanes. Keeping independent output rows in+ * the lanes preserves the scalar operation order within every row. */+static inline void colibri_i4_rows4(const uint8_t *q4,int rb,int o,int byte,+ __m128 *lo,__m128 *hi){const __m128i m4=_mm_set1_epi8(0x0F), b8=_mm_set1_epi8(8);- __m128 a0=_mm_setzero_ps(),a1=_mm_setzero_ps(); int i=0;- for(;i+16<=I;i+=16){- __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1)));- __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);- __m128i q=_mm_sub_epi8(_mm_unpacklo_epi8(lo,hi),b8);- a0=COLIBRI_FMA(colibri_sse41_loadu_ps(x+i),- _mm_cvtepi32_ps(_mm_cvtepi8_epi32(q)),a0);- a1=COLIBRI_FMA(colibri_sse41_loadu_ps(x+i+4),- _mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_srli_si128(q,4))),a1);- a0=COLIBRI_FMA(colibri_sse41_loadu_ps(x+i+8),- _mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_srli_si128(q,8))),a0);- a1=COLIBRI_FMA(colibri_sse41_loadu_ps(x+i+12),- _mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_srli_si128(q,12))),a1);- }- float sum=colibri_sse41_hsum_ps(_mm_add_ps(a0,a1));- for(;i+1<I;i+=2){ uint8_t b=w[i>>1];- sum+=x[i]*(float)((int)(b&15)-8)+x[i+1]*(float)((int)(b>>4)-8); }- if(i<I) sum+=x[i]*(float)((int)(w[i>>1]&15)-8);- return sum;+ uint32_t packed=(uint32_t)q4[(int64_t)(o+0)*rb+byte]+ |(uint32_t)q4[(int64_t)(o+1)*rb+byte]<<8+ |(uint32_t)q4[(int64_t)(o+2)*rb+byte]<<16+ |(uint32_t)q4[(int64_t)(o+3)*rb+byte]<<24;+ __m128i by=_mm_cvtsi32_si128((int)packed);+ __m128i qlo=_mm_sub_epi8(_mm_and_si128(by,m4),b8);+ __m128i qhi=_mm_sub_epi8(_mm_and_si128(_mm_srli_epi16(by,4),m4),b8);+ *lo=_mm_cvtepi32_ps(_mm_cvtepi8_epi32(qlo));+ *hi=_mm_cvtepi32_ps(_mm_cvtepi8_epi32(qhi));}-/* Gate and up share the same activation row. Keep both projections in flight- * so the SSE4.1 path loads that row once. */-static inline void dot_i4f_sse41_pair(const uint8_t *wg,const uint8_t *wu,- const float *x,int I,float *g,float *u){- const __m128i m4=_mm_set1_epi8(0x0F), b8=_mm_set1_epi8(8);- __m128 g0=_mm_setzero_ps(),g1=_mm_setzero_ps();- __m128 u0=_mm_setzero_ps(),u1=_mm_setzero_ps(); int i=0;- for(;i+16<=I;i+=16){- __m128 x0=colibri_sse41_loadu_ps(x+i), x1=colibri_sse41_loadu_ps(x+i+4);- __m128 x2=colibri_sse41_loadu_ps(x+i+8), x3=colibri_sse41_loadu_ps(x+i+12);- __m128i bg=_mm_loadl_epi64((const __m128i*)(wg+(i>>1)));- __m128i lg=_mm_and_si128(bg,m4), hg=_mm_and_si128(_mm_srli_epi16(bg,4),m4);- __m128i qg=_mm_sub_epi8(_mm_unpacklo_epi8(lg,hg),b8);- g0=COLIBRI_FMA(x0,_mm_cvtepi32_ps(_mm_cvtepi8_epi32(qg)),g0);- g1=COLIBRI_FMA(x1,_mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_srli_si128(qg,4))),g1);- g0=COLIBRI_FMA(x2,_mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_srli_si128(qg,8))),g0);- g1=COLIBRI_FMA(x3,_mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_srli_si128(qg,12))),g1);- __m128i bu=_mm_loadl_epi64((const __m128i*)(wu+(i>>1)));- __m128i lu=_mm_and_si128(bu,m4), hu=_mm_and_si128(_mm_srli_epi16(bu,4),m4);- __m128i qu=_mm_sub_epi8(_mm_unpacklo_epi8(lu,hu),b8);- u0=COLIBRI_FMA(x0,_mm_cvtepi32_ps(_mm_cvtepi8_epi32(qu)),u0);- u1=COLIBRI_FMA(x1,_mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_srli_si128(qu,4))),u1);- u0=COLIBRI_FMA(x2,_mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_srli_si128(qu,8))),u0);- u1=COLIBRI_FMA(x3,_mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_srli_si128(qu,12))),u1);- }- float ag=colibri_sse41_hsum_ps(_mm_add_ps(g0,g1));- float au=colibri_sse41_hsum_ps(_mm_add_ps(u0,u1));- for(;i+1<I;i+=2){ uint8_t bg=wg[i>>1],bu=wu[i>>1];- ag+=x[i]*(float)((int)(bg&15)-8)+x[i+1]*(float)((int)(bg>>4)-8);- au+=x[i]*(float)((int)(bu&15)-8)+x[i+1]*(float)((int)(bu>>4)-8); }- if(i<I){- ag+=x[i]*(float)((int)(wg[i>>1]&15)-8);- au+=x[i]*(float)((int)(wu[i>>1]&15)-8);+static inline __m128 colibri_f32_rows4(const float *p,int stride,int o,int i){+ return _mm_set_ps(p[(int64_t)(o+3)*stride+i],p[(int64_t)(o+2)*stride+i],+ p[(int64_t)(o+1)*stride+i],p[(int64_t)(o+0)*stride+i]);+}++/* Process four output rows at once without a horizontal reduction. Each lane+ * uses the scalar kernel's pair sum, scale multiply, and accumulator add in+ * the same order, so the result can remain byte-identical on pre-FMA CPUs. */+static void matmul_i4_grouped_sse41_rows4(float *y,const float *x,+ const uint8_t *q4,const float *scale,+ int S,int I,int O,int gs,int rb,int ng,+ int o4){+ #pragma omp parallel for schedule(static)+ for(int o=0;o<o4;o+=4){+ for(int s=0;s<S;s++){+ const float *xs=x+(int64_t)s*I; __m128 a=_mm_setzero_ps();+ for(int g=0;g*gs<I;g++){+ int base=g*gs,end=base+gs; if(end>I) end=I;+ __m128 sc=colibri_f32_rows4(scale,ng,o,g); int i=base;+ for(;i+1<end;i+=2){+ __m128 lo,hi; colibri_i4_rows4(q4,rb,o,i>>1,&lo,&hi);+ __m128 pair=_mm_add_ps(_mm_mul_ps(_mm_set1_ps(xs[i]),lo),+ _mm_mul_ps(_mm_set1_ps(xs[i+1]),hi));+ a=_mm_add_ps(a,_mm_mul_ps(pair,sc));+ }+ if(i<end){+ __m128 lo,hi; colibri_i4_rows4(q4,rb,o,i>>1,&lo,&hi); (void)hi;+ a=_mm_add_ps(a,_mm_mul_ps(_mm_mul_ps(_mm_set1_ps(xs[i]),lo),sc));+ }+ }+ colibri_sse41_storeu_ps(y+(int64_t)s*O+o,a);+ }}- *g=ag; *u=au;}#endif/* ---- AVX-512 int4->float accumulator -------------------------------------- */@@ -207,8 +199,17 @@ static void matmul_i4(float *y, const float *x, const uint8_t *q4, const float *static void matmul_i4_grouped(float *y, const float *x, const uint8_t *q4, const float *scale,int S, int I, int O, int gs){int rb=(I+1)/2; int ng=(I+gs-1)/gs;+ int o0=0;+#if defined(__SSE4_1__) && !defined(__AVX2__)+ /* Even group sizes keep every group start on a low-nibble boundary. */+ if(!(gs&1)){+ o0=O&~3;+ if(o0) matmul_i4_grouped_sse41_rows4(y,x,q4,scale,S,I,O,gs,rb,ng,o0);+ if(o0==O) return;+ }+#endif#pragma omp parallel for schedule(static)- for(int o=0;o<O;o++){+ for(int o=o0;o<O;o++){const uint8_t *w=q4+(int64_t)o*rb;const float *scl=scale+(int64_t)o*ng;for(int s=0;s<S;s++){@@ -235,13 +236,6 @@ static void matmul_i4_grouped(float *y, const float *x, const uint8_t *q4, const* the glm_tiny token oracle, depend on build flags rather than* on the code. */a=fmaf(hsum256(acc),sc,a);-#elif defined(__SSE4_1__)- /* Group starts must be nibble-aligned before the packed row- * can be passed as an independent SSE4.1 dot product. */- if(!(base&1)){- a+=dot_i4f_sse41(w+(base>>1),xs+base,glen)*sc;- i=base+glen;- }#endiffor(; i<base+glen; i+=2){if(i+1<base+glen){ uint8_t byte=w[i>>1];diff --git a/c/sse41_kernels.h b/c/sse41_kernels.hindex 60ec53ae..6c31dcce 100644--- a/c/sse41_kernels.h+++ b/c/sse41_kernels.h@@ -47,13 +47,6 @@static inline __m128 colibri_sse41_loadu_ps(const float *p) { return _mm_loadu_ps(p); }static inline void colibri_sse41_storeu_ps(float *p, __m128 v) { _mm_storeu_ps(p, v); }-/* Fixed four-lane reduction used by the grouped-int4 GEMV kernels. */-static inline float colibri_sse41_hsum_ps(__m128 v) {- v = _mm_add_ps(v, _mm_movehl_ps(v, v));- v = _mm_add_ss(v, _mm_shuffle_ps(v, v, 1));- return _mm_cvtss_f32(v);-}-/** Min/max (SSE 4.1 native). Identical to AVX2, just narrower width.*/diff --git a/c/tests/test_i4_grouped.c b/c/tests/test_i4_grouped.cindex 26eadcc4..a6dd3f33 100644--- a/c/tests/test_i4_grouped.c+++ b/c/tests/test_i4_grouped.c@@ -12,11 +12,9 @@* nibble edges 0 and 15 (which decode to -8 and +7 β an offset encoding, NOT* two's complement; getting this backwards is silent and looks like noise).*- * FP note: the kernel sums each group in f32 (AVX2 accumulator + scalar tail)- * while the reference sums in double, so we compare against a relative epsilon- * rather than bit-exactly. The tolerance is tight enough that a wrong scale- * index, a wrong group boundary or a swapped nibble cannot hide under it β- * those are O(1) relative errors, not O(1e-6). */+ * FP note: the kernel sums in f32 while the format reference sums in double, so+ * we compare those two against a relative epsilon. Forced-SSE builds separately+ * require byte identity with the scalar f32 operation tree. */#define main coli_glm_main_unused#include "../colibri.c"#undef main@@ -53,15 +51,54 @@ static void ref_grouped(double *y, double *mag, const float *x, const uint8_t *q}}+/* Scalar contract: this is the scalar grouped kernel's operation tree, not a+ * higher-precision format reference. The forced-SSE builds compare its output+ * byte for byte with the four-output-row SIMD path. */+#ifdef COLI_I4_GROUPED_SCALAR_EXACT+static void ref_grouped_scalar(float *y,const float *x,const uint8_t *q4,+ const float *scale,int S,int I,int O,int gs){+ int rb=(I+1)/2,ng=(I+gs-1)/gs;+ for(int o=0;o<O;o++){+ const uint8_t *w=q4+(int64_t)o*rb;+ const float *scl=scale+(int64_t)o*ng;+ for(int s=0;s<S;s++){+ const float *xs=x+(int64_t)s*I; float a=0;+ for(int g=0;g*gs<I;g++){+ int base=g*gs,glen=gs; if(base+glen>I) glen=I-base;+ float sc=scl[g];+ for(int i=base;i<base+glen;i+=2){+ if(i+1<base+glen){+ uint8_t byte=w[i>>1];+ a+=(xs[i]*(float)((int)(byte&0xF)-8)+ +xs[i+1]*(float)((int)(byte>>4)-8))*sc;+ }else{+ uint8_t byte=w[i>>1];+ a+=xs[i]*(float)((int)(byte&0xF)-8)*sc;+ }+ }+ }+ y[(int64_t)s*O+o]=a;+ }+ }+}+#endif+static int check(const char *name, int S, int I, int O, int gs, int fill_edges){int rb=(I+1)/2, ng=(I+gs-1)/gs;uint8_t *q4=malloc((size_t)O*rb);float *scale=malloc((size_t)O*ng*sizeof(float));float *x=malloc((size_t)S*I*sizeof(float));float *y=malloc((size_t)S*O*sizeof(float));+#ifdef COLI_I4_GROUPED_SCALAR_EXACT+ float *ys=malloc((size_t)S*O*sizeof(float));+#endifdouble *yr=malloc((size_t)S*O*sizeof(double));double *ym=malloc((size_t)S*O*sizeof(double));- if(!q4||!scale||!x||!y||!yr||!ym){ fprintf(stderr,"%s: OOM\n",name); return 1; }+ if(!q4||!scale||!x||!y||!yr||!ym+#ifdef COLI_I4_GROUPED_SCALAR_EXACT+ ||!ys+#endif+ ){ fprintf(stderr,"%s: OOM\n",name); return 1; }for(size_t i=0;i<(size_t)O*rb;i++) q4[i]=(uint8_t)(xr()&0xFF);if(fill_edges){@@ -75,6 +112,9 @@ static int check(const char *name, int S, int I, int O, int gs, int fill_edges){matmul_i4_grouped(y,x,q4,scale,S,I,O,gs);ref_grouped(yr,ym,x,q4,scale,S,I,O,gs);+#ifdef COLI_I4_GROUPED_SCALAR_EXACT+ ref_grouped_scalar(ys,x,q4,scale,S,I,O,gs);+#endifint bad=0; double worst=0;for(int i=0;i<S*O;i++){@@ -87,9 +127,26 @@ static int check(const char *name, int S, int I, int O, int gs, int fill_edges){bad++;}}+#ifdef COLI_I4_GROUPED_SCALAR_EXACT+ if(memcmp(y,ys,(size_t)S*O*sizeof(float))!=0){+ int first=0;+ while(first<S*O && memcmp(y+first,ys+first,sizeof(float))==0) first++;+ fprintf(stderr,"%s: FAIL SSE4.1 != scalar at [%d]: %.9g vs %.9g\n",+ name,first,(double)y[first],(double)ys[first]);+ bad++;+ }+ free(ys);+#endiffree(q4);free(scale);free(x);free(y);free(yr);free(ym);if(bad){ fprintf(stderr,"%s: FAIL (%d/%d mismatched, worst rel %.3g)\n",name,bad,S*O,worst); return 1; }- printf(" %-42s ok (S=%d I=%d O=%d gs=%d ng=%d, worst rel %.2g)\n",name,S,I,O,gs,ng,worst);+ printf(" %-42s ok (S=%d I=%d O=%d gs=%d ng=%d, worst rel %.2g%s)\n",+ name,S,I,O,gs,ng,worst,+#ifdef COLI_I4_GROUPED_SCALAR_EXACT+ ", bit-exact vs scalar"+#else+ ""+#endif+ );return 0;}@@ -98,16 +155,12 @@ static int check(const char *name, int S, int I, int O, int gs, int fill_edges){** - Correctness, always: both outputs must match the double reference within* the same magnitude-relative epsilon as the unfused kernel.- * - Bit-exactness, only when I % gs == 0: then every group is covered by the- * same SIMD or scalar body in both kernels, so- * the results agree to the last bit. This is the shape the real g64- * checkpoints have (I = 2048 / 6144, gs = 64), i.e. the production path.+ * - Bit-exactness: forced-SSE builds require every shape to match two scalar+ * calls. Other builds retain the aligned-shape invariant used by AVX2.*- * With a PARTIAL last group the group tail falls to scalar code, and the- * compiler is free to contract/reassociate the fused body differently from the- * single-matrix one. The results then differ by ~1e-7 -- rounding, not logic- * (which of gate/up "differs" is arbitrary, the tell that it is FP luck).- * Demanding bit-exactness there would report a compiler artifact as a bug. */+ * On AVX2, a PARTIAL last group falls to scalar code and the fused body can+ * round differently from the single-matrix one. The forced-SSE rows4 path has+ * no horizontal reduction or group tail and is exact for those shapes too. */#ifdef COLI_HAVE_GROUPED_PAIRstatic int check_pair(const char *name, int S, int I, int O, int gs){int rb=(I+1)/2, ng=(I+gs-1)/gs;@@ -143,18 +196,20 @@ static int check_pair(const char *name, int S, int I, int O, int gs){}if(yg[i]!=rg[i]||yu[i]!=ru[i]) exact=0;}- /* Aligned shapes use the same accumulation order as the unfused kernel, so- * bit-exactness is a real invariant there and worth asserting. */- if(I%gs==0 && !exact){- fprintf(stderr,"%s: FAIL fused != unfused bitwise on an ALIGNED shape "- "(no scalar tail runs here; the orders must match)\n",name);+ int require_exact=I%gs==0;+#ifdef COLI_I4_GROUPED_SCALAR_EXACT+ require_exact=1;+#endif+ if(require_exact && !exact){+ fprintf(stderr,"%s: FAIL fused != unfused bitwise "+ "(the operation order must match)\n",name);bad++;}free(qg);free(qu);free(sg);free(su);free(x);free(yg);free(yu);free(rg);free(ru);free(dg);free(du);free(mg);free(mu);if(bad){ fprintf(stderr,"%s: FAIL (%d mismatched)\n",name,bad); return 1; }printf(" %-42s ok (S=%d I=%d O=%d gs=%d, worst rel %.2g%s)\n",name,S,I,O,gs,worst,- I%gs==0?", bit-exact vs unfused":"");+ require_exact?", bit-exact vs unfused":"");return 0;}#endif