Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
perf(sse41): vectorize grouped-int4 GEMV
codex 20260829-144356-2670143244d357448aa3e
diff --git a/c/Makefile b/c/Makefileindex fe37e6bb..d3f34064 100644--- a/c/Makefile+++ b/c/Makefile@@ -491,11 +491,10 @@ endifTEST_RULES := $(shell sed -n 's|^tests/\(test_[a-z0-9_]*\)\$$(EXE):.*|\1|p' $(firstword $(MAKEFILE_LIST)))# test_uring is Linux-only. V4 engine tests are appended below only on supported# x86-64 Linux/Windows and aarch64 Linux hosts; the V4 infrastructure tests have-# unconditional rules and therefore remain portable gates. test_gsgemv_sse41/-# test_qgemv_sse41/test_olmoe_dot_i8_16_sse41 force -msse4.1/-mno-avx2/-mno-fma,-# which only exist as gcc/clang flags on x86 -- unconditionally in TEST_BINS-# they take down `check` on arm64 (e.g. macOS):-# "unsupported option '-msse4.1' for target arm64-...".+# unconditional rules and therefore remain portable gates. The four forced+# SSE4.1 tests use -msse4.1/-mno-avx2/-mno-fma, which only exist as gcc/clang+# flags on x86 -- unconditionally in TEST_BINS they take down `check` on+# arm64 (e.g. macOS): "unsupported option '-msse4.1' for target arm64-...".# Appended back below only on x86-64 hosts, alongside the other conditional# platform tests already handled the same way.# test_qwen38_tier_engine drives qwen38.c over the generated FP8 fixture@@ -503,13 +502,16 @@ TEST_RULES := $(shell sed -n 's|^tests/\(test_[a-z0-9_]*\)\$$(EXE):.*|\1|p' $(fTEST_EXCLUDE = test_uring test_deepseek_v4 test_v4_ownership test_v4_serve_framing \test_segment_adapters_registration test_segment_adapters_real \test_edge_adapters_registration test_edge_adapters_real \- test_gsgemv_sse41 test_qgemv_sse41 test_olmoe_dot_i8_16_sse41 test_qwen38_tier_engine+ test_qwen38_tier_engine \+ test_gsgemv_sse41 test_qgemv_sse41 test_olmoe_dot_i8_16_sse41 \+ test_i4_grouped_sse41TEST_BINS = $(addprefix tests/,$(addsuffix $(EXE),$(filter-out $(TEST_EXCLUDE),$(TEST_RULES))))ifneq (,$(LINUX))TEST_BINS += tests/test_uring$(EXE)endififneq (,$(X86_64))-TEST_BINS += tests/test_gsgemv_sse41$(EXE) tests/test_qgemv_sse41$(EXE) tests/test_olmoe_dot_i8_16_sse41$(EXE)+TEST_BINS += tests/test_gsgemv_sse41$(EXE) tests/test_qgemv_sse41$(EXE) \+ tests/test_olmoe_dot_i8_16_sse41$(EXE) tests/test_i4_grouped_sse41$(EXE)endififeq ($(COLI_V4_SUPPORTED),1)TEST_BINS += tests/test_v4_hybrid_policy$(EXE) tests/test_k3_fill_budget$(EXE) tests/test_v4_bank_pair$(EXE)@@ -1476,8 +1478,12 @@ tests/test_qwen36_cache_index$(EXE): tests/test_qwen36_cache_index.c qwen36.c extests/test_idot$(EXE): tests/test_idot.c colibri.c oracle.h st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h idot.h sample.h kv_persist.h telemetry.h route_trace.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)-tests/test_i4_grouped$(EXE): tests/test_i4_grouped.c colibri.c oracle.h st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h idot.h sample.h kv_persist.h telemetry.h route_trace.h- $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+tests/test_i4_grouped$(EXE): tests/test_i4_grouped.c colibri.c oracle.h st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h idot.h sse41_kernels.h sample.h kv_persist.h telemetry.h route_trace.h+ $(CC) $(CFLAGS) -DCOLI_HAVE_GROUPED_PAIR $< -o $@ $(LDFLAGS)++# Force the grouped-int4 oracle onto the SSE4.1 tier even on an AVX2 host.+tests/test_i4_grouped_sse41$(EXE): tests/test_i4_grouped.c colibri.c oracle.h st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h idot.h sse41_kernels.h sample.h kv_persist.h telemetry.h route_trace.h+ $(CC) $(CFLAGS) -DCOLI_HAVE_GROUPED_PAIR -msse4.1 -mno-avx2 -mno-fma $< -o $@ $(LDFLAGS)tests/test_stops$(EXE): tests/test_stops.c colibri.c oracle.h st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h idot.h sample.h kv_persist.h telemetry.h route_trace.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)diff --git a/c/colibri.c b/c/colibri.cindex a96ccba4..ead9853c 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -1048,6 +1048,12 @@ static void matmul_i4_grouped_pair(float *yg, float *yu, const float *x,/* Same pinning as matmul_i4_grouped in quant.h: contraction* here is the compiler's choice, so the result was too. */ag=fmaf(hsum256(accg),scg,ag); au=fmaf(hsum256(accu),scu,au);+#elif defined(__SSE4_1__)+ if(!(base&1)){+ float dg,du;+ dot_i4f_sse41_pair(wg+(base>>1),wu2+(base>>1),xs+base,glen,&dg,&du);+ ag+=dg*scg; au+=du*scu; i=base+glen;+ }#endiffor(; i+1<base+glen; i+=2){uint8_t bg=wg[i>>1], bu=wu2[i>>1];diff --git a/c/quant.h b/c/quant.hindex 91cf7549..251eb5e5 100644--- a/c/quant.h+++ b/c/quant.h@@ -15,6 +15,73 @@#endif#include "idot.h" /* SIMD prelude + the integer dot kernels, shared with qwen36 */++#if defined(__SSE4_1__)+#include <immintrin.h>+#include "sse41_kernels.h"++/* Exact-activation W4A32 dot for x86 CPUs below AVX2. Packed weights use+ * offset nibbles: 0 means -8 and 15 means +7. */+static inline float dot_i4f_sse41(const uint8_t *w,const float *x,int I){+ const __m128i m4=_mm_set1_epi8(0x0F), b8=_mm_set1_epi8(8);+ __m128 a0=_mm_setzero_ps(),a1=_mm_setzero_ps(); int i=0;+ for(;i+16<=I;i+=16){+ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1)));+ __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);+ __m128i q=_mm_sub_epi8(_mm_unpacklo_epi8(lo,hi),b8);+ a0=COLIBRI_FMA(colibri_sse41_loadu_ps(x+i),+ _mm_cvtepi32_ps(_mm_cvtepi8_epi32(q)),a0);+ a1=COLIBRI_FMA(colibri_sse41_loadu_ps(x+i+4),+ _mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_srli_si128(q,4))),a1);+ a0=COLIBRI_FMA(colibri_sse41_loadu_ps(x+i+8),+ _mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_srli_si128(q,8))),a0);+ a1=COLIBRI_FMA(colibri_sse41_loadu_ps(x+i+12),+ _mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_srli_si128(q,12))),a1);+ }+ float sum=colibri_sse41_hsum_ps(_mm_add_ps(a0,a1));+ for(;i+1<I;i+=2){ uint8_t b=w[i>>1];+ sum+=x[i]*(float)((int)(b&15)-8)+x[i+1]*(float)((int)(b>>4)-8); }+ if(i<I) sum+=x[i]*(float)((int)(w[i>>1]&15)-8);+ return sum;+}++/* Gate and up share the same activation row. Keep both projections in flight+ * so the SSE4.1 path loads that row once. */+static inline void dot_i4f_sse41_pair(const uint8_t *wg,const uint8_t *wu,+ const float *x,int I,float *g,float *u){+ const __m128i m4=_mm_set1_epi8(0x0F), b8=_mm_set1_epi8(8);+ __m128 g0=_mm_setzero_ps(),g1=_mm_setzero_ps();+ __m128 u0=_mm_setzero_ps(),u1=_mm_setzero_ps(); int i=0;+ for(;i+16<=I;i+=16){+ __m128 x0=colibri_sse41_loadu_ps(x+i), x1=colibri_sse41_loadu_ps(x+i+4);+ __m128 x2=colibri_sse41_loadu_ps(x+i+8), x3=colibri_sse41_loadu_ps(x+i+12);+ __m128i bg=_mm_loadl_epi64((const __m128i*)(wg+(i>>1)));+ __m128i lg=_mm_and_si128(bg,m4), hg=_mm_and_si128(_mm_srli_epi16(bg,4),m4);+ __m128i qg=_mm_sub_epi8(_mm_unpacklo_epi8(lg,hg),b8);+ g0=COLIBRI_FMA(x0,_mm_cvtepi32_ps(_mm_cvtepi8_epi32(qg)),g0);+ g1=COLIBRI_FMA(x1,_mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_srli_si128(qg,4))),g1);+ g0=COLIBRI_FMA(x2,_mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_srli_si128(qg,8))),g0);+ g1=COLIBRI_FMA(x3,_mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_srli_si128(qg,12))),g1);+ __m128i bu=_mm_loadl_epi64((const __m128i*)(wu+(i>>1)));+ __m128i lu=_mm_and_si128(bu,m4), hu=_mm_and_si128(_mm_srli_epi16(bu,4),m4);+ __m128i qu=_mm_sub_epi8(_mm_unpacklo_epi8(lu,hu),b8);+ u0=COLIBRI_FMA(x0,_mm_cvtepi32_ps(_mm_cvtepi8_epi32(qu)),u0);+ u1=COLIBRI_FMA(x1,_mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_srli_si128(qu,4))),u1);+ u0=COLIBRI_FMA(x2,_mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_srli_si128(qu,8))),u0);+ u1=COLIBRI_FMA(x3,_mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_srli_si128(qu,12))),u1);+ }+ float ag=colibri_sse41_hsum_ps(_mm_add_ps(g0,g1));+ float au=colibri_sse41_hsum_ps(_mm_add_ps(u0,u1));+ for(;i+1<I;i+=2){ uint8_t bg=wg[i>>1],bu=wu[i>>1];+ ag+=x[i]*(float)((int)(bg&15)-8)+x[i+1]*(float)((int)(bg>>4)-8);+ au+=x[i]*(float)((int)(bu&15)-8)+x[i+1]*(float)((int)(bu>>4)-8); }+ if(i<I){+ ag+=x[i]*(float)((int)(wg[i>>1]&15)-8);+ au+=x[i]*(float)((int)(wu[i>>1]&15)-8);+ }+ *g=ag; *u=au;+}+#endif/* ---- AVX-512 int4->float accumulator -------------------------------------- */#if defined(__AVX512F__) && defined(__AVX512BW__)static int g_i4_acc512=1;@@ -168,6 +235,13 @@ static void matmul_i4_grouped(float *y, const float *x, const uint8_t *q4, const* the glm_tiny token oracle, depend on build flags rather than* on the code. */a=fmaf(hsum256(acc),sc,a);+#elif defined(__SSE4_1__)+ /* Group starts must be nibble-aligned before the packed row+ * can be passed as an independent SSE4.1 dot product. */+ if(!(base&1)){+ a+=dot_i4f_sse41(w+(base>>1),xs+base,glen)*sc;+ i=base+glen;+ }#endiffor(; i<base+glen; i+=2){if(i+1<base+glen){ uint8_t byte=w[i>>1];diff --git a/c/sse41_kernels.h b/c/sse41_kernels.hindex 6c31dcce..60ec53ae 100644--- a/c/sse41_kernels.h+++ b/c/sse41_kernels.h@@ -47,6 +47,13 @@static inline __m128 colibri_sse41_loadu_ps(const float *p) { return _mm_loadu_ps(p); }static inline void colibri_sse41_storeu_ps(float *p, __m128 v) { _mm_storeu_ps(p, v); }+/* Fixed four-lane reduction used by the grouped-int4 GEMV kernels. */+static inline float colibri_sse41_hsum_ps(__m128 v) {+ v = _mm_add_ps(v, _mm_movehl_ps(v, v));+ v = _mm_add_ss(v, _mm_shuffle_ps(v, v, 1));+ return _mm_cvtss_f32(v);+}+/** Min/max (SSE 4.1 native). Identical to AVX2, just narrower width.*/diff --git a/c/tests/test_i4_grouped.c b/c/tests/test_i4_grouped.cindex 04e8d887..26eadcc4 100644--- a/c/tests/test_i4_grouped.c+++ b/c/tests/test_i4_grouped.c@@ -99,7 +99,7 @@ static int check(const char *name, int S, int I, int O, int gs, int fill_edges){* - Correctness, always: both outputs must match the double reference within* the same magnitude-relative epsilon as the unfused kernel.* - Bit-exactness, only when I % gs == 0: then every group is covered by the- * AVX2 body, whose accumulation order is identical to the unfused kernel, so+ * same SIMD or scalar body in both kernels, so* the results agree to the last bit. This is the shape the real g64* checkpoints have (I = 2048 / 6144, gs = 64), i.e. the production path.*@@ -143,8 +143,8 @@ static int check_pair(const char *name, int S, int I, int O, int gs){}if(yg[i]!=rg[i]||yu[i]!=ru[i]) exact=0;}- /* Aligned shapes run entirely through the AVX2 body: same order as unfused,- * so bit-exactness is a real invariant there and worth asserting. */+ /* Aligned shapes use the same accumulation order as the unfused kernel, so+ * bit-exactness is a real invariant there and worth asserting. */if(I%gs==0 && !exact){fprintf(stderr,"%s: FAIL fused != unfused bitwise on an ALIGNED shape ""(no scalar tail runs here; the orders must match)\n",name);