Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

perf(sse41): vectorize grouped-int4 GEMV

codex 20260829-144356-2670143244d357448aa3e
diff --git a/c/Makefile b/c/Makefile
index fe37e6bb..d3f34064 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -491,11 +491,10 @@ endif
TEST_RULES := $(shell sed -n 's|^tests/\(test_[a-z0-9_]*\)\$$(EXE):.*|\1|p' $(firstword $(MAKEFILE_LIST)))
# test_uring is Linux-only. V4 engine tests are appended below only on supported
# x86-64 Linux/Windows and aarch64 Linux hosts; the V4 infrastructure tests have
-# unconditional rules and therefore remain portable gates. test_gsgemv_sse41/
-# test_qgemv_sse41/test_olmoe_dot_i8_16_sse41 force -msse4.1/-mno-avx2/-mno-fma,
-# which only exist as gcc/clang flags on x86 -- unconditionally in TEST_BINS
-# they take down `check` on arm64 (e.g. macOS):
-# "unsupported option '-msse4.1' for target arm64-...".
+# unconditional rules and therefore remain portable gates. The four forced
+# SSE4.1 tests use -msse4.1/-mno-avx2/-mno-fma, which only exist as gcc/clang
+# flags on x86 -- unconditionally in TEST_BINS they take down `check` on
+# arm64 (e.g. macOS): "unsupported option '-msse4.1' for target arm64-...".
# Appended back below only on x86-64 hosts, alongside the other conditional
# platform tests already handled the same way.
# test_qwen38_tier_engine drives qwen38.c over the generated FP8 fixture
@@ -503,13 +502,16 @@ TEST_RULES := $(shell sed -n 's|^tests/\(test_[a-z0-9_]*\)\$$(EXE):.*|\1|p' $(f
TEST_EXCLUDE = test_uring test_deepseek_v4 test_v4_ownership test_v4_serve_framing \
test_segment_adapters_registration test_segment_adapters_real \
test_edge_adapters_registration test_edge_adapters_real \
- test_gsgemv_sse41 test_qgemv_sse41 test_olmoe_dot_i8_16_sse41 test_qwen38_tier_engine
+ test_qwen38_tier_engine \
+ test_gsgemv_sse41 test_qgemv_sse41 test_olmoe_dot_i8_16_sse41 \
+ test_i4_grouped_sse41
TEST_BINS = $(addprefix tests/,$(addsuffix $(EXE),$(filter-out $(TEST_EXCLUDE),$(TEST_RULES))))
ifneq (,$(LINUX))
TEST_BINS += tests/test_uring$(EXE)
endif
ifneq (,$(X86_64))
-TEST_BINS += tests/test_gsgemv_sse41$(EXE) tests/test_qgemv_sse41$(EXE) tests/test_olmoe_dot_i8_16_sse41$(EXE)
+TEST_BINS += tests/test_gsgemv_sse41$(EXE) tests/test_qgemv_sse41$(EXE) \
+ tests/test_olmoe_dot_i8_16_sse41$(EXE) tests/test_i4_grouped_sse41$(EXE)
endif
ifeq ($(COLI_V4_SUPPORTED),1)
TEST_BINS += tests/test_v4_hybrid_policy$(EXE) tests/test_k3_fill_budget$(EXE) tests/test_v4_bank_pair$(EXE)
@@ -1476,8 +1478,12 @@ tests/test_qwen36_cache_index$(EXE): tests/test_qwen36_cache_index.c qwen36.c ex
tests/test_idot$(EXE): tests/test_idot.c colibri.c oracle.h st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h idot.h sample.h kv_persist.h telemetry.h route_trace.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
-tests/test_i4_grouped$(EXE): tests/test_i4_grouped.c colibri.c oracle.h st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h idot.h sample.h kv_persist.h telemetry.h route_trace.h
- $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+tests/test_i4_grouped$(EXE): tests/test_i4_grouped.c colibri.c oracle.h st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h idot.h sse41_kernels.h sample.h kv_persist.h telemetry.h route_trace.h
+ $(CC) $(CFLAGS) -DCOLI_HAVE_GROUPED_PAIR $< -o $@ $(LDFLAGS)
+
+# Force the grouped-int4 oracle onto the SSE4.1 tier even on an AVX2 host.
+tests/test_i4_grouped_sse41$(EXE): tests/test_i4_grouped.c colibri.c oracle.h st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h idot.h sse41_kernels.h sample.h kv_persist.h telemetry.h route_trace.h
+ $(CC) $(CFLAGS) -DCOLI_HAVE_GROUPED_PAIR -msse4.1 -mno-avx2 -mno-fma $< -o $@ $(LDFLAGS)
tests/test_stops$(EXE): tests/test_stops.c colibri.c oracle.h st.h uring.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h quant.h idot.h sample.h kv_persist.h telemetry.h route_trace.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
diff --git a/c/colibri.c b/c/colibri.c
index a96ccba4..ead9853c 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -1048,6 +1048,12 @@ static void matmul_i4_grouped_pair(float *yg, float *yu, const float *x,
/* Same pinning as matmul_i4_grouped in quant.h: contraction
* here is the compiler's choice, so the result was too. */
ag=fmaf(hsum256(accg),scg,ag); au=fmaf(hsum256(accu),scu,au);
+#elif defined(__SSE4_1__)
+ if(!(base&1)){
+ float dg,du;
+ dot_i4f_sse41_pair(wg+(base>>1),wu2+(base>>1),xs+base,glen,&dg,&du);
+ ag+=dg*scg; au+=du*scu; i=base+glen;
+ }
#endif
for(; i+1<base+glen; i+=2){
uint8_t bg=wg[i>>1], bu=wu2[i>>1];
diff --git a/c/quant.h b/c/quant.h
index 91cf7549..251eb5e5 100644
--- a/c/quant.h
+++ b/c/quant.h
@@ -15,6 +15,73 @@
#endif
#include "idot.h" /* SIMD prelude + the integer dot kernels, shared with qwen36 */
+
+#if defined(__SSE4_1__)
+#include <immintrin.h>
+#include "sse41_kernels.h"
+
+/* Exact-activation W4A32 dot for x86 CPUs below AVX2. Packed weights use
+ * offset nibbles: 0 means -8 and 15 means +7. */
+static inline float dot_i4f_sse41(const uint8_t *w,const float *x,int I){
+ const __m128i m4=_mm_set1_epi8(0x0F), b8=_mm_set1_epi8(8);
+ __m128 a0=_mm_setzero_ps(),a1=_mm_setzero_ps(); int i=0;
+ for(;i+16<=I;i+=16){
+ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1)));
+ __m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
+ __m128i q=_mm_sub_epi8(_mm_unpacklo_epi8(lo,hi),b8);
+ a0=COLIBRI_FMA(colibri_sse41_loadu_ps(x+i),
+ _mm_cvtepi32_ps(_mm_cvtepi8_epi32(q)),a0);
+ a1=COLIBRI_FMA(colibri_sse41_loadu_ps(x+i+4),
+ _mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_srli_si128(q,4))),a1);
+ a0=COLIBRI_FMA(colibri_sse41_loadu_ps(x+i+8),
+ _mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_srli_si128(q,8))),a0);
+ a1=COLIBRI_FMA(colibri_sse41_loadu_ps(x+i+12),
+ _mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_srli_si128(q,12))),a1);
+ }
+ float sum=colibri_sse41_hsum_ps(_mm_add_ps(a0,a1));
+ for(;i+1<I;i+=2){ uint8_t b=w[i>>1];
+ sum+=x[i]*(float)((int)(b&15)-8)+x[i+1]*(float)((int)(b>>4)-8); }
+ if(i<I) sum+=x[i]*(float)((int)(w[i>>1]&15)-8);
+ return sum;
+}
+
+/* Gate and up share the same activation row. Keep both projections in flight
+ * so the SSE4.1 path loads that row once. */
+static inline void dot_i4f_sse41_pair(const uint8_t *wg,const uint8_t *wu,
+ const float *x,int I,float *g,float *u){
+ const __m128i m4=_mm_set1_epi8(0x0F), b8=_mm_set1_epi8(8);
+ __m128 g0=_mm_setzero_ps(),g1=_mm_setzero_ps();
+ __m128 u0=_mm_setzero_ps(),u1=_mm_setzero_ps(); int i=0;
+ for(;i+16<=I;i+=16){
+ __m128 x0=colibri_sse41_loadu_ps(x+i), x1=colibri_sse41_loadu_ps(x+i+4);
+ __m128 x2=colibri_sse41_loadu_ps(x+i+8), x3=colibri_sse41_loadu_ps(x+i+12);
+ __m128i bg=_mm_loadl_epi64((const __m128i*)(wg+(i>>1)));
+ __m128i lg=_mm_and_si128(bg,m4), hg=_mm_and_si128(_mm_srli_epi16(bg,4),m4);
+ __m128i qg=_mm_sub_epi8(_mm_unpacklo_epi8(lg,hg),b8);
+ g0=COLIBRI_FMA(x0,_mm_cvtepi32_ps(_mm_cvtepi8_epi32(qg)),g0);
+ g1=COLIBRI_FMA(x1,_mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_srli_si128(qg,4))),g1);
+ g0=COLIBRI_FMA(x2,_mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_srli_si128(qg,8))),g0);
+ g1=COLIBRI_FMA(x3,_mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_srli_si128(qg,12))),g1);
+ __m128i bu=_mm_loadl_epi64((const __m128i*)(wu+(i>>1)));
+ __m128i lu=_mm_and_si128(bu,m4), hu=_mm_and_si128(_mm_srli_epi16(bu,4),m4);
+ __m128i qu=_mm_sub_epi8(_mm_unpacklo_epi8(lu,hu),b8);
+ u0=COLIBRI_FMA(x0,_mm_cvtepi32_ps(_mm_cvtepi8_epi32(qu)),u0);
+ u1=COLIBRI_FMA(x1,_mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_srli_si128(qu,4))),u1);
+ u0=COLIBRI_FMA(x2,_mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_srli_si128(qu,8))),u0);
+ u1=COLIBRI_FMA(x3,_mm_cvtepi32_ps(_mm_cvtepi8_epi32(_mm_srli_si128(qu,12))),u1);
+ }
+ float ag=colibri_sse41_hsum_ps(_mm_add_ps(g0,g1));
+ float au=colibri_sse41_hsum_ps(_mm_add_ps(u0,u1));
+ for(;i+1<I;i+=2){ uint8_t bg=wg[i>>1],bu=wu[i>>1];
+ ag+=x[i]*(float)((int)(bg&15)-8)+x[i+1]*(float)((int)(bg>>4)-8);
+ au+=x[i]*(float)((int)(bu&15)-8)+x[i+1]*(float)((int)(bu>>4)-8); }
+ if(i<I){
+ ag+=x[i]*(float)((int)(wg[i>>1]&15)-8);
+ au+=x[i]*(float)((int)(wu[i>>1]&15)-8);
+ }
+ *g=ag; *u=au;
+}
+#endif
/* ---- AVX-512 int4->float accumulator -------------------------------------- */
#if defined(__AVX512F__) && defined(__AVX512BW__)
static int g_i4_acc512=1;
@@ -168,6 +235,13 @@ static void matmul_i4_grouped(float *y, const float *x, const uint8_t *q4, const
* the glm_tiny token oracle, depend on build flags rather than
* on the code. */
a=fmaf(hsum256(acc),sc,a);
+#elif defined(__SSE4_1__)
+ /* Group starts must be nibble-aligned before the packed row
+ * can be passed as an independent SSE4.1 dot product. */
+ if(!(base&1)){
+ a+=dot_i4f_sse41(w+(base>>1),xs+base,glen)*sc;
+ i=base+glen;
+ }
#endif
for(; i<base+glen; i+=2){
if(i+1<base+glen){ uint8_t byte=w[i>>1];
diff --git a/c/sse41_kernels.h b/c/sse41_kernels.h
index 6c31dcce..60ec53ae 100644
--- a/c/sse41_kernels.h
+++ b/c/sse41_kernels.h
@@ -47,6 +47,13 @@
static inline __m128 colibri_sse41_loadu_ps(const float *p) { return _mm_loadu_ps(p); }
static inline void colibri_sse41_storeu_ps(float *p, __m128 v) { _mm_storeu_ps(p, v); }
+/* Fixed four-lane reduction used by the grouped-int4 GEMV kernels. */
+static inline float colibri_sse41_hsum_ps(__m128 v) {
+ v = _mm_add_ps(v, _mm_movehl_ps(v, v));
+ v = _mm_add_ss(v, _mm_shuffle_ps(v, v, 1));
+ return _mm_cvtss_f32(v);
+}
+
/*
* Min/max (SSE 4.1 native). Identical to AVX2, just narrower width.
*/
diff --git a/c/tests/test_i4_grouped.c b/c/tests/test_i4_grouped.c
index 04e8d887..26eadcc4 100644
--- a/c/tests/test_i4_grouped.c
+++ b/c/tests/test_i4_grouped.c
@@ -99,7 +99,7 @@ static int check(const char *name, int S, int I, int O, int gs, int fill_edges){
* - Correctness, always: both outputs must match the double reference within
* the same magnitude-relative epsilon as the unfused kernel.
* - Bit-exactness, only when I % gs == 0: then every group is covered by the
- * AVX2 body, whose accumulation order is identical to the unfused kernel, so
+ * same SIMD or scalar body in both kernels, so
* the results agree to the last bit. This is the shape the real g64
* checkpoints have (I = 2048 / 6144, gs = 64), i.e. the production path.
*
@@ -143,8 +143,8 @@ static int check_pair(const char *name, int S, int I, int O, int gs){
}
if(yg[i]!=rg[i]||yu[i]!=ru[i]) exact=0;
}
- /* Aligned shapes run entirely through the AVX2 body: same order as unfused,
- * so bit-exactness is a real invariant there and worth asserting. */
+ /* Aligned shapes use the same accumulation order as the unfused kernel, so
+ * bit-exactness is a real invariant there and worth asserting. */
if(I%gs==0 && !exact){
fprintf(stderr,"%s: FAIL fused != unfused bitwise on an ALIGNED shape "
"(no scalar tail runs here; the orders must match)\n",name);