Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

refactor(sse41): collect grouped-int4 kernels in the shared header

codex 20260923-120015-13315222673b242215ea98
diff --git a/c/colibri.c b/c/colibri.c
index 839d82b1..678a30f2 100644
--- a/c/colibri.c
+++ b/c/colibri.c
@@ -1024,47 +1024,6 @@ static void *xzalloc(size_t n, const char *what){
/* Fused gate+up for grouped int4 (fmt=4): computes both yg[S,O] and yu[S,O] from
* the same x[S,I], reading x once instead of twice — saves ~33% of expert-matmul time at decode.
* The per-group scale logic matches matmul_i4_grouped exactly. */
-#if defined(__SSE4_1__) && !defined(__AVX2__)
-static void matmul_i4_grouped_pair_sse41_rows4(float *yg,float *yu,const float *x,
- const uint8_t *qg,const float *sg,
- const uint8_t *qu,const float *su,
- int S,int I,int O,int gs,int rb,
- int ng,int o4){
- #pragma omp parallel for schedule(static)
- for(int o=0;o<o4;o+=4){
- for(int s=0;s<S;s++){
- const float *xs=x+(int64_t)s*I;
- __m128 ag=_mm_setzero_ps(),au=_mm_setzero_ps();
- for(int g=0;g*gs<I;g++){
- int base=g*gs,end=base+gs; if(end>I) end=I;
- __m128 scg=colibri_f32_rows4(sg,ng,o,g);
- __m128 scu=colibri_f32_rows4(su,ng,o,g); int i=base;
- for(;i+1<end;i+=2){
- __m128 gl,gh,ul,uh;
- colibri_i4_rows4(qg,rb,o,i>>1,&gl,&gh);
- colibri_i4_rows4(qu,rb,o,i>>1,&ul,&uh);
- __m128 x0=_mm_set1_ps(xs[i]),x1=_mm_set1_ps(xs[i+1]);
- __m128 gp=_mm_add_ps(_mm_mul_ps(x0,gl),_mm_mul_ps(x1,gh));
- __m128 up=_mm_add_ps(_mm_mul_ps(x0,ul),_mm_mul_ps(x1,uh));
- ag=_mm_add_ps(ag,_mm_mul_ps(gp,scg));
- au=_mm_add_ps(au,_mm_mul_ps(up,scu));
- }
- if(i<end){
- __m128 gl,gh,ul,uh;
- colibri_i4_rows4(qg,rb,o,i>>1,&gl,&gh);
- colibri_i4_rows4(qu,rb,o,i>>1,&ul,&uh); (void)gh; (void)uh;
- __m128 xi=_mm_set1_ps(xs[i]);
- ag=_mm_add_ps(ag,_mm_mul_ps(_mm_mul_ps(xi,gl),scg));
- au=_mm_add_ps(au,_mm_mul_ps(_mm_mul_ps(xi,ul),scu));
- }
- }
- colibri_sse41_storeu_ps(yg+(int64_t)s*O+o,ag);
- colibri_sse41_storeu_ps(yu+(int64_t)s*O+o,au);
- }
- }
-}
-#endif
-
static void matmul_i4_grouped_pair(float *yg, float *yu, const float *x,
const uint8_t *qg, const float *sg,
const uint8_t *qu, const float *su,
diff --git a/c/quant.h b/c/quant.h
index 2fde48e1..a62c36ff 100644
--- a/c/quant.h
+++ b/c/quant.h
@@ -17,63 +17,9 @@
#include "idot.h" /* SIMD prelude + the integer dot kernels, shared with qwen36 */
#if defined(__SSE4_1__)
-#include <immintrin.h>
#include "sse41_kernels.h"
#endif
-#if defined(__SSE4_1__) && !defined(__AVX2__)
-/* Load one packed byte from each of four output rows, then unpack their low and
- * high offset nibbles into four f32 lanes. Keeping independent output rows in
- * the lanes preserves the scalar operation order within every row. */
-static inline void colibri_i4_rows4(const uint8_t *q4,int rb,int o,int byte,
- __m128 *lo,__m128 *hi){
- const __m128i m4=_mm_set1_epi8(0x0F), b8=_mm_set1_epi8(8);
- uint32_t packed=(uint32_t)q4[(int64_t)(o+0)*rb+byte]
- |(uint32_t)q4[(int64_t)(o+1)*rb+byte]<<8
- |(uint32_t)q4[(int64_t)(o+2)*rb+byte]<<16
- |(uint32_t)q4[(int64_t)(o+3)*rb+byte]<<24;
- __m128i by=_mm_cvtsi32_si128((int)packed);
- __m128i qlo=_mm_sub_epi8(_mm_and_si128(by,m4),b8);
- __m128i qhi=_mm_sub_epi8(_mm_and_si128(_mm_srli_epi16(by,4),m4),b8);
- *lo=_mm_cvtepi32_ps(_mm_cvtepi8_epi32(qlo));
- *hi=_mm_cvtepi32_ps(_mm_cvtepi8_epi32(qhi));
-}
-
-static inline __m128 colibri_f32_rows4(const float *p,int stride,int o,int i){
- return _mm_set_ps(p[(int64_t)(o+3)*stride+i],p[(int64_t)(o+2)*stride+i],
- p[(int64_t)(o+1)*stride+i],p[(int64_t)(o+0)*stride+i]);
-}
-
-/* Process four output rows at once without a horizontal reduction. Each lane
- * uses the scalar kernel's pair sum, scale multiply, and accumulator add in
- * the same order, so the result can remain byte-identical on pre-FMA CPUs. */
-static void matmul_i4_grouped_sse41_rows4(float *y,const float *x,
- const uint8_t *q4,const float *scale,
- int S,int I,int O,int gs,int rb,int ng,
- int o4){
- #pragma omp parallel for schedule(static)
- for(int o=0;o<o4;o+=4){
- for(int s=0;s<S;s++){
- const float *xs=x+(int64_t)s*I; __m128 a=_mm_setzero_ps();
- for(int g=0;g*gs<I;g++){
- int base=g*gs,end=base+gs; if(end>I) end=I;
- __m128 sc=colibri_f32_rows4(scale,ng,o,g); int i=base;
- for(;i+1<end;i+=2){
- __m128 lo,hi; colibri_i4_rows4(q4,rb,o,i>>1,&lo,&hi);
- __m128 pair=_mm_add_ps(_mm_mul_ps(_mm_set1_ps(xs[i]),lo),
- _mm_mul_ps(_mm_set1_ps(xs[i+1]),hi));
- a=_mm_add_ps(a,_mm_mul_ps(pair,sc));
- }
- if(i<end){
- __m128 lo,hi; colibri_i4_rows4(q4,rb,o,i>>1,&lo,&hi); (void)hi;
- a=_mm_add_ps(a,_mm_mul_ps(_mm_mul_ps(_mm_set1_ps(xs[i]),lo),sc));
- }
- }
- colibri_sse41_storeu_ps(y+(int64_t)s*O+o,a);
- }
- }
-}
-#endif
/* ---- AVX-512 int4->float accumulator -------------------------------------- */
#if defined(__AVX512F__) && defined(__AVX512BW__)
static int g_i4_acc512=1;
diff --git a/c/sse41_kernels.h b/c/sse41_kernels.h
index 6c31dcce..4f93eee4 100644
--- a/c/sse41_kernels.h
+++ b/c/sse41_kernels.h
@@ -22,6 +22,9 @@
*/
#if defined(__SSE2__)
+#include <immintrin.h>
+#include <stdint.h>
+
/*
* COLIBRI_FMA: emulate FMA on non-FMA hardware.
*
@@ -58,5 +61,102 @@ static inline __m128 colibri_sse41_max_ps(__m128 a, __m128 b) { return _mm_max_p
*/
static inline void colibri_sse41_prefetch(const void *p) { _mm_prefetch(p, _MM_HINT_T0); }
+/* Grouped-int4 kernels for the SSE4.1 tier. Callers select even group sizes
+ * and pass a multiple of four output rows; scalar dispatch handles the rest.
+ * Keep the scalar pair-sum, scale-multiply and accumulator-add order intact. */
+#if defined(__SSE4_1__) && !defined(__AVX2__)
+/* Load one packed byte from each of four output rows, then unpack their low and
+ * high offset nibbles into four f32 lanes. Keeping independent output rows in
+ * the lanes preserves the scalar operation order within every row. */
+static inline void colibri_sse41_i4_rows4(const uint8_t *q4,int rb,int o,int byte,
+ __m128 *lo,__m128 *hi){
+ const __m128i m4=_mm_set1_epi8(0x0F), b8=_mm_set1_epi8(8);
+ /* Read exactly one byte per row, including when a row is one byte wide. */
+ __m128i by=_mm_cvtsi32_si128(q4[(int64_t)(o+0)*rb+byte]);
+ by=_mm_insert_epi8(by,q4[(int64_t)(o+1)*rb+byte],1);
+ by=_mm_insert_epi8(by,q4[(int64_t)(o+2)*rb+byte],2);
+ by=_mm_insert_epi8(by,q4[(int64_t)(o+3)*rb+byte],3);
+ __m128i qlo=_mm_sub_epi8(_mm_and_si128(by,m4),b8);
+ __m128i qhi=_mm_sub_epi8(_mm_and_si128(_mm_srli_epi16(by,4),m4),b8);
+ *lo=_mm_cvtepi32_ps(_mm_cvtepi8_epi32(qlo));
+ *hi=_mm_cvtepi32_ps(_mm_cvtepi8_epi32(qhi));
+}
+
+static inline __m128 colibri_sse41_f32_rows4(const float *p,int stride,int o,int i){
+ return _mm_set_ps(p[(int64_t)(o+3)*stride+i],p[(int64_t)(o+2)*stride+i],
+ p[(int64_t)(o+1)*stride+i],p[(int64_t)(o+0)*stride+i]);
+}
+
+/* Process four output rows at once without a horizontal reduction. Each lane
+ * uses the scalar kernel's pair sum, scale multiply, and accumulator add in
+ * the same order, so the result can remain byte-identical on pre-FMA CPUs. */
+static void matmul_i4_grouped_sse41_rows4(float *y,const float *x,
+ const uint8_t *q4,const float *scale,
+ int S,int I,int O,int gs,int rb,int ng,
+ int o4){
+ #pragma omp parallel for schedule(static)
+ for(int o=0;o<o4;o+=4){
+ for(int s=0;s<S;s++){
+ const float *xs=x+(int64_t)s*I; __m128 a=_mm_setzero_ps();
+ for(int g=0;g*gs<I;g++){
+ int base=g*gs,end=base+gs; if(end>I) end=I;
+ __m128 sc=colibri_sse41_f32_rows4(scale,ng,o,g); int i=base;
+ for(;i+1<end;i+=2){
+ __m128 lo,hi; colibri_sse41_i4_rows4(q4,rb,o,i>>1,&lo,&hi);
+ __m128 pair=_mm_add_ps(_mm_mul_ps(_mm_set1_ps(xs[i]),lo),
+ _mm_mul_ps(_mm_set1_ps(xs[i+1]),hi));
+ a=_mm_add_ps(a,_mm_mul_ps(pair,sc));
+ }
+ if(i<end){
+ __m128 lo,hi; colibri_sse41_i4_rows4(q4,rb,o,i>>1,&lo,&hi); (void)hi;
+ a=_mm_add_ps(a,_mm_mul_ps(_mm_mul_ps(_mm_set1_ps(xs[i]),lo),sc));
+ }
+ }
+ colibri_sse41_storeu_ps(y+(int64_t)s*O+o,a);
+ }
+ }
+}
+
+/* Fused gate/up shares activation loads while keeping independent accumulators. */
+static void matmul_i4_grouped_pair_sse41_rows4(float *yg,float *yu,const float *x,
+ const uint8_t *qg,const float *sg,
+ const uint8_t *qu,const float *su,
+ int S,int I,int O,int gs,int rb,
+ int ng,int o4){
+ #pragma omp parallel for schedule(static)
+ for(int o=0;o<o4;o+=4){
+ for(int s=0;s<S;s++){
+ const float *xs=x+(int64_t)s*I;
+ __m128 ag=_mm_setzero_ps(),au=_mm_setzero_ps();
+ for(int g=0;g*gs<I;g++){
+ int base=g*gs,end=base+gs; if(end>I) end=I;
+ __m128 scg=colibri_sse41_f32_rows4(sg,ng,o,g);
+ __m128 scu=colibri_sse41_f32_rows4(su,ng,o,g); int i=base;
+ for(;i+1<end;i+=2){
+ __m128 gl,gh,ul,uh;
+ colibri_sse41_i4_rows4(qg,rb,o,i>>1,&gl,&gh);
+ colibri_sse41_i4_rows4(qu,rb,o,i>>1,&ul,&uh);
+ __m128 x0=_mm_set1_ps(xs[i]),x1=_mm_set1_ps(xs[i+1]);
+ __m128 gp=_mm_add_ps(_mm_mul_ps(x0,gl),_mm_mul_ps(x1,gh));
+ __m128 up=_mm_add_ps(_mm_mul_ps(x0,ul),_mm_mul_ps(x1,uh));
+ ag=_mm_add_ps(ag,_mm_mul_ps(gp,scg));
+ au=_mm_add_ps(au,_mm_mul_ps(up,scu));
+ }
+ if(i<end){
+ __m128 gl,gh,ul,uh;
+ colibri_sse41_i4_rows4(qg,rb,o,i>>1,&gl,&gh);
+ colibri_sse41_i4_rows4(qu,rb,o,i>>1,&ul,&uh); (void)gh; (void)uh;
+ __m128 xi=_mm_set1_ps(xs[i]);
+ ag=_mm_add_ps(ag,_mm_mul_ps(_mm_mul_ps(xi,gl),scg));
+ au=_mm_add_ps(au,_mm_mul_ps(_mm_mul_ps(xi,ul),scu));
+ }
+ }
+ colibri_sse41_storeu_ps(yg+(int64_t)s*O+o,ag);
+ colibri_sse41_storeu_ps(yu+(int64_t)s*O+o,au);
+ }
+ }
+}
+#endif /* __SSE4_1__ && !__AVX2__ */
+
#endif /* __SSE2__ */
#endif /* COLIBRI_SSE41_KERNELS_H */
diff --git a/c/tests/test_i4_grouped.c b/c/tests/test_i4_grouped.c
index a6dd3f33..f76c8918 100644
--- a/c/tests/test_i4_grouped.c
+++ b/c/tests/test_i4_grouped.c
@@ -199,6 +199,8 @@ static int check_pair(const char *name, int S, int I, int O, int gs){
int require_exact=I%gs==0;
#ifdef COLI_I4_GROUPED_SCALAR_EXACT
require_exact=1;
+ exact=memcmp(yg,rg,(size_t)S*O*sizeof(float))==0
+ && memcmp(yu,ru,(size_t)S*O*sizeof(float))==0;
#endif
if(require_exact && !exact){
fprintf(stderr,"%s: FAIL fused != unfused bitwise "
@@ -214,9 +216,47 @@ static int check_pair(const char *name, int S, int I, int O, int gs){
}
#endif
+#if defined(__SSE4_1__) && !defined(__AVX2__)
+/* Exercise every packed byte in every lane, with tight and strided rows.
+ * The last byte of the last row sits at the allocation boundary so ASan also
+ * catches a wider load that would read beyond the packed matrix. */
+static int check_rows4_unpack(void){
+ const int strides[]={1,3}, offsets[]={0,1};
+ for(int c=0;c<2;c++){
+ int rb=strides[c],o=offsets[c];
+ size_t n=(size_t)(o+4)*rb;
+ uint8_t *q4=malloc(n);
+ if(!q4){ fprintf(stderr,"rows4 unpack: OOM\n"); return 1; }
+ for(int value=0;value<256;value++){
+ for(size_t i=0;i<n;i++) q4[i]=(uint8_t)(value+37*i);
+ for(int byte=0;byte<rb;byte++){
+ __m128 lo,hi; float l[4],h[4];
+ colibri_sse41_i4_rows4(q4,rb,o,byte,&lo,&hi);
+ _mm_storeu_ps(l,lo); _mm_storeu_ps(h,hi);
+ for(int row=0;row<4;row++){
+ unsigned packed=q4[(o+row)*rb+byte];
+ if(l[row]!=(float)((int)(packed&15)-8)
+ || h[row]!=(float)((int)(packed>>4)-8)){
+ fprintf(stderr,"rows4 unpack: rb=%d o=%d byte=%d row=%d value=%u\n",
+ rb,o,byte,row,packed);
+ free(q4); return 1;
+ }
+ }
+ }
+ }
+ free(q4);
+ }
+ printf(" rows4 unpack: all 256 byte values per lane, tight/strided rows ok\n");
+ return 0;
+}
+#endif
+
int main(void){
int fail=0;
printf("test_i4_grouped: matmul_i4_grouped vs plain-C dequant reference\n");
+#if defined(__SSE4_1__) && !defined(__AVX2__)
+ fail|=check_rows4_unpack();
+#endif
/* the shape the g64 checkpoints actually use */
fail|=check("gs=64, I multiple of gs", 2, 512, 8, 64, 0);
@@ -240,6 +280,7 @@ int main(void){
/* batch: S>1 exercises the per-s inner loop against a shared scale row */
fail|=check("gs=64, batch S=8", 8, 320, 6, 64, 0);
+ fail|=check("one-byte rows, odd I=1", 1, 1, 4, 2, 0);
#ifdef COLI_HAVE_GROUPED_PAIR
printf("test_i4_grouped: matmul_i4_grouped_pair (fused gate+up) vs two separate calls\n");
@@ -248,6 +289,7 @@ int main(void){
fail|=check_pair("pair: gs=64, odd I (I=201)", 2, 201, 4, 64);
fail|=check_pair("pair: gs=64, decode S=1", 1, 320, 6, 64);
fail|=check_pair("pair: gs=128, I=512", 2, 512, 4, 128);
+ fail|=check_pair("pair: one-byte rows, odd I=1", 1, 1, 4, 2);
#endif
if(fail){ printf("test_i4_grouped: FAIL\n"); return 1; }