Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
refactor(sse41): collect grouped-int4 kernels in the shared header
codex 20260923-120015-13315222673b242215ea98
diff --git a/c/colibri.c b/c/colibri.cindex 839d82b1..678a30f2 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -1024,47 +1024,6 @@ static void *xzalloc(size_t n, const char *what){/* Fused gate+up for grouped int4 (fmt=4): computes both yg[S,O] and yu[S,O] from* the same x[S,I], reading x once instead of twice — saves ~33% of expert-matmul time at decode.* The per-group scale logic matches matmul_i4_grouped exactly. */-#if defined(__SSE4_1__) && !defined(__AVX2__)-static void matmul_i4_grouped_pair_sse41_rows4(float *yg,float *yu,const float *x,- const uint8_t *qg,const float *sg,- const uint8_t *qu,const float *su,- int S,int I,int O,int gs,int rb,- int ng,int o4){- #pragma omp parallel for schedule(static)- for(int o=0;o<o4;o+=4){- for(int s=0;s<S;s++){- const float *xs=x+(int64_t)s*I;- __m128 ag=_mm_setzero_ps(),au=_mm_setzero_ps();- for(int g=0;g*gs<I;g++){- int base=g*gs,end=base+gs; if(end>I) end=I;- __m128 scg=colibri_f32_rows4(sg,ng,o,g);- __m128 scu=colibri_f32_rows4(su,ng,o,g); int i=base;- for(;i+1<end;i+=2){- __m128 gl,gh,ul,uh;- colibri_i4_rows4(qg,rb,o,i>>1,&gl,&gh);- colibri_i4_rows4(qu,rb,o,i>>1,&ul,&uh);- __m128 x0=_mm_set1_ps(xs[i]),x1=_mm_set1_ps(xs[i+1]);- __m128 gp=_mm_add_ps(_mm_mul_ps(x0,gl),_mm_mul_ps(x1,gh));- __m128 up=_mm_add_ps(_mm_mul_ps(x0,ul),_mm_mul_ps(x1,uh));- ag=_mm_add_ps(ag,_mm_mul_ps(gp,scg));- au=_mm_add_ps(au,_mm_mul_ps(up,scu));- }- if(i<end){- __m128 gl,gh,ul,uh;- colibri_i4_rows4(qg,rb,o,i>>1,&gl,&gh);- colibri_i4_rows4(qu,rb,o,i>>1,&ul,&uh); (void)gh; (void)uh;- __m128 xi=_mm_set1_ps(xs[i]);- ag=_mm_add_ps(ag,_mm_mul_ps(_mm_mul_ps(xi,gl),scg));- au=_mm_add_ps(au,_mm_mul_ps(_mm_mul_ps(xi,ul),scu));- }- }- colibri_sse41_storeu_ps(yg+(int64_t)s*O+o,ag);- colibri_sse41_storeu_ps(yu+(int64_t)s*O+o,au);- }- }-}-#endif-static void matmul_i4_grouped_pair(float *yg, float *yu, const float *x,const uint8_t *qg, const float *sg,const uint8_t *qu, const float *su,diff --git a/c/quant.h b/c/quant.hindex 2fde48e1..a62c36ff 100644--- a/c/quant.h+++ b/c/quant.h@@ -17,63 +17,9 @@#include "idot.h" /* SIMD prelude + the integer dot kernels, shared with qwen36 */#if defined(__SSE4_1__)-#include <immintrin.h>#include "sse41_kernels.h"#endif-#if defined(__SSE4_1__) && !defined(__AVX2__)-/* Load one packed byte from each of four output rows, then unpack their low and- * high offset nibbles into four f32 lanes. Keeping independent output rows in- * the lanes preserves the scalar operation order within every row. */-static inline void colibri_i4_rows4(const uint8_t *q4,int rb,int o,int byte,- __m128 *lo,__m128 *hi){- const __m128i m4=_mm_set1_epi8(0x0F), b8=_mm_set1_epi8(8);- uint32_t packed=(uint32_t)q4[(int64_t)(o+0)*rb+byte]- |(uint32_t)q4[(int64_t)(o+1)*rb+byte]<<8- |(uint32_t)q4[(int64_t)(o+2)*rb+byte]<<16- |(uint32_t)q4[(int64_t)(o+3)*rb+byte]<<24;- __m128i by=_mm_cvtsi32_si128((int)packed);- __m128i qlo=_mm_sub_epi8(_mm_and_si128(by,m4),b8);- __m128i qhi=_mm_sub_epi8(_mm_and_si128(_mm_srli_epi16(by,4),m4),b8);- *lo=_mm_cvtepi32_ps(_mm_cvtepi8_epi32(qlo));- *hi=_mm_cvtepi32_ps(_mm_cvtepi8_epi32(qhi));-}--static inline __m128 colibri_f32_rows4(const float *p,int stride,int o,int i){- return _mm_set_ps(p[(int64_t)(o+3)*stride+i],p[(int64_t)(o+2)*stride+i],- p[(int64_t)(o+1)*stride+i],p[(int64_t)(o+0)*stride+i]);-}--/* Process four output rows at once without a horizontal reduction. Each lane- * uses the scalar kernel's pair sum, scale multiply, and accumulator add in- * the same order, so the result can remain byte-identical on pre-FMA CPUs. */-static void matmul_i4_grouped_sse41_rows4(float *y,const float *x,- const uint8_t *q4,const float *scale,- int S,int I,int O,int gs,int rb,int ng,- int o4){- #pragma omp parallel for schedule(static)- for(int o=0;o<o4;o+=4){- for(int s=0;s<S;s++){- const float *xs=x+(int64_t)s*I; __m128 a=_mm_setzero_ps();- for(int g=0;g*gs<I;g++){- int base=g*gs,end=base+gs; if(end>I) end=I;- __m128 sc=colibri_f32_rows4(scale,ng,o,g); int i=base;- for(;i+1<end;i+=2){- __m128 lo,hi; colibri_i4_rows4(q4,rb,o,i>>1,&lo,&hi);- __m128 pair=_mm_add_ps(_mm_mul_ps(_mm_set1_ps(xs[i]),lo),- _mm_mul_ps(_mm_set1_ps(xs[i+1]),hi));- a=_mm_add_ps(a,_mm_mul_ps(pair,sc));- }- if(i<end){- __m128 lo,hi; colibri_i4_rows4(q4,rb,o,i>>1,&lo,&hi); (void)hi;- a=_mm_add_ps(a,_mm_mul_ps(_mm_mul_ps(_mm_set1_ps(xs[i]),lo),sc));- }- }- colibri_sse41_storeu_ps(y+(int64_t)s*O+o,a);- }- }-}-#endif/* ---- AVX-512 int4->float accumulator -------------------------------------- */#if defined(__AVX512F__) && defined(__AVX512BW__)static int g_i4_acc512=1;diff --git a/c/sse41_kernels.h b/c/sse41_kernels.hindex 6c31dcce..4f93eee4 100644--- a/c/sse41_kernels.h+++ b/c/sse41_kernels.h@@ -22,6 +22,9 @@*/#if defined(__SSE2__)+#include <immintrin.h>+#include <stdint.h>+/** COLIBRI_FMA: emulate FMA on non-FMA hardware.*@@ -58,5 +61,102 @@ static inline __m128 colibri_sse41_max_ps(__m128 a, __m128 b) { return _mm_max_p*/static inline void colibri_sse41_prefetch(const void *p) { _mm_prefetch(p, _MM_HINT_T0); }+/* Grouped-int4 kernels for the SSE4.1 tier. Callers select even group sizes+ * and pass a multiple of four output rows; scalar dispatch handles the rest.+ * Keep the scalar pair-sum, scale-multiply and accumulator-add order intact. */+#if defined(__SSE4_1__) && !defined(__AVX2__)+/* Load one packed byte from each of four output rows, then unpack their low and+ * high offset nibbles into four f32 lanes. Keeping independent output rows in+ * the lanes preserves the scalar operation order within every row. */+static inline void colibri_sse41_i4_rows4(const uint8_t *q4,int rb,int o,int byte,+ __m128 *lo,__m128 *hi){+ const __m128i m4=_mm_set1_epi8(0x0F), b8=_mm_set1_epi8(8);+ /* Read exactly one byte per row, including when a row is one byte wide. */+ __m128i by=_mm_cvtsi32_si128(q4[(int64_t)(o+0)*rb+byte]);+ by=_mm_insert_epi8(by,q4[(int64_t)(o+1)*rb+byte],1);+ by=_mm_insert_epi8(by,q4[(int64_t)(o+2)*rb+byte],2);+ by=_mm_insert_epi8(by,q4[(int64_t)(o+3)*rb+byte],3);+ __m128i qlo=_mm_sub_epi8(_mm_and_si128(by,m4),b8);+ __m128i qhi=_mm_sub_epi8(_mm_and_si128(_mm_srli_epi16(by,4),m4),b8);+ *lo=_mm_cvtepi32_ps(_mm_cvtepi8_epi32(qlo));+ *hi=_mm_cvtepi32_ps(_mm_cvtepi8_epi32(qhi));+}++static inline __m128 colibri_sse41_f32_rows4(const float *p,int stride,int o,int i){+ return _mm_set_ps(p[(int64_t)(o+3)*stride+i],p[(int64_t)(o+2)*stride+i],+ p[(int64_t)(o+1)*stride+i],p[(int64_t)(o+0)*stride+i]);+}++/* Process four output rows at once without a horizontal reduction. Each lane+ * uses the scalar kernel's pair sum, scale multiply, and accumulator add in+ * the same order, so the result can remain byte-identical on pre-FMA CPUs. */+static void matmul_i4_grouped_sse41_rows4(float *y,const float *x,+ const uint8_t *q4,const float *scale,+ int S,int I,int O,int gs,int rb,int ng,+ int o4){+ #pragma omp parallel for schedule(static)+ for(int o=0;o<o4;o+=4){+ for(int s=0;s<S;s++){+ const float *xs=x+(int64_t)s*I; __m128 a=_mm_setzero_ps();+ for(int g=0;g*gs<I;g++){+ int base=g*gs,end=base+gs; if(end>I) end=I;+ __m128 sc=colibri_sse41_f32_rows4(scale,ng,o,g); int i=base;+ for(;i+1<end;i+=2){+ __m128 lo,hi; colibri_sse41_i4_rows4(q4,rb,o,i>>1,&lo,&hi);+ __m128 pair=_mm_add_ps(_mm_mul_ps(_mm_set1_ps(xs[i]),lo),+ _mm_mul_ps(_mm_set1_ps(xs[i+1]),hi));+ a=_mm_add_ps(a,_mm_mul_ps(pair,sc));+ }+ if(i<end){+ __m128 lo,hi; colibri_sse41_i4_rows4(q4,rb,o,i>>1,&lo,&hi); (void)hi;+ a=_mm_add_ps(a,_mm_mul_ps(_mm_mul_ps(_mm_set1_ps(xs[i]),lo),sc));+ }+ }+ colibri_sse41_storeu_ps(y+(int64_t)s*O+o,a);+ }+ }+}++/* Fused gate/up shares activation loads while keeping independent accumulators. */+static void matmul_i4_grouped_pair_sse41_rows4(float *yg,float *yu,const float *x,+ const uint8_t *qg,const float *sg,+ const uint8_t *qu,const float *su,+ int S,int I,int O,int gs,int rb,+ int ng,int o4){+ #pragma omp parallel for schedule(static)+ for(int o=0;o<o4;o+=4){+ for(int s=0;s<S;s++){+ const float *xs=x+(int64_t)s*I;+ __m128 ag=_mm_setzero_ps(),au=_mm_setzero_ps();+ for(int g=0;g*gs<I;g++){+ int base=g*gs,end=base+gs; if(end>I) end=I;+ __m128 scg=colibri_sse41_f32_rows4(sg,ng,o,g);+ __m128 scu=colibri_sse41_f32_rows4(su,ng,o,g); int i=base;+ for(;i+1<end;i+=2){+ __m128 gl,gh,ul,uh;+ colibri_sse41_i4_rows4(qg,rb,o,i>>1,&gl,&gh);+ colibri_sse41_i4_rows4(qu,rb,o,i>>1,&ul,&uh);+ __m128 x0=_mm_set1_ps(xs[i]),x1=_mm_set1_ps(xs[i+1]);+ __m128 gp=_mm_add_ps(_mm_mul_ps(x0,gl),_mm_mul_ps(x1,gh));+ __m128 up=_mm_add_ps(_mm_mul_ps(x0,ul),_mm_mul_ps(x1,uh));+ ag=_mm_add_ps(ag,_mm_mul_ps(gp,scg));+ au=_mm_add_ps(au,_mm_mul_ps(up,scu));+ }+ if(i<end){+ __m128 gl,gh,ul,uh;+ colibri_sse41_i4_rows4(qg,rb,o,i>>1,&gl,&gh);+ colibri_sse41_i4_rows4(qu,rb,o,i>>1,&ul,&uh); (void)gh; (void)uh;+ __m128 xi=_mm_set1_ps(xs[i]);+ ag=_mm_add_ps(ag,_mm_mul_ps(_mm_mul_ps(xi,gl),scg));+ au=_mm_add_ps(au,_mm_mul_ps(_mm_mul_ps(xi,ul),scu));+ }+ }+ colibri_sse41_storeu_ps(yg+(int64_t)s*O+o,ag);+ colibri_sse41_storeu_ps(yu+(int64_t)s*O+o,au);+ }+ }+}+#endif /* __SSE4_1__ && !__AVX2__ */+#endif /* __SSE2__ */#endif /* COLIBRI_SSE41_KERNELS_H */diff --git a/c/tests/test_i4_grouped.c b/c/tests/test_i4_grouped.cindex a6dd3f33..f76c8918 100644--- a/c/tests/test_i4_grouped.c+++ b/c/tests/test_i4_grouped.c@@ -199,6 +199,8 @@ static int check_pair(const char *name, int S, int I, int O, int gs){int require_exact=I%gs==0;#ifdef COLI_I4_GROUPED_SCALAR_EXACTrequire_exact=1;+ exact=memcmp(yg,rg,(size_t)S*O*sizeof(float))==0+ && memcmp(yu,ru,(size_t)S*O*sizeof(float))==0;#endifif(require_exact && !exact){fprintf(stderr,"%s: FAIL fused != unfused bitwise "@@ -214,9 +216,47 @@ static int check_pair(const char *name, int S, int I, int O, int gs){}#endif+#if defined(__SSE4_1__) && !defined(__AVX2__)+/* Exercise every packed byte in every lane, with tight and strided rows.+ * The last byte of the last row sits at the allocation boundary so ASan also+ * catches a wider load that would read beyond the packed matrix. */+static int check_rows4_unpack(void){+ const int strides[]={1,3}, offsets[]={0,1};+ for(int c=0;c<2;c++){+ int rb=strides[c],o=offsets[c];+ size_t n=(size_t)(o+4)*rb;+ uint8_t *q4=malloc(n);+ if(!q4){ fprintf(stderr,"rows4 unpack: OOM\n"); return 1; }+ for(int value=0;value<256;value++){+ for(size_t i=0;i<n;i++) q4[i]=(uint8_t)(value+37*i);+ for(int byte=0;byte<rb;byte++){+ __m128 lo,hi; float l[4],h[4];+ colibri_sse41_i4_rows4(q4,rb,o,byte,&lo,&hi);+ _mm_storeu_ps(l,lo); _mm_storeu_ps(h,hi);+ for(int row=0;row<4;row++){+ unsigned packed=q4[(o+row)*rb+byte];+ if(l[row]!=(float)((int)(packed&15)-8)+ || h[row]!=(float)((int)(packed>>4)-8)){+ fprintf(stderr,"rows4 unpack: rb=%d o=%d byte=%d row=%d value=%u\n",+ rb,o,byte,row,packed);+ free(q4); return 1;+ }+ }+ }+ }+ free(q4);+ }+ printf(" rows4 unpack: all 256 byte values per lane, tight/strided rows ok\n");+ return 0;+}+#endif+int main(void){int fail=0;printf("test_i4_grouped: matmul_i4_grouped vs plain-C dequant reference\n");+#if defined(__SSE4_1__) && !defined(__AVX2__)+ fail|=check_rows4_unpack();+#endif/* the shape the g64 checkpoints actually use */fail|=check("gs=64, I multiple of gs", 2, 512, 8, 64, 0);@@ -240,6 +280,7 @@ int main(void){/* batch: S>1 exercises the per-s inner loop against a shared scale row */fail|=check("gs=64, batch S=8", 8, 320, 6, 64, 0);+ fail|=check("one-byte rows, odd I=1", 1, 1, 4, 2, 0);#ifdef COLI_HAVE_GROUPED_PAIRprintf("test_i4_grouped: matmul_i4_grouped_pair (fused gate+up) vs two separate calls\n");@@ -248,6 +289,7 @@ int main(void){fail|=check_pair("pair: gs=64, odd I (I=201)", 2, 201, 4, 64);fail|=check_pair("pair: gs=64, decode S=1", 1, 320, 6, 64);fail|=check_pair("pair: gs=128, I=512", 2, 512, 4, 128);+ fail|=check_pair("pair: one-byte rows, odd I=1", 1, 1, 4, 2);#endifif(fail){ printf("test_i4_grouped: FAIL\n"); return 1; }