Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
/* Exactness test for the grouped-int4 kernel (fmt=4, one f32 scale per `gs`
* elements along I) against a plain-C reference that dequantizes and multiplies
* in double.
*
* Why this test exists: matmul_i4_grouped is the REFERENCE the CUDA fmt=4 path
* (#298) is expected to reproduce, and it had no test of its own. Debugging a
* backend against an unverified oracle means two moving targets. Anyone porting
* fmt=4 to a new backend can now diff against a kernel that is known exact here.
*
* Covers: I a clean multiple of gs, I with a partial last group (the `glen`
* clamp), odd I (the nibble tail), gs larger than I (single group), and the
* nibble edges 0 and 15 (which decode to -8 and +7 — an offset encoding, NOT
* two's complement; getting this backwards is silent and looks like noise).
*
* FP note: the kernel sums in f32 while the format reference sums in double, so
* we compare those two against a relative epsilon. Forced-SSE builds separately
* require byte identity with the scalar f32 operation tree. */
#define main coli_glm_main_unused
#include "../colibri.c"
#undef main
static uint32_t rng_state=0xC0FFEEu;
static uint32_t xr(void){ rng_state^=rng_state<<13; rng_state^=rng_state>>17; rng_state^=rng_state<<5; return rng_state; }
static float frand(void){ return (float)((int)(xr()%2001)-1000)/1000.0f; }
/* Reference: dequantize nibble -> (v-8)*scale[group], accumulate in double.
* Deliberately the dumbest possible expression of the format. */
static void ref_grouped(double *y, double *mag, const float *x, const uint8_t *q4,
const float *scale, int S, int I, int O, int gs){
int rb=(I+1)/2, ng=(I+gs-1)/gs;
for(int o=0;o<O;o++){
const uint8_t *w=q4+(int64_t)o*rb;
const float *scl=scale+(int64_t)o*ng;
for(int s=0;s<S;s++){
const float *xs=x+(int64_t)s*I; double a=0, m=0;
for(int i=0;i<I;i++){
uint8_t byte=w[i>>1];
int nib=(i&1)?(int)(byte>>4):(int)(byte&0xF);
double term=(double)xs[i] * (double)(nib-8) * (double)scl[i/gs];
a += term; m += fabs(term);
}
y[(int64_t)s*O+o]=a;
/* Sum of |terms|: the scale the f32 rounding error actually lives on.
* Comparing against |result| instead would flag pure cancellation --
* a dot product of signed terms can land near zero, and then a 1e-6
* absolute error reads as a 1e-3 relative one. That is the accumulator's
* precision, not a kernel defect. A wrong scale index or group boundary
* shifts the result by a fraction OF THE TERMS, so it is caught here. */
mag[(int64_t)s*O+o]=m;
}
}
}
/* Scalar contract: this is the scalar grouped kernel's operation tree, not a
* higher-precision format reference. The forced-SSE builds compare its output
* byte for byte with the four-output-row SIMD path. */
#ifdef COLI_I4_GROUPED_SCALAR_EXACT
static void ref_grouped_scalar(float *y,const float *x,const uint8_t *q4,
const float *scale,int S,int I,int O,int gs){
int rb=(I+1)/2,ng=(I+gs-1)/gs;
for(int o=0;o<O;o++){
const uint8_t *w=q4+(int64_t)o*rb;
const float *scl=scale+(int64_t)o*ng;
for(int s=0;s<S;s++){
const float *xs=x+(int64_t)s*I; float a=0;
for(int g=0;g*gs<I;g++){
int base=g*gs,glen=gs; if(base+glen>I) glen=I-base;
float sc=scl[g];
for(int i=base;i<base+glen;i+=2){
if(i+1<base+glen){
uint8_t byte=w[i>>1];
a+=(xs[i]*(float)((int)(byte&0xF)-8)
+xs[i+1]*(float)((int)(byte>>4)-8))*sc;
}else{
uint8_t byte=w[i>>1];
a+=xs[i]*(float)((int)(byte&0xF)-8)*sc;
}
}
}
y[(int64_t)s*O+o]=a;
}
}
}
#endif
static int check(const char *name, int S, int I, int O, int gs, int fill_edges){
int rb=(I+1)/2, ng=(I+gs-1)/gs;
uint8_t *q4=malloc((size_t)O*rb);
float *scale=malloc((size_t)O*ng*sizeof(float));
float *x=malloc((size_t)S*I*sizeof(float));
float *y=malloc((size_t)S*O*sizeof(float));
#ifdef COLI_I4_GROUPED_SCALAR_EXACT
float *ys=malloc((size_t)S*O*sizeof(float));
#endif
double *yr=malloc((size_t)S*O*sizeof(double));
double *ym=malloc((size_t)S*O*sizeof(double));
if(!q4||!scale||!x||!y||!yr||!ym
#ifdef COLI_I4_GROUPED_SCALAR_EXACT
||!ys
#endif
){ fprintf(stderr,"%s: OOM\n",name); return 1; }
for(size_t i=0;i<(size_t)O*rb;i++) q4[i]=(uint8_t)(xr()&0xFF);
if(fill_edges){
/* nibble extremes: 0x0F -> +7, 0x00 -> -8. A two's-complement misread
* turns 15 into -1 instead of +7 and the error is data-dependent noise. */
for(size_t i=0;i<(size_t)O*rb && i<64;i++) q4[i]=(i&1)?0x00:0xFF;
}
/* scales span a few orders of magnitude: a wrong group index shows up big */
for(int i=0;i<O*ng;i++) scale[i]=(0.001f+(float)(xr()%1000)/1000.0f)*((xr()&1)?1.f:-1.f);
for(int i=0;i<S*I;i++) x[i]=frand();
matmul_i4_grouped(y,x,q4,scale,S,I,O,gs);
ref_grouped(yr,ym,x,q4,scale,S,I,O,gs);
#ifdef COLI_I4_GROUPED_SCALAR_EXACT
ref_grouped_scalar(ys,x,q4,scale,S,I,O,gs);
#endif
int bad=0; double worst=0;
for(int i=0;i<S*O;i++){
double d=fabs((double)y[i]-yr[i]);
double rel = ym[i]>1e-30 ? d/ym[i] : d; /* error relative to the summed magnitude */
if(rel>worst) worst=rel;
if(rel>1e-6){
if(bad<3) fprintf(stderr,"%s: [%d] got %.9g want %.9g (|terms| %.3g, rel %.3g)\n",
name,i,(double)y[i],yr[i],ym[i],rel);
bad++;
}
}
#ifdef COLI_I4_GROUPED_SCALAR_EXACT
if(memcmp(y,ys,(size_t)S*O*sizeof(float))!=0){
int first=0;
while(first<S*O && memcmp(y+first,ys+first,sizeof(float))==0) first++;
fprintf(stderr,"%s: FAIL SSE4.1 != scalar at [%d]: %.9g vs %.9g\n",
name,first,(double)y[first],(double)ys[first]);
bad++;
}
free(ys);
#endif
free(q4);free(scale);free(x);free(y);free(yr);free(ym);
if(bad){ fprintf(stderr,"%s: FAIL (%d/%d mismatched, worst rel %.3g)\n",name,bad,S*O,worst); return 1; }
printf(" %-42s ok (S=%d I=%d O=%d gs=%d ng=%d, worst rel %.2g%s)\n",
name,S,I,O,gs,ng,worst,
#ifdef COLI_I4_GROUPED_SCALAR_EXACT
", bit-exact vs scalar"
#else
""
#endif
);
return 0;
}
/* matmul_i4_grouped_pair (fused gate+up, #298) reads x once instead of twice.
* Checked two ways, because "identical" is only true where it can be:
*
* - Correctness, always: both outputs must match the double reference within
* the same magnitude-relative epsilon as the unfused kernel.
* - Bit-exactness: forced-SSE builds require every shape to match two scalar
* calls. Other builds retain the aligned-shape invariant used by AVX2.
*
* On AVX2, a PARTIAL last group falls to scalar code and the fused body can
* round differently from the single-matrix one. The forced-SSE rows4 path has
* no horizontal reduction or group tail and is exact for those shapes too. */
#ifdef COLI_HAVE_GROUPED_PAIR
static int check_pair(const char *name, int S, int I, int O, int gs){
int rb=(I+1)/2, ng=(I+gs-1)/gs;
uint8_t *qg=malloc((size_t)O*rb), *qu=malloc((size_t)O*rb);
float *sg=malloc((size_t)O*ng*sizeof(float)), *su=malloc((size_t)O*ng*sizeof(float));
float *x=malloc((size_t)S*I*sizeof(float));
float *yg=malloc((size_t)S*O*sizeof(float)), *yu=malloc((size_t)S*O*sizeof(float));
float *rg=malloc((size_t)S*O*sizeof(float)), *ru=malloc((size_t)S*O*sizeof(float));
double *dg=malloc((size_t)S*O*sizeof(double)), *du=malloc((size_t)S*O*sizeof(double));
double *mg=malloc((size_t)S*O*sizeof(double)), *mu=malloc((size_t)S*O*sizeof(double));
if(!qg||!qu||!sg||!su||!x||!yg||!yu||!rg||!ru||!dg||!du||!mg||!mu){ fprintf(stderr,"%s: OOM\n",name); return 1; }
for(size_t i=0;i<(size_t)O*rb;i++){ qg[i]=(uint8_t)(xr()&0xFF); qu[i]=(uint8_t)(xr()&0xFF); }
for(int i=0;i<O*ng;i++){ sg[i]=frand(); su[i]=frand(); }
for(int i=0;i<S*I;i++) x[i]=frand();
matmul_i4_grouped_pair(yg,yu,x,qg,sg,qu,su,S,I,O,gs);
matmul_i4_grouped(rg,x,qg,sg,S,I,O,gs);
matmul_i4_grouped(ru,x,qu,su,S,I,O,gs);
ref_grouped(dg,mg,x,qg,sg,S,I,O,gs);
ref_grouped(du,mu,x,qu,su,S,I,O,gs);
int bad=0, exact=1; double worst=0;
for(int i=0;i<S*O;i++){
double eg = mg[i]>1e-30 ? fabs((double)yg[i]-dg[i])/mg[i] : fabs((double)yg[i]-dg[i]);
double eu = mu[i]>1e-30 ? fabs((double)yu[i]-du[i])/mu[i] : fabs((double)yu[i]-du[i]);
if(eg>worst) worst=eg;
if(eu>worst) worst=eu;
if(eg>1e-6||eu>1e-6){
if(bad<3) fprintf(stderr,"%s: [%d] gate %.9g/%.9g up %.9g/%.9g (rel %.3g/%.3g)\n",
name,i,(double)yg[i],dg[i],(double)yu[i],du[i],eg,eu);
bad++;
}
if(yg[i]!=rg[i]||yu[i]!=ru[i]) exact=0;
}
int require_exact=I%gs==0;
#ifdef COLI_I4_GROUPED_SCALAR_EXACT
require_exact=1;
exact=memcmp(yg,rg,(size_t)S*O*sizeof(float))==0
&& memcmp(yu,ru,(size_t)S*O*sizeof(float))==0;
#endif
if(require_exact && !exact){
fprintf(stderr,"%s: FAIL fused != unfused bitwise "
"(the operation order must match)\n",name);
bad++;
}
free(qg);free(qu);free(sg);free(su);free(x);free(yg);free(yu);free(rg);free(ru);
free(dg);free(du);free(mg);free(mu);
if(bad){ fprintf(stderr,"%s: FAIL (%d mismatched)\n",name,bad); return 1; }
printf(" %-42s ok (S=%d I=%d O=%d gs=%d, worst rel %.2g%s)\n",name,S,I,O,gs,worst,
require_exact?", bit-exact vs unfused":"");
return 0;
}
#endif
#if defined(__SSE4_1__) && !defined(__AVX2__)
/* Exercise every packed byte in every lane, with tight and strided rows.
* The last byte of the last row sits at the allocation boundary so ASan also
* catches a wider load that would read beyond the packed matrix. */
static int check_rows4_unpack(void){
const int strides[]={1,3}, offsets[]={0,1};
for(int c=0;c<2;c++){
int rb=strides[c],o=offsets[c];
size_t n=(size_t)(o+4)*rb;
uint8_t *q4=malloc(n);
if(!q4){ fprintf(stderr,"rows4 unpack: OOM\n"); return 1; }
for(int value=0;value<256;value++){
for(size_t i=0;i<n;i++) q4[i]=(uint8_t)(value+37*i);
for(int byte=0;byte<rb;byte++){
__m128 lo,hi; float l[4],h[4];
colibri_sse41_i4_rows4(q4,rb,o,byte,&lo,&hi);
_mm_storeu_ps(l,lo); _mm_storeu_ps(h,hi);
for(int row=0;row<4;row++){
unsigned packed=q4[(o+row)*rb+byte];
if(l[row]!=(float)((int)(packed&15)-8)
|| h[row]!=(float)((int)(packed>>4)-8)){
fprintf(stderr,"rows4 unpack: rb=%d o=%d byte=%d row=%d value=%u\n",
rb,o,byte,row,packed);
free(q4); return 1;
}
}
}
}
free(q4);
}
printf(" rows4 unpack: all 256 byte values per lane, tight/strided rows ok\n");
return 0;
}
#endif
int main(void){
int fail=0;
printf("test_i4_grouped: matmul_i4_grouped vs plain-C dequant reference\n");
#if defined(__SSE4_1__) && !defined(__AVX2__)
fail|=check_rows4_unpack();
#endif
/* the shape the g64 checkpoints actually use */
fail|=check("gs=64, I multiple of gs", 2, 512, 8, 64, 0);
fail|=check("gs=64, single row single token", 1, 128, 1, 64, 0);
fail|=check("gs=64, nibble edges (0x00/0xFF)", 1, 256, 4, 64, 1);
/* partial last group: glen clamp, the classic off-by-one */
fail|=check("gs=64, partial last group (I=200)", 2, 200, 4, 64, 0);
fail|=check("gs=64, I just over a group (I=65)", 1, 65, 3, 64, 0);
fail|=check("gs=64, I one under a group (I=63)", 1, 63, 3, 64, 0);
/* odd I: the scalar nibble tail (i+1 == I) */
fail|=check("gs=64, odd I (I=201)", 2, 201, 4, 64, 0);
fail|=check("gs=16, odd I (I=33)", 1, 33, 2, 16, 0);
/* gs > I: everything in one group */
fail|=check("gs=128 > I=64 (single group)", 1, 64, 4, 128, 0);
/* the other documented group size */
fail|=check("gs=128, I multiple of gs", 2, 512, 4, 128, 0);
/* batch: S>1 exercises the per-s inner loop against a shared scale row */
fail|=check("gs=64, batch S=8", 8, 320, 6, 64, 0);
fail|=check("one-byte rows, odd I=1", 1, 1, 4, 2, 0);
#ifdef COLI_HAVE_GROUPED_PAIR
printf("test_i4_grouped: matmul_i4_grouped_pair (fused gate+up) vs two separate calls\n");
fail|=check_pair("pair: gs=64, I multiple of gs", 2, 512, 8, 64);
fail|=check_pair("pair: gs=64, partial last group",2, 200, 4, 64);
fail|=check_pair("pair: gs=64, odd I (I=201)", 2, 201, 4, 64);
fail|=check_pair("pair: gs=64, decode S=1", 1, 320, 6, 64);
fail|=check_pair("pair: gs=128, I=512", 2, 512, 4, 128);
fail|=check_pair("pair: one-byte rows, odd I=1", 1, 1, 4, 2);
#endif
if(fail){ printf("test_i4_grouped: FAIL\n"); return 1; }
printf("test_i4_grouped: ok\n");
return 0;
}