Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
quant: NEON arm for batched matmul_fp8 (bit-exact vs scalar)
mfethe13a3ba2b2abd3
diff --git a/c/quant.h b/c/quant.hindex a62c36ff..e7db670e 100644--- a/c/quant.h+++ b/c/quant.h@@ -572,8 +572,8 @@ static inline __m256 bf16_decode8(const uint16_t *p) {So clang gets the fast kernel and GCC keeps upstream's, which is exact onevery arch tested. */#if defined(__clang__)-static void matmul_fp8(float *y, const float *x, const uint8_t *q8, const float *bscale,- int S, int I, int O){+static void matmul_fp8_scalar(float *y, const float *x, const uint8_t *q8, const float *bscale,+ int S, int I, int O){int64_t nblkI = fp8_nblk(I);/* Four output rows per pass, each with its own accumulator. Every row'saddition sequence is identical to the one-row form - same operands, same@@ -619,8 +619,8 @@ static void matmul_fp8(float *y, const float *x, const uint8_t *q8, const float#else/* GCC and everything else: upstream's one-row kernel, unchanged. Exact onevery -march tested; see the note above for why it is not simply replaced. */-static void matmul_fp8(float *y, const float *x, const uint8_t *q8, const float *bscale,- int S, int I, int O){+static void matmul_fp8_scalar(float *y, const float *x, const uint8_t *q8, const float *bscale,+ int S, int I, int O){int64_t nblkI = fp8_nblk(I);#pragma omp parallel for schedule(static)for(int o=0;o<O;o++){@@ -642,6 +642,72 @@ static void matmul_fp8(float *y, const float *x, const uint8_t *q8, const float}#endif+#ifdef __ARM_NEON+/* Batched (prefill) NEON arm: 16 output rows x 4 tokens per pass. Each block's+ 16 rows are decoded once into a column-major tile; each row's per-block sum is+ then one vfmaq chain over i = base..base+blen-1 from zero -- the same fused+ multiply-adds, in the same order, that clang (-ffp-contract=on) and GCC+ (-ffp-contract=fast, the arm64 default) emit for the scalar form -- and the+ double-precision block combine is unchanged, so the output is bit-identical to+ matmul_fp8_scalar (test_fp8_passthrough checks it). A 16-row tile never+ straddles a FP8_BLOCK row block. Rows past O are zero-padded and never+ stored; token lanes past S clamp onto the last token and are never summed.+ S < 4 stays scalar: one token cannot amortise the tile decode. */+static void matmul_fp8_neon(float *y, const float *x, const uint8_t *q8, const float *bscale,+ int S, int I, int O){+ int64_t nblkI = fp8_nblk(I);+ #pragma omp parallel for schedule(static)+ for(int o=0;o<O;o+=16){+ int rn = O-o<16 ? O-o : 16;+ const uint8_t *w = q8 + (int64_t)o*I;+ const float *scl = bscale + ((int64_t)o/FP8_BLOCK)*nblkI;+ float wt[FP8_BLOCK*16]; /* wt[i*16+r] */+ double a[64*16];+ for(int s0=0;s0<S;s0+=64){+ int sn = S-s0<64 ? S-s0 : 64;+ memset(a,0,sizeof(double)*16*sn);+ for(int64_t bi=0; bi*FP8_BLOCK<I; bi++){+ int base=(int)(bi*FP8_BLOCK), blen=I-base<FP8_BLOCK ? I-base : FP8_BLOCK;+ double sc=scl[bi];+ for(int r=0;r<16;r++) for(int i=0;i<blen;i++)+ wt[i*16+r] = r<rn ? e4m3_decode(w[(int64_t)r*I+base+i]) : 0.f;+ for(int s=0;s<sn;s+=4){+ const float *xt[4];+ float32x4_t c[4][4];+ for(int t=0;t<4;t++){+ xt[t] = x + (int64_t)(s0+(s+t<sn ? s+t : sn-1))*I + base;+ for(int k=0;k<4;k++) c[t][k]=vdupq_n_f32(0.f);+ }+ for(int i=0;i<blen;i++){+ const float *p = wt+i*16;+ float32x4_t w0=vld1q_f32(p),w1=vld1q_f32(p+4),w2=vld1q_f32(p+8),w3=vld1q_f32(p+12);+ for(int t=0;t<4;t++){+ float32x4_t xv=vdupq_n_f32(xt[t][i]);+ c[t][0]=vfmaq_f32(c[t][0],w0,xv); c[t][1]=vfmaq_f32(c[t][1],w1,xv);+ c[t][2]=vfmaq_f32(c[t][2],w2,xv); c[t][3]=vfmaq_f32(c[t][3],w3,xv);+ }+ }+ for(int t=0;t<4 && s+t<sn;t++){+ float acc[16]; double *as=a+(s+t)*16;+ for(int k=0;k<4;k++) vst1q_f32(acc+4*k,c[t][k]);+ for(int r=0;r<16;r++) as[r] += (double)acc[r]*sc;+ }+ }+ }+ for(int s=0;s<sn;s++) for(int r=0;r<rn;r++) y[(int64_t)(s0+s)*O+o+r]=(float)a[s*16+r];+ }+ }+}+#endif++static void matmul_fp8(float *y, const float *x, const uint8_t *q8, const float *bscale,+ int S, int I, int O){+#ifdef __ARM_NEON+ if(S>=4){ matmul_fp8_neon(y,x,q8,bscale,S,I,O); return; }+#endif+ matmul_fp8_scalar(y,x,q8,bscale,S,I,O);+}+/* f32 planare (fmt=2): stesso ordine di accumulazione di matmul_i4 (sequenza* 0..63 in blocchi di 8 lane, stessa lane per elemento) -> bit-identico. */diff --git a/c/tests/test_fp8_passthrough.c b/c/tests/test_fp8_passthrough.cindex e83b638b..12ca4131 100644--- a/c/tests/test_fp8_passthrough.c+++ b/c/tests/test_fp8_passthrough.c@@ -151,6 +151,28 @@ static void test_nan_propagation(void){CHECK(isnan(y[1])); /* contaminated row propagates NaN */}+/* matmul_fp8 routes S>=4 through the NEON batch arm on arm64: it must be+ * bit-identical to the scalar kernel, token by token. Odd O/I/S cover the+ * zero-padded row tile, the clamped token lanes and the 64-token chunk edge;+ * one NaN byte checks the propagation policy survives the tile decode. */+static void run_batch_exact(int S, int I, int O){+ uint8_t *q8=malloc((size_t)O*I);+ int64_t nblk=fp8_nblk(O)*fp8_nblk(I);+ float *bscale=malloc((size_t)nblk*sizeof(float)), *x=malloc((size_t)S*I*sizeof(float));+ float *y=malloc((size_t)S*O*sizeof(float)), *y1=malloc((size_t)S*O*sizeof(float));+ for(int64_t i=0;i<(int64_t)O*I;i++) q8[i]=rndbyte_nonan();+ q8[(int64_t)(O-1)*I + I/2] = 0x7F;+ for(int64_t b=0;b<nblk;b++) bscale[b]=rndf();+ for(int64_t i=0;i<(int64_t)S*I;i++) x[i]=rndf();+ matmul_fp8(y, x, q8, bscale, S, I, O);+ matmul_fp8_scalar(y1, x, q8, bscale, S, I, O);+ int same=1;+ for(int64_t i=0;i<(int64_t)S*O;i++) same &= isnan(y1[i]) ? isnan(y[i]) : !memcmp(y+i,y1+i,sizeof(float));+ if(!same) printf(" batch S=%d I=%d O=%d not bit-identical to scalar\n", S, I, O);+ CHECK(same);+ free(q8); free(bscale); free(x); free(y); free(y1);+}+int main(void){test_lut();run_block_case(2048, 6144, "block gate/up-shaped O=2048 I=6144 (spec example)");@@ -161,6 +183,8 @@ int main(void){run_block_case(1, 1, "degenerate 1x1 (single sub-block)");run_block_case(384, 6144, "non-square block grid nblkO=3 nblkI=48 (stride audit)");test_nan_propagation();+ int bs[][3]={{4,128,16},{5,200,130},{7,129,37},{64,1000,257},{65,4096,128},{130,2048,512}};+ for(unsigned k=0;k<sizeof bs/sizeof*bs;k++) run_batch_exact(bs[k][0],bs[k][1],bs[k][2]);if(fails){ printf("fp8 passthrough CPU tests: %d FAILED\n", fails); return 1; }printf("fp8 passthrough CPU tests: ok\n");return 0;