Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

quant: NEON arm for batched matmul_fp8 (bit-exact vs scalar)

mfethe13a3ba2b2abd3
diff --git a/c/quant.h b/c/quant.h
index a62c36ff..e7db670e 100644
--- a/c/quant.h
+++ b/c/quant.h
@@ -572,8 +572,8 @@ static inline __m256 bf16_decode8(const uint16_t *p) {
So clang gets the fast kernel and GCC keeps upstream's, which is exact on
every arch tested. */
#if defined(__clang__)
-static void matmul_fp8(float *y, const float *x, const uint8_t *q8, const float *bscale,
- int S, int I, int O){
+static void matmul_fp8_scalar(float *y, const float *x, const uint8_t *q8, const float *bscale,
+ int S, int I, int O){
int64_t nblkI = fp8_nblk(I);
/* Four output rows per pass, each with its own accumulator. Every row's
addition sequence is identical to the one-row form - same operands, same
@@ -619,8 +619,8 @@ static void matmul_fp8(float *y, const float *x, const uint8_t *q8, const float
#else
/* GCC and everything else: upstream's one-row kernel, unchanged. Exact on
every -march tested; see the note above for why it is not simply replaced. */
-static void matmul_fp8(float *y, const float *x, const uint8_t *q8, const float *bscale,
- int S, int I, int O){
+static void matmul_fp8_scalar(float *y, const float *x, const uint8_t *q8, const float *bscale,
+ int S, int I, int O){
int64_t nblkI = fp8_nblk(I);
#pragma omp parallel for schedule(static)
for(int o=0;o<O;o++){
@@ -642,6 +642,72 @@ static void matmul_fp8(float *y, const float *x, const uint8_t *q8, const float
}
#endif
+#ifdef __ARM_NEON
+/* Batched (prefill) NEON arm: 16 output rows x 4 tokens per pass. Each block's
+ 16 rows are decoded once into a column-major tile; each row's per-block sum is
+ then one vfmaq chain over i = base..base+blen-1 from zero -- the same fused
+ multiply-adds, in the same order, that clang (-ffp-contract=on) and GCC
+ (-ffp-contract=fast, the arm64 default) emit for the scalar form -- and the
+ double-precision block combine is unchanged, so the output is bit-identical to
+ matmul_fp8_scalar (test_fp8_passthrough checks it). A 16-row tile never
+ straddles a FP8_BLOCK row block. Rows past O are zero-padded and never
+ stored; token lanes past S clamp onto the last token and are never summed.
+ S < 4 stays scalar: one token cannot amortise the tile decode. */
+static void matmul_fp8_neon(float *y, const float *x, const uint8_t *q8, const float *bscale,
+ int S, int I, int O){
+ int64_t nblkI = fp8_nblk(I);
+ #pragma omp parallel for schedule(static)
+ for(int o=0;o<O;o+=16){
+ int rn = O-o<16 ? O-o : 16;
+ const uint8_t *w = q8 + (int64_t)o*I;
+ const float *scl = bscale + ((int64_t)o/FP8_BLOCK)*nblkI;
+ float wt[FP8_BLOCK*16]; /* wt[i*16+r] */
+ double a[64*16];
+ for(int s0=0;s0<S;s0+=64){
+ int sn = S-s0<64 ? S-s0 : 64;
+ memset(a,0,sizeof(double)*16*sn);
+ for(int64_t bi=0; bi*FP8_BLOCK<I; bi++){
+ int base=(int)(bi*FP8_BLOCK), blen=I-base<FP8_BLOCK ? I-base : FP8_BLOCK;
+ double sc=scl[bi];
+ for(int r=0;r<16;r++) for(int i=0;i<blen;i++)
+ wt[i*16+r] = r<rn ? e4m3_decode(w[(int64_t)r*I+base+i]) : 0.f;
+ for(int s=0;s<sn;s+=4){
+ const float *xt[4];
+ float32x4_t c[4][4];
+ for(int t=0;t<4;t++){
+ xt[t] = x + (int64_t)(s0+(s+t<sn ? s+t : sn-1))*I + base;
+ for(int k=0;k<4;k++) c[t][k]=vdupq_n_f32(0.f);
+ }
+ for(int i=0;i<blen;i++){
+ const float *p = wt+i*16;
+ float32x4_t w0=vld1q_f32(p),w1=vld1q_f32(p+4),w2=vld1q_f32(p+8),w3=vld1q_f32(p+12);
+ for(int t=0;t<4;t++){
+ float32x4_t xv=vdupq_n_f32(xt[t][i]);
+ c[t][0]=vfmaq_f32(c[t][0],w0,xv); c[t][1]=vfmaq_f32(c[t][1],w1,xv);
+ c[t][2]=vfmaq_f32(c[t][2],w2,xv); c[t][3]=vfmaq_f32(c[t][3],w3,xv);
+ }
+ }
+ for(int t=0;t<4 && s+t<sn;t++){
+ float acc[16]; double *as=a+(s+t)*16;
+ for(int k=0;k<4;k++) vst1q_f32(acc+4*k,c[t][k]);
+ for(int r=0;r<16;r++) as[r] += (double)acc[r]*sc;
+ }
+ }
+ }
+ for(int s=0;s<sn;s++) for(int r=0;r<rn;r++) y[(int64_t)(s0+s)*O+o+r]=(float)a[s*16+r];
+ }
+ }
+}
+#endif
+
+static void matmul_fp8(float *y, const float *x, const uint8_t *q8, const float *bscale,
+ int S, int I, int O){
+#ifdef __ARM_NEON
+ if(S>=4){ matmul_fp8_neon(y,x,q8,bscale,S,I,O); return; }
+#endif
+ matmul_fp8_scalar(y,x,q8,bscale,S,I,O);
+}
+
/* f32 planare (fmt=2): stesso ordine di accumulazione di matmul_i4 (sequenza
* 0..63 in blocchi di 8 lane, stessa lane per elemento) -> bit-identico. */
diff --git a/c/tests/test_fp8_passthrough.c b/c/tests/test_fp8_passthrough.c
index e83b638b..12ca4131 100644
--- a/c/tests/test_fp8_passthrough.c
+++ b/c/tests/test_fp8_passthrough.c
@@ -151,6 +151,28 @@ static void test_nan_propagation(void){
CHECK(isnan(y[1])); /* contaminated row propagates NaN */
}
+/* matmul_fp8 routes S>=4 through the NEON batch arm on arm64: it must be
+ * bit-identical to the scalar kernel, token by token. Odd O/I/S cover the
+ * zero-padded row tile, the clamped token lanes and the 64-token chunk edge;
+ * one NaN byte checks the propagation policy survives the tile decode. */
+static void run_batch_exact(int S, int I, int O){
+ uint8_t *q8=malloc((size_t)O*I);
+ int64_t nblk=fp8_nblk(O)*fp8_nblk(I);
+ float *bscale=malloc((size_t)nblk*sizeof(float)), *x=malloc((size_t)S*I*sizeof(float));
+ float *y=malloc((size_t)S*O*sizeof(float)), *y1=malloc((size_t)S*O*sizeof(float));
+ for(int64_t i=0;i<(int64_t)O*I;i++) q8[i]=rndbyte_nonan();
+ q8[(int64_t)(O-1)*I + I/2] = 0x7F;
+ for(int64_t b=0;b<nblk;b++) bscale[b]=rndf();
+ for(int64_t i=0;i<(int64_t)S*I;i++) x[i]=rndf();
+ matmul_fp8(y, x, q8, bscale, S, I, O);
+ matmul_fp8_scalar(y1, x, q8, bscale, S, I, O);
+ int same=1;
+ for(int64_t i=0;i<(int64_t)S*O;i++) same &= isnan(y1[i]) ? isnan(y[i]) : !memcmp(y+i,y1+i,sizeof(float));
+ if(!same) printf(" batch S=%d I=%d O=%d not bit-identical to scalar\n", S, I, O);
+ CHECK(same);
+ free(q8); free(bscale); free(x); free(y); free(y1);
+}
+
int main(void){
test_lut();
run_block_case(2048, 6144, "block gate/up-shaped O=2048 I=6144 (spec example)");
@@ -161,6 +183,8 @@ int main(void){
run_block_case(1, 1, "degenerate 1x1 (single sub-block)");
run_block_case(384, 6144, "non-square block grid nblkO=3 nblkI=48 (stride audit)");
test_nan_propagation();
+ int bs[][3]={{4,128,16},{5,200,130},{7,129,37},{64,1000,257},{65,4096,128},{130,2048,512}};
+ for(unsigned k=0;k<sizeof bs/sizeof*bs;k++) run_batch_exact(bs[k][0],bs[k][1],bs[k][2]);
if(fails){ printf("fp8 passthrough CPU tests: %d FAILED\n", fails); return 1; }
printf("fp8 passthrough CPU tests: ok\n");
return 0;