Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

quant: pin the scalar fp8 chain fused (fmaf) on aarch64; full-mantissa test inputs

Michael Fethe2e3e0c7b2f2
diff --git a/c/quant.h b/c/quant.h
index e7db670e..5fbc004b 100644
--- a/c/quant.h
+++ b/c/quant.h
@@ -599,10 +599,20 @@ static void matmul_fp8_scalar(float *y, const float *x, const uint8_t *q8, const
float acc0=0,acc1=0,acc2=0,acc3=0;
for(int i=base;i<base+blen;i++){
float xv=xs[i];
+#if defined(__aarch64__)
+ /* SVE review: explicit fma pins the chain FUSED regardless of
+ -ffp-contract or the auto-vectoriser (gcc on SVE lowers a
+ plain acc += w*x to unfused fadda and diverges from vfmaq). */
+ acc0 = __builtin_fmaf(e4m3_decode(w0[i]),xv,acc0);
+ acc1 = __builtin_fmaf(e4m3_decode(w1[i]),xv,acc1);
+ acc2 = __builtin_fmaf(e4m3_decode(w2[i]),xv,acc2);
+ acc3 = __builtin_fmaf(e4m3_decode(w3[i]),xv,acc3);
+#else
acc0 += e4m3_decode(w0[i])*xv;
acc1 += e4m3_decode(w1[i])*xv;
acc2 += e4m3_decode(w2[i])*xv;
acc3 += e4m3_decode(w3[i])*xv;
+#endif
}
a0 += (double)acc0*scl0[bi];
a1 += (double)acc1*scl1[bi];
@@ -633,7 +643,12 @@ static void matmul_fp8_scalar(float *y, const float *x, const uint8_t *q8, const
for(int64_t bi=0; bi*FP8_BLOCK<I; bi++){
int base=(int)(bi*FP8_BLOCK); int blen=FP8_BLOCK; if(base+blen>I) blen=I-base;
float sc=scl[bi]; float acc=0;
+#if defined(__aarch64__)
+ /* see the SVE note in the clang arm above */
+ for(int i=base;i<base+blen;i++) acc = __builtin_fmaf(e4m3_decode(w[i]),xs[i],acc);
+#else
for(int i=base;i<base+blen;i++) acc += e4m3_decode(w[i])*xs[i];
+#endif
a += (double)acc*sc;
}
y[(int64_t)s*O+o]=(float)a;
diff --git a/c/tests/test_fp8_passthrough.c b/c/tests/test_fp8_passthrough.c
index 12ca4131..34594dc9 100644
--- a/c/tests/test_fp8_passthrough.c
+++ b/c/tests/test_fp8_passthrough.c
@@ -155,6 +155,14 @@ static void test_nan_propagation(void){
* bit-identical to the scalar kernel, token by token. Odd O/I/S cover the
* zero-padded row tile, the clamped token lanes and the 64-token chunk edge;
* one NaN byte checks the propagation policy survives the tile decode. */
+/* full 23-bit-mantissa x in [1,2): e4m3 (4-bit mantissa) x such a value needs
+ * 27 bits, so w*x is NOT exactly representable and a fused multiply-add chain
+ * (vfmaq / fmaf / fmas) rounds differently from an unfused mul-then-add chain.
+ * The old rndf() inputs (20-bit mantissa) made every product exact, which is
+ * why this test could not see fused-vs-unfused divergence (SVE fadda review). */
+static float rndf_mant(void){ rng ^= rng << 13; rng ^= rng >> 7; rng ^= rng << 17;
+ uint32_t u = (uint32_t)rng; union { uint32_t b; float f; } v;
+ v.b = 0x3F800000u | (u & 0x7FFFFFu); return v.f; }
static void run_batch_exact(int S, int I, int O){
uint8_t *q8=malloc((size_t)O*I);
int64_t nblk=fp8_nblk(O)*fp8_nblk(I);
@@ -163,7 +171,7 @@ static void run_batch_exact(int S, int I, int O){
for(int64_t i=0;i<(int64_t)O*I;i++) q8[i]=rndbyte_nonan();
q8[(int64_t)(O-1)*I + I/2] = 0x7F;
for(int64_t b=0;b<nblk;b++) bscale[b]=rndf();
- for(int64_t i=0;i<(int64_t)S*I;i++) x[i]=rndf();
+ for(int64_t i=0;i<(int64_t)S*I;i++) x[i]=rndf_mant();
matmul_fp8(y, x, q8, bscale, S, I, O);
matmul_fp8_scalar(y1, x, q8, bscale, S, I, O);
int same=1;