Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

quant: the f32 matmul runs eight fmaf chains at once, with the same bits on every FMA build (#442)

RayanR000e783ff4e3519
diff --git a/c/Makefile b/c/Makefile
index 49db8d14..00db6f83 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -2232,6 +2232,14 @@ tests/bench_i4p_gidot$(EXE): tests/bench_i4p_gidot.c sse41_kernels.h colibri.c o
tests/bench_gemv_stream$(EXE): tests/bench_gemv_stream.c sse41_kernels.h colibri.c oracle.h st.h uring.h json.h tok.h tok_unicode.h tok_unicode_deepseek.h compat.h grammar.h tier.h quant.h fp8_format.h idot.h sample.h kv_persist.h telemetry.h route_trace.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+# bench_matmul_f32: microbenchmark (pre-#442 scalar vs row-blocked fmaf f32 matmul at
+# router shapes), NOT a test gate. Build on demand: make tests/bench_matmul_f32
+tests/bench_matmul_f32$(EXE): tests/bench_matmul_f32.c quant.h idot.h sse41_kernels.h
+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+
+tests/test_matmul_f32$(EXE): tests/test_matmul_f32.c quant.h idot.h sse41_kernels.h
+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+
# bench_mla_simd: microbenchmark (scalar vs AVX2/NEON MLA-absorb reductions, #442),
# NOT a test gate. Build on demand: make tests/bench_mla_simd
tests/bench_mla_simd$(EXE): tests/bench_mla_simd.c sse41_kernels.h colibri.c oracle.h st.h uring.h json.h tok.h tok_unicode.h tok_unicode_deepseek.h compat.h grammar.h tier.h quant.h fp8_format.h idot.h sample.h kv_persist.h telemetry.h route_trace.h
diff --git a/c/quant.h b/c/quant.h
index a62c36ff..b8028a5d 100644
--- a/c/quant.h
+++ b/c/quant.h
@@ -72,11 +72,32 @@ static int i4_acc512_selftest(void){
#endif
/* ---- y[S,O] = x[S,I] @ W^T, W[O,I] f32 ---------------------------------- */
+#if defined(__FMA__) || defined(__ARM_FEATURE_FMA)
+/* Every output is one fmaf chain over i in order, so the bits are the same on
+ * any FMA target and at any vector width. The scalar `a+=x*w` form they replace
+ * had toolchain-dependent bits: gcc and Apple clang vectorize the multiply and
+ * keep the adds in order (unfused), fusing only the tail; x86 clang fuses all.
+ * Eight rows per pass give eight independent chains instead of one (#442). */
+static void matmul(float *y, const float *x, const float *W, int S, int I, int O){
+ int O8=O&~7;
+ #pragma omp parallel for schedule(static)
+ for (int o=0;o<O8;o+=8){ const float *w0=W+(int64_t)o*I, *w1=w0+I, *w2=w1+I, *w3=w2+I, *w4=w3+I, *w5=w4+I, *w6=w5+I, *w7=w6+I;
+ for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a0=0,a1=0,a2=0,a3=0,a4=0,a5=0,a6=0,a7=0;
+ for(int i=0;i<I;i++){ float xi=xs[i];
+ a0=fmaf(xi,w0[i],a0); a1=fmaf(xi,w1[i],a1); a2=fmaf(xi,w2[i],a2); a3=fmaf(xi,w3[i],a3);
+ a4=fmaf(xi,w4[i],a4); a5=fmaf(xi,w5[i],a5); a6=fmaf(xi,w6[i],a6); a7=fmaf(xi,w7[i],a7); }
+ float *ys=y+(int64_t)s*O+o; ys[0]=a0; ys[1]=a1; ys[2]=a2; ys[3]=a3; ys[4]=a4; ys[5]=a5; ys[6]=a6; ys[7]=a7; } }
+ for (int o=O8;o<O;o++){ const float *w=W+(int64_t)o*I;
+ for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; for(int i=0;i<I;i++) a=fmaf(xs[i],w[i],a); y[(int64_t)s*O+o]=a; } }
+}
+#else
+/* No hardware FMA: fmaf would be a libm call, so keep the plain loop. */
static void matmul(float *y, const float *x, const float *W, int S, int I, int O){
#pragma omp parallel for schedule(static)
for (int o=0;o<O;o++){ const float *w=W+(int64_t)o*I;
for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; for(int i=0;i<I;i++) a+=xs[i]*w[i]; y[(int64_t)s*O+o]=a; } }
}
+#endif
/* ---- y[S,O] = x[S,I] @ W^T, W int8 per-row + scale[O] ------------------- */
static void matmul_q(float *y, const float *x, const int8_t *q, const float *scale, int S, int I, int O){
diff --git a/c/tests/bench_matmul_f32.c b/c/tests/bench_matmul_f32.c
new file mode 100644
index 00000000..241275c4
--- /dev/null
+++ b/c/tests/bench_matmul_f32.c
@@ -0,0 +1,64 @@
+/* Microbenchmark: the pre-#442 scalar f32 matmul vs quant.h's fmaf row-blocked
+ * matmul, at the GLM-5.2 router shape (colibri.c, D=6144 E=256) and a smaller
+ * router (D=2048 E=64), for decode (S=1) and a prefill chunk (S=16).
+ *
+ * NOT a unit test: tests/test_matmul_f32 gates exactness. Each arm runs
+ * SAMPLES timed samples (each the mean of enough calls to last ~50 ms) after
+ * one warm-up sample; the median is reported, arms interleaved per sample.
+ *
+ * Run: make tests/bench_matmul_f32 && ./tests/bench_matmul_f32
+ * OMP_NUM_THREADS=1 ./tests/bench_matmul_f32 (single thread) */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <math.h>
+#include <time.h>
+#include "../quant.h"
+
+#define SAMPLES 5
+
+/* ---- OLD: verbatim pre-#442 quant.h matmul ---- */
+static void matmul_old(float *y, const float *x, const float *W, int S, int I, int O){
+ #pragma omp parallel for schedule(static)
+ for (int o=0;o<O;o++){ const float *w=W+(int64_t)o*I;
+ for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; for(int i=0;i<I;i++) a+=xs[i]*w[i]; y[(int64_t)s*O+o]=a; } }
+}
+
+static double now_s(void){ struct timespec t; clock_gettime(CLOCK_MONOTONIC,&t); return t.tv_sec+t.tv_nsec*1e-9; }
+static int cmp_d(const void *a, const void *b){ double x=*(const double*)a, y=*(const double*)b; return (x>y)-(x<y); }
+
+typedef void (*mm_fn)(float*,const float*,const float*,int,int,int);
+static double sample_us(mm_fn f, float *y, const float *x, const float *W, int S, int I, int O, int reps){
+ mm_fn volatile call=f; /* opaque call: without OpenMP the unused result would be optimized away */
+ double t=now_s(); for (int r=0;r<reps;r++) call(y,x,W,S,I,O); return (now_s()-t)/reps*1e6;
+}
+
+int main(void){
+ static const struct { const char *name; int S, I, O; } shapes[]={
+ {"GLM-5.2 router decode", 1, 6144, 256}, {"GLM-5.2 router prefill", 16, 6144, 256},
+ {"small router decode", 1, 2048, 64}, {"small router prefill", 16, 2048, 64} };
+ uint64_t rng=0x9E3779B97F4A7C15ull;
+#ifdef _OPENMP
+ printf("threads: %d\n", omp_get_max_threads());
+#else
+ printf("threads: 1 (no OpenMP)\n");
+#endif
+ for (size_t k=0;k<sizeof shapes/sizeof shapes[0];k++){
+ int S=shapes[k].S, I=shapes[k].I, O=shapes[k].O;
+ float *x=malloc(sizeof(float)*(size_t)S*I), *W=malloc(sizeof(float)*(size_t)I*O), *y=malloc(sizeof(float)*(size_t)S*O);
+ for (size_t i=0;i<(size_t)S*I;i++){ rng^=rng<<13; rng^=rng>>7; rng^=rng<<17; x[i]=(float)(rng>>40)/(float)(1<<24)*2.f-1.f; }
+ for (size_t i=0;i<(size_t)I*O;i++){ rng^=rng<<13; rng^=rng>>7; rng^=rng<<17; W[i]=((float)(rng>>40)/(float)(1<<24)*2.f-1.f)*0.05f; }
+ int reps=1; while (sample_us(matmul_old,y,x,W,S,I,O,reps)*reps<50e3 && reps<1<<20) reps*=2;
+ sample_us(matmul_old,y,x,W,S,I,O,reps); sample_us(matmul,y,x,W,S,I,O,reps); /* warm-up */
+ double to[SAMPLES], tn[SAMPLES];
+ for (int r=0;r<SAMPLES;r++){ to[r]=sample_us(matmul_old,y,x,W,S,I,O,reps); tn[r]=sample_us(matmul,y,x,W,S,I,O,reps); }
+ printf("%-24s S=%-2d I=%d O=%d old:", shapes[k].name, S, I, O);
+ for (int r=0;r<SAMPLES;r++) printf(" %.2f", to[r]);
+ printf(" new:"); for (int r=0;r<SAMPLES;r++) printf(" %.2f", tn[r]);
+ qsort(to,SAMPLES,sizeof(double),cmp_d); qsort(tn,SAMPLES,sizeof(double),cmp_d);
+ printf(" us/call; median %.2f -> %.2f (%.2fx)\n", to[SAMPLES/2], tn[SAMPLES/2], to[SAMPLES/2]/tn[SAMPLES/2]);
+ free(x); free(W); free(y);
+ }
+ return 0;
+}
diff --git a/c/tests/test_matmul_f32.c b/c/tests/test_matmul_f32.c
new file mode 100644
index 00000000..77fbd48b
--- /dev/null
+++ b/c/tests/test_matmul_f32.c
@@ -0,0 +1,50 @@
+/* f32 matmul exactness gate (#442).
+ *
+ * On FMA targets quant.h's matmul computes each output as one fmaf chain over
+ * i in order, eight rows at a time. The row blocking must not change a single
+ * bit: this compares it with memcmp against the one-row fmaf chain, over shapes
+ * that exercise the eight-row blocks, the leftover rows and short rows. Without
+ * FMA matmul keeps the plain `a+=x*w` loop, and the reference follows it.
+ *
+ * Build: make -C c tests/test_matmul_f32 */
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <math.h>
+#include "../quant.h"
+
+static uint64_t rng = 0x2545F4914F6CDD1Dull;
+static float rnd_f(void){ rng^=rng<<13; rng^=rng>>7; rng^=rng<<17; return (float)(rng>>40)/(float)(1<<24)*2.f-1.f; }
+
+static void ref_matmul(float *y, const float *x, const float *W, int S, int I, int O){
+ for (int o=0;o<O;o++){ const float *w=W+(int64_t)o*I;
+ for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0;
+#if defined(__FMA__) || defined(__ARM_FEATURE_FMA)
+ for(int i=0;i<I;i++) a=fmaf(xs[i],w[i],a);
+#else
+ for(int i=0;i<I;i++) a+=xs[i]*w[i];
+#endif
+ y[(int64_t)s*O+o]=a; } }
+}
+
+int main(void){
+ static const int shapes[][3]={ {1,1,1}, {1,5,8}, {1,7,13}, {3,33,17}, {2,1027,41},
+ {1,2048,64}, {4,2048,64}, {5,6144,256}, {1,6144,257} };
+ int fails=0;
+ for (size_t k=0;k<sizeof shapes/sizeof shapes[0];k++){
+ int S=shapes[k][0], I=shapes[k][1], O=shapes[k][2];
+ float *x=malloc(sizeof(float)*(size_t)S*I), *W=malloc(sizeof(float)*(size_t)I*O);
+ float *y=malloc(sizeof(float)*(size_t)S*O), *r=malloc(sizeof(float)*(size_t)S*O);
+ for (size_t i=0;i<(size_t)S*I;i++) x[i]=rnd_f();
+ for (size_t i=0;i<(size_t)I*O;i++) W[i]=rnd_f()*0.05f;
+ matmul(y,x,W,S,I,O); ref_matmul(r,x,W,S,I,O);
+ int bad=0; for (int i=0;i<S*O;i++) bad+=memcmp(&y[i],&r[i],sizeof(float))!=0;
+ printf("%s matmul S=%d I=%d O=%d: %d/%d outputs differ from the reference\n",
+ bad?"FAIL":"PASS", S, I, O, bad, S*O);
+ fails+=bad!=0;
+ free(x); free(W); free(y); free(r);
+ }
+ printf("\n%s\n", fails ? "FAILED" : "ALL PASS");
+ return fails?1:0;
+}