Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
quant: the f32 matmul runs eight fmaf chains at once, with the same bits on every FMA build (#442)
RayanR000e783ff4e3519
diff --git a/c/Makefile b/c/Makefileindex 49db8d14..00db6f83 100644--- a/c/Makefile+++ b/c/Makefile@@ -2232,6 +2232,14 @@ tests/bench_i4p_gidot$(EXE): tests/bench_i4p_gidot.c sse41_kernels.h colibri.c otests/bench_gemv_stream$(EXE): tests/bench_gemv_stream.c sse41_kernels.h colibri.c oracle.h st.h uring.h json.h tok.h tok_unicode.h tok_unicode_deepseek.h compat.h grammar.h tier.h quant.h fp8_format.h idot.h sample.h kv_persist.h telemetry.h route_trace.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+# bench_matmul_f32: microbenchmark (pre-#442 scalar vs row-blocked fmaf f32 matmul at+# router shapes), NOT a test gate. Build on demand: make tests/bench_matmul_f32+tests/bench_matmul_f32$(EXE): tests/bench_matmul_f32.c quant.h idot.h sse41_kernels.h+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)++tests/test_matmul_f32$(EXE): tests/test_matmul_f32.c quant.h idot.h sse41_kernels.h+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+# bench_mla_simd: microbenchmark (scalar vs AVX2/NEON MLA-absorb reductions, #442),# NOT a test gate. Build on demand: make tests/bench_mla_simdtests/bench_mla_simd$(EXE): tests/bench_mla_simd.c sse41_kernels.h colibri.c oracle.h st.h uring.h json.h tok.h tok_unicode.h tok_unicode_deepseek.h compat.h grammar.h tier.h quant.h fp8_format.h idot.h sample.h kv_persist.h telemetry.h route_trace.hdiff --git a/c/quant.h b/c/quant.hindex a62c36ff..b8028a5d 100644--- a/c/quant.h+++ b/c/quant.h@@ -72,11 +72,32 @@ static int i4_acc512_selftest(void){#endif/* ---- y[S,O] = x[S,I] @ W^T, W[O,I] f32 ---------------------------------- */+#if defined(__FMA__) || defined(__ARM_FEATURE_FMA)+/* Every output is one fmaf chain over i in order, so the bits are the same on+ * any FMA target and at any vector width. The scalar `a+=x*w` form they replace+ * had toolchain-dependent bits: gcc and Apple clang vectorize the multiply and+ * keep the adds in order (unfused), fusing only the tail; x86 clang fuses all.+ * Eight rows per pass give eight independent chains instead of one (#442). */+static void matmul(float *y, const float *x, const float *W, int S, int I, int O){+ int O8=O&~7;+ #pragma omp parallel for schedule(static)+ for (int o=0;o<O8;o+=8){ const float *w0=W+(int64_t)o*I, *w1=w0+I, *w2=w1+I, *w3=w2+I, *w4=w3+I, *w5=w4+I, *w6=w5+I, *w7=w6+I;+ for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a0=0,a1=0,a2=0,a3=0,a4=0,a5=0,a6=0,a7=0;+ for(int i=0;i<I;i++){ float xi=xs[i];+ a0=fmaf(xi,w0[i],a0); a1=fmaf(xi,w1[i],a1); a2=fmaf(xi,w2[i],a2); a3=fmaf(xi,w3[i],a3);+ a4=fmaf(xi,w4[i],a4); a5=fmaf(xi,w5[i],a5); a6=fmaf(xi,w6[i],a6); a7=fmaf(xi,w7[i],a7); }+ float *ys=y+(int64_t)s*O+o; ys[0]=a0; ys[1]=a1; ys[2]=a2; ys[3]=a3; ys[4]=a4; ys[5]=a5; ys[6]=a6; ys[7]=a7; } }+ for (int o=O8;o<O;o++){ const float *w=W+(int64_t)o*I;+ for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; for(int i=0;i<I;i++) a=fmaf(xs[i],w[i],a); y[(int64_t)s*O+o]=a; } }+}+#else+/* No hardware FMA: fmaf would be a libm call, so keep the plain loop. */static void matmul(float *y, const float *x, const float *W, int S, int I, int O){#pragma omp parallel for schedule(static)for (int o=0;o<O;o++){ const float *w=W+(int64_t)o*I;for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; for(int i=0;i<I;i++) a+=xs[i]*w[i]; y[(int64_t)s*O+o]=a; } }}+#endif/* ---- y[S,O] = x[S,I] @ W^T, W int8 per-row + scale[O] ------------------- */static void matmul_q(float *y, const float *x, const int8_t *q, const float *scale, int S, int I, int O){diff --git a/c/tests/bench_matmul_f32.c b/c/tests/bench_matmul_f32.cnew file mode 100644index 00000000..241275c4--- /dev/null+++ b/c/tests/bench_matmul_f32.c@@ -0,0 +1,64 @@+/* Microbenchmark: the pre-#442 scalar f32 matmul vs quant.h's fmaf row-blocked+ * matmul, at the GLM-5.2 router shape (colibri.c, D=6144 E=256) and a smaller+ * router (D=2048 E=64), for decode (S=1) and a prefill chunk (S=16).+ *+ * NOT a unit test: tests/test_matmul_f32 gates exactness. Each arm runs+ * SAMPLES timed samples (each the mean of enough calls to last ~50 ms) after+ * one warm-up sample; the median is reported, arms interleaved per sample.+ *+ * Run: make tests/bench_matmul_f32 && ./tests/bench_matmul_f32+ * OMP_NUM_THREADS=1 ./tests/bench_matmul_f32 (single thread) */+#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <stdint.h>+#include <math.h>+#include <time.h>+#include "../quant.h"++#define SAMPLES 5++/* ---- OLD: verbatim pre-#442 quant.h matmul ---- */+static void matmul_old(float *y, const float *x, const float *W, int S, int I, int O){+ #pragma omp parallel for schedule(static)+ for (int o=0;o<O;o++){ const float *w=W+(int64_t)o*I;+ for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; for(int i=0;i<I;i++) a+=xs[i]*w[i]; y[(int64_t)s*O+o]=a; } }+}++static double now_s(void){ struct timespec t; clock_gettime(CLOCK_MONOTONIC,&t); return t.tv_sec+t.tv_nsec*1e-9; }+static int cmp_d(const void *a, const void *b){ double x=*(const double*)a, y=*(const double*)b; return (x>y)-(x<y); }++typedef void (*mm_fn)(float*,const float*,const float*,int,int,int);+static double sample_us(mm_fn f, float *y, const float *x, const float *W, int S, int I, int O, int reps){+ mm_fn volatile call=f; /* opaque call: without OpenMP the unused result would be optimized away */+ double t=now_s(); for (int r=0;r<reps;r++) call(y,x,W,S,I,O); return (now_s()-t)/reps*1e6;+}++int main(void){+ static const struct { const char *name; int S, I, O; } shapes[]={+ {"GLM-5.2 router decode", 1, 6144, 256}, {"GLM-5.2 router prefill", 16, 6144, 256},+ {"small router decode", 1, 2048, 64}, {"small router prefill", 16, 2048, 64} };+ uint64_t rng=0x9E3779B97F4A7C15ull;+#ifdef _OPENMP+ printf("threads: %d\n", omp_get_max_threads());+#else+ printf("threads: 1 (no OpenMP)\n");+#endif+ for (size_t k=0;k<sizeof shapes/sizeof shapes[0];k++){+ int S=shapes[k].S, I=shapes[k].I, O=shapes[k].O;+ float *x=malloc(sizeof(float)*(size_t)S*I), *W=malloc(sizeof(float)*(size_t)I*O), *y=malloc(sizeof(float)*(size_t)S*O);+ for (size_t i=0;i<(size_t)S*I;i++){ rng^=rng<<13; rng^=rng>>7; rng^=rng<<17; x[i]=(float)(rng>>40)/(float)(1<<24)*2.f-1.f; }+ for (size_t i=0;i<(size_t)I*O;i++){ rng^=rng<<13; rng^=rng>>7; rng^=rng<<17; W[i]=((float)(rng>>40)/(float)(1<<24)*2.f-1.f)*0.05f; }+ int reps=1; while (sample_us(matmul_old,y,x,W,S,I,O,reps)*reps<50e3 && reps<1<<20) reps*=2;+ sample_us(matmul_old,y,x,W,S,I,O,reps); sample_us(matmul,y,x,W,S,I,O,reps); /* warm-up */+ double to[SAMPLES], tn[SAMPLES];+ for (int r=0;r<SAMPLES;r++){ to[r]=sample_us(matmul_old,y,x,W,S,I,O,reps); tn[r]=sample_us(matmul,y,x,W,S,I,O,reps); }+ printf("%-24s S=%-2d I=%d O=%d old:", shapes[k].name, S, I, O);+ for (int r=0;r<SAMPLES;r++) printf(" %.2f", to[r]);+ printf(" new:"); for (int r=0;r<SAMPLES;r++) printf(" %.2f", tn[r]);+ qsort(to,SAMPLES,sizeof(double),cmp_d); qsort(tn,SAMPLES,sizeof(double),cmp_d);+ printf(" us/call; median %.2f -> %.2f (%.2fx)\n", to[SAMPLES/2], tn[SAMPLES/2], to[SAMPLES/2]/tn[SAMPLES/2]);+ free(x); free(W); free(y);+ }+ return 0;+}diff --git a/c/tests/test_matmul_f32.c b/c/tests/test_matmul_f32.cnew file mode 100644index 00000000..77fbd48b--- /dev/null+++ b/c/tests/test_matmul_f32.c@@ -0,0 +1,50 @@+/* f32 matmul exactness gate (#442).+ *+ * On FMA targets quant.h's matmul computes each output as one fmaf chain over+ * i in order, eight rows at a time. The row blocking must not change a single+ * bit: this compares it with memcmp against the one-row fmaf chain, over shapes+ * that exercise the eight-row blocks, the leftover rows and short rows. Without+ * FMA matmul keeps the plain `a+=x*w` loop, and the reference follows it.+ *+ * Build: make -C c tests/test_matmul_f32 */+#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <stdint.h>+#include <math.h>+#include "../quant.h"++static uint64_t rng = 0x2545F4914F6CDD1Dull;+static float rnd_f(void){ rng^=rng<<13; rng^=rng>>7; rng^=rng<<17; return (float)(rng>>40)/(float)(1<<24)*2.f-1.f; }++static void ref_matmul(float *y, const float *x, const float *W, int S, int I, int O){+ for (int o=0;o<O;o++){ const float *w=W+(int64_t)o*I;+ for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0;+#if defined(__FMA__) || defined(__ARM_FEATURE_FMA)+ for(int i=0;i<I;i++) a=fmaf(xs[i],w[i],a);+#else+ for(int i=0;i<I;i++) a+=xs[i]*w[i];+#endif+ y[(int64_t)s*O+o]=a; } }+}++int main(void){+ static const int shapes[][3]={ {1,1,1}, {1,5,8}, {1,7,13}, {3,33,17}, {2,1027,41},+ {1,2048,64}, {4,2048,64}, {5,6144,256}, {1,6144,257} };+ int fails=0;+ for (size_t k=0;k<sizeof shapes/sizeof shapes[0];k++){+ int S=shapes[k][0], I=shapes[k][1], O=shapes[k][2];+ float *x=malloc(sizeof(float)*(size_t)S*I), *W=malloc(sizeof(float)*(size_t)I*O);+ float *y=malloc(sizeof(float)*(size_t)S*O), *r=malloc(sizeof(float)*(size_t)S*O);+ for (size_t i=0;i<(size_t)S*I;i++) x[i]=rnd_f();+ for (size_t i=0;i<(size_t)I*O;i++) W[i]=rnd_f()*0.05f;+ matmul(y,x,W,S,I,O); ref_matmul(r,x,W,S,I,O);+ int bad=0; for (int i=0;i<S*O;i++) bad+=memcmp(&y[i],&r[i],sizeof(float))!=0;+ printf("%s matmul S=%d I=%d O=%d: %d/%d outputs differ from the reference\n",+ bad?"FAIL":"PASS", S, I, O, bad, S*O);+ fails+=bad!=0;+ free(x); free(W); free(y); free(r);+ }+ printf("\n%s\n", fails ? "FAILED" : "ALL PASS");+ return fails?1:0;+}