Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
matmul_f32.h: one vectorized f32 matmul for quant.h, olmoe, inkling and qwen36 (#442)
RayanR000c875b9e344b4
diff --git a/c/Makefile b/c/Makefileindex 1530e1be..0a957a15 100644--- a/c/Makefile+++ b/c/Makefile@@ -821,7 +821,7 @@ endifendif.build-config: ;-colibri$(EXE): colibri.c sse41_kernels.h exact_dot.h oracle.h pin_pool.h cli_args.h st.h uring.h json.h tok.h tok_unicode.h tok_unicode_deepseek.h compat.h grammar.h quant.h fp8_format.h idot.h sample.h kv_persist.h telemetry.h route_trace.h omp_tune.h kv_fp8.h kv_tq.h abl.h backend_cuda.h backend_metal.h backend_vulkan.h backend_xdna.h decode_batch.h edge_adapters.h edge_runtime.h edge_tok_internal.h evidence_digest.h schema_gbnf.h segment_adapter_internal.h segment_adapters.h segment_runtime.h tier.h $(CUDA_OBJ) $(METAL_OBJ) $(VK_OBJ) $(VK_SPV) $(XDNA_OBJ) .build-config+colibri$(EXE): colibri.c matmul_f32.h sse41_kernels.h exact_dot.h oracle.h pin_pool.h cli_args.h st.h uring.h json.h tok.h tok_unicode.h tok_unicode_deepseek.h compat.h grammar.h quant.h fp8_format.h idot.h sample.h kv_persist.h telemetry.h route_trace.h omp_tune.h kv_fp8.h kv_tq.h abl.h backend_cuda.h backend_metal.h backend_vulkan.h backend_xdna.h decode_batch.h edge_adapters.h edge_runtime.h edge_tok_internal.h evidence_digest.h schema_gbnf.h segment_adapter_internal.h segment_adapters.h segment_runtime.h tier.h $(CUDA_OBJ) $(METAL_OBJ) $(VK_OBJ) $(VK_SPV) $(XDNA_OBJ) .build-config$(CC) $(CFLAGS) colibri.c $(CUDA_OBJ) $(METAL_OBJ) $(VK_OBJ) $(XDNA_OBJ) -o colibri$(EXE) $(LDFLAGS)# Vulkan backend object (plain C + vulkan headers) and its SPIR-V shaders.@@ -1170,7 +1170,7 @@ tests/bench_cuda_resident_batch$(EXE): tests/bench_cuda_resident_batch.cu backen# the GPU sits idle. Same guard #783 put on kimi_k3, which since gaining an# MXFP4 expert path no longer needs it. tests/test_makefile_cuda_scope.py# asserts this shape.-olmoe$(EXE): olmoe.c cli_args.h st.h json.h compat.h sample.h tok.h tok_unicode.h tok_unicode_deepseek.h tok_unicode_o200k.h omp_tune.h kv_prefix.h pin_pool.h route_trace.h serve_codec.h serve_budget.h edge_adapters.h edge_runtime.h edge_tok_internal.h fused_simd.h segment_adapter_internal.h segment_adapters.h segment_runtime.h sse41_kernels.h+olmoe$(EXE): olmoe.c matmul_f32.h cli_args.h st.h json.h compat.h sample.h tok.h tok_unicode.h tok_unicode_deepseek.h tok_unicode_o200k.h omp_tune.h kv_prefix.h pin_pool.h route_trace.h serve_codec.h serve_budget.h edge_adapters.h edge_runtime.h edge_tok_internal.h fused_simd.h segment_adapter_internal.h segment_adapters.h segment_runtime.h sse41_kernels.h$(CC) $(NOCUDA_CFLAGS) olmoe.c -o olmoe$(EXE) $(NOCUDA_LDFLAGS)# Qwen3.6-35B-A3B engine (hybrid Gated Attention + Gated DeltaNet + streaming@@ -1212,7 +1212,7 @@ QWEN36_CFLAGS += -DQWEN36_QPACKendif# Rebuild when CUDA_DLL changes; otherwise an existing CPU-only executable can# be reported as up to date despite selecting the Windows CUDA DLL tier.-qwen36$(EXE): qwen36.c affine_quant.h backend_metal.h qwen36_qpack.h qwen38_vision.h sse41_kernels.h decode_batch.h serve_poll.h cli_args.h qwen36_tier.h expert_ffn.h idot.h st.h json.h compat.h omp_tune.h kv_prefix.h pin_pool.h edge_adapter_internal.h edge_adapters.h edge_runtime.h segment_adapter_internal.h segment_adapters.h segment_runtime.h gsgemv.h qgemv.h $(QWEN36_TIER_SRC) $(QWEN36_QPACK_SRC) $(CUDA_OBJ) $(METAL_OBJ) .build-config+qwen36$(EXE): qwen36.c matmul_f32.h affine_quant.h backend_metal.h qwen36_qpack.h qwen38_vision.h sse41_kernels.h decode_batch.h serve_poll.h cli_args.h qwen36_tier.h expert_ffn.h idot.h st.h json.h compat.h omp_tune.h kv_prefix.h pin_pool.h edge_adapter_internal.h edge_adapters.h edge_runtime.h segment_adapter_internal.h segment_adapters.h segment_runtime.h gsgemv.h qgemv.h $(QWEN36_TIER_SRC) $(QWEN36_QPACK_SRC) $(CUDA_OBJ) $(METAL_OBJ) .build-config$(CC) $(QWEN36_CFLAGS) qwen36.c $(QWEN36_TIER_SRC) $(QWEN36_QPACK_SRC) $(CUDA_OBJ) $(METAL_OBJ) -o qwen36$(EXE) $(QWEN36_LDFLAGS)# Qwen-Image-2.1 text-to-image (docs/qwen-image.md). It reads the diffusers@@ -1232,7 +1232,7 @@ tests/test_qi_gemm$(EXE): tests/test_qi_gemm.c qi_gemm.h# DeepSeek V4.1 Flash: one file, like every other portable engine. The fp4 experts# stream from the official checkpoint and quant.h's mxfp4 kernel reads them as they# are, so there is no conversion target here to go with it.-deepseek_v41$(EXE): deepseek_v41.c sse41_kernels.h cli_args.h st.h json.h tok.h tok_unicode.h tok_unicode_deepseek.h compat.h omp_tune.h kv_prefix.h pin_pool.h quant.h fp8_format.h idot.h \+deepseek_v41$(EXE): deepseek_v41.c matmul_f32.h sse41_kernels.h cli_args.h st.h json.h tok.h tok_unicode.h tok_unicode_deepseek.h compat.h omp_tune.h kv_prefix.h pin_pool.h quant.h fp8_format.h idot.h \sparse_attn.h hyper_connections.h serve_codec.h serve_poll.h .build-config$(CC) $(CFLAGS) deepseek_v41.c -o deepseek_v41$(EXE) $(LDFLAGS)@@ -1240,7 +1240,7 @@ deepseek_v41$(EXE): deepseek_v41.c sse41_kernels.h cli_args.h st.h json.h tok.h# loader and SERVE=1 protocol. With CUDA=1 it links the same expert tier as# qwen36 (fp8 streaming mode: hot experts get VRAM copies, the RAM LRU stays);# without it the tier header's inline stubs keep the build toolkit-free.-qwen38$(EXE): qwen38.c sse41_kernels.h pin_pool.h cli_args.h qwen38_core.h kv_prefix.h qwen38_nfc.h qwen38_nfc_tables.h st.h json.h compat.h omp_tune.h quant.h fp8_format.h idot.h route_trace.h tok.h tok_unicode.h tok_unicode_deepseek.h tok_unicode_o200k.h serve_codec.h edge_adapter_internal.h edge_adapters.h edge_runtime.h qwen38_vision.h segment_adapter_internal.h segment_adapters.h segment_runtime.h qwen36_tier.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)+qwen38$(EXE): qwen38.c matmul_f32.h sse41_kernels.h pin_pool.h cli_args.h qwen38_core.h kv_prefix.h qwen38_nfc.h qwen38_nfc_tables.h st.h json.h compat.h omp_tune.h quant.h fp8_format.h idot.h route_trace.h tok.h tok_unicode.h tok_unicode_deepseek.h tok_unicode_o200k.h serve_codec.h edge_adapter_internal.h edge_adapters.h edge_runtime.h qwen38_vision.h segment_adapter_internal.h segment_adapters.h segment_runtime.h qwen36_tier.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)$(CC) $(QWEN36_CFLAGS) qwen38.c $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o qwen38$(EXE) $(QWEN36_LDFLAGS).PHONY: qwen38-tiny-generate qwen38-tiny-check@@ -1354,7 +1354,7 @@ tests/test_qwen36_tokenizer$(EXE): tests/test_qwen36_tokenizer.c gsgemv.h qgemv.tests/bench_qwen36_dense_batch$(EXE): tests/bench_qwen36_dense_batch.c gsgemv.h qgemv.h sse41_kernels.h qwen36.c qwen38_vision.h qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)$(CC) $(QWEN36_CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(QWEN36_LDFLAGS)-inkling$(EXE): inkling.c cli_args.h st.h json.h tok.h tok_unicode.h tok_unicode_deepseek.h tok_unicode_o200k.h compat.h omp_tune.h route_trace.h kv_prefix.h pin_pool.h serve_codec.h serve_budget.h backend_cuda_ink.h backend_metal.h edge_adapters.h edge_runtime.h edge_tok_internal.h segment_adapter_internal.h segment_adapters.h segment_runtime.h $(INK_CUDA_OBJ) $(METAL_OBJ)+inkling$(EXE): inkling.c matmul_f32.h cli_args.h st.h json.h tok.h tok_unicode.h tok_unicode_deepseek.h tok_unicode_o200k.h compat.h omp_tune.h route_trace.h kv_prefix.h pin_pool.h serve_codec.h serve_budget.h backend_cuda_ink.h backend_metal.h edge_adapters.h edge_runtime.h edge_tok_internal.h segment_adapter_internal.h segment_adapters.h segment_runtime.h $(INK_CUDA_OBJ) $(METAL_OBJ)$(CC) $(CFLAGS) inkling.c $(INK_CUDA_OBJ) $(METAL_OBJ) -o inkling$(EXE) $(LDFLAGS)# ENGINES WITHOUT A CUDA BACKEND (#783).@@ -1371,10 +1371,10 @@ NOCUDA_LDFLAGS = $(filter-out -lcudart -lstdc++ -lcuda -L$(CUDA_HOME)/lib64 \# GLM-5.3-Flash: routed experts stream from the int4-gs64 container.# METAL=1 accelerates resident matrices and routed MoE; CPU remains fallback.-glm53$(EXE): glm53.c sse41_kernels.h decode_batch.h pin_pool.h cli_args.h st.h json.h stop_ids.h tok.h tok_unicode.h tok_unicode_deepseek.h tok_unicode_o200k.h compat.h omp_tune.h serve_poll.h route_trace.h quant.h fp8_format.h idot.h hyper_connections.h delta_attention.h sparse_index.h vision_tower.h backend_metal.h backend_vulkan.h edge_adapter_internal.h edge_adapters.h edge_runtime.h segment_adapter_internal.h segment_adapters.h segment_runtime.h $(METAL_OBJ) $(VK_OBJ) $(VK_SPV)+glm53$(EXE): glm53.c matmul_f32.h sse41_kernels.h decode_batch.h pin_pool.h cli_args.h st.h json.h stop_ids.h tok.h tok_unicode.h tok_unicode_deepseek.h tok_unicode_o200k.h compat.h omp_tune.h serve_poll.h route_trace.h quant.h fp8_format.h idot.h hyper_connections.h delta_attention.h sparse_index.h vision_tower.h backend_metal.h backend_vulkan.h edge_adapter_internal.h edge_adapters.h edge_runtime.h segment_adapter_internal.h segment_adapters.h segment_runtime.h $(METAL_OBJ) $(VK_OBJ) $(VK_SPV)$(CC) $(CFLAGS) glm53.c $(METAL_OBJ) $(VK_OBJ) -o glm53$(EXE) $(LDFLAGS)-kimi_k3$(EXE): kimi_k3.c sse41_kernels.h serve_budget.h cli_args.h st.h json.h tok.h tok_unicode.h tok_unicode_deepseek.h tok_unicode_o200k.h compat.h quant.h fp8_format.h idot.h omp_tune.h route_trace.h kv_prefix.h pin_pool.h serve_codec.h backend_cuda.h backend_metal.h backend_vulkan.h edge_adapters.h edge_runtime.h edge_tok_internal.h hybrid_split.h segment_adapter_internal.h segment_adapters.h segment_runtime.h $(CUDA_OBJ) $(VK_OBJ) $(VK_SPV) $(METAL_OBJ)+kimi_k3$(EXE): kimi_k3.c matmul_f32.h sse41_kernels.h serve_budget.h cli_args.h st.h json.h tok.h tok_unicode.h tok_unicode_deepseek.h tok_unicode_o200k.h compat.h quant.h fp8_format.h idot.h omp_tune.h route_trace.h kv_prefix.h pin_pool.h serve_codec.h backend_cuda.h backend_metal.h backend_vulkan.h edge_adapters.h edge_runtime.h edge_tok_internal.h hybrid_split.h segment_adapter_internal.h segment_adapters.h segment_runtime.h $(CUDA_OBJ) $(VK_OBJ) $(VK_SPV) $(METAL_OBJ)$(CC) $(CFLAGS) kimi_k3.c $(CUDA_OBJ) $(VK_OBJ) $(METAL_OBJ) -o kimi_k3$(EXE) $(LDFLAGS)# Use a baseline that matches the compiler target. macOS already targets a@@ -2282,10 +2282,10 @@ tests/bench_gemv_stream$(EXE): tests/bench_gemv_stream.c sse41_kernels.h colibri# bench_matmul_f32: microbenchmark (pre-#442 scalar vs row-blocked fmaf f32 matmul at# router shapes), NOT a test gate. Build on demand: make tests/bench_matmul_f32-tests/bench_matmul_f32$(EXE): tests/bench_matmul_f32.c quant.h idot.h sse41_kernels.h+tests/bench_matmul_f32$(EXE): tests/bench_matmul_f32.c quant.h matmul_f32.h idot.h sse41_kernels.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)-tests/test_matmul_f32$(EXE): tests/test_matmul_f32.c quant.h idot.h sse41_kernels.h+tests/test_matmul_f32$(EXE): tests/test_matmul_f32.c quant.h matmul_f32.h idot.h sse41_kernels.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)# bench_mla_simd: microbenchmark (scalar vs AVX2/NEON MLA-absorb reductions, #442),diff --git a/c/inkling.c b/c/inkling.cindex c7ce91ff..2b142a36 100644--- a/c/inkling.c+++ b/c/inkling.c@@ -187,19 +187,7 @@ static float siluf(float x) { return x / (1.f + expf(-x)); }* cumulato p. 0 = spento (default): tutti i topk, calcolo invariato. */static float g_topp = 0.f;-/* y[S,O] = x[S,I] @ W^T, W row-major [O,I] */-static void matmul(float *y, const float *x, const float *W, int S, int I, int O) {- #pragma omp parallel for schedule(static)- for (int o = 0; o < O; o++) {- const float *w = W + (int64_t)o * I;- for (int s = 0; s < S; s++) {- const float *xs = x + (int64_t)s * I;- float acc = 0.f;- for (int i = 0; i < I; i++) acc += xs[i] * w[i];- y[(int64_t)s * O + o] = acc;- }- }-}+#include "matmul_f32.h" /* y[S,O] = x[S,I] @ W^T, W [O,I] f32 row-major */#if defined(__AVX512BF16__) && defined(__AVX512F__)#include <immintrin.h>diff --git a/c/matmul_f32.h b/c/matmul_f32.hnew file mode 100644index 00000000..e4bcdc34--- /dev/null+++ b/c/matmul_f32.h@@ -0,0 +1,50 @@+/* matmul_f32.h — y[S,O] = x[S,I] @ W^T with W[O,I] f32 (header-only, static).+ * One kernel for every engine that keeps f32 matrices: quant.h (colibri, glm53,+ * kimi_k3, deepseek_v4/v41, qwen38) and olmoe, inkling and qwen36, which used+ * to carry their own copy of the loop (#442). */+#ifndef COLI_MATMUL_F32_H+#define COLI_MATMUL_F32_H++#include <math.h>+#include <stdint.h>++/* Each dot product runs in MATMUL_F32_LANES fixed lanes: lane j accumulates+ * the elements i == j (mod lanes) in order, then the lanes are summed in a+ * fixed halving tree. Every lane is its own chain, so the compiler vectorizes+ * the loop without reassociating anything, and the result does not depend on+ * the vector width: NEON runs 8 chains of 4, AVX2 4 of 8, AVX-512 2 of 16, and+ * all of them produce the same bits.+ *+ * The one-chain loops this replaces were either scalar (`a+=x*w`, which has+ * toolchain-dependent bits: gcc and Apple clang keep the adds unfused, x86+ * clang fuses them) or `#pragma omp simd reduction`, which lets the compiler+ * pick the order, so the bits change with the target. */+#define MATMUL_F32_LANES 32++#if defined(__FMA__) || defined(__ARM_FEATURE_FMA)+#define MATMUL_F32_MADD(a, x, w) fmaf((x), (w), (a))+#else+/* No hardware FMA: fmaf would be a libm call. The lanes still fix the order. */+#define MATMUL_F32_MADD(a, x, w) ((a) + (x) * (w))+#endif++static inline float dot_f32_lanes(const float *x, const float *w, int I){+ float a[MATMUL_F32_LANES] = {0};+ int i = 0;+ for (; i + MATMUL_F32_LANES <= I; i += MATMUL_F32_LANES)+ for (int j = 0; j < MATMUL_F32_LANES; j++) a[j] = MATMUL_F32_MADD(a[j], x[i+j], w[i+j]);+ for (int j = 0; i + j < I; j++) a[j] = MATMUL_F32_MADD(a[j], x[i+j], w[i+j]);+ for (int h = MATMUL_F32_LANES/2; h > 0; h >>= 1)+ for (int j = 0; j < h; j++) a[j] += a[j+h];+ return a[0];+}++static void matmul(float *y, const float *x, const float *W, int S, int I, int O){+ #pragma omp parallel for schedule(static)+ for (int o = 0; o < O; o++){+ const float *w = W + (int64_t)o * I;+ for (int s = 0; s < S; s++) y[(int64_t)s * O + o] = dot_f32_lanes(x + (int64_t)s * I, w, I);+ }+}++#endifdiff --git a/c/olmoe.c b/c/olmoe.cindex d56b606d..72adefee 100644--- a/c/olmoe.c+++ b/c/olmoe.c@@ -283,20 +283,7 @@ static float g_temp = 0.7f; /* TEMP env overrides */static float g_nuc = 0.95f; /* NUCLEUS env overrides */#include "sample.h"-/* y[S,O] = x[S,I] @ W^T, W e' [O,I] row-major */-static void matmul(float *y, const float *x, const float *W, int S, int I, int O) {- #pragma omp parallel for schedule(static)- for (int o = 0; o < O; o++) {- const float *w = W + (int64_t)o * I;- for (int s = 0; s < S; s++) {- const float *xs = x + (int64_t)s * I;- float acc = 0.f;- #pragma omp simd reduction(+:acc)- for (int i = 0; i < I; i++) acc += xs[i] * w[i];- y[(int64_t)s * O + o] = acc;- }- }-}+#include "matmul_f32.h" /* y[S,O] = x[S,I] @ W^T, W [O,I] f32 row-major *//* y[1,O] = x[1,I] @ W^T con W quantizzato: q[O,I] int8 + scala per riga.* W[o,i] ~= q[o,i]*scale[o] -> y[o] = scale[o] * sum_i x[i]*q[o,i].diff --git a/c/quant.h b/c/quant.hindex b8028a5d..2284610a 100644--- a/c/quant.h+++ b/c/quant.h@@ -71,33 +71,8 @@ static int i4_acc512_selftest(void){}#endif-/* ---- y[S,O] = x[S,I] @ W^T, W[O,I] f32 ---------------------------------- */-#if defined(__FMA__) || defined(__ARM_FEATURE_FMA)-/* Every output is one fmaf chain over i in order, so the bits are the same on- * any FMA target and at any vector width. The scalar `a+=x*w` form they replace- * had toolchain-dependent bits: gcc and Apple clang vectorize the multiply and- * keep the adds in order (unfused), fusing only the tail; x86 clang fuses all.- * Eight rows per pass give eight independent chains instead of one (#442). */-static void matmul(float *y, const float *x, const float *W, int S, int I, int O){- int O8=O&~7;- #pragma omp parallel for schedule(static)- for (int o=0;o<O8;o+=8){ const float *w0=W+(int64_t)o*I, *w1=w0+I, *w2=w1+I, *w3=w2+I, *w4=w3+I, *w5=w4+I, *w6=w5+I, *w7=w6+I;- for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a0=0,a1=0,a2=0,a3=0,a4=0,a5=0,a6=0,a7=0;- for(int i=0;i<I;i++){ float xi=xs[i];- a0=fmaf(xi,w0[i],a0); a1=fmaf(xi,w1[i],a1); a2=fmaf(xi,w2[i],a2); a3=fmaf(xi,w3[i],a3);- a4=fmaf(xi,w4[i],a4); a5=fmaf(xi,w5[i],a5); a6=fmaf(xi,w6[i],a6); a7=fmaf(xi,w7[i],a7); }- float *ys=y+(int64_t)s*O+o; ys[0]=a0; ys[1]=a1; ys[2]=a2; ys[3]=a3; ys[4]=a4; ys[5]=a5; ys[6]=a6; ys[7]=a7; } }- for (int o=O8;o<O;o++){ const float *w=W+(int64_t)o*I;- for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; for(int i=0;i<I;i++) a=fmaf(xs[i],w[i],a); y[(int64_t)s*O+o]=a; } }-}-#else-/* No hardware FMA: fmaf would be a libm call, so keep the plain loop. */-static void matmul(float *y, const float *x, const float *W, int S, int I, int O){- #pragma omp parallel for schedule(static)- for (int o=0;o<O;o++){ const float *w=W+(int64_t)o*I;- for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; for(int i=0;i<I;i++) a+=xs[i]*w[i]; y[(int64_t)s*O+o]=a; } }-}-#endif+/* ---- y[S,O] = x[S,I] @ W^T, W[O,I] f32: matmul_f32.h ------------------- */+#include "matmul_f32.h"/* ---- y[S,O] = x[S,I] @ W^T, W int8 per-row + scale[O] ------------------- */static void matmul_q(float *y, const float *x, const int8_t *q, const float *scale, int S, int I, int O){diff --git a/c/qwen36.c b/c/qwen36.cindex 6d2b27a7..b9c14200 100644--- a/c/qwen36.c+++ b/c/qwen36.c@@ -974,19 +974,7 @@ static void tm_report(void){}static float *falloc(int64_t n) { float *p = malloc(n*sizeof(float)); if(!p){fprintf(stderr,"OOM %ld\n",(long)n);exit(1);} return p; }-/* y[S,O] = x[S,I] @ W^T, W is [O,I] row-major */-static void matmul(float *y, const float *x, const float *W, int S, int I, int O) {- #pragma omp parallel for schedule(static)- for (int o = 0; o < O; o++) {- const float *w = W + (int64_t)o * I;- for (int s = 0; s < S; s++) {- const float *xs = x + (int64_t)s * I;- float acc = 0.f;- for (int i = 0; i < I; i++) acc += xs[i] * w[i];- y[(int64_t)s * O + o] = acc;- }- }-}+#include "matmul_f32.h" /* y[S,O] = x[S,I] @ W^T, W [O,I] f32 row-major *//* y[1,O] = x[1,I] @ W^T with W quantized: q[O,I] int8 + scale per row.* matmul_q lives in qgemv.h so tests/test_qgemv.c can link the exact kerneldiff --git a/c/tests/bench_matmul_f32.c b/c/tests/bench_matmul_f32.cindex 241275c4..43da5357 100644--- a/c/tests/bench_matmul_f32.c+++ b/c/tests/bench_matmul_f32.c@@ -1,6 +1,10 @@-/* Microbenchmark: the pre-#442 scalar f32 matmul vs quant.h's fmaf row-blocked- * matmul, at the GLM-5.2 router shape (colibri.c, D=6144 E=256) and a smaller- * router (D=2048 E=64), for decode (S=1) and a prefill chunk (S=16).+/* Microbenchmark: matmul_f32.h's lane kernel against the f32 loops it replaced,+ * at the shapes the engines run it on: routers (GLM-5.2 in colibri.c, Inkling),+ * Qwen3.6's DeltaNet a/b projections and OLMoE's attention and lm_head.+ * scalar the plain `a+=x*w` loop (quant.h before #442, inkling, qwen36)+ * ompsimd the same loop under `#pragma omp simd reduction` (olmoe)+ * fmaf8 eight rows of one fmaf chain each (quant.h after #1790)+ * lanes matmul_f32.h** NOT a unit test: tests/test_matmul_f32 gates exactness. Each arm runs* SAMPLES timed samples (each the mean of enough calls to last ~50 ms) after@@ -18,12 +22,31 @@#define SAMPLES 5-/* ---- OLD: verbatim pre-#442 quant.h matmul ---- */-static void matmul_old(float *y, const float *x, const float *W, int S, int I, int O){+static void matmul_scalar(float *y, const float *x, const float *W, int S, int I, int O){#pragma omp parallel for schedule(static)for (int o=0;o<O;o++){ const float *w=W+(int64_t)o*I;for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; for(int i=0;i<I;i++) a+=xs[i]*w[i]; y[(int64_t)s*O+o]=a; } }}+static void matmul_ompsimd(float *y, const float *x, const float *W, int S, int I, int O){+ #pragma omp parallel for schedule(static)+ for (int o=0;o<O;o++){ const float *w=W+(int64_t)o*I;+ for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0;+ #pragma omp simd reduction(+:a)+ for(int i=0;i<I;i++) a+=xs[i]*w[i];+ y[(int64_t)s*O+o]=a; } }+}+static void matmul_fmaf8(float *y, const float *x, const float *W, int S, int I, int O){+ int O8=O&~7;+ #pragma omp parallel for schedule(static)+ for (int o=0;o<O8;o+=8){ const float *w0=W+(int64_t)o*I, *w1=w0+I, *w2=w1+I, *w3=w2+I, *w4=w3+I, *w5=w4+I, *w6=w5+I, *w7=w6+I;+ for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a0=0,a1=0,a2=0,a3=0,a4=0,a5=0,a6=0,a7=0;+ for(int i=0;i<I;i++){ float xi=xs[i];+ a0=fmaf(xi,w0[i],a0); a1=fmaf(xi,w1[i],a1); a2=fmaf(xi,w2[i],a2); a3=fmaf(xi,w3[i],a3);+ a4=fmaf(xi,w4[i],a4); a5=fmaf(xi,w5[i],a5); a6=fmaf(xi,w6[i],a6); a7=fmaf(xi,w7[i],a7); }+ float *ys=y+(int64_t)s*O+o; ys[0]=a0; ys[1]=a1; ys[2]=a2; ys[3]=a3; ys[4]=a4; ys[5]=a5; ys[6]=a6; ys[7]=a7; } }+ for (int o=O8;o<O;o++){ const float *w=W+(int64_t)o*I;+ for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; for(int i=0;i<I;i++) a=fmaf(xs[i],w[i],a); y[(int64_t)s*O+o]=a; } }+}static double now_s(void){ struct timespec t; clock_gettime(CLOCK_MONOTONIC,&t); return t.tv_sec+t.tv_nsec*1e-9; }static int cmp_d(const void *a, const void *b){ double x=*(const double*)a, y=*(const double*)b; return (x>y)-(x<y); }@@ -36,28 +59,38 @@ static double sample_us(mm_fn f, float *y, const float *x, const float *W, int Sint main(void){static const struct { const char *name; int S, I, O; } shapes[]={- {"GLM-5.2 router decode", 1, 6144, 256}, {"GLM-5.2 router prefill", 16, 6144, 256},- {"small router decode", 1, 2048, 64}, {"small router prefill", 16, 2048, 64} };+ {"GLM-5.2 router decode", 1, 6144, 256}, {"GLM-5.2 router prefill", 16, 6144, 256},+ {"Inkling router decode", 1, 6144, 258}, {"Qwen3.6 DeltaNet a/b", 1, 2048, 32},+ {"OLMoE router decode", 1, 2048, 64}, {"OLMoE q/k/v/o decode", 1, 2048, 2048},+ {"OLMoE q/k/v/o prefill", 16, 2048, 2048}, {"OLMoE lm_head decode", 1, 2048, 50304} };+ static const mm_fn arm[]={ matmul_scalar, matmul_ompsimd, matmul_fmaf8, matmul };+ static const char *arm_name[]={ "scalar", "ompsimd", "fmaf8", "lanes" };+ enum { ARMS=4 };uint64_t rng=0x9E3779B97F4A7C15ull;#ifdef _OPENMPprintf("threads: %d\n", omp_get_max_threads());#elseprintf("threads: 1 (no OpenMP)\n");#endif+ printf("%-24s %-22s", "shape", "");+ for (int a=0;a<ARMS;a++) printf(" %9s", arm_name[a]);+ printf(" median us/call; lanes vs each\n");for (size_t k=0;k<sizeof shapes/sizeof shapes[0];k++){int S=shapes[k].S, I=shapes[k].I, O=shapes[k].O;float *x=malloc(sizeof(float)*(size_t)S*I), *W=malloc(sizeof(float)*(size_t)I*O), *y=malloc(sizeof(float)*(size_t)S*O);for (size_t i=0;i<(size_t)S*I;i++){ rng^=rng<<13; rng^=rng>>7; rng^=rng<<17; x[i]=(float)(rng>>40)/(float)(1<<24)*2.f-1.f; }for (size_t i=0;i<(size_t)I*O;i++){ rng^=rng<<13; rng^=rng>>7; rng^=rng<<17; W[i]=((float)(rng>>40)/(float)(1<<24)*2.f-1.f)*0.05f; }- int reps=1; while (sample_us(matmul_old,y,x,W,S,I,O,reps)*reps<50e3 && reps<1<<20) reps*=2;- sample_us(matmul_old,y,x,W,S,I,O,reps); sample_us(matmul,y,x,W,S,I,O,reps); /* warm-up */- double to[SAMPLES], tn[SAMPLES];- for (int r=0;r<SAMPLES;r++){ to[r]=sample_us(matmul_old,y,x,W,S,I,O,reps); tn[r]=sample_us(matmul,y,x,W,S,I,O,reps); }- printf("%-24s S=%-2d I=%d O=%d old:", shapes[k].name, S, I, O);- for (int r=0;r<SAMPLES;r++) printf(" %.2f", to[r]);- printf(" new:"); for (int r=0;r<SAMPLES;r++) printf(" %.2f", tn[r]);- qsort(to,SAMPLES,sizeof(double),cmp_d); qsort(tn,SAMPLES,sizeof(double),cmp_d);- printf(" us/call; median %.2f -> %.2f (%.2fx)\n", to[SAMPLES/2], tn[SAMPLES/2], to[SAMPLES/2]/tn[SAMPLES/2]);+ int reps=1; while (sample_us(matmul_scalar,y,x,W,S,I,O,reps)*reps<50e3 && reps<1<<20) reps*=2;+ for (int a=0;a<ARMS;a++) sample_us(arm[a],y,x,W,S,I,O,reps); /* warm-up */+ double t[ARMS][SAMPLES];+ for (int r=0;r<SAMPLES;r++) for (int a=0;a<ARMS;a++) t[a][r]=sample_us(arm[a],y,x,W,S,I,O,reps);+ for (int a=0;a<ARMS;a++) qsort(t[a],SAMPLES,sizeof(double),cmp_d);+ double lanes=t[ARMS-1][SAMPLES/2];+ printf("%-24s S=%-2d I=%-4d O=%-6d ", shapes[k].name, S, I, O);+ for (int a=0;a<ARMS;a++) printf(" %9.2f", t[a][SAMPLES/2]);+ printf(" ");+ for (int a=0;a<ARMS-1;a++) printf(" %.2fx", t[a][SAMPLES/2]/lanes);+ printf("\n");free(x); free(W); free(y);}return 0;diff --git a/c/tests/test_matmul_f32.c b/c/tests/test_matmul_f32.cindex 77fbd48b..3e8c3649 100644--- a/c/tests/test_matmul_f32.c+++ b/c/tests/test_matmul_f32.c@@ -1,10 +1,11 @@/* f32 matmul exactness gate (#442).*- * On FMA targets quant.h's matmul computes each output as one fmaf chain over- * i in order, eight rows at a time. The row blocking must not change a single- * bit: this compares it with memcmp against the one-row fmaf chain, over shapes- * that exercise the eight-row blocks, the leftover rows and short rows. Without- * FMA matmul keeps the plain `a+=x*w` loop, and the reference follows it.+ * matmul_f32.h computes each output in MATMUL_F32_LANES fixed lanes (lane j+ * takes i == j mod lanes, in order) and sums the lanes in a halving tree. The+ * vectorized loop must give exactly the bits of that definition written out+ * one element at a time: this compares them with memcmp over shapes that+ * exercise full lane blocks, a partial last block and rows shorter than one+ * block. With FMA every lane step is an fmaf, without it `a+x*w`.** Build: make -C c tests/test_matmul_f32 */#include <stdio.h>@@ -19,17 +20,20 @@ static float rnd_f(void){ rng^=rng<<13; rng^=rng>>7; rng^=rng<<17; return (floatstatic void ref_matmul(float *y, const float *x, const float *W, int S, int I, int O){for (int o=0;o<O;o++){ const float *w=W+(int64_t)o*I;- for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0;+ for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; volatile float a[MATMUL_F32_LANES]={0};+ for (int i=0;i<I;i++){ int j=i%MATMUL_F32_LANES;#if defined(__FMA__) || defined(__ARM_FEATURE_FMA)- for(int i=0;i<I;i++) a=fmaf(xs[i],w[i],a);+ a[j]=fmaf(xs[i],w[i],a[j]);#else- for(int i=0;i<I;i++) a+=xs[i]*w[i];+ a[j]=a[j]+xs[i]*w[i];#endif- y[(int64_t)s*O+o]=a; } }+ }+ for (int h=MATMUL_F32_LANES/2;h>0;h>>=1) for (int j=0;j<h;j++) a[j]=a[j]+a[j+h];+ y[(int64_t)s*O+o]=a[0]; } }}int main(void){- static const int shapes[][3]={ {1,1,1}, {1,5,8}, {1,7,13}, {3,33,17}, {2,1027,41},+ static const int shapes[][3]={ {1,1,1}, {1,5,8}, {1,7,13}, {1,31,3}, {1,32,3}, {3,33,17}, {2,1027,41}, {1,2048,32},{1,2048,64}, {4,2048,64}, {5,6144,256}, {1,6144,257} };int fails=0;for (size_t k=0;k<sizeof shapes/sizeof shapes[0];k++){