Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

matmul_f32.h: one vectorized f32 matmul for quant.h, olmoe, inkling and qwen36 (#442)

RayanR000c875b9e344b4
diff --git a/c/Makefile b/c/Makefile
index 1530e1be..0a957a15 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -821,7 +821,7 @@ endif
endif
.build-config: ;
-colibri$(EXE): colibri.c sse41_kernels.h exact_dot.h oracle.h pin_pool.h cli_args.h st.h uring.h json.h tok.h tok_unicode.h tok_unicode_deepseek.h compat.h grammar.h quant.h fp8_format.h idot.h sample.h kv_persist.h telemetry.h route_trace.h omp_tune.h kv_fp8.h kv_tq.h abl.h backend_cuda.h backend_metal.h backend_vulkan.h backend_xdna.h decode_batch.h edge_adapters.h edge_runtime.h edge_tok_internal.h evidence_digest.h schema_gbnf.h segment_adapter_internal.h segment_adapters.h segment_runtime.h tier.h $(CUDA_OBJ) $(METAL_OBJ) $(VK_OBJ) $(VK_SPV) $(XDNA_OBJ) .build-config
+colibri$(EXE): colibri.c matmul_f32.h sse41_kernels.h exact_dot.h oracle.h pin_pool.h cli_args.h st.h uring.h json.h tok.h tok_unicode.h tok_unicode_deepseek.h compat.h grammar.h quant.h fp8_format.h idot.h sample.h kv_persist.h telemetry.h route_trace.h omp_tune.h kv_fp8.h kv_tq.h abl.h backend_cuda.h backend_metal.h backend_vulkan.h backend_xdna.h decode_batch.h edge_adapters.h edge_runtime.h edge_tok_internal.h evidence_digest.h schema_gbnf.h segment_adapter_internal.h segment_adapters.h segment_runtime.h tier.h $(CUDA_OBJ) $(METAL_OBJ) $(VK_OBJ) $(VK_SPV) $(XDNA_OBJ) .build-config
$(CC) $(CFLAGS) colibri.c $(CUDA_OBJ) $(METAL_OBJ) $(VK_OBJ) $(XDNA_OBJ) -o colibri$(EXE) $(LDFLAGS)
# Vulkan backend object (plain C + vulkan headers) and its SPIR-V shaders.
@@ -1170,7 +1170,7 @@ tests/bench_cuda_resident_batch$(EXE): tests/bench_cuda_resident_batch.cu backen
# the GPU sits idle. Same guard #783 put on kimi_k3, which since gaining an
# MXFP4 expert path no longer needs it. tests/test_makefile_cuda_scope.py
# asserts this shape.
-olmoe$(EXE): olmoe.c cli_args.h st.h json.h compat.h sample.h tok.h tok_unicode.h tok_unicode_deepseek.h tok_unicode_o200k.h omp_tune.h kv_prefix.h pin_pool.h route_trace.h serve_codec.h serve_budget.h edge_adapters.h edge_runtime.h edge_tok_internal.h fused_simd.h segment_adapter_internal.h segment_adapters.h segment_runtime.h sse41_kernels.h
+olmoe$(EXE): olmoe.c matmul_f32.h cli_args.h st.h json.h compat.h sample.h tok.h tok_unicode.h tok_unicode_deepseek.h tok_unicode_o200k.h omp_tune.h kv_prefix.h pin_pool.h route_trace.h serve_codec.h serve_budget.h edge_adapters.h edge_runtime.h edge_tok_internal.h fused_simd.h segment_adapter_internal.h segment_adapters.h segment_runtime.h sse41_kernels.h
$(CC) $(NOCUDA_CFLAGS) olmoe.c -o olmoe$(EXE) $(NOCUDA_LDFLAGS)
# Qwen3.6-35B-A3B engine (hybrid Gated Attention + Gated DeltaNet + streaming
@@ -1212,7 +1212,7 @@ QWEN36_CFLAGS += -DQWEN36_QPACK
endif
# Rebuild when CUDA_DLL changes; otherwise an existing CPU-only executable can
# be reported as up to date despite selecting the Windows CUDA DLL tier.
-qwen36$(EXE): qwen36.c affine_quant.h backend_metal.h qwen36_qpack.h qwen38_vision.h sse41_kernels.h decode_batch.h serve_poll.h cli_args.h qwen36_tier.h expert_ffn.h idot.h st.h json.h compat.h omp_tune.h kv_prefix.h pin_pool.h edge_adapter_internal.h edge_adapters.h edge_runtime.h segment_adapter_internal.h segment_adapters.h segment_runtime.h gsgemv.h qgemv.h $(QWEN36_TIER_SRC) $(QWEN36_QPACK_SRC) $(CUDA_OBJ) $(METAL_OBJ) .build-config
+qwen36$(EXE): qwen36.c matmul_f32.h affine_quant.h backend_metal.h qwen36_qpack.h qwen38_vision.h sse41_kernels.h decode_batch.h serve_poll.h cli_args.h qwen36_tier.h expert_ffn.h idot.h st.h json.h compat.h omp_tune.h kv_prefix.h pin_pool.h edge_adapter_internal.h edge_adapters.h edge_runtime.h segment_adapter_internal.h segment_adapters.h segment_runtime.h gsgemv.h qgemv.h $(QWEN36_TIER_SRC) $(QWEN36_QPACK_SRC) $(CUDA_OBJ) $(METAL_OBJ) .build-config
$(CC) $(QWEN36_CFLAGS) qwen36.c $(QWEN36_TIER_SRC) $(QWEN36_QPACK_SRC) $(CUDA_OBJ) $(METAL_OBJ) -o qwen36$(EXE) $(QWEN36_LDFLAGS)
# Qwen-Image-2.1 text-to-image (docs/qwen-image.md). It reads the diffusers
@@ -1232,7 +1232,7 @@ tests/test_qi_gemm$(EXE): tests/test_qi_gemm.c qi_gemm.h
# DeepSeek V4.1 Flash: one file, like every other portable engine. The fp4 experts
# stream from the official checkpoint and quant.h's mxfp4 kernel reads them as they
# are, so there is no conversion target here to go with it.
-deepseek_v41$(EXE): deepseek_v41.c sse41_kernels.h cli_args.h st.h json.h tok.h tok_unicode.h tok_unicode_deepseek.h compat.h omp_tune.h kv_prefix.h pin_pool.h quant.h fp8_format.h idot.h \
+deepseek_v41$(EXE): deepseek_v41.c matmul_f32.h sse41_kernels.h cli_args.h st.h json.h tok.h tok_unicode.h tok_unicode_deepseek.h compat.h omp_tune.h kv_prefix.h pin_pool.h quant.h fp8_format.h idot.h \
sparse_attn.h hyper_connections.h serve_codec.h serve_poll.h .build-config
$(CC) $(CFLAGS) deepseek_v41.c -o deepseek_v41$(EXE) $(LDFLAGS)
@@ -1240,7 +1240,7 @@ deepseek_v41$(EXE): deepseek_v41.c sse41_kernels.h cli_args.h st.h json.h tok.h
# loader and SERVE=1 protocol. With CUDA=1 it links the same expert tier as
# qwen36 (fp8 streaming mode: hot experts get VRAM copies, the RAM LRU stays);
# without it the tier header's inline stubs keep the build toolkit-free.
-qwen38$(EXE): qwen38.c sse41_kernels.h pin_pool.h cli_args.h qwen38_core.h kv_prefix.h qwen38_nfc.h qwen38_nfc_tables.h st.h json.h compat.h omp_tune.h quant.h fp8_format.h idot.h route_trace.h tok.h tok_unicode.h tok_unicode_deepseek.h tok_unicode_o200k.h serve_codec.h edge_adapter_internal.h edge_adapters.h edge_runtime.h qwen38_vision.h segment_adapter_internal.h segment_adapters.h segment_runtime.h qwen36_tier.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)
+qwen38$(EXE): qwen38.c matmul_f32.h sse41_kernels.h pin_pool.h cli_args.h qwen38_core.h kv_prefix.h qwen38_nfc.h qwen38_nfc_tables.h st.h json.h compat.h omp_tune.h quant.h fp8_format.h idot.h route_trace.h tok.h tok_unicode.h tok_unicode_deepseek.h tok_unicode_o200k.h serve_codec.h edge_adapter_internal.h edge_adapters.h edge_runtime.h qwen38_vision.h segment_adapter_internal.h segment_adapters.h segment_runtime.h qwen36_tier.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)
$(CC) $(QWEN36_CFLAGS) qwen38.c $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o qwen38$(EXE) $(QWEN36_LDFLAGS)
.PHONY: qwen38-tiny-generate qwen38-tiny-check
@@ -1354,7 +1354,7 @@ tests/test_qwen36_tokenizer$(EXE): tests/test_qwen36_tokenizer.c gsgemv.h qgemv.
tests/bench_qwen36_dense_batch$(EXE): tests/bench_qwen36_dense_batch.c gsgemv.h qgemv.h sse41_kernels.h qwen36.c qwen38_vision.h qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)
$(CC) $(QWEN36_CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(QWEN36_LDFLAGS)
-inkling$(EXE): inkling.c cli_args.h st.h json.h tok.h tok_unicode.h tok_unicode_deepseek.h tok_unicode_o200k.h compat.h omp_tune.h route_trace.h kv_prefix.h pin_pool.h serve_codec.h serve_budget.h backend_cuda_ink.h backend_metal.h edge_adapters.h edge_runtime.h edge_tok_internal.h segment_adapter_internal.h segment_adapters.h segment_runtime.h $(INK_CUDA_OBJ) $(METAL_OBJ)
+inkling$(EXE): inkling.c matmul_f32.h cli_args.h st.h json.h tok.h tok_unicode.h tok_unicode_deepseek.h tok_unicode_o200k.h compat.h omp_tune.h route_trace.h kv_prefix.h pin_pool.h serve_codec.h serve_budget.h backend_cuda_ink.h backend_metal.h edge_adapters.h edge_runtime.h edge_tok_internal.h segment_adapter_internal.h segment_adapters.h segment_runtime.h $(INK_CUDA_OBJ) $(METAL_OBJ)
$(CC) $(CFLAGS) inkling.c $(INK_CUDA_OBJ) $(METAL_OBJ) -o inkling$(EXE) $(LDFLAGS)
# ENGINES WITHOUT A CUDA BACKEND (#783).
@@ -1371,10 +1371,10 @@ NOCUDA_LDFLAGS = $(filter-out -lcudart -lstdc++ -lcuda -L$(CUDA_HOME)/lib64 \
# GLM-5.3-Flash: routed experts stream from the int4-gs64 container.
# METAL=1 accelerates resident matrices and routed MoE; CPU remains fallback.
-glm53$(EXE): glm53.c sse41_kernels.h decode_batch.h pin_pool.h cli_args.h st.h json.h stop_ids.h tok.h tok_unicode.h tok_unicode_deepseek.h tok_unicode_o200k.h compat.h omp_tune.h serve_poll.h route_trace.h quant.h fp8_format.h idot.h hyper_connections.h delta_attention.h sparse_index.h vision_tower.h backend_metal.h backend_vulkan.h edge_adapter_internal.h edge_adapters.h edge_runtime.h segment_adapter_internal.h segment_adapters.h segment_runtime.h $(METAL_OBJ) $(VK_OBJ) $(VK_SPV)
+glm53$(EXE): glm53.c matmul_f32.h sse41_kernels.h decode_batch.h pin_pool.h cli_args.h st.h json.h stop_ids.h tok.h tok_unicode.h tok_unicode_deepseek.h tok_unicode_o200k.h compat.h omp_tune.h serve_poll.h route_trace.h quant.h fp8_format.h idot.h hyper_connections.h delta_attention.h sparse_index.h vision_tower.h backend_metal.h backend_vulkan.h edge_adapter_internal.h edge_adapters.h edge_runtime.h segment_adapter_internal.h segment_adapters.h segment_runtime.h $(METAL_OBJ) $(VK_OBJ) $(VK_SPV)
$(CC) $(CFLAGS) glm53.c $(METAL_OBJ) $(VK_OBJ) -o glm53$(EXE) $(LDFLAGS)
-kimi_k3$(EXE): kimi_k3.c sse41_kernels.h serve_budget.h cli_args.h st.h json.h tok.h tok_unicode.h tok_unicode_deepseek.h tok_unicode_o200k.h compat.h quant.h fp8_format.h idot.h omp_tune.h route_trace.h kv_prefix.h pin_pool.h serve_codec.h backend_cuda.h backend_metal.h backend_vulkan.h edge_adapters.h edge_runtime.h edge_tok_internal.h hybrid_split.h segment_adapter_internal.h segment_adapters.h segment_runtime.h $(CUDA_OBJ) $(VK_OBJ) $(VK_SPV) $(METAL_OBJ)
+kimi_k3$(EXE): kimi_k3.c matmul_f32.h sse41_kernels.h serve_budget.h cli_args.h st.h json.h tok.h tok_unicode.h tok_unicode_deepseek.h tok_unicode_o200k.h compat.h quant.h fp8_format.h idot.h omp_tune.h route_trace.h kv_prefix.h pin_pool.h serve_codec.h backend_cuda.h backend_metal.h backend_vulkan.h edge_adapters.h edge_runtime.h edge_tok_internal.h hybrid_split.h segment_adapter_internal.h segment_adapters.h segment_runtime.h $(CUDA_OBJ) $(VK_OBJ) $(VK_SPV) $(METAL_OBJ)
$(CC) $(CFLAGS) kimi_k3.c $(CUDA_OBJ) $(VK_OBJ) $(METAL_OBJ) -o kimi_k3$(EXE) $(LDFLAGS)
# Use a baseline that matches the compiler target. macOS already targets a
@@ -2282,10 +2282,10 @@ tests/bench_gemv_stream$(EXE): tests/bench_gemv_stream.c sse41_kernels.h colibri
# bench_matmul_f32: microbenchmark (pre-#442 scalar vs row-blocked fmaf f32 matmul at
# router shapes), NOT a test gate. Build on demand: make tests/bench_matmul_f32
-tests/bench_matmul_f32$(EXE): tests/bench_matmul_f32.c quant.h idot.h sse41_kernels.h
+tests/bench_matmul_f32$(EXE): tests/bench_matmul_f32.c quant.h matmul_f32.h idot.h sse41_kernels.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
-tests/test_matmul_f32$(EXE): tests/test_matmul_f32.c quant.h idot.h sse41_kernels.h
+tests/test_matmul_f32$(EXE): tests/test_matmul_f32.c quant.h matmul_f32.h idot.h sse41_kernels.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
# bench_mla_simd: microbenchmark (scalar vs AVX2/NEON MLA-absorb reductions, #442),
diff --git a/c/inkling.c b/c/inkling.c
index c7ce91ff..2b142a36 100644
--- a/c/inkling.c
+++ b/c/inkling.c
@@ -187,19 +187,7 @@ static float siluf(float x) { return x / (1.f + expf(-x)); }
* cumulato p. 0 = spento (default): tutti i topk, calcolo invariato. */
static float g_topp = 0.f;
-/* y[S,O] = x[S,I] @ W^T, W row-major [O,I] */
-static void matmul(float *y, const float *x, const float *W, int S, int I, int O) {
- #pragma omp parallel for schedule(static)
- for (int o = 0; o < O; o++) {
- const float *w = W + (int64_t)o * I;
- for (int s = 0; s < S; s++) {
- const float *xs = x + (int64_t)s * I;
- float acc = 0.f;
- for (int i = 0; i < I; i++) acc += xs[i] * w[i];
- y[(int64_t)s * O + o] = acc;
- }
- }
-}
+#include "matmul_f32.h" /* y[S,O] = x[S,I] @ W^T, W [O,I] f32 row-major */
#if defined(__AVX512BF16__) && defined(__AVX512F__)
#include <immintrin.h>
diff --git a/c/matmul_f32.h b/c/matmul_f32.h
new file mode 100644
index 00000000..e4bcdc34
--- /dev/null
+++ b/c/matmul_f32.h
@@ -0,0 +1,50 @@
+/* matmul_f32.h — y[S,O] = x[S,I] @ W^T with W[O,I] f32 (header-only, static).
+ * One kernel for every engine that keeps f32 matrices: quant.h (colibri, glm53,
+ * kimi_k3, deepseek_v4/v41, qwen38) and olmoe, inkling and qwen36, which used
+ * to carry their own copy of the loop (#442). */
+#ifndef COLI_MATMUL_F32_H
+#define COLI_MATMUL_F32_H
+
+#include <math.h>
+#include <stdint.h>
+
+/* Each dot product runs in MATMUL_F32_LANES fixed lanes: lane j accumulates
+ * the elements i == j (mod lanes) in order, then the lanes are summed in a
+ * fixed halving tree. Every lane is its own chain, so the compiler vectorizes
+ * the loop without reassociating anything, and the result does not depend on
+ * the vector width: NEON runs 8 chains of 4, AVX2 4 of 8, AVX-512 2 of 16, and
+ * all of them produce the same bits.
+ *
+ * The one-chain loops this replaces were either scalar (`a+=x*w`, which has
+ * toolchain-dependent bits: gcc and Apple clang keep the adds unfused, x86
+ * clang fuses them) or `#pragma omp simd reduction`, which lets the compiler
+ * pick the order, so the bits change with the target. */
+#define MATMUL_F32_LANES 32
+
+#if defined(__FMA__) || defined(__ARM_FEATURE_FMA)
+#define MATMUL_F32_MADD(a, x, w) fmaf((x), (w), (a))
+#else
+/* No hardware FMA: fmaf would be a libm call. The lanes still fix the order. */
+#define MATMUL_F32_MADD(a, x, w) ((a) + (x) * (w))
+#endif
+
+static inline float dot_f32_lanes(const float *x, const float *w, int I){
+ float a[MATMUL_F32_LANES] = {0};
+ int i = 0;
+ for (; i + MATMUL_F32_LANES <= I; i += MATMUL_F32_LANES)
+ for (int j = 0; j < MATMUL_F32_LANES; j++) a[j] = MATMUL_F32_MADD(a[j], x[i+j], w[i+j]);
+ for (int j = 0; i + j < I; j++) a[j] = MATMUL_F32_MADD(a[j], x[i+j], w[i+j]);
+ for (int h = MATMUL_F32_LANES/2; h > 0; h >>= 1)
+ for (int j = 0; j < h; j++) a[j] += a[j+h];
+ return a[0];
+}
+
+static void matmul(float *y, const float *x, const float *W, int S, int I, int O){
+ #pragma omp parallel for schedule(static)
+ for (int o = 0; o < O; o++){
+ const float *w = W + (int64_t)o * I;
+ for (int s = 0; s < S; s++) y[(int64_t)s * O + o] = dot_f32_lanes(x + (int64_t)s * I, w, I);
+ }
+}
+
+#endif
diff --git a/c/olmoe.c b/c/olmoe.c
index d56b606d..72adefee 100644
--- a/c/olmoe.c
+++ b/c/olmoe.c
@@ -283,20 +283,7 @@ static float g_temp = 0.7f; /* TEMP env overrides */
static float g_nuc = 0.95f; /* NUCLEUS env overrides */
#include "sample.h"
-/* y[S,O] = x[S,I] @ W^T, W e' [O,I] row-major */
-static void matmul(float *y, const float *x, const float *W, int S, int I, int O) {
- #pragma omp parallel for schedule(static)
- for (int o = 0; o < O; o++) {
- const float *w = W + (int64_t)o * I;
- for (int s = 0; s < S; s++) {
- const float *xs = x + (int64_t)s * I;
- float acc = 0.f;
- #pragma omp simd reduction(+:acc)
- for (int i = 0; i < I; i++) acc += xs[i] * w[i];
- y[(int64_t)s * O + o] = acc;
- }
- }
-}
+#include "matmul_f32.h" /* y[S,O] = x[S,I] @ W^T, W [O,I] f32 row-major */
/* y[1,O] = x[1,I] @ W^T con W quantizzato: q[O,I] int8 + scala per riga.
* W[o,i] ~= q[o,i]*scale[o] -> y[o] = scale[o] * sum_i x[i]*q[o,i].
diff --git a/c/quant.h b/c/quant.h
index b8028a5d..2284610a 100644
--- a/c/quant.h
+++ b/c/quant.h
@@ -71,33 +71,8 @@ static int i4_acc512_selftest(void){
}
#endif
-/* ---- y[S,O] = x[S,I] @ W^T, W[O,I] f32 ---------------------------------- */
-#if defined(__FMA__) || defined(__ARM_FEATURE_FMA)
-/* Every output is one fmaf chain over i in order, so the bits are the same on
- * any FMA target and at any vector width. The scalar `a+=x*w` form they replace
- * had toolchain-dependent bits: gcc and Apple clang vectorize the multiply and
- * keep the adds in order (unfused), fusing only the tail; x86 clang fuses all.
- * Eight rows per pass give eight independent chains instead of one (#442). */
-static void matmul(float *y, const float *x, const float *W, int S, int I, int O){
- int O8=O&~7;
- #pragma omp parallel for schedule(static)
- for (int o=0;o<O8;o+=8){ const float *w0=W+(int64_t)o*I, *w1=w0+I, *w2=w1+I, *w3=w2+I, *w4=w3+I, *w5=w4+I, *w6=w5+I, *w7=w6+I;
- for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a0=0,a1=0,a2=0,a3=0,a4=0,a5=0,a6=0,a7=0;
- for(int i=0;i<I;i++){ float xi=xs[i];
- a0=fmaf(xi,w0[i],a0); a1=fmaf(xi,w1[i],a1); a2=fmaf(xi,w2[i],a2); a3=fmaf(xi,w3[i],a3);
- a4=fmaf(xi,w4[i],a4); a5=fmaf(xi,w5[i],a5); a6=fmaf(xi,w6[i],a6); a7=fmaf(xi,w7[i],a7); }
- float *ys=y+(int64_t)s*O+o; ys[0]=a0; ys[1]=a1; ys[2]=a2; ys[3]=a3; ys[4]=a4; ys[5]=a5; ys[6]=a6; ys[7]=a7; } }
- for (int o=O8;o<O;o++){ const float *w=W+(int64_t)o*I;
- for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; for(int i=0;i<I;i++) a=fmaf(xs[i],w[i],a); y[(int64_t)s*O+o]=a; } }
-}
-#else
-/* No hardware FMA: fmaf would be a libm call, so keep the plain loop. */
-static void matmul(float *y, const float *x, const float *W, int S, int I, int O){
- #pragma omp parallel for schedule(static)
- for (int o=0;o<O;o++){ const float *w=W+(int64_t)o*I;
- for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; for(int i=0;i<I;i++) a+=xs[i]*w[i]; y[(int64_t)s*O+o]=a; } }
-}
-#endif
+/* ---- y[S,O] = x[S,I] @ W^T, W[O,I] f32: matmul_f32.h ------------------- */
+#include "matmul_f32.h"
/* ---- y[S,O] = x[S,I] @ W^T, W int8 per-row + scale[O] ------------------- */
static void matmul_q(float *y, const float *x, const int8_t *q, const float *scale, int S, int I, int O){
diff --git a/c/qwen36.c b/c/qwen36.c
index 6d2b27a7..b9c14200 100644
--- a/c/qwen36.c
+++ b/c/qwen36.c
@@ -974,19 +974,7 @@ static void tm_report(void){
}
static float *falloc(int64_t n) { float *p = malloc(n*sizeof(float)); if(!p){fprintf(stderr,"OOM %ld\n",(long)n);exit(1);} return p; }
-/* y[S,O] = x[S,I] @ W^T, W is [O,I] row-major */
-static void matmul(float *y, const float *x, const float *W, int S, int I, int O) {
- #pragma omp parallel for schedule(static)
- for (int o = 0; o < O; o++) {
- const float *w = W + (int64_t)o * I;
- for (int s = 0; s < S; s++) {
- const float *xs = x + (int64_t)s * I;
- float acc = 0.f;
- for (int i = 0; i < I; i++) acc += xs[i] * w[i];
- y[(int64_t)s * O + o] = acc;
- }
- }
-}
+#include "matmul_f32.h" /* y[S,O] = x[S,I] @ W^T, W [O,I] f32 row-major */
/* y[1,O] = x[1,I] @ W^T with W quantized: q[O,I] int8 + scale per row.
* matmul_q lives in qgemv.h so tests/test_qgemv.c can link the exact kernel
diff --git a/c/tests/bench_matmul_f32.c b/c/tests/bench_matmul_f32.c
index 241275c4..43da5357 100644
--- a/c/tests/bench_matmul_f32.c
+++ b/c/tests/bench_matmul_f32.c
@@ -1,6 +1,10 @@
-/* Microbenchmark: the pre-#442 scalar f32 matmul vs quant.h's fmaf row-blocked
- * matmul, at the GLM-5.2 router shape (colibri.c, D=6144 E=256) and a smaller
- * router (D=2048 E=64), for decode (S=1) and a prefill chunk (S=16).
+/* Microbenchmark: matmul_f32.h's lane kernel against the f32 loops it replaced,
+ * at the shapes the engines run it on: routers (GLM-5.2 in colibri.c, Inkling),
+ * Qwen3.6's DeltaNet a/b projections and OLMoE's attention and lm_head.
+ * scalar the plain `a+=x*w` loop (quant.h before #442, inkling, qwen36)
+ * ompsimd the same loop under `#pragma omp simd reduction` (olmoe)
+ * fmaf8 eight rows of one fmaf chain each (quant.h after #1790)
+ * lanes matmul_f32.h
*
* NOT a unit test: tests/test_matmul_f32 gates exactness. Each arm runs
* SAMPLES timed samples (each the mean of enough calls to last ~50 ms) after
@@ -18,12 +22,31 @@
#define SAMPLES 5
-/* ---- OLD: verbatim pre-#442 quant.h matmul ---- */
-static void matmul_old(float *y, const float *x, const float *W, int S, int I, int O){
+static void matmul_scalar(float *y, const float *x, const float *W, int S, int I, int O){
#pragma omp parallel for schedule(static)
for (int o=0;o<O;o++){ const float *w=W+(int64_t)o*I;
for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; for(int i=0;i<I;i++) a+=xs[i]*w[i]; y[(int64_t)s*O+o]=a; } }
}
+static void matmul_ompsimd(float *y, const float *x, const float *W, int S, int I, int O){
+ #pragma omp parallel for schedule(static)
+ for (int o=0;o<O;o++){ const float *w=W+(int64_t)o*I;
+ for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0;
+ #pragma omp simd reduction(+:a)
+ for(int i=0;i<I;i++) a+=xs[i]*w[i];
+ y[(int64_t)s*O+o]=a; } }
+}
+static void matmul_fmaf8(float *y, const float *x, const float *W, int S, int I, int O){
+ int O8=O&~7;
+ #pragma omp parallel for schedule(static)
+ for (int o=0;o<O8;o+=8){ const float *w0=W+(int64_t)o*I, *w1=w0+I, *w2=w1+I, *w3=w2+I, *w4=w3+I, *w5=w4+I, *w6=w5+I, *w7=w6+I;
+ for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a0=0,a1=0,a2=0,a3=0,a4=0,a5=0,a6=0,a7=0;
+ for(int i=0;i<I;i++){ float xi=xs[i];
+ a0=fmaf(xi,w0[i],a0); a1=fmaf(xi,w1[i],a1); a2=fmaf(xi,w2[i],a2); a3=fmaf(xi,w3[i],a3);
+ a4=fmaf(xi,w4[i],a4); a5=fmaf(xi,w5[i],a5); a6=fmaf(xi,w6[i],a6); a7=fmaf(xi,w7[i],a7); }
+ float *ys=y+(int64_t)s*O+o; ys[0]=a0; ys[1]=a1; ys[2]=a2; ys[3]=a3; ys[4]=a4; ys[5]=a5; ys[6]=a6; ys[7]=a7; } }
+ for (int o=O8;o<O;o++){ const float *w=W+(int64_t)o*I;
+ for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; for(int i=0;i<I;i++) a=fmaf(xs[i],w[i],a); y[(int64_t)s*O+o]=a; } }
+}
static double now_s(void){ struct timespec t; clock_gettime(CLOCK_MONOTONIC,&t); return t.tv_sec+t.tv_nsec*1e-9; }
static int cmp_d(const void *a, const void *b){ double x=*(const double*)a, y=*(const double*)b; return (x>y)-(x<y); }
@@ -36,28 +59,38 @@ static double sample_us(mm_fn f, float *y, const float *x, const float *W, int S
int main(void){
static const struct { const char *name; int S, I, O; } shapes[]={
- {"GLM-5.2 router decode", 1, 6144, 256}, {"GLM-5.2 router prefill", 16, 6144, 256},
- {"small router decode", 1, 2048, 64}, {"small router prefill", 16, 2048, 64} };
+ {"GLM-5.2 router decode", 1, 6144, 256}, {"GLM-5.2 router prefill", 16, 6144, 256},
+ {"Inkling router decode", 1, 6144, 258}, {"Qwen3.6 DeltaNet a/b", 1, 2048, 32},
+ {"OLMoE router decode", 1, 2048, 64}, {"OLMoE q/k/v/o decode", 1, 2048, 2048},
+ {"OLMoE q/k/v/o prefill", 16, 2048, 2048}, {"OLMoE lm_head decode", 1, 2048, 50304} };
+ static const mm_fn arm[]={ matmul_scalar, matmul_ompsimd, matmul_fmaf8, matmul };
+ static const char *arm_name[]={ "scalar", "ompsimd", "fmaf8", "lanes" };
+ enum { ARMS=4 };
uint64_t rng=0x9E3779B97F4A7C15ull;
#ifdef _OPENMP
printf("threads: %d\n", omp_get_max_threads());
#else
printf("threads: 1 (no OpenMP)\n");
#endif
+ printf("%-24s %-22s", "shape", "");
+ for (int a=0;a<ARMS;a++) printf(" %9s", arm_name[a]);
+ printf(" median us/call; lanes vs each\n");
for (size_t k=0;k<sizeof shapes/sizeof shapes[0];k++){
int S=shapes[k].S, I=shapes[k].I, O=shapes[k].O;
float *x=malloc(sizeof(float)*(size_t)S*I), *W=malloc(sizeof(float)*(size_t)I*O), *y=malloc(sizeof(float)*(size_t)S*O);
for (size_t i=0;i<(size_t)S*I;i++){ rng^=rng<<13; rng^=rng>>7; rng^=rng<<17; x[i]=(float)(rng>>40)/(float)(1<<24)*2.f-1.f; }
for (size_t i=0;i<(size_t)I*O;i++){ rng^=rng<<13; rng^=rng>>7; rng^=rng<<17; W[i]=((float)(rng>>40)/(float)(1<<24)*2.f-1.f)*0.05f; }
- int reps=1; while (sample_us(matmul_old,y,x,W,S,I,O,reps)*reps<50e3 && reps<1<<20) reps*=2;
- sample_us(matmul_old,y,x,W,S,I,O,reps); sample_us(matmul,y,x,W,S,I,O,reps); /* warm-up */
- double to[SAMPLES], tn[SAMPLES];
- for (int r=0;r<SAMPLES;r++){ to[r]=sample_us(matmul_old,y,x,W,S,I,O,reps); tn[r]=sample_us(matmul,y,x,W,S,I,O,reps); }
- printf("%-24s S=%-2d I=%d O=%d old:", shapes[k].name, S, I, O);
- for (int r=0;r<SAMPLES;r++) printf(" %.2f", to[r]);
- printf(" new:"); for (int r=0;r<SAMPLES;r++) printf(" %.2f", tn[r]);
- qsort(to,SAMPLES,sizeof(double),cmp_d); qsort(tn,SAMPLES,sizeof(double),cmp_d);
- printf(" us/call; median %.2f -> %.2f (%.2fx)\n", to[SAMPLES/2], tn[SAMPLES/2], to[SAMPLES/2]/tn[SAMPLES/2]);
+ int reps=1; while (sample_us(matmul_scalar,y,x,W,S,I,O,reps)*reps<50e3 && reps<1<<20) reps*=2;
+ for (int a=0;a<ARMS;a++) sample_us(arm[a],y,x,W,S,I,O,reps); /* warm-up */
+ double t[ARMS][SAMPLES];
+ for (int r=0;r<SAMPLES;r++) for (int a=0;a<ARMS;a++) t[a][r]=sample_us(arm[a],y,x,W,S,I,O,reps);
+ for (int a=0;a<ARMS;a++) qsort(t[a],SAMPLES,sizeof(double),cmp_d);
+ double lanes=t[ARMS-1][SAMPLES/2];
+ printf("%-24s S=%-2d I=%-4d O=%-6d ", shapes[k].name, S, I, O);
+ for (int a=0;a<ARMS;a++) printf(" %9.2f", t[a][SAMPLES/2]);
+ printf(" ");
+ for (int a=0;a<ARMS-1;a++) printf(" %.2fx", t[a][SAMPLES/2]/lanes);
+ printf("\n");
free(x); free(W); free(y);
}
return 0;
diff --git a/c/tests/test_matmul_f32.c b/c/tests/test_matmul_f32.c
index 77fbd48b..3e8c3649 100644
--- a/c/tests/test_matmul_f32.c
+++ b/c/tests/test_matmul_f32.c
@@ -1,10 +1,11 @@
/* f32 matmul exactness gate (#442).
*
- * On FMA targets quant.h's matmul computes each output as one fmaf chain over
- * i in order, eight rows at a time. The row blocking must not change a single
- * bit: this compares it with memcmp against the one-row fmaf chain, over shapes
- * that exercise the eight-row blocks, the leftover rows and short rows. Without
- * FMA matmul keeps the plain `a+=x*w` loop, and the reference follows it.
+ * matmul_f32.h computes each output in MATMUL_F32_LANES fixed lanes (lane j
+ * takes i == j mod lanes, in order) and sums the lanes in a halving tree. The
+ * vectorized loop must give exactly the bits of that definition written out
+ * one element at a time: this compares them with memcmp over shapes that
+ * exercise full lane blocks, a partial last block and rows shorter than one
+ * block. With FMA every lane step is an fmaf, without it `a+x*w`.
*
* Build: make -C c tests/test_matmul_f32 */
#include <stdio.h>
@@ -19,17 +20,20 @@ static float rnd_f(void){ rng^=rng<<13; rng^=rng>>7; rng^=rng<<17; return (float
static void ref_matmul(float *y, const float *x, const float *W, int S, int I, int O){
for (int o=0;o<O;o++){ const float *w=W+(int64_t)o*I;
- for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0;
+ for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; volatile float a[MATMUL_F32_LANES]={0};
+ for (int i=0;i<I;i++){ int j=i%MATMUL_F32_LANES;
#if defined(__FMA__) || defined(__ARM_FEATURE_FMA)
- for(int i=0;i<I;i++) a=fmaf(xs[i],w[i],a);
+ a[j]=fmaf(xs[i],w[i],a[j]);
#else
- for(int i=0;i<I;i++) a+=xs[i]*w[i];
+ a[j]=a[j]+xs[i]*w[i];
#endif
- y[(int64_t)s*O+o]=a; } }
+ }
+ for (int h=MATMUL_F32_LANES/2;h>0;h>>=1) for (int j=0;j<h;j++) a[j]=a[j]+a[j+h];
+ y[(int64_t)s*O+o]=a[0]; } }
}
int main(void){
- static const int shapes[][3]={ {1,1,1}, {1,5,8}, {1,7,13}, {3,33,17}, {2,1027,41},
+ static const int shapes[][3]={ {1,1,1}, {1,5,8}, {1,7,13}, {1,31,3}, {1,32,3}, {3,33,17}, {2,1027,41}, {1,2048,32},
{1,2048,64}, {4,2048,64}, {5,6144,256}, {1,6144,257} };
int fails=0;
for (size_t k=0;k<sizeof shapes/sizeof shapes[0];k++){