Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
bench: decode-regime GEMV bandwidth vs the read ceiling, with a frozen A/B baseline
Mario Gutierrez46cb37cfc512
diff --git a/c/tests/bench_gemv_stream.c b/c/tests/bench_gemv_stream.cnew file mode 100644index 00000000..34140e76--- /dev/null+++ b/c/tests/bench_gemv_stream.c@@ -0,0 +1,360 @@+/* Microbenchmark: decode-regime GEMV bandwidth vs the machine's read ceiling.+ *+ * bench_idot measures the kernels' warm-cache compute ceiling. Decode lives in the+ * opposite regime: expert weights do NOT fit in cache, every token streams them from+ * RAM, and arithmetic intensity is ~1 MAC/byte. This measures the REAL quant.h row+ * loops (matmul_q int8, matmul_i4 int4, their own internal OMP) over a working set+ * far larger than the LLC, cycling experts so no call sees a warm weight — then a+ * read baseline in the same process, same pass, so the ratio cancels host noise.+ *+ * The int8-vs-int4 GB/s comparison is the sharp edge: equal GB/s means the decode+ * GEMV is bandwidth-bound (int4 = free 2x tokens/byte); int4 well below int8 means+ * the unpack is compute-bound on this ISA and kernel work has headroom.+ *+ * Arms: f32-act int8/int4 (IDOT=0 paths), idot int8/int4 (production decode), a FROZEN+ * baseline copy of today's plain-AVX2 dot_i4i8 (any future kernel change A/Bs against+ * it), a DEINTERLEAVED-x candidate kernel (no unpacks; bit-exact, gated inline), a warm+ * single-expert arm (ALU ceiling vs streaming), and a parallel-read ceiling.+ *+ * Run: make tests/bench_gemv_stream ARCH=native && OMP_NUM_THREADS=N ./tests/bench_gemv_stream+ * (not in TEST_BINS -- a measurement, not a gate. On shared vCPUs the read ceiling+ * scales with N and never shows the physical RAM roof, so absolute GB/s there is+ * indicative only; the within-pass ratios are the signal. Physical AVX2-only silicon+ * is what decides whether the deinterleaved candidate is worth landing.) */+#define main coli_glm_main_unused+#include "../colibri.c"+#undef main+#include <stdint.h>+#include <string.h>++/* FROZEN baseline: verbatim copy of dev's plain-AVX2 dot_i4i8 as of 2026-08-09,+ * bench_idot's old-vs-new pattern. While quant.h is unchanged the ratio prints 1.00x;+ * any future kernel change A/Bs against this copy in one process over identical frozen+ * inputs, so host noise cancels in the ratio. */+#ifdef __AVX2__+static inline int32_t dot_i4i8_old(const uint8_t *w4, const int8_t *x, int I) {+ int32_t sum = 0; int i = 0;+ const __m128i m4 = _mm_set1_epi8(0x0F); const __m256i b8 = _mm256_set1_epi8(8);+ const __m256i ones = _mm256_set1_epi16(1);+ __m256i acc = _mm256_setzero_si256();+ for (; i + 32 <= I; i += 32) {+ __m128i by = _mm_loadu_si128((const __m128i*)(w4 + (i >> 1)));+ __m128i lo = _mm_and_si128(by, m4), hi = _mm_and_si128(_mm_srli_epi16(by, 4), m4);+ __m128i n0 = _mm_unpacklo_epi8(lo, hi), n1 = _mm_unpackhi_epi8(lo, hi);+ __m256i wv = _mm256_sub_epi8(_mm256_set_m128i(n1, n0), b8);+ __m256i xv = _mm256_loadu_si256((const __m256i*)(x + i));+ __m256i p = _mm256_maddubs_epi16(_mm256_sign_epi8(wv, wv), _mm256_sign_epi8(xv, wv));+ acc = _mm256_add_epi32(acc, _mm256_madd_epi16(p, ones));+ }+ sum = hsum256_i32(acc);+ for (; i < I; i++) { int8_t w = (int8_t)((i & 1) ? (w4[i >> 1] >> 4) : (w4[i >> 1] & 0x0F)) - 8; sum += w * x[i]; }+ return sum;+}+static void matmul_i4_idot_old(float *y, const int8_t *xq, const float *sx, const uint8_t *q4,+ const float *scale, int S, int I, int O) {+ int rb = (I + 1) / 2;+ #pragma omp parallel for schedule(static)+ for (int o = 0; o < O; o++) { const uint8_t *w = q4 + (int64_t)o * rb; float sc = scale[o];+ for (int s = 0; s < S; s++) y[(int64_t)s * O + o] = (float)dot_i4i8_old(w, xq + (int64_t)s * I, I) * sc * sx[s]; }+}+#endif++/* CANDIDATE dot_i4i8_de: x pre-deinterleaved (even/odd elements) so lo/hi nibbles are+ * used straight after and/srli — no unpacks, no lane insert. The deinterleave runs once+ * per activation vector and amortises over every row of the projection. Products are+ * identical and integer adds are associative, so the result must stay bit-identical. */+#ifdef __AVX2__+static inline int32_t dot_i4i8_de(const uint8_t *w4, const int8_t *xe, const int8_t *xo, int I) {+ int32_t sum = 0; int i = 0;+ const __m256i m4 = _mm256_set1_epi8(0x0F); const __m256i b8 = _mm256_set1_epi8(8);+ const __m256i ones = _mm256_set1_epi16(1);+ __m256i a0 = _mm256_setzero_si256(), a1 = _mm256_setzero_si256();+ for (; i + 64 <= I; i += 64) {+ __m256i by = _mm256_loadu_si256((const __m256i*)(w4 + (i >> 1))); /* 32 B = 64 elems */+ __m256i we = _mm256_sub_epi8(_mm256_and_si256(by, m4), b8);+ __m256i wo = _mm256_sub_epi8(_mm256_and_si256(_mm256_srli_epi16(by, 4), m4), b8);+ __m256i xev = _mm256_loadu_si256((const __m256i*)(xe + (i >> 1)));+ __m256i xov = _mm256_loadu_si256((const __m256i*)(xo + (i >> 1)));+ a0 = _mm256_add_epi32(a0, _mm256_madd_epi16(_mm256_maddubs_epi16(_mm256_sign_epi8(we, we), _mm256_sign_epi8(xev, we)), ones));+ a1 = _mm256_add_epi32(a1, _mm256_madd_epi16(_mm256_maddubs_epi16(_mm256_sign_epi8(wo, wo), _mm256_sign_epi8(xov, wo)), ones));+ }+ sum = hsum256_i32(_mm256_add_epi32(a0, a1));+ for (; i < I; i++) { int8_t w = (int8_t)((i & 1) ? (w4[i >> 1] >> 4) : (w4[i >> 1] & 0x0F)) - 8;+ sum += w * ((i & 1) ? xo[i >> 1] : xe[i >> 1]); }+ return sum;+}+static void i4_deinterleave(const int8_t *x, int I, int8_t *xe, int8_t *xo) {+ int j = 0;+ for (; j + 2 <= I; j += 2) { xe[j >> 1] = x[j]; xo[j >> 1] = x[j + 1]; }+ if (I & 1) xe[I >> 1] = x[I - 1];+}+static void matmul_i4_idot_de(float *y, const int8_t *xq, const float *sx, const uint8_t *q4,+ const float *scale, int I, int O, int8_t *xe, int8_t *xo) {+ int rb = (I + 1) / 2; /* S=1 decode shape */+ i4_deinterleave(xq, I, xe, xo);+ #pragma omp parallel for schedule(static)+ for (int o = 0; o < O; o++)+ y[o] = (float)dot_i4i8_de(q4 + (int64_t)o * rb, xe, xo, I) * scale[o] * sx[0];+}+#endif++static uint32_t rs = 0x2545F491u;+static uint32_t xr(void) { rs ^= rs << 13; rs ^= rs >> 17; rs ^= rs << 5; return rs; }+static double tnow(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec + t.tv_nsec * 1e-9; }+static int cmpd(const void *a, const void *b) { double x = *(const double*)a, y = *(const double*)b; return x < y ? -1 : x > y; }+static double median5(double *v) { qsort(v, 5, sizeof(double), cmpd); return v[2]; }++/* GLM-like expert: gate/up [INTER,HID], down [HID,INTER]; top-K experts per token */+enum { HID = 5120, INTER = 1536, NEXP = 32, TOPK = 8, TOKENS = 24, REPS = 5 };++int main(void) {+ const int64_t w8_per = (int64_t)INTER * HID * 2 + (int64_t)HID * INTER; /* int8 bytes/expert */+ const int64_t w4_per = w8_per / 2;+ printf("bench_gemv_stream: %d experts x %.1f MB int8 (working set %.2f GB), top-%d, S=1\n",+ NEXP, w8_per / 1e6, NEXP * w8_per / 1e9, TOPK);++ /* one contiguous slab per format, cut into per-expert views */+ int8_t *w8 = malloc((size_t)NEXP * w8_per);+ uint8_t *w4 = malloc((size_t)NEXP * w4_per);+ float *sc = malloc((size_t)NEXP * (INTER * 2 + HID) * sizeof(float));+ float *x = malloc((size_t)HID * sizeof(float));+ float *g = malloc((size_t)INTER * sizeof(float));+ float *u = malloc((size_t)INTER * sizeof(float));+ float *d = malloc((size_t)HID * sizeof(float));+ int8_t *xq = malloc((size_t)HID); /* idot activations */+ int8_t *gq = malloc((size_t)INTER);+ float *rd = malloc((size_t)1 << 30); /* 1 GiB read baseline */+ if (!w8 || !w4 || !sc || !x || !g || !u || !d || !xq || !gq || !rd) { fprintf(stderr, "OOM\n"); return 1; }+ for (int i = 0; i < HID; i++) xq[i] = (int8_t)(xr() % 15) - 7;+ for (int i = 0; i < INTER; i++) gq[i] = (int8_t)(xr() % 15) - 7;+ float sx1 = 0.031f; /* one activation scale, S=1 */+ for (int64_t i = 0; i < (int64_t)NEXP * w8_per; i++) w8[i] = (int8_t)(xr() & 0xff);+ for (int64_t i = 0; i < (int64_t)NEXP * w4_per; i++) w4[i] = (uint8_t)(xr() & 0xff);+ for (int64_t i = 0; i < (int64_t)NEXP * (INTER * 2 + HID); i++) sc[i] = 1.0f + (float)(xr() & 7) * 0.01f;+ for (int i = 0; i < HID; i++) x[i] = (float)(int)(xr() % 17) - 8.0f;+ for (int64_t i = 0; i < ((int64_t)1 << 30) / 4; i++) rd[i] = (float)(xr() & 0xff);++ double sink = 0; /* defeats DCE across arms */++ /* --- arm A: int8 GEMV, cold experts (round-robin so no reuse across calls) --- */+ double a5[5];+ for (int r = 0; r < 5; r++) {+ int e = 0; double t0 = tnow();+ for (int t = 0; t < TOKENS; t++)+ for (int k = 0; k < TOPK; k++, e = (e + 1) % NEXP) {+ const int8_t *base = w8 + (int64_t)e * w8_per;+ const float *s = sc + (int64_t)e * (INTER * 2 + HID);+ matmul_q(g, x, base, s, 1, HID, INTER);+ matmul_q(u, x, base + (int64_t)INTER * HID, s + INTER, 1, HID, INTER);+ for (int i = 0; i < INTER; i++) g[i] *= u[i]; /* keep g live like the FFN does */+ matmul_q(d, g, base + (int64_t)INTER * HID * 2, s + INTER * 2, 1, INTER, HID);+ sink += d[0] + d[HID - 1];+ }+ a5[r] = (double)TOKENS * TOPK * w8_per / (tnow() - t0);+ }+ double gbs8 = median5(a5) / 1e9;++ /* --- arm B: same shape, int4 packed --- */+ double b5[5];+ for (int r = 0; r < 5; r++) {+ int e = 0; double t0 = tnow();+ for (int t = 0; t < TOKENS; t++)+ for (int k = 0; k < TOPK; k++, e = (e + 1) % NEXP) {+ const uint8_t *base = w4 + (int64_t)e * w4_per;+ const float *s = sc + (int64_t)e * (INTER * 2 + HID);+ matmul_i4(g, x, base, s, 1, HID, INTER);+ matmul_i4(u, x, base + (int64_t)INTER * HID / 2, s + INTER, 1, HID, INTER);+ for (int i = 0; i < INTER; i++) g[i] *= u[i];+ matmul_i4(d, g, base + (int64_t)INTER * HID, s + INTER * 2, 1, INTER, HID);+ sink += d[0] + d[HID - 1];+ }+ b5[r] = (double)TOKENS * TOPK * w4_per / (tnow() - t0);+ }+ double gbs4 = median5(b5) / 1e9;++ /* --- arm E: the PRODUCTION decode path — idot int8×int8 (dot_i8i8) --- */+ double e5[5];+ for (int r = 0; r < 5; r++) {+ int e = 0; double t0 = tnow();+ for (int t = 0; t < TOKENS; t++)+ for (int k = 0; k < TOPK; k++, e = (e + 1) % NEXP) {+ const int8_t *base = w8 + (int64_t)e * w8_per;+ const float *s = sc + (int64_t)e * (INTER * 2 + HID);+ matmul_q_idot(g, xq, &sx1, base, s, 1, HID, INTER);+ matmul_q_idot(u, xq, &sx1, base + (int64_t)INTER * HID, s + INTER, 1, HID, INTER);+ matmul_q_idot(d, gq, &sx1, base + (int64_t)INTER * HID * 2, s + INTER * 2, 1, INTER, HID);+ sink += d[0] + d[HID - 1];+ }+ e5[r] = (double)TOKENS * TOPK * w8_per / (tnow() - t0);+ }+ double gbs8i = median5(e5) / 1e9;++ /* --- arm F: idot int4 (dot_i4i8) — the GLM fmt-4 decode kernel --- */+ double f5[5];+ for (int r = 0; r < 5; r++) {+ int e = 0; double t0 = tnow();+ for (int t = 0; t < TOKENS; t++)+ for (int k = 0; k < TOPK; k++, e = (e + 1) % NEXP) {+ const uint8_t *base = w4 + (int64_t)e * w4_per;+ const float *s = sc + (int64_t)e * (INTER * 2 + HID);+ matmul_i4_idot(g, xq, &sx1, base, s, 1, HID, INTER);+ matmul_i4_idot(u, xq, &sx1, base + (int64_t)INTER * HID / 2, s + INTER, 1, HID, INTER);+ matmul_i4_idot(d, gq, &sx1, base + (int64_t)INTER * HID, s + INTER * 2, 1, INTER, HID);+ sink += d[0] + d[HID - 1];+ }+ f5[r] = (double)TOKENS * TOPK * w4_per / (tnow() - t0);+ }+ double gbs4i = median5(f5) / 1e9;++ /* --- arm G: the OLD single-accumulator idot int4, same inputs — the A/B --- */+ double g5[5]; double gbs4o = 0;+#ifdef __AVX2__+ for (int r = 0; r < 5; r++) {+ int e = 0; double t0 = tnow();+ for (int t = 0; t < TOKENS; t++)+ for (int k = 0; k < TOPK; k++, e = (e + 1) % NEXP) {+ const uint8_t *base = w4 + (int64_t)e * w4_per;+ const float *s = sc + (int64_t)e * (INTER * 2 + HID);+ matmul_i4_idot_old(g, xq, &sx1, base, s, 1, HID, INTER);+ matmul_i4_idot_old(u, xq, &sx1, base + (int64_t)INTER * HID / 2, s + INTER, 1, HID, INTER);+ matmul_i4_idot_old(d, gq, &sx1, base + (int64_t)INTER * HID, s + INTER * 2, 1, INTER, HID);+ sink += d[0] + d[HID - 1];+ }+ g5[r] = (double)TOKENS * TOPK * w4_per / (tnow() - t0);+ }+ gbs4o = median5(g5) / 1e9;++ /* bit-exactness spot check across the whole working set, old vs current */+ {+ int64_t rows_checked = 0;+ for (int e = 0; e < NEXP; e++) {+ const uint8_t *base = w4 + (int64_t)e * w4_per;+ for (int o = 0; o < INTER; o += 97) { /* stride: cover all experts cheaply */+ int32_t a = dot_i4i8(base + (int64_t)o * (HID / 2), xq, HID);+ int32_t b = dot_i4i8_old(base + (int64_t)o * (HID / 2), xq, HID);+ if (a != b) { printf("EXACTNESS FAIL expert %d row %d: %d != %d\n", e, o, a, b); return 1; }+ rows_checked++;+ }+ }+ printf(" old-vs-new bit-exact on %lld sampled rows\n", (long long)rows_checked);+ }+#endif++ /* --- arm H: the deinterleaved-x candidate, same inputs --- */+ double h5[5]; double gbs4d = 0;+#ifdef __AVX2__+ {+ int8_t *xeb = malloc((size_t)(HID + 1) / 2), *xob = malloc((size_t)(HID + 1) / 2);+ int8_t *geb = malloc((size_t)(INTER + 1) / 2), *gob = malloc((size_t)(INTER + 1) / 2);+ if (!xeb || !xob || !geb || !gob) { fprintf(stderr, "OOM de\n"); return 1; }+ for (int r = 0; r < 5; r++) {+ int e = 0; double t0 = tnow();+ for (int t = 0; t < TOKENS; t++)+ for (int k = 0; k < TOPK; k++, e = (e + 1) % NEXP) {+ const uint8_t *base = w4 + (int64_t)e * w4_per;+ const float *s = sc + (int64_t)e * (INTER * 2 + HID);+ matmul_i4_idot_de(g, xq, &sx1, base, s, HID, INTER, xeb, xob);+ matmul_i4_idot_de(u, xq, &sx1, base + (int64_t)INTER * HID / 2, s + INTER, HID, INTER, xeb, xob);+ matmul_i4_idot_de(d, gq, &sx1, base + (int64_t)INTER * HID, s + INTER * 2, INTER, HID, geb, gob);+ sink += d[0] + d[HID - 1];+ }+ h5[r] = (double)TOKENS * TOPK * w4_per / (tnow() - t0);+ }+ gbs4d = median5(h5) / 1e9;++ /* exactness: candidate vs current, whole sampled set + ragged tails */+ i4_deinterleave(xq, HID, xeb, xob);+ int64_t rows = 0;+ for (int e = 0; e < NEXP; e++) {+ const uint8_t *base = w4 + (int64_t)e * w4_per;+ for (int o = 0; o < INTER; o += 97) {+ int32_t a = dot_i4i8(base + (int64_t)o * (HID / 2), xq, HID);+ int32_t b = dot_i4i8_de(base + (int64_t)o * (HID / 2), xeb, xob, HID);+ if (a != b) { printf("DE EXACTNESS FAIL e%d o%d: %d != %d\n", e, o, a, b); return 1; }+ rows++;+ }+ }+ for (int I2 = 1; I2 <= 200; I2++) { /* ragged sizes, both parities */+ i4_deinterleave(xq, I2, xeb, xob);+ int32_t a = dot_i4i8(w4, xq, I2), b = dot_i4i8_de(w4, xeb, xob, I2);+ if (a != b) { printf("DE TAIL FAIL I=%d: %d != %d\n", I2, a, b); return 1; }+ }+ printf(" candidate bit-exact on %lld sampled rows + I=1..200 tails\n", (long long)rows);+ free(xeb); free(xob); free(geb); free(gob);+ }+#endif++ /* --- arm C: warm sanity — one expert, must beat arm A by a wide margin or the+ * "cold" arms were never actually streaming RAM --- */+ double c5[5];+ for (int r = 0; r < 5; r++) {+ double t0 = tnow();+ for (int t = 0; t < TOKENS * TOPK; t++) {+ matmul_q(g, x, w8, sc, 1, HID, INTER);+ sink += g[0];+ }+ c5[r] = (double)TOKENS * TOPK * INTER * HID / (tnow() - t0);+ }+ double gbsw = median5(c5) / 1e9;++ /* --- arm W: warm idot4, current vs deinterleaved — one gate matrix in cache.+ * Separates the ALU ceiling from the streaming cost: if DE wins big here but+ * only 9% cold, the cold arms are memory-limited and kernel µops are moot. --- */+#ifdef __AVX2__+ {+ int8_t *xeb = malloc((size_t)(HID + 1) / 2), *xob = malloc((size_t)(HID + 1) / 2);+ double wc5[5], wd5[5];+ for (int r = 0; r < 5; r++) {+ double t0 = tnow();+ for (int t = 0; t < TOKENS * TOPK; t++) {+ matmul_i4_idot(g, xq, &sx1, w4, sc, 1, HID, INTER);+ sink += g[0];+ }+ wc5[r] = (double)TOKENS * TOPK * ((int64_t)INTER * HID / 2) / (tnow() - t0);+ }+ for (int r = 0; r < 5; r++) {+ double t0 = tnow();+ for (int t = 0; t < TOKENS * TOPK; t++) {+ matmul_i4_idot_de(g, xq, &sx1, w4, sc, HID, INTER, xeb, xob);+ sink += g[0];+ }+ wd5[r] = (double)TOKENS * TOPK * ((int64_t)INTER * HID / 2) / (tnow() - t0);+ }+ printf(" warm idot4 cur : %6.2f GB/s warm idot4 DEINT: %6.2f GB/s (DE/cur warm: %.2fx)\n",+ median5(wc5) / 1e9, median5(wd5) / 1e9, median5(wd5) / median5(wc5));+ free(xeb); free(xob);+ }+#endif++ /* --- arm D: read ceiling — every thread streams the 1 GiB buffer --- */+ double d5[5];+ for (int r = 0; r < 5; r++) {+ double t0 = tnow(); double acc = 0;+ #pragma omp parallel for reduction(+:acc) schedule(static)+ for (int64_t i = 0; i < ((int64_t)1 << 30) / 4; i += 8) acc += rd[i];+ d5[r] = (double)(1 << 30) / (tnow() - t0);+ sink += acc;+ }+ double gbsr = median5(d5) / 1e9;++ printf(" f32-act int8 cold: %6.2f GB/s of weights (IDOT=0 path)\n", gbs8);+ printf(" f32-act int4 cold: %6.2f GB/s of weights\n", gbs4);+ printf(" idot int8 cold : %6.2f GB/s of weights (production decode, dot_i8i8)\n", gbs8i);+ printf(" idot int4 cold : %6.2f GB/s of weights (production decode, dot_i4i8)\n", gbs4i);+ if (gbs4o > 0)+ printf(" idot int4 BASE : %6.2f GB/s of weights (frozen baseline copy; 1.00x while quant.h is unchanged)\n",+ gbs4o);+ if (gbs4d > 0)+ printf(" idot int4 DEINT : %6.2f GB/s of weights (deinterleaved x) vs current: %.2fx tokens vs idot8: %.2fx\n",+ gbs4d, gbs4d / gbs4i, 2.0 * gbs4d / gbs8i);+ printf(" int8 GEMV warm : %6.2f GB/s (kernel ceiling, weights in cache)\n", gbsw);+ printf(" read ceiling : %6.2f GB/s (parallel strided sum, 1 GiB)\n", gbsr);+ printf(" efficiency vs read: f32i8 %.0f%% f32i4 %.0f%% idot8 %.0f%% idot4 %.0f%%\n",+ 100.0 * gbs8 / gbsr, 100.0 * gbs4 / gbsr, 100.0 * gbs8i / gbsr, 100.0 * gbs4i / gbsr);+ printf(" tokens/s, idot4 vs idot8 (same weights, half the bytes): %.2fx\n", 2.0 * gbs4i / gbs8i);+ printf(" (sink %.3g)\n", sink);+ if (gbsw < gbs8 * 0.95) { printf("SANITY FAIL: warm below cold — measurement broken\n"); return 1; }+ printf(" regime of the f32-act path: %s (warm ceiling at %.0f%% of read ceiling)\n",+ gbsw < 0.7 * gbsr ? "COMPUTE-bound" : "bandwidth-bound", 100.0 * gbsw / gbsr);+ return 0;+}