Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
/* Microbenchmark: decode-regime GEMV bandwidth vs the machine's read ceiling.
*
* bench_idot measures the kernels' warm-cache compute ceiling. Decode lives in the
* opposite regime: expert weights do NOT fit in cache, every token streams them from
* RAM, and arithmetic intensity is ~1 MAC/byte. This measures the REAL quant.h row
* loops (matmul_q int8, matmul_i4 int4, their own internal OMP) over a working set
* far larger than the LLC, cycling experts so no call sees a warm weight — then a
* read baseline in the same process, same pass, so the ratio cancels host noise.
*
* The int8-vs-int4 GB/s comparison is the sharp edge: equal GB/s means the decode
* GEMV is bandwidth-bound (int4 = free 2x tokens/byte); int4 well below int8 means
* the unpack is compute-bound on this ISA and kernel work has headroom.
*
* Arms: f32-act int8/int4 (IDOT=0 paths), idot int8/int4 (production decode), a FROZEN
* baseline copy of today's plain-AVX2 dot_i4i8 (any future kernel change A/Bs against
* it), a DEINTERLEAVED-x candidate kernel (no unpacks; bit-exact, gated inline), a warm
* single-expert arm (ALU ceiling vs streaming), and a parallel-read ceiling.
*
* Run: make tests/bench_gemv_stream ARCH=native && OMP_NUM_THREADS=N ./tests/bench_gemv_stream
* (not in TEST_BINS -- a measurement, not a gate. On shared vCPUs the read ceiling
* scales with N and never shows the physical RAM roof, so absolute GB/s there is
* indicative only; the within-pass ratios are the signal. Physical AVX2-only silicon
* is what decides whether the deinterleaved candidate is worth landing.) */
#define main coli_glm_main_unused
#include "../colibri.c"
#undef main
#include <stdint.h>
#include <string.h>
/* FROZEN baseline: verbatim copy of dev's plain-AVX2 dot_i4i8 as of 2026-08-09,
* bench_idot's old-vs-new pattern. While quant.h is unchanged the ratio prints 1.00x;
* any future kernel change A/Bs against this copy in one process over identical frozen
* inputs, so host noise cancels in the ratio. */
#ifdef __AVX2__
static inline int32_t dot_i4i8_old(const uint8_t *w4, const int8_t *x, int I) {
int32_t sum = 0; int i = 0;
const __m128i m4 = _mm_set1_epi8(0x0F); const __m256i b8 = _mm256_set1_epi8(8);
const __m256i ones = _mm256_set1_epi16(1);
__m256i acc = _mm256_setzero_si256();
for (; i + 32 <= I; i += 32) {
__m128i by = _mm_loadu_si128((const __m128i*)(w4 + (i >> 1)));
__m128i lo = _mm_and_si128(by, m4), hi = _mm_and_si128(_mm_srli_epi16(by, 4), m4);
__m128i n0 = _mm_unpacklo_epi8(lo, hi), n1 = _mm_unpackhi_epi8(lo, hi);
__m256i wv = _mm256_sub_epi8(_mm256_set_m128i(n1, n0), b8);
__m256i xv = _mm256_loadu_si256((const __m256i*)(x + i));
__m256i p = _mm256_maddubs_epi16(_mm256_sign_epi8(wv, wv), _mm256_sign_epi8(xv, wv));
acc = _mm256_add_epi32(acc, _mm256_madd_epi16(p, ones));
}
sum = hsum256_i32(acc);
for (; i < I; i++) { int8_t w = (int8_t)((i & 1) ? (w4[i >> 1] >> 4) : (w4[i >> 1] & 0x0F)) - 8; sum += w * x[i]; }
return sum;
}
static void matmul_i4_idot_old(float *y, const int8_t *xq, const float *sx, const uint8_t *q4,
const float *scale, int S, int I, int O) {
int rb = (I + 1) / 2;
#pragma omp parallel for schedule(static)
for (int o = 0; o < O; o++) { const uint8_t *w = q4 + (int64_t)o * rb; float sc = scale[o];
for (int s = 0; s < S; s++) y[(int64_t)s * O + o] = (float)dot_i4i8_old(w, xq + (int64_t)s * I, I) * sc * sx[s]; }
}
#endif
/* CANDIDATE dot_i4i8_de: x pre-deinterleaved (even/odd elements) so lo/hi nibbles are
* used straight after and/srli — no unpacks, no lane insert. The deinterleave runs once
* per activation vector and amortises over every row of the projection. Products are
* identical and integer adds are associative, so the result must stay bit-identical. */
#ifdef __AVX2__
static inline int32_t dot_i4i8_de(const uint8_t *w4, const int8_t *xe, const int8_t *xo, int I) {
int32_t sum = 0; int i = 0;
const __m256i m4 = _mm256_set1_epi8(0x0F); const __m256i b8 = _mm256_set1_epi8(8);
const __m256i ones = _mm256_set1_epi16(1);
__m256i a0 = _mm256_setzero_si256(), a1 = _mm256_setzero_si256();
for (; i + 64 <= I; i += 64) {
__m256i by = _mm256_loadu_si256((const __m256i*)(w4 + (i >> 1))); /* 32 B = 64 elems */
__m256i we = _mm256_sub_epi8(_mm256_and_si256(by, m4), b8);
__m256i wo = _mm256_sub_epi8(_mm256_and_si256(_mm256_srli_epi16(by, 4), m4), b8);
__m256i xev = _mm256_loadu_si256((const __m256i*)(xe + (i >> 1)));
__m256i xov = _mm256_loadu_si256((const __m256i*)(xo + (i >> 1)));
a0 = _mm256_add_epi32(a0, _mm256_madd_epi16(_mm256_maddubs_epi16(_mm256_sign_epi8(we, we), _mm256_sign_epi8(xev, we)), ones));
a1 = _mm256_add_epi32(a1, _mm256_madd_epi16(_mm256_maddubs_epi16(_mm256_sign_epi8(wo, wo), _mm256_sign_epi8(xov, wo)), ones));
}
sum = hsum256_i32(_mm256_add_epi32(a0, a1));
for (; i < I; i++) { int8_t w = (int8_t)((i & 1) ? (w4[i >> 1] >> 4) : (w4[i >> 1] & 0x0F)) - 8;
sum += w * ((i & 1) ? xo[i >> 1] : xe[i >> 1]); }
return sum;
}
static void i4_deinterleave(const int8_t *x, int I, int8_t *xe, int8_t *xo) {
int j = 0;
for (; j + 2 <= I; j += 2) { xe[j >> 1] = x[j]; xo[j >> 1] = x[j + 1]; }
if (I & 1) xe[I >> 1] = x[I - 1];
}
static void matmul_i4_idot_de(float *y, const int8_t *xq, const float *sx, const uint8_t *q4,
const float *scale, int I, int O, int8_t *xe, int8_t *xo) {
int rb = (I + 1) / 2; /* S=1 decode shape */
i4_deinterleave(xq, I, xe, xo);
#pragma omp parallel for schedule(static)
for (int o = 0; o < O; o++)
y[o] = (float)dot_i4i8_de(q4 + (int64_t)o * rb, xe, xo, I) * scale[o] * sx[0];
}
#endif
static uint32_t rs = 0x2545F491u;
static uint32_t xr(void) { rs ^= rs << 13; rs ^= rs >> 17; rs ^= rs << 5; return rs; }
static double tnow(void) { struct timespec t; clock_gettime(CLOCK_MONOTONIC, &t); return t.tv_sec + t.tv_nsec * 1e-9; }
static int cmpd(const void *a, const void *b) { double x = *(const double*)a, y = *(const double*)b; return x < y ? -1 : x > y; }
static double median5(double *v) { qsort(v, 5, sizeof(double), cmpd); return v[2]; }
/* GLM-like expert: gate/up [INTER,HID], down [HID,INTER]; top-K experts per token */
enum { HID = 5120, INTER = 1536, NEXP = 32, TOPK = 8, TOKENS = 24, REPS = 5 };
int main(void) {
const int64_t w8_per = (int64_t)INTER * HID * 2 + (int64_t)HID * INTER; /* int8 bytes/expert */
const int64_t w4_per = w8_per / 2;
printf("bench_gemv_stream: %d experts x %.1f MB int8 (working set %.2f GB), top-%d, S=1\n",
NEXP, w8_per / 1e6, NEXP * w8_per / 1e9, TOPK);
/* one contiguous slab per format, cut into per-expert views */
int8_t *w8 = malloc((size_t)NEXP * w8_per);
uint8_t *w4 = malloc((size_t)NEXP * w4_per);
float *sc = malloc((size_t)NEXP * (INTER * 2 + HID) * sizeof(float));
float *x = malloc((size_t)HID * sizeof(float));
float *g = malloc((size_t)INTER * sizeof(float));
float *u = malloc((size_t)INTER * sizeof(float));
float *d = malloc((size_t)HID * sizeof(float));
int8_t *xq = malloc((size_t)HID); /* idot activations */
int8_t *gq = malloc((size_t)INTER);
float *rd = malloc((size_t)1 << 30); /* 1 GiB read baseline */
if (!w8 || !w4 || !sc || !x || !g || !u || !d || !xq || !gq || !rd) { fprintf(stderr, "OOM\n"); return 1; }
for (int i = 0; i < HID; i++) xq[i] = (int8_t)(xr() % 15) - 7;
for (int i = 0; i < INTER; i++) gq[i] = (int8_t)(xr() % 15) - 7;
float sx1 = 0.031f; /* one activation scale, S=1 */
for (int64_t i = 0; i < (int64_t)NEXP * w8_per; i++) w8[i] = (int8_t)(xr() & 0xff);
for (int64_t i = 0; i < (int64_t)NEXP * w4_per; i++) w4[i] = (uint8_t)(xr() & 0xff);
for (int64_t i = 0; i < (int64_t)NEXP * (INTER * 2 + HID); i++) sc[i] = 1.0f + (float)(xr() & 7) * 0.01f;
for (int i = 0; i < HID; i++) x[i] = (float)(int)(xr() % 17) - 8.0f;
for (int64_t i = 0; i < ((int64_t)1 << 30) / 4; i++) rd[i] = (float)(xr() & 0xff);
double sink = 0; /* defeats DCE across arms */
/* --- arm A: int8 GEMV, cold experts (round-robin so no reuse across calls) --- */
double a5[5];
for (int r = 0; r < 5; r++) {
int e = 0; double t0 = tnow();
for (int t = 0; t < TOKENS; t++)
for (int k = 0; k < TOPK; k++, e = (e + 1) % NEXP) {
const int8_t *base = w8 + (int64_t)e * w8_per;
const float *s = sc + (int64_t)e * (INTER * 2 + HID);
matmul_q(g, x, base, s, 1, HID, INTER);
matmul_q(u, x, base + (int64_t)INTER * HID, s + INTER, 1, HID, INTER);
for (int i = 0; i < INTER; i++) g[i] *= u[i]; /* keep g live like the FFN does */
matmul_q(d, g, base + (int64_t)INTER * HID * 2, s + INTER * 2, 1, INTER, HID);
sink += d[0] + d[HID - 1];
}
a5[r] = (double)TOKENS * TOPK * w8_per / (tnow() - t0);
}
double gbs8 = median5(a5) / 1e9;
/* --- arm B: same shape, int4 packed --- */
double b5[5];
for (int r = 0; r < 5; r++) {
int e = 0; double t0 = tnow();
for (int t = 0; t < TOKENS; t++)
for (int k = 0; k < TOPK; k++, e = (e + 1) % NEXP) {
const uint8_t *base = w4 + (int64_t)e * w4_per;
const float *s = sc + (int64_t)e * (INTER * 2 + HID);
matmul_i4(g, x, base, s, 1, HID, INTER);
matmul_i4(u, x, base + (int64_t)INTER * HID / 2, s + INTER, 1, HID, INTER);
for (int i = 0; i < INTER; i++) g[i] *= u[i];
matmul_i4(d, g, base + (int64_t)INTER * HID, s + INTER * 2, 1, INTER, HID);
sink += d[0] + d[HID - 1];
}
b5[r] = (double)TOKENS * TOPK * w4_per / (tnow() - t0);
}
double gbs4 = median5(b5) / 1e9;
/* --- arm E: the PRODUCTION decode path — idot int8×int8 (dot_i8i8) --- */
double e5[5];
for (int r = 0; r < 5; r++) {
int e = 0; double t0 = tnow();
for (int t = 0; t < TOKENS; t++)
for (int k = 0; k < TOPK; k++, e = (e + 1) % NEXP) {
const int8_t *base = w8 + (int64_t)e * w8_per;
const float *s = sc + (int64_t)e * (INTER * 2 + HID);
matmul_q_idot(g, xq, &sx1, base, s, 1, HID, INTER);
matmul_q_idot(u, xq, &sx1, base + (int64_t)INTER * HID, s + INTER, 1, HID, INTER);
matmul_q_idot(d, gq, &sx1, base + (int64_t)INTER * HID * 2, s + INTER * 2, 1, INTER, HID);
sink += d[0] + d[HID - 1];
}
e5[r] = (double)TOKENS * TOPK * w8_per / (tnow() - t0);
}
double gbs8i = median5(e5) / 1e9;
/* --- arm F: idot int4 (dot_i4i8) — the GLM fmt-4 decode kernel --- */
double f5[5];
for (int r = 0; r < 5; r++) {
int e = 0; double t0 = tnow();
for (int t = 0; t < TOKENS; t++)
for (int k = 0; k < TOPK; k++, e = (e + 1) % NEXP) {
const uint8_t *base = w4 + (int64_t)e * w4_per;
const float *s = sc + (int64_t)e * (INTER * 2 + HID);
matmul_i4_idot(g, xq, &sx1, base, s, 1, HID, INTER);
matmul_i4_idot(u, xq, &sx1, base + (int64_t)INTER * HID / 2, s + INTER, 1, HID, INTER);
matmul_i4_idot(d, gq, &sx1, base + (int64_t)INTER * HID, s + INTER * 2, 1, INTER, HID);
sink += d[0] + d[HID - 1];
}
f5[r] = (double)TOKENS * TOPK * w4_per / (tnow() - t0);
}
double gbs4i = median5(f5) / 1e9;
/* --- arm G: the OLD single-accumulator idot int4, same inputs — the A/B --- */
double g5[5]; double gbs4o = 0;
#ifdef __AVX2__
for (int r = 0; r < 5; r++) {
int e = 0; double t0 = tnow();
for (int t = 0; t < TOKENS; t++)
for (int k = 0; k < TOPK; k++, e = (e + 1) % NEXP) {
const uint8_t *base = w4 + (int64_t)e * w4_per;
const float *s = sc + (int64_t)e * (INTER * 2 + HID);
matmul_i4_idot_old(g, xq, &sx1, base, s, 1, HID, INTER);
matmul_i4_idot_old(u, xq, &sx1, base + (int64_t)INTER * HID / 2, s + INTER, 1, HID, INTER);
matmul_i4_idot_old(d, gq, &sx1, base + (int64_t)INTER * HID, s + INTER * 2, 1, INTER, HID);
sink += d[0] + d[HID - 1];
}
g5[r] = (double)TOKENS * TOPK * w4_per / (tnow() - t0);
}
gbs4o = median5(g5) / 1e9;
/* bit-exactness spot check across the whole working set, old vs current */
{
int64_t rows_checked = 0;
for (int e = 0; e < NEXP; e++) {
const uint8_t *base = w4 + (int64_t)e * w4_per;
for (int o = 0; o < INTER; o += 97) { /* stride: cover all experts cheaply */
int32_t a = dot_i4i8(base + (int64_t)o * (HID / 2), xq, HID);
int32_t b = dot_i4i8_old(base + (int64_t)o * (HID / 2), xq, HID);
if (a != b) { printf("EXACTNESS FAIL expert %d row %d: %d != %d\n", e, o, a, b); return 1; }
rows_checked++;
}
}
printf(" old-vs-new bit-exact on %lld sampled rows\n", (long long)rows_checked);
}
#endif
/* --- arm H: the deinterleaved-x candidate, same inputs --- */
double h5[5]; double gbs4d = 0;
#ifdef __AVX2__
{
int8_t *xeb = malloc((size_t)(HID + 1) / 2), *xob = malloc((size_t)(HID + 1) / 2);
int8_t *geb = malloc((size_t)(INTER + 1) / 2), *gob = malloc((size_t)(INTER + 1) / 2);
if (!xeb || !xob || !geb || !gob) { fprintf(stderr, "OOM de\n"); return 1; }
for (int r = 0; r < 5; r++) {
int e = 0; double t0 = tnow();
for (int t = 0; t < TOKENS; t++)
for (int k = 0; k < TOPK; k++, e = (e + 1) % NEXP) {
const uint8_t *base = w4 + (int64_t)e * w4_per;
const float *s = sc + (int64_t)e * (INTER * 2 + HID);
matmul_i4_idot_de(g, xq, &sx1, base, s, HID, INTER, xeb, xob);
matmul_i4_idot_de(u, xq, &sx1, base + (int64_t)INTER * HID / 2, s + INTER, HID, INTER, xeb, xob);
matmul_i4_idot_de(d, gq, &sx1, base + (int64_t)INTER * HID, s + INTER * 2, INTER, HID, geb, gob);
sink += d[0] + d[HID - 1];
}
h5[r] = (double)TOKENS * TOPK * w4_per / (tnow() - t0);
}
gbs4d = median5(h5) / 1e9;
/* exactness: candidate vs current, whole sampled set + ragged tails */
i4_deinterleave(xq, HID, xeb, xob);
int64_t rows = 0;
for (int e = 0; e < NEXP; e++) {
const uint8_t *base = w4 + (int64_t)e * w4_per;
for (int o = 0; o < INTER; o += 97) {
int32_t a = dot_i4i8(base + (int64_t)o * (HID / 2), xq, HID);
int32_t b = dot_i4i8_de(base + (int64_t)o * (HID / 2), xeb, xob, HID);
if (a != b) { printf("DE EXACTNESS FAIL e%d o%d: %d != %d\n", e, o, a, b); return 1; }
rows++;
}
}
for (int I2 = 1; I2 <= 200; I2++) { /* ragged sizes, both parities */
i4_deinterleave(xq, I2, xeb, xob);
int32_t a = dot_i4i8(w4, xq, I2), b = dot_i4i8_de(w4, xeb, xob, I2);
if (a != b) { printf("DE TAIL FAIL I=%d: %d != %d\n", I2, a, b); return 1; }
}
printf(" candidate bit-exact on %lld sampled rows + I=1..200 tails\n", (long long)rows);
free(xeb); free(xob); free(geb); free(gob);
}
#endif
/* --- arm C: warm sanity — one expert, must beat arm A by a wide margin or the
* "cold" arms were never actually streaming RAM --- */
double c5[5];
for (int r = 0; r < 5; r++) {
double t0 = tnow();
for (int t = 0; t < TOKENS * TOPK; t++) {
matmul_q(g, x, w8, sc, 1, HID, INTER);
sink += g[0];
}
c5[r] = (double)TOKENS * TOPK * INTER * HID / (tnow() - t0);
}
double gbsw = median5(c5) / 1e9;
/* --- arm W: warm idot4, current vs deinterleaved — one gate matrix in cache.
* Separates the ALU ceiling from the streaming cost: if DE wins big here but
* only 9% cold, the cold arms are memory-limited and kernel µops are moot. --- */
#ifdef __AVX2__
{
int8_t *xeb = malloc((size_t)(HID + 1) / 2), *xob = malloc((size_t)(HID + 1) / 2);
double wc5[5], wd5[5];
for (int r = 0; r < 5; r++) {
double t0 = tnow();
for (int t = 0; t < TOKENS * TOPK; t++) {
matmul_i4_idot(g, xq, &sx1, w4, sc, 1, HID, INTER);
sink += g[0];
}
wc5[r] = (double)TOKENS * TOPK * ((int64_t)INTER * HID / 2) / (tnow() - t0);
}
for (int r = 0; r < 5; r++) {
double t0 = tnow();
for (int t = 0; t < TOKENS * TOPK; t++) {
matmul_i4_idot_de(g, xq, &sx1, w4, sc, HID, INTER, xeb, xob);
sink += g[0];
}
wd5[r] = (double)TOKENS * TOPK * ((int64_t)INTER * HID / 2) / (tnow() - t0);
}
printf(" warm idot4 cur : %6.2f GB/s warm idot4 DEINT: %6.2f GB/s (DE/cur warm: %.2fx)\n",
median5(wc5) / 1e9, median5(wd5) / 1e9, median5(wd5) / median5(wc5));
free(xeb); free(xob);
}
#endif
/* --- arm D: read ceiling — every thread streams the 1 GiB buffer --- */
double d5[5];
for (int r = 0; r < 5; r++) {
double t0 = tnow(); double acc = 0;
#pragma omp parallel for reduction(+:acc) schedule(static)
for (int64_t i = 0; i < ((int64_t)1 << 30) / 4; i += 8) acc += rd[i];
d5[r] = (double)(1 << 30) / (tnow() - t0);
sink += acc;
}
double gbsr = median5(d5) / 1e9;
printf(" f32-act int8 cold: %6.2f GB/s of weights (IDOT=0 path)\n", gbs8);
printf(" f32-act int4 cold: %6.2f GB/s of weights\n", gbs4);
printf(" idot int8 cold : %6.2f GB/s of weights (production decode, dot_i8i8)\n", gbs8i);
printf(" idot int4 cold : %6.2f GB/s of weights (production decode, dot_i4i8)\n", gbs4i);
if (gbs4o > 0)
printf(" idot int4 BASE : %6.2f GB/s of weights (frozen baseline copy; 1.00x while quant.h is unchanged)\n",
gbs4o);
if (gbs4d > 0)
printf(" idot int4 DEINT : %6.2f GB/s of weights (deinterleaved x) vs current: %.2fx tokens vs idot8: %.2fx\n",
gbs4d, gbs4d / gbs4i, 2.0 * gbs4d / gbs8i);
printf(" int8 GEMV warm : %6.2f GB/s (kernel ceiling, weights in cache)\n", gbsw);
printf(" read ceiling : %6.2f GB/s (parallel strided sum, 1 GiB)\n", gbsr);
printf(" efficiency vs read: f32i8 %.0f%% f32i4 %.0f%% idot8 %.0f%% idot4 %.0f%%\n",
100.0 * gbs8 / gbsr, 100.0 * gbs4 / gbsr, 100.0 * gbs8i / gbsr, 100.0 * gbs4i / gbsr);
printf(" tokens/s, idot4 vs idot8 (same weights, half the bytes): %.2fx\n", 2.0 * gbs4i / gbs8i);
printf(" (sink %.3g)\n", sink);
if (gbsw < gbs8 * 0.95) { printf("SANITY FAIL: warm below cold — measurement broken\n"); return 1; }
printf(" regime of the f32-act path: %s (warm ceiling at %.0f%% of read ceiling)\n",
gbsw < 0.7 * gbsr ? "COMPUTE-bound" : "bandwidth-bound", 100.0 * gbsw / gbsr);
return 0;
}