Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
Add the benchmark the previous commit's numbers come from
ZacharyZcR4dbada96a699
diff --git a/c/.gitignore b/c/.gitignoreindex d942612a..356fdfc7 100644--- a/c/.gitignore+++ b/c/.gitignore@@ -12,6 +12,7 @@ tools/librans_c.sotools/librans_c.dylibtools/rans_c.dlltests/fuzz_rans+tests/bench_omp_graintests/*.dSYM/tests/test_dsa_selecttests/test_i4_groupeddiff --git a/c/Makefile b/c/Makefileindex 56390905..43a5be42 100644--- a/c/Makefile+++ b/c/Makefile@@ -727,6 +727,18 @@ efficiency-cuda:efficiency-report:$(PYTHON) tests/test_efficiency_report.py++# What a parallel region costs before its body runs. Backs the `if(S > 1)`+# clause in the DSA indexer: at decode that region has one iteration, and the+# fork/join is paid anyway. Not a gate -- a measurement, run it when changing+# how the engine parallelises.+# make -C c bench-omp-grain+# Honours OMP_NUM_THREADS/OMP_WAIT_POLICY, so measure with the settings the+# engine actually runs under (omp_tune.h sizes the team to physical cores).+bench-omp-grain: tests/bench_omp_grain.c+ $(CC) $(CFLAGS) $< -o tests/bench_omp_grain $(LDFLAGS)+ ./tests/bench_omp_grain+# Local validation: one portable CPU build and dependency-free tests.check:$(MAKE) clean@@ -763,5 +775,8 @@ bench: iobench$(EXE)@if [ -n "$(ARGS)" ]; then ./iobench$(EXE) $(ARGS); else echo "built iobench$(EXE) — run: ./iobench$(EXE) <file> <MB> <iters> <threads> <direct 0|1>"; fi.PHONY: all colibri glm iq3 rans fuzz-rans cuda-test hip-test gpu-compile cuda-bench cuda-dll portable test-c test-python test check clean install uninstall bench+# Own line: see the TEST_RULES note above on shared .PHONY lines.+.PHONY: bench-omp-grain+tests/test_kv_prefix$(EXE): tests/test_kv_prefix.c kv_prefix.h$(CC) $(CFLAGS) tests/test_kv_prefix.c -o tests/test_kv_prefix$(EXE) $(LDFLAGS)diff --git a/c/tests/bench_omp_grain.c b/c/tests/bench_omp_grain.cnew file mode 100644index 00000000..81932be6--- /dev/null+++ b/c/tests/bench_omp_grain.c@@ -0,0 +1,101 @@+/* bench_omp_grain — what a parallel region costs before it does any work.+ *+ * The engine wraps work in `#pragma omp parallel for` without asking how much+ * work is in there. That is right for an expert matmul and wrong for a loop+ * with one iteration: OpenMP still forks and joins the team, and at decode+ * (S=1) several of the engine's regions have exactly that shape.+ *+ * This measures the floor: a region whose body does nothing measurable, so the+ * number IS the fork/join cost. Multiply by regions-per-token times layers to+ * see what it means for a real model.+ *+ * make -C c bench-omp-grain+ *+ * Reports nanoseconds per region for the team as configured (OMP_NUM_THREADS),+ * against the same loop run inline via `if(0)`. Runs long enough that the+ * difference is not timer noise -- the engine-level tiny-model replay is 15 ms+ * end to end and cannot resolve this at all.+ */+#define _GNU_SOURCE+#include <stdio.h>+#include <stdlib.h>+#include <string.h>+#include <time.h>+#ifdef _OPENMP+#include <omp.h>+#endif++static double now_s(void) {+ struct timespec t;+ clock_gettime(CLOCK_MONOTONIC, &t);+ return (double)t.tv_sec + 1e-9 * (double)t.tv_nsec;+}++/* Sink the loop body writes to, so neither version can be optimised away. */+static volatile long g_sink;++/* One iteration per region: the decode shape (S == 1). */+static double bench_region(long reps, int iters, int parallel) {+ long acc = 0;+ double t0 = now_s();+ for (long r = 0; r < reps; r++) {+#pragma omp parallel for reduction(+ : acc) if (parallel)+ for (int i = 0; i < iters; i++) acc += i + r;+ }+ double dt = now_s() - t0;+ g_sink = acc;+ return dt;+}++/* Median of n runs: the first is always slower (team not yet warm) and a+ * scheduler hiccup should not become the headline number. */+static int cmp_d(const void *a, const void *b) {+ double x = *(const double *)a, y = *(const double *)b;+ return x < y ? -1 : x > y ? 1 : 0;+}++static double median_ns(long reps, int iters, int parallel, int runs) {+ double *v = malloc((size_t)runs * sizeof(double));+ if (!v) return -1;+ for (int k = 0; k < runs; k++) v[k] = bench_region(reps, iters, parallel) / (double)reps * 1e9;+ qsort(v, (size_t)runs, sizeof(double), cmp_d);+ double m = v[runs / 2];+ free(v);+ return m;+}++int main(int argc, char **argv) {+ long reps = argc > 1 ? atol(argv[1]) : 200000;+ int runs = argc > 2 ? atoi(argv[2]) : 7;+ if (reps < 1) reps = 1;+ if (runs < 1) runs = 1;++ int nthreads = 1;+#ifdef _OPENMP+#pragma omp parallel+ {+#pragma omp master+ nthreads = omp_get_num_threads();+ }+#else+ printf("built without OpenMP -- nothing to measure\n");+ return 0;+#endif++ printf("bench_omp_grain: %ld regions per sample, %d samples, team of %d\n", reps, runs, nthreads);+ printf(" %-12s %14s %14s %12s\n", "iterations", "parallel (ns)", "inline (ns)", "overhead");+ for (int iters = 1; iters <= 64; iters *= 8) {+ double par = median_ns(reps, iters, 1, runs);+ double seq = median_ns(reps, iters, 0, runs);+ printf(" %-12d %14.1f %14.1f %11.1fx\n", iters, par, seq, seq > 0 ? par / seq : 0.0);+ }++ /* The number the engine cares about: one iteration, which is what S=1+ * decode gives every per-position region. */+ double one_par = median_ns(reps, 1, 1, runs);+ double one_seq = median_ns(reps, 1, 0, runs);+ double cost = one_par - one_seq;+ printf("\n fork/join floor: %.0f ns per region (team of %d)\n", cost, nthreads);+ printf(" a 78-layer model paying this once per layer per token: %.2f ms/token\n", cost * 78.0 / 1e6);+ return 0;+}