Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

Add the benchmark the previous commit's numbers come from

ZacharyZcR4dbada96a699
diff --git a/c/.gitignore b/c/.gitignore
index d942612a..356fdfc7 100644
--- a/c/.gitignore
+++ b/c/.gitignore
@@ -12,6 +12,7 @@ tools/librans_c.so
tools/librans_c.dylib
tools/rans_c.dll
tests/fuzz_rans
+tests/bench_omp_grain
tests/*.dSYM/
tests/test_dsa_select
tests/test_i4_grouped
diff --git a/c/Makefile b/c/Makefile
index 56390905..43a5be42 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -727,6 +727,18 @@ efficiency-cuda:
efficiency-report:
$(PYTHON) tests/test_efficiency_report.py
+
+# What a parallel region costs before its body runs. Backs the `if(S > 1)`
+# clause in the DSA indexer: at decode that region has one iteration, and the
+# fork/join is paid anyway. Not a gate -- a measurement, run it when changing
+# how the engine parallelises.
+# make -C c bench-omp-grain
+# Honours OMP_NUM_THREADS/OMP_WAIT_POLICY, so measure with the settings the
+# engine actually runs under (omp_tune.h sizes the team to physical cores).
+bench-omp-grain: tests/bench_omp_grain.c
+ $(CC) $(CFLAGS) $< -o tests/bench_omp_grain $(LDFLAGS)
+ ./tests/bench_omp_grain
+
# Local validation: one portable CPU build and dependency-free tests.
check:
$(MAKE) clean
@@ -763,5 +775,8 @@ bench: iobench$(EXE)
@if [ -n "$(ARGS)" ]; then ./iobench$(EXE) $(ARGS); else echo "built iobench$(EXE) — run: ./iobench$(EXE) <file> <MB> <iters> <threads> <direct 0|1>"; fi
.PHONY: all colibri glm iq3 rans fuzz-rans cuda-test hip-test gpu-compile cuda-bench cuda-dll portable test-c test-python test check clean install uninstall bench
+# Own line: see the TEST_RULES note above on shared .PHONY lines.
+.PHONY: bench-omp-grain
+
tests/test_kv_prefix$(EXE): tests/test_kv_prefix.c kv_prefix.h
$(CC) $(CFLAGS) tests/test_kv_prefix.c -o tests/test_kv_prefix$(EXE) $(LDFLAGS)
diff --git a/c/tests/bench_omp_grain.c b/c/tests/bench_omp_grain.c
new file mode 100644
index 00000000..81932be6
--- /dev/null
+++ b/c/tests/bench_omp_grain.c
@@ -0,0 +1,101 @@
+/* bench_omp_grain — what a parallel region costs before it does any work.
+ *
+ * The engine wraps work in `#pragma omp parallel for` without asking how much
+ * work is in there. That is right for an expert matmul and wrong for a loop
+ * with one iteration: OpenMP still forks and joins the team, and at decode
+ * (S=1) several of the engine's regions have exactly that shape.
+ *
+ * This measures the floor: a region whose body does nothing measurable, so the
+ * number IS the fork/join cost. Multiply by regions-per-token times layers to
+ * see what it means for a real model.
+ *
+ * make -C c bench-omp-grain
+ *
+ * Reports nanoseconds per region for the team as configured (OMP_NUM_THREADS),
+ * against the same loop run inline via `if(0)`. Runs long enough that the
+ * difference is not timer noise -- the engine-level tiny-model replay is 15 ms
+ * end to end and cannot resolve this at all.
+ */
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#ifdef _OPENMP
+#include <omp.h>
+#endif
+
+static double now_s(void) {
+ struct timespec t;
+ clock_gettime(CLOCK_MONOTONIC, &t);
+ return (double)t.tv_sec + 1e-9 * (double)t.tv_nsec;
+}
+
+/* Sink the loop body writes to, so neither version can be optimised away. */
+static volatile long g_sink;
+
+/* One iteration per region: the decode shape (S == 1). */
+static double bench_region(long reps, int iters, int parallel) {
+ long acc = 0;
+ double t0 = now_s();
+ for (long r = 0; r < reps; r++) {
+#pragma omp parallel for reduction(+ : acc) if (parallel)
+ for (int i = 0; i < iters; i++) acc += i + r;
+ }
+ double dt = now_s() - t0;
+ g_sink = acc;
+ return dt;
+}
+
+/* Median of n runs: the first is always slower (team not yet warm) and a
+ * scheduler hiccup should not become the headline number. */
+static int cmp_d(const void *a, const void *b) {
+ double x = *(const double *)a, y = *(const double *)b;
+ return x < y ? -1 : x > y ? 1 : 0;
+}
+
+static double median_ns(long reps, int iters, int parallel, int runs) {
+ double *v = malloc((size_t)runs * sizeof(double));
+ if (!v) return -1;
+ for (int k = 0; k < runs; k++) v[k] = bench_region(reps, iters, parallel) / (double)reps * 1e9;
+ qsort(v, (size_t)runs, sizeof(double), cmp_d);
+ double m = v[runs / 2];
+ free(v);
+ return m;
+}
+
+int main(int argc, char **argv) {
+ long reps = argc > 1 ? atol(argv[1]) : 200000;
+ int runs = argc > 2 ? atoi(argv[2]) : 7;
+ if (reps < 1) reps = 1;
+ if (runs < 1) runs = 1;
+
+ int nthreads = 1;
+#ifdef _OPENMP
+#pragma omp parallel
+ {
+#pragma omp master
+ nthreads = omp_get_num_threads();
+ }
+#else
+ printf("built without OpenMP -- nothing to measure\n");
+ return 0;
+#endif
+
+ printf("bench_omp_grain: %ld regions per sample, %d samples, team of %d\n", reps, runs, nthreads);
+ printf(" %-12s %14s %14s %12s\n", "iterations", "parallel (ns)", "inline (ns)", "overhead");
+ for (int iters = 1; iters <= 64; iters *= 8) {
+ double par = median_ns(reps, iters, 1, runs);
+ double seq = median_ns(reps, iters, 0, runs);
+ printf(" %-12d %14.1f %14.1f %11.1fx\n", iters, par, seq, seq > 0 ? par / seq : 0.0);
+ }
+
+ /* The number the engine cares about: one iteration, which is what S=1
+ * decode gives every per-position region. */
+ double one_par = median_ns(reps, 1, 1, runs);
+ double one_seq = median_ns(reps, 1, 0, runs);
+ double cost = one_par - one_seq;
+ printf("\n fork/join floor: %.0f ns per region (team of %d)\n", cost, nthreads);
+ printf(" a 78-layer model paying this once per layer per token: %.2f ms/token\n", cost * 78.0 / 1e6);
+ return 0;
+}