Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
mimo: a prefill row no longer depends on the rows that share its block
JustVugg6cf7910b84ec
diff --git a/c/mimo.c b/c/mimo.cindex 21193b7a..3580ffe1 100644--- a/c/mimo.c+++ b/c/mimo.c@@ -1032,10 +1032,15 @@ static void moe(Model *m, int li, const float *xn, int n, float *out) {* up of every expert, then down of every expert -- split in row blocks, instead* of three OpenMP regions per expert (a thousand a token on Flash, and every* barrier costs a scheduling round on a busy machine). The kernel inside a- * task is the same matmul_mxfp4 on the same rows, so the numbers do not move. */+ * task is the same matmul_mxfp4 on the same rows, so the numbers do not move.+ *+ * Every (row, choice) pair has its own slot in the buffers below, across all+ * the groups of the block, so the weighted sum can wait for the last group+ * and run row by row in that row's own routing order. */int cap = m->cache[li].cap;int *row = xmalloc((size_t)n * K * sizeof(int), "expert rows index");float *rw = xmalloc((size_t)n * K * sizeof(float), "expert row weights");+ int *pair = xmalloc((size_t)n * K * sizeof(int), "expert pair of a choice");int *first = xmalloc((size_t)(cap + 1) * sizeof(int), "expert row offsets");float *xg = xmalloc((size_t)n * K * H * sizeof(float), "expert inputs");float *g = xmalloc((size_t)n * K * MI * sizeof(float), "expert gate");@@ -1045,20 +1050,25 @@ static void moe(Model *m, int li, const float *xn, int n, float *out) {= m->idot ? matmul_mxfp4_i8 : matmul_mxfp4;const int BLOCK_ROWS = 256;Slot *slots[4096];+ int total = 0; /* pairs placed so far, over every group */for (int b = 0; b < nu; b += cap) {int nb = nu - b < cap ? nu - b : cap;experts_ensure(m, li, uni + b, nb, slots);double t0 = now_s();- int total = 0;+ int from = total;for (int j = 0; j < nb; j++) {int e = uni[b + j];first[j] = total;for (int t = 0; t < n; t++)for (int q = 0; q < K; q++)- if (sel[(size_t)t * K + q] == e) { row[total] = t; rw[total] = wt[(size_t)t * K + q]; total++; }+ if (sel[(size_t)t * K + q] == e) {+ row[total] = t; rw[total] = wt[(size_t)t * K + q];+ pair[(size_t)t * K + q] = total;+ total++;+ }}first[nb] = total;- for (int i = 0; i < total; i++) memcpy(xg + (size_t)i * H, xn + (size_t)row[i] * H, (size_t)H * sizeof(float));+ for (int i = from; i < total; i++) memcpy(xg + (size_t)i * H, xn + (size_t)row[i] * H, (size_t)H * sizeof(float));int up_blocks = (MI + BLOCK_ROWS - 1) / BLOCK_ROWS, down_blocks = (H + BLOCK_ROWS - 1) / BLOCK_ROWS;int rb_mi = MI / 2, gb_mi = MI / 32, rb_h = H / 2, gb_h = H / 32;#ifdef COLI_VULKAN@@ -1088,7 +1098,7 @@ static void moe(Model *m, int li, const float *xn, int n, float *out) {memcpy(dst + (size_t)i * MI + o0, scratch + (size_t)i * rows, (size_t)rows * sizeof(float));if (scratch != tmp) free(scratch);}- for (size_t i = 0; i < (size_t)total * MI; i++) g[i] = silu(g[i]) * u[i];+ for (size_t i = (size_t)from * MI; i < (size_t)total * MI; i++) g[i] = silu(g[i]) * u[i];#ifdef COLI_VULKANvk_experts_down(m, li, uni + b, nb, first, g, y, vk_gu, vk_dn);#endif@@ -1111,16 +1121,27 @@ static void moe(Model *m, int li, const float *xn, int n, float *out) {memcpy(dst + (size_t)i * H + o0, scratch + (size_t)i * rows, (size_t)rows * sizeof(float));if (scratch != tmp) free(scratch);}- /* the weighted sum, expert by expert in the order they were chosen */- for (int j = 0; j < nb; j++)- for (int i = first[j]; i < first[j + 1]; i++) {- float *o = out + (size_t)row[i] * H;- const float *yy = y + (size_t)i * H;- for (int d = 0; d < H; d++) o[d] += rw[i] * yy[d];- }m->t_expert += now_s() - t0;}- free(first);+ /* The weighted sum, row by row, each row's experts in the order ITS router+ * chose them. A row's sum used to follow the block's first-use order, which+ * depends on the rows that happen to share the block: the same position+ * came out with different low bits in a 64-row prefill block and in a+ * one-row resume, so a prompt resumed from a prefix was not the prompt+ * computed cold. Now a row is a function of its own inputs and nothing else,+ * whatever the block boundaries. One row (a decode step) adds in the same+ * order as before: its first-use order IS its routing order. */+ double t1 = now_s();+ for (int t = 0; t < n; t++) {+ float *o = out + (size_t)t * H;+ for (int q = 0; q < K; q++) {+ int i = pair[(size_t)t * K + q];+ const float *yy = y + (size_t)i * H;+ for (int d = 0; d < H; d++) o[d] += rw[i] * yy[d];+ }+ }+ m->t_expert += now_s() - t1;+ free(first); free(pair);free(xg); free(g); free(u); free(y); free(row); free(rw); free(uni); free(sel); free(wt);}diff --git a/c/tests/mimo_tiny_harness.py b/c/tests/mimo_tiny_harness.pyindex 2df3bf8b..1b5f5598 100644--- a/c/tests/mimo_tiny_harness.py+++ b/c/tests/mimo_tiny_harness.py@@ -14,6 +14,11 @@ What is checked, all token-exact (argmax, no tolerances):weights, an expert cache that evicts at every layer, prefill in blocks of 3and of 1 (the sliding-window ring across blocks), buffered expert reads- determinism (two identical runs, identical tokens)+ - block invariance, BIT for bit: the logits of every position are the same+ bytes whatever the prefill blocks (64, 3, 1) and the expert cache (one that+ evicts at every layer), text and picture. A prompt resumed from a prefix or+ a Brio photo is computed in other blocks than the same prompt cold, so this+ is what lets the two agree to the last bit rather than to the argmax- with the vision fixture (make_mimo_tiny.py --vision): the picture throughthe tower, greedy and teacher-forced, f32 and native BF16, and the negatedpicture giving the reference's own answer for it@@ -126,6 +131,41 @@ def teacher_forced(binary, fixture, ids, env, vocab, image=None):for p in range(len(ids))]+def logits_bytes(binary, fixture, ids, env, image=None):+ with tempfile.NamedTemporaryFile(suffix=".f32", delete=False) as tmp:+ path = tmp.name+ try:+ run_engine(binary, fixture, ids, 0, env, extra_env={"MIMO_LOGITS": path},+ image=image)+ with open(path, "rb") as fh:+ return fh.read()+ finally:+ os.unlink(path)+++def check_block_invariance(binary, fixture, reference):+ """A row is a function of its own inputs, never of the rows that share its+ prefill block: the logits must be the same BYTES for every block size."""+ variants = {"blocks of 3": {"MIMO_CHUNK": "3"},+ "one token at a time": {"MIMO_CHUNK": "1"},+ "cache 4 (evicts at every layer)": {"MIMO_CAP": "4"}}+ inputs = [(name, case["greedy_full_ids"], None)+ for name, case in reference["cases"].items()]+ if "image" in reference:+ image = reference["image"]+ inputs.append(("image", image["greedy_full_ids"],+ (fixture / "patches.f32", image["grid_h"], image["grid_w"])))+ for name, ids, image in inputs:+ base = logits_bytes(binary, fixture, ids, reference["engine_env"], image)+ for label, extra in variants.items():+ env = dict(reference["engine_env"], **extra)+ if logits_bytes(binary, fixture, ids, env, image) != base:+ raise AssertionError(f"block invariance: case '{name}', {label}: the "+ f"logits are not the bytes of the default blocks")+ print(f"ok block invariance: {len(inputs)} cases x {len(variants)} variants, "+ f"bit-identical logits")++def check_teacher_forcing(binary, fixture, reference, vocab):for name, case in reference["cases"].items():got = teacher_forced(binary, fixture, case["greedy_full_ids"],@@ -268,6 +308,7 @@ def main():check_greedy(args.binary, fixture, reference)check_exact_variants(args.binary, fixture, reference)check_determinism(args.binary, fixture, reference)+ check_block_invariance(args.binary, fixture, reference)check_teacher_forcing(args.binary, fixture, reference, config["vocab_size"])check_oracle_bites(args.binary, fixture, reference,config["n_routed_experts"])