Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

mimo: a prefill row no longer depends on the rows that share its block

JustVugg6cf7910b84ec
diff --git a/c/mimo.c b/c/mimo.c
index 21193b7a..3580ffe1 100644
--- a/c/mimo.c
+++ b/c/mimo.c
@@ -1032,10 +1032,15 @@ static void moe(Model *m, int li, const float *xn, int n, float *out) {
* up of every expert, then down of every expert -- split in row blocks, instead
* of three OpenMP regions per expert (a thousand a token on Flash, and every
* barrier costs a scheduling round on a busy machine). The kernel inside a
- * task is the same matmul_mxfp4 on the same rows, so the numbers do not move. */
+ * task is the same matmul_mxfp4 on the same rows, so the numbers do not move.
+ *
+ * Every (row, choice) pair has its own slot in the buffers below, across all
+ * the groups of the block, so the weighted sum can wait for the last group
+ * and run row by row in that row's own routing order. */
int cap = m->cache[li].cap;
int *row = xmalloc((size_t)n * K * sizeof(int), "expert rows index");
float *rw = xmalloc((size_t)n * K * sizeof(float), "expert row weights");
+ int *pair = xmalloc((size_t)n * K * sizeof(int), "expert pair of a choice");
int *first = xmalloc((size_t)(cap + 1) * sizeof(int), "expert row offsets");
float *xg = xmalloc((size_t)n * K * H * sizeof(float), "expert inputs");
float *g = xmalloc((size_t)n * K * MI * sizeof(float), "expert gate");
@@ -1045,20 +1050,25 @@ static void moe(Model *m, int li, const float *xn, int n, float *out) {
= m->idot ? matmul_mxfp4_i8 : matmul_mxfp4;
const int BLOCK_ROWS = 256;
Slot *slots[4096];
+ int total = 0; /* pairs placed so far, over every group */
for (int b = 0; b < nu; b += cap) {
int nb = nu - b < cap ? nu - b : cap;
experts_ensure(m, li, uni + b, nb, slots);
double t0 = now_s();
- int total = 0;
+ int from = total;
for (int j = 0; j < nb; j++) {
int e = uni[b + j];
first[j] = total;
for (int t = 0; t < n; t++)
for (int q = 0; q < K; q++)
- if (sel[(size_t)t * K + q] == e) { row[total] = t; rw[total] = wt[(size_t)t * K + q]; total++; }
+ if (sel[(size_t)t * K + q] == e) {
+ row[total] = t; rw[total] = wt[(size_t)t * K + q];
+ pair[(size_t)t * K + q] = total;
+ total++;
+ }
}
first[nb] = total;
- for (int i = 0; i < total; i++) memcpy(xg + (size_t)i * H, xn + (size_t)row[i] * H, (size_t)H * sizeof(float));
+ for (int i = from; i < total; i++) memcpy(xg + (size_t)i * H, xn + (size_t)row[i] * H, (size_t)H * sizeof(float));
int up_blocks = (MI + BLOCK_ROWS - 1) / BLOCK_ROWS, down_blocks = (H + BLOCK_ROWS - 1) / BLOCK_ROWS;
int rb_mi = MI / 2, gb_mi = MI / 32, rb_h = H / 2, gb_h = H / 32;
#ifdef COLI_VULKAN
@@ -1088,7 +1098,7 @@ static void moe(Model *m, int li, const float *xn, int n, float *out) {
memcpy(dst + (size_t)i * MI + o0, scratch + (size_t)i * rows, (size_t)rows * sizeof(float));
if (scratch != tmp) free(scratch);
}
- for (size_t i = 0; i < (size_t)total * MI; i++) g[i] = silu(g[i]) * u[i];
+ for (size_t i = (size_t)from * MI; i < (size_t)total * MI; i++) g[i] = silu(g[i]) * u[i];
#ifdef COLI_VULKAN
vk_experts_down(m, li, uni + b, nb, first, g, y, vk_gu, vk_dn);
#endif
@@ -1111,16 +1121,27 @@ static void moe(Model *m, int li, const float *xn, int n, float *out) {
memcpy(dst + (size_t)i * H + o0, scratch + (size_t)i * rows, (size_t)rows * sizeof(float));
if (scratch != tmp) free(scratch);
}
- /* the weighted sum, expert by expert in the order they were chosen */
- for (int j = 0; j < nb; j++)
- for (int i = first[j]; i < first[j + 1]; i++) {
- float *o = out + (size_t)row[i] * H;
- const float *yy = y + (size_t)i * H;
- for (int d = 0; d < H; d++) o[d] += rw[i] * yy[d];
- }
m->t_expert += now_s() - t0;
}
- free(first);
+ /* The weighted sum, row by row, each row's experts in the order ITS router
+ * chose them. A row's sum used to follow the block's first-use order, which
+ * depends on the rows that happen to share the block: the same position
+ * came out with different low bits in a 64-row prefill block and in a
+ * one-row resume, so a prompt resumed from a prefix was not the prompt
+ * computed cold. Now a row is a function of its own inputs and nothing else,
+ * whatever the block boundaries. One row (a decode step) adds in the same
+ * order as before: its first-use order IS its routing order. */
+ double t1 = now_s();
+ for (int t = 0; t < n; t++) {
+ float *o = out + (size_t)t * H;
+ for (int q = 0; q < K; q++) {
+ int i = pair[(size_t)t * K + q];
+ const float *yy = y + (size_t)i * H;
+ for (int d = 0; d < H; d++) o[d] += rw[i] * yy[d];
+ }
+ }
+ m->t_expert += now_s() - t1;
+ free(first); free(pair);
free(xg); free(g); free(u); free(y); free(row); free(rw); free(uni); free(sel); free(wt);
}
diff --git a/c/tests/mimo_tiny_harness.py b/c/tests/mimo_tiny_harness.py
index 2df3bf8b..1b5f5598 100644
--- a/c/tests/mimo_tiny_harness.py
+++ b/c/tests/mimo_tiny_harness.py
@@ -14,6 +14,11 @@ What is checked, all token-exact (argmax, no tolerances):
weights, an expert cache that evicts at every layer, prefill in blocks of 3
and of 1 (the sliding-window ring across blocks), buffered expert reads
- determinism (two identical runs, identical tokens)
+ - block invariance, BIT for bit: the logits of every position are the same
+ bytes whatever the prefill blocks (64, 3, 1) and the expert cache (one that
+ evicts at every layer), text and picture. A prompt resumed from a prefix or
+ a Brio photo is computed in other blocks than the same prompt cold, so this
+ is what lets the two agree to the last bit rather than to the argmax
- with the vision fixture (make_mimo_tiny.py --vision): the picture through
the tower, greedy and teacher-forced, f32 and native BF16, and the negated
picture giving the reference's own answer for it
@@ -126,6 +131,41 @@ def teacher_forced(binary, fixture, ids, env, vocab, image=None):
for p in range(len(ids))]
+def logits_bytes(binary, fixture, ids, env, image=None):
+ with tempfile.NamedTemporaryFile(suffix=".f32", delete=False) as tmp:
+ path = tmp.name
+ try:
+ run_engine(binary, fixture, ids, 0, env, extra_env={"MIMO_LOGITS": path},
+ image=image)
+ with open(path, "rb") as fh:
+ return fh.read()
+ finally:
+ os.unlink(path)
+
+
+def check_block_invariance(binary, fixture, reference):
+ """A row is a function of its own inputs, never of the rows that share its
+ prefill block: the logits must be the same BYTES for every block size."""
+ variants = {"blocks of 3": {"MIMO_CHUNK": "3"},
+ "one token at a time": {"MIMO_CHUNK": "1"},
+ "cache 4 (evicts at every layer)": {"MIMO_CAP": "4"}}
+ inputs = [(name, case["greedy_full_ids"], None)
+ for name, case in reference["cases"].items()]
+ if "image" in reference:
+ image = reference["image"]
+ inputs.append(("image", image["greedy_full_ids"],
+ (fixture / "patches.f32", image["grid_h"], image["grid_w"])))
+ for name, ids, image in inputs:
+ base = logits_bytes(binary, fixture, ids, reference["engine_env"], image)
+ for label, extra in variants.items():
+ env = dict(reference["engine_env"], **extra)
+ if logits_bytes(binary, fixture, ids, env, image) != base:
+ raise AssertionError(f"block invariance: case '{name}', {label}: the "
+ f"logits are not the bytes of the default blocks")
+ print(f"ok block invariance: {len(inputs)} cases x {len(variants)} variants, "
+ f"bit-identical logits")
+
+
def check_teacher_forcing(binary, fixture, reference, vocab):
for name, case in reference["cases"].items():
got = teacher_forced(binary, fixture, case["greedy_full_ids"],
@@ -268,6 +308,7 @@ def main():
check_greedy(args.binary, fixture, reference)
check_exact_variants(args.binary, fixture, reference)
check_determinism(args.binary, fixture, reference)
+ check_block_invariance(args.binary, fixture, reference)
check_teacher_forcing(args.binary, fixture, reference, config["vocab_size"])
check_oracle_bites(args.binary, fixture, reference,
config["n_routed_experts"])