Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
mimo: Brio mode -- logprobs read-out, pin photos with the window rings
JustVugg3b800e2bb415
diff --git a/c/Makefile b/c/Makefileindex adf7ac3d..51342701 100644--- a/c/Makefile+++ b/c/Makefile@@ -1282,6 +1282,14 @@ mimo-tiny-generate:mimo-tiny-check: mimo-tiny-generate mimo$(EXE)$(PYTHON) tests/mimo_tiny_harness.py --binary ./mimo$(EXE) --fixture ./mimo_tiny+# The same fixture served: KV prefix reuse, Brio photos (the windowed layers' rings,+# pictures, nested and stale photos), and /v1/brio and chat logprobs through the+# gateway. Every comparison is against a cold engine, to the last printed digit.+.PHONY: mimo-tiny-serve-check+mimo-tiny-serve-check: mimo-tiny-generate mimo$(EXE)+ $(PYTHON) -m unittest -v tests.test_mimo_prefix_serve tests.test_brio_serve.MimoBrioServe \+ tests.test_brio_api.MimoBrioEndToEnd+# Qwen3.8-Flash-Next text-only sibling. qwen38.c owns the direct checkpoint# loader and SERVE=1 protocol. With CUDA=1 it links the same expert tier as# qwen36 (fp8 streaming mode: hot experts get VRAM copies, the RAM LRU stays);diff --git a/c/mimo.c b/c/mimo.cindex 3580ffe1..c837b4bc 100644--- a/c/mimo.c+++ b/c/mimo.c@@ -57,6 +57,7 @@#include "quant.h"#include "omp_tune.h"#include "kv_prefix.h"+#include "pin_pool.h"#include "tok.h"#include "serve_codec.h"#include "serve_poll.h"@@ -1207,19 +1208,69 @@ static void forward(Model *m, const int *ids, int n, float *logits, int all_rowsfree(h); free(xn); free(tmp);}+/* The read-out of a prefill, for the logprobs channel (SUBMIT logprobs=k): one+ * ECHO frame per prompt position this prefill feeds, in position order, before+ * any DATA. The logits at position p predict the token at p + 1, so the frame+ * of a position carries the log-probability of the token that is actually+ * there, over the whole vocabulary, and not only when it is among the top k:+ * that is what scores an option the model would never have written.+ *+ * The first position fed has no predictor among the rows computed here. On a+ * cold prompt it is position 0, which has nothing to condition on, and its+ * frame carries " nan 0". Resumed from a photo, `first` is the photo's own last+ * logits, which predict exactly that token. There is no third case: a prompt+ * read out never resumes from a live prefix, because the positions before it+ * would have no frame at all (see the serve loop).+ *+ * This is its own argument and not a mode of something else: it asks for every+ * row's logits and changes nothing about how the rows are computed. */+typedef struct {+ const char *id;+ int k;+ Tok *tok;+ const float *first;+} Echo;++static void echo_frame(const Echo *e, int pos, int token, const float *lo, int vocab) {+ char tail[1024], piece[512];+ coli_logprob_tail(tail, sizeof(tail), lo, vocab, token, e->k);+ int n = tok_decode(e->tok, &token, 1, piece, (int)sizeof(piece));+ if (n < 0) n = 0;+ printf("ECHO %s %d %d%s\n", e->id, n, pos, tail);+ if (n > 0) fwrite(piece, 1, (size_t)n, stdout);+ fputc('\n', stdout);+ fflush(stdout);+}+/* A prompt in blocks of MIMO_CHUNK rows (default 64): the last block's last- * row (or every row, for the teacher-forcing dump) comes back in `logits`. */+ * row (or every row, for the teacher-forcing dump) comes back in `logits`.+ * With `echo`, every row of every block goes through the head and is read out+ * as it comes; only one block of logits is ever held, not the prompt's. */static void prefill(Model *m, const int *ids, int n, float *logits, float *all_logits,- const ImageRows *img) {+ const ImageRows *img, const Echo *echo) {int chunk = env_int("MIMO_CHUNK", 64);if (chunk < 1) chunk = 1;- int used = 0;+ int used = 0, V = m->c.vocab;+ float *rows = NULL;+ if (echo && !all_logits) {+ rows = xmalloc((size_t)(n < chunk ? n : chunk) * V * sizeof(float), "read-out logits");+ echo_frame(echo, m->pos, ids[0], echo->first, V);+ }for (int b = 0; b < n; b += chunk) {int nb = n - b < chunk ? n - b : chunk;- if (all_logits) forward(m, ids + b, nb, all_logits + (size_t)b * m->c.vocab, 1, img, &used);+ if (all_logits) forward(m, ids + b, nb, all_logits + (size_t)b * V, 1, img, &used);+ else if (rows) {+ int p0 = m->pos;+ forward(m, ids + b, nb, rows, 1, img, &used);+ for (int t = 0; t < nb; t++) {+ if (b + t + 1 < n) echo_frame(echo, p0 + t + 1, ids[b + t + 1], rows + (size_t)t * V, V);+ else if (logits) memcpy(logits, rows + (size_t)t * V, (size_t)V * sizeof(float));+ }+ }else forward(m, ids + b, nb, b + nb == n ? logits : NULL, 0, img, &used);}- if (all_logits && logits) memcpy(logits, all_logits + (size_t)(n - 1) * m->c.vocab, (size_t)m->c.vocab * sizeof(float));+ free(rows);+ if (all_logits && logits) memcpy(logits, all_logits + (size_t)(n - 1) * V, (size_t)V * sizeof(float));}/* ----------------------------------------------------------------- vision ---- */@@ -1288,14 +1339,150 @@ static int sample(const float *logits, int vocab, float temperature, float top_preturn pick;}+/* ---------------------------------------------------------------- photos ---- */++/* SUBMIT pin=1: a photo of the state at the end of this prompt, so that the+ * prompts that begin with it (the options of a closed question) resume from+ * there instead of computing it again. pin_pool.h keeps several, nested, and+ * restores the deepest one that is a strict prefix of the new prompt.+ *+ * WHAT A LAYER CARRIES BETWEEN TOKENS, and so what the photo must hold:+ *+ * full attention K/V rows indexed by position. Nothing after the photo+ * writes below it, so they stay where they are, and the+ * photo holds only the token ids; kv_prefix_holds() is what+ * says they are still the photo's rows (an unrelated prompt+ * in between rewrites them from position 0).+ * sliding window a RING of `rows` slots, position p in slot p % rows, and+ * the position each slot holds. Every option overwrites the+ * ring slots of the photo's last positions -- its own token at+ * P lands where P - rows was -- so after one option the ring+ * no longer describes the photo. The ring, keys, values and+ * positions, is copied into the photo and copied back.+ * the position m->pos, which is the photo's length.+ *+ * Nothing else carries over: the sinks are weights, the router has no state,+ * and the expert cache changes the time, never the numbers. On Flash the rings+ * of the 39 windowed layers are 128 x 8 x (192 + 128) floats each, about 50 MB+ * a photo, so COLI_PIN_SLOTS (default 4) is what bounds the memory. */+typedef struct {+ int n_layers;+ int rows[MIMO_MAX_LAYERS]; /* 0: a full-attention layer */+ size_t kd[MIMO_MAX_LAYERS], vd[MIMO_MAX_LAYERS];+ float *K[MIMO_MAX_LAYERS], *V[MIMO_MAX_LAYERS];+ int *ring_pos[MIMO_MAX_LAYERS];+} PinState;++static ColiPinPool g_pins;++static void pin_state_free(void *v) {+ PinState *st = (PinState *)v;+ if (!st) return;+ for (int li = 0; li < st->n_layers; li++) { free(st->K[li]); free(st->V[li]); free(st->ring_pos[li]); }+ free(st);+}++/* Copies the rings into `st` (allocated on first use). NULL when memory runs+ * out: a photo is an optimisation and never the reason a request fails. */+static PinState *pin_state_save(Model *m, PinState *st) {+ Cfg *c = &m->c;+ if (st && st->n_layers != c->n_layers) { pin_state_free(st); st = NULL; }+ if (!st) {+ st = calloc(1, sizeof(*st));+ if (!st) return NULL;+ st->n_layers = c->n_layers;+ for (int li = 0; li < c->n_layers; li++) {+ if (!c->swa[li]) continue;+ Layer *l = &m->L[li];+ st->rows[li] = l->rows;+ st->kd[li] = (size_t)l->rows * c->kv_heads[1] * c->head_dim[1];+ st->vd[li] = (size_t)l->rows * c->kv_heads[1] * c->v_dim[1];+ st->K[li] = malloc(st->kd[li] * sizeof(float));+ st->V[li] = malloc(st->vd[li] * sizeof(float));+ st->ring_pos[li] = malloc((size_t)l->rows * sizeof(int));+ if (!st->K[li] || !st->V[li] || !st->ring_pos[li]) { pin_state_free(st); return NULL; }+ }+ }+ for (int li = 0; li < c->n_layers; li++) {+ if (!st->rows[li]) continue;+ Layer *l = &m->L[li];+ memcpy(st->K[li], l->K, st->kd[li] * sizeof(float));+ memcpy(st->V[li], l->V, st->vd[li] * sizeof(float));+ memcpy(st->ring_pos[li], l->ring_pos, (size_t)l->rows * sizeof(int));+ }+ return st;+}++static int pin_state_load(Model *m, const PinState *st) {+ Cfg *c = &m->c;+ if (!st || st->n_layers != c->n_layers) return 0;+ for (int li = 0; li < c->n_layers; li++)+ if (st->rows[li] != (c->swa[li] ? m->L[li].rows : 0)) return 0;+ for (int li = 0; li < c->n_layers; li++) {+ if (!st->rows[li]) continue;+ Layer *l = &m->L[li];+ memcpy(l->K, st->K[li], st->kd[li] * sizeof(float));+ memcpy(l->V, st->V[li], st->vd[li] * sizeof(float));+ memcpy(l->ring_pos, st->ring_pos[li], (size_t)l->rows * sizeof(int));+ }+ return 1;+}++/* Only a request that asks for it takes a photo, and it goes into the pool,+ * where no other request writes: an ordinary prompt can make a photo stale+ * (it rewrites the rows the photo stands on), never replace it. */+static void pin_save(Model *m, const int *ids, int n, const float *logits) {+ coli_pin_pool_init(&g_pins, m->c.vocab);+ ColiPin *k = coli_pin_store(&g_pins, ids, n, logits);+ if (!k) return;+ PinState *st = pin_state_save(m, (PinState *)k->state);+ if (!st) { k->state = NULL; k->len = 0; return; } /* without its rings it would lie */+ k->state = st;+ fprintf(stderr, "[PIN] photo of %d tokens\n", n);+ fflush(stderr);+}++/* The deepest photo that is a strict prefix of this prompt and whose rows the+ * state still holds, put back: the rings, the position and the record. Returns+ * its length and its logits (the predictor of the first fresh token), or 0.+ * A photo whose rows are gone cannot help any more: it is released, memory and+ * all, and the next deepest is tried instead of giving up. */+static int pin_restore(Model *m, const int *ids, int n, const float **logit) {+ *logit = NULL;+ int s = coli_pin_best(&g_pins, ids, n);+ while (s >= 0) {+ ColiPin *k = &g_pins.slot[s];+ if (k->logit && kv_prefix_holds(&m->kvp, k->ids, k->len) &&+ pin_state_load(m, (const PinState *)k->state)) {+ m->pos = k->len;+ kv_prefix_clear(&m->kvp);+ kv_prefix_record(&m->kvp, k->ids, 0, k->len);+ *logit = k->logit;+ coli_pin_touch(&g_pins, s);+ return k->len;+ }+ fprintf(stderr, "[PIN] released the photo of %d tokens: the state no longer holds it\n", k->len);+ k->len = 0;+ pin_state_free(k->state);+ k->state = NULL;+ s = coli_pin_best(&g_pins, ids, n);+ }+ return 0;+}+/* ------------------------------------------------------------------ serve ---- */+/* The 7th numeric field (the byte count of a payload extension) is accepted, but+ * only as 0: the gateway puts it ahead of the logprobs= key on chat requests, and+ * this engine takes no grammar, so any other value is still a bad header. */static const ColiServeWireProfile mimo_wire = {.max_header_bytes = 511,.max_payload_bytes = 1u << 26,+ .max_extension_bytes = 0,.max_tokens = 1 << 20,.require_exact_lf = 1,.require_finite_sampling = 1,+ .allow_extension_bytes = 1,};static void serve_line(const char *format, ...) {@@ -1348,12 +1535,14 @@ static void serve_hits(Model *m) {free(hex); free(bitmap);}-/* max_tokens is a ceiling; generation needs at least one free position */-static int serve_budget(int prompt, int requested, int context) {+/* max_tokens is a ceiling; generation needs at least one free position. A+ * read-only request (max_tokens=0, legal only with logprobs) generates nothing,+ * so its prompt may fill the context. */+static int serve_budget(int prompt, int requested, int context, int logprobs) {if (prompt < 1 || prompt > context) return -1;- int budget = requested > 0 ? requested : 256;+ int budget = requested > 0 ? requested : (logprobs > 0 ? 0 : 256);int room = context - prompt;- if (room == 0) return -1;+ if (budget > 0 && room == 0) return -1;return budget < room ? budget : room;}@@ -1409,7 +1598,7 @@ static void serve_loop(Model *m, Tok *tokenizer, const char *dir) {uint64_t hits0 = m->hits, miss0 = m->miss, fw0 = m->forwards;int n_prompt = tok_encode(tokenizer, (const char *)command.payload, (int)command.payload_bytes,ids, m->ctx + 1);- int budget = serve_budget(n_prompt, command.max_tokens, m->ctx);+ int budget = serve_budget(n_prompt, command.max_tokens, m->ctx, command.logprobs);if (budget < 0) {char message[160];snprintf(message, sizeof(message), "CONTEXT_EXCEEDED prompt_tokens=%d requested=%d capacity=%d",@@ -1438,13 +1627,38 @@ static void serve_loop(Model *m, Tok *tokenizer, const char *dir) {continue;}}- /* A chat client resends the whole transcript: if this prompt begins with- * the ids the state was built from, only the tail is fed. An image- * refuses it -- the pad ids do not describe the picture. */+ /* Where this prompt starts. A chat client resends the whole transcript:+ * if this prompt begins with the ids the state was built from, only the+ * tail is fed (COLI_KV_PREFIX=0 turns that off). A photo (SUBMIT pin=1+ * earlier) resumes a prompt that begins with it.+ *+ * With the read-out on (logprobs=k) only a photo will do. The read-out+ * owes a frame to every position it does not inherit a predictor for,+ * and a live prefix comes with none: the frames before it would simply+ * be missing. So such a prompt resumes from the deepest photo, which+ * carries the logits that predict its first fresh token, or is read out+ * from position 0. It resumes at the photo even when the live state+ * shares more: two options share the text before them, and stopping at+ * that would leave the first option token without its predictor.+ *+ * An image refuses both: the pad ids do not describe the picture. */int prefix_on = env_int("COLI_KV_PREFIX", 1) != 0;- int reuse = (!pending && prefix_on) ? kv_prefix_reuse(&m->kvp, ids, n_prompt) : 0;+ int reuse = 0;+ const float *pin_logit = NULL;+ if (!pending) {+ if (command.logprobs > 0) reuse = pin_restore(m, ids, n_prompt, &pin_logit);+ else {+ reuse = prefix_on ? kv_prefix_reuse(&m->kvp, ids, n_prompt) : 0;+ if (!reuse) reuse = pin_restore(m, ids, n_prompt, &pin_logit);+ }+ }if (getenv("COLI_PREFIX_LOG"))- fprintf(stderr, "[PREFIX] %s %d of %d prompt tokens\n", reuse ? "reusing" : "no reuse,", reuse, n_prompt);+ fprintf(stderr, "[PREFIX] %s %d of %d prompt tokens%s\n", reuse ? "reusing" : "no reuse,", reuse,+ n_prompt, pin_logit ? " (photo)" : "");+ if (pin_logit) {+ fprintf(stderr, "[PIN] resumed from the photo of %d tokens, %d to prefill\n", reuse, n_prompt - reuse);+ fflush(stderr);+ }if (!reuse) model_reset(m);coli_serve_write_accept(stdout, command.id, n_prompt);if (pending) {@@ -1452,8 +1666,16 @@ static void serve_loop(Model *m, Tok *tokenizer, const char *dir) {free(pending); pending = NULL;}ImageRows img = { image, image_rows };- prefill(m, ids + reuse, n_prompt - reuse, logits, NULL, image ? &img : NULL);- if (image) { kv_prefix_taint(&m->kvp); free(image); }+ Echo echo = { command.id, command.logprobs, tokenizer, pin_logit };+ prefill(m, ids + reuse, n_prompt - reuse, logits, NULL, image ? &img : NULL,+ command.logprobs > 0 ? &echo : NULL);+ if (image) kv_prefix_taint(&m->kvp);+ if (command.pin) {+ /* a photo is described by its ids, and these do not describe the picture */+ if (image) fprintf(stderr, "[PIN] no photo: the prompt carries a picture\n");+ else pin_save(m, ids, n_prompt, logits);+ }+ free(image);int emitted = 0, limited = 1, cancelled = 0, done_early = 0;char piece[512];while (emitted < budget && !cancelled && !done_early) {@@ -1462,7 +1684,14 @@ static void serve_loop(Model *m, Tok *tokenizer, const char *dir) {for (int i = 0; i < n_eos; i++) if (token == eos[i]) stop = 1;if (stop) { limited = 0; break; }int written = tok_decode(tokenizer, &token, 1, piece, (int)sizeof(piece));- if (written > 0) coli_serve_write_data(stdout, command.id, piece, (size_t)written);+ if (written > 0) {+ if (command.logprobs > 0) {+ /* the distribution the token was drawn from: sample() only reads it */+ char tail[1024];+ coli_logprob_tail(tail, sizeof(tail), logits, c->vocab, token, command.logprobs);+ coli_serve_write_data_lp(stdout, command.id, piece, (size_t)written, tail);+ } else coli_serve_write_data(stdout, command.id, piece, (size_t)written);+ }emitted++;while (coli_serve_stdin_ready()) {ColiServeCommand control;@@ -1474,7 +1703,7 @@ static void serve_loop(Model *m, Tok *tokenizer, const char *dir) {coli_serve_command_dispose(&control);}if (cancelled || done_early || emitted >= budget) break;- prefill(m, &token, 1, logits, NULL, NULL);+ prefill(m, &token, 1, logits, NULL, NULL, NULL);}if (cancelled) {coli_serve_write_error(stdout, command.id, "CANCELLED");@@ -1631,7 +1860,7 @@ int main(int argc, char **argv) {}ImageRows img = { image, image_rows };double tp = now_s();- prefill(m, ids, n, logits, all, image ? &img : NULL);+ prefill(m, ids, n, logits, all, image ? &img : NULL, NULL);double prefill_s = now_s() - tp;if (dump) {FILE *f = fopen(dump, "wb");@@ -1656,7 +1885,7 @@ int main(int argc, char **argv) {fflush(stdout);produced++;if (stop && have_tok && !ids_text) break;- if (g + 1 < ngen) prefill(m, &token, 1, logits, NULL, NULL);+ if (g + 1 < ngen) prefill(m, &token, 1, logits, NULL, NULL, NULL);}printf("\n");fflush(stdout);diff --git a/c/tests/mimo_serve_fixture.py b/c/tests/mimo_serve_fixture.pynew file mode 100644index 00000000..d873f0a8--- /dev/null+++ b/c/tests/mimo_serve_fixture.py@@ -0,0 +1,81 @@+"""The tiny MiMo fixture, served: the same weights plus a tokenizer.++tools/make_mimo_tiny.py ships no tokenizer.json on purpose -- the vendor oracle+(tests/mimo_tiny_harness.py) reads raw token ids on stdout and refuses a fixture+that has one. Serve mode goes through text, so the serve tests work on a copy:+the fixture's files plus a one-token-per-byte tokenizer, with the three vision+markers as added tokens at the ids the fixture's config gives them, so an image+prompt can be written as text.++The copy is made once per process, in a temporary directory that is removed at+exit, and the source fixture is never touched.+"""+import atexit+import json+import os+import shutil+import sys+import tempfile+from pathlib import Path++sys.path.insert(0, str(Path(__file__).resolve().parent))+from prefix_serve_harness import byte_level_vocab # noqa: E402++HERE = Path(__file__).resolve().parent.parent+ENGINE = Path(os.environ.get("MIMO_BINARY",+ HERE / ("mimo.exe" if sys.platform == "win32" else "mimo")))+FIXTURE = Path(os.environ.get("MIMO_TINY", HERE / "mimo_tiny"))++IMAGE_PAD = "<|image_pad|>"+VISION_START = "<|vision_start|>"+VISION_END = "<|vision_end|>"++_served = None+++def available():+ return ENGINE.is_file() and (FIXTURE / "config.json").is_file()+++def has_vision():+ return (FIXTURE / "patches.f32").is_file() and (FIXTURE / "image.json").is_file()+++def served_fixture():+ """The serve-mode copy of FIXTURE, built on first use."""+ global _served+ if _served is not None:+ return _served+ root = Path(tempfile.mkdtemp(prefix="mimo-serve-"))+ atexit.register(shutil.rmtree, root, True)+ out = root / "fixture"+ shutil.copytree(FIXTURE, out)+ config = json.loads((FIXTURE / "config.json").read_text())+ added = []+ for key, text in (("image_token_id", IMAGE_PAD), ("vision_start_token_id", VISION_START),+ ("vision_end_token_id", VISION_END)):+ if isinstance(config.get(key), int):+ added.append({"id": config[key], "content": text, "special": True})+ (out / "tokenizer.json").write_text(json.dumps({+ "version": "1.0",+ "added_tokens": added,+ "model": {"type": "BPE", "vocab": byte_level_vocab(256), "merges": []},+ }), encoding="utf-8")+ _served = out+ return out+++def image_payload():+ """(patch bytes, grid_h, grid_w, pad count) of the fixture's picture."""+ meta = json.loads((FIXTURE / "image.json").read_text())+ return ((FIXTURE / "patches.f32").read_bytes(), meta["grid_h"], meta["grid_w"],+ meta["tokens"])+++def engine_env(**extra):+ """What every serve test runs the engine with: the served copy, a small+ context, two threads and no OpenMP re-exec."""+ env = {"SNAP": str(served_fixture()), "SERVE": "1", "CTX": "256",+ "OMP_NUM_THREADS": "2", "COLI_NO_OMP_TUNE": "1"}+ env.update(extra)+ return envdiff --git a/c/tests/test_brio_serve.py b/c/tests/test_brio_serve.pyindex 86f6bca9..76ce9992 100644--- a/c/tests/test_brio_serve.py+++ b/c/tests/test_brio_serve.py@@ -14,17 +14,25 @@ byte-identical frames it produced before the channel existed. Chat is the modeeveryone uses; the scoring channel is opt-in and must stay that way. The checkis a byte diff of the frames, not a reading of the text.-Runs against the CONVERTED tiny olmoe fixture, olmoe_tiny_c, that the-`olmoe-tiny-check` CI job builds and runs this module on, so it needs no-checkpoint. From c/:+The scenarios are a contract, written once and run per engine:++- OLMoE, against the CONVERTED tiny fixture olmoe_tiny_c that the+ `olmoe-tiny-check` CI job builds and runs this module on. From c/:python3 tools/make_olmoe_tiny.py --output olmoe_tinypython3 tools/convert_olmoe_merged.py --model olmoe_tiny --out olmoe_tiny_cpython3 tools/make_edge_tiny_tokenizer.py --vocab-size 128 olmoe_tiny_c-olmoe_tiny is the HF source checkpoint the engine cannot load, and it has a-config.json too: the guard below looks for the tokenizer, which only the-converted fixture has, so a missing fixture skips instead of failing (#1742).+ olmoe_tiny is the HF source checkpoint the engine cannot load, and it has a+ config.json too: the guard below looks for the tokenizer, which only the+ converted fixture has, so a missing fixture skips instead of failing (#1742).++- MiMo-V2.6, against the tiny fixture of `make mimo-tiny-generate`, served+ through tests/mimo_serve_fixture.py (a copy with a byte tokenizer). Its+ windowed layers and its pictures have their own cases in+ tests/test_mimo_prefix_serve.py.++Neither needs a checkpoint, and an engine that is not built skips."""import jsonimport os@@ -33,6 +41,9 @@ import sysimport unittestfrom pathlib import Path+sys.path.insert(0, str(Path(__file__).resolve().parent))+import mimo_serve_fixture # noqa: E402+HERE = Path(__file__).resolve().parent.parentENGINE = HERE / ("olmoe.exe" if sys.platform == "win32" else "olmoe")FIXTURE = Path(os.environ.get("OLMOE_TINY", HERE / "olmoe_tiny_c"))@@ -40,16 +51,20 @@ PROMPT = "Context: the release is late and the tests are red.\nQuestion: ship?\nOPTION = " no"+def olmoe_engine():+ return Engine(ENGINE, [], dict(SNAP=str(FIXTURE), SERVE="1",+ TOK=str(FIXTURE / "tokenizer.json"),+ COLI_NO_OMP_TUNE="1", OMP_NUM_THREADS="2"))++class Engine:"""One serve-mode engine, driven over the wire the gateway uses."""- def __init__(self):- env = dict(os.environ, SNAP=str(FIXTURE), SERVE="1",- TOK=str(FIXTURE / "tokenizer.json"),- COLI_NO_OMP_TUNE="1", OMP_NUM_THREADS="2")- self.p = subprocess.Popen([str(ENGINE)], env=env, stdin=subprocess.PIPE,- stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,- bufsize=0)+ def __init__(self, binary, argv, env):+ env = dict(os.environ, **env)+ self.p = subprocess.Popen([str(binary)] + [str(a) for a in argv], env=env,+ stdin=subprocess.PIPE, stdout=subprocess.PIPE,+ stderr=subprocess.DEVNULL, bufsize=0)while True:line = self.readline()if line is None:@@ -101,6 +116,7 @@ class Engine:passself.p.terminate()self.p.wait(timeout=30)+ self.p.stdout.close()def strip_volatile(frames):@@ -110,14 +126,15 @@ def strip_volatile(frames):if not f.startswith(("STAT", "PROF", "DONE", "TIERS", "HWINFO", "EMAP", "HITS"))][email protected]((FIXTURE / "tokenizer.json").is_file(),- "converted tiny olmoe fixture is absent: see the module "- "docstring (make_olmoe_tiny.py, convert_olmoe_merged.py, "- "make_edge_tiny_tokenizer.py)")[email protected](ENGINE.is_file(), "olmoe engine is not built")-class BrioServe(unittest.TestCase):+class BrioServeContract:+ """The scenarios. Mix into a TestCase that provides engine(); deliberately not+ a TestCase itself, or unittest would run them once more with no engine."""++ def engine(self): # pragma: no cover - overridden+ raise NotImplementedError+def test_a_snapshot_scores_exactly_like_a_cold_recompute(self):- warm = Engine()+ warm = self.engine()try:_, prefix = warm.submit(1, PROMPT, 0, " logprobs=1 pin=1")_, pinned = warm.submit(2, PROMPT + OPTION, 0, " logprobs=1")@@ -126,7 +143,7 @@ class BrioServe(unittest.TestCase):self.assertTrue(prefix, "the warm pass read nothing")tail_starts = max(prefix) + 1- cold = Engine()+ cold = self.engine()try:_, fresh = cold.submit(1, PROMPT + OPTION, 0, " logprobs=1")finally:@@ -170,7 +187,7 @@ class BrioServe(unittest.TestCase):cold recompute."""other = "Context: the invoice is overdue by ninety days.\nQuestion: escalate?\nAnswer:"- engine = Engine()+ engine = self.engine()try:_, prefix = engine.submit(1, PROMPT, 0, " logprobs=1 pin=1")engine.submit(2, other, 4) # overwrites the record with other ids@@ -180,7 +197,7 @@ class BrioServe(unittest.TestCase):self.assertTrue(prefix, "the warm pass read nothing")tail_starts = max(prefix) + 1- cold = Engine()+ cold = self.engine()try:_, fresh = cold.submit(1, PROMPT + OPTION, 0, " logprobs=1")finally:@@ -199,7 +216,7 @@ class BrioServe(unittest.TestCase):def test_the_snapshot_only_reads_the_fresh_tail(self):"""The point of the snapshot: the shared prefix is not read again."""- engine = Engine()+ engine = self.engine()try:_, prefix = engine.submit(1, PROMPT, 0, " logprobs=1 pin=1")_, tail = engine.submit(2, PROMPT + OPTION, 0, " logprobs=1")@@ -217,7 +234,7 @@ class BrioServe(unittest.TestCase):of every frame, so the same id has to be reused, and a second turn on alive engine would also start from a different cached state."""def frames_for(extension):- engine = Engine()+ engine = self.engine()try:frames, _ = engine.submit(1, PROMPT, 4, extension)finally:@@ -244,7 +261,7 @@ class BrioServe(unittest.TestCase):def test_read_only_generates_nothing(self):"""max_tokens=0 means read the prompt and stop: no DATA frame at all."""- engine = Engine()+ engine = self.engine()try:frames, _ = engine.submit(1, PROMPT, 0, " logprobs=1")finally:@@ -256,7 +273,7 @@ class BrioServe(unittest.TestCase):def test_read_only_needs_the_channel(self):"""max_tokens=0 without logprobs stays a malformed request."""- engine = Engine()+ engine = self.engine()try:frames, _ = engine.submit(1, PROMPT, 0)finally:@@ -265,5 +282,23 @@ class BrioServe(unittest.TestCase):"max_tokens=0 was accepted without the logprobs channel")[email protected]((FIXTURE / "tokenizer.json").is_file(),+ "converted tiny olmoe fixture is absent: see the module "+ "docstring (make_olmoe_tiny.py, convert_olmoe_merged.py, "+ "make_edge_tiny_tokenizer.py)")[email protected](ENGINE.is_file(), "olmoe engine is not built")+class BrioServe(BrioServeContract, unittest.TestCase):+ def engine(self):+ return olmoe_engine()++[email protected](mimo_serve_fixture.available(),+ "mimo is not built or the tiny MiMo fixture is absent "+ "(make mimo mimo-tiny-generate)")+class MimoBrioServe(BrioServeContract, unittest.TestCase):+ def engine(self):+ return Engine(mimo_serve_fixture.ENGINE, ["8"], mimo_serve_fixture.engine_env())++if __name__ == "__main__":unittest.main()diff --git a/c/tests/test_mimo_prefix_serve.py b/c/tests/test_mimo_prefix_serve.pynew file mode 100644index 00000000..545d75ed--- /dev/null+++ b/c/tests/test_mimo_prefix_serve.py@@ -0,0 +1,331 @@+"""KV prefix reuse and Brio photos on MiMo-V2.6, against the tiny fixture.++MiMo is the engine where a photo is more than token ids. Most of its layers+attend a sliding window, and a windowed layer keeps its keys in a RING: position+p in slot p % window. The photo of a prompt of P tokens holds the ring as it was+at P; the first option then writes its own positions into the slots of P - window+and onwards, so after one option the ring no longer describes the photo. A photo+that kept only the ids -- enough for the full-attention layers, whose rows below+P nobody touches -- would hand the second option a ring from the first. The+fixture's window is 8 tokens, and every prompt below outgrows it.++What is checked, all against a cold engine and to the last printed digit of+every ECHO frame (the log-probability of the token and the whole top-k):++ - the shared prefix-reuse contract (tests/prefix_serve_harness.py);+ - options that cross the window resume from the photo exactly, one after the+ other, with an ordinary chat turn in between that must not take the photo+ away, and each one PROCESSES ONLY ITS OWN TOKENS: its read-out starts at the+ photo's end and the engine says it prefilled the option and nothing more.+ The numbers alone would not show a photo that silently fell back to a full+ recompute; the positions and the count do;+ - nested photos (a state, then a question) and a sibling question that+ overwrites the deeper one's rows: the engine falls back to the shallower;+ - a photo whose rows an unrelated prompt overwrote is released, not restored;+ - pictures: a prompt with an image never resumes from a photo, and a prompt+ with an image takes none, because token ids do not describe the picture;+ - the read-out covers every position from 0 unless a photo covers the prefix,+ which is what the gateway's `echo` needs, and never resumes from a live+ prefix;+ - asking for logprobs changes no generated byte;+ - a read-only request may fill the context, a generating one may not.++Needs the built engine and the fixture: `make mimo mimo-tiny-generate`.+"""+import json+import math+import re+import sys+import unittest+from pathlib import Path++sys.path.insert(0, str(Path(__file__).resolve().parent))+import mimo_serve_fixture as fixture # noqa: E402+from prefix_serve_harness import PrefixReuseContract, ServeEngine # noqa: E402++STATE = "Context: the release is late, the tests are red and the changelog is empty.\n"+QUESTION = STATE + "Question: should we ship it today?\nAnswer:"+SIBLING = STATE + "Question: who has to sign it off?\nAnswer:"+OPTIONS = (" only after the tests pass and the changelog says what changed",+ " yes",+ " no, not today",+ " only after the tests pass and the changelog says what changed")+++def window():+ config = json.loads((fixture.FIXTURE / "config.json").read_text())+ return int(config["sliding_window"])+++class Client(ServeEngine):+ """A serve engine read frame by frame: the ECHO frames are what is compared."""++ def __init__(self, log_prefix=True, **env):+ super().__init__(fixture.ENGINE, ["8"], fixture.engine_env(**env),+ log_prefix=log_prefix)++ def image(self, rid):+ payload, grid_h, grid_w, _pads = fixture.image_payload()+ self.p.stdin.write(f"IMAGE {rid} {len(payload)} {grid_h} {grid_w}\n".encode()+ + payload + b"\n")+ self.p.stdin.flush()++ def submit(self, rid, prompt, max_tokens=0, ext=" logprobs=3"):+ """Returns {kind: ..., "echo": {pos: tail}, "data": [(bytes, tail)], "end": line}.++ A tail is the frame's numeric part after the position, compared as text:+ the log-probability and every top-k candidate, as the engine printed them."""+ data = prompt.encode()+ self.p.stdin.write(f"SUBMIT {rid} 0 {len(data)} {max_tokens} 0 1{ext}\n".encode()+ + data + b"\n")+ self.p.stdin.flush()+ out = {"echo": {}, "data": [], "accept": None, "end": None}+ while True:+ line = self.p.stdout.readline()+ if not line:+ raise RuntimeError("engine closed mid-request:\n" + "".join(self._err)[-2000:])+ fields = line.decode("latin-1").split()+ if not fields:+ continue+ if fields[0] in ("ECHO", "DATA"):+ payload = self.p.stdout.read(int(fields[2]))+ self.p.stdout.read(1)+ if fields[0] == "ECHO":+ pos = int(fields[3])+ if pos in out["echo"]:+ raise AssertionError(f"position {pos} read out twice")+ out["echo"][pos] = " ".join(fields[4:])+ else:+ out["data"].append((payload, " ".join(fields[3:])))+ elif fields[0] == "ACCEPT":+ out["accept"] = int(fields[2])+ elif fields[0] in ("DONE", "ERROR"):+ out["end"] = fields+ return out+++def resumed(log):+ """[(photo length, tokens prefilled)] in order, from the engine's own account."""+ return [(int(a), int(b)) for a, b in+ re.findall(r"\[PIN\] resumed from the photo of (\d+) tokens, (\d+) to prefill", log)]++[email protected](fixture.available(),+ "mimo is not built or the tiny MiMo fixture is absent "+ "(make mimo mimo-tiny-generate)")+class MimoPrefixServeTest(PrefixReuseContract, unittest.TestCase):+ # The fixture's vocabulary (320) is wider than the byte tokenizer the served+ # copy carries, so a generated reply does not always survive decode and+ # re-encode: the replay scenario may then legitimately not reuse.+ REPLAY_REUSES = False++ def spawn(self, log_prefix=True, reuse=True):+ return ServeEngine(fixture.ENGINE, ["8"], fixture.engine_env(),+ log_prefix=log_prefix, reuse=reuse)++[email protected](fixture.available(),+ "mimo is not built or the tiny MiMo fixture is absent "+ "(make mimo mimo-tiny-generate)")+class MimoBrioPhotoTest(unittest.TestCase):+ def client(self, **env):+ c = Client(**env)+ self.addCleanup(c.p.stdout.close)+ self.addCleanup(c.p.stderr.close)+ return c++ def cold(self, prompt, image=False):+ """The prompt read out by a fresh engine that never saw a photo."""+ c = self.client(log_prefix=False)+ if image:+ c.image(1)+ got = c.submit(1, prompt)+ c.close()+ return got++ def assertSameReadOut(self, got, cold, positions, what):+ for pos in positions:+ self.assertIn(pos, cold["echo"], f"{what}: the cold run did not read position {pos}")+ self.assertIn(pos, got["echo"], f"{what}: position {pos} was not read out")+ self.assertEqual(got["echo"][pos], cold["echo"][pos],+ f"{what}: position {pos} differs from a cold recompute "+ f"-- the state resumed is not the state it claims to be")++ def test_options_that_cross_the_window_resume_from_the_photo_exactly(self):+ w = window()+ self.assertGreater(len(QUESTION.encode()), 2 * w,+ "the photo must outgrow the window, or its ring never wraps")+ self.assertGreater(len(OPTIONS[0].encode()), 4 * w,+ "the long option must slide the window past the photo")+ eng = self.client()+ photo = eng.submit(1, QUESTION, ext=" logprobs=3 pin=1")+ n = photo["accept"]+ self.assertEqual(n, len(QUESTION.encode()))+ warm = []+ for i, option in enumerate(OPTIONS):+ warm.append(eng.submit(10 + i, QUESTION + option))+ if i == 1:+ # An ordinary chat turn that extends the photo. The live state holds+ # the option before it, so it resumes from the photo too -- and it+ # must not take the photo away from the options after it.+ chat = eng.submit(20, QUESTION + " maybe", max_tokens=4, ext="")+ self.assertEqual(chat["end"][0], "DONE")+ self.assertEqual(chat["echo"], {}, "a chat turn was read out unasked")+ log = eng.close()++ self.assertIn("[PIN] photo of %d tokens" % n, log)+ self.assertEqual(log.count("[PIN] photo of"), 1, "only the pin=1 request takes a photo")+ lengths = [len(option.encode()) for option in OPTIONS]+ # The saving, from the engine's own account: every option resumed from the+ # photo and prefilled ITS tokens, not the shared prefix again.+ expected = [(n, k) for k in lengths]+ expected.insert(2, (n, len(" maybe"))) # the chat turn+ self.assertEqual(resumed(log), expected,+ "an option did not resume from the photo:\n" + log)+ for option, k, got in zip(OPTIONS, lengths, warm):+ with self.subTest(option=option):+ # ...and from the wire: the read-out starts exactly where the photo+ # ends -- the first option token predicted by the photo's own logits+ # -- and covers the option and nothing before it.+ self.assertEqual(sorted(got["echo"]), list(range(n, n + k)))+ self.assertFalse(any(t.startswith("nan") for t in got["echo"].values()),+ "an option token lost its predictor")+ cold = self.cold(QUESTION + option)+ self.assertEqual(sorted(cold["echo"]), list(range(0, n + k)),+ "a cold read-out covers every position")+ self.assertSameReadOut(got, cold, range(n, n + k), repr(option))++ def test_nested_photos_and_a_sibling_question(self):+ eng = self.client()+ s = eng.submit(1, STATE, ext=" logprobs=1 pin=1")["accept"]+ q = eng.submit(2, QUESTION, ext=" logprobs=1 pin=1")["accept"]+ first = eng.submit(3, QUESTION + OPTIONS[2])+ # Same state, another question: resumes from the state, and its prefill+ # overwrites the rows the deeper photo stood on.+ sibling_pin = eng.submit(4, SIBLING, ext=" logprobs=1 pin=1")+ sibling = eng.submit(5, SIBLING + " the release manager")+ back = eng.submit(6, QUESTION + OPTIONS[1])+ log = eng.close()++ self.assertEqual(sorted(first["echo"])[0], q)+ self.assertEqual(sorted(sibling_pin["echo"])[0], s)+ self.assertEqual(sorted(sibling["echo"])[0], sibling_pin["accept"])+ # The question's photo is stale now: released, and the state's taken instead+ # of starting over.+ self.assertIn("[PIN] released the photo of %d tokens" % q, log)+ self.assertEqual(sorted(back["echo"])[0], s)+ for got, prompt, start in ((first, QUESTION + OPTIONS[2], q),+ (sibling, SIBLING + " the release manager",+ sibling_pin["accept"]),+ (back, QUESTION + OPTIONS[1], s)):+ cold = self.cold(prompt)+ self.assertSameReadOut(got, cold, range(start, len(prompt.encode())), prompt)++ def test_a_photo_whose_rows_were_overwritten_is_not_restored(self):+ other = "Context: the invoice is overdue by ninety days.\nQuestion: escalate?\nAnswer:"+ eng = self.client()+ n = eng.submit(1, QUESTION, ext=" logprobs=1 pin=1")["accept"]+ eng.submit(2, other, max_tokens=4, ext="")+ after = eng.submit(3, QUESTION + OPTIONS[2])+ log = eng.close()+ self.assertIn("[PIN] released the photo of %d tokens" % n, log)+ self.assertEqual(resumed(log), [])+ total = len((QUESTION + OPTIONS[2]).encode())+ self.assertEqual(sorted(after["echo"]), list(range(total)),+ "with no photo the read-out starts over from position 0")+ self.assertSameReadOut(after, self.cold(QUESTION + OPTIONS[2]), range(1, total),+ "after a divergent prompt")++ @unittest.skipUnless(fixture.has_vision(), "the fixture carries no picture")+ def test_an_image_prompt_never_resumes_from_a_photo(self):+ """The prompt begins with the photo's ids, so the ids alone would say+ resume. They do not describe the picture, and the image rows are spliced+ by their order in the whole prompt: the engine starts over."""+ _payload, _h, _w, pads = fixture.image_payload()+ prompt = (QUESTION + " " + fixture.VISION_START + fixture.IMAGE_PAD * pads+ + fixture.VISION_END + " so:")+ eng = self.client()+ eng.submit(1, QUESTION, ext=" logprobs=1 pin=1")+ eng.image(2)+ got = eng.submit(2, prompt)+ log = eng.close()+ self.assertEqual(got["end"][0], "DONE", got["end"])+ self.assertEqual(resumed(log), [], "an image prompt resumed from a photo:\n" + log)+ self.assertEqual(sorted(got["echo"]), list(range(got["accept"])))+ cold = self.cold(prompt, image=True)+ self.assertSameReadOut(got, cold, range(1, got["accept"]), "image prompt")++ @unittest.skipUnless(fixture.has_vision(), "the fixture carries no picture")+ def test_a_prompt_with_a_picture_takes_no_photo(self):+ """pin=1 on a prompt with a picture: the photo would be filed under ids+ that do not describe the picture, and a later TEXT prompt with the same ids+ (the pads as plain tokens) would resume from a state that saw an image."""+ _payload, _h, _w, pads = fixture.image_payload()+ pictured = (STATE + fixture.VISION_START + fixture.IMAGE_PAD * pads+ + fixture.VISION_END + "\nAnswer:")+ eng = self.client()+ eng.image(1)+ eng.submit(1, pictured, ext=" logprobs=1 pin=1")+ text_only = eng.submit(2, pictured + OPTIONS[1])+ log = eng.close()+ self.assertIn("[PIN] no photo: the prompt carries a picture", log)+ self.assertNotIn("[PIN] photo of", log)+ self.assertEqual(resumed(log), [])+ total = text_only["accept"]+ self.assertEqual(sorted(text_only["echo"]), list(range(total)),+ "a text prompt resumed from a state that saw a picture")+ cold = self.cold(pictured + OPTIONS[1])+ self.assertSameReadOut(text_only, cold, range(1, total), "text after a picture")++ def test_the_read_out_covers_every_position_unless_a_photo_does(self):+ """What the gateway's `echo` needs: one frame per prompt position, the+ first carrying nan because nothing precedes it. A live prefix brings no+ predictor for its last position, so a read-out never resumes from one."""+ opening = "The capital of France is"+ eng = self.client()+ eng.submit(1, opening, max_tokens=1, ext="")+ got = eng.submit(2, opening + " a city, and the capital of Spain is")+ log = eng.close()+ self.assertNotIn("[PREFIX] reusing", log, "a read-out resumed from a live prefix")+ n = got["accept"]+ self.assertEqual(sorted(got["echo"]), list(range(n)))+ self.assertTrue(got["echo"][0].startswith("nan 0"), got["echo"][0])+ for pos in range(1, n):+ lp = float(got["echo"][pos].split()[0])+ self.assertTrue(math.isfinite(lp) and lp <= 0.0, (pos, lp))+ self.assertEqual(int(got["echo"][pos].split()[1]), 3, "the top-k asked for")++ def test_logprobs_change_no_generated_byte(self):+ prompt = "Streaming experts from disk means that the decode rate depends on"+ plain = self.client()+ a = plain.submit(1, prompt, max_tokens=12, ext="")+ plain.close()+ scored = self.client()+ b = scored.submit(1, prompt, max_tokens=12, ext=" logprobs=4")+ scored.close()+ self.assertTrue(a["data"], "nothing was generated")+ self.assertEqual([d for d, _t in a["data"]], [d for d, _t in b["data"]])+ self.assertEqual(a["end"][:4], b["end"][:4])+ self.assertTrue(all(t == "" for _d, t in a["data"]), "a tail nobody asked for")+ for _d, tail in b["data"]:+ fields = tail.split()+ self.assertEqual(int(fields[1]), 4)+ self.assertEqual(len(fields), 2 + 2 * 4)++ def test_a_read_only_request_may_fill_the_context(self):+ eng = self.client(CTX="32")+ full = eng.submit(1, "a" * 32)+ self.assertEqual(full["end"][:4], ["DONE", "1", "STAT", "0"])+ self.assertEqual(sorted(full["echo"]), list(range(32)))+ self.assertFalse(full["data"])+ refused = eng.submit(2, "a" * 32, max_tokens=1, ext="")+ self.assertEqual(refused["end"][:3], ["ERROR", "2", "CONTEXT_EXCEEDED"])+ over = eng.submit(3, "a" * 33)+ self.assertEqual(over["end"][:3], ["ERROR", "3", "CONTEXT_EXCEEDED"])+ self.assertFalse(over["echo"])+ self.assertEqual(eng.submit(4, "a" * 4, max_tokens=1, ext="")["end"][0], "DONE")+ eng.close()+++if __name__ == "__main__":+ unittest.main()