Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

mimo: Brio mode -- logprobs read-out, pin photos with the window rings

JustVugg3b800e2bb415
diff --git a/c/Makefile b/c/Makefile
index adf7ac3d..51342701 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -1282,6 +1282,14 @@ mimo-tiny-generate:
mimo-tiny-check: mimo-tiny-generate mimo$(EXE)
$(PYTHON) tests/mimo_tiny_harness.py --binary ./mimo$(EXE) --fixture ./mimo_tiny
+# The same fixture served: KV prefix reuse, Brio photos (the windowed layers' rings,
+# pictures, nested and stale photos), and /v1/brio and chat logprobs through the
+# gateway. Every comparison is against a cold engine, to the last printed digit.
+.PHONY: mimo-tiny-serve-check
+mimo-tiny-serve-check: mimo-tiny-generate mimo$(EXE)
+ $(PYTHON) -m unittest -v tests.test_mimo_prefix_serve tests.test_brio_serve.MimoBrioServe \
+ tests.test_brio_api.MimoBrioEndToEnd
+
# Qwen3.8-Flash-Next text-only sibling. qwen38.c owns the direct checkpoint
# loader and SERVE=1 protocol. With CUDA=1 it links the same expert tier as
# qwen36 (fp8 streaming mode: hot experts get VRAM copies, the RAM LRU stays);
diff --git a/c/mimo.c b/c/mimo.c
index 3580ffe1..c837b4bc 100644
--- a/c/mimo.c
+++ b/c/mimo.c
@@ -57,6 +57,7 @@
#include "quant.h"
#include "omp_tune.h"
#include "kv_prefix.h"
+#include "pin_pool.h"
#include "tok.h"
#include "serve_codec.h"
#include "serve_poll.h"
@@ -1207,19 +1208,69 @@ static void forward(Model *m, const int *ids, int n, float *logits, int all_rows
free(h); free(xn); free(tmp);
}
+/* The read-out of a prefill, for the logprobs channel (SUBMIT logprobs=k): one
+ * ECHO frame per prompt position this prefill feeds, in position order, before
+ * any DATA. The logits at position p predict the token at p + 1, so the frame
+ * of a position carries the log-probability of the token that is actually
+ * there, over the whole vocabulary, and not only when it is among the top k:
+ * that is what scores an option the model would never have written.
+ *
+ * The first position fed has no predictor among the rows computed here. On a
+ * cold prompt it is position 0, which has nothing to condition on, and its
+ * frame carries " nan 0". Resumed from a photo, `first` is the photo's own last
+ * logits, which predict exactly that token. There is no third case: a prompt
+ * read out never resumes from a live prefix, because the positions before it
+ * would have no frame at all (see the serve loop).
+ *
+ * This is its own argument and not a mode of something else: it asks for every
+ * row's logits and changes nothing about how the rows are computed. */
+typedef struct {
+ const char *id;
+ int k;
+ Tok *tok;
+ const float *first;
+} Echo;
+
+static void echo_frame(const Echo *e, int pos, int token, const float *lo, int vocab) {
+ char tail[1024], piece[512];
+ coli_logprob_tail(tail, sizeof(tail), lo, vocab, token, e->k);
+ int n = tok_decode(e->tok, &token, 1, piece, (int)sizeof(piece));
+ if (n < 0) n = 0;
+ printf("ECHO %s %d %d%s\n", e->id, n, pos, tail);
+ if (n > 0) fwrite(piece, 1, (size_t)n, stdout);
+ fputc('\n', stdout);
+ fflush(stdout);
+}
+
/* A prompt in blocks of MIMO_CHUNK rows (default 64): the last block's last
- * row (or every row, for the teacher-forcing dump) comes back in `logits`. */
+ * row (or every row, for the teacher-forcing dump) comes back in `logits`.
+ * With `echo`, every row of every block goes through the head and is read out
+ * as it comes; only one block of logits is ever held, not the prompt's. */
static void prefill(Model *m, const int *ids, int n, float *logits, float *all_logits,
- const ImageRows *img) {
+ const ImageRows *img, const Echo *echo) {
int chunk = env_int("MIMO_CHUNK", 64);
if (chunk < 1) chunk = 1;
- int used = 0;
+ int used = 0, V = m->c.vocab;
+ float *rows = NULL;
+ if (echo && !all_logits) {
+ rows = xmalloc((size_t)(n < chunk ? n : chunk) * V * sizeof(float), "read-out logits");
+ echo_frame(echo, m->pos, ids[0], echo->first, V);
+ }
for (int b = 0; b < n; b += chunk) {
int nb = n - b < chunk ? n - b : chunk;
- if (all_logits) forward(m, ids + b, nb, all_logits + (size_t)b * m->c.vocab, 1, img, &used);
+ if (all_logits) forward(m, ids + b, nb, all_logits + (size_t)b * V, 1, img, &used);
+ else if (rows) {
+ int p0 = m->pos;
+ forward(m, ids + b, nb, rows, 1, img, &used);
+ for (int t = 0; t < nb; t++) {
+ if (b + t + 1 < n) echo_frame(echo, p0 + t + 1, ids[b + t + 1], rows + (size_t)t * V, V);
+ else if (logits) memcpy(logits, rows + (size_t)t * V, (size_t)V * sizeof(float));
+ }
+ }
else forward(m, ids + b, nb, b + nb == n ? logits : NULL, 0, img, &used);
}
- if (all_logits && logits) memcpy(logits, all_logits + (size_t)(n - 1) * m->c.vocab, (size_t)m->c.vocab * sizeof(float));
+ free(rows);
+ if (all_logits && logits) memcpy(logits, all_logits + (size_t)(n - 1) * V, (size_t)V * sizeof(float));
}
/* ----------------------------------------------------------------- vision ---- */
@@ -1288,14 +1339,150 @@ static int sample(const float *logits, int vocab, float temperature, float top_p
return pick;
}
+/* ---------------------------------------------------------------- photos ---- */
+
+/* SUBMIT pin=1: a photo of the state at the end of this prompt, so that the
+ * prompts that begin with it (the options of a closed question) resume from
+ * there instead of computing it again. pin_pool.h keeps several, nested, and
+ * restores the deepest one that is a strict prefix of the new prompt.
+ *
+ * WHAT A LAYER CARRIES BETWEEN TOKENS, and so what the photo must hold:
+ *
+ * full attention K/V rows indexed by position. Nothing after the photo
+ * writes below it, so they stay where they are, and the
+ * photo holds only the token ids; kv_prefix_holds() is what
+ * says they are still the photo's rows (an unrelated prompt
+ * in between rewrites them from position 0).
+ * sliding window a RING of `rows` slots, position p in slot p % rows, and
+ * the position each slot holds. Every option overwrites the
+ * ring slots of the photo's last positions -- its own token at
+ * P lands where P - rows was -- so after one option the ring
+ * no longer describes the photo. The ring, keys, values and
+ * positions, is copied into the photo and copied back.
+ * the position m->pos, which is the photo's length.
+ *
+ * Nothing else carries over: the sinks are weights, the router has no state,
+ * and the expert cache changes the time, never the numbers. On Flash the rings
+ * of the 39 windowed layers are 128 x 8 x (192 + 128) floats each, about 50 MB
+ * a photo, so COLI_PIN_SLOTS (default 4) is what bounds the memory. */
+typedef struct {
+ int n_layers;
+ int rows[MIMO_MAX_LAYERS]; /* 0: a full-attention layer */
+ size_t kd[MIMO_MAX_LAYERS], vd[MIMO_MAX_LAYERS];
+ float *K[MIMO_MAX_LAYERS], *V[MIMO_MAX_LAYERS];
+ int *ring_pos[MIMO_MAX_LAYERS];
+} PinState;
+
+static ColiPinPool g_pins;
+
+static void pin_state_free(void *v) {
+ PinState *st = (PinState *)v;
+ if (!st) return;
+ for (int li = 0; li < st->n_layers; li++) { free(st->K[li]); free(st->V[li]); free(st->ring_pos[li]); }
+ free(st);
+}
+
+/* Copies the rings into `st` (allocated on first use). NULL when memory runs
+ * out: a photo is an optimisation and never the reason a request fails. */
+static PinState *pin_state_save(Model *m, PinState *st) {
+ Cfg *c = &m->c;
+ if (st && st->n_layers != c->n_layers) { pin_state_free(st); st = NULL; }
+ if (!st) {
+ st = calloc(1, sizeof(*st));
+ if (!st) return NULL;
+ st->n_layers = c->n_layers;
+ for (int li = 0; li < c->n_layers; li++) {
+ if (!c->swa[li]) continue;
+ Layer *l = &m->L[li];
+ st->rows[li] = l->rows;
+ st->kd[li] = (size_t)l->rows * c->kv_heads[1] * c->head_dim[1];
+ st->vd[li] = (size_t)l->rows * c->kv_heads[1] * c->v_dim[1];
+ st->K[li] = malloc(st->kd[li] * sizeof(float));
+ st->V[li] = malloc(st->vd[li] * sizeof(float));
+ st->ring_pos[li] = malloc((size_t)l->rows * sizeof(int));
+ if (!st->K[li] || !st->V[li] || !st->ring_pos[li]) { pin_state_free(st); return NULL; }
+ }
+ }
+ for (int li = 0; li < c->n_layers; li++) {
+ if (!st->rows[li]) continue;
+ Layer *l = &m->L[li];
+ memcpy(st->K[li], l->K, st->kd[li] * sizeof(float));
+ memcpy(st->V[li], l->V, st->vd[li] * sizeof(float));
+ memcpy(st->ring_pos[li], l->ring_pos, (size_t)l->rows * sizeof(int));
+ }
+ return st;
+}
+
+static int pin_state_load(Model *m, const PinState *st) {
+ Cfg *c = &m->c;
+ if (!st || st->n_layers != c->n_layers) return 0;
+ for (int li = 0; li < c->n_layers; li++)
+ if (st->rows[li] != (c->swa[li] ? m->L[li].rows : 0)) return 0;
+ for (int li = 0; li < c->n_layers; li++) {
+ if (!st->rows[li]) continue;
+ Layer *l = &m->L[li];
+ memcpy(l->K, st->K[li], st->kd[li] * sizeof(float));
+ memcpy(l->V, st->V[li], st->vd[li] * sizeof(float));
+ memcpy(l->ring_pos, st->ring_pos[li], (size_t)l->rows * sizeof(int));
+ }
+ return 1;
+}
+
+/* Only a request that asks for it takes a photo, and it goes into the pool,
+ * where no other request writes: an ordinary prompt can make a photo stale
+ * (it rewrites the rows the photo stands on), never replace it. */
+static void pin_save(Model *m, const int *ids, int n, const float *logits) {
+ coli_pin_pool_init(&g_pins, m->c.vocab);
+ ColiPin *k = coli_pin_store(&g_pins, ids, n, logits);
+ if (!k) return;
+ PinState *st = pin_state_save(m, (PinState *)k->state);
+ if (!st) { k->state = NULL; k->len = 0; return; } /* without its rings it would lie */
+ k->state = st;
+ fprintf(stderr, "[PIN] photo of %d tokens\n", n);
+ fflush(stderr);
+}
+
+/* The deepest photo that is a strict prefix of this prompt and whose rows the
+ * state still holds, put back: the rings, the position and the record. Returns
+ * its length and its logits (the predictor of the first fresh token), or 0.
+ * A photo whose rows are gone cannot help any more: it is released, memory and
+ * all, and the next deepest is tried instead of giving up. */
+static int pin_restore(Model *m, const int *ids, int n, const float **logit) {
+ *logit = NULL;
+ int s = coli_pin_best(&g_pins, ids, n);
+ while (s >= 0) {
+ ColiPin *k = &g_pins.slot[s];
+ if (k->logit && kv_prefix_holds(&m->kvp, k->ids, k->len) &&
+ pin_state_load(m, (const PinState *)k->state)) {
+ m->pos = k->len;
+ kv_prefix_clear(&m->kvp);
+ kv_prefix_record(&m->kvp, k->ids, 0, k->len);
+ *logit = k->logit;
+ coli_pin_touch(&g_pins, s);
+ return k->len;
+ }
+ fprintf(stderr, "[PIN] released the photo of %d tokens: the state no longer holds it\n", k->len);
+ k->len = 0;
+ pin_state_free(k->state);
+ k->state = NULL;
+ s = coli_pin_best(&g_pins, ids, n);
+ }
+ return 0;
+}
+
/* ------------------------------------------------------------------ serve ---- */
+/* The 7th numeric field (the byte count of a payload extension) is accepted, but
+ * only as 0: the gateway puts it ahead of the logprobs= key on chat requests, and
+ * this engine takes no grammar, so any other value is still a bad header. */
static const ColiServeWireProfile mimo_wire = {
.max_header_bytes = 511,
.max_payload_bytes = 1u << 26,
+ .max_extension_bytes = 0,
.max_tokens = 1 << 20,
.require_exact_lf = 1,
.require_finite_sampling = 1,
+ .allow_extension_bytes = 1,
};
static void serve_line(const char *format, ...) {
@@ -1348,12 +1535,14 @@ static void serve_hits(Model *m) {
free(hex); free(bitmap);
}
-/* max_tokens is a ceiling; generation needs at least one free position */
-static int serve_budget(int prompt, int requested, int context) {
+/* max_tokens is a ceiling; generation needs at least one free position. A
+ * read-only request (max_tokens=0, legal only with logprobs) generates nothing,
+ * so its prompt may fill the context. */
+static int serve_budget(int prompt, int requested, int context, int logprobs) {
if (prompt < 1 || prompt > context) return -1;
- int budget = requested > 0 ? requested : 256;
+ int budget = requested > 0 ? requested : (logprobs > 0 ? 0 : 256);
int room = context - prompt;
- if (room == 0) return -1;
+ if (budget > 0 && room == 0) return -1;
return budget < room ? budget : room;
}
@@ -1409,7 +1598,7 @@ static void serve_loop(Model *m, Tok *tokenizer, const char *dir) {
uint64_t hits0 = m->hits, miss0 = m->miss, fw0 = m->forwards;
int n_prompt = tok_encode(tokenizer, (const char *)command.payload, (int)command.payload_bytes,
ids, m->ctx + 1);
- int budget = serve_budget(n_prompt, command.max_tokens, m->ctx);
+ int budget = serve_budget(n_prompt, command.max_tokens, m->ctx, command.logprobs);
if (budget < 0) {
char message[160];
snprintf(message, sizeof(message), "CONTEXT_EXCEEDED prompt_tokens=%d requested=%d capacity=%d",
@@ -1438,13 +1627,38 @@ static void serve_loop(Model *m, Tok *tokenizer, const char *dir) {
continue;
}
}
- /* A chat client resends the whole transcript: if this prompt begins with
- * the ids the state was built from, only the tail is fed. An image
- * refuses it -- the pad ids do not describe the picture. */
+ /* Where this prompt starts. A chat client resends the whole transcript:
+ * if this prompt begins with the ids the state was built from, only the
+ * tail is fed (COLI_KV_PREFIX=0 turns that off). A photo (SUBMIT pin=1
+ * earlier) resumes a prompt that begins with it.
+ *
+ * With the read-out on (logprobs=k) only a photo will do. The read-out
+ * owes a frame to every position it does not inherit a predictor for,
+ * and a live prefix comes with none: the frames before it would simply
+ * be missing. So such a prompt resumes from the deepest photo, which
+ * carries the logits that predict its first fresh token, or is read out
+ * from position 0. It resumes at the photo even when the live state
+ * shares more: two options share the text before them, and stopping at
+ * that would leave the first option token without its predictor.
+ *
+ * An image refuses both: the pad ids do not describe the picture. */
int prefix_on = env_int("COLI_KV_PREFIX", 1) != 0;
- int reuse = (!pending && prefix_on) ? kv_prefix_reuse(&m->kvp, ids, n_prompt) : 0;
+ int reuse = 0;
+ const float *pin_logit = NULL;
+ if (!pending) {
+ if (command.logprobs > 0) reuse = pin_restore(m, ids, n_prompt, &pin_logit);
+ else {
+ reuse = prefix_on ? kv_prefix_reuse(&m->kvp, ids, n_prompt) : 0;
+ if (!reuse) reuse = pin_restore(m, ids, n_prompt, &pin_logit);
+ }
+ }
if (getenv("COLI_PREFIX_LOG"))
- fprintf(stderr, "[PREFIX] %s %d of %d prompt tokens\n", reuse ? "reusing" : "no reuse,", reuse, n_prompt);
+ fprintf(stderr, "[PREFIX] %s %d of %d prompt tokens%s\n", reuse ? "reusing" : "no reuse,", reuse,
+ n_prompt, pin_logit ? " (photo)" : "");
+ if (pin_logit) {
+ fprintf(stderr, "[PIN] resumed from the photo of %d tokens, %d to prefill\n", reuse, n_prompt - reuse);
+ fflush(stderr);
+ }
if (!reuse) model_reset(m);
coli_serve_write_accept(stdout, command.id, n_prompt);
if (pending) {
@@ -1452,8 +1666,16 @@ static void serve_loop(Model *m, Tok *tokenizer, const char *dir) {
free(pending); pending = NULL;
}
ImageRows img = { image, image_rows };
- prefill(m, ids + reuse, n_prompt - reuse, logits, NULL, image ? &img : NULL);
- if (image) { kv_prefix_taint(&m->kvp); free(image); }
+ Echo echo = { command.id, command.logprobs, tokenizer, pin_logit };
+ prefill(m, ids + reuse, n_prompt - reuse, logits, NULL, image ? &img : NULL,
+ command.logprobs > 0 ? &echo : NULL);
+ if (image) kv_prefix_taint(&m->kvp);
+ if (command.pin) {
+ /* a photo is described by its ids, and these do not describe the picture */
+ if (image) fprintf(stderr, "[PIN] no photo: the prompt carries a picture\n");
+ else pin_save(m, ids, n_prompt, logits);
+ }
+ free(image);
int emitted = 0, limited = 1, cancelled = 0, done_early = 0;
char piece[512];
while (emitted < budget && !cancelled && !done_early) {
@@ -1462,7 +1684,14 @@ static void serve_loop(Model *m, Tok *tokenizer, const char *dir) {
for (int i = 0; i < n_eos; i++) if (token == eos[i]) stop = 1;
if (stop) { limited = 0; break; }
int written = tok_decode(tokenizer, &token, 1, piece, (int)sizeof(piece));
- if (written > 0) coli_serve_write_data(stdout, command.id, piece, (size_t)written);
+ if (written > 0) {
+ if (command.logprobs > 0) {
+ /* the distribution the token was drawn from: sample() only reads it */
+ char tail[1024];
+ coli_logprob_tail(tail, sizeof(tail), logits, c->vocab, token, command.logprobs);
+ coli_serve_write_data_lp(stdout, command.id, piece, (size_t)written, tail);
+ } else coli_serve_write_data(stdout, command.id, piece, (size_t)written);
+ }
emitted++;
while (coli_serve_stdin_ready()) {
ColiServeCommand control;
@@ -1474,7 +1703,7 @@ static void serve_loop(Model *m, Tok *tokenizer, const char *dir) {
coli_serve_command_dispose(&control);
}
if (cancelled || done_early || emitted >= budget) break;
- prefill(m, &token, 1, logits, NULL, NULL);
+ prefill(m, &token, 1, logits, NULL, NULL, NULL);
}
if (cancelled) {
coli_serve_write_error(stdout, command.id, "CANCELLED");
@@ -1631,7 +1860,7 @@ int main(int argc, char **argv) {
}
ImageRows img = { image, image_rows };
double tp = now_s();
- prefill(m, ids, n, logits, all, image ? &img : NULL);
+ prefill(m, ids, n, logits, all, image ? &img : NULL, NULL);
double prefill_s = now_s() - tp;
if (dump) {
FILE *f = fopen(dump, "wb");
@@ -1656,7 +1885,7 @@ int main(int argc, char **argv) {
fflush(stdout);
produced++;
if (stop && have_tok && !ids_text) break;
- if (g + 1 < ngen) prefill(m, &token, 1, logits, NULL, NULL);
+ if (g + 1 < ngen) prefill(m, &token, 1, logits, NULL, NULL, NULL);
}
printf("\n");
fflush(stdout);
diff --git a/c/tests/mimo_serve_fixture.py b/c/tests/mimo_serve_fixture.py
new file mode 100644
index 00000000..d873f0a8
--- /dev/null
+++ b/c/tests/mimo_serve_fixture.py
@@ -0,0 +1,81 @@
+"""The tiny MiMo fixture, served: the same weights plus a tokenizer.
+
+tools/make_mimo_tiny.py ships no tokenizer.json on purpose -- the vendor oracle
+(tests/mimo_tiny_harness.py) reads raw token ids on stdout and refuses a fixture
+that has one. Serve mode goes through text, so the serve tests work on a copy:
+the fixture's files plus a one-token-per-byte tokenizer, with the three vision
+markers as added tokens at the ids the fixture's config gives them, so an image
+prompt can be written as text.
+
+The copy is made once per process, in a temporary directory that is removed at
+exit, and the source fixture is never touched.
+"""
+import atexit
+import json
+import os
+import shutil
+import sys
+import tempfile
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from prefix_serve_harness import byte_level_vocab # noqa: E402
+
+HERE = Path(__file__).resolve().parent.parent
+ENGINE = Path(os.environ.get("MIMO_BINARY",
+ HERE / ("mimo.exe" if sys.platform == "win32" else "mimo")))
+FIXTURE = Path(os.environ.get("MIMO_TINY", HERE / "mimo_tiny"))
+
+IMAGE_PAD = "<|image_pad|>"
+VISION_START = "<|vision_start|>"
+VISION_END = "<|vision_end|>"
+
+_served = None
+
+
+def available():
+ return ENGINE.is_file() and (FIXTURE / "config.json").is_file()
+
+
+def has_vision():
+ return (FIXTURE / "patches.f32").is_file() and (FIXTURE / "image.json").is_file()
+
+
+def served_fixture():
+ """The serve-mode copy of FIXTURE, built on first use."""
+ global _served
+ if _served is not None:
+ return _served
+ root = Path(tempfile.mkdtemp(prefix="mimo-serve-"))
+ atexit.register(shutil.rmtree, root, True)
+ out = root / "fixture"
+ shutil.copytree(FIXTURE, out)
+ config = json.loads((FIXTURE / "config.json").read_text())
+ added = []
+ for key, text in (("image_token_id", IMAGE_PAD), ("vision_start_token_id", VISION_START),
+ ("vision_end_token_id", VISION_END)):
+ if isinstance(config.get(key), int):
+ added.append({"id": config[key], "content": text, "special": True})
+ (out / "tokenizer.json").write_text(json.dumps({
+ "version": "1.0",
+ "added_tokens": added,
+ "model": {"type": "BPE", "vocab": byte_level_vocab(256), "merges": []},
+ }), encoding="utf-8")
+ _served = out
+ return out
+
+
+def image_payload():
+ """(patch bytes, grid_h, grid_w, pad count) of the fixture's picture."""
+ meta = json.loads((FIXTURE / "image.json").read_text())
+ return ((FIXTURE / "patches.f32").read_bytes(), meta["grid_h"], meta["grid_w"],
+ meta["tokens"])
+
+
+def engine_env(**extra):
+ """What every serve test runs the engine with: the served copy, a small
+ context, two threads and no OpenMP re-exec."""
+ env = {"SNAP": str(served_fixture()), "SERVE": "1", "CTX": "256",
+ "OMP_NUM_THREADS": "2", "COLI_NO_OMP_TUNE": "1"}
+ env.update(extra)
+ return env
diff --git a/c/tests/test_brio_serve.py b/c/tests/test_brio_serve.py
index 86f6bca9..76ce9992 100644
--- a/c/tests/test_brio_serve.py
+++ b/c/tests/test_brio_serve.py
@@ -14,17 +14,25 @@ byte-identical frames it produced before the channel existed. Chat is the mode
everyone uses; the scoring channel is opt-in and must stay that way. The check
is a byte diff of the frames, not a reading of the text.
-Runs against the CONVERTED tiny olmoe fixture, olmoe_tiny_c, that the
-`olmoe-tiny-check` CI job builds and runs this module on, so it needs no
-checkpoint. From c/:
+The scenarios are a contract, written once and run per engine:
+
+- OLMoE, against the CONVERTED tiny fixture olmoe_tiny_c that the
+ `olmoe-tiny-check` CI job builds and runs this module on. From c/:
python3 tools/make_olmoe_tiny.py --output olmoe_tiny
python3 tools/convert_olmoe_merged.py --model olmoe_tiny --out olmoe_tiny_c
python3 tools/make_edge_tiny_tokenizer.py --vocab-size 128 olmoe_tiny_c
-olmoe_tiny is the HF source checkpoint the engine cannot load, and it has a
-config.json too: the guard below looks for the tokenizer, which only the
-converted fixture has, so a missing fixture skips instead of failing (#1742).
+ olmoe_tiny is the HF source checkpoint the engine cannot load, and it has a
+ config.json too: the guard below looks for the tokenizer, which only the
+ converted fixture has, so a missing fixture skips instead of failing (#1742).
+
+- MiMo-V2.6, against the tiny fixture of `make mimo-tiny-generate`, served
+ through tests/mimo_serve_fixture.py (a copy with a byte tokenizer). Its
+ windowed layers and its pictures have their own cases in
+ tests/test_mimo_prefix_serve.py.
+
+Neither needs a checkpoint, and an engine that is not built skips.
"""
import json
import os
@@ -33,6 +41,9 @@ import sys
import unittest
from pathlib import Path
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+import mimo_serve_fixture # noqa: E402
+
HERE = Path(__file__).resolve().parent.parent
ENGINE = HERE / ("olmoe.exe" if sys.platform == "win32" else "olmoe")
FIXTURE = Path(os.environ.get("OLMOE_TINY", HERE / "olmoe_tiny_c"))
@@ -40,16 +51,20 @@ PROMPT = "Context: the release is late and the tests are red.\nQuestion: ship?\n
OPTION = " no"
+def olmoe_engine():
+ return Engine(ENGINE, [], dict(SNAP=str(FIXTURE), SERVE="1",
+ TOK=str(FIXTURE / "tokenizer.json"),
+ COLI_NO_OMP_TUNE="1", OMP_NUM_THREADS="2"))
+
+
class Engine:
"""One serve-mode engine, driven over the wire the gateway uses."""
- def __init__(self):
- env = dict(os.environ, SNAP=str(FIXTURE), SERVE="1",
- TOK=str(FIXTURE / "tokenizer.json"),
- COLI_NO_OMP_TUNE="1", OMP_NUM_THREADS="2")
- self.p = subprocess.Popen([str(ENGINE)], env=env, stdin=subprocess.PIPE,
- stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
- bufsize=0)
+ def __init__(self, binary, argv, env):
+ env = dict(os.environ, **env)
+ self.p = subprocess.Popen([str(binary)] + [str(a) for a in argv], env=env,
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+ stderr=subprocess.DEVNULL, bufsize=0)
while True:
line = self.readline()
if line is None:
@@ -101,6 +116,7 @@ class Engine:
pass
self.p.terminate()
self.p.wait(timeout=30)
+ self.p.stdout.close()
def strip_volatile(frames):
@@ -110,14 +126,15 @@ def strip_volatile(frames):
if not f.startswith(("STAT", "PROF", "DONE", "TIERS", "HWINFO", "EMAP", "HITS"))]
[email protected]((FIXTURE / "tokenizer.json").is_file(),
- "converted tiny olmoe fixture is absent: see the module "
- "docstring (make_olmoe_tiny.py, convert_olmoe_merged.py, "
- "make_edge_tiny_tokenizer.py)")
[email protected](ENGINE.is_file(), "olmoe engine is not built")
-class BrioServe(unittest.TestCase):
+class BrioServeContract:
+ """The scenarios. Mix into a TestCase that provides engine(); deliberately not
+ a TestCase itself, or unittest would run them once more with no engine."""
+
+ def engine(self): # pragma: no cover - overridden
+ raise NotImplementedError
+
def test_a_snapshot_scores_exactly_like_a_cold_recompute(self):
- warm = Engine()
+ warm = self.engine()
try:
_, prefix = warm.submit(1, PROMPT, 0, " logprobs=1 pin=1")
_, pinned = warm.submit(2, PROMPT + OPTION, 0, " logprobs=1")
@@ -126,7 +143,7 @@ class BrioServe(unittest.TestCase):
self.assertTrue(prefix, "the warm pass read nothing")
tail_starts = max(prefix) + 1
- cold = Engine()
+ cold = self.engine()
try:
_, fresh = cold.submit(1, PROMPT + OPTION, 0, " logprobs=1")
finally:
@@ -170,7 +187,7 @@ class BrioServe(unittest.TestCase):
cold recompute."""
other = "Context: the invoice is overdue by ninety days.\nQuestion: escalate?\nAnswer:"
- engine = Engine()
+ engine = self.engine()
try:
_, prefix = engine.submit(1, PROMPT, 0, " logprobs=1 pin=1")
engine.submit(2, other, 4) # overwrites the record with other ids
@@ -180,7 +197,7 @@ class BrioServe(unittest.TestCase):
self.assertTrue(prefix, "the warm pass read nothing")
tail_starts = max(prefix) + 1
- cold = Engine()
+ cold = self.engine()
try:
_, fresh = cold.submit(1, PROMPT + OPTION, 0, " logprobs=1")
finally:
@@ -199,7 +216,7 @@ class BrioServe(unittest.TestCase):
def test_the_snapshot_only_reads_the_fresh_tail(self):
"""The point of the snapshot: the shared prefix is not read again."""
- engine = Engine()
+ engine = self.engine()
try:
_, prefix = engine.submit(1, PROMPT, 0, " logprobs=1 pin=1")
_, tail = engine.submit(2, PROMPT + OPTION, 0, " logprobs=1")
@@ -217,7 +234,7 @@ class BrioServe(unittest.TestCase):
of every frame, so the same id has to be reused, and a second turn on a
live engine would also start from a different cached state."""
def frames_for(extension):
- engine = Engine()
+ engine = self.engine()
try:
frames, _ = engine.submit(1, PROMPT, 4, extension)
finally:
@@ -244,7 +261,7 @@ class BrioServe(unittest.TestCase):
def test_read_only_generates_nothing(self):
"""max_tokens=0 means read the prompt and stop: no DATA frame at all."""
- engine = Engine()
+ engine = self.engine()
try:
frames, _ = engine.submit(1, PROMPT, 0, " logprobs=1")
finally:
@@ -256,7 +273,7 @@ class BrioServe(unittest.TestCase):
def test_read_only_needs_the_channel(self):
"""max_tokens=0 without logprobs stays a malformed request."""
- engine = Engine()
+ engine = self.engine()
try:
frames, _ = engine.submit(1, PROMPT, 0)
finally:
@@ -265,5 +282,23 @@ class BrioServe(unittest.TestCase):
"max_tokens=0 was accepted without the logprobs channel")
[email protected]((FIXTURE / "tokenizer.json").is_file(),
+ "converted tiny olmoe fixture is absent: see the module "
+ "docstring (make_olmoe_tiny.py, convert_olmoe_merged.py, "
+ "make_edge_tiny_tokenizer.py)")
[email protected](ENGINE.is_file(), "olmoe engine is not built")
+class BrioServe(BrioServeContract, unittest.TestCase):
+ def engine(self):
+ return olmoe_engine()
+
+
[email protected](mimo_serve_fixture.available(),
+ "mimo is not built or the tiny MiMo fixture is absent "
+ "(make mimo mimo-tiny-generate)")
+class MimoBrioServe(BrioServeContract, unittest.TestCase):
+ def engine(self):
+ return Engine(mimo_serve_fixture.ENGINE, ["8"], mimo_serve_fixture.engine_env())
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/c/tests/test_mimo_prefix_serve.py b/c/tests/test_mimo_prefix_serve.py
new file mode 100644
index 00000000..545d75ed
--- /dev/null
+++ b/c/tests/test_mimo_prefix_serve.py
@@ -0,0 +1,331 @@
+"""KV prefix reuse and Brio photos on MiMo-V2.6, against the tiny fixture.
+
+MiMo is the engine where a photo is more than token ids. Most of its layers
+attend a sliding window, and a windowed layer keeps its keys in a RING: position
+p in slot p % window. The photo of a prompt of P tokens holds the ring as it was
+at P; the first option then writes its own positions into the slots of P - window
+and onwards, so after one option the ring no longer describes the photo. A photo
+that kept only the ids -- enough for the full-attention layers, whose rows below
+P nobody touches -- would hand the second option a ring from the first. The
+fixture's window is 8 tokens, and every prompt below outgrows it.
+
+What is checked, all against a cold engine and to the last printed digit of
+every ECHO frame (the log-probability of the token and the whole top-k):
+
+ - the shared prefix-reuse contract (tests/prefix_serve_harness.py);
+ - options that cross the window resume from the photo exactly, one after the
+ other, with an ordinary chat turn in between that must not take the photo
+ away, and each one PROCESSES ONLY ITS OWN TOKENS: its read-out starts at the
+ photo's end and the engine says it prefilled the option and nothing more.
+ The numbers alone would not show a photo that silently fell back to a full
+ recompute; the positions and the count do;
+ - nested photos (a state, then a question) and a sibling question that
+ overwrites the deeper one's rows: the engine falls back to the shallower;
+ - a photo whose rows an unrelated prompt overwrote is released, not restored;
+ - pictures: a prompt with an image never resumes from a photo, and a prompt
+ with an image takes none, because token ids do not describe the picture;
+ - the read-out covers every position from 0 unless a photo covers the prefix,
+ which is what the gateway's `echo` needs, and never resumes from a live
+ prefix;
+ - asking for logprobs changes no generated byte;
+ - a read-only request may fill the context, a generating one may not.
+
+Needs the built engine and the fixture: `make mimo mimo-tiny-generate`.
+"""
+import json
+import math
+import re
+import sys
+import unittest
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+import mimo_serve_fixture as fixture # noqa: E402
+from prefix_serve_harness import PrefixReuseContract, ServeEngine # noqa: E402
+
+STATE = "Context: the release is late, the tests are red and the changelog is empty.\n"
+QUESTION = STATE + "Question: should we ship it today?\nAnswer:"
+SIBLING = STATE + "Question: who has to sign it off?\nAnswer:"
+OPTIONS = (" only after the tests pass and the changelog says what changed",
+ " yes",
+ " no, not today",
+ " only after the tests pass and the changelog says what changed")
+
+
+def window():
+ config = json.loads((fixture.FIXTURE / "config.json").read_text())
+ return int(config["sliding_window"])
+
+
+class Client(ServeEngine):
+ """A serve engine read frame by frame: the ECHO frames are what is compared."""
+
+ def __init__(self, log_prefix=True, **env):
+ super().__init__(fixture.ENGINE, ["8"], fixture.engine_env(**env),
+ log_prefix=log_prefix)
+
+ def image(self, rid):
+ payload, grid_h, grid_w, _pads = fixture.image_payload()
+ self.p.stdin.write(f"IMAGE {rid} {len(payload)} {grid_h} {grid_w}\n".encode()
+ + payload + b"\n")
+ self.p.stdin.flush()
+
+ def submit(self, rid, prompt, max_tokens=0, ext=" logprobs=3"):
+ """Returns {kind: ..., "echo": {pos: tail}, "data": [(bytes, tail)], "end": line}.
+
+ A tail is the frame's numeric part after the position, compared as text:
+ the log-probability and every top-k candidate, as the engine printed them."""
+ data = prompt.encode()
+ self.p.stdin.write(f"SUBMIT {rid} 0 {len(data)} {max_tokens} 0 1{ext}\n".encode()
+ + data + b"\n")
+ self.p.stdin.flush()
+ out = {"echo": {}, "data": [], "accept": None, "end": None}
+ while True:
+ line = self.p.stdout.readline()
+ if not line:
+ raise RuntimeError("engine closed mid-request:\n" + "".join(self._err)[-2000:])
+ fields = line.decode("latin-1").split()
+ if not fields:
+ continue
+ if fields[0] in ("ECHO", "DATA"):
+ payload = self.p.stdout.read(int(fields[2]))
+ self.p.stdout.read(1)
+ if fields[0] == "ECHO":
+ pos = int(fields[3])
+ if pos in out["echo"]:
+ raise AssertionError(f"position {pos} read out twice")
+ out["echo"][pos] = " ".join(fields[4:])
+ else:
+ out["data"].append((payload, " ".join(fields[3:])))
+ elif fields[0] == "ACCEPT":
+ out["accept"] = int(fields[2])
+ elif fields[0] in ("DONE", "ERROR"):
+ out["end"] = fields
+ return out
+
+
+def resumed(log):
+ """[(photo length, tokens prefilled)] in order, from the engine's own account."""
+ return [(int(a), int(b)) for a, b in
+ re.findall(r"\[PIN\] resumed from the photo of (\d+) tokens, (\d+) to prefill", log)]
+
+
[email protected](fixture.available(),
+ "mimo is not built or the tiny MiMo fixture is absent "
+ "(make mimo mimo-tiny-generate)")
+class MimoPrefixServeTest(PrefixReuseContract, unittest.TestCase):
+ # The fixture's vocabulary (320) is wider than the byte tokenizer the served
+ # copy carries, so a generated reply does not always survive decode and
+ # re-encode: the replay scenario may then legitimately not reuse.
+ REPLAY_REUSES = False
+
+ def spawn(self, log_prefix=True, reuse=True):
+ return ServeEngine(fixture.ENGINE, ["8"], fixture.engine_env(),
+ log_prefix=log_prefix, reuse=reuse)
+
+
[email protected](fixture.available(),
+ "mimo is not built or the tiny MiMo fixture is absent "
+ "(make mimo mimo-tiny-generate)")
+class MimoBrioPhotoTest(unittest.TestCase):
+ def client(self, **env):
+ c = Client(**env)
+ self.addCleanup(c.p.stdout.close)
+ self.addCleanup(c.p.stderr.close)
+ return c
+
+ def cold(self, prompt, image=False):
+ """The prompt read out by a fresh engine that never saw a photo."""
+ c = self.client(log_prefix=False)
+ if image:
+ c.image(1)
+ got = c.submit(1, prompt)
+ c.close()
+ return got
+
+ def assertSameReadOut(self, got, cold, positions, what):
+ for pos in positions:
+ self.assertIn(pos, cold["echo"], f"{what}: the cold run did not read position {pos}")
+ self.assertIn(pos, got["echo"], f"{what}: position {pos} was not read out")
+ self.assertEqual(got["echo"][pos], cold["echo"][pos],
+ f"{what}: position {pos} differs from a cold recompute "
+ f"-- the state resumed is not the state it claims to be")
+
+ def test_options_that_cross_the_window_resume_from_the_photo_exactly(self):
+ w = window()
+ self.assertGreater(len(QUESTION.encode()), 2 * w,
+ "the photo must outgrow the window, or its ring never wraps")
+ self.assertGreater(len(OPTIONS[0].encode()), 4 * w,
+ "the long option must slide the window past the photo")
+ eng = self.client()
+ photo = eng.submit(1, QUESTION, ext=" logprobs=3 pin=1")
+ n = photo["accept"]
+ self.assertEqual(n, len(QUESTION.encode()))
+ warm = []
+ for i, option in enumerate(OPTIONS):
+ warm.append(eng.submit(10 + i, QUESTION + option))
+ if i == 1:
+ # An ordinary chat turn that extends the photo. The live state holds
+ # the option before it, so it resumes from the photo too -- and it
+ # must not take the photo away from the options after it.
+ chat = eng.submit(20, QUESTION + " maybe", max_tokens=4, ext="")
+ self.assertEqual(chat["end"][0], "DONE")
+ self.assertEqual(chat["echo"], {}, "a chat turn was read out unasked")
+ log = eng.close()
+
+ self.assertIn("[PIN] photo of %d tokens" % n, log)
+ self.assertEqual(log.count("[PIN] photo of"), 1, "only the pin=1 request takes a photo")
+ lengths = [len(option.encode()) for option in OPTIONS]
+ # The saving, from the engine's own account: every option resumed from the
+ # photo and prefilled ITS tokens, not the shared prefix again.
+ expected = [(n, k) for k in lengths]
+ expected.insert(2, (n, len(" maybe"))) # the chat turn
+ self.assertEqual(resumed(log), expected,
+ "an option did not resume from the photo:\n" + log)
+ for option, k, got in zip(OPTIONS, lengths, warm):
+ with self.subTest(option=option):
+ # ...and from the wire: the read-out starts exactly where the photo
+ # ends -- the first option token predicted by the photo's own logits
+ # -- and covers the option and nothing before it.
+ self.assertEqual(sorted(got["echo"]), list(range(n, n + k)))
+ self.assertFalse(any(t.startswith("nan") for t in got["echo"].values()),
+ "an option token lost its predictor")
+ cold = self.cold(QUESTION + option)
+ self.assertEqual(sorted(cold["echo"]), list(range(0, n + k)),
+ "a cold read-out covers every position")
+ self.assertSameReadOut(got, cold, range(n, n + k), repr(option))
+
+ def test_nested_photos_and_a_sibling_question(self):
+ eng = self.client()
+ s = eng.submit(1, STATE, ext=" logprobs=1 pin=1")["accept"]
+ q = eng.submit(2, QUESTION, ext=" logprobs=1 pin=1")["accept"]
+ first = eng.submit(3, QUESTION + OPTIONS[2])
+ # Same state, another question: resumes from the state, and its prefill
+ # overwrites the rows the deeper photo stood on.
+ sibling_pin = eng.submit(4, SIBLING, ext=" logprobs=1 pin=1")
+ sibling = eng.submit(5, SIBLING + " the release manager")
+ back = eng.submit(6, QUESTION + OPTIONS[1])
+ log = eng.close()
+
+ self.assertEqual(sorted(first["echo"])[0], q)
+ self.assertEqual(sorted(sibling_pin["echo"])[0], s)
+ self.assertEqual(sorted(sibling["echo"])[0], sibling_pin["accept"])
+ # The question's photo is stale now: released, and the state's taken instead
+ # of starting over.
+ self.assertIn("[PIN] released the photo of %d tokens" % q, log)
+ self.assertEqual(sorted(back["echo"])[0], s)
+ for got, prompt, start in ((first, QUESTION + OPTIONS[2], q),
+ (sibling, SIBLING + " the release manager",
+ sibling_pin["accept"]),
+ (back, QUESTION + OPTIONS[1], s)):
+ cold = self.cold(prompt)
+ self.assertSameReadOut(got, cold, range(start, len(prompt.encode())), prompt)
+
+ def test_a_photo_whose_rows_were_overwritten_is_not_restored(self):
+ other = "Context: the invoice is overdue by ninety days.\nQuestion: escalate?\nAnswer:"
+ eng = self.client()
+ n = eng.submit(1, QUESTION, ext=" logprobs=1 pin=1")["accept"]
+ eng.submit(2, other, max_tokens=4, ext="")
+ after = eng.submit(3, QUESTION + OPTIONS[2])
+ log = eng.close()
+ self.assertIn("[PIN] released the photo of %d tokens" % n, log)
+ self.assertEqual(resumed(log), [])
+ total = len((QUESTION + OPTIONS[2]).encode())
+ self.assertEqual(sorted(after["echo"]), list(range(total)),
+ "with no photo the read-out starts over from position 0")
+ self.assertSameReadOut(after, self.cold(QUESTION + OPTIONS[2]), range(1, total),
+ "after a divergent prompt")
+
+ @unittest.skipUnless(fixture.has_vision(), "the fixture carries no picture")
+ def test_an_image_prompt_never_resumes_from_a_photo(self):
+ """The prompt begins with the photo's ids, so the ids alone would say
+ resume. They do not describe the picture, and the image rows are spliced
+ by their order in the whole prompt: the engine starts over."""
+ _payload, _h, _w, pads = fixture.image_payload()
+ prompt = (QUESTION + " " + fixture.VISION_START + fixture.IMAGE_PAD * pads
+ + fixture.VISION_END + " so:")
+ eng = self.client()
+ eng.submit(1, QUESTION, ext=" logprobs=1 pin=1")
+ eng.image(2)
+ got = eng.submit(2, prompt)
+ log = eng.close()
+ self.assertEqual(got["end"][0], "DONE", got["end"])
+ self.assertEqual(resumed(log), [], "an image prompt resumed from a photo:\n" + log)
+ self.assertEqual(sorted(got["echo"]), list(range(got["accept"])))
+ cold = self.cold(prompt, image=True)
+ self.assertSameReadOut(got, cold, range(1, got["accept"]), "image prompt")
+
+ @unittest.skipUnless(fixture.has_vision(), "the fixture carries no picture")
+ def test_a_prompt_with_a_picture_takes_no_photo(self):
+ """pin=1 on a prompt with a picture: the photo would be filed under ids
+ that do not describe the picture, and a later TEXT prompt with the same ids
+ (the pads as plain tokens) would resume from a state that saw an image."""
+ _payload, _h, _w, pads = fixture.image_payload()
+ pictured = (STATE + fixture.VISION_START + fixture.IMAGE_PAD * pads
+ + fixture.VISION_END + "\nAnswer:")
+ eng = self.client()
+ eng.image(1)
+ eng.submit(1, pictured, ext=" logprobs=1 pin=1")
+ text_only = eng.submit(2, pictured + OPTIONS[1])
+ log = eng.close()
+ self.assertIn("[PIN] no photo: the prompt carries a picture", log)
+ self.assertNotIn("[PIN] photo of", log)
+ self.assertEqual(resumed(log), [])
+ total = text_only["accept"]
+ self.assertEqual(sorted(text_only["echo"]), list(range(total)),
+ "a text prompt resumed from a state that saw a picture")
+ cold = self.cold(pictured + OPTIONS[1])
+ self.assertSameReadOut(text_only, cold, range(1, total), "text after a picture")
+
+ def test_the_read_out_covers_every_position_unless_a_photo_does(self):
+ """What the gateway's `echo` needs: one frame per prompt position, the
+ first carrying nan because nothing precedes it. A live prefix brings no
+ predictor for its last position, so a read-out never resumes from one."""
+ opening = "The capital of France is"
+ eng = self.client()
+ eng.submit(1, opening, max_tokens=1, ext="")
+ got = eng.submit(2, opening + " a city, and the capital of Spain is")
+ log = eng.close()
+ self.assertNotIn("[PREFIX] reusing", log, "a read-out resumed from a live prefix")
+ n = got["accept"]
+ self.assertEqual(sorted(got["echo"]), list(range(n)))
+ self.assertTrue(got["echo"][0].startswith("nan 0"), got["echo"][0])
+ for pos in range(1, n):
+ lp = float(got["echo"][pos].split()[0])
+ self.assertTrue(math.isfinite(lp) and lp <= 0.0, (pos, lp))
+ self.assertEqual(int(got["echo"][pos].split()[1]), 3, "the top-k asked for")
+
+ def test_logprobs_change_no_generated_byte(self):
+ prompt = "Streaming experts from disk means that the decode rate depends on"
+ plain = self.client()
+ a = plain.submit(1, prompt, max_tokens=12, ext="")
+ plain.close()
+ scored = self.client()
+ b = scored.submit(1, prompt, max_tokens=12, ext=" logprobs=4")
+ scored.close()
+ self.assertTrue(a["data"], "nothing was generated")
+ self.assertEqual([d for d, _t in a["data"]], [d for d, _t in b["data"]])
+ self.assertEqual(a["end"][:4], b["end"][:4])
+ self.assertTrue(all(t == "" for _d, t in a["data"]), "a tail nobody asked for")
+ for _d, tail in b["data"]:
+ fields = tail.split()
+ self.assertEqual(int(fields[1]), 4)
+ self.assertEqual(len(fields), 2 + 2 * 4)
+
+ def test_a_read_only_request_may_fill_the_context(self):
+ eng = self.client(CTX="32")
+ full = eng.submit(1, "a" * 32)
+ self.assertEqual(full["end"][:4], ["DONE", "1", "STAT", "0"])
+ self.assertEqual(sorted(full["echo"]), list(range(32)))
+ self.assertFalse(full["data"])
+ refused = eng.submit(2, "a" * 32, max_tokens=1, ext="")
+ self.assertEqual(refused["end"][:3], ["ERROR", "2", "CONTEXT_EXCEEDED"])
+ over = eng.submit(3, "a" * 33)
+ self.assertEqual(over["end"][:3], ["ERROR", "3", "CONTEXT_EXCEEDED"])
+ self.assertFalse(over["echo"])
+ self.assertEqual(eng.submit(4, "a" * 4, max_tokens=1, ext="")["end"][0], "DONE")
+ eng.close()
+
+
+if __name__ == "__main__":
+ unittest.main()