Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
"""KV prefix reuse and Brio photos on MiMo-V2.6, against the tiny fixture.
MiMo is the engine where a photo is more than token ids. Most of its layers
attend a sliding window, and a windowed layer keeps its keys in a RING: position
p in slot p % window. The photo of a prompt of P tokens holds the ring as it was
at P; the first option then writes its own positions into the slots of P - window
and onwards, so after one option the ring no longer describes the photo. A photo
that kept only the ids -- enough for the full-attention layers, whose rows below
P nobody touches -- would hand the second option a ring from the first. The
fixture's window is 8 tokens, and every prompt below outgrows it.
What is checked, all against a cold engine and to the last printed digit of
every ECHO frame (the log-probability of the token and the whole top-k):
- the shared prefix-reuse contract (tests/prefix_serve_harness.py);
- options that cross the window resume from the photo exactly, one after the
other, with an ordinary chat turn in between that must not take the photo
away, and each one PROCESSES ONLY ITS OWN TOKENS: its read-out starts at the
photo's end and the engine says it prefilled the option and nothing more.
The numbers alone would not show a photo that silently fell back to a full
recompute; the positions and the count do;
- nested photos (a state, then a question) and a sibling question that
overwrites the deeper one's rows: the engine falls back to the shallower;
- a photo whose rows an unrelated prompt overwrote is released, not restored;
- pictures: a prompt with an image never resumes from a photo, and a prompt
with an image takes none, because token ids do not describe the picture;
- the read-out covers every position from 0 unless a photo covers the prefix,
which is what the gateway's `echo` needs, and never resumes from a live
prefix;
- asking for logprobs changes no generated byte;
- a read-only request may fill the context, a generating one may not.
Needs the built engine and the fixture: `make mimo mimo-tiny-generate`.
"""
import json
import math
import re
import sys
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
import mimo_serve_fixture as fixture # noqa: E402
from prefix_serve_harness import PrefixReuseContract, ServeEngine # noqa: E402
STATE = "Context: the release is late, the tests are red and the changelog is empty.\n"
QUESTION = STATE + "Question: should we ship it today?\nAnswer:"
SIBLING = STATE + "Question: who has to sign it off?\nAnswer:"
OPTIONS = (" only after the tests pass and the changelog says what changed",
" yes",
" no, not today",
" only after the tests pass and the changelog says what changed")
def window():
config = json.loads((fixture.FIXTURE / "config.json").read_text())
return int(config["sliding_window"])
class Client(ServeEngine):
"""A serve engine read frame by frame: the ECHO frames are what is compared."""
def __init__(self, log_prefix=True, **env):
super().__init__(fixture.ENGINE, ["8"], fixture.engine_env(**env),
log_prefix=log_prefix)
def image(self, rid):
payload, grid_h, grid_w, _pads = fixture.image_payload()
self.p.stdin.write(f"IMAGE {rid} {len(payload)} {grid_h} {grid_w}\n".encode()
+ payload + b"\n")
self.p.stdin.flush()
def submit(self, rid, prompt, max_tokens=0, ext=" logprobs=3"):
"""Returns {kind: ..., "echo": {pos: tail}, "data": [(bytes, tail)], "end": line}.
A tail is the frame's numeric part after the position, compared as text:
the log-probability and every top-k candidate, as the engine printed them."""
data = prompt.encode()
self.p.stdin.write(f"SUBMIT {rid} 0 {len(data)} {max_tokens} 0 1{ext}\n".encode()
+ data + b"\n")
self.p.stdin.flush()
out = {"echo": {}, "data": [], "accept": None, "end": None}
while True:
line = self.p.stdout.readline()
if not line:
raise RuntimeError("engine closed mid-request:\n" + "".join(self._err)[-2000:])
fields = line.decode("latin-1").split()
if not fields:
continue
if fields[0] in ("ECHO", "DATA"):
payload = self.p.stdout.read(int(fields[2]))
self.p.stdout.read(1)
if fields[0] == "ECHO":
pos = int(fields[3])
if pos in out["echo"]:
raise AssertionError(f"position {pos} read out twice")
out["echo"][pos] = " ".join(fields[4:])
else:
out["data"].append((payload, " ".join(fields[3:])))
elif fields[0] == "ACCEPT":
out["accept"] = int(fields[2])
elif fields[0] in ("DONE", "ERROR"):
out["end"] = fields
return out
def resumed(log):
"""[(photo length, tokens prefilled)] in order, from the engine's own account."""
return [(int(a), int(b)) for a, b in
re.findall(r"\[PIN\] resumed from the photo of (\d+) tokens, (\d+) to prefill", log)]
@unittest.skipUnless(fixture.available(),
"mimo is not built or the tiny MiMo fixture is absent "
"(make mimo mimo-tiny-generate)")
class MimoPrefixServeTest(PrefixReuseContract, unittest.TestCase):
# The fixture's vocabulary (320) is wider than the byte tokenizer the served
# copy carries, so a generated reply does not always survive decode and
# re-encode: the replay scenario may then legitimately not reuse.
REPLAY_REUSES = False
def spawn(self, log_prefix=True, reuse=True):
return ServeEngine(fixture.ENGINE, ["8"], fixture.engine_env(),
log_prefix=log_prefix, reuse=reuse)
@unittest.skipUnless(fixture.available(),
"mimo is not built or the tiny MiMo fixture is absent "
"(make mimo mimo-tiny-generate)")
class MimoBrioPhotoTest(unittest.TestCase):
def client(self, **env):
c = Client(**env)
self.addCleanup(c.p.stdout.close)
self.addCleanup(c.p.stderr.close)
return c
def cold(self, prompt, image=False):
"""The prompt read out by a fresh engine that never saw a photo."""
c = self.client(log_prefix=False)
if image:
c.image(1)
got = c.submit(1, prompt)
c.close()
return got
def assertSameReadOut(self, got, cold, positions, what):
for pos in positions:
self.assertIn(pos, cold["echo"], f"{what}: the cold run did not read position {pos}")
self.assertIn(pos, got["echo"], f"{what}: position {pos} was not read out")
self.assertEqual(got["echo"][pos], cold["echo"][pos],
f"{what}: position {pos} differs from a cold recompute "
f"-- the state resumed is not the state it claims to be")
def test_options_that_cross_the_window_resume_from_the_photo_exactly(self):
w = window()
self.assertGreater(len(QUESTION.encode()), 2 * w,
"the photo must outgrow the window, or its ring never wraps")
self.assertGreater(len(OPTIONS[0].encode()), 4 * w,
"the long option must slide the window past the photo")
eng = self.client()
photo = eng.submit(1, QUESTION, ext=" logprobs=3 pin=1")
n = photo["accept"]
self.assertEqual(n, len(QUESTION.encode()))
warm = []
for i, option in enumerate(OPTIONS):
warm.append(eng.submit(10 + i, QUESTION + option))
if i == 1:
# An ordinary chat turn that extends the photo. The live state holds
# the option before it, so it resumes from the photo too -- and it
# must not take the photo away from the options after it.
chat = eng.submit(20, QUESTION + " maybe", max_tokens=4, ext="")
self.assertEqual(chat["end"][0], "DONE")
self.assertEqual(chat["echo"], {}, "a chat turn was read out unasked")
log = eng.close()
self.assertIn("[PIN] photo of %d tokens" % n, log)
self.assertEqual(log.count("[PIN] photo of"), 1, "only the pin=1 request takes a photo")
lengths = [len(option.encode()) for option in OPTIONS]
# The saving, from the engine's own account: every option resumed from the
# photo and prefilled ITS tokens, not the shared prefix again.
expected = [(n, k) for k in lengths]
expected.insert(2, (n, len(" maybe"))) # the chat turn
self.assertEqual(resumed(log), expected,
"an option did not resume from the photo:\n" + log)
for option, k, got in zip(OPTIONS, lengths, warm):
with self.subTest(option=option):
# ...and from the wire: the read-out starts exactly where the photo
# ends -- the first option token predicted by the photo's own logits
# -- and covers the option and nothing before it.
self.assertEqual(sorted(got["echo"]), list(range(n, n + k)))
self.assertFalse(any(t.startswith("nan") for t in got["echo"].values()),
"an option token lost its predictor")
cold = self.cold(QUESTION + option)
self.assertEqual(sorted(cold["echo"]), list(range(0, n + k)),
"a cold read-out covers every position")
self.assertSameReadOut(got, cold, range(n, n + k), repr(option))
def test_nested_photos_and_a_sibling_question(self):
eng = self.client()
s = eng.submit(1, STATE, ext=" logprobs=1 pin=1")["accept"]
q = eng.submit(2, QUESTION, ext=" logprobs=1 pin=1")["accept"]
first = eng.submit(3, QUESTION + OPTIONS[2])
# Same state, another question: resumes from the state, and its prefill
# overwrites the rows the deeper photo stood on.
sibling_pin = eng.submit(4, SIBLING, ext=" logprobs=1 pin=1")
sibling = eng.submit(5, SIBLING + " the release manager")
back = eng.submit(6, QUESTION + OPTIONS[1])
log = eng.close()
self.assertEqual(sorted(first["echo"])[0], q)
self.assertEqual(sorted(sibling_pin["echo"])[0], s)
self.assertEqual(sorted(sibling["echo"])[0], sibling_pin["accept"])
# The question's photo is stale now: released, and the state's taken instead
# of starting over.
self.assertIn("[PIN] released the photo of %d tokens" % q, log)
self.assertEqual(sorted(back["echo"])[0], s)
for got, prompt, start in ((first, QUESTION + OPTIONS[2], q),
(sibling, SIBLING + " the release manager",
sibling_pin["accept"]),
(back, QUESTION + OPTIONS[1], s)):
cold = self.cold(prompt)
self.assertSameReadOut(got, cold, range(start, len(prompt.encode())), prompt)
def test_a_photo_whose_rows_were_overwritten_is_not_restored(self):
other = "Context: the invoice is overdue by ninety days.\nQuestion: escalate?\nAnswer:"
eng = self.client()
n = eng.submit(1, QUESTION, ext=" logprobs=1 pin=1")["accept"]
eng.submit(2, other, max_tokens=4, ext="")
after = eng.submit(3, QUESTION + OPTIONS[2])
log = eng.close()
self.assertIn("[PIN] released the photo of %d tokens" % n, log)
self.assertEqual(resumed(log), [])
total = len((QUESTION + OPTIONS[2]).encode())
self.assertEqual(sorted(after["echo"]), list(range(total)),
"with no photo the read-out starts over from position 0")
self.assertSameReadOut(after, self.cold(QUESTION + OPTIONS[2]), range(1, total),
"after a divergent prompt")
@unittest.skipUnless(fixture.has_vision(), "the fixture carries no picture")
def test_an_image_prompt_never_resumes_from_a_photo(self):
"""The prompt begins with the photo's ids, so the ids alone would say
resume. They do not describe the picture, and the image rows are spliced
by their order in the whole prompt: the engine starts over."""
_payload, _h, _w, pads = fixture.image_payload()
prompt = (QUESTION + " " + fixture.VISION_START + fixture.IMAGE_PAD * pads
+ fixture.VISION_END + " so:")
eng = self.client()
eng.submit(1, QUESTION, ext=" logprobs=1 pin=1")
eng.image(2)
got = eng.submit(2, prompt)
log = eng.close()
self.assertEqual(got["end"][0], "DONE", got["end"])
self.assertEqual(resumed(log), [], "an image prompt resumed from a photo:\n" + log)
self.assertEqual(sorted(got["echo"]), list(range(got["accept"])))
cold = self.cold(prompt, image=True)
self.assertSameReadOut(got, cold, range(1, got["accept"]), "image prompt")
@unittest.skipUnless(fixture.has_vision(), "the fixture carries no picture")
def test_a_prompt_with_a_picture_takes_no_photo(self):
"""pin=1 on a prompt with a picture: the photo would be filed under ids
that do not describe the picture, and a later TEXT prompt with the same ids
(the pads as plain tokens) would resume from a state that saw an image."""
_payload, _h, _w, pads = fixture.image_payload()
pictured = (STATE + fixture.VISION_START + fixture.IMAGE_PAD * pads
+ fixture.VISION_END + "\nAnswer:")
eng = self.client()
eng.image(1)
eng.submit(1, pictured, ext=" logprobs=1 pin=1")
text_only = eng.submit(2, pictured + OPTIONS[1])
log = eng.close()
self.assertIn("[PIN] no photo: the prompt carries a picture", log)
self.assertNotIn("[PIN] photo of", log)
self.assertEqual(resumed(log), [])
total = text_only["accept"]
self.assertEqual(sorted(text_only["echo"]), list(range(total)),
"a text prompt resumed from a state that saw a picture")
cold = self.cold(pictured + OPTIONS[1])
self.assertSameReadOut(text_only, cold, range(1, total), "text after a picture")
def test_the_read_out_covers_every_position_unless_a_photo_does(self):
"""What the gateway's `echo` needs: one frame per prompt position, the
first carrying nan because nothing precedes it. A live prefix brings no
predictor for its last position, so a read-out never resumes from one."""
opening = "The capital of France is"
eng = self.client()
eng.submit(1, opening, max_tokens=1, ext="")
got = eng.submit(2, opening + " a city, and the capital of Spain is")
log = eng.close()
self.assertNotIn("[PREFIX] reusing", log, "a read-out resumed from a live prefix")
n = got["accept"]
self.assertEqual(sorted(got["echo"]), list(range(n)))
self.assertTrue(got["echo"][0].startswith("nan 0"), got["echo"][0])
for pos in range(1, n):
lp = float(got["echo"][pos].split()[0])
self.assertTrue(math.isfinite(lp) and lp <= 0.0, (pos, lp))
self.assertEqual(int(got["echo"][pos].split()[1]), 3, "the top-k asked for")
def test_logprobs_change_no_generated_byte(self):
prompt = "Streaming experts from disk means that the decode rate depends on"
plain = self.client()
a = plain.submit(1, prompt, max_tokens=12, ext="")
plain.close()
scored = self.client()
b = scored.submit(1, prompt, max_tokens=12, ext=" logprobs=4")
scored.close()
self.assertTrue(a["data"], "nothing was generated")
self.assertEqual([d for d, _t in a["data"]], [d for d, _t in b["data"]])
self.assertEqual(a["end"][:4], b["end"][:4])
self.assertTrue(all(t == "" for _d, t in a["data"]), "a tail nobody asked for")
for _d, tail in b["data"]:
fields = tail.split()
self.assertEqual(int(fields[1]), 4)
self.assertEqual(len(fields), 2 + 2 * 4)
def test_a_read_only_request_may_fill_the_context(self):
eng = self.client(CTX="32")
full = eng.submit(1, "a" * 32)
self.assertEqual(full["end"][:4], ["DONE", "1", "STAT", "0"])
self.assertEqual(sorted(full["echo"]), list(range(32)))
self.assertFalse(full["data"])
refused = eng.submit(2, "a" * 32, max_tokens=1, ext="")
self.assertEqual(refused["end"][:3], ["ERROR", "2", "CONTEXT_EXCEEDED"])
over = eng.submit(3, "a" * 33)
self.assertEqual(over["end"][:3], ["ERROR", "3", "CONTEXT_EXCEEDED"])
self.assertFalse(over["echo"])
self.assertEqual(eng.submit(4, "a" * 4, max_tokens=1, ext="")["end"][0], "DONE")
eng.close()
if __name__ == "__main__":
unittest.main()