Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

mimo: gateway, registry, build and release

JustVugg429551a313ee
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ac865f4a..7e531b7e 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -231,7 +231,7 @@ jobs:
run: |
cd c
rc=0
- ENGINES="colibri glm53 inkling kimi_k3 olmoe qwen36 qwen38 deepseek_v41 qwenimage"
+ ENGINES="colibri glm53 inkling kimi_k3 olmoe qwen36 qwen38 deepseek_v41 mimo qwenimage"
if [ "${{ matrix.v4 }}" = "1" ]; then ENGINES="$ENGINES deepseek-v4"; fi
for t in $ENGINES; do
echo "::group::$t"
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index ff3055e5..2fcc7ead 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -67,10 +67,10 @@ jobs:
- name: Build engines
run: |
cd c
- for t in colibri glm53 inkling kimi_k3 olmoe qwen36 qwen38 deepseek_v41 qwenimage; do
+ for t in colibri glm53 inkling kimi_k3 olmoe qwen36 qwen38 deepseek_v41 mimo qwenimage; do
make $t ${{ matrix.make_args }}
done
- ls -lh colibri${{ matrix.ext }} glm53${{ matrix.ext }} inkling${{ matrix.ext }} kimi_k3${{ matrix.ext }} olmoe${{ matrix.ext }} qwen36${{ matrix.ext }} qwen38${{ matrix.ext }} deepseek_v41${{ matrix.ext }}
+ ls -lh colibri${{ matrix.ext }} glm53${{ matrix.ext }} inkling${{ matrix.ext }} kimi_k3${{ matrix.ext }} olmoe${{ matrix.ext }} qwen36${{ matrix.ext }} qwen38${{ matrix.ext }} deepseek_v41${{ matrix.ext }} mimo${{ matrix.ext }}
# DeepSeek V4 has its own Makefile and its own target name (`deepseek-v4`,
# producing `deepseek_v4`), so it never joined the loop above -- and #858 is
@@ -133,6 +133,7 @@ jobs:
cp c/qwen38${{ matrix.ext }} dist/qwen38${{ matrix.ext }}
cp c/qwenimage${{ matrix.ext }} dist/qwenimage${{ matrix.ext }}
cp c/deepseek_v41${{ matrix.ext }} dist/deepseek_v41${{ matrix.ext }}
+ cp c/mimo${{ matrix.ext }} dist/mimo${{ matrix.ext }}
# deepseek_v4 -- underscore, not the hyphen of the make target: engine_for()
# looks for "deepseek_v4" next to coli. Guarded, not unconditional, because
# the macos-arm64 archive has no such engine to ship (#858).
@@ -192,7 +193,7 @@ jobs:
# a green build, green tests, and an archive with no engine for the model the
# user actually had. Presence is not enough -- assert they are executable, and
# that the launcher's own resolver finds them where it looks (next to coli).
- SIBLINGS="inkling kimi_k3 olmoe qwen36 qwen38"
+ SIBLINGS="inkling kimi_k3 olmoe qwen36 qwen38 mimo"
# deepseek_v4 only where the engine builds (COLI_V4_SUPPORTED); asserting it
# unconditionally would fail the macos-arm64 archive for shipping an engine
# that platform cannot have.
diff --git a/c/coli b/c/coli
index b707109b..cee8fbed 100755
--- a/c/coli
+++ b/c/coli
@@ -538,7 +538,7 @@ def env_for_engine(a, arch, plan=None):
# #855; before that `grep -c RAM_GB c/kimi_k3.c` returned 0, so `coli chat
# --ram 242` on Kimi K3 set an environment variable nobody looked at and the
# user's session ran itself out of memory with the flag apparently set.
- if arch in ("deepseek_v4", "deepseek_v41", "kimi", "glm53", "olmoe"):
+ if arch in ("deepseek_v4", "deepseek_v41", "kimi", "glm53", "olmoe", "mimo"):
if a.ram: env["RAM_GB"] = str(a.ram)
if arch == "glm53":
# I densi vanno a int4 di default: sono 9,7 B parametri su 321, e in
diff --git a/c/family_registry.py b/c/family_registry.py
index bffcafad..be74bdf7 100644
--- a/c/family_registry.py
+++ b/c/family_registry.py
@@ -818,6 +818,61 @@ def _dsv41_geometry(config, context, _model_dir):
return PlannerGeometry(state, fixed, workspace, experts)
+def _mimo_geometry(config, context, _model_dir):
+ """MiMo-V2.6: what mimo.c allocates per context (kv_alloc), in f32.
+
+ full-attention layers context * kv_heads * (head_dim + v_head_dim) * 4
+ sliding-window layers min(window, context) * swa_kv_heads * (swa_head_dim +
+ swa_v_head_dim) * 4, a ring that never grows (fixed)
+
+ hybrid_layer_pattern says which is which (1 = sliding window). On Flash 39 of
+ the 48 layers are windowed, so a 1M context costs the KV of 9 layers.
+ Workspace mirrors forward()'s per-block buffers at the default 64-row block,
+ plus the full-attention score row per head.
+ """
+ layers = _required_int(config, "num_hidden_layers", "mimo")
+ experts = _required_int(config, "n_routed_experts", "mimo")
+ hidden = _required_int(config, "hidden_size", "mimo")
+ kv = _required_int(config, "num_key_value_heads", "mimo")
+ head_dim = _required_int(config, "head_dim", "mimo")
+ v_dim = config.get("v_head_dim", head_dim)
+ swa_kv = config.get("swa_num_key_value_heads", kv)
+ swa_hd = config.get("swa_head_dim", head_dim)
+ swa_vd = config.get("swa_v_head_dim", v_dim)
+ heads = _required_int(config, "num_attention_heads", "mimo")
+ for name, value in (("v_head_dim", v_dim), ("swa_num_key_value_heads", swa_kv),
+ ("swa_head_dim", swa_hd), ("swa_v_head_dim", swa_vd)):
+ if not isinstance(value, int) or isinstance(value, bool) or value < 1:
+ raise ValueError(f"mimo: {name} must be a positive integer")
+ pattern = config.get("hybrid_layer_pattern")
+ if not isinstance(pattern, list) or len(pattern) != layers:
+ raise ValueError("mimo: hybrid_layer_pattern must list every layer")
+ windowed = sum(1 for kind in pattern if kind == 1)
+ window = _required_int(config, "sliding_window", "mimo") if windowed else 0
+ state = (layers - windowed) * context * kv * (head_dim + v_dim) * 4
+ fixed = windowed * min(window, context) * swa_kv * (swa_hd + swa_vd) * 4
+ block = 64
+ workspace = (block * hidden * 6 + block * heads * (head_dim + v_dim) * 2
+ + heads * context) * 4
+ return PlannerGeometry(state, fixed, workspace, experts)
+
+
+# What the checkpoint carries and mimo.c never loads: the three MTP layers (no
+# speculative decoding yet) and the audio encoder (text and images only).
+_MIMO_NOT_LOADED = re.compile(r"^(?:model\.mtp\.|audio_encoder\.|speech_embeddings\.)")
+
+
+def _mimo_expert_inventory(name, size, _config, _dtype=None):
+ match = _GLM_EXPERT.search(name)
+ if match is None or name.startswith("model.mtp."):
+ return ()
+ return ((int(match.group(1)), int(match.group(2)), size),)
+
+
+def _mimo_resident_inventory(name, size, _config, _dtype=None):
+ return 0 if _MIMO_NOT_LOADED.match(name) else size
+
+
_DSV41_MTP_EXPERT = re.compile(r"^mtp\.(\d+)\.ffn\.experts\.(\d+)\.")
@@ -1465,6 +1520,52 @@ FAMILIES = (
# coli chat, coli serve and coli web already use.
has_cli_adapter=False,
),
+ FamilyDescriptor(
+ id="mimo",
+ model_types=("mimo_v2",),
+ display_name="MiMo-V2.6 Flash",
+ display_scale="309B",
+ # XiaomiMiMo/MiMo-V2.6-{Flash,Pro}: same architecture, two sizes.
+ reference_experts=256,
+ display_variants=(
+ DisplayVariant((("hidden_size", 4096), ("num_hidden_layers", 48),
+ ("n_routed_experts", 256)),
+ "MiMo-V2.6 Flash", "309B"),
+ DisplayVariant((("hidden_size", 6144), ("num_hidden_layers", 70),
+ ("n_routed_experts", 384)),
+ "MiMo-V2.6 Pro", "1.02T", model_id="mimo-v2.6-pro"),
+ ),
+ engine_artifact="mimo",
+ engine_aliases=(),
+ engine_group="mimo",
+ internal_arch="mimo",
+ build_target="mimo",
+ process_names=("mimo",),
+ default_model_id="mimo-v2.6-flash",
+ cli_adapter="mimo",
+ gateway_adapter="mimo",
+ planner_id="mimo",
+ planner_geometry=_mimo_geometry,
+ planner_unsupported_reason="",
+ # CPU-only: mimo.c links no accelerator backend (see deepseek_v41 above for
+ # why the planner must not offer a VRAM tier the engine cannot use).
+ supports_accelerator=False,
+ expert_inventory=_mimo_expert_inventory,
+ resident_inventory=_mimo_resident_inventory,
+ config_section="root",
+ # top-8 routing: the engine raises any smaller cache to one routing step
+ limits=FamilyLimits(8192, 1048576, 1024, 16384, 1, 16, "CTX"),
+ # tools (the XML call form, parse_mimo_tool_calls), thinking (on by default,
+ # as the template has it); no grammars, no audio in or out
+ capabilities=FamilyCapabilities(True, False, False, True),
+ has_gateway_adapter=True,
+ # the engine is driven through the gateway (SERVE); coli run has no mimo
+ # branch and would fall through to GLM's binary, as for deepseek_v41
+ has_cli_adapter=False,
+ # the gateway's cue with thinking on (render_chat_mimo): the template's
+ # ChatML with no newline after <|im_end|>, and the <think> the model writes
+ tune_prompt_template="<|im_start|>user\n{prompt}<|im_end|><|im_start|>assistant\n<think>",
+ ),
FamilyDescriptor(
id="qwen_image",
# A diffusers pipeline has no config.json at its root: the family is
diff --git a/c/openai_server.py b/c/openai_server.py
index fdab0091..9b74cd47 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -966,6 +966,8 @@ def _parse_arch_tool_calls(reply, tools, tool_reply, track_spans):
return parse_k3_tool_calls(reply, tools) + (None, None) # pre-#1147 engines
if chat_flavor() in ("qwen36", "qwen38"):
return parse_qwen_tool_calls(reply, tools) + (None, None)
+ if ARCH == "mimo":
+ return parse_mimo_tool_calls(reply, tools) + (None, None)
return _parse_tool_calls(reply, tools, track_spans)
@@ -1938,6 +1940,172 @@ def parse_qwen_tool_calls(reply, tools=None):
parse_qwen38_tool_calls = parse_qwen_tool_calls
+# ---- MiMo-V2.6 (Xiaomi) -------------------------------------------------------------------
+# ChatML without a newline after <|im_end|>, every assistant turn carrying its <think> block
+# (empty or not), tools declared in their own system turn, calls written INLINE
+# (<tool_call><function=N><parameter=K>V</parameter></function></tool_call>, no newlines) and
+# tool results as a plain `tool` turn. All of it from the release's chat_template.jinja
+# (XiaomiMiMo/MiMo-V2.6-Flash-MOPD); tests/test_mimo_chat_template.py renders that template
+# with jinja2 and compares byte for byte.
+
+MIMO_TOOLS_HEAD = "You are provided with the following tools:\n\n<tools>"
+
+
+def _mimo_tools(tools):
+ """render_tools(): each tool object as JSON, key order kept (transformers' tojson)."""
+ return (MIMO_TOOLS_HEAD
+ + "".join("\n" + json.dumps(tool, ensure_ascii=False) for tool in tools)
+ + "\n</tools>")
+
+
+def _mimo_value(value):
+ """render_value(): a string as-is, anything else as JSON."""
+ return value if isinstance(value, str) else json.dumps(value, ensure_ascii=False)
+
+
+def _mimo_tool_calls(calls, index):
+ out = []
+ for position, call in enumerate(calls):
+ where = f"messages.{index}.tool_calls.{position}"
+ if not isinstance(call, dict):
+ raise APIError(400, "Each tool call must be an object.", where)
+ fn = call.get("function", call.get("custom", call))
+ if not isinstance(fn, dict) or not isinstance(fn.get("name"), str) or not fn["name"]:
+ raise APIError(400, "A tool call needs a `function.name`.", f"{where}.function")
+ out.append(f"<tool_call><function={fn['name']}>")
+ if isinstance(fn.get("input"), str):
+ out.append(fn["input"])
+ else:
+ args = fn.get("arguments")
+ # An OpenAI client sends the arguments back as a JSON string. The template
+ # would print that string raw, a shape the model never writes; read as the
+ # object it is, the call comes back in the <parameter=...> form the model
+ # produced, which is also what keeps the resent history on the KV prefix.
+ if isinstance(args, str) and args.strip():
+ try:
+ args = json.loads(args)
+ except (TypeError, ValueError):
+ raise APIError(400, "`function.arguments` must be a JSON object.",
+ f"{where}.function.arguments")
+ if isinstance(args, dict):
+ for key, value in args.items():
+ out.append(f"<parameter={key}>{_mimo_value(value)}</parameter>")
+ elif args not in (None, "", {}):
+ raise APIError(400, "`function.arguments` must be a JSON object.",
+ f"{where}.function.arguments")
+ out.append("</function></tool_call>")
+ return "".join(out)
+
+
+def render_chat_mimo(messages, enable_thinking=True, reasoning_effort=None, tools=None,
+ tool_choice=None, add_generation_prompt=True):
+ """MiMo-V2.6's chat template.
+
+ One deliberate addition: with thinking on, the generation cue ends in `<think>`. The
+ template leaves that token to the model, and the model writes it first on every turn
+ it was trained on (each assistant turn of the template opens with it); putting it in
+ the prompt is what lets the reasoning split start inside the reasoning, where the
+ model is. With thinking off the template's own `<think></think>` closes it.
+
+ add_generation_prompt=False continues a trailing assistant turn: its past-turn render
+ minus the closing <|im_end|>, and no cue."""
+ if not isinstance(messages, list) or not messages:
+ raise APIError(400, "`messages` must be a non-empty array.", "messages")
+ if tool_choice in ("none",):
+ tools = None
+ if tools is not None and not isinstance(tools, list):
+ raise APIError(400, "`tools` must be an array.", "tools")
+ parts = []
+ if tools:
+ parts.append(f"<|im_start|>system\n{_mimo_tools(tools)}<|im_end|>")
+ last = len(messages) - 1
+ for index, message in enumerate(messages):
+ if not isinstance(message, dict):
+ raise APIError(400, "Each message must be an object.", f"messages.{index}")
+ role = message.get("role")
+ if role == "developer":
+ role = "system"
+ if role not in ("system", "user", "assistant", "tool"):
+ raise APIError(400, f"Unsupported role {role!r}.", f"messages.{index}.role")
+ raw = message.get("content")
+ body = content_text(raw, f"messages.{index}.content") if raw is not None else ""
+ if role == "assistant":
+ reasoning = message.get("reasoning_content")
+ if reasoning is not None and not isinstance(reasoning, str):
+ raise APIError(400, "`reasoning_content` must be a string.",
+ f"messages.{index}.reasoning_content")
+ turn = f"<|im_start|>assistant\n<think>{reasoning or ''}</think>{body}"
+ calls = message.get("tool_calls")
+ if calls:
+ if not isinstance(calls, list):
+ raise APIError(400, "`tool_calls` must be an array.",
+ f"messages.{index}.tool_calls")
+ turn += _mimo_tool_calls(calls, index)
+ if not add_generation_prompt and index == last:
+ parts.append(turn) # the open turn: no terminator, no cue
+ return "".join(parts)
+ parts.append(turn + "<|im_end|>")
+ continue
+ parts.append(f"<|im_start|>{role}\n{body}<|im_end|>")
+ if add_generation_prompt:
+ parts.append("<|im_start|>assistant\n")
+ parts.append("<think>" if enable_thinking else "<think></think>")
+ return "".join(parts)
+
+
+MIMO_CALL_RE = re.compile(
+ r"<tool_call>\s*<function=([^>\n]+)>(.*?)</function>\s*</tool_call>", re.S)
+MIMO_PARAM_RE = re.compile(r"<parameter=([^>\n]+)>(.*?)</parameter>", re.S)
+
+
+def parse_mimo_tool_calls(reply, tools=None):
+ """MiMo's calls back into OpenAI `tool_calls`.
+
+ Parameters are inline (`<parameter=K>V</parameter>`); a value that arrives on its own
+ lines, Qwen-style, loses exactly one newline on each side. Types come from the declared
+ schema as for Qwen: a string parameter stays text, anything else is read as JSON. A
+ body with no parameter tags but a JSON object in it is the template's other spelling
+ (`arguments` as a string) and is taken as the arguments."""
+ schema = {}
+ for tool in (tools or []):
+ fn = tool.get("function", tool) if isinstance(tool, dict) else {}
+ name = fn.get("name")
+ params = (fn.get("parameters") or {}).get("properties") or {}
+ if isinstance(name, str) and name and isinstance(params, dict):
+ schema[name] = params
+
+ def make_call(name, body):
+ args = {}
+ found = MIMO_PARAM_RE.findall(body)
+ for key, raw in found:
+ key = key.strip()
+ if raw.startswith("\n"):
+ raw = raw[1:]
+ if raw.endswith("\n"):
+ raw = raw[:-1]
+ declared = (schema.get(name) or {}).get(key) or {}
+ kind = declared.get("type") if isinstance(declared, dict) else None
+ args[key] = _coerce_arg(raw, kind if isinstance(kind, str) else None) \
+ if kind not in (None, "string") else raw
+ if not found and body.strip():
+ try:
+ parsed = json.loads(body)
+ if isinstance(parsed, dict):
+ args = parsed
+ except (TypeError, ValueError):
+ pass
+ return {"id": f"call_{uuid.uuid4().hex[:24]}", "type": "function",
+ "function": {"name": name, "arguments": json.dumps(args, ensure_ascii=False)}}
+
+ text = reply or ""
+ calls = [make_call(m.group(1).strip(), m.group(2)) for m in MIMO_CALL_RE.finditer(text)]
+ if not calls and tools and ("<tool_call>" in text or "<function=" in text):
+ sys.stderr.write("[api] mimo tool markers present but no call parsed -- "
+ "possibly truncated or mangled output\n")
+ sys.stderr.flush()
+ return MIMO_CALL_RE.sub("", text).strip(), calls
+
+
def render_chat_qwen38(messages, enable_thinking=True, reasoning_effort=None, tools=None,
tool_choice=None, add_generation_prompt=True):
"""Text-only Qwen3.8 chat-template subset with native reasoning hints.
@@ -2904,7 +3072,7 @@ def render_chat_dsv41(messages, enable_thinking=False, reasoning_effort=None, to
# on by default, and a family without its open-turn shape yet must not start rejecting requests
# nobody opted into. Each renderer adds itself here in the same commit that derives its shape.
CONTINUATION_FAMILIES = {"glm53", "qwen38", "qwen36", "glm", "olmoe", "deepseek_v4", "inkling",
- "kimi", "deepseek_v41"}
+ "kimi", "deepseek_v41", "mimo"}
def resolve_generation_prompt(messages, body):
@@ -3016,6 +3184,9 @@ def render_chat_for_arch(messages, enable_thinking=False, reasoning_effort=None,
if ARCH == "deepseek_v41":
return render_chat_dsv41(messages, enable_thinking, reasoning_effort, tools,
tool_choice, add_generation_prompt)
+ if ARCH == "mimo":
+ return render_chat_mimo(messages, enable_thinking, reasoning_effort, tools,
+ tool_choice, add_generation_prompt)
return render_chat(messages, enable_thinking, reasoning_effort, tools, tool_choice)
@@ -4265,8 +4436,8 @@ def cap_for_arch(arch, cap, env=None, model=None):
planned = 0
if planned >= 1:
return planned
- if arch == "deepseek_v41" and model is not None:
- # V4.1 only reads its argv cap, not RAM_GB. Without --auto-tier the
+ if arch in ("deepseek_v41", "mimo") and model is not None:
+ # V4.1 and MiMo only read their argv cap, not RAM_GB. Without --auto-tier the
# legacy eight slots silently discarded both --ram and RAM_GB (#1666).
from resource_plan import build_plan
settings = env if env is not None else os.environ
@@ -4277,9 +4448,10 @@ def cap_for_arch(arch, cap, env=None, model=None):
gpu_indices=[])
slots = plan["tiers"]["ram"]["cache_slots_per_layer"]
if slots < 1:
- raise ValueError("DeepSeek V4.1 RAM budget cannot hold one expert slot per layer")
- print(f"[v41] RAM plan: {slots} expert cache slots/layer; --cap overrides",
- file=sys.stderr)
+ raise ValueError(f"{family_by_id(arch).display_name} RAM budget cannot hold one "
+ f"expert slot per layer")
+ print(f"[{'v41' if arch == 'deepseek_v41' else arch}] RAM plan: {slots} expert cache "
+ f"slots/layer; --cap overrides", file=sys.stderr)
return slots
return family_by_id(arch).limits.implicit_cap
@@ -6666,7 +6838,8 @@ class APIHandler(BaseHTTPRequestHandler):
# preserve the older opt-in default for the other families.
if chat_flavor() == "qwen38":
reasoning_effort = "xhigh"
- elif os.environ.get("COLI_THINK", "0") == "1":
+ elif ARCH == "mimo" or os.environ.get("COLI_THINK", "0") == "1":
+ # MiMo-V2.6's template thinks unless told not to (enable_thinking false)
reasoning_effort = "high"
enable_thinking = body.get("enable_thinking", reasoning_effort not in (None, "none"))
if not isinstance(enable_thinking, bool):
@@ -6718,6 +6891,12 @@ class APIHandler(BaseHTTPRequestHandler):
ceiling = os.environ.get("V41_MAX_IMAGE_TOKENS")
messages, images = expand_dsv41_images(messages, model_dir,
int(ceiling) if ceiling else None)
+ elif ARCH == "mimo":
+ # MiMo-V2.6's ViT reads the Qwen2-VL processor's patches, with the same
+ # placeholders: the Qwen expander serves it unchanged.
+ ceiling = os.environ.get("MIMO_MAX_IMAGE_TOKENS")
+ messages, images = expand_qwen38_images(messages, model_dir,
+ int(ceiling) if ceiling else None)
elif ARCH == "qwen38" or (ARCH == "qwen36" and qwen36_has_vision(model_dir)):
# Qwen3.5/3.6/3.8 share the tower and the preprocessor, so qwen36
# checkpoints converted with their tower take the same path (#1757).
@@ -6748,7 +6927,7 @@ class APIHandler(BaseHTTPRequestHandler):
raise APIError(400, "`thinking` must be an object.", "thinking")
enable_thinking = bool(thinking and thinking.get("type") == "enabled")
if not enable_thinking and thinking is None:
- if chat_flavor() == "qwen38":
+ if chat_flavor() == "qwen38" or ARCH == "mimo":
enable_thinking = True
elif os.environ.get("COLI_THINK", "0") == "1":
enable_thinking = True
diff --git a/c/tests/fixtures/mimo_v26_chat_template.jinja b/c/tests/fixtures/mimo_v26_chat_template.jinja
new file mode 100644
index 00000000..92597ade
--- /dev/null
+++ b/c/tests/fixtures/mimo_v26_chat_template.jinja
@@ -0,0 +1,95 @@
+{%- macro render_value(value) -%}
+ {%- if value is string -%}
+ {{- value -}}
+ {%- else -%}
+ {{- value | tojson(ensure_ascii=False) -}}
+ {%- endif -%}
+{%- endmacro -%}
+
+{%- macro render_content(message_content) -%}
+ {%- if message_content is string -%}
+ {{- message_content -}}
+ {%- elif message_content is iterable -%}
+ {%- for part in message_content -%}
+ {%- if part is not mapping -%}
+ {{- part -}}
+ {%- elif part['type'] == 'image' or 'image' in part or 'image_url' in part -%}
+ {{- '<|vision_start|><|image_pad|><|vision_end|>' -}}
+ {%- elif part['type'] == 'audio' or part['type'] == 'input_audio' or 'audio' in part or 'audio_url' in part or 'input_audio' in part -%}
+ {{- '<|mimo_audio_start|><|audio_pad|><|mimo_audio_end|>' -}}
+ {%- elif part['type'] == 'video' or 'video' in part or 'video_url' in part -%}
+ {{- '<|vision_start|><|video_pad|><|vision_end|>' -}}
+ {%- elif 'text' in part -%}
+ {{- part['text'] -}}
+ {%- endif -%}
+ {%- endfor -%}
+ {%- endif -%}
+{%- endmacro -%}
+
+{%- macro render_tools(tools) -%}
+ {{- 'You are provided with the following tools:\n\n<tools>' -}}
+ {%- for tool in tools -%}
+ {{- '\n' ~ (tool | tojson(ensure_ascii=False)) -}}
+ {%- endfor -%}
+ {{- '\n</tools>' -}}
+{%- endmacro -%}
+
+{%- macro render_tool_calls(tool_calls) -%}
+ {%- for tool_call in tool_calls -%}
+ {%- if tool_call.function is defined -%}
+ {%- set tool_call = tool_call.function -%}
+ {%- elif tool_call.custom is defined -%}
+ {%- set tool_call = tool_call.custom -%}
+ {%- endif -%}
+ {{- '<tool_call><function=' ~ tool_call.name ~ '>' -}}
+ {%- if tool_call.input is defined and tool_call.input is string -%}
+ {{- tool_call.input -}}
+ {%- elif tool_call.arguments -%}
+ {%- if tool_call.arguments is string -%}
+ {{- tool_call.arguments -}}
+ {%- else -%}
+ {%- for args_name, args_value in tool_call.arguments | items -%}
+ {{- '<parameter=' ~ args_name ~ '>' ~ render_value(args_value) ~ '</parameter>' -}}
+ {%- endfor -%}
+ {%- endif -%}
+ {%- endif -%}
+ {{- '</function></tool_call>' -}}
+ {%- endfor -%}
+{%- endmacro -%}
+
+{%- macro render_assistant_message(message) -%}
+ {%- set content = render_content(message.content) -%}
+ {%- set reasoning = message.reasoning_content if message.reasoning_content is string else '' -%}
+ {{- '<|im_start|>assistant\n<think>' ~ reasoning ~ '</think>' ~ content -}}
+ {%- if message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 -%}
+ {{- render_tool_calls(message.tool_calls) -}}
+ {%- endif -%}
+ {{- '<|im_end|>' -}}
+{%- endmacro -%}
+
+{%- if tools is defined and tools is iterable and tools | length > 0 -%}
+ {{- '<|im_start|>system\n' ~ render_tools(tools) ~ '<|im_end|>' -}}
+{%- endif -%}
+
+{%- for message in messages -%}
+ {%- if message.role == 'assistant' -%}
+ {{- render_assistant_message(message) -}}
+ {%- else -%}
+ {%- set body = render_content(message.content) -%}
+ {{- '<|im_start|>' ~ message.role ~ '\n' ~ body -}}
+ {%- if message.tools is defined and message.tools is iterable and message.tools | length > 0 -%}
+ {%- if body -%}
+ {{- '\n\n' -}}
+ {%- endif -%}
+ {{- render_tools(message.tools) -}}
+ {%- endif -%}
+ {{- '<|im_end|>' -}}
+ {%- endif -%}
+{%- endfor -%}
+
+{%- if add_generation_prompt -%}
+ {{- '<|im_start|>assistant\n' -}}
+ {%- if enable_thinking is false -%}
+ {{- '<think></think>' -}}
+ {%- endif -%}
+{%- endif -%}
diff --git a/c/tests/segment_conformance_fixtures.c b/c/tests/segment_conformance_fixtures.c
index 9c9ff910..81231130 100644
--- a/c/tests/segment_conformance_fixtures.c
+++ b/c/tests/segment_conformance_fixtures.c
@@ -127,6 +127,15 @@ static const ColiSegmentConformanceFixture g_fixtures[] = {
COLI_SEGMENT_FIXTURE_VISION_TOWER,
6, 8, 4, 64, UINT32_C(0x44533431),
},
+ {
+ "mimo", "MiMo-V2.6", "fixture/mimo-kv-ring-sink-v1",
+ "full-attention KV + sliding-window KV ring with sink logits + vision",
+ "tools/make_mimo_tiny.py",
+ COLI_SEGMENT_FIXTURE_KV |
+ COLI_SEGMENT_FIXTURE_SLIDING_RING |
+ COLI_SEGMENT_FIXTURE_VISION_TOWER,
+ 3, 8, 6, 64, UINT32_C(0x4d494d4f),
+ },
};
static int fail(char *error, size_t error_size, const char *message) {
@@ -378,6 +387,7 @@ DECLARE_OPEN_WRAPPER(qwen36, 5)
DECLARE_OPEN_WRAPPER(qwen38, 6)
DECLARE_OPEN_WRAPPER(deepseek_v4, 7)
DECLARE_OPEN_WRAPPER(deepseek_v41, 8)
+DECLARE_OPEN_WRAPPER(mimo, 9)
#define FIXTURE_ADAPTER(name) \
{ \
@@ -404,6 +414,7 @@ static const ColiSegmentAdapter g_adapters[] = {
FIXTURE_ADAPTER(qwen38),
FIXTURE_ADAPTER(deepseek_v4),
FIXTURE_ADAPTER(deepseek_v41),
+ FIXTURE_ADAPTER(mimo),
};
int coli_segment_conformance_register_fixtures(void) {
diff --git a/c/tests/segment_conformance_manifest.json b/c/tests/segment_conformance_manifest.json
index 2d2bdb49..ab51dee7 100644
--- a/c/tests/segment_conformance_manifest.json
+++ b/c/tests/segment_conformance_manifest.json
@@ -138,6 +138,21 @@
"path": "tools/make_dsv41_tiny.py"
},
"real_adapter_required": true
+ },
+ {
+ "family_id": "mimo",
+ "state_schema": "fixture/mimo-kv-ring-sink-v1",
+ "state_components": [
+ "full_attention_kv",
+ "window_attention",
+ "attention_sink",
+ "vision"
+ ],
+ "oracle": {
+ "kind": "generated_tiny",
+ "path": "tools/make_mimo_tiny.py"
+ },
+ "real_adapter_required": true
}
]
}
diff --git a/c/tests/test_family_registry.py b/c/tests/test_family_registry.py
index a41ac219..d500cb4c 100644
--- a/c/tests/test_family_registry.py
+++ b/c/tests/test_family_registry.py
@@ -1032,6 +1032,8 @@ class FamilyRegistryTest(unittest.TestCase):
# V4.1 ships its chat encoding as a Python module (encoding/encoding.py),
# not a jinja template, so the replay prompt stays the bare text like V4.
"deepseek_v41": "hello {world}",
+ # MiMo-V2.6: render_chat_mimo's cue with thinking on (its template's default)
+ "mimo": "<|im_start|>user\nhello {world}<|im_end|><|im_start|>assistant\n<think>",
# An image model has no chat template and nothing to tune; the
# template is the identity and `coli tune` refuses the modality.
"qwen_image": "hello {world}",
diff --git a/c/tests/test_mimo_chat_template.py b/c/tests/test_mimo_chat_template.py
new file mode 100644
index 00000000..90db3636
--- /dev/null
+++ b/c/tests/test_mimo_chat_template.py
@@ -0,0 +1,224 @@
+#!/usr/bin/env python3
+"""The gateway's MiMo-V2.6 renderer and tool-call parser against the release's template.
+
+The gateway renders prompts by hand instead of running jinja per request, and a hand copy
+can drift from the original without anyone noticing: the model answers anyway. Here the
+real template (tests/fixtures/mimo_v26_chat_template.jinja, MIT, from
+XiaomiMiMo/MiMo-V2.6-Flash-MOPD @ 2479e2d0, sha256 11ea52e1...) is rendered with jinja2 the
+way transformers does and compared byte for byte with render_chat_mimo.
+
+Two documented differences are asserted, not hidden:
+ - with thinking on the gateway's cue ends in `<think>`, the first token the model writes
+ on every turn of this template; the template leaves it to the model;
+ - an OpenAI client sends tool-call arguments as a JSON string, which the template would
+ print raw; the gateway reads it as the object it is, which is what the template renders
+ when the arguments ARE an object. The reference side of each case gets the object.
+"""
+import copy
+import hashlib
+import json
+import sys
+import unittest
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE.parent))
+
+import openai_server # noqa: E402
+
+TEMPLATE = HERE / "fixtures" / "mimo_v26_chat_template.jinja"
+TEMPLATE_SHA256 = "11ea52e156de38a458e6b7720ad45915d65b97d4ec979a09f55e3c9bd1b4d059"
+
+try:
+ import jinja2
+ import jinja2.sandbox
+except ImportError: # pragma: no cover
+ jinja2 = None
+
+WEATHER = {"type": "function", "function": {
+ "name": "get_weather", "description": "Il meteo di una citta'",
+ "parameters": {"type": "object", "properties": {
+ "city": {"type": "string"}, "days": {"type": "integer"},
+ "units": {"type": "array", "items": {"type": "string"}}},
+ "required": ["city"]}}}
+SEARCH = {"type": "function", "function": {
+ "name": "search", "parameters": {"type": "object", "properties": {
+ "query": {"type": "string"}}}}}
+
+CASES = {
+ "one user turn": {"messages": [{"role": "user", "content": "ciao"}]},
+ "system and user": {"messages": [
+ {"role": "system", "content": "Sei conciso."},
+ {"role": "user", "content": "capitale della Francia?"}]},
+ "history with an answer": {"messages": [
+ {"role": "user", "content": "1+1?"},
+ {"role": "assistant", "content": "2"},
+ {"role": "user", "content": "e 2+2?"}]},
+ "history with reasoning": {"messages": [
+ {"role": "user", "content": "1+1?"},
+ {"role": "assistant", "content": "2", "reasoning_content": "uno piu' uno"},
+ {"role": "user", "content": "e 2+2?"}]},
+ "text parts": {"messages": [
+ {"role": "user", "content": [{"type": "text", "text": "prima "},
+ {"type": "text", "text": "seconda"}]}]},
+ "tools declared": {"tools": [WEATHER, SEARCH], "messages": [
+ {"role": "system", "content": "Usa gli strumenti."},
+ {"role": "user", "content": "Che tempo fa a Roma?"}]},
+ "an agent loop": {"tools": [WEATHER, SEARCH], "messages": [
+ {"role": "user", "content": "Meteo a Roma e Milano per 3 giorni?"},
+ {"role": "assistant", "content": "", "tool_calls": [
+ {"id": "a", "type": "function", "function": {
+ "name": "get_weather", "arguments": {"city": "Roma", "days": 3,
+ "units": ["C", "mm"]}}},
+ {"id": "b", "type": "function", "function": {
+ "name": "get_weather", "arguments": {"city": "Milano", "days": 3}}}]},
+ {"role": "tool", "tool_call_id": "a", "content": "sole, 24 gradi"},
+ {"role": "tool", "tool_call_id": "b", "content": "pioggia, 17 gradi"},
+ {"role": "assistant", "content": "A Roma sole, a Milano pioggia."},
+ {"role": "user", "content": "E Napoli?"}]},
+ "an image placeholder": {"messages": [
+ {"role": "user", "content": "<|vision_start|>" + "<|image_pad|>" * 6
+ + "<|vision_end|>Cosa vedi?"}]},
+}
+
+
+def reference_render(case, add_generation_prompt=True, enable_thinking=None):
+ """The template, rendered the way transformers' apply_chat_template does."""
+ env = jinja2.sandbox.ImmutableSandboxedEnvironment(
+ trim_blocks=True, lstrip_blocks=True, extensions=["jinja2.ext.loopcontrols"])
+
+ def tojson(x, ensure_ascii=False, indent=None, separators=None, sort_keys=False):
+ return json.dumps(x, ensure_ascii=ensure_ascii, indent=indent,
+ separators=separators, sort_keys=sort_keys)
+ env.filters["tojson"] = tojson
+ template = env.from_string(TEMPLATE.read_text(encoding="utf-8"))
+ kwargs = {"messages": case["messages"], "add_generation_prompt": add_generation_prompt}
+ if case.get("tools"):
+ kwargs["tools"] = case["tools"]
+ if enable_thinking is not None:
+ kwargs["enable_thinking"] = enable_thinking
+ return template.render(**kwargs)
+
+
+def as_client_sends(messages):
+ """The same history with tool-call arguments as JSON strings, the OpenAI shape."""
+ out = copy.deepcopy(messages)
+ for message in out:
+ for call in message.get("tool_calls") or []:
+ fn = call["function"]
+ if isinstance(fn.get("arguments"), dict):
+ fn["arguments"] = json.dumps(fn["arguments"], ensure_ascii=False)
+ return out
+
+
[email protected](jinja2, "jinja2 is not installed")
+class MimoTemplateTest(unittest.TestCase):
+
+ def setUp(self):
+ self.saved_arch = openai_server.ARCH
+ openai_server.ARCH = "mimo"
+
+ def tearDown(self):
+ openai_server.ARCH = self.saved_arch
+
+ def test_template_fixture_is_the_release(self):
+ digest = hashlib.sha256(TEMPLATE.read_bytes()).hexdigest()
+ self.assertEqual(digest, TEMPLATE_SHA256)
+
+ def test_thinking_off_is_byte_identical(self):
+ for name, case in CASES.items():
+ with self.subTest(name):
+ want = reference_render(case, enable_thinking=False)
+ got = openai_server.render_chat_mimo(
+ as_client_sends(case["messages"]), enable_thinking=False,
+ tools=case.get("tools"))
+ self.assertEqual(got, want)
+
+ def test_thinking_on_adds_only_the_opening_think(self):
+ for name, case in CASES.items():
+ with self.subTest(name):
+ want = reference_render(case) # enable_thinking undefined: thinking
+ got = openai_server.render_chat_mimo(
+ as_client_sends(case["messages"]), enable_thinking=True,
+ tools=case.get("tools"))
+ self.assertEqual(got, want + "<think>")
+
+ def test_open_turn_is_the_past_turn_without_its_terminator(self):
+ case = {"messages": [
+ {"role": "user", "content": "Scrivi una poesia"},
+ {"role": "assistant", "content": "Nel mezzo del", "reasoning_content": "Dante"}]}
+ full = reference_render(case, add_generation_prompt=False)
+ self.assertTrue(full.endswith("<|im_end|>"))
+ got = openai_server.render_chat_mimo(case["messages"], add_generation_prompt=False)
+ self.assertEqual(got, full[:-len("<|im_end|>")])
+
+ def test_dispatch_and_continuation(self):
+ prompt = openai_server.render_chat_for_arch(
+ [{"role": "user", "content": "ciao"}], enable_thinking=False)
+ self.assertEqual(prompt, "<|im_start|>user\nciao<|im_end|><|im_start|>assistant\n"
+ "<think></think>")
+ self.assertIn("mimo", openai_server.CONTINUATION_FAMILIES)
+
+ def test_tool_choice_none_offers_no_tools(self):
+ got = openai_server.render_chat_mimo([{"role": "user", "content": "x"}],
+ tools=[WEATHER], tool_choice="none")
+ self.assertNotIn("<tools>", got)
+
+ def test_bad_arguments_are_a_400(self):
+ messages = [{"role": "assistant", "content": "", "tool_calls": [
+ {"function": {"name": "f", "arguments": "{not json"}}]}]
+ with self.assertRaises(openai_server.APIError) as caught:
+ openai_server.render_chat_mimo(messages)
+ self.assertEqual(caught.exception.status, 400)
+
+
+class MimoToolParseTest(unittest.TestCase):
+
+ def test_inline_calls_with_typed_arguments(self):
+ reply = ("Controllo.<tool_call><function=get_weather><parameter=city>Roma</parameter>"
+ "<parameter=days>3</parameter><parameter=units>[\"C\", \"mm\"]</parameter>"
+ "</function></tool_call><tool_call><function=search><parameter=query>42"
+ "</parameter></function></tool_call>")
+ text, calls = openai_server.parse_mimo_tool_calls(reply, [WEATHER, SEARCH])
+ self.assertEqual(text, "Controllo.")
+ self.assertEqual([c["function"]["name"] for c in calls], ["get_weather", "search"])
+ self.assertEqual(json.loads(calls[0]["function"]["arguments"]),
+ {"city": "Roma", "days": 3, "units": ["C", "mm"]})
+ # a string parameter stays a string even when it looks like a number
+ self.assertEqual(json.loads(calls[1]["function"]["arguments"]), {"query": "42"})
+
+ def test_values_on_their_own_lines(self):
+ reply = ("<tool_call>\n<function=get_weather>\n<parameter=city>\nRoma\n</parameter>\n"
+ "</function>\n</tool_call>")
+ _text, calls = openai_server.parse_mimo_tool_calls(reply, [WEATHER])
+ self.assertEqual(json.loads(calls[0]["function"]["arguments"]), {"city": "Roma"})
+
+ def test_json_body_is_the_arguments(self):
+ reply = '<tool_call><function=get_weather>{"city": "Roma"}</function></tool_call>'
+ _text, calls = openai_server.parse_mimo_tool_calls(reply, [WEATHER])
+ self.assertEqual(json.loads(calls[0]["function"]["arguments"]), {"city": "Roma"})
+
+ def test_round_trip_through_the_renderer(self):
+ reply = ("<tool_call><function=get_weather><parameter=city>Roma</parameter>"
+ "<parameter=days>3</parameter></function></tool_call>")
+ _text, calls = openai_server.parse_mimo_tool_calls(reply, [WEATHER])
+ rendered = openai_server.render_chat_mimo(
+ [{"role": "user", "content": "x"},
+ {"role": "assistant", "content": "", "tool_calls": calls}],
+ add_generation_prompt=False)
+ self.assertTrue(rendered.endswith("<think></think>" + reply))
+
+ def test_dispatch_in_parse_arch_tool_calls(self):
+ saved = openai_server.ARCH
+ openai_server.ARCH = "mimo"
+ try:
+ _text, calls = openai_server.parse_arch_tool_calls(
+ "<tool_call><function=search><parameter=query>x</parameter></function>"
+ "</tool_call>", [SEARCH])
+ finally:
+ openai_server.ARCH = saved
+ self.assertEqual(calls[0]["function"]["name"], "search")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/c/tests/test_segment_conformance.c b/c/tests/test_segment_conformance.c
index 7693dbbc..a1635628 100644
--- a/c/tests/test_segment_conformance.c
+++ b/c/tests/test_segment_conformance.c
@@ -8,7 +8,7 @@
* other is how this test last failed for a reason unrelated to what it checks. */
static const char *const kExpectedFamilies[] = {
"glm", "glm53", "inkling", "kimi", "olmoe", "qwen36", "qwen38", "deepseek_v4",
- "deepseek_v41",
+ "deepseek_v41", "mimo",
};
static int expected_family(const char *family_id) {
diff --git a/c/tools/clean.py b/c/tools/clean.py
index 517b93a9..aff3b82b 100644
--- a/c/tools/clean.py
+++ b/c/tools/clean.py
@@ -35,6 +35,7 @@ FILES = [
"coli_hip.dll", "coli_hip.lib", "coli_hip.exp", "coli_hip.pdb",
"deepseek_v4", "deepseek_v4.exe", "deepseek_v4.cflags", "deepseek_v4.cudaflags",
"deepseek_v41", "deepseek_v41.exe",
+ "mimo", "mimo.exe",
"qwenimage", "qwenimage.exe",
"native_quant.o", "native_quant_parallel.o", "native_quant_dual.o",
"native_quant_batch_avx512.o", "native_quant_fp4_rows16.o",
diff --git a/c/tools/convert_fp8_to_int4.py b/c/tools/convert_fp8_to_int4.py
index 4dbbb889..3cb12b91 100644
--- a/c/tools/convert_fp8_to_int4.py
+++ b/c/tools/convert_fp8_to_int4.py
@@ -66,6 +66,9 @@ OTHER_FAMILY_PATHS = {
"deepseek_v4": "DeepSeek V4: see docs/deepseek-v4.md, section Download",
"deepseek_v41": "DeepSeek V4.1: no conversion needed -- its experts already ship fp4 and its dense fp8; run tools/prepare_dsv41.py once, see docs/deepseek-v41.md",
"deepseek_v41_text": "DeepSeek V4.1: no conversion needed -- see docs/deepseek-v41.md",
+ "mimo_v2": "MiMo-V2.6 is NOT converted: its experts already ship MXFP4 and its "
+ "dense FP8/BF16, and the engine reads the release as downloaded; "
+ "see docs/mimo.md",
"qwenimage21pipeline": "Qwen-Image-2.1 is NOT converted: the image engine reads the "
"diffusers checkpoint as downloaded; see docs/qwen-image.md",
}