Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

Merge branch 'dev' into fix/tool-choice-required-renderers

Yi-111-a4cab875b4e75
diff --cc c/openai_server.py
index 0410b4a8,3d6847b4..4261818b
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@@ -1962,6 -1999,172 +2023,178 @@@ def parse_qwen_tool_calls(reply, tools=
parse_qwen38_tool_calls = parse_qwen_tool_calls
+ # ---- MiMo-V2.6 (Xiaomi) -------------------------------------------------------------------
+ # ChatML without a newline after <|im_end|>, every assistant turn carrying its <think> block
+ # (empty or not), tools declared in their own system turn, calls written INLINE
+ # (<tool_call><function=N><parameter=K>V</parameter></function></tool_call>, no newlines) and
+ # tool results as a plain `tool` turn. All of it from the release's chat_template.jinja
+ # (XiaomiMiMo/MiMo-V2.6-Flash-MOPD); tests/test_mimo_chat_template.py renders that template
+ # with jinja2 and compares byte for byte.
+
+ MIMO_TOOLS_HEAD = "You are provided with the following tools:\n\n<tools>"
+
+
+ def _mimo_tools(tools):
+ """render_tools(): each tool object as JSON, key order kept (transformers' tojson)."""
+ return (MIMO_TOOLS_HEAD
+ + "".join("\n" + json.dumps(tool, ensure_ascii=False) for tool in tools)
+ + "\n</tools>")
+
+
+ def _mimo_value(value):
+ """render_value(): a string as-is, anything else as JSON."""
+ return value if isinstance(value, str) else json.dumps(value, ensure_ascii=False)
+
+
+ def _mimo_tool_calls(calls, index):
+ out = []
+ for position, call in enumerate(calls):
+ where = f"messages.{index}.tool_calls.{position}"
+ if not isinstance(call, dict):
+ raise APIError(400, "Each tool call must be an object.", where)
+ fn = call.get("function", call.get("custom", call))
+ if not isinstance(fn, dict) or not isinstance(fn.get("name"), str) or not fn["name"]:
+ raise APIError(400, "A tool call needs a `function.name`.", f"{where}.function")
+ out.append(f"<tool_call><function={fn['name']}>")
+ if isinstance(fn.get("input"), str):
+ out.append(fn["input"])
+ else:
+ args = fn.get("arguments")
+ # An OpenAI client sends the arguments back as a JSON string. The template
+ # would print that string raw, a shape the model never writes; read as the
+ # object it is, the call comes back in the <parameter=...> form the model
+ # produced, which is also what keeps the resent history on the KV prefix.
+ if isinstance(args, str) and args.strip():
+ try:
+ args = json.loads(args)
+ except (TypeError, ValueError):
+ raise APIError(400, "`function.arguments` must be a JSON object.",
+ f"{where}.function.arguments")
+ if isinstance(args, dict):
+ for key, value in args.items():
+ out.append(f"<parameter={key}>{_mimo_value(value)}</parameter>")
+ elif args not in (None, "", {}):
+ raise APIError(400, "`function.arguments` must be a JSON object.",
+ f"{where}.function.arguments")
+ out.append("</function></tool_call>")
+ return "".join(out)
+
+
+ def render_chat_mimo(messages, enable_thinking=True, reasoning_effort=None, tools=None,
+ tool_choice=None, add_generation_prompt=True):
+ """MiMo-V2.6's chat template.
+
+ One deliberate addition: with thinking on, the generation cue ends in `<think>`. The
+ template leaves that token to the model, and the model writes it first on every turn
+ it was trained on (each assistant turn of the template opens with it); putting it in
+ the prompt is what lets the reasoning split start inside the reasoning, where the
+ model is. With thinking off the template's own `<think></think>` closes it.
+
+ add_generation_prompt=False continues a trailing assistant turn: its past-turn render
+ minus the closing <|im_end|>, and no cue."""
+ if not isinstance(messages, list) or not messages:
+ raise APIError(400, "`messages` must be a non-empty array.", "messages")
+ if tool_choice in ("none",):
+ tools = None
+ if tools is not None and not isinstance(tools, list):
+ raise APIError(400, "`tools` must be an array.", "tools")
+ parts = []
+ if tools:
- parts.append(f"<|im_start|>system\n{_mimo_tools(tools)}<|im_end|>")
++ tools_text = _mimo_tools(tools)
++ # In the tool turn, after </tools> and before its <|im_end|>: "the functions
++ # above" has to be true, and MiMo's declaration block is a whole system turn
++ # of its own, so there is nothing to put the line outside of but the frame.
++ if tool_choice == "required":
++ tools_text += TOOL_CHOICE_REQUIRED_INSTRUCTION
++ parts.append(f"<|im_start|>system\n{tools_text}<|im_end|>")
+ last = len(messages) - 1
+ for index, message in enumerate(messages):
+ if not isinstance(message, dict):
+ raise APIError(400, "Each message must be an object.", f"messages.{index}")
+ role = message.get("role")
+ if role == "developer":
+ role = "system"
+ if role not in ("system", "user", "assistant", "tool"):
+ raise APIError(400, f"Unsupported role {role!r}.", f"messages.{index}.role")
+ raw = message.get("content")
+ body = content_text(raw, f"messages.{index}.content") if raw is not None else ""
+ if role == "assistant":
+ reasoning = message.get("reasoning_content")
+ if reasoning is not None and not isinstance(reasoning, str):
+ raise APIError(400, "`reasoning_content` must be a string.",
+ f"messages.{index}.reasoning_content")
+ turn = f"<|im_start|>assistant\n<think>{reasoning or ''}</think>{body}"
+ calls = message.get("tool_calls")
+ if calls:
+ if not isinstance(calls, list):
+ raise APIError(400, "`tool_calls` must be an array.",
+ f"messages.{index}.tool_calls")
+ turn += _mimo_tool_calls(calls, index)
+ if not add_generation_prompt and index == last:
+ parts.append(turn) # the open turn: no terminator, no cue
+ return "".join(parts)
+ parts.append(turn + "<|im_end|>")
+ continue
+ parts.append(f"<|im_start|>{role}\n{body}<|im_end|>")
+ if add_generation_prompt:
+ parts.append("<|im_start|>assistant\n")
+ parts.append("<think>" if enable_thinking else "<think></think>")
+ return "".join(parts)
+
+
+ MIMO_CALL_RE = re.compile(
+ r"<tool_call>\s*<function=([^>\n]+)>(.*?)</function>\s*</tool_call>", re.S)
+ MIMO_PARAM_RE = re.compile(r"<parameter=([^>\n]+)>(.*?)</parameter>", re.S)
+
+
+ def parse_mimo_tool_calls(reply, tools=None):
+ """MiMo's calls back into OpenAI `tool_calls`.
+
+ Parameters are inline (`<parameter=K>V</parameter>`); a value that arrives on its own
+ lines, Qwen-style, loses exactly one newline on each side. Types come from the declared
+ schema as for Qwen: a string parameter stays text, anything else is read as JSON. A
+ body with no parameter tags but a JSON object in it is the template's other spelling
+ (`arguments` as a string) and is taken as the arguments."""
+ schema = {}
+ for tool in (tools or []):
+ fn = tool.get("function", tool) if isinstance(tool, dict) else {}
+ name = fn.get("name")
+ params = (fn.get("parameters") or {}).get("properties") or {}
+ if isinstance(name, str) and name and isinstance(params, dict):
+ schema[name] = params
+
+ def make_call(name, body):
+ args = {}
+ found = MIMO_PARAM_RE.findall(body)
+ for key, raw in found:
+ key = key.strip()
+ if raw.startswith("\n"):
+ raw = raw[1:]
+ if raw.endswith("\n"):
+ raw = raw[:-1]
+ declared = (schema.get(name) or {}).get(key) or {}
+ kind = declared.get("type") if isinstance(declared, dict) else None
+ args[key] = _coerce_arg(raw, kind if isinstance(kind, str) else None) \
+ if kind not in (None, "string") else raw
+ if not found and body.strip():
+ try:
+ parsed = json.loads(body)
+ if isinstance(parsed, dict):
+ args = parsed
+ except (TypeError, ValueError):
+ pass
+ return {"id": f"call_{uuid.uuid4().hex[:24]}", "type": "function",
+ "function": {"name": name, "arguments": json.dumps(args, ensure_ascii=False)}}
+
+ text = reply or ""
+ calls = [make_call(m.group(1).strip(), m.group(2)) for m in MIMO_CALL_RE.finditer(text)]
+ if not calls and tools and ("<tool_call>" in text or "<function=" in text):
+ sys.stderr.write("[api] mimo tool markers present but no call parsed -- "
+ "possibly truncated or mangled output\n")
+ sys.stderr.flush()
+ return MIMO_CALL_RE.sub("", text).strip(), calls
+
+
def render_chat_qwen38(messages, enable_thinking=True, reasoning_effort=None, tools=None,
tool_choice=None, add_generation_prompt=True):
"""Text-only Qwen3.8 chat-template subset with native reasoning hints.
diff --cc c/tests/test_openai_server.py
index 56f17fe4,589e7289..97532a1a
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@@ -29,8 -29,8 +29,9 @@@ from openai_server import (APIError, AP
parse_qwen_tool_calls,
read_engine_turn, render_chat, render_chat_qwen, render_chat_for_arch,
render_chat_glm53, render_chat_inkling, render_chat_kimi,
-- render_chat_olmoe, render_chat_qwen,
-- render_chat_qwen38, render_chat_v4, render_chat_dsv41,
++ render_chat_olmoe, render_chat_qwen,
++ render_chat_qwen38, render_chat_v4, render_chat_dsv41,
++ render_chat_mimo,
_dsv4_tool_calls, serve,
resolve_generation_prompt, split_thinking_reply,
detect_chat_flavor, qwen36_has_vision,
@@@ -38,8 -38,8 +39,9 @@@
parse_tool_calls_spans, parse_arch_tool_calls_spans,
THINK_OPEN, THINK_CLOSE,
_compose_span_maps, _cut_span_map, _project_span,
- stop_policy, tune_child_env,
- TOOL_CHOICE_REQUIRED_INSTRUCTION)
- _keepalive_choice,
- stop_policy, tune_child_env)
++ _keepalive_choice,
++ stop_policy, tune_child_env,
++ TOOL_CHOICE_REQUIRED_INSTRUCTION)
def echo_record(pos, data, lp, topk):
@@@ -3661,86 -3789,6 +3791,97 @@@ class ToolChoiceTest(unittest.TestCase)
with self.assertRaises(APIError):
generation_options({"messages": [], "tool_choice": "required"}, 128)
+ # #1698 Tier 2: `required` was prompt-level on four renderers and dropped by
+ # the four others that accept it, so a client got the tools and no instruction
+ # to use them. These pin the one-line difference per renderer, because the
+ # failure is silent: the request succeeds, the model just answers in prose.
+
+ # The renderers that render a tool block and therefore have somewhere to put
+ # the instruction. Kimi K3 is absent on purpose: its wire format carries a
+ # dedicated tool-choice message rather than prose (tested below).
+ BLOCK_RENDERERS = (render_chat, render_chat_v4, render_chat_dsv41, render_chat_qwen,
- render_chat_qwen38, render_chat_glm53)
++ render_chat_qwen38, render_chat_glm53, render_chat_mimo)
+
+ def _auto_and_required(self, render, **kwargs):
+ messages = [{"role": "user", "content": "Where is order 7?"}]
+ return (render(messages, tools=ORDER_TOOL, tool_choice="auto", **kwargs),
+ render(messages, tools=ORDER_TOOL, tool_choice="required", **kwargs))
+
+ def test_required_is_auto_plus_exactly_the_instruction(self):
+ for render in self.BLOCK_RENDERERS:
+ with self.subTest(renderer=render.__name__):
+ auto, required = self._auto_and_required(render)
+ self.assertEqual(required.count(TOOL_CHOICE_REQUIRED_INSTRUCTION), 1)
+ # Nothing else moves: the instruction is the whole difference
+ # between `auto` and `required` on every one of these families.
+ self.assertEqual(
+ required.replace(TOOL_CHOICE_REQUIRED_INSTRUCTION, "", 1), auto)
+
+ def test_required_instruction_comes_after_the_declarations(self):
+ # "the functions above" has to be true, so the instruction goes after the
+ # last declared function and not inside or before the tool block.
+ for render in self.BLOCK_RENDERERS:
+ with self.subTest(renderer=render.__name__):
+ _, required = self._auto_and_required(render)
+ self.assertLess(required.rindex("lookup_order"),
+ required.index(TOOL_CHOICE_REQUIRED_INSTRUCTION))
+
+ def test_required_sits_between_the_tool_block_and_the_system_text(self):
+ # Qwen3.6/3.8 build one system turn as [tool block][client system text];
+ # the instruction belongs to the tool half, so it goes between the two
+ # and not after the client's own text.
+ system = [{"role": "system", "content": "Be terse."},
+ {"role": "user", "content": "Where is order 7?"}]
+ for render in (render_chat_qwen, render_chat_qwen38):
+ with self.subTest(renderer=render.__name__):
+ required = render(system, tools=ORDER_TOOL, tool_choice="required")
+ self.assertLess(required.index("# Tools"),
+ required.index(TOOL_CHOICE_REQUIRED_INSTRUCTION))
+ self.assertLess(required.index(TOOL_CHOICE_REQUIRED_INSTRUCTION),
+ required.index("Be terse."))
+
++ def test_mimo_required_stays_inside_the_tool_system_turn(self):
++ # MiMo's declaration block is a system turn of its own, so there is
++ # nowhere outside it to put the line: it has to land after </tools> and
++ # before that turn's <|im_end|>, or the frame closes before the model
++ # has read it.
++ _, required = self._auto_and_required(render_chat_mimo)
++ close = required.index("</tools>")
++ self.assertLess(close, required.index(TOOL_CHOICE_REQUIRED_INSTRUCTION))
++ self.assertLess(required.index(TOOL_CHOICE_REQUIRED_INSTRUCTION),
++ required.index("<|im_end|>"))
++
+ def test_olmoe_required_under_the_tool_fallback(self):
+ with patch("openai_server._TOOL_FALLBACK", True):
+ auto, required = self._auto_and_required(render_chat_olmoe)
+ self.assertEqual(required.count(TOOL_CHOICE_REQUIRED_INSTRUCTION), 1)
+ self.assertEqual(required.replace(TOOL_CHOICE_REQUIRED_INSTRUCTION, "", 1), auto)
+ self.assertLess(required.rindex("lookup_order"),
+ required.index(TOOL_CHOICE_REQUIRED_INSTRUCTION))
+
+ def test_olmoe_without_the_fallback_refuses_required(self):
+ # No tool block, so no instruction to give: the honest outcome is the
+ # 400, not a `required` that is accepted and then ignored.
+ with patch("openai_server._TOOL_FALLBACK", False):
+ with self.assertRaisesRegex(APIError, "Tool use"):
+ render_chat_olmoe([{"role": "user", "content": "Hi"}], tools=ORDER_TOOL,
+ tool_choice="required")
+
+ def test_inkling_refuses_required(self):
+ with self.assertRaisesRegex(APIError, "not wired up"):
+ render_chat_inkling([{"role": "user", "content": "Hi"}], tools=ORDER_TOOL,
+ tool_choice="required")
+
+ def test_kimi_keeps_its_own_required_wording(self):
+ # Same policy, different wire format: a dedicated tool-choice message
+ # instead of prose appended to the tool block.
+ required = render_chat_kimi([{"role": "user", "content": "Hi"}],
+ tools=ORDER_TOOL, tool_choice="required")
+ self.assertIn("tool-choiceThe system is invoked with `tool_choice=required`.",
+ required)
+ self.assertIn("You MUST call tools in the next message.", required)
+ self.assertNotIn(TOOL_CHOICE_REQUIRED_INSTRUCTION, required)
+
class TrailingAssistantTurnTest(unittest.TestCase):
"""A trailing `assistant` message is a turn to CONTINUE, not one already finished.
diff --cc docs/api.md
index 2b046992,e8cd2327..d065fae2
--- a/docs/api.md
+++ b/docs/api.md
@@@ -248,22 -247,15 +248,31 @@@ Protocol support does not guarantee tha
tool syntax; `COLI_TOOL_SALVAGE=1` is an opt-in recovery path for malformed GLM
int4 tool calls. DeepSeek V4 uses its strict native DSML parser instead.
+`tool_choice: "required"` is a prompt-level instruction, not a sampling
+constraint. Every renderer that offers a tool block appends the same one line to
+it, and no renderer filters tokens or forces the sampler, so a model that
+answers in prose anyway has done nothing the API said was impossible.
+Grammar forcing is not a remedy: that path feeds a draft the engine then
+verifies, so a schema the engine cannot compile costs the speedup and nothing
+else. An engine with no tool block to attach the instruction to (Inkling, OLMoE
+without `COLI_TOOL_FALLBACK=1`) answers HTTP 400 rather than accept the choice
+and ignore it.
+
+A forced choice, `tool_choice: {"type": "function", "function": {"name": …}}`,
+is applied per engine and the engines do not agree on how: some narrow the
+offered tools to the named one, some keep the full list and name the tool in
+prose instead, and some (Qwen3.6, Qwen3.8) do neither. Read the rendered prompt
+rather than assuming the request was honoured.
+
+ For GLM calls that will execute tools, an OpenAI chat request may set
-`"strict_tool_calls": true`. This opt-in accepts only complete `<tool_call>`
++`"strict_tool_calls": true`. This opt-in accepts only complete `<​tool_call>`
+ blocks whose function and arguments match the declared tool schema. It returns
+ HTTP 502 with code `invalid_model_tool_call` for incomplete, duplicate, unknown,
+ or invalid arguments and for a tool call cut off by the generation limit. It
+ never recovers or salvages a malformed call. The default recovery behavior is
+ unchanged. Strict mode currently requires `stream: false` and does not support
+ `logprobs`; these combinations receive HTTP 400 before generation.
+
When a reverse proxy or MagicDNS hostname preserves a public `Host` header,
trust that exact hostname with repeatable `--allowed-host` options. The
comma-separated `COLI_ALLOWED_HOSTS` environment variable is equivalent: