Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
Merge branch 'dev' into fix/tool-choice-required-renderers
Yi-111-a4cab875b4e75
diff --cc c/openai_server.pyindex 0410b4a8,3d6847b4..4261818b--- a/c/openai_server.py+++ b/c/openai_server.py@@@ -1962,6 -1999,172 +2023,178 @@@ def parse_qwen_tool_calls(reply, tools=parse_qwen38_tool_calls = parse_qwen_tool_calls+ # ---- MiMo-V2.6 (Xiaomi) -------------------------------------------------------------------+ # ChatML without a newline after <|im_end|>, every assistant turn carrying its <think> block+ # (empty or not), tools declared in their own system turn, calls written INLINE+ # (<tool_call><function=N><parameter=K>V</parameter></function></tool_call>, no newlines) and+ # tool results as a plain `tool` turn. All of it from the release's chat_template.jinja+ # (XiaomiMiMo/MiMo-V2.6-Flash-MOPD); tests/test_mimo_chat_template.py renders that template+ # with jinja2 and compares byte for byte.++ MIMO_TOOLS_HEAD = "You are provided with the following tools:\n\n<tools>"+++ def _mimo_tools(tools):+ """render_tools(): each tool object as JSON, key order kept (transformers' tojson)."""+ return (MIMO_TOOLS_HEAD+ + "".join("\n" + json.dumps(tool, ensure_ascii=False) for tool in tools)+ + "\n</tools>")+++ def _mimo_value(value):+ """render_value(): a string as-is, anything else as JSON."""+ return value if isinstance(value, str) else json.dumps(value, ensure_ascii=False)+++ def _mimo_tool_calls(calls, index):+ out = []+ for position, call in enumerate(calls):+ where = f"messages.{index}.tool_calls.{position}"+ if not isinstance(call, dict):+ raise APIError(400, "Each tool call must be an object.", where)+ fn = call.get("function", call.get("custom", call))+ if not isinstance(fn, dict) or not isinstance(fn.get("name"), str) or not fn["name"]:+ raise APIError(400, "A tool call needs a `function.name`.", f"{where}.function")+ out.append(f"<tool_call><function={fn['name']}>")+ if isinstance(fn.get("input"), str):+ out.append(fn["input"])+ else:+ args = fn.get("arguments")+ # An OpenAI client sends the arguments back as a JSON string. The template+ # would print that string raw, a shape the model never writes; read as the+ # object it is, the call comes back in the <parameter=...> form the model+ # produced, which is also what keeps the resent history on the KV prefix.+ if isinstance(args, str) and args.strip():+ try:+ args = json.loads(args)+ except (TypeError, ValueError):+ raise APIError(400, "`function.arguments` must be a JSON object.",+ f"{where}.function.arguments")+ if isinstance(args, dict):+ for key, value in args.items():+ out.append(f"<parameter={key}>{_mimo_value(value)}</parameter>")+ elif args not in (None, "", {}):+ raise APIError(400, "`function.arguments` must be a JSON object.",+ f"{where}.function.arguments")+ out.append("</function></tool_call>")+ return "".join(out)+++ def render_chat_mimo(messages, enable_thinking=True, reasoning_effort=None, tools=None,+ tool_choice=None, add_generation_prompt=True):+ """MiMo-V2.6's chat template.++ One deliberate addition: with thinking on, the generation cue ends in `<think>`. The+ template leaves that token to the model, and the model writes it first on every turn+ it was trained on (each assistant turn of the template opens with it); putting it in+ the prompt is what lets the reasoning split start inside the reasoning, where the+ model is. With thinking off the template's own `<think></think>` closes it.++ add_generation_prompt=False continues a trailing assistant turn: its past-turn render+ minus the closing <|im_end|>, and no cue."""+ if not isinstance(messages, list) or not messages:+ raise APIError(400, "`messages` must be a non-empty array.", "messages")+ if tool_choice in ("none",):+ tools = None+ if tools is not None and not isinstance(tools, list):+ raise APIError(400, "`tools` must be an array.", "tools")+ parts = []+ if tools:- parts.append(f"<|im_start|>system\n{_mimo_tools(tools)}<|im_end|>")++ tools_text = _mimo_tools(tools)++ # In the tool turn, after </tools> and before its <|im_end|>: "the functions++ # above" has to be true, and MiMo's declaration block is a whole system turn++ # of its own, so there is nothing to put the line outside of but the frame.++ if tool_choice == "required":++ tools_text += TOOL_CHOICE_REQUIRED_INSTRUCTION++ parts.append(f"<|im_start|>system\n{tools_text}<|im_end|>")+ last = len(messages) - 1+ for index, message in enumerate(messages):+ if not isinstance(message, dict):+ raise APIError(400, "Each message must be an object.", f"messages.{index}")+ role = message.get("role")+ if role == "developer":+ role = "system"+ if role not in ("system", "user", "assistant", "tool"):+ raise APIError(400, f"Unsupported role {role!r}.", f"messages.{index}.role")+ raw = message.get("content")+ body = content_text(raw, f"messages.{index}.content") if raw is not None else ""+ if role == "assistant":+ reasoning = message.get("reasoning_content")+ if reasoning is not None and not isinstance(reasoning, str):+ raise APIError(400, "`reasoning_content` must be a string.",+ f"messages.{index}.reasoning_content")+ turn = f"<|im_start|>assistant\n<think>{reasoning or ''}</think>{body}"+ calls = message.get("tool_calls")+ if calls:+ if not isinstance(calls, list):+ raise APIError(400, "`tool_calls` must be an array.",+ f"messages.{index}.tool_calls")+ turn += _mimo_tool_calls(calls, index)+ if not add_generation_prompt and index == last:+ parts.append(turn) # the open turn: no terminator, no cue+ return "".join(parts)+ parts.append(turn + "<|im_end|>")+ continue+ parts.append(f"<|im_start|>{role}\n{body}<|im_end|>")+ if add_generation_prompt:+ parts.append("<|im_start|>assistant\n")+ parts.append("<think>" if enable_thinking else "<think></think>")+ return "".join(parts)+++ MIMO_CALL_RE = re.compile(+ r"<tool_call>\s*<function=([^>\n]+)>(.*?)</function>\s*</tool_call>", re.S)+ MIMO_PARAM_RE = re.compile(r"<parameter=([^>\n]+)>(.*?)</parameter>", re.S)+++ def parse_mimo_tool_calls(reply, tools=None):+ """MiMo's calls back into OpenAI `tool_calls`.++ Parameters are inline (`<parameter=K>V</parameter>`); a value that arrives on its own+ lines, Qwen-style, loses exactly one newline on each side. Types come from the declared+ schema as for Qwen: a string parameter stays text, anything else is read as JSON. A+ body with no parameter tags but a JSON object in it is the template's other spelling+ (`arguments` as a string) and is taken as the arguments."""+ schema = {}+ for tool in (tools or []):+ fn = tool.get("function", tool) if isinstance(tool, dict) else {}+ name = fn.get("name")+ params = (fn.get("parameters") or {}).get("properties") or {}+ if isinstance(name, str) and name and isinstance(params, dict):+ schema[name] = params++ def make_call(name, body):+ args = {}+ found = MIMO_PARAM_RE.findall(body)+ for key, raw in found:+ key = key.strip()+ if raw.startswith("\n"):+ raw = raw[1:]+ if raw.endswith("\n"):+ raw = raw[:-1]+ declared = (schema.get(name) or {}).get(key) or {}+ kind = declared.get("type") if isinstance(declared, dict) else None+ args[key] = _coerce_arg(raw, kind if isinstance(kind, str) else None) \+ if kind not in (None, "string") else raw+ if not found and body.strip():+ try:+ parsed = json.loads(body)+ if isinstance(parsed, dict):+ args = parsed+ except (TypeError, ValueError):+ pass+ return {"id": f"call_{uuid.uuid4().hex[:24]}", "type": "function",+ "function": {"name": name, "arguments": json.dumps(args, ensure_ascii=False)}}++ text = reply or ""+ calls = [make_call(m.group(1).strip(), m.group(2)) for m in MIMO_CALL_RE.finditer(text)]+ if not calls and tools and ("<tool_call>" in text or "<function=" in text):+ sys.stderr.write("[api] mimo tool markers present but no call parsed -- "+ "possibly truncated or mangled output\n")+ sys.stderr.flush()+ return MIMO_CALL_RE.sub("", text).strip(), calls++def render_chat_qwen38(messages, enable_thinking=True, reasoning_effort=None, tools=None,tool_choice=None, add_generation_prompt=True):"""Text-only Qwen3.8 chat-template subset with native reasoning hints.diff --cc c/tests/test_openai_server.pyindex 56f17fe4,589e7289..97532a1a--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@@ -29,8 -29,8 +29,9 @@@ from openai_server import (APIError, APparse_qwen_tool_calls,read_engine_turn, render_chat, render_chat_qwen, render_chat_for_arch,render_chat_glm53, render_chat_inkling, render_chat_kimi,-- render_chat_olmoe, render_chat_qwen,-- render_chat_qwen38, render_chat_v4, render_chat_dsv41,++ render_chat_olmoe, render_chat_qwen,++ render_chat_qwen38, render_chat_v4, render_chat_dsv41,++ render_chat_mimo,_dsv4_tool_calls, serve,resolve_generation_prompt, split_thinking_reply,detect_chat_flavor, qwen36_has_vision,@@@ -38,8 -38,8 +39,9 @@@parse_tool_calls_spans, parse_arch_tool_calls_spans,THINK_OPEN, THINK_CLOSE,_compose_span_maps, _cut_span_map, _project_span,- stop_policy, tune_child_env,- TOOL_CHOICE_REQUIRED_INSTRUCTION)- _keepalive_choice,- stop_policy, tune_child_env)++ _keepalive_choice,++ stop_policy, tune_child_env,++ TOOL_CHOICE_REQUIRED_INSTRUCTION)def echo_record(pos, data, lp, topk):@@@ -3661,86 -3789,6 +3791,97 @@@ class ToolChoiceTest(unittest.TestCase)with self.assertRaises(APIError):generation_options({"messages": [], "tool_choice": "required"}, 128)+ # #1698 Tier 2: `required` was prompt-level on four renderers and dropped by+ # the four others that accept it, so a client got the tools and no instruction+ # to use them. These pin the one-line difference per renderer, because the+ # failure is silent: the request succeeds, the model just answers in prose.++ # The renderers that render a tool block and therefore have somewhere to put+ # the instruction. Kimi K3 is absent on purpose: its wire format carries a+ # dedicated tool-choice message rather than prose (tested below).+ BLOCK_RENDERERS = (render_chat, render_chat_v4, render_chat_dsv41, render_chat_qwen,- render_chat_qwen38, render_chat_glm53)++ render_chat_qwen38, render_chat_glm53, render_chat_mimo)++ def _auto_and_required(self, render, **kwargs):+ messages = [{"role": "user", "content": "Where is order 7?"}]+ return (render(messages, tools=ORDER_TOOL, tool_choice="auto", **kwargs),+ render(messages, tools=ORDER_TOOL, tool_choice="required", **kwargs))++ def test_required_is_auto_plus_exactly_the_instruction(self):+ for render in self.BLOCK_RENDERERS:+ with self.subTest(renderer=render.__name__):+ auto, required = self._auto_and_required(render)+ self.assertEqual(required.count(TOOL_CHOICE_REQUIRED_INSTRUCTION), 1)+ # Nothing else moves: the instruction is the whole difference+ # between `auto` and `required` on every one of these families.+ self.assertEqual(+ required.replace(TOOL_CHOICE_REQUIRED_INSTRUCTION, "", 1), auto)++ def test_required_instruction_comes_after_the_declarations(self):+ # "the functions above" has to be true, so the instruction goes after the+ # last declared function and not inside or before the tool block.+ for render in self.BLOCK_RENDERERS:+ with self.subTest(renderer=render.__name__):+ _, required = self._auto_and_required(render)+ self.assertLess(required.rindex("lookup_order"),+ required.index(TOOL_CHOICE_REQUIRED_INSTRUCTION))++ def test_required_sits_between_the_tool_block_and_the_system_text(self):+ # Qwen3.6/3.8 build one system turn as [tool block][client system text];+ # the instruction belongs to the tool half, so it goes between the two+ # and not after the client's own text.+ system = [{"role": "system", "content": "Be terse."},+ {"role": "user", "content": "Where is order 7?"}]+ for render in (render_chat_qwen, render_chat_qwen38):+ with self.subTest(renderer=render.__name__):+ required = render(system, tools=ORDER_TOOL, tool_choice="required")+ self.assertLess(required.index("# Tools"),+ required.index(TOOL_CHOICE_REQUIRED_INSTRUCTION))+ self.assertLess(required.index(TOOL_CHOICE_REQUIRED_INSTRUCTION),+ required.index("Be terse."))+++ def test_mimo_required_stays_inside_the_tool_system_turn(self):++ # MiMo's declaration block is a system turn of its own, so there is++ # nowhere outside it to put the line: it has to land after </tools> and++ # before that turn's <|im_end|>, or the frame closes before the model++ # has read it.++ _, required = self._auto_and_required(render_chat_mimo)++ close = required.index("</tools>")++ self.assertLess(close, required.index(TOOL_CHOICE_REQUIRED_INSTRUCTION))++ self.assertLess(required.index(TOOL_CHOICE_REQUIRED_INSTRUCTION),++ required.index("<|im_end|>"))+++ def test_olmoe_required_under_the_tool_fallback(self):+ with patch("openai_server._TOOL_FALLBACK", True):+ auto, required = self._auto_and_required(render_chat_olmoe)+ self.assertEqual(required.count(TOOL_CHOICE_REQUIRED_INSTRUCTION), 1)+ self.assertEqual(required.replace(TOOL_CHOICE_REQUIRED_INSTRUCTION, "", 1), auto)+ self.assertLess(required.rindex("lookup_order"),+ required.index(TOOL_CHOICE_REQUIRED_INSTRUCTION))++ def test_olmoe_without_the_fallback_refuses_required(self):+ # No tool block, so no instruction to give: the honest outcome is the+ # 400, not a `required` that is accepted and then ignored.+ with patch("openai_server._TOOL_FALLBACK", False):+ with self.assertRaisesRegex(APIError, "Tool use"):+ render_chat_olmoe([{"role": "user", "content": "Hi"}], tools=ORDER_TOOL,+ tool_choice="required")++ def test_inkling_refuses_required(self):+ with self.assertRaisesRegex(APIError, "not wired up"):+ render_chat_inkling([{"role": "user", "content": "Hi"}], tools=ORDER_TOOL,+ tool_choice="required")++ def test_kimi_keeps_its_own_required_wording(self):+ # Same policy, different wire format: a dedicated tool-choice message+ # instead of prose appended to the tool block.+ required = render_chat_kimi([{"role": "user", "content": "Hi"}],+ tools=ORDER_TOOL, tool_choice="required")+ self.assertIn("tool-choiceThe system is invoked with `tool_choice=required`.",+ required)+ self.assertIn("You MUST call tools in the next message.", required)+ self.assertNotIn(TOOL_CHOICE_REQUIRED_INSTRUCTION, required)+class TrailingAssistantTurnTest(unittest.TestCase):"""A trailing `assistant` message is a turn to CONTINUE, not one already finished.diff --cc docs/api.mdindex 2b046992,e8cd2327..d065fae2--- a/docs/api.md+++ b/docs/api.md@@@ -248,22 -247,15 +248,31 @@@ Protocol support does not guarantee thatool syntax; `COLI_TOOL_SALVAGE=1` is an opt-in recovery path for malformed GLMint4 tool calls. DeepSeek V4 uses its strict native DSML parser instead.+`tool_choice: "required"` is a prompt-level instruction, not a sampling+constraint. Every renderer that offers a tool block appends the same one line to+it, and no renderer filters tokens or forces the sampler, so a model that+answers in prose anyway has done nothing the API said was impossible.+Grammar forcing is not a remedy: that path feeds a draft the engine then+verifies, so a schema the engine cannot compile costs the speedup and nothing+else. An engine with no tool block to attach the instruction to (Inkling, OLMoE+without `COLI_TOOL_FALLBACK=1`) answers HTTP 400 rather than accept the choice+and ignore it.++A forced choice, `tool_choice: {"type": "function", "function": {"name": …}}`,+is applied per engine and the engines do not agree on how: some narrow the+offered tools to the named one, some keep the full list and name the tool in+prose instead, and some (Qwen3.6, Qwen3.8) do neither. Read the rendered prompt+rather than assuming the request was honoured.++ For GLM calls that will execute tools, an OpenAI chat request may set-`"strict_tool_calls": true`. This opt-in accepts only complete `<tool_call>`++`"strict_tool_calls": true`. This opt-in accepts only complete `<tool_call>`+ blocks whose function and arguments match the declared tool schema. It returns+ HTTP 502 with code `invalid_model_tool_call` for incomplete, duplicate, unknown,+ or invalid arguments and for a tool call cut off by the generation limit. It+ never recovers or salvages a malformed call. The default recovery behavior is+ unchanged. Strict mode currently requires `stream: false` and does not support+ `logprobs`; these combinations receive HTTP 400 before generation.+When a reverse proxy or MagicDNS hostname preserves a public `Host` header,trust that exact hostname with repeatable `--allowed-host` options. Thecomma-separated `COLI_ALLOWED_HOSTS` environment variable is equivalent: