Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
feat(api): opt-in prompt-injected tool translation for OLMoE and Qwen3.6
basil-k-aji-dev073dc4ddbed4
diff --git a/c/openai_server.py b/c/openai_server.pyindex 7476fc08..b43eb7cd 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -251,6 +251,63 @@ _BOX_RE = re.compile(re.escape(BOX_START) + r"(.*?)" + re.escape(BOX_END), re.D_ARG_RE = re.compile(r"<arg_key>([^<]*)</arg_key><arg_value>(.*?)</arg_value>", re.DOTALL)_NAME_RE = re.compile(r"\s*([A-Za-z0-9_.\-]+)")_TAG_RE = re.compile(r"</?arg_key>|</?arg_value>")+++def _fallback_tool_preamble(tools):+ """Tool declaration for a family with no native tool tokens.++ Mirrors the GLM-5.2 block because ``parse_tool_calls`` -- the parser these+ families fall back to in ``parse_arch_tool_calls`` -- reads exactly that+ wire format. Asking for a format the parser does not accept would produce+ tool calls nobody can read back.+ """+ out = ["You have access to the following functions. Call one only when it "+ "is needed to answer the user.\n\n<tools>\n"]+ for tool in tools:+ fn = tool.get("function", tool) if isinstance(tool, dict) else {}+ out.append(json.dumps(fn, ensure_ascii=False) + "\n")+ out.append("</tools>\n\nTo call a function, reply with the call and nothing "+ "else, in this exact format:\n" + BOX_START + "{function-name}"+ "<arg_key>{arg-key}</arg_key><arg_value>{arg-value}</arg_value>"+ + BOX_END)+ return "".join(out)+++def _fallback_tool_calls(tool_calls, index):+ """Render assistant tool_calls in the format parse_tool_calls() reads."""+ out = []+ for position, call in enumerate(tool_calls or []):+ if not isinstance(call, dict):+ raise APIError(400, "Each tool call must be an object.",+ f"messages.{index}.tool_calls.{position}")+ fn = call.get("function", call)+ if not isinstance(fn, dict):+ raise APIError(400, "`function` must be an object.",+ f"messages.{index}.tool_calls.{position}.function")+ name = fn.get("name")+ if not isinstance(name, str) or not name:+ raise APIError(400, "`function.name` must be a non-empty string.",+ f"messages.{index}.tool_calls.{position}.function.name")+ args = fn.get("arguments", "{}")+ if isinstance(args, str):+ try:+ args = json.loads(args) if args else {}+ except (json.JSONDecodeError, TypeError, ValueError):+ raise APIError(400, "`function.arguments` must be a JSON object.",+ f"messages.{index}.tool_calls.{position}.function.arguments")+ out.append(BOX_START + name)+ for key, value in (args or {}).items():+ rendered = value if isinstance(value, str) else json.dumps(+ value, ensure_ascii=False)+ out.append(f"<arg_key>{key}</arg_key><arg_value>{rendered}</arg_value>")+ out.append(BOX_END)+ return "".join(out)+++def _fallback_tool_result(message, index):+ """Render a role:"tool" message as prose these templates can carry."""+ body = content_text(message.get("content"), f"messages.{index}.content")+ return TR_OPEN + body + TR_CLOSE# A closing tag the model started but never finished ("</tool_cal", "</tool"), at end of reply._PARTIAL_END_RE = re.compile(r"<(?:/(?:t(?:o(?:o(?:l(?:_(?:c(?:a(?:l)?)?)?)?)?)?)?)?)?\Z")@@ -258,6 +315,15 @@ _PARTIAL_END_RE = re.compile(r"<(?:/(?:t(?:o(?:o(?:l(?:_(?:c(?:a(?:l)?)?)?)?)?)?# <arg_key>K</arg_key><arg_value> structure. Default OFF (never rewrites well-formed output)._SALVAGE = os.environ.get("COLI_TOOL_SALVAGE", "0") == "1"+# Families whose chat template has no tool syntax at all (OLMoE, Qwen3.6) refuse+# tools[] and role:"tool" rather than invent a format. COLI_TOOL_FALLBACK=1 opts+# into a prompt-injected translation for them: the declaration block, the prior+# assistant calls and the tool results are written as ordinary turns, in the+# same wire format parse_tool_calls() already reads back (#1378). Default OFF --+# these models were never trained on tool syntax, so this trades a clean 400 for+# output the parser may or may not recognise.+_TOOL_FALLBACK = os.environ.get("COLI_TOOL_FALLBACK", "0") == "1"+def _tool_param_order(tools):"""name -> ordered param names (required first) from the request schema, for de-mangling."""@@ -1187,17 +1253,25 @@ def render_chat_olmoe(messages, enable_thinking=False, reasoning_effort=None, to"""if not isinstance(messages, list) or not messages:raise APIError(400, "`messages` must be a non-empty array.", "messages")- if tools or tool_choice not in (None, "none"):- raise APIError(400, "Tool use is not wired up for the OLMoE engine yet.",- "tools", "unsupported_parameter")+ if tool_choice == "none":+ tools = None+ if (tools or tool_choice not in (None, "none")) and not _TOOL_FALLBACK:+ raise APIError(400, "Tool use is not wired up for the OLMoE engine yet. "+ "Set COLI_TOOL_FALLBACK=1 to opt into prompt-injected "+ "tool translation.", "tools", "unsupported_parameter")boundary = "|||IP_ADDRESS|||" # bos_token == eos_token in this tokenizerparts = [boundary]+ if tools and _TOOL_FALLBACK:+ parts.append(f"<|system|>\n{_fallback_tool_preamble(tools)}\n")last = len(messages) - 1for index, message in enumerate(messages):if not isinstance(message, dict):raise APIError(400, "Each message must be an object.", f"messages.{index}")role = message.get("role")- if role not in ("system", "developer", "user", "assistant"):+ allowed = ("system", "developer", "user", "assistant")+ if _TOOL_FALLBACK:+ allowed += ("tool",)+ if role not in allowed:raise APIError(400, f"Unsupported role {role!r}.", f"messages.{index}.role")raw = message.get("content")text = content_text(raw, f"messages.{index}.content") if raw is not None else ""@@ -1205,8 +1279,13 @@ def render_chat_olmoe(messages, enable_thinking=False, reasoning_effort=None, toparts.append(f"<|system|>\n{text}\n")elif role == "user":parts.append(f"<|user|>\n{text}\n")+ elif role == "tool":+ # No tool role in this template: the result rides in as a user turn.+ parts.append(f"<|user|>\n{_fallback_tool_result(message, index)}\n")else:- parts.append(f"<|assistant|>\n{text}{boundary}")+ calls = (_fallback_tool_calls(message.get("tool_calls"), index)+ if _TOOL_FALLBACK else "")+ parts.append(f"<|assistant|>\n{text}{calls}{boundary}")if index != last:parts.append("\n")parts.append("<|assistant|>\n")@@ -1224,20 +1303,36 @@ def render_chat_qwen(messages, enable_thinking=False, reasoning_effort=None, toomirrored here byte for byte."""if not isinstance(messages, list) or not messages:raise APIError(400, "`messages` must be a non-empty array.", "messages")- if tools or tool_choice not in (None, "none"):- raise APIError(400, "Tool use is not wired up for the qwen36 engine yet.",- "tools", "unsupported_parameter")+ if tool_choice == "none":+ tools = None+ if (tools or tool_choice not in (None, "none")) and not _TOOL_FALLBACK:+ raise APIError(400, "Tool use is not wired up for the qwen36 engine yet. "+ "Set COLI_TOOL_FALLBACK=1 to opt into prompt-injected "+ "tool translation.", "tools", "unsupported_parameter")parts = []+ if tools and _TOOL_FALLBACK:+ parts.append("<|im_start|>system\n"+ + _fallback_tool_preamble(tools) + "<|im_end|>\n")for index, message in enumerate(messages):if not isinstance(message, dict):raise APIError(400, "Each message must be an object.", f"messages.{index}")role = message.get("role")if role == "developer":role = "system"- if role not in ("system", "user", "assistant"):+ allowed = ("system", "user", "assistant")+ if _TOOL_FALLBACK:+ allowed += ("tool",)+ if role not in allowed:raise APIError(400, f"Unsupported role {role!r}.", f"messages.{index}.role")raw = message.get("content")text = content_text(raw, f"messages.{index}.content") if raw is not None else ""+ if role == "tool":+ # No tool role in this template: the result rides in as a user turn.+ parts.append("<|im_start|>user\n"+ + _fallback_tool_result(message, index) + "<|im_end|>\n")+ continue+ if role == "assistant" and _TOOL_FALLBACK:+ text += _fallback_tool_calls(message.get("tool_calls"), index)parts.append(f"<|im_start|>{role}\n{text}<|im_end|>\n")parts.append("<|im_start|>assistant\n")parts.append("<think>\n" if enable_thinking else "<think>\n\n</think>\n\n")diff --git a/c/tests/test_openai_tools_fallback_e2e.py b/c/tests/test_openai_tools_fallback_e2e.pynew file mode 100644index 00000000..894d59c3--- /dev/null+++ b/c/tests/test_openai_tools_fallback_e2e.py@@ -0,0 +1,222 @@+"""Prompt-injected tool translation for families with no native tool syntax.++OLMoE and Qwen3.6 have no tool tokens in their chat templates, so the gateway+answers 400 for `tools[]` and for a `role: "tool"` turn rather than invent a+format. `COLI_TOOL_FALLBACK=1` opts into a translation that writes the+declaration, the prior assistant calls and the tool results as ordinary turns,+in the GLM wire format `parse_tool_calls()` already reads back (#1378).++These tests pin both halves of that switch: the default still refuses, and with+the flag set a full two-turn agent loop completes through+`/v1/chat/completions`. Runs against a mock engine speaking the SERVE wire+protocol, so no checkpoint is needed.+"""+import json+import os+import socket+import subprocess+import sys+import tempfile+import unittest+import urllib.error+import urllib.request+from pathlib import Path++HERE = Path(__file__).resolve().parent+SERVER = HERE.parent / "openai_server.py"+sys.path.insert(0, str(HERE.parent))++# The call the injected preamble asks for, in the format parse_tool_calls reads.+CALL = ("<tool_call>get_weather"+ "<arg_key>location</arg_key><arg_value>Rome</arg_value>"+ "<arg_key>unit</arg_key><arg_value>celsius</arg_value>"+ "</tool_call>")++MOCK_ENGINE = r'''#!/usr/bin/env python3+import sys, os+out, inp = sys.stdout.buffer, sys.stdin.buffer+out.write(b"\x01\x01READY\x01\x01\n" + b"STAT 0 0 0 0 0\n"); out.flush()++CALL = ("<tool_call>get_weather"+ "<arg_key>location</arg_key><arg_value>Rome</arg_value>"+ "<arg_key>unit</arg_key><arg_value>celsius</arg_value>"+ "</tool_call>")++def reply(rid, text):+ data = text.encode("utf-8")+ out.write(("DATA %s %d\n" % (rid, len(data))).encode() + data + b"\n"); out.flush()+ out.write(("DONE %s STAT %d 1.0 50.0 10.0 42 0\n" % (rid, len(text.split()))).encode())+ out.flush()++while True:+ line = inp.readline()+ if not line: break+ f = line.decode().strip().split()+ if not f or f[0] != "SUBMIT": continue+ rid, plen = f[1], int(f[3])+ prompt = inp.read(plen).decode("utf-8", "replace"); inp.read(1)+ with open(os.environ["MOCK_LOG"], "a") as log:+ log.write(prompt + "\n\x00\n")+ if "<tool_response>" in prompt:+ reply(rid, "25 degrees and sunny in Rome.")+ elif "weather in Rome" in prompt:+ reply(rid, CALL)+ else:+ reply(rid, "Hello from the mock engine.")+'''++TOOLS = [{"type": "function", "function": {+ "name": "get_weather",+ "description": "Current weather for a city",+ "parameters": {"type": "object",+ "properties": {"location": {"type": "string"},+ "unit": {"type": "string"}},+ "required": ["location"]}}}]+++class _FallbackBase(unittest.TestCase):+ """Boots the gateway for one arch, with COLI_TOOL_FALLBACK under test."""++ arch = None+ # config.json model_type for the family (family_registry.model_types), which+ # is not always the --arch id: qwen36's checkpoints say "qwen3_5_moe".+ model_type = None+ fallback = "1"++ @classmethod+ def setUpClass(cls):+ if cls.arch is None:+ raise unittest.SkipTest("base class")+ cls.tmp = tempfile.TemporaryDirectory()+ (Path(cls.tmp.name) / "config.json").write_text(+ json.dumps({"model_type": cls.model_type or cls.arch}), encoding="utf-8")+ mock = Path(cls.tmp.name) / "mock_engine.py"+ mock.write_text(MOCK_ENGINE)+ mock.chmod(0o755)+ cls.mock_log = Path(cls.tmp.name) / "prompts.log"+ cls.mock_log.touch()+ with socket.socket() as probe:+ probe.bind(("127.0.0.1", 0))+ cls.port = probe.getsockname()[1]+ env = dict(os.environ, MOCK_LOG=str(cls.mock_log),+ COLI_TOOL_FALLBACK=cls.fallback)+ env.pop("COLI_API_KEY", None)+ cls.server = subprocess.Popen(+ [sys.executable, str(SERVER), "--model", cls.tmp.name,+ "--engine", str(mock), "--arch", cls.arch, "--port", str(cls.port)],+ env=env, stderr=subprocess.DEVNULL)+ cls.base = f"http://127.0.0.1:{cls.port}/v1"+ for _ in range(100):+ try:+ with urllib.request.urlopen(cls.base + "/models", timeout=2):+ pass+ return+ except OSError:+ if cls.server.poll() is not None:+ raise RuntimeError("gateway exited during startup")+ import time+ time.sleep(0.1)+ raise RuntimeError("gateway did not come up")++ @classmethod+ def tearDownClass(cls):+ if getattr(cls, "server", None) is None:+ return+ cls.server.terminate()+ cls.server.wait(timeout=5)+ cls.tmp.cleanup()++ def post(self, body):+ req = urllib.request.Request(+ self.base + "/chat/completions", json.dumps(body).encode(),+ {"Content-Type": "application/json"})+ with urllib.request.urlopen(req, timeout=30) as resp:+ return json.loads(resp.read())++ def model_id(self):+ with urllib.request.urlopen(self.base + "/models", timeout=5) as resp:+ return json.loads(resp.read())["data"][0]["id"]+++class _TwoTurnLoop(_FallbackBase):+ """The two-turn loop the issue asks for, with the flag on."""++ def test_tool_call_parsed_back(self):+ out = self.post({"model": self.model_id(),+ "messages": [{"role": "user", "content": "weather in Rome?"}],+ "tools": TOOLS, "temperature": 0, "max_tokens": 128})+ choice = out["choices"][0]+ self.assertEqual(choice["finish_reason"], "tool_calls")+ calls = choice["message"].get("tool_calls") or []+ self.assertEqual(len(calls), 1)+ self.assertEqual(calls[0]["function"]["name"], "get_weather")+ self.assertEqual(json.loads(calls[0]["function"]["arguments"]),+ {"location": "Rome", "unit": "celsius"})++ def test_second_turn_completes(self):+ mid = self.model_id()+ msgs = [{"role": "user", "content": "weather in Rome?"}]+ out = self.post({"model": mid, "messages": msgs, "tools": TOOLS,+ "temperature": 0, "max_tokens": 128})+ calls = out["choices"][0]["message"]["tool_calls"]+ msgs.append({"role": "assistant", "content": None, "tool_calls": calls})+ msgs.append({"role": "tool", "tool_call_id": calls[0]["id"],+ "content": json.dumps({"temp_c": 25})})+ out2 = self.post({"model": mid, "messages": msgs, "tools": TOOLS,+ "temperature": 0, "max_tokens": 128})+ self.assertIn("25", out2["choices"][0]["message"]["content"] or "")++ prompts = self.mock_log.read_text()+ self.assertIn("<tools>", prompts) # declaration was injected+ self.assertIn("<tool_call>get_weather", prompts) # prior call replayed+ self.assertIn("<tool_response>", prompts) # result reached the prompt+++class OlmoeToolFallbackE2E(_TwoTurnLoop):+ arch = "olmoe"+++class Qwen36ToolFallbackE2E(_TwoTurnLoop):+ arch = "qwen36"+ model_type = "qwen3_5_moe"+++class _RefusedByDefault(_FallbackBase):+ """With the flag unset the gateway must still refuse, not half-support."""++ fallback = "0"++ def _expect_400(self, body):+ with self.assertRaises(urllib.error.HTTPError) as caught:+ self.post(body)+ self.assertEqual(caught.exception.code, 400)+ return json.loads(caught.exception.read())++ def test_tools_refused(self):+ err = self._expect_400({"model": self.model_id(),+ "messages": [{"role": "user", "content": "hi"}],+ "tools": TOOLS, "max_tokens": 16})+ self.assertIn("COLI_TOOL_FALLBACK", json.dumps(err))++ def test_tool_role_refused(self):+ self._expect_400({"model": self.model_id(),+ "messages": [{"role": "user", "content": "hi"},+ {"role": "tool", "content": "18C"}],+ "max_tokens": 16})+++class OlmoeToolRefusedByDefault(_RefusedByDefault):+ arch = "olmoe"+++class Qwen36ToolRefusedByDefault(_RefusedByDefault):+ arch = "qwen36"+ model_type = "qwen3_5_moe"+++# The shared base classes must not run as tests themselves.+del _FallbackBase, _TwoTurnLoop, _RefusedByDefault+++if __name__ == "__main__":+ unittest.main()