Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

feat(api): opt-in prompt-injected tool translation for OLMoE and Qwen3.6

basil-k-aji-dev073dc4ddbed4
diff --git a/c/openai_server.py b/c/openai_server.py
index 7476fc08..b43eb7cd 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -251,6 +251,63 @@ _BOX_RE = re.compile(re.escape(BOX_START) + r"(.*?)" + re.escape(BOX_END), re.D
_ARG_RE = re.compile(r"<arg_key>([^<]*)</arg_key><arg_value>(.*?)</arg_value>", re.DOTALL)
_NAME_RE = re.compile(r"\s*([A-Za-z0-9_.\-]+)")
_TAG_RE = re.compile(r"</?arg_key>|</?arg_value>")
+
+
+def _fallback_tool_preamble(tools):
+ """Tool declaration for a family with no native tool tokens.
+
+ Mirrors the GLM-5.2 block because ``parse_tool_calls`` -- the parser these
+ families fall back to in ``parse_arch_tool_calls`` -- reads exactly that
+ wire format. Asking for a format the parser does not accept would produce
+ tool calls nobody can read back.
+ """
+ out = ["You have access to the following functions. Call one only when it "
+ "is needed to answer the user.\n\n<tools>\n"]
+ for tool in tools:
+ fn = tool.get("function", tool) if isinstance(tool, dict) else {}
+ out.append(json.dumps(fn, ensure_ascii=False) + "\n")
+ out.append("</tools>\n\nTo call a function, reply with the call and nothing "
+ "else, in this exact format:\n" + BOX_START + "{function-name}"
+ "<arg_key>{arg-key}</arg_key><arg_value>{arg-value}</arg_value>"
+ + BOX_END)
+ return "".join(out)
+
+
+def _fallback_tool_calls(tool_calls, index):
+ """Render assistant tool_calls in the format parse_tool_calls() reads."""
+ out = []
+ for position, call in enumerate(tool_calls or []):
+ if not isinstance(call, dict):
+ raise APIError(400, "Each tool call must be an object.",
+ f"messages.{index}.tool_calls.{position}")
+ fn = call.get("function", call)
+ if not isinstance(fn, dict):
+ raise APIError(400, "`function` must be an object.",
+ f"messages.{index}.tool_calls.{position}.function")
+ name = fn.get("name")
+ if not isinstance(name, str) or not name:
+ raise APIError(400, "`function.name` must be a non-empty string.",
+ f"messages.{index}.tool_calls.{position}.function.name")
+ args = fn.get("arguments", "{}")
+ if isinstance(args, str):
+ try:
+ args = json.loads(args) if args else {}
+ except (json.JSONDecodeError, TypeError, ValueError):
+ raise APIError(400, "`function.arguments` must be a JSON object.",
+ f"messages.{index}.tool_calls.{position}.function.arguments")
+ out.append(BOX_START + name)
+ for key, value in (args or {}).items():
+ rendered = value if isinstance(value, str) else json.dumps(
+ value, ensure_ascii=False)
+ out.append(f"<arg_key>{key}</arg_key><arg_value>{rendered}</arg_value>")
+ out.append(BOX_END)
+ return "".join(out)
+
+
+def _fallback_tool_result(message, index):
+ """Render a role:"tool" message as prose these templates can carry."""
+ body = content_text(message.get("content"), f"messages.{index}.content")
+ return TR_OPEN + body + TR_CLOSE
# A closing tag the model started but never finished ("</tool_cal", "</tool"), at end of reply.
_PARTIAL_END_RE = re.compile(r"<(?:/(?:t(?:o(?:o(?:l(?:_(?:c(?:a(?:l)?)?)?)?)?)?)?)?)?\Z")
@@ -258,6 +315,15 @@ _PARTIAL_END_RE = re.compile(r"<(?:/(?:t(?:o(?:o(?:l(?:_(?:c(?:a(?:l)?)?)?)?)?)?
# <arg_key>K</arg_key><arg_value> structure. Default OFF (never rewrites well-formed output).
_SALVAGE = os.environ.get("COLI_TOOL_SALVAGE", "0") == "1"
+# Families whose chat template has no tool syntax at all (OLMoE, Qwen3.6) refuse
+# tools[] and role:"tool" rather than invent a format. COLI_TOOL_FALLBACK=1 opts
+# into a prompt-injected translation for them: the declaration block, the prior
+# assistant calls and the tool results are written as ordinary turns, in the
+# same wire format parse_tool_calls() already reads back (#1378). Default OFF --
+# these models were never trained on tool syntax, so this trades a clean 400 for
+# output the parser may or may not recognise.
+_TOOL_FALLBACK = os.environ.get("COLI_TOOL_FALLBACK", "0") == "1"
+
def _tool_param_order(tools):
"""name -> ordered param names (required first) from the request schema, for de-mangling."""
@@ -1187,17 +1253,25 @@ def render_chat_olmoe(messages, enable_thinking=False, reasoning_effort=None, to
"""
if not isinstance(messages, list) or not messages:
raise APIError(400, "`messages` must be a non-empty array.", "messages")
- if tools or tool_choice not in (None, "none"):
- raise APIError(400, "Tool use is not wired up for the OLMoE engine yet.",
- "tools", "unsupported_parameter")
+ if tool_choice == "none":
+ tools = None
+ if (tools or tool_choice not in (None, "none")) and not _TOOL_FALLBACK:
+ raise APIError(400, "Tool use is not wired up for the OLMoE engine yet. "
+ "Set COLI_TOOL_FALLBACK=1 to opt into prompt-injected "
+ "tool translation.", "tools", "unsupported_parameter")
boundary = "|||IP_ADDRESS|||" # bos_token == eos_token in this tokenizer
parts = [boundary]
+ if tools and _TOOL_FALLBACK:
+ parts.append(f"<|system|>\n{_fallback_tool_preamble(tools)}\n")
last = len(messages) - 1
for index, message in enumerate(messages):
if not isinstance(message, dict):
raise APIError(400, "Each message must be an object.", f"messages.{index}")
role = message.get("role")
- if role not in ("system", "developer", "user", "assistant"):
+ allowed = ("system", "developer", "user", "assistant")
+ if _TOOL_FALLBACK:
+ allowed += ("tool",)
+ if role not in allowed:
raise APIError(400, f"Unsupported role {role!r}.", f"messages.{index}.role")
raw = message.get("content")
text = content_text(raw, f"messages.{index}.content") if raw is not None else ""
@@ -1205,8 +1279,13 @@ def render_chat_olmoe(messages, enable_thinking=False, reasoning_effort=None, to
parts.append(f"<|system|>\n{text}\n")
elif role == "user":
parts.append(f"<|user|>\n{text}\n")
+ elif role == "tool":
+ # No tool role in this template: the result rides in as a user turn.
+ parts.append(f"<|user|>\n{_fallback_tool_result(message, index)}\n")
else:
- parts.append(f"<|assistant|>\n{text}{boundary}")
+ calls = (_fallback_tool_calls(message.get("tool_calls"), index)
+ if _TOOL_FALLBACK else "")
+ parts.append(f"<|assistant|>\n{text}{calls}{boundary}")
if index != last:
parts.append("\n")
parts.append("<|assistant|>\n")
@@ -1224,20 +1303,36 @@ def render_chat_qwen(messages, enable_thinking=False, reasoning_effort=None, too
mirrored here byte for byte."""
if not isinstance(messages, list) or not messages:
raise APIError(400, "`messages` must be a non-empty array.", "messages")
- if tools or tool_choice not in (None, "none"):
- raise APIError(400, "Tool use is not wired up for the qwen36 engine yet.",
- "tools", "unsupported_parameter")
+ if tool_choice == "none":
+ tools = None
+ if (tools or tool_choice not in (None, "none")) and not _TOOL_FALLBACK:
+ raise APIError(400, "Tool use is not wired up for the qwen36 engine yet. "
+ "Set COLI_TOOL_FALLBACK=1 to opt into prompt-injected "
+ "tool translation.", "tools", "unsupported_parameter")
parts = []
+ if tools and _TOOL_FALLBACK:
+ parts.append("<|im_start|>system\n"
+ + _fallback_tool_preamble(tools) + "<|im_end|>\n")
for index, message in enumerate(messages):
if not isinstance(message, dict):
raise APIError(400, "Each message must be an object.", f"messages.{index}")
role = message.get("role")
if role == "developer":
role = "system"
- if role not in ("system", "user", "assistant"):
+ allowed = ("system", "user", "assistant")
+ if _TOOL_FALLBACK:
+ allowed += ("tool",)
+ if role not in allowed:
raise APIError(400, f"Unsupported role {role!r}.", f"messages.{index}.role")
raw = message.get("content")
text = content_text(raw, f"messages.{index}.content") if raw is not None else ""
+ if role == "tool":
+ # No tool role in this template: the result rides in as a user turn.
+ parts.append("<|im_start|>user\n"
+ + _fallback_tool_result(message, index) + "<|im_end|>\n")
+ continue
+ if role == "assistant" and _TOOL_FALLBACK:
+ text += _fallback_tool_calls(message.get("tool_calls"), index)
parts.append(f"<|im_start|>{role}\n{text}<|im_end|>\n")
parts.append("<|im_start|>assistant\n")
parts.append("<think>\n" if enable_thinking else "<think>\n\n</think>\n\n")
diff --git a/c/tests/test_openai_tools_fallback_e2e.py b/c/tests/test_openai_tools_fallback_e2e.py
new file mode 100644
index 00000000..894d59c3
--- /dev/null
+++ b/c/tests/test_openai_tools_fallback_e2e.py
@@ -0,0 +1,222 @@
+"""Prompt-injected tool translation for families with no native tool syntax.
+
+OLMoE and Qwen3.6 have no tool tokens in their chat templates, so the gateway
+answers 400 for `tools[]` and for a `role: "tool"` turn rather than invent a
+format. `COLI_TOOL_FALLBACK=1` opts into a translation that writes the
+declaration, the prior assistant calls and the tool results as ordinary turns,
+in the GLM wire format `parse_tool_calls()` already reads back (#1378).
+
+These tests pin both halves of that switch: the default still refuses, and with
+the flag set a full two-turn agent loop completes through
+`/v1/chat/completions`. Runs against a mock engine speaking the SERVE wire
+protocol, so no checkpoint is needed.
+"""
+import json
+import os
+import socket
+import subprocess
+import sys
+import tempfile
+import unittest
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+SERVER = HERE.parent / "openai_server.py"
+sys.path.insert(0, str(HERE.parent))
+
+# The call the injected preamble asks for, in the format parse_tool_calls reads.
+CALL = ("<tool_call>get_weather"
+ "<arg_key>location</arg_key><arg_value>Rome</arg_value>"
+ "<arg_key>unit</arg_key><arg_value>celsius</arg_value>"
+ "</tool_call>")
+
+MOCK_ENGINE = r'''#!/usr/bin/env python3
+import sys, os
+out, inp = sys.stdout.buffer, sys.stdin.buffer
+out.write(b"\x01\x01READY\x01\x01\n" + b"STAT 0 0 0 0 0\n"); out.flush()
+
+CALL = ("<tool_call>get_weather"
+ "<arg_key>location</arg_key><arg_value>Rome</arg_value>"
+ "<arg_key>unit</arg_key><arg_value>celsius</arg_value>"
+ "</tool_call>")
+
+def reply(rid, text):
+ data = text.encode("utf-8")
+ out.write(("DATA %s %d\n" % (rid, len(data))).encode() + data + b"\n"); out.flush()
+ out.write(("DONE %s STAT %d 1.0 50.0 10.0 42 0\n" % (rid, len(text.split()))).encode())
+ out.flush()
+
+while True:
+ line = inp.readline()
+ if not line: break
+ f = line.decode().strip().split()
+ if not f or f[0] != "SUBMIT": continue
+ rid, plen = f[1], int(f[3])
+ prompt = inp.read(plen).decode("utf-8", "replace"); inp.read(1)
+ with open(os.environ["MOCK_LOG"], "a") as log:
+ log.write(prompt + "\n\x00\n")
+ if "<tool_response>" in prompt:
+ reply(rid, "25 degrees and sunny in Rome.")
+ elif "weather in Rome" in prompt:
+ reply(rid, CALL)
+ else:
+ reply(rid, "Hello from the mock engine.")
+'''
+
+TOOLS = [{"type": "function", "function": {
+ "name": "get_weather",
+ "description": "Current weather for a city",
+ "parameters": {"type": "object",
+ "properties": {"location": {"type": "string"},
+ "unit": {"type": "string"}},
+ "required": ["location"]}}}]
+
+
+class _FallbackBase(unittest.TestCase):
+ """Boots the gateway for one arch, with COLI_TOOL_FALLBACK under test."""
+
+ arch = None
+ # config.json model_type for the family (family_registry.model_types), which
+ # is not always the --arch id: qwen36's checkpoints say "qwen3_5_moe".
+ model_type = None
+ fallback = "1"
+
+ @classmethod
+ def setUpClass(cls):
+ if cls.arch is None:
+ raise unittest.SkipTest("base class")
+ cls.tmp = tempfile.TemporaryDirectory()
+ (Path(cls.tmp.name) / "config.json").write_text(
+ json.dumps({"model_type": cls.model_type or cls.arch}), encoding="utf-8")
+ mock = Path(cls.tmp.name) / "mock_engine.py"
+ mock.write_text(MOCK_ENGINE)
+ mock.chmod(0o755)
+ cls.mock_log = Path(cls.tmp.name) / "prompts.log"
+ cls.mock_log.touch()
+ with socket.socket() as probe:
+ probe.bind(("127.0.0.1", 0))
+ cls.port = probe.getsockname()[1]
+ env = dict(os.environ, MOCK_LOG=str(cls.mock_log),
+ COLI_TOOL_FALLBACK=cls.fallback)
+ env.pop("COLI_API_KEY", None)
+ cls.server = subprocess.Popen(
+ [sys.executable, str(SERVER), "--model", cls.tmp.name,
+ "--engine", str(mock), "--arch", cls.arch, "--port", str(cls.port)],
+ env=env, stderr=subprocess.DEVNULL)
+ cls.base = f"http://127.0.0.1:{cls.port}/v1"
+ for _ in range(100):
+ try:
+ with urllib.request.urlopen(cls.base + "/models", timeout=2):
+ pass
+ return
+ except OSError:
+ if cls.server.poll() is not None:
+ raise RuntimeError("gateway exited during startup")
+ import time
+ time.sleep(0.1)
+ raise RuntimeError("gateway did not come up")
+
+ @classmethod
+ def tearDownClass(cls):
+ if getattr(cls, "server", None) is None:
+ return
+ cls.server.terminate()
+ cls.server.wait(timeout=5)
+ cls.tmp.cleanup()
+
+ def post(self, body):
+ req = urllib.request.Request(
+ self.base + "/chat/completions", json.dumps(body).encode(),
+ {"Content-Type": "application/json"})
+ with urllib.request.urlopen(req, timeout=30) as resp:
+ return json.loads(resp.read())
+
+ def model_id(self):
+ with urllib.request.urlopen(self.base + "/models", timeout=5) as resp:
+ return json.loads(resp.read())["data"][0]["id"]
+
+
+class _TwoTurnLoop(_FallbackBase):
+ """The two-turn loop the issue asks for, with the flag on."""
+
+ def test_tool_call_parsed_back(self):
+ out = self.post({"model": self.model_id(),
+ "messages": [{"role": "user", "content": "weather in Rome?"}],
+ "tools": TOOLS, "temperature": 0, "max_tokens": 128})
+ choice = out["choices"][0]
+ self.assertEqual(choice["finish_reason"], "tool_calls")
+ calls = choice["message"].get("tool_calls") or []
+ self.assertEqual(len(calls), 1)
+ self.assertEqual(calls[0]["function"]["name"], "get_weather")
+ self.assertEqual(json.loads(calls[0]["function"]["arguments"]),
+ {"location": "Rome", "unit": "celsius"})
+
+ def test_second_turn_completes(self):
+ mid = self.model_id()
+ msgs = [{"role": "user", "content": "weather in Rome?"}]
+ out = self.post({"model": mid, "messages": msgs, "tools": TOOLS,
+ "temperature": 0, "max_tokens": 128})
+ calls = out["choices"][0]["message"]["tool_calls"]
+ msgs.append({"role": "assistant", "content": None, "tool_calls": calls})
+ msgs.append({"role": "tool", "tool_call_id": calls[0]["id"],
+ "content": json.dumps({"temp_c": 25})})
+ out2 = self.post({"model": mid, "messages": msgs, "tools": TOOLS,
+ "temperature": 0, "max_tokens": 128})
+ self.assertIn("25", out2["choices"][0]["message"]["content"] or "")
+
+ prompts = self.mock_log.read_text()
+ self.assertIn("<tools>", prompts) # declaration was injected
+ self.assertIn("<tool_call>get_weather", prompts) # prior call replayed
+ self.assertIn("<tool_response>", prompts) # result reached the prompt
+
+
+class OlmoeToolFallbackE2E(_TwoTurnLoop):
+ arch = "olmoe"
+
+
+class Qwen36ToolFallbackE2E(_TwoTurnLoop):
+ arch = "qwen36"
+ model_type = "qwen3_5_moe"
+
+
+class _RefusedByDefault(_FallbackBase):
+ """With the flag unset the gateway must still refuse, not half-support."""
+
+ fallback = "0"
+
+ def _expect_400(self, body):
+ with self.assertRaises(urllib.error.HTTPError) as caught:
+ self.post(body)
+ self.assertEqual(caught.exception.code, 400)
+ return json.loads(caught.exception.read())
+
+ def test_tools_refused(self):
+ err = self._expect_400({"model": self.model_id(),
+ "messages": [{"role": "user", "content": "hi"}],
+ "tools": TOOLS, "max_tokens": 16})
+ self.assertIn("COLI_TOOL_FALLBACK", json.dumps(err))
+
+ def test_tool_role_refused(self):
+ self._expect_400({"model": self.model_id(),
+ "messages": [{"role": "user", "content": "hi"},
+ {"role": "tool", "content": "18C"}],
+ "max_tokens": 16})
+
+
+class OlmoeToolRefusedByDefault(_RefusedByDefault):
+ arch = "olmoe"
+
+
+class Qwen36ToolRefusedByDefault(_RefusedByDefault):
+ arch = "qwen36"
+ model_type = "qwen3_5_moe"
+
+
+# The shared base classes must not run as tests themselves.
+del _FallbackBase, _TwoTurnLoop, _RefusedByDefault
+
+
+if __name__ == "__main__":
+ unittest.main()