Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
"""End-to-end tool calling for OLMoE and Qwen3.6.
OLMoE retains opt-in prompt-injected tool translation, while Qwen3.6
supports native tool calling without COLI_TOOL_FALLBACK.
Both implementations are exercised through /v1/chat/completions using
a mock SERVE engine, without requiring a checkpoint.
"""
import json
import os
import socket
import subprocess
import sys
import tempfile
import unittest
import urllib.error
import urllib.request
from pathlib import Path
HERE = Path(__file__).resolve().parent
SERVER = HERE.parent / "openai_server.py"
sys.path.insert(0, str(HERE.parent))
# The call the injected preamble asks for, in the format parse_tool_calls reads.
CALL = ("<tool_call>get_weather"
"<arg_key>location</arg_key><arg_value>Rome</arg_value>"
"<arg_key>unit</arg_key><arg_value>celsius</arg_value>"
"</tool_call>")
MOCK_ENGINE = r'''#!/usr/bin/env python3
import sys, os, re
out, inp = sys.stdout.buffer, sys.stdin.buffer
out.write(b"\x01\x01READY\x01\x01\n" + b"STAT 0 0 0 0 0\n"); out.flush()
CALL = ("<tool_call>get_weather"
"<arg_key>location</arg_key><arg_value>Rome</arg_value>"
"<arg_key>unit</arg_key><arg_value>celsius</arg_value>"
"</tool_call>")
QWEN_CALL = (
"<tool_call>"
"<function=get_weather>"
"<parameter=location>\nRome\n</parameter>"
"<parameter=unit>\ncelsius\n</parameter>"
"</function>"
"</tool_call>"
)
NATIVE_QWEN = os.environ.get("MOCK_NATIVE_QWEN") == "1"
def reply(rid, text):
data = text.encode("utf-8")
out.write(("DATA %s %d\n" % (rid, len(data))).encode() + data + b"\n"); out.flush()
out.write(("DONE %s STAT %d 1.0 50.0 10.0 42 0\n" % (rid, len(text.split()))).encode())
out.flush()
while True:
line = inp.readline()
if not line: break
f = line.decode().strip().split()
if not f or f[0] != "SUBMIT": continue
rid, plen = f[1], int(f[3])
prompt = inp.read(plen).decode("utf-8", "replace"); inp.read(1)
with open(os.environ["MOCK_LOG"], "a") as log:
log.write(prompt + "\n\x00\n")
if NATIVE_QWEN and re.search(
r'<\|im_start\|>user\n'
r'<tool_response>\n'
r'\s*\{"temp_c"\s*:\s*25\}\s*\n'
r'</tool_response><\|im_end\|>',
prompt,
):
reply(rid, "25 degrees and sunny in Rome.")
elif not NATIVE_QWEN and "<tool_response>" in prompt:
reply(rid, "25 degrees and sunny in Rome.")
elif "weather in Rome" in prompt:
reply(rid, QWEN_CALL if NATIVE_QWEN else CALL)
else:
reply(rid, "Hello from the mock engine.")
'''
TOOLS = [{"type": "function", "function": {
"name": "get_weather",
"description": "Current weather for a city",
"parameters": {"type": "object",
"properties": {"location": {"type": "string"},
"unit": {"type": "string"}},
"required": ["location"]}}}]
@unittest.skipUnless(os.name == "posix",
"the mock engine is a shebang script the gateway execs directly; "
"Windows CreateProcess cannot run it. The gateway logic under test "
"is platform-independent and covered by the POSIX CI jobs.")
class _FallbackBase(unittest.TestCase):
"""Boots the gateway for one arch, with COLI_TOOL_FALLBACK under test."""
arch = None
# config.json model_type for the family (family_registry.model_types), which
# is not always the --arch id: qwen36's checkpoints say "qwen3_5_moe".
model_type = None
fallback = "1"
@classmethod
def setUpClass(cls):
if cls.arch is None:
raise unittest.SkipTest("base class")
cls.tmp = tempfile.TemporaryDirectory()
(Path(cls.tmp.name) / "config.json").write_text(
json.dumps({"model_type": cls.model_type or cls.arch}), encoding="utf-8")
mock = Path(cls.tmp.name) / "mock_engine.py"
mock.write_text(MOCK_ENGINE)
mock.chmod(0o755)
cls.mock_log = Path(cls.tmp.name) / "prompts.log"
cls.mock_log.touch()
with socket.socket() as probe:
probe.bind(("127.0.0.1", 0))
cls.port = probe.getsockname()[1]
env = dict(os.environ, MOCK_LOG=str(cls.mock_log),
COLI_TOOL_FALLBACK=cls.fallback,
MOCK_NATIVE_QWEN="1" if cls.arch == "qwen36" else "0")
env.pop("COLI_API_KEY", None)
cls.server = subprocess.Popen(
[sys.executable, str(SERVER), "--model", cls.tmp.name,
"--engine", str(mock), "--arch", cls.arch, "--port", str(cls.port)],
env=env, stderr=subprocess.DEVNULL)
cls.base = f"http://127.0.0.1:{cls.port}/v1"
for _ in range(100):
try:
with urllib.request.urlopen(cls.base + "/models", timeout=2):
pass
return
except OSError:
if cls.server.poll() is not None:
raise RuntimeError("gateway exited during startup")
import time
time.sleep(0.1)
raise RuntimeError("gateway did not come up")
@classmethod
def tearDownClass(cls):
if getattr(cls, "server", None) is None:
return
cls.server.terminate()
cls.server.wait(timeout=5)
cls.tmp.cleanup()
def post(self, body):
req = urllib.request.Request(
self.base + "/chat/completions", json.dumps(body).encode(),
{"Content-Type": "application/json"})
with urllib.request.urlopen(req, timeout=30) as resp:
return json.loads(resp.read())
def model_id(self):
with urllib.request.urlopen(self.base + "/models", timeout=5) as resp:
return json.loads(resp.read())["data"][0]["id"]
class _TwoTurnLoop(_FallbackBase):
"""The two-turn loop the issue asks for, with the flag on."""
def test_tool_call_parsed_back(self):
out = self.post({"model": self.model_id(),
"messages": [{"role": "user", "content": "weather in Rome?"}],
"tools": TOOLS, "temperature": 0, "max_tokens": 128})
choice = out["choices"][0]
self.assertEqual(choice["finish_reason"], "tool_calls")
calls = choice["message"].get("tool_calls") or []
self.assertEqual(len(calls), 1)
self.assertEqual(calls[0]["function"]["name"], "get_weather")
self.assertEqual(json.loads(calls[0]["function"]["arguments"]),
{"location": "Rome", "unit": "celsius"})
def test_second_turn_completes(self):
mid = self.model_id()
msgs = [{"role": "user", "content": "weather in Rome?"}]
out = self.post({"model": mid, "messages": msgs, "tools": TOOLS,
"temperature": 0, "max_tokens": 128})
calls = out["choices"][0]["message"]["tool_calls"]
msgs.append({"role": "assistant", "content": None, "tool_calls": calls})
msgs.append({"role": "tool", "tool_call_id": calls[0]["id"],
"content": json.dumps({"temp_c": 25})})
out2 = self.post({"model": mid, "messages": msgs, "tools": TOOLS,
"temperature": 0, "max_tokens": 128})
self.assertIn("25", out2["choices"][0]["message"]["content"] or "")
prompts = self.mock_log.read_text()
self.assertIn("<tools>", prompts) # declaration was injected
self.assertIn("<tool_call>get_weather", prompts) # prior call replayed
self.assertIn("<tool_response>", prompts) # result reached the prompt
class OlmoeToolFallbackE2E(_TwoTurnLoop):
arch = "olmoe"
class Qwen36NativeToolsE2E(_TwoTurnLoop):
"""Native Qwen3.6 tool calling works without the fallback flag."""
arch = "qwen36"
model_type = "qwen3_5_moe"
fallback = "0"
def test_second_turn_completes(self):
mid = self.model_id()
msgs = [{"role": "user", "content": "weather in Rome?"}]
out = self.post({
"model": mid,
"messages": msgs,
"tools": TOOLS,
"temperature": 0,
"max_tokens": 128,
})
choice = out["choices"][0]
self.assertEqual(choice["finish_reason"], "tool_calls")
calls = choice["message"]["tool_calls"]
self.assertEqual(len(calls), 1)
msgs.append({
"role": "assistant",
"content": None,
"tool_calls": calls,
})
msgs.append({
"role": "tool",
"tool_call_id": calls[0]["id"],
"content": json.dumps({"temp_c": 25}),
})
out2 = self.post({
"model": mid,
"messages": msgs,
"tools": TOOLS,
"temperature": 0,
"max_tokens": 128,
})
self.assertEqual(out2["choices"][0]["finish_reason"], "stop")
self.assertEqual(
out2["choices"][0]["message"]["content"],
"25 degrees and sunny in Rome.",
)
prompts = self.mock_log.read_text()
second_prompt = prompts.split("\x00")[-2]
call_marker = "<function=get_weather>"
response_marker = (
'<|im_start|>user\n'
'<tool_response>\n'
'{"temp_c": 25}\n'
'</tool_response><|im_end|>'
)
self.assertIn(call_marker, second_prompt)
self.assertIn(response_marker, second_prompt)
self.assertLess(
second_prompt.index(call_marker),
second_prompt.index(response_marker),
)
class _RefusedByDefault(_FallbackBase):
"""With the flag unset the gateway must still refuse, not half-support."""
fallback = "0"
def _expect_400(self, body):
with self.assertRaises(urllib.error.HTTPError) as caught:
self.post(body)
self.assertEqual(caught.exception.code, 400)
return json.loads(caught.exception.read())
def test_tools_refused(self):
err = self._expect_400({"model": self.model_id(),
"messages": [{"role": "user", "content": "hi"}],
"tools": TOOLS, "max_tokens": 16})
self.assertIn("COLI_TOOL_FALLBACK", json.dumps(err))
def test_tool_role_refused(self):
self._expect_400({"model": self.model_id(),
"messages": [{"role": "user", "content": "hi"},
{"role": "tool", "content": "18C"}],
"max_tokens": 16})
class OlmoeToolRefusedByDefault(_RefusedByDefault):
arch = "olmoe"
class Qwen36ToolAcceptedByDefault(_FallbackBase):
"""Qwen3.6 accepts native tools without the fallback flag."""
arch = "qwen36"
model_type = "qwen3_5_moe"
fallback = "0"
def test_tools_accepted(self):
out = self.post({
"model": self.model_id(),
"messages": [
{"role": "user", "content": "weather in Rome?"}
],
"tools": TOOLS,
"temperature": 0,
"max_tokens": 128,
})
choice = out["choices"][0]
self.assertEqual(choice["finish_reason"], "tool_calls")
calls = choice["message"]["tool_calls"]
self.assertEqual(len(calls), 1)
self.assertEqual(
calls[0]["function"]["name"],
"get_weather",
)
def test_tool_role_accepted(self):
out = self.post({
"model": self.model_id(),
"messages": [
{
"role": "user",
"content": "weather in Rome?",
},
{
"role": "assistant",
"content": None,
"tool_calls": [{
"id": "call_test",
"type": "function",
"function": {
"name": "get_weather",
"arguments": json.dumps({
"location": "Rome",
"unit": "celsius",
}),
},
}],
},
{
"role": "tool",
"tool_call_id": "call_test",
"content": json.dumps({"temp_c": 25}),
},
],
"tools": TOOLS,
"temperature": 0,
"max_tokens": 128,
})
choice = out["choices"][0]
self.assertEqual(choice["finish_reason"], "stop")
self.assertIn(
"25",
choice["message"]["content"] or "",
)
# The shared base classes must not run as tests themselves.
del _FallbackBase, _TwoTurnLoop, _RefusedByDefault
if __name__ == "__main__":
unittest.main()