Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(server): OLMoE streaming dropped the answer into reasoning (#984)

JustVugg88a75657f301
diff --git a/c/openai_server.py b/c/openai_server.py
index de1edf2e..4e901768 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -2818,6 +2818,15 @@ class APIHandler(BaseHTTPRequestHandler):
enable_thinking = body.get("enable_thinking", reasoning_effort not in (None, "none"))
if not isinstance(enable_thinking, bool):
raise APIError(400, "`enable_thinking` must be a boolean.", "enable_thinking")
+ if ARCH == "olmoe" and enable_thinking:
+ # OLMoE's template has no thinking mode (render_chat_olmoe: "accepted
+ # but unused"), so the engine never emits <think>/</think>. Left on,
+ # the reasoning splitter files the ENTIRE answer as reasoning_content
+ # and streams an empty `content` -- the drop reported in #984, which
+ # bit streaming (ThinkingStreamSplit stays in thinking mode forever)
+ # while non-streaming happened to survive. Make the template's "unused"
+ # true end-to-end instead of trusting every path to opt out.
+ enable_thinking = False
tools = body.get("tools") or body.get("functions") or None
tool_choice = body.get("tool_choice")
audio_clips = [] if ARCH == "inkling" else None
@@ -2844,6 +2853,8 @@ class APIHandler(BaseHTTPRequestHandler):
enable_thinking = bool(thinking and thinking.get("type") == "enabled")
if not enable_thinking and thinking is None and os.environ.get("COLI_THINK", "0") == "1":
enable_thinking = True
+ if ARCH == "olmoe":
+ enable_thinking = False # #984: OLMoE has no thinking mode (see the OpenAI path)
if body.get("max_tokens") is None:
raise APIError(400, "`max_tokens` is required.", "max_tokens")
# Reuse the OpenAI path's own validation by handing it an equivalent body.
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index beca59e1..d9c02788 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -914,6 +914,36 @@ class HTTPTest(unittest.TestCase):
})
self.assertEqual(caught.exception.code, 400)
+ def test_olmoe_never_files_the_answer_as_reasoning(self):
+ # #984: OLMoE has no thinking mode, so its engine never emits </think>.
+ # With thinking left on, the reasoning splitter kept the whole answer in
+ # reasoning_content and streamed an empty `content` -- content dropped.
+ # Forcing thinking off for olmoe must return the answer as content
+ # whether or not the client asked for reasoning, streaming or not.
+ for streaming in (False, True):
+ with self.subTest(stream=streaming), \
+ patch("openai_server.ARCH", "olmoe"):
+ with self.request("/v1/chat/completions", {
+ "model": "test-model",
+ "messages": [{"role": "user", "content": "Hi"}],
+ "reasoning_effort": "high", # a client asking to think
+ "stream": streaming,
+ }) as response:
+ raw = response.read().decode()
+ if streaming:
+ payloads = [json.loads(line[6:]) for line in raw.splitlines()
+ if line.startswith("data: ") and line != "data: [DONE]"]
+ content = "".join((c.get("delta") or {}).get("content", "")
+ for p in payloads for c in p["choices"])
+ reasoning = "".join((c.get("delta") or {}).get("reasoning_content", "")
+ for p in payloads for c in p["choices"])
+ else:
+ msg = json.loads(raw)["choices"][0]["message"]
+ content = msg.get("content") or ""
+ reasoning = msg.get("reasoning_content") or ""
+ self.assertIn("Hé", content, "the answer must arrive as content")
+ self.assertEqual(reasoning, "", "olmoe must not produce reasoning")
+
def test_streaming_chat_completion(self):
with self.request("/v1/chat/completions", {
"model": "test-model", "messages": [{"role": "user", "content": "Hi"}],