Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(server): OLMoE streaming dropped the answer into reasoning (#984)
JustVugg88a75657f301
diff --git a/c/openai_server.py b/c/openai_server.pyindex de1edf2e..4e901768 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -2818,6 +2818,15 @@ class APIHandler(BaseHTTPRequestHandler):enable_thinking = body.get("enable_thinking", reasoning_effort not in (None, "none"))if not isinstance(enable_thinking, bool):raise APIError(400, "`enable_thinking` must be a boolean.", "enable_thinking")+ if ARCH == "olmoe" and enable_thinking:+ # OLMoE's template has no thinking mode (render_chat_olmoe: "accepted+ # but unused"), so the engine never emits <think>/</think>. Left on,+ # the reasoning splitter files the ENTIRE answer as reasoning_content+ # and streams an empty `content` -- the drop reported in #984, which+ # bit streaming (ThinkingStreamSplit stays in thinking mode forever)+ # while non-streaming happened to survive. Make the template's "unused"+ # true end-to-end instead of trusting every path to opt out.+ enable_thinking = Falsetools = body.get("tools") or body.get("functions") or Nonetool_choice = body.get("tool_choice")audio_clips = [] if ARCH == "inkling" else None@@ -2844,6 +2853,8 @@ class APIHandler(BaseHTTPRequestHandler):enable_thinking = bool(thinking and thinking.get("type") == "enabled")if not enable_thinking and thinking is None and os.environ.get("COLI_THINK", "0") == "1":enable_thinking = True+ if ARCH == "olmoe":+ enable_thinking = False # #984: OLMoE has no thinking mode (see the OpenAI path)if body.get("max_tokens") is None:raise APIError(400, "`max_tokens` is required.", "max_tokens")# Reuse the OpenAI path's own validation by handing it an equivalent body.diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pyindex beca59e1..d9c02788 100644--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@ -914,6 +914,36 @@ class HTTPTest(unittest.TestCase):})self.assertEqual(caught.exception.code, 400)+ def test_olmoe_never_files_the_answer_as_reasoning(self):+ # #984: OLMoE has no thinking mode, so its engine never emits </think>.+ # With thinking left on, the reasoning splitter kept the whole answer in+ # reasoning_content and streamed an empty `content` -- content dropped.+ # Forcing thinking off for olmoe must return the answer as content+ # whether or not the client asked for reasoning, streaming or not.+ for streaming in (False, True):+ with self.subTest(stream=streaming), \+ patch("openai_server.ARCH", "olmoe"):+ with self.request("/v1/chat/completions", {+ "model": "test-model",+ "messages": [{"role": "user", "content": "Hi"}],+ "reasoning_effort": "high", # a client asking to think+ "stream": streaming,+ }) as response:+ raw = response.read().decode()+ if streaming:+ payloads = [json.loads(line[6:]) for line in raw.splitlines()+ if line.startswith("data: ") and line != "data: [DONE]"]+ content = "".join((c.get("delta") or {}).get("content", "")+ for p in payloads for c in p["choices"])+ reasoning = "".join((c.get("delta") or {}).get("reasoning_content", "")+ for p in payloads for c in p["choices"])+ else:+ msg = json.loads(raw)["choices"][0]["message"]+ content = msg.get("content") or ""+ reasoning = msg.get("reasoning_content") or ""+ self.assertIn("Hé", content, "the answer must arrive as content")+ self.assertEqual(reasoning, "", "olmoe must not produce reasoning")+def test_streaming_chat_completion(self):with self.request("/v1/chat/completions", {"model": "test-model", "messages": [{"role": "user", "content": "Hi"}],