Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(api): stream the completions keepalive as a text chunk, not a delta
benmaster8298957b71594b
diff --git a/c/openai_server.py b/c/openai_server.pyindex fdab0091..c8816325 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -3792,6 +3792,23 @@ def _json_float(value):return value if math.isfinite(value) else None+def _keepalive_choice(chat, visible):+ """The streamed keepalive's single choice, in the endpoint's own chunk shape.++ A chat chunk (`chat.completion.chunk`) carries a `delta`; a legacy+ `/v1/completions` chunk (`text_completion`) carries `text`, never `delta`.+ Emitting a `delta` on the completions stream produces a chunk with no `text`,+ which a strict client (the OpenAI SDK models `CompletionChoice.text` as+ required) rejects. Chat has a side channel for the diagnostic marker+ (reasoning_content, off the visible answer); completions does not, so its+ keepalive stays an empty `text` — a "." there would land in the completion —+ and still resets the client's idle timer."""+ if chat:+ return {"index": 0, "delta": {"reasoning_content": "." if visible else ""},+ "logprobs": None, "finish_reason": None}+ return {"index": 0, "text": "", "logprobs": None, "finish_reason": None}++def _order_echo_records(prompt_records):"""Place each prompt-echo record at its own wire `pos` index rather than trusting theorder the frames arrived in, and return `(payload bytes, record)` pairs -- the same@@ -6466,10 +6483,7 @@ class APIHandler(BaseHTTPRequestHandler):# cold prefill. COLI_VISIBLE_KEEPALIVE=1 restores the old visible "." for# diagnosing whether keepalives are being delivered at all.visible = os.environ.get("COLI_VISIBLE_KEEPALIVE") == "1"- ping = [{"index": 0,- "delta": ({"reasoning_content": "." if visible else ""} if chat- else {"content": ""}),- "logprobs": None, "finish_reason": None}]+ ping = [_keepalive_choice(chat, visible)]while not ka_stop.wait(1.0):if not connected:returndiff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pyindex 72eaf450..0dadc31f 100644--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@ -38,6 +38,7 @@ from openai_server import (APIError, APIHandler, APIServer, ClientCancelled,parse_tool_calls_spans, parse_arch_tool_calls_spans,THINK_OPEN, THINK_CLOSE,_compose_span_maps, _cut_span_map, _project_span,+ _keepalive_choice,stop_policy, tune_child_env)@@ -886,6 +887,22 @@ class TemplateTest(unittest.TestCase):with self.subTest(value=value), self.assertRaises(APIError):generation_options({"stop": value}, 8)+ def test_keepalive_choice_matches_the_endpoint_chunk_shape(self):+ # Chat chunks carry a `delta`; the diagnostic marker rides reasoning_content,+ # off the visible answer.+ chat = _keepalive_choice(True, False)+ self.assertEqual(chat["delta"], {"reasoning_content": ""})+ self.assertNotIn("text", chat)+ self.assertEqual(_keepalive_choice(True, True)["delta"],+ {"reasoning_content": "."})+ # Legacy /v1/completions chunks carry `text`, never `delta`, or a strict+ # client (OpenAI SDK: CompletionChoice.text is required) rejects the ping.+ # No side channel, so the marker never appears: a "." would land in the text.+ for visible in (False, True):+ comp = _keepalive_choice(False, visible)+ self.assertEqual(comp["text"], "")+ self.assertNotIn("delta", comp)+def test_glm_chat_defaults_role_stops_without_changing_other_policies(self):with patch("openai_server.ARCH", "glm"):self.assertEqual(stop_policy({}, True), (DEFAULT_CHAT_STOP_SEQUENCES, True))