Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(api): stream the completions keepalive as a text chunk, not a delta

benmaster8298957b71594b
diff --git a/c/openai_server.py b/c/openai_server.py
index fdab0091..c8816325 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -3792,6 +3792,23 @@ def _json_float(value):
return value if math.isfinite(value) else None
+def _keepalive_choice(chat, visible):
+ """The streamed keepalive's single choice, in the endpoint's own chunk shape.
+
+ A chat chunk (`chat.completion.chunk`) carries a `delta`; a legacy
+ `/v1/completions` chunk (`text_completion`) carries `text`, never `delta`.
+ Emitting a `delta` on the completions stream produces a chunk with no `text`,
+ which a strict client (the OpenAI SDK models `CompletionChoice.text` as
+ required) rejects. Chat has a side channel for the diagnostic marker
+ (reasoning_content, off the visible answer); completions does not, so its
+ keepalive stays an empty `text` — a "." there would land in the completion —
+ and still resets the client's idle timer."""
+ if chat:
+ return {"index": 0, "delta": {"reasoning_content": "." if visible else ""},
+ "logprobs": None, "finish_reason": None}
+ return {"index": 0, "text": "", "logprobs": None, "finish_reason": None}
+
+
def _order_echo_records(prompt_records):
"""Place each prompt-echo record at its own wire `pos` index rather than trusting the
order the frames arrived in, and return `(payload bytes, record)` pairs -- the same
@@ -6466,10 +6483,7 @@ class APIHandler(BaseHTTPRequestHandler):
# cold prefill. COLI_VISIBLE_KEEPALIVE=1 restores the old visible "." for
# diagnosing whether keepalives are being delivered at all.
visible = os.environ.get("COLI_VISIBLE_KEEPALIVE") == "1"
- ping = [{"index": 0,
- "delta": ({"reasoning_content": "." if visible else ""} if chat
- else {"content": ""}),
- "logprobs": None, "finish_reason": None}]
+ ping = [_keepalive_choice(chat, visible)]
while not ka_stop.wait(1.0):
if not connected:
return
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index 72eaf450..0dadc31f 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -38,6 +38,7 @@ from openai_server import (APIError, APIHandler, APIServer, ClientCancelled,
parse_tool_calls_spans, parse_arch_tool_calls_spans,
THINK_OPEN, THINK_CLOSE,
_compose_span_maps, _cut_span_map, _project_span,
+ _keepalive_choice,
stop_policy, tune_child_env)
@@ -886,6 +887,22 @@ class TemplateTest(unittest.TestCase):
with self.subTest(value=value), self.assertRaises(APIError):
generation_options({"stop": value}, 8)
+ def test_keepalive_choice_matches_the_endpoint_chunk_shape(self):
+ # Chat chunks carry a `delta`; the diagnostic marker rides reasoning_content,
+ # off the visible answer.
+ chat = _keepalive_choice(True, False)
+ self.assertEqual(chat["delta"], {"reasoning_content": ""})
+ self.assertNotIn("text", chat)
+ self.assertEqual(_keepalive_choice(True, True)["delta"],
+ {"reasoning_content": "."})
+ # Legacy /v1/completions chunks carry `text`, never `delta`, or a strict
+ # client (OpenAI SDK: CompletionChoice.text is required) rejects the ping.
+ # No side channel, so the marker never appears: a "." would land in the text.
+ for visible in (False, True):
+ comp = _keepalive_choice(False, visible)
+ self.assertEqual(comp["text"], "")
+ self.assertNotIn("delta", comp)
+
def test_glm_chat_defaults_role_stops_without_changing_other_policies(self):
with patch("openai_server.ARCH", "glm"):
self.assertEqual(stop_policy({}, True), (DEFAULT_CHAT_STOP_SEQUENCES, True))