Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

Stop stream keepalive threads on generation errors and cancellation

ZacharyZcR0d91cdc40781
diff --git a/c/openai_server.py b/c/openai_server.py
index 651607c1..abe4c22e 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -4526,7 +4526,8 @@ class APIHandler(BaseHTTPRequestHandler):
completion_id = id_prefix + uuid.uuid4().hex
created = int(time.time())
- with self.server.scheduler.admit(self.client_disconnected, cache_slot) as admission:
+ with self.server.scheduler.admit(self.client_disconnected, cache_slot) as admission, \
+ contextlib.ExitStack() as stream_cleanup:
queue_wait, cache_slot = admission
queue_headers = {"x-colibri-queue-wait-ms": str(round(queue_wait * 1000))}
if not stream:
@@ -4677,6 +4678,8 @@ class APIHandler(BaseHTTPRequestHandler):
"logprobs": None, "finish_reason": None}])
ka_thread[0] = threading.Thread(target=_keepalive, daemon=True)
ka_thread[0].start()
+ stream_cleanup.callback(ka_thread[0].join, timeout=2)
+ stream_cleanup.callback(ka_stop.set)
if chat and tools:
# Suppress tool-call markers from the streamed content and parse the authoritative
# calls from the FULL reply after generation. Hold back a marker-length tail so a
@@ -4959,7 +4962,8 @@ class APIHandler(BaseHTTPRequestHandler):
reason = "tool_calls" if calls else ("length" if stats["length_limited"] else "stop")
return content, self.ANTHROPIC_STOP[reason]
- with self.server.scheduler.admit(self.client_disconnected, cache_slot) as admission:
+ with self.server.scheduler.admit(self.client_disconnected, cache_slot) as admission, \
+ contextlib.ExitStack() as stream_cleanup:
queue_wait, cache_slot = admission
queue_headers = {"x-colibri-queue-wait-ms": str(round(queue_wait * 1000))}
if not stream:
@@ -5040,6 +5044,8 @@ class APIHandler(BaseHTTPRequestHandler):
"content_block": {"type": "text", "text": ""}})
ka_thread = threading.Thread(target=keepalive, daemon=True)
ka_thread.start()
+ stream_cleanup.callback(ka_thread.join, timeout=2)
+ stream_cleanup.callback(ka_stop.set)
raw = []
sideband = ToolSideband(ARCH == "kimi" and bool(tools), stop_sequences,
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index 10a56b36..a66a2af0 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -2859,6 +2859,44 @@ class KeepAliveFramingTest(unittest.TestCase):
self.assertIn("event: message_stop", raw)
self.assertNotIn("<STILL-OPEN>", raw)
+ def test_stream_exit_stops_keepalive_before_releasing_slot(self):
+ class CancelledEngine(_ExplodingEngine):
+ def generate(self, *args, **kwargs):
+ try:
+ return super().generate(*args, **kwargs)
+ except RuntimeError:
+ raise ClientCancelled()
+
+ original_thread = threading.Thread
+ for path in ("/v1/chat/completions", "/v1/messages"):
+ for engine_type, outcome in ((_ExplodingEngine, "failed"),
+ (CancelledEngine, "cancelled")):
+ with self.subTest(path=path, outcome=outcome):
+ pumps = []
+ def thread_factory(*args, **kwargs):
+ thread = original_thread(*args, **kwargs)
+ target = kwargs.get("target")
+ if getattr(target, "__name__", "") in ("_keepalive", "keepalive"):
+ stop = next(cell.cell_contents for cell in target.__closure__
+ if isinstance(cell.cell_contents, threading.Event))
+ pumps.append((thread, stop))
+ return thread
+ server = self._server(engine_type())
+ try:
+ with patch.object(threading, "Thread", side_effect=thread_factory):
+ status, _ = self._post(self._conn(server),
+ dict(self.CHAT, stream=True, max_tokens=16), path=path)
+ self.assertEqual(status, 200)
+ self.assertEqual(len(pumps), 1)
+ self.assertFalse(pumps[0][0].is_alive(), "keepalive survived stream exit")
+ stats = server.scheduler.snapshot()
+ self.assertEqual(stats[outcome], 1)
+ self.assertEqual((stats["active"], stats["completed"]), (0, 0))
+ finally:
+ for thread, stop in pumps:
+ stop.set()
+ thread.join(2)
+
def test_engine_failure_after_commit_does_not_splice_a_second_response(self):
"""Once the 200 is out, a 500 status line would land inside the event stream."""
server = self._server(_ExplodingEngine())