Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
Stop stream keepalive threads on generation errors and cancellation
ZacharyZcR0d91cdc40781
diff --git a/c/openai_server.py b/c/openai_server.pyindex 651607c1..abe4c22e 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -4526,7 +4526,8 @@ class APIHandler(BaseHTTPRequestHandler):completion_id = id_prefix + uuid.uuid4().hexcreated = int(time.time())- with self.server.scheduler.admit(self.client_disconnected, cache_slot) as admission:+ with self.server.scheduler.admit(self.client_disconnected, cache_slot) as admission, \+ contextlib.ExitStack() as stream_cleanup:queue_wait, cache_slot = admissionqueue_headers = {"x-colibri-queue-wait-ms": str(round(queue_wait * 1000))}if not stream:@@ -4677,6 +4678,8 @@ class APIHandler(BaseHTTPRequestHandler):"logprobs": None, "finish_reason": None}])ka_thread[0] = threading.Thread(target=_keepalive, daemon=True)ka_thread[0].start()+ stream_cleanup.callback(ka_thread[0].join, timeout=2)+ stream_cleanup.callback(ka_stop.set)if chat and tools:# Suppress tool-call markers from the streamed content and parse the authoritative# calls from the FULL reply after generation. Hold back a marker-length tail so a@@ -4959,7 +4962,8 @@ class APIHandler(BaseHTTPRequestHandler):reason = "tool_calls" if calls else ("length" if stats["length_limited"] else "stop")return content, self.ANTHROPIC_STOP[reason]- with self.server.scheduler.admit(self.client_disconnected, cache_slot) as admission:+ with self.server.scheduler.admit(self.client_disconnected, cache_slot) as admission, \+ contextlib.ExitStack() as stream_cleanup:queue_wait, cache_slot = admissionqueue_headers = {"x-colibri-queue-wait-ms": str(round(queue_wait * 1000))}if not stream:@@ -5040,6 +5044,8 @@ class APIHandler(BaseHTTPRequestHandler):"content_block": {"type": "text", "text": ""}})ka_thread = threading.Thread(target=keepalive, daemon=True)ka_thread.start()+ stream_cleanup.callback(ka_thread.join, timeout=2)+ stream_cleanup.callback(ka_stop.set)raw = []sideband = ToolSideband(ARCH == "kimi" and bool(tools), stop_sequences,diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pyindex 10a56b36..a66a2af0 100644--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@ -2859,6 +2859,44 @@ class KeepAliveFramingTest(unittest.TestCase):self.assertIn("event: message_stop", raw)self.assertNotIn("<STILL-OPEN>", raw)+ def test_stream_exit_stops_keepalive_before_releasing_slot(self):+ class CancelledEngine(_ExplodingEngine):+ def generate(self, *args, **kwargs):+ try:+ return super().generate(*args, **kwargs)+ except RuntimeError:+ raise ClientCancelled()++ original_thread = threading.Thread+ for path in ("/v1/chat/completions", "/v1/messages"):+ for engine_type, outcome in ((_ExplodingEngine, "failed"),+ (CancelledEngine, "cancelled")):+ with self.subTest(path=path, outcome=outcome):+ pumps = []+ def thread_factory(*args, **kwargs):+ thread = original_thread(*args, **kwargs)+ target = kwargs.get("target")+ if getattr(target, "__name__", "") in ("_keepalive", "keepalive"):+ stop = next(cell.cell_contents for cell in target.__closure__+ if isinstance(cell.cell_contents, threading.Event))+ pumps.append((thread, stop))+ return thread+ server = self._server(engine_type())+ try:+ with patch.object(threading, "Thread", side_effect=thread_factory):+ status, _ = self._post(self._conn(server),+ dict(self.CHAT, stream=True, max_tokens=16), path=path)+ self.assertEqual(status, 200)+ self.assertEqual(len(pumps), 1)+ self.assertFalse(pumps[0][0].is_alive(), "keepalive survived stream exit")+ stats = server.scheduler.snapshot()+ self.assertEqual(stats[outcome], 1)+ self.assertEqual((stats["active"], stats["completed"]), (0, 0))+ finally:+ for thread, stop in pumps:+ stop.set()+ thread.join(2)+def test_engine_failure_after_commit_does_not_splice_a_second_response(self):"""Once the 200 is out, a 500 status line would land inside the event stream."""server = self._server(_ExplodingEngine())