Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(api): cancel a generation that disconnects before the first frame (#908)
Gourav Kargwalaa4c51a14c32
diff --git a/c/openai_server.py b/c/openai_server.pyindex c4787e2e..8dcd35e9 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -1723,7 +1723,21 @@ class Engine:on_accept(info)while True:- kind, value = events.get()+ try:+ kind, value = events.get(timeout=0.05)+ except queue.Empty:+ # #908: cancelled() is only polled in the "data" branch, so a+ # client that disconnects before the engine's first DATA frame+ # (it is still prefilling) never cancels: the CANCEL never went+ # out, the turn ran to its token limit, and this thread stayed+ # blocked until the engine emitted something. Poll the callback+ # while idle so a pre-first-frame disconnect cancels too.+ if not cancel_sent and not stop_sent and cancelled and cancelled():+ cancel_sent = True+ with self.write_lock:+ self.process.stdin.write(f"CANCEL {request_id}\n".encode())+ self.process.stdin.flush()+ continueif kind == "accept":if accepted:raise RuntimeError("engine sent a duplicate ACCEPT frame")diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pyindex 5e2836b2..928a76a9 100644--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@ -601,6 +601,50 @@ class DispatcherTest(unittest.TestCase):self.assertEqual(output, ["x"])self.assertEqual(process.writes[-1].split(), [b"CANCEL", request_id])+ def test_cancels_generation_before_first_frame(self):+ # #908: a client that disconnects while the engine is still prefilling+ # (no DATA frame has arrived) must cancel too. cancelled() used to be+ # polled only in the "data" branch, so the CANCEL never went out and+ # the turn ran to its token limit while this thread stayed blocked.+ # The fake engine emits nothing until it sees CANCEL -- exactly the+ # pre-first-frame regime -- and must still get one.+ request_id = None++ def respond(process, frame):+ nonlocal request_id+ fields = frame.split()+ if fields[0] == b"SUBMIT":+ request_id = fields[1]+ elif fields[0] == b"CANCEL":+ self.assertEqual(fields[1], request_id)+ process.stdout.feed(b"ERROR " + request_id + b" CANCELLED\n")++ process = FakeProcess(respond)+ with patch("openai_server.subprocess.Popen", return_value=process):+ engine = Engine("glm", "model")+ flag = {"cancelled": False}+ outcome = []++ def generate():+ try:+ engine.generate("hello", 8, 0.7, 0.9, lambda _: None,+ cancelled=lambda: flag["cancelled"])+ except ClientCancelled:+ outcome.append("cancelled")++ thread = threading.Thread(target=generate)+ thread.start()+ for _ in range(200):+ if any(frame.startswith(b"SUBMIT") for frame in process.writes):+ break+ time.sleep(0.01)+ flag["cancelled"] = True+ thread.join(timeout=2)+ self.assertFalse(thread.is_alive())+ engine.close()+ self.assertEqual(outcome, ["cancelled"])+ self.assertEqual(process.writes[-1].split(), [b"CANCEL", request_id])+def test_stops_generation_through_successful_done_path(self):request_id = None