Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(api): cancel a generation that disconnects before the first frame (#908)

Gourav Kargwalaa4c51a14c32
diff --git a/c/openai_server.py b/c/openai_server.py
index c4787e2e..8dcd35e9 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -1723,7 +1723,21 @@ class Engine:
on_accept(info)
while True:
- kind, value = events.get()
+ try:
+ kind, value = events.get(timeout=0.05)
+ except queue.Empty:
+ # #908: cancelled() is only polled in the "data" branch, so a
+ # client that disconnects before the engine's first DATA frame
+ # (it is still prefilling) never cancels: the CANCEL never went
+ # out, the turn ran to its token limit, and this thread stayed
+ # blocked until the engine emitted something. Poll the callback
+ # while idle so a pre-first-frame disconnect cancels too.
+ if not cancel_sent and not stop_sent and cancelled and cancelled():
+ cancel_sent = True
+ with self.write_lock:
+ self.process.stdin.write(f"CANCEL {request_id}\n".encode())
+ self.process.stdin.flush()
+ continue
if kind == "accept":
if accepted:
raise RuntimeError("engine sent a duplicate ACCEPT frame")
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index 5e2836b2..928a76a9 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -601,6 +601,50 @@ class DispatcherTest(unittest.TestCase):
self.assertEqual(output, ["x"])
self.assertEqual(process.writes[-1].split(), [b"CANCEL", request_id])
+ def test_cancels_generation_before_first_frame(self):
+ # #908: a client that disconnects while the engine is still prefilling
+ # (no DATA frame has arrived) must cancel too. cancelled() used to be
+ # polled only in the "data" branch, so the CANCEL never went out and
+ # the turn ran to its token limit while this thread stayed blocked.
+ # The fake engine emits nothing until it sees CANCEL -- exactly the
+ # pre-first-frame regime -- and must still get one.
+ request_id = None
+
+ def respond(process, frame):
+ nonlocal request_id
+ fields = frame.split()
+ if fields[0] == b"SUBMIT":
+ request_id = fields[1]
+ elif fields[0] == b"CANCEL":
+ self.assertEqual(fields[1], request_id)
+ process.stdout.feed(b"ERROR " + request_id + b" CANCELLED\n")
+
+ process = FakeProcess(respond)
+ with patch("openai_server.subprocess.Popen", return_value=process):
+ engine = Engine("glm", "model")
+ flag = {"cancelled": False}
+ outcome = []
+
+ def generate():
+ try:
+ engine.generate("hello", 8, 0.7, 0.9, lambda _: None,
+ cancelled=lambda: flag["cancelled"])
+ except ClientCancelled:
+ outcome.append("cancelled")
+
+ thread = threading.Thread(target=generate)
+ thread.start()
+ for _ in range(200):
+ if any(frame.startswith(b"SUBMIT") for frame in process.writes):
+ break
+ time.sleep(0.01)
+ flag["cancelled"] = True
+ thread.join(timeout=2)
+ self.assertFalse(thread.is_alive())
+ engine.close()
+ self.assertEqual(outcome, ["cancelled"])
+ self.assertEqual(process.writes[-1].split(), [b"CANCEL", request_id])
+
def test_stops_generation_through_successful_done_path(self):
request_id = None