Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

serve: a client hanging up is not a server error

JustVuggf5fcbc1a36ed
diff --git a/c/openai_server.py b/c/openai_server.py
index ec946b2a..28903240 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -1722,7 +1722,22 @@ class APIHandler(BaseHTTPRequestHandler):
instead of asking each early return to remember."""
self._committed = False
self._body_read = False
- super().handle_one_request()
+ try:
+ super().handle_one_request()
+ except (BrokenPipeError, ConnectionResetError):
+ # The client hung up mid-response. That is not an error here, it is
+ # how HTTP clients behave: `coli chat` polls /health while the model
+ # loads and drops each connection as soon as it has its answer, and
+ # Ctrl-C during a stream closes the socket by design -- the banner
+ # tells the user to do exactly that. Without this, socketserver's
+ # handler prints a full traceback per occurrence, so a normal start
+ # buried the loading spinner under BrokenPipeError stack traces and
+ # every cancelled answer looked like a crash.
+ #
+ # Caught here rather than in send_json() so it also covers the SSE
+ # writes in the streaming path, which is where Ctrl-C lands.
+ self.close_connection = True
+ return
if not self.close_connection:
self._drain_request_body()
diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.py
index 77225a6e..5465499d 100644
--- a/c/tests/test_openai_server.py
+++ b/c/tests/test_openai_server.py
@@ -5,6 +5,9 @@ import math
import socket
import tempfile
import threading
+import sys
+import time
+import struct
import unittest
from unittest.mock import patch
from urllib.error import HTTPError
@@ -858,6 +861,58 @@ class HTTPTest(unittest.TestCase):
self.assertEqual(caught.exception.code, 400)
+class ClientHangupTest(unittest.TestCase):
+ """A client that disconnects mid-response must not print a traceback.
+
+ `coli chat` polls /health on a 2 s timeout while the model loads and drops
+ each connection the moment it has an answer; Ctrl-C during a stream closes
+ the socket by design -- the banner tells the user to do exactly that. Both
+ reach the handler as BrokenPipeError, and socketserver logs an unhandled
+ exception per occurrence, so a normal DeepSeek V4 start buried the loading
+ spinner under stack traces and every cancelled answer looked like a crash.
+ """
+
+ def setUp(self):
+ self.engine = FakeEngine()
+ self.server = APIServer(("127.0.0.1", 0), self.engine, "test-model",
+ None, 16, kv_slots=1)
+ self.errors = []
+ # socketserver routes an escaped exception here; the base class prints
+ # it to stderr. Recording instead of printing is what lets the test
+ # assert on it rather than on captured output.
+ self.server.handle_error = lambda request, address: self.errors.append(
+ sys.exc_info()[1])
+ self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
+ self.thread.start()
+ self.addCleanup(self.thread.join, 2)
+ self.addCleanup(self.server.server_close)
+ self.addCleanup(self.server.shutdown)
+ self.addCleanup(self.server.scheduler.close)
+
+ def _hang_up_after_request(self, path):
+ """Send a request, then close without reading the response."""
+ sock = socket.create_connection(("127.0.0.1", self.server.server_port), 2)
+ sock.sendall(f"GET {path} HTTP/1.1\r\nHost: 127.0.0.1\r\n"
+ f"Connection: close\r\n\r\n".encode())
+ sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,
+ struct.pack("ii", 1, 0)) # RST rather than a clean FIN
+ sock.close()
+ time.sleep(0.3)
+
+ def test_hangup_on_health_is_not_an_error(self):
+ for _ in range(5):
+ self._hang_up_after_request("/health")
+ self.assertEqual(self.errors, [],
+ "client disconnect surfaced as a server error")
+
+ def test_the_server_still_answers_afterwards(self):
+ """The real damage would be a handler thread lost to the exception."""
+ self._hang_up_after_request("/health")
+ with urlopen(f"http://127.0.0.1:{self.server.server_port}/health",
+ timeout=2) as response:
+ self.assertEqual(json.load(response)["status"], "ok")
+
+
class StaticServingTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()