Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
serve: a client hanging up is not a server error
JustVuggf5fcbc1a36ed
diff --git a/c/openai_server.py b/c/openai_server.pyindex ec946b2a..28903240 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -1722,7 +1722,22 @@ class APIHandler(BaseHTTPRequestHandler):instead of asking each early return to remember."""self._committed = Falseself._body_read = False- super().handle_one_request()+ try:+ super().handle_one_request()+ except (BrokenPipeError, ConnectionResetError):+ # The client hung up mid-response. That is not an error here, it is+ # how HTTP clients behave: `coli chat` polls /health while the model+ # loads and drops each connection as soon as it has its answer, and+ # Ctrl-C during a stream closes the socket by design -- the banner+ # tells the user to do exactly that. Without this, socketserver's+ # handler prints a full traceback per occurrence, so a normal start+ # buried the loading spinner under BrokenPipeError stack traces and+ # every cancelled answer looked like a crash.+ #+ # Caught here rather than in send_json() so it also covers the SSE+ # writes in the streaming path, which is where Ctrl-C lands.+ self.close_connection = True+ returnif not self.close_connection:self._drain_request_body()diff --git a/c/tests/test_openai_server.py b/c/tests/test_openai_server.pyindex 77225a6e..5465499d 100644--- a/c/tests/test_openai_server.py+++ b/c/tests/test_openai_server.py@@ -5,6 +5,9 @@ import mathimport socketimport tempfileimport threading+import sys+import time+import structimport unittestfrom unittest.mock import patchfrom urllib.error import HTTPError@@ -858,6 +861,58 @@ class HTTPTest(unittest.TestCase):self.assertEqual(caught.exception.code, 400)+class ClientHangupTest(unittest.TestCase):+ """A client that disconnects mid-response must not print a traceback.++ `coli chat` polls /health on a 2 s timeout while the model loads and drops+ each connection the moment it has an answer; Ctrl-C during a stream closes+ the socket by design -- the banner tells the user to do exactly that. Both+ reach the handler as BrokenPipeError, and socketserver logs an unhandled+ exception per occurrence, so a normal DeepSeek V4 start buried the loading+ spinner under stack traces and every cancelled answer looked like a crash.+ """++ def setUp(self):+ self.engine = FakeEngine()+ self.server = APIServer(("127.0.0.1", 0), self.engine, "test-model",+ None, 16, kv_slots=1)+ self.errors = []+ # socketserver routes an escaped exception here; the base class prints+ # it to stderr. Recording instead of printing is what lets the test+ # assert on it rather than on captured output.+ self.server.handle_error = lambda request, address: self.errors.append(+ sys.exc_info()[1])+ self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)+ self.thread.start()+ self.addCleanup(self.thread.join, 2)+ self.addCleanup(self.server.server_close)+ self.addCleanup(self.server.shutdown)+ self.addCleanup(self.server.scheduler.close)++ def _hang_up_after_request(self, path):+ """Send a request, then close without reading the response."""+ sock = socket.create_connection(("127.0.0.1", self.server.server_port), 2)+ sock.sendall(f"GET {path} HTTP/1.1\r\nHost: 127.0.0.1\r\n"+ f"Connection: close\r\n\r\n".encode())+ sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER,+ struct.pack("ii", 1, 0)) # RST rather than a clean FIN+ sock.close()+ time.sleep(0.3)++ def test_hangup_on_health_is_not_an_error(self):+ for _ in range(5):+ self._hang_up_after_request("/health")+ self.assertEqual(self.errors, [],+ "client disconnect surfaced as a server error")++ def test_the_server_still_answers_afterwards(self):+ """The real damage would be a handler thread lost to the exception."""+ self._hang_up_after_request("/health")+ with urlopen(f"http://127.0.0.1:{self.server.server_port}/health",+ timeout=2) as response:+ self.assertEqual(json.load(response)["status"], "ok")++class StaticServingTest(unittest.TestCase):def setUp(self):self.tmp = tempfile.TemporaryDirectory()