Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(api): enforce read deadlines between socket receives
Rudy Celekli8dead8192641
diff --git a/c/openai_server.py b/c/openai_server.pyindex 2838036f..6cb31ed3 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -5994,13 +5994,44 @@ class _DeadlineReader:raise TimeoutError("request read deadline exceeded")self._sock.settimeout(min(self._per_read, left))- def readline(self, *args):- self._arm()- return self._raw.readline(*args)+ def readline(self, size=-1):+ chunks = []+ remaining = size+ while remaining != 0:+ self._arm()+ # peek() does at most one raw socket read. Consume only bytes it+ # already buffered, so readline cannot renew one socket timeout+ # internally while a peer drips an unfinished header.+ buffered = self._raw.peek(1)+ if not buffered:+ break+ newline = buffered.find(b"\n")+ take = newline + 1 if newline >= 0 else len(buffered)+ if remaining > 0:+ take = min(take, remaining)+ chunk = self._raw.read1(take)+ chunks.append(chunk)+ if remaining > 0:+ remaining -= len(chunk)+ if chunk.endswith(b"\n"):+ break+ return b"".join(chunks)- def read(self, *args):- self._arm()- return self._raw.read(*args)+ def read(self, size=-1):+ chunks = []+ remaining = size+ while remaining != 0:+ self._arm()+ # BufferedReader.read(size) can perform many recv calls, each+ # renewing the same socket timeout. read1() does at most one;+ # arm the shrinking absolute budget again before the next.+ chunk = self._raw.read1(min(remaining, 65536) if remaining > 0 else 65536)+ if not chunk:+ break+ chunks.append(chunk)+ if remaining > 0:+ remaining -= len(chunk)+ return b"".join(chunks)def __getattr__(self, name):return getattr(self._raw, name)@@ -6055,6 +6086,8 @@ class APIHandler(BaseHTTPRequestHandler):self.timeout, self.READ_DEADLINE)try:super().handle_one_request()+ if not self.close_connection:+ self._drain_request_body()except TimeoutError:# The read budget ran out. Say so and close; do not answer, because# we never received a complete request to answer.@@ -6083,8 +6116,6 @@ class APIHandler(BaseHTTPRequestHandler):# writes in the streaming path, which is where Ctrl-C lands.self.close_connection = Truereturn- if not self.close_connection:- self._drain_request_body()def send_response(self, code, message=None):"""Single choke point for "the status line is out". Overriding here rather than@@ -6375,7 +6406,14 @@ class APIHandler(BaseHTTPRequestHandler):try:self._check_host()self.require_auth()- body = self.read_json()+ try:+ body = self.read_json()+ except TimeoutError:+ # Only request intake is on this deadline. A later timeout+ # from the engine keeps the existing structured 500 response.+ self.close_connection = True+ self.log_error("request read deadline exceeded")+ returnpath = urlsplit(self.path).path# A client written for Jev sends "jev-latest": on that route the# served model answers whatever name was asked for.diff --git a/c/tests/test_request_read_deadline.py b/c/tests/test_request_read_deadline.pynew file mode 100644index 00000000..392d5ed3--- /dev/null+++ b/c/tests/test_request_read_deadline.py@@ -0,0 +1,120 @@+"""A continuously arriving body or header cannot renew the read deadline."""+import socket+import threading+import time+import unittest+from unittest.mock import patch++from openai_server import APIHandler, APIServer, _DeadlineReader+from test_openai_server import ScriptedEngine, _error_body, _post_completions, _spawn_test_server+++class RequestReadDeadlineTest(unittest.TestCase):+ def stream(self):+ client, server = socket.socketpair()+ raw = server.makefile("rb")+ self.addCleanup(client.close)+ self.addCleanup(server.close)+ self.addCleanup(raw.close)+ return client, _DeadlineReader(raw, server, 1, 0.25)++ def assert_drip_expires(self, method):+ client, reader = self.stream()+ stopped = threading.Event()++ def drip():+ for _ in range(30):+ if stopped.wait(0.04):+ break+ try:+ client.sendall(b"x")+ except OSError:+ break+ try:+ client.shutdown(socket.SHUT_WR)+ except OSError:+ pass++ writer = threading.Thread(target=drip)+ writer.start()+ try:+ started = time.monotonic()+ with self.assertRaises(TimeoutError):+ getattr(reader, method)(30)+ self.assertLess(time.monotonic() - started, 0.8)+ finally:+ stopped.set()+ writer.join(timeout=2)++ def test_body_read_deadline_expires_during_one_buffered_read(self):+ self.assert_drip_expires("read")++ def test_header_deadline_expires_during_one_buffered_readline(self):+ self.assert_drip_expires("readline")++ def test_readline_preserves_following_pipelined_bytes_and_limits(self):+ client, reader = self.stream()+ client.sendall(b"first\nsecond\nbody")+ client.shutdown(socket.SHUT_WR)+ self.assertEqual(reader.readline(3), b"fir")+ self.assertEqual(reader.readline(), b"st\n")+ self.assertEqual(reader.readline(), b"second\n")+ self.assertEqual(reader.read(4), b"body")+ self.assertEqual(reader.read(1), b"")++ def test_zero_length_reads_do_not_consume_input(self):+ client, reader = self.stream()+ client.sendall(b"ok\n")+ self.assertEqual(reader.read(0), b"")+ self.assertEqual(reader.readline(0), b"")+ self.assertEqual(reader.readline(), b"ok\n")++ def test_slow_http_body_closes_without_misreporting_an_engine_error(self):+ engine = ScriptedEngine()+ with patch.object(APIHandler, "READ_DEADLINE", 0.25):+ server = APIServer(("127.0.0.1", 0), engine, "test-model", None, 16)+ serving = threading.Thread(target=server.serve_forever, args=(0.01,), daemon=True)+ serving.start()+ stopped = threading.Event()+ try:+ with socket.create_connection(server.server_address, timeout=2) as client:+ client.sendall(b"POST /v1/completions HTTP/1.1\r\nHost: localhost\r\n"+ b"Content-Length: 30\r\n\r\n")++ def drip():+ while not stopped.wait(0.04):+ try:+ client.sendall(b"x")+ except OSError:+ break++ writer = threading.Thread(target=drip)+ writer.start()+ try:+ started = time.monotonic()+ self.assertEqual(client.recv(65536), b"")+ self.assertLess(time.monotonic() - started, 0.8)+ finally:+ stopped.set()+ writer.join(timeout=2)+ self.assertEqual(engine.calls, [])+ finally:+ server.scheduler.close()+ server.shutdown()+ server.server_close()+ serving.join(timeout=2)++ def test_engine_timeout_keeps_the_existing_structured_error(self):+ class TimedOutEngine(ScriptedEngine):+ def generate(self, *args, **kwargs):+ raise TimeoutError("engine operation timed out")++ base = _spawn_test_server(self, TimedOutEngine())+ status, error = _error_body(self, lambda: _post_completions(+ base, {"model": "test-model", "prompt": "hello"}))+ self.assertEqual(status, 500)+ self.assertEqual(error["code"], "engine_error")+++if __name__ == "__main__":+ unittest.main()