Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(api): enforce read deadlines between socket receives

Rudy Celekli8dead8192641
diff --git a/c/openai_server.py b/c/openai_server.py
index 2838036f..6cb31ed3 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -5994,13 +5994,44 @@ class _DeadlineReader:
raise TimeoutError("request read deadline exceeded")
self._sock.settimeout(min(self._per_read, left))
- def readline(self, *args):
- self._arm()
- return self._raw.readline(*args)
+ def readline(self, size=-1):
+ chunks = []
+ remaining = size
+ while remaining != 0:
+ self._arm()
+ # peek() does at most one raw socket read. Consume only bytes it
+ # already buffered, so readline cannot renew one socket timeout
+ # internally while a peer drips an unfinished header.
+ buffered = self._raw.peek(1)
+ if not buffered:
+ break
+ newline = buffered.find(b"\n")
+ take = newline + 1 if newline >= 0 else len(buffered)
+ if remaining > 0:
+ take = min(take, remaining)
+ chunk = self._raw.read1(take)
+ chunks.append(chunk)
+ if remaining > 0:
+ remaining -= len(chunk)
+ if chunk.endswith(b"\n"):
+ break
+ return b"".join(chunks)
- def read(self, *args):
- self._arm()
- return self._raw.read(*args)
+ def read(self, size=-1):
+ chunks = []
+ remaining = size
+ while remaining != 0:
+ self._arm()
+ # BufferedReader.read(size) can perform many recv calls, each
+ # renewing the same socket timeout. read1() does at most one;
+ # arm the shrinking absolute budget again before the next.
+ chunk = self._raw.read1(min(remaining, 65536) if remaining > 0 else 65536)
+ if not chunk:
+ break
+ chunks.append(chunk)
+ if remaining > 0:
+ remaining -= len(chunk)
+ return b"".join(chunks)
def __getattr__(self, name):
return getattr(self._raw, name)
@@ -6055,6 +6086,8 @@ class APIHandler(BaseHTTPRequestHandler):
self.timeout, self.READ_DEADLINE)
try:
super().handle_one_request()
+ if not self.close_connection:
+ self._drain_request_body()
except TimeoutError:
# The read budget ran out. Say so and close; do not answer, because
# we never received a complete request to answer.
@@ -6083,8 +6116,6 @@ class APIHandler(BaseHTTPRequestHandler):
# writes in the streaming path, which is where Ctrl-C lands.
self.close_connection = True
return
- if not self.close_connection:
- self._drain_request_body()
def send_response(self, code, message=None):
"""Single choke point for "the status line is out". Overriding here rather than
@@ -6375,7 +6406,14 @@ class APIHandler(BaseHTTPRequestHandler):
try:
self._check_host()
self.require_auth()
- body = self.read_json()
+ try:
+ body = self.read_json()
+ except TimeoutError:
+ # Only request intake is on this deadline. A later timeout
+ # from the engine keeps the existing structured 500 response.
+ self.close_connection = True
+ self.log_error("request read deadline exceeded")
+ return
path = urlsplit(self.path).path
# A client written for Jev sends "jev-latest": on that route the
# served model answers whatever name was asked for.
diff --git a/c/tests/test_request_read_deadline.py b/c/tests/test_request_read_deadline.py
new file mode 100644
index 00000000..392d5ed3
--- /dev/null
+++ b/c/tests/test_request_read_deadline.py
@@ -0,0 +1,120 @@
+"""A continuously arriving body or header cannot renew the read deadline."""
+import socket
+import threading
+import time
+import unittest
+from unittest.mock import patch
+
+from openai_server import APIHandler, APIServer, _DeadlineReader
+from test_openai_server import ScriptedEngine, _error_body, _post_completions, _spawn_test_server
+
+
+class RequestReadDeadlineTest(unittest.TestCase):
+ def stream(self):
+ client, server = socket.socketpair()
+ raw = server.makefile("rb")
+ self.addCleanup(client.close)
+ self.addCleanup(server.close)
+ self.addCleanup(raw.close)
+ return client, _DeadlineReader(raw, server, 1, 0.25)
+
+ def assert_drip_expires(self, method):
+ client, reader = self.stream()
+ stopped = threading.Event()
+
+ def drip():
+ for _ in range(30):
+ if stopped.wait(0.04):
+ break
+ try:
+ client.sendall(b"x")
+ except OSError:
+ break
+ try:
+ client.shutdown(socket.SHUT_WR)
+ except OSError:
+ pass
+
+ writer = threading.Thread(target=drip)
+ writer.start()
+ try:
+ started = time.monotonic()
+ with self.assertRaises(TimeoutError):
+ getattr(reader, method)(30)
+ self.assertLess(time.monotonic() - started, 0.8)
+ finally:
+ stopped.set()
+ writer.join(timeout=2)
+
+ def test_body_read_deadline_expires_during_one_buffered_read(self):
+ self.assert_drip_expires("read")
+
+ def test_header_deadline_expires_during_one_buffered_readline(self):
+ self.assert_drip_expires("readline")
+
+ def test_readline_preserves_following_pipelined_bytes_and_limits(self):
+ client, reader = self.stream()
+ client.sendall(b"first\nsecond\nbody")
+ client.shutdown(socket.SHUT_WR)
+ self.assertEqual(reader.readline(3), b"fir")
+ self.assertEqual(reader.readline(), b"st\n")
+ self.assertEqual(reader.readline(), b"second\n")
+ self.assertEqual(reader.read(4), b"body")
+ self.assertEqual(reader.read(1), b"")
+
+ def test_zero_length_reads_do_not_consume_input(self):
+ client, reader = self.stream()
+ client.sendall(b"ok\n")
+ self.assertEqual(reader.read(0), b"")
+ self.assertEqual(reader.readline(0), b"")
+ self.assertEqual(reader.readline(), b"ok\n")
+
+ def test_slow_http_body_closes_without_misreporting_an_engine_error(self):
+ engine = ScriptedEngine()
+ with patch.object(APIHandler, "READ_DEADLINE", 0.25):
+ server = APIServer(("127.0.0.1", 0), engine, "test-model", None, 16)
+ serving = threading.Thread(target=server.serve_forever, args=(0.01,), daemon=True)
+ serving.start()
+ stopped = threading.Event()
+ try:
+ with socket.create_connection(server.server_address, timeout=2) as client:
+ client.sendall(b"POST /v1/completions HTTP/1.1\r\nHost: localhost\r\n"
+ b"Content-Length: 30\r\n\r\n")
+
+ def drip():
+ while not stopped.wait(0.04):
+ try:
+ client.sendall(b"x")
+ except OSError:
+ break
+
+ writer = threading.Thread(target=drip)
+ writer.start()
+ try:
+ started = time.monotonic()
+ self.assertEqual(client.recv(65536), b"")
+ self.assertLess(time.monotonic() - started, 0.8)
+ finally:
+ stopped.set()
+ writer.join(timeout=2)
+ self.assertEqual(engine.calls, [])
+ finally:
+ server.scheduler.close()
+ server.shutdown()
+ server.server_close()
+ serving.join(timeout=2)
+
+ def test_engine_timeout_keeps_the_existing_structured_error(self):
+ class TimedOutEngine(ScriptedEngine):
+ def generate(self, *args, **kwargs):
+ raise TimeoutError("engine operation timed out")
+
+ base = _spawn_test_server(self, TimedOutEngine())
+ status, error = _error_body(self, lambda: _post_completions(
+ base, {"model": "test-model", "prompt": "hello"}))
+ self.assertEqual(status, 500)
+ self.assertEqual(error["code"], "engine_error")
+
+
+if __name__ == "__main__":
+ unittest.main()