Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C
4.9 KB
Raw Blame
"""A continuously arriving body or header cannot renew the read deadline."""
import socket
import threading
import time
import unittest
from unittest.mock import patch

from openai_server import APIHandler, APIServer, _DeadlineReader
from test_openai_server import ScriptedEngine, _error_body, _post_completions, _spawn_test_server


class RequestReadDeadlineTest(unittest.TestCase):
    def stream(self):
        client, server = socket.socketpair()
        raw = server.makefile("rb")
        self.addCleanup(client.close)
        self.addCleanup(server.close)
        self.addCleanup(raw.close)
        return client, _DeadlineReader(raw, server, 1, 0.25)

    def assert_drip_expires(self, method):
        client, reader = self.stream()
        stopped = threading.Event()

        def drip():
            for _ in range(30):
                if stopped.wait(0.04):
                    break
                try:
                    client.sendall(b"x")
                except OSError:
                    break
            try:
                client.shutdown(socket.SHUT_WR)
            except OSError:
                pass

        writer = threading.Thread(target=drip)
        writer.start()
        try:
            started = time.monotonic()
            with self.assertRaises(TimeoutError):
                getattr(reader, method)(30)
            self.assertLess(time.monotonic() - started, 0.8)
        finally:
            stopped.set()
            writer.join(timeout=2)

    def test_body_read_deadline_expires_during_one_buffered_read(self):
        self.assert_drip_expires("read")

    def test_header_deadline_expires_during_one_buffered_readline(self):
        self.assert_drip_expires("readline")

    def test_readline_preserves_following_pipelined_bytes_and_limits(self):
        client, reader = self.stream()
        client.sendall(b"first\nsecond\nbody")
        client.shutdown(socket.SHUT_WR)
        self.assertEqual(reader.readline(3), b"fir")
        self.assertEqual(reader.readline(), b"st\n")
        self.assertEqual(reader.readline(), b"second\n")
        self.assertEqual(reader.read(4), b"body")
        self.assertEqual(reader.read(1), b"")

    def test_zero_length_reads_do_not_consume_input(self):
        client, reader = self.stream()
        client.sendall(b"ok\n")
        self.assertEqual(reader.read(0), b"")
        self.assertEqual(reader.readline(0), b"")
        self.assertEqual(reader.readline(), b"ok\n")

    def test_slow_http_body_closes_without_misreporting_an_engine_error(self):
        engine = ScriptedEngine()
        with patch.object(APIHandler, "READ_DEADLINE", 0.25):
            server = APIServer(("127.0.0.1", 0), engine, "test-model", None, 16)
            serving = threading.Thread(target=server.serve_forever, args=(0.01,), daemon=True)
            serving.start()
            stopped = threading.Event()
            try:
                with socket.create_connection(server.server_address, timeout=2) as client:
                    client.sendall(b"POST /v1/completions HTTP/1.1\r\nHost: localhost\r\n"
                                   b"Content-Length: 30\r\n\r\n")

                    def drip():
                        while not stopped.wait(0.04):
                            try:
                                client.sendall(b"x")
                            except OSError:
                                break

                    writer = threading.Thread(target=drip)
                    writer.start()
                    try:
                        started = time.monotonic()
                        try:
                            data = client.recv(65536)
                        except ConnectionResetError:
                            # the server closed with a dripped byte still unread: the
                            # kernel resets instead of a clean close (seen on macOS);
                            # either way nothing was answered
                            data = b""
                        self.assertEqual(data, b"")
                        self.assertLess(time.monotonic() - started, 0.8)
                    finally:
                        stopped.set()
                        writer.join(timeout=2)
                self.assertEqual(engine.calls, [])
            finally:
                server.scheduler.close()
                server.shutdown()
                server.server_close()
                serving.join(timeout=2)

    def test_engine_timeout_keeps_the_existing_structured_error(self):
        class TimedOutEngine(ScriptedEngine):
            def generate(self, *args, **kwargs):
                raise TimeoutError("engine operation timed out")

        base = _spawn_test_server(self, TimedOutEngine())
        status, error = _error_body(self, lambda: _post_completions(
            base, {"model": "test-model", "prompt": "hello"}))
        self.assertEqual(status, 500)
        self.assertEqual(error["code"], "engine_error")


if __name__ == "__main__":
    unittest.main()