Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(serve): set TCP_NODELAY on every accepted connection

Jayanth4f466f4bb7de
diff --git a/laya/serve.py b/laya/serve.py
index 2cd30922..334b5d27 100644
--- a/laya/serve.py
+++ b/laya/serve.py
@@ -391,13 +391,24 @@ def create_app(router: Optional[Any] = None):
def main() -> None:
+ import socket
+
import uvicorn
+ from uvicorn.protocols.http.auto import AutoHTTPProtocol
+
+ class NoDelayHTTPProtocol(AutoHTTPProtocol):
+ # asyncio only sets TCP_NODELAY when the accepted socket reports IPPROTO_TCP, and
+ # on macOS and Windows it reports 0, so Nagle delays small responses (#620).
+ def connection_made(self, transport):
+ transport.get_extra_info("socket").setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
+ super().connection_made(transport)
uvicorn.run(
create_app(),
host=os.environ.get("LAYA_HOST", "0.0.0.0"),
port=_resolve_port(),
log_level=os.environ.get("LAYA_LOG_LEVEL", "info"),
+ http=NoDelayHTTPProtocol,
)
diff --git a/tests/test_serve.py b/tests/test_serve.py
index ae52aae2..2604bcf1 100644
--- a/tests/test_serve.py
+++ b/tests/test_serve.py
@@ -529,3 +529,39 @@ def test_admission_slot_is_released_after_inference(monkeypatch):
client = TestClient(create_app(router=FakeRouter()))
assert client.post("/v1/systemone", json=REQ).status_code == 200
assert client.post("/v1/systemone", json=REQ).status_code == 200
+
+
+def test_accepted_connections_set_tcp_nodelay(monkeypatch):
+ """#620: asyncio skips TCP_NODELAY when an accepted socket reports proto 0, as it
+ does on macOS and Windows, so Nagle held back small responses by about 50 ms."""
+ import asyncio
+ import socket
+
+ import uvicorn
+
+ import laya.serve
+
+ captured = {}
+ monkeypatch.setattr(uvicorn, "run", lambda app, **kwargs: captured.update(kwargs))
+ monkeypatch.setattr(laya.serve, "create_app", lambda: None)
+ laya.serve.main()
+
+ async def drive():
+ config = uvicorn.Config(create_app(router=FakeRouter()), host="127.0.0.1", port=0,
+ http=captured["http"], log_level="warning")
+ server = uvicorn.Server(config)
+ serving = asyncio.ensure_future(server.serve())
+ while not server.started:
+ await asyncio.sleep(0.01)
+ port = server.servers[0].sockets[0].getsockname()[1]
+ _, writer = await asyncio.open_connection("127.0.0.1", port)
+ while not server.server_state.connections:
+ await asyncio.sleep(0.01)
+ (conn,) = server.server_state.connections
+ nodelay = conn.transport.get_extra_info("socket").getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY)
+ writer.close()
+ server.should_exit = True
+ await serving
+ return nodelay
+
+ assert asyncio.run(drive())