Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(serve): set TCP_NODELAY on every accepted connection
Jayanth4f466f4bb7de
diff --git a/laya/serve.py b/laya/serve.pyindex 2cd30922..334b5d27 100644--- a/laya/serve.py+++ b/laya/serve.py@@ -391,13 +391,24 @@ def create_app(router: Optional[Any] = None):def main() -> None:+ import socket+import uvicorn+ from uvicorn.protocols.http.auto import AutoHTTPProtocol++ class NoDelayHTTPProtocol(AutoHTTPProtocol):+ # asyncio only sets TCP_NODELAY when the accepted socket reports IPPROTO_TCP, and+ # on macOS and Windows it reports 0, so Nagle delays small responses (#620).+ def connection_made(self, transport):+ transport.get_extra_info("socket").setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)+ super().connection_made(transport)uvicorn.run(create_app(),host=os.environ.get("LAYA_HOST", "0.0.0.0"),port=_resolve_port(),log_level=os.environ.get("LAYA_LOG_LEVEL", "info"),+ http=NoDelayHTTPProtocol,)diff --git a/tests/test_serve.py b/tests/test_serve.pyindex ae52aae2..2604bcf1 100644--- a/tests/test_serve.py+++ b/tests/test_serve.py@@ -529,3 +529,39 @@ def test_admission_slot_is_released_after_inference(monkeypatch):client = TestClient(create_app(router=FakeRouter()))assert client.post("/v1/systemone", json=REQ).status_code == 200assert client.post("/v1/systemone", json=REQ).status_code == 200+++def test_accepted_connections_set_tcp_nodelay(monkeypatch):+ """#620: asyncio skips TCP_NODELAY when an accepted socket reports proto 0, as it+ does on macOS and Windows, so Nagle held back small responses by about 50 ms."""+ import asyncio+ import socket++ import uvicorn++ import laya.serve++ captured = {}+ monkeypatch.setattr(uvicorn, "run", lambda app, **kwargs: captured.update(kwargs))+ monkeypatch.setattr(laya.serve, "create_app", lambda: None)+ laya.serve.main()++ async def drive():+ config = uvicorn.Config(create_app(router=FakeRouter()), host="127.0.0.1", port=0,+ http=captured["http"], log_level="warning")+ server = uvicorn.Server(config)+ serving = asyncio.ensure_future(server.serve())+ while not server.started:+ await asyncio.sleep(0.01)+ port = server.servers[0].sockets[0].getsockname()[1]+ _, writer = await asyncio.open_connection("127.0.0.1", port)+ while not server.server_state.connections:+ await asyncio.sleep(0.01)+ (conn,) = server.server_state.connections+ nodelay = conn.transport.get_extra_info("socket").getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY)+ writer.close()+ server.should_exit = True+ await serving+ return nodelay++ assert asyncio.run(drive())