Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(serve): run inference off the event loop (#230)

PerryLink07e2b3bb653f
diff --git a/laya/serve.py b/laya/serve.py
index 57bb460f..15d341bb 100644
--- a/laya/serve.py
+++ b/laya/serve.py
@@ -111,12 +111,28 @@ def build_router():
def create_app(router: Optional[Any] = None):
"""Build the FastAPI app. Pass a Router to inject one (tests); otherwise one
is built from the environment (and preloaded) at app-creation time."""
+ import asyncio
+ from concurrent.futures import ThreadPoolExecutor
+
from fastapi import FastAPI, Header, HTTPException, Request
if router is None:
router = build_router()
api_key = os.environ.get("LAYA_API_KEY") or None
+ # Inference is synchronous torch, and a CPU call takes hundreds of milliseconds to
+ # seconds, so it must not run on the event loop: one request would stall every
+ # other client, `GET /health` included. One worker, because one forward pass at a
+ # time is what a single CPU or GPU Agent wants (the Router already guards checkpoint
+ # lifecycle, and leaves `Agent.system_one` unguarded deliberately so concurrent
+ # predictions can share a checkpoint -- a GPU-shaped choice this endpoint does not
+ # rely on). `loop.run_in_executor` is the API the issue asked for.
+ pool = ThreadPoolExecutor(max_workers=1, thread_name_prefix="laya-infer")
+ # Created on first request, not here: an `asyncio.Lock` binds to the loop that is
+ # running when it is first awaited, and `create_app` may be called before that loop
+ # exists (module scope, TestClient startup, a preload script).
+ gate: Optional[asyncio.Lock] = None
+
app = FastAPI(
title="laya-serve",
summary="Laya System-1 decisions over the TypeSafe Jev /v1/systemone protocol",
@@ -138,6 +154,7 @@ def create_app(router: Optional[Any] = None):
@app.post("/v1/systemone")
async def systemone(request: Request, authorization: Optional[str] = Header(default=None)):
+ nonlocal gate
_check_auth(authorization)
body = await request.json()
if not isinstance(body, dict) or "questions" not in body:
@@ -145,10 +162,15 @@ def create_app(router: Optional[Any] = None):
state = body.get("state")
questions = body["questions"]
model = _resolve_model(body.get("model"))
+ if gate is None:
+ gate = asyncio.Lock()
try:
# Laya's result is already Jev-shaped: {model, answers, usage, routing}.
# hs-jev decodes `answers` and `usage` and ignores the rest.
- return router.predict(state, questions, model=model)
+ async with gate:
+ loop = asyncio.get_running_loop()
+ return await loop.run_in_executor(
+ pool, lambda: router.predict(state, questions, model=model))
except HTTPException:
raise
except Exception as e: # noqa: BLE001 -- surface model/tokenizer errors as 422
diff --git a/tests/test_serve.py b/tests/test_serve.py
index 6d0be4c2..bba6a431 100644
--- a/tests/test_serve.py
+++ b/tests/test_serve.py
@@ -119,3 +119,91 @@ def test_thread_limit(monkeypatch):
assert _apply_thread_limit() == 8
import torch
assert torch.get_num_threads() == 8
+
+
+# The endpoint is `async def` and inference is synchronous torch, which on CPU takes
+# hundreds of milliseconds to seconds. Calling it from the coroutine puts that work on
+# the event loop, so every other client -- `GET /health` included -- waits for it.
+# Driving the app directly on a loop (`httpx.ASGITransport`) makes the difference
+# observable: offloaded work runs on a worker thread, inline work runs on the loop's own
+# `MainThread`. `TestClient` cannot see this, because it runs the loop in a portal thread
+# and hands each call its own, so a blocking endpoint still looks concurrent there.
+class SlowRouter(FakeRouter):
+ """Sleeps like a CPU forward pass and records the thread it ran on."""
+
+ def __init__(self, seconds=0.25):
+ super().__init__()
+ self.seconds = seconds
+ self.threads = []
+
+ def predict(self, state, questions, model=None):
+ import threading
+ import time
+ self.threads.append(threading.current_thread().name)
+ time.sleep(self.seconds)
+ return super().predict(state, questions, model=model)
+
+
+def test_inference_runs_off_the_event_loop(monkeypatch):
+ import asyncio
+ import threading
+
+ import httpx
+
+ monkeypatch.delenv("LAYA_API_KEY", raising=False)
+ fake = FakeRouter()
+ seen = []
+ real_predict = fake.predict
+
+ def recording_predict(state, questions, model=None):
+ seen.append(threading.current_thread().name)
+ return real_predict(state, questions, model=model)
+
+ fake.predict = recording_predict
+ app = create_app(router=fake)
+
+ async def drive():
+ transport = httpx.ASGITransport(app=app)
+ async with httpx.AsyncClient(transport=transport, base_url="http://t") as client:
+ return await client.post("/v1/systemone", json=REQ)
+
+ response = asyncio.run(drive())
+
+ assert response.status_code == 200, response.text
+ assert seen, "predict was never called"
+ assert "MainThread" not in seen, (
+ "predict ran on the event loop thread: %s -- one request would stall every "
+ "other client, including GET /health" % seen)
+
+
+def test_health_stays_available_during_inference(monkeypatch):
+ """A request in flight must not stop the app answering `GET /health`."""
+ import asyncio
+
+ import httpx
+
+ monkeypatch.delenv("LAYA_API_KEY", raising=False)
+ fake = SlowRouter(seconds=0.25)
+ app = create_app(router=fake)
+ seen = {}
+
+ async def drive():
+ transport = httpx.ASGITransport(app=app)
+ async with httpx.AsyncClient(transport=transport, base_url="http://t") as client:
+ await client.post("/v1/systemone", json=REQ) # warm up
+
+ async def slow_request():
+ seen["slow"] = (await client.post("/v1/systemone", json=REQ)).status_code
+
+ async def health():
+ r = await client.get("/health")
+ seen["health"] = r.status_code
+ seen["payload"] = r.json()
+
+ await asyncio.gather(slow_request(), health())
+
+ asyncio.run(drive())
+
+ assert seen["slow"] == 200
+ assert seen["health"] == 200 and seen["payload"]["status"] == "ok"
+ assert fake.threads and "MainThread" not in fake.threads, fake.threads