Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(serve): run inference off the event loop (#230)
PerryLink07e2b3bb653f
diff --git a/laya/serve.py b/laya/serve.pyindex 57bb460f..15d341bb 100644--- a/laya/serve.py+++ b/laya/serve.py@@ -111,12 +111,28 @@ def build_router():def create_app(router: Optional[Any] = None):"""Build the FastAPI app. Pass a Router to inject one (tests); otherwise oneis built from the environment (and preloaded) at app-creation time."""+ import asyncio+ from concurrent.futures import ThreadPoolExecutor+from fastapi import FastAPI, Header, HTTPException, Requestif router is None:router = build_router()api_key = os.environ.get("LAYA_API_KEY") or None+ # Inference is synchronous torch, and a CPU call takes hundreds of milliseconds to+ # seconds, so it must not run on the event loop: one request would stall every+ # other client, `GET /health` included. One worker, because one forward pass at a+ # time is what a single CPU or GPU Agent wants (the Router already guards checkpoint+ # lifecycle, and leaves `Agent.system_one` unguarded deliberately so concurrent+ # predictions can share a checkpoint -- a GPU-shaped choice this endpoint does not+ # rely on). `loop.run_in_executor` is the API the issue asked for.+ pool = ThreadPoolExecutor(max_workers=1, thread_name_prefix="laya-infer")+ # Created on first request, not here: an `asyncio.Lock` binds to the loop that is+ # running when it is first awaited, and `create_app` may be called before that loop+ # exists (module scope, TestClient startup, a preload script).+ gate: Optional[asyncio.Lock] = None+app = FastAPI(title="laya-serve",summary="Laya System-1 decisions over the TypeSafe Jev /v1/systemone protocol",@@ -138,6 +154,7 @@ def create_app(router: Optional[Any] = None):@app.post("/v1/systemone")async def systemone(request: Request, authorization: Optional[str] = Header(default=None)):+ nonlocal gate_check_auth(authorization)body = await request.json()if not isinstance(body, dict) or "questions" not in body:@@ -145,10 +162,15 @@ def create_app(router: Optional[Any] = None):state = body.get("state")questions = body["questions"]model = _resolve_model(body.get("model"))+ if gate is None:+ gate = asyncio.Lock()try:# Laya's result is already Jev-shaped: {model, answers, usage, routing}.# hs-jev decodes `answers` and `usage` and ignores the rest.- return router.predict(state, questions, model=model)+ async with gate:+ loop = asyncio.get_running_loop()+ return await loop.run_in_executor(+ pool, lambda: router.predict(state, questions, model=model))except HTTPException:raiseexcept Exception as e: # noqa: BLE001 -- surface model/tokenizer errors as 422diff --git a/tests/test_serve.py b/tests/test_serve.pyindex 6d0be4c2..bba6a431 100644--- a/tests/test_serve.py+++ b/tests/test_serve.py@@ -119,3 +119,91 @@ def test_thread_limit(monkeypatch):assert _apply_thread_limit() == 8import torchassert torch.get_num_threads() == 8+++# The endpoint is `async def` and inference is synchronous torch, which on CPU takes+# hundreds of milliseconds to seconds. Calling it from the coroutine puts that work on+# the event loop, so every other client -- `GET /health` included -- waits for it.+# Driving the app directly on a loop (`httpx.ASGITransport`) makes the difference+# observable: offloaded work runs on a worker thread, inline work runs on the loop's own+# `MainThread`. `TestClient` cannot see this, because it runs the loop in a portal thread+# and hands each call its own, so a blocking endpoint still looks concurrent there.+class SlowRouter(FakeRouter):+ """Sleeps like a CPU forward pass and records the thread it ran on."""++ def __init__(self, seconds=0.25):+ super().__init__()+ self.seconds = seconds+ self.threads = []++ def predict(self, state, questions, model=None):+ import threading+ import time+ self.threads.append(threading.current_thread().name)+ time.sleep(self.seconds)+ return super().predict(state, questions, model=model)+++def test_inference_runs_off_the_event_loop(monkeypatch):+ import asyncio+ import threading++ import httpx++ monkeypatch.delenv("LAYA_API_KEY", raising=False)+ fake = FakeRouter()+ seen = []+ real_predict = fake.predict++ def recording_predict(state, questions, model=None):+ seen.append(threading.current_thread().name)+ return real_predict(state, questions, model=model)++ fake.predict = recording_predict+ app = create_app(router=fake)++ async def drive():+ transport = httpx.ASGITransport(app=app)+ async with httpx.AsyncClient(transport=transport, base_url="http://t") as client:+ return await client.post("/v1/systemone", json=REQ)++ response = asyncio.run(drive())++ assert response.status_code == 200, response.text+ assert seen, "predict was never called"+ assert "MainThread" not in seen, (+ "predict ran on the event loop thread: %s -- one request would stall every "+ "other client, including GET /health" % seen)+++def test_health_stays_available_during_inference(monkeypatch):+ """A request in flight must not stop the app answering `GET /health`."""+ import asyncio++ import httpx++ monkeypatch.delenv("LAYA_API_KEY", raising=False)+ fake = SlowRouter(seconds=0.25)+ app = create_app(router=fake)+ seen = {}++ async def drive():+ transport = httpx.ASGITransport(app=app)+ async with httpx.AsyncClient(transport=transport, base_url="http://t") as client:+ await client.post("/v1/systemone", json=REQ) # warm up++ async def slow_request():+ seen["slow"] = (await client.post("/v1/systemone", json=REQ)).status_code++ async def health():+ r = await client.get("/health")+ seen["health"] = r.status_code+ seen["payload"] = r.json()++ await asyncio.gather(slow_request(), health())++ asyncio.run(drive())++ assert seen["slow"] == 200+ assert seen["health"] == 200 and seen["payload"]["status"] == "ok"+ assert fake.threads and "MainThread" not in fake.threads, fake.threads