Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
fix(serve): bound concurrent requests with a non-blocking admission check
bunnysayzzbc7a09c88465
diff --git a/laya/serve.py b/laya/serve.pyindex afecb966..8b825715 100644--- a/laya/serve.py+++ b/laya/serve.py@@ -26,6 +26,8 @@ env var meaning default``LAYA_AUTO_TASK`` auto-route to the typed-decisions checkpoint 0``LAYA_API_KEY`` if set, require ``Authorization: Bearer <it>`` (none)``LAYA_LOG_LEVEL`` uvicorn log level info+``LAYA_MAX_CONCURRENT`` cap on requests past auth at once; excess 16+ gets 503 (see below)====================== ============================================ =========Imports of heavy dependencies (fastapi, uvicorn, torch via Router) are all@@ -57,6 +59,10 @@ _KNOWN_MODELS = {"english", "multilingual", "typed-decisions"}MAX_QUESTIONS = 64MAX_STATE_CHARS = 50000MAX_BODY_BYTES = 2 * 1024 * 1024+# Cap on requests past auth at once. Each one can buffer up to MAX_BODY_BYTES+# before inference, so without a bound many concurrent near-cap requests OOM+# the worker even though every request is individually valid (#330).+DEFAULT_MAX_CONCURRENT = 16# Public Hugging Face ids, accepted so a client can name a checkpoint. The root bundle is# deliberately absent: the documented ``convaiinnovations/laya`` value means# "let the Router choose", rather than pinning the English checkpoint.@@ -92,6 +98,18 @@ def _resolve_model(model: Optional[str]) -> Optional[str]:return key if key in _KNOWN_MODELS else None+def _resolve_max_concurrent() -> int:+ """Bound on requests past auth at once, from LAYA_MAX_CONCURRENT."""+ raw = os.environ.get("LAYA_MAX_CONCURRENT")+ if not raw:+ return DEFAULT_MAX_CONCURRENT+ try:+ n = int(raw)+ except ValueError:+ return DEFAULT_MAX_CONCURRENT+ return n if n > 0 else DEFAULT_MAX_CONCURRENT++def _resolve_port() -> int:"""Port from LAYA_PORT, validated. Exits with a message instead of a traceback."""raw = os.environ.get("LAYA_PORT", "8000")@@ -205,6 +223,13 @@ def create_app(router: Optional[Any] = None):# running when it is first awaited, and `create_app` may be called before that loop# exists (module scope, TestClient startup, a preload script).gate: Optional[asyncio.Lock] = None+ # Admission bound, same late-creation reason as the gate. Checked before any+ # body byte is read and held through inference, so the bodies buffered at+ # once stay bounded no matter how many clients connect (#330). The inference+ # gate is still joined only after the body is complete, so a slow client+ # holds an admission slot but never an inference slot.+ max_concurrent = _resolve_max_concurrent()+ admission: Optional[asyncio.Semaphore] = None@asynccontextmanagerasync def lifespan(_app: FastAPI):@@ -247,8 +272,22 @@ def create_app(router: Optional[Any] = None):@app.post("/v1/systemone")async def systemone(request: Request, authorization: Optional[str] = Header(default=None)):- nonlocal gate+ nonlocal gate, admission_check_auth(authorization)+ if admission is None:+ admission = asyncio.Semaphore(max_concurrent)+ if admission.locked():+ # Non-blocking: excess load is refused rather than queued, so the+ # buffered bodies stay within the bound above.+ raise HTTPException(status_code=503, detail="server busy, try again later")+ await admission.acquire()+ try:+ return await _systemone_inner(request)+ finally:+ admission.release()++ async def _systemone_inner(request: Request):+ nonlocal gate# A declared length over the cap is rejected before anything is read; the# streaming cap below is what actually enforces it, for bodies that declare# no length or understate it.diff --git a/tests/test_serve.py b/tests/test_serve.pyindex f2b80abb..3c9df0f4 100644--- a/tests/test_serve.py+++ b/tests/test_serve.py@@ -402,3 +402,65 @@ def test_inference_timing_headers():assert "X-Inference-Time-Ms" in res.headersdur = float(res.headers["X-Inference-Time-Ms"])assert dur >= 0.0+++class GatedRouter(FakeRouter):+ """Blocks inside predict until released, so a second request arrives while+ the first still holds its admission slot (#330)."""++ def __init__(self):+ super().__init__()+ import threading+ self.entered = threading.Event()+ self.release = threading.Event()++ def predict(self, state, questions, model=None):+ self.entered.set()+ assert self.release.wait(timeout=10), "test did not release the router"+ return super().predict(state, questions, model=model)+++def test_admission_bound_refuses_with_503_when_full(monkeypatch):+ """With one admission slot and inference blocked, a second concurrent+ request gets 503 instead of queueing another body in memory."""+ import asyncio++ import httpx++ monkeypatch.delenv("LAYA_API_KEY", raising=False)+ monkeypatch.setenv("LAYA_MAX_CONCURRENT", "1")+ fake = GatedRouter()+ app = create_app(router=fake)+ seen = {}++ async def drive():+ transport = httpx.ASGITransport(app=app)+ async with httpx.AsyncClient(transport=transport, base_url="http://t") as client:+ first = asyncio.ensure_future(client.post("/v1/systemone", json=REQ))+ # Poll: a blocking wait here would stall the loop the first+ # request needs to reach inference.+ for _ in range(200):+ if fake.entered.is_set():+ break+ await asyncio.sleep(0.05)+ assert fake.entered.is_set(), "first request never reached inference"+ # Give the first request a moment to settle past the gate too, so the+ # second request deterministically finds the slot taken.+ await asyncio.sleep(0.2)+ seen["second"] = (await client.post("/v1/systemone", json=REQ)).status_code+ fake.release.set()+ seen["first"] = (await first).status_code++ asyncio.run(drive())++ assert seen["second"] == 503, seen+ assert seen["first"] == 200, seen+++def test_admission_slot_is_released_after_inference(monkeypatch):+ """Slots are reusable: sequential requests with a bound of one all pass."""+ monkeypatch.delenv("LAYA_API_KEY", raising=False)+ monkeypatch.setenv("LAYA_MAX_CONCURRENT", "1")+ client = TestClient(create_app(router=FakeRouter()))+ assert client.post("/v1/systemone", json=REQ).status_code == 200+ assert client.post("/v1/systemone", json=REQ).status_code == 200