Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(serve): bound concurrent requests with a non-blocking admission check

bunnysayzzbc7a09c88465
diff --git a/laya/serve.py b/laya/serve.py
index afecb966..8b825715 100644
--- a/laya/serve.py
+++ b/laya/serve.py
@@ -26,6 +26,8 @@ env var meaning default
``LAYA_AUTO_TASK`` auto-route to the typed-decisions checkpoint 0
``LAYA_API_KEY`` if set, require ``Authorization: Bearer <it>`` (none)
``LAYA_LOG_LEVEL`` uvicorn log level info
+``LAYA_MAX_CONCURRENT`` cap on requests past auth at once; excess 16
+ gets 503 (see below)
====================== ============================================ =========
Imports of heavy dependencies (fastapi, uvicorn, torch via Router) are all
@@ -57,6 +59,10 @@ _KNOWN_MODELS = {"english", "multilingual", "typed-decisions"}
MAX_QUESTIONS = 64
MAX_STATE_CHARS = 50000
MAX_BODY_BYTES = 2 * 1024 * 1024
+# Cap on requests past auth at once. Each one can buffer up to MAX_BODY_BYTES
+# before inference, so without a bound many concurrent near-cap requests OOM
+# the worker even though every request is individually valid (#330).
+DEFAULT_MAX_CONCURRENT = 16
# Public Hugging Face ids, accepted so a client can name a checkpoint. The root bundle is
# deliberately absent: the documented ``convaiinnovations/laya`` value means
# "let the Router choose", rather than pinning the English checkpoint.
@@ -92,6 +98,18 @@ def _resolve_model(model: Optional[str]) -> Optional[str]:
return key if key in _KNOWN_MODELS else None
+def _resolve_max_concurrent() -> int:
+ """Bound on requests past auth at once, from LAYA_MAX_CONCURRENT."""
+ raw = os.environ.get("LAYA_MAX_CONCURRENT")
+ if not raw:
+ return DEFAULT_MAX_CONCURRENT
+ try:
+ n = int(raw)
+ except ValueError:
+ return DEFAULT_MAX_CONCURRENT
+ return n if n > 0 else DEFAULT_MAX_CONCURRENT
+
+
def _resolve_port() -> int:
"""Port from LAYA_PORT, validated. Exits with a message instead of a traceback."""
raw = os.environ.get("LAYA_PORT", "8000")
@@ -205,6 +223,13 @@ def create_app(router: Optional[Any] = None):
# running when it is first awaited, and `create_app` may be called before that loop
# exists (module scope, TestClient startup, a preload script).
gate: Optional[asyncio.Lock] = None
+ # Admission bound, same late-creation reason as the gate. Checked before any
+ # body byte is read and held through inference, so the bodies buffered at
+ # once stay bounded no matter how many clients connect (#330). The inference
+ # gate is still joined only after the body is complete, so a slow client
+ # holds an admission slot but never an inference slot.
+ max_concurrent = _resolve_max_concurrent()
+ admission: Optional[asyncio.Semaphore] = None
@asynccontextmanager
async def lifespan(_app: FastAPI):
@@ -247,8 +272,22 @@ def create_app(router: Optional[Any] = None):
@app.post("/v1/systemone")
async def systemone(request: Request, authorization: Optional[str] = Header(default=None)):
- nonlocal gate
+ nonlocal gate, admission
_check_auth(authorization)
+ if admission is None:
+ admission = asyncio.Semaphore(max_concurrent)
+ if admission.locked():
+ # Non-blocking: excess load is refused rather than queued, so the
+ # buffered bodies stay within the bound above.
+ raise HTTPException(status_code=503, detail="server busy, try again later")
+ await admission.acquire()
+ try:
+ return await _systemone_inner(request)
+ finally:
+ admission.release()
+
+ async def _systemone_inner(request: Request):
+ nonlocal gate
# A declared length over the cap is rejected before anything is read; the
# streaming cap below is what actually enforces it, for bodies that declare
# no length or understate it.
diff --git a/tests/test_serve.py b/tests/test_serve.py
index f2b80abb..3c9df0f4 100644
--- a/tests/test_serve.py
+++ b/tests/test_serve.py
@@ -402,3 +402,65 @@ def test_inference_timing_headers():
assert "X-Inference-Time-Ms" in res.headers
dur = float(res.headers["X-Inference-Time-Ms"])
assert dur >= 0.0
+
+
+class GatedRouter(FakeRouter):
+ """Blocks inside predict until released, so a second request arrives while
+ the first still holds its admission slot (#330)."""
+
+ def __init__(self):
+ super().__init__()
+ import threading
+ self.entered = threading.Event()
+ self.release = threading.Event()
+
+ def predict(self, state, questions, model=None):
+ self.entered.set()
+ assert self.release.wait(timeout=10), "test did not release the router"
+ return super().predict(state, questions, model=model)
+
+
+def test_admission_bound_refuses_with_503_when_full(monkeypatch):
+ """With one admission slot and inference blocked, a second concurrent
+ request gets 503 instead of queueing another body in memory."""
+ import asyncio
+
+ import httpx
+
+ monkeypatch.delenv("LAYA_API_KEY", raising=False)
+ monkeypatch.setenv("LAYA_MAX_CONCURRENT", "1")
+ fake = GatedRouter()
+ app = create_app(router=fake)
+ seen = {}
+
+ async def drive():
+ transport = httpx.ASGITransport(app=app)
+ async with httpx.AsyncClient(transport=transport, base_url="http://t") as client:
+ first = asyncio.ensure_future(client.post("/v1/systemone", json=REQ))
+ # Poll: a blocking wait here would stall the loop the first
+ # request needs to reach inference.
+ for _ in range(200):
+ if fake.entered.is_set():
+ break
+ await asyncio.sleep(0.05)
+ assert fake.entered.is_set(), "first request never reached inference"
+ # Give the first request a moment to settle past the gate too, so the
+ # second request deterministically finds the slot taken.
+ await asyncio.sleep(0.2)
+ seen["second"] = (await client.post("/v1/systemone", json=REQ)).status_code
+ fake.release.set()
+ seen["first"] = (await first).status_code
+
+ asyncio.run(drive())
+
+ assert seen["second"] == 503, seen
+ assert seen["first"] == 200, seen
+
+
+def test_admission_slot_is_released_after_inference(monkeypatch):
+ """Slots are reusable: sequential requests with a bound of one all pass."""
+ monkeypatch.delenv("LAYA_API_KEY", raising=False)
+ monkeypatch.setenv("LAYA_MAX_CONCURRENT", "1")
+ client = TestClient(create_app(router=FakeRouter()))
+ assert client.post("/v1/systemone", json=REQ).status_code == 200
+ assert client.post("/v1/systemone", json=REQ).status_code == 200