Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(serve): send Retry-After with the 503 admission refusal

bunnysayzza4494a2f31cf
diff --git a/laya/serve.py b/laya/serve.py
index 2cd30922..cc0abe37 100644
--- a/laya/serve.py
+++ b/laya/serve.py
@@ -321,8 +321,11 @@ def create_app(router: Optional[Any] = None):
admission = asyncio.Semaphore(max_concurrent)
if admission.locked():
# Non-blocking: excess load is refused rather than queued, so the
- # buffered bodies stay within the bound above.
- raise HTTPException(status_code=503, detail="server busy, try again later")
+ # buffered bodies stay within the bound above. Retry-After tells
+ # well-behaved clients when a slot is likely free: admission turns
+ # over at inference speed, so one second is the honest hint.
+ raise HTTPException(status_code=503, detail="server busy, try again later",
+ headers={"Retry-After": "1"})
await admission.acquire()
try:
return await _systemone_inner(request)
diff --git a/tests/test_serve.py b/tests/test_serve.py
index ae52aae2..8eb0008f 100644
--- a/tests/test_serve.py
+++ b/tests/test_serve.py
@@ -512,13 +512,16 @@ def test_admission_bound_refuses_with_503_when_full(monkeypatch):
# Give the first request a moment to settle past the gate too, so the
# second request deterministically finds the slot taken.
await asyncio.sleep(0.2)
- seen["second"] = (await client.post("/v1/systemone", json=REQ)).status_code
+ second = await client.post("/v1/systemone", json=REQ)
+ seen["second"] = second.status_code
+ seen["retry_after"] = second.headers.get("retry-after")
fake.release.set()
seen["first"] = (await first).status_code
asyncio.run(drive())
assert seen["second"] == 503, seen
+ assert seen["retry_after"] == "1", seen
assert seen["first"] == 200, seen