Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

fix(serve): bound the request-body read so a stalled upload frees its slot

thread3d7bdd1e7260ac
diff --git a/docs/http-api.md b/docs/http-api.md
index c9d805b8..a645bf3f 100644
--- a/docs/http-api.md
+++ b/docs/http-api.md
@@ -45,6 +45,7 @@ Everything is environment variables, so one image serves a laptop dev run and a
| `LAYA_API_KEY` | if set, require `Authorization: Bearer <key>` | none |
| `LAYA_LOG_LEVEL` | uvicorn log level | `info` |
| `LAYA_MAX_CONCURRENT` | requests admitted past auth at once; excess gets `503` | `16` |
+| `LAYA_BODY_READ_TIMEOUT` | seconds a request may spend delivering its body before the server answers `408` and releases the admission slot it was holding. Unparseable, non-positive or infinite input falls back | `10` |
| `LAYA_MAX_BATCH_TOKENS` | tokens one `/v1/systemone/batch` FORWARD PASS may collate (`states` x questions x row width); a larger batch is split into several passes, not refused | `131072` |
| `LAYA_JEV_STRICT` | serve the strict Jev wire contract: no root `routing`, no per-answer `action` / `answer_confidence`, no `confidence` on noul answers, and `usage` reduced to `input_tokens` + `output_tokens`. For clients that validate the response against the Jev contract with no extra fields | `0` |
| `LAYA_TOURNAMENT_GROUP` | answer a `choice` with more options than this by elimination, in groups of this size, on `/v1/systemone` and `/v1/systemone/batch`, and raise their option cap from 100 to 255. See [Large choices](#large-choices-laya_tournament_group). Unset, or outside 2 to 100, is off | `0` |
@@ -451,7 +452,11 @@ never runs on the event loop: requests are handed to a single-worker executor, w
forward pass at a time -- the shape a single checkpoint on one device wants. Admission (the
`LAYA_MAX_CONCURRENT` semaphore) is checked before any body byte is read and held through
inference; the inference gate is joined only after the body is complete, so a slow client holds an
-admission slot but never an inference slot.
+admission slot but never an inference slot. That ordering is what bounds how many bodies are
+buffered at once, so the body read carries its own deadline (`LAYA_BODY_READ_TIMEOUT`, 10 s): a peer
+that sends a request head and then stops writing answers `408` and gives the slot back, rather than
+holding one of the `LAYA_MAX_CONCURRENT` slots -- and so both decision routes -- out of service for
+as long as it keeps the socket open.
## Not (yet) here
diff --git a/laya/serve.py b/laya/serve.py
index 788b1f4b..5ea5b96a 100644
--- a/laya/serve.py
+++ b/laya/serve.py
@@ -48,6 +48,9 @@ env var meaning defaul
explicit ``model=`` like a built-in.
``LAYA_MAX_TOKEN_BUDGET`` cap on per-request max_len / head_max_len 8192
``LAYA_MAX_BATCH_TOKENS`` cap on batch forward tokens (states x q x len) 131072
+``LAYA_BODY_READ_TIMEOUT`` seconds a request may spend delivering its body 10
+ before the server answers 408 and releases its
+ admission slot (see below)
``LAYA_JEV_STRICT`` if set, serve the strict Jev wire contract: no 0
root `routing`, no per-answer `action` /
`answer_confidence`, no `confidence` on noul
@@ -91,6 +94,17 @@ MAX_QUESTIONS = 64
MAX_STATE_CHARS = 50000
MAX_BATCH_STATES = 64
MAX_BODY_BYTES = 2 * 1024 * 1024
+# How long a request may take to deliver its body. Admission is taken BEFORE the body is read --
+# that ordering is deliberate, it is what bounds how many bodies are buffered at once -- but it also
+# means a client that sends a request head and one partial body chunk and then goes quiet holds one
+# of the `LAYA_MAX_CONCURRENT` slots for as long as the socket stays open. There is no deadline at
+# any other layer: uvicorn has no request-body timeout, and `async for chunk in request.stream()`
+# waits as long as the peer does. So without the bound below, `LAYA_MAX_CONCURRENT` stalled uploads
+# (16 by default, a few bytes each) keep BOTH decision routes answering 503 for as long as the
+# attacker keeps the sockets open, while `/health` -- which takes no admission slot -- keeps telling
+# the orchestrator the process is healthy, so nothing ever restarts it. Bounding the read turns an
+# indefinitely held slot into a slot held for at most this long, and releases it either way.
+DEFAULT_BODY_READ_TIMEOUT = 10.0
# `/v1/systemone/batch` collates `states x questions` rows into ONE tensor, and each of the caps
# above bounds only one factor: 64 states of 64 questions is 4096 rows from a 4.1 KB body, with every
# documented limit satisfied, and `batch_size` defaults to `None`, which `predict_batch` documents as
@@ -426,6 +440,18 @@ def _resolve_max_concurrent() -> int:
return n if n > 0 else DEFAULT_MAX_CONCURRENT
+def _resolve_body_read_timeout() -> float:
+ """Seconds a request may spend delivering its body, from LAYA_BODY_READ_TIMEOUT."""
+ raw = os.environ.get("LAYA_BODY_READ_TIMEOUT")
+ if not raw or not raw.strip():
+ return DEFAULT_BODY_READ_TIMEOUT
+ try:
+ value = float(raw)
+ except ValueError:
+ return DEFAULT_BODY_READ_TIMEOUT
+ return value if 0.0 < value < float("inf") else DEFAULT_BODY_READ_TIMEOUT
+
+
def _resolve_tournament_group() -> int:
"""Group size for choices answered by elimination, from LAYA_TOURNAMENT_GROUP; 0 is off (#302).
@@ -1041,7 +1067,7 @@ def _has_lone_surrogate(value: Any) -> bool:
return False
-async def _read_body_capped(request: Any) -> bytes:
+async def _read_body_capped(request: Any, timeout: Optional[float] = None) -> bytes:
"""Read the request body, refusing to buffer more than ``MAX_BODY_BYTES``.
``Content-Length`` cannot be the only gate. It is a value the client chooses,
@@ -1050,24 +1076,48 @@ async def _read_body_capped(request: Any) -> bytes:
read into memory in full, whatever its size. The body is streamed here and
abandoned as soon as it exceeds the cap, so the limit holds for every framing
rather than only for clients that announce their length honestly.
+
+ The read is also bounded in TIME (``LAYA_BODY_READ_TIMEOUT``, 10 s), because the
+ admission slot is taken before this runs: a peer that sends a request head and
+ then stops writing would otherwise hold one of the ``LAYA_MAX_CONCURRENT`` slots
+ for as long as it keeps the socket open. Nothing below the application layer
+ bounds that -- uvicorn has no request-body timeout -- so without this a handful
+ of idle sockets take the whole decision API out of service while ``/health``,
+ which takes no slot, still reports the process as healthy. A body that does not
+ arrive in time answers ``408`` and the slot is released by the unwinding
+ ``async with _admit()``, exactly as a ``413`` already was.
"""
+ import asyncio
+
from fastapi import HTTPException
- total = 0
- chunks = []
- async for chunk in request.stream():
- if not chunk:
- continue
- total += len(chunk)
- if total > MAX_BODY_BYTES:
- # Stop reading rather than draining the rest: the peer is already over
- # the limit and nothing further can make the request acceptable.
- raise HTTPException(status_code=413, detail="request body too large")
- chunks.append(chunk)
- return b"".join(chunks)
+ async def collect() -> bytes:
+ total = 0
+ chunks = []
+ async for chunk in request.stream():
+ if not chunk:
+ continue
+ total += len(chunk)
+ if total > MAX_BODY_BYTES:
+ # Stop reading rather than draining the rest: the peer is already over
+ # the limit and nothing further can make the request acceptable.
+ raise HTTPException(status_code=413, detail="request body too large")
+ chunks.append(chunk)
+ return b"".join(chunks)
+
+ try:
+ return await asyncio.wait_for(
+ collect(), timeout=_resolve_body_read_timeout() if timeout is None else timeout)
+ except asyncio.TimeoutError:
+ # Cancelling `collect()` cancels the stream read, so the peer's socket is not
+ # left half-consumed; the response below is what it sees.
+ raise HTTPException(
+ status_code=408,
+ detail="request body was not received in time",
+ ) from None
-async def _read_json_body(request: Any) -> Any:
+async def _read_json_body(request: Any, timeout: Optional[float] = None) -> Any:
"""Read the request body and decode JSON, enforcing size caps and parse error guards."""
from fastapi import HTTPException
@@ -1077,7 +1127,7 @@ async def _read_json_body(request: Any) -> Any:
raise HTTPException(status_code=413, detail="request body too large")
except ValueError:
pass
- raw = await _read_body_capped(request)
+ raw = await _read_body_capped(request, timeout)
try:
return json.loads(raw)
except (ValueError, RecursionError):
@@ -1171,6 +1221,7 @@ def create_app(router: Optional[Any] = None):
# gate is still joined only after the body is complete, so a slow client
# holds an admission slot but never an inference slot.
max_concurrent = _resolve_max_concurrent()
+ body_read_timeout = _resolve_body_read_timeout()
admission: Optional[asyncio.Semaphore] = None
idle_unload_seconds = _resolve_idle_unload_seconds()
@@ -1337,7 +1388,7 @@ def create_app(router: Optional[Any] = None):
async def _systemone_inner(request: Request):
nonlocal gate
_mark_request()
- body = await _read_json_body(request)
+ body = await _read_json_body(request, body_read_timeout)
if not isinstance(body, dict) or "questions" not in body:
raise HTTPException(status_code=400, detail="request body must be an object with a 'questions' field")
state = body.get("state")
@@ -1434,7 +1485,7 @@ def create_app(router: Optional[Any] = None):
async def _systemone_batch_inner(request: Request):
nonlocal gate
_mark_request()
- body = await _read_json_body(request)
+ body = await _read_json_body(request, body_read_timeout)
if not isinstance(body, dict) or "questions" not in body or "states" not in body:
raise HTTPException(
status_code=400,
diff --git a/tests/test_serve.py b/tests/test_serve.py
index 6ba279bf..4c277ec2 100644
--- a/tests/test_serve.py
+++ b/tests/test_serve.py
@@ -27,6 +27,7 @@ from laya.serve import ( # noqa: E402
BATCH_BODY_ITEM_CONTROLS,
BODY_CONTROLS,
BODY_REFUSALS,
+ DEFAULT_BODY_READ_TIMEOUT,
DEFAULT_LOG_LEVEL,
DEFAULT_MAX_TOKEN_BUDGET,
DEFAULT_PORT,
@@ -37,6 +38,7 @@ from laya.serve import ( # noqa: E402
_check_request_limits,
_env_bool,
_LONE_SURROGATE_DETAIL,
+ _resolve_body_read_timeout,
_resolve_log_level,
_resolve_max_token_budget,
_resolve_idle_unload_seconds,
@@ -4218,3 +4220,94 @@ def test_extra_models_pin_over_http_and_do_not_leak_into_routing(monkeypatch):
assert plain.status_code == 200, plain.text
assert plain.json()["routing"]["model"] in ("english", "multilingual", "typed-decisions")
assert "my-ckpt" not in built[1:] or built[1:] == [], built
+
+
+# --------------------------------------------------------------------------- body-read deadline
+#
+# `_admit()` takes a `LAYA_MAX_CONCURRENT` slot BEFORE the body is read, which is what bounds how
+# many bodies are buffered at once. A peer that sends a request head and one partial chunk and then
+# stops writing used to hold that slot for as long as it kept the socket open, so a few idle sockets
+# took both decision routes out of service while `/health` kept reporting the process healthy.
+# `_read_body_capped` now bounds the read, answers 408, and lets the `async with _admit()` unwind.
+
+_ASGI_BODY = json.dumps({
+ "state": "hi",
+ "questions": {"q": {"type": "noul", "instructions": "is it true?"}},
+}).encode()
+
+
+def _asgi_scope(path):
+ return {"type": "http", "asgi": {"version": "3.0", "spec_version": "2.1"},
+ "http_version": "1.1", "method": "POST", "scheme": "http",
+ "path": path, "raw_path": path.encode(), "query_string": b"",
+ "root_path": "", "headers": [(b"content-type", b"application/json")],
+ "client": ("127.0.0.1", 5000), "server": ("127.0.0.1", 8000)}
+
+
+async def _asgi_post(app, path, body, *, stall_after=None, timeout=10.0):
+ """One ASGI POST, returning (status, body). `stall_after` sends that many bytes then hangs."""
+ messages = []
+ state = {"sent": False}
+
+ async def receive():
+ if not state["sent"]:
+ state["sent"] = True
+ if stall_after is None:
+ return {"type": "http.request", "body": body, "more_body": False}
+ return {"type": "http.request", "body": body[:stall_after], "more_body": True}
+ if stall_after is not None:
+ await asyncio.Event().wait() # the peer never writes another byte
+ return {"type": "http.request", "body": b"", "more_body": False}
+
+ async def send(message):
+ messages.append(message)
+
+ await asyncio.wait_for(app(_asgi_scope(path), receive, send), timeout=timeout)
+ status = next(m["status"] for m in messages if m["type"] == "http.response.start")
+ payload = b"".join(m.get("body", b"") for m in messages if m["type"] == "http.response.body")
+ return status, payload
+
+
+def test_a_stalled_body_answers_408_and_frees_its_admission_slot(monkeypatch):
+ # One slot, so the second request proves the first one gave its slot back rather than merely
+ # that a second request is allowed to take one.
+ monkeypatch.setenv("LAYA_MAX_CONCURRENT", "1")
+ monkeypatch.setenv("LAYA_BODY_READ_TIMEOUT", "0.05")
+ app = create_app(router=FakeRouter())
+
+ async def run():
+ stalled = await _asgi_post(app, "/v1/systemone", _ASGI_BODY, stall_after=10)
+ served = await _asgi_post(app, "/v1/systemone", _ASGI_BODY)
+ return stalled, served
+
+ (stall_status, stall_body), (served_status, _) = asyncio.run(run())
+ assert stall_status == 408, stall_body
+ assert b"not received in time" in stall_body
+ assert served_status == 200, served_status
+
+
[email protected]("raw, want", [
+ (None, DEFAULT_BODY_READ_TIMEOUT),
+ ("2.5", 2.5),
+ ("", DEFAULT_BODY_READ_TIMEOUT),
+ (" ", DEFAULT_BODY_READ_TIMEOUT),
+ ("0", DEFAULT_BODY_READ_TIMEOUT),
+ ("-1", DEFAULT_BODY_READ_TIMEOUT),
+ ("nope", DEFAULT_BODY_READ_TIMEOUT),
+ ("inf", DEFAULT_BODY_READ_TIMEOUT),
+ ("nan", DEFAULT_BODY_READ_TIMEOUT),
+])
+def test_body_read_timeout_resolution(monkeypatch, raw, want):
+ monkeypatch.delenv("LAYA_BODY_READ_TIMEOUT", raising=False)
+ if raw is not None:
+ monkeypatch.setenv("LAYA_BODY_READ_TIMEOUT", raw)
+ assert _resolve_body_read_timeout() == want
+
+
+def test_a_body_delivered_in_time_still_answers(monkeypatch):
+ # The deadline adds no behaviour to the ordinary path: same status, same payload.
+ monkeypatch.setenv("LAYA_BODY_READ_TIMEOUT", "5")
+ app = create_app(router=FakeRouter())
+ status, payload = asyncio.run(_asgi_post(app, "/v1/systemone", _ASGI_BODY))
+ assert status == 200
+ assert json.loads(payload)["usage"]["input_tokens"] == 42