Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Merge main into this branch and resolve conflicts

Nandakishorf664d3f1f9cd
diff --cc laya/serve.py
index 8b825715,0db4cdb0..2cd30922
--- a/laya/serve.py
+++ b/laya/serve.py
@@@ -59,10 -57,10 +59,15 @@@ _KNOWN_MODELS = {"english", "multilingu
MAX_QUESTIONS = 64
MAX_STATE_CHARS = 50000
MAX_BODY_BYTES = 2 * 1024 * 1024
+ # HTTP-only amplification guard; the library keeps its head_max_len-aware budget.
+ MAX_CHOICE_OPTIONS = 100
+ MAX_SCORE_LEVELS = 32
+ MAX_TOTAL_OPTIONS = 512
++
+# Cap on requests past auth at once. Each one can buffer up to MAX_BODY_BYTES
+# before inference, so without a bound many concurrent near-cap requests OOM
+# the worker even though every request is individually valid (#330).
+DEFAULT_MAX_CONCURRENT = 16
# Public Hugging Face ids, accepted so a client can name a checkpoint. The root bundle is
# deliberately absent: the documented ``convaiinnovations/laya`` value means
# "let the Router choose", rather than pinning the English checkpoint.
diff --cc tests/test_serve.py
index 3c9df0f4,d83594af..ae52aae2
--- a/tests/test_serve.py
+++ b/tests/test_serve.py
@@@ -404,63 -444,26 +444,88 @@@ def test_inference_timing_headers()
assert dur >= 0.0
+ def test_a_missing_state_is_rejected_rather_than_answered():
+ """No `state` key, or `"state": null`, must be a 400 and not a decision about "null".
+
+ `serialize_state(None)` is `json.dumps(None)` -- the four characters `null` -- so the request
+ was answered as a decision about that literal text: HTTP 200, byte-identical to sending
+ `"state": "null"`, and at ~0.94 confidence on the real checkpoint. The caller gets an answer
+ about a state they never supplied, with nothing in the response to say so.
+ """
+ router = FakeRouter()
+ client = TestClient(create_app(router=router))
+ questions = {"dept": {"type": "choice", "instructions": "which?",
+ "criteria": {"billing": "invoices"}}}
+
+ for body in ({"questions": questions}, # no state key
+ {"state": None, "questions": questions}): # explicit null
+ res = client.post("/v1/systemone", json=body)
+ assert res.status_code == 400, (body, res.status_code, res.text)
+ assert "'state' is required" in res.text, res.text
+
+ # a state that IS a string is the caller's business, including the text "null" and ""
+ for state in ("null", "", "0"):
+ res = client.post("/v1/systemone", json={"state": state, "questions": questions})
+ assert res.status_code == 200, (state, res.status_code, res.text)
++
++
+class GatedRouter(FakeRouter):
+ """Blocks inside predict until released, so a second request arrives while
+ the first still holds its admission slot (#330)."""
+
+ def __init__(self):
+ super().__init__()
+ import threading
+ self.entered = threading.Event()
+ self.release = threading.Event()
+
+ def predict(self, state, questions, model=None):
+ self.entered.set()
+ assert self.release.wait(timeout=10), "test did not release the router"
+ return super().predict(state, questions, model=model)
+
+
+def test_admission_bound_refuses_with_503_when_full(monkeypatch):
+ """With one admission slot and inference blocked, a second concurrent
+ request gets 503 instead of queueing another body in memory."""
+ import asyncio
+
+ import httpx
+
+ monkeypatch.delenv("LAYA_API_KEY", raising=False)
+ monkeypatch.setenv("LAYA_MAX_CONCURRENT", "1")
+ fake = GatedRouter()
+ app = create_app(router=fake)
+ seen = {}
+
+ async def drive():
+ transport = httpx.ASGITransport(app=app)
+ async with httpx.AsyncClient(transport=transport, base_url="http://t") as client:
+ first = asyncio.ensure_future(client.post("/v1/systemone", json=REQ))
+ # Poll: a blocking wait here would stall the loop the first
+ # request needs to reach inference.
+ for _ in range(200):
+ if fake.entered.is_set():
+ break
+ await asyncio.sleep(0.05)
+ assert fake.entered.is_set(), "first request never reached inference"
+ # Give the first request a moment to settle past the gate too, so the
+ # second request deterministically finds the slot taken.
+ await asyncio.sleep(0.2)
+ seen["second"] = (await client.post("/v1/systemone", json=REQ)).status_code
+ fake.release.set()
+ seen["first"] = (await first).status_code
+
+ asyncio.run(drive())
+
+ assert seen["second"] == 503, seen
+ assert seen["first"] == 200, seen
+
+
+def test_admission_slot_is_released_after_inference(monkeypatch):
+ """Slots are reusable: sequential requests with a bound of one all pass."""
+ monkeypatch.delenv("LAYA_API_KEY", raising=False)
+ monkeypatch.setenv("LAYA_MAX_CONCURRENT", "1")
+ client = TestClient(create_app(router=FakeRouter()))
+ assert client.post("/v1/systemone", json=REQ).status_code == 200
+ assert client.post("/v1/systemone", json=REQ).status_code == 200