Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.
Python calibrationclassificationdecision-modelhuggingfacejevmodernbertmultilingualnlppythonpytorchroutingtyped-decisionszero-shot
Merge pull request #454 from PerryLink/fix/serve-surrogate
Nandakishor33e8dd534ea7
diff --cc laya/serve.pyindex 58a10516,7637dcb3..7e7847fa--- a/laya/serve.py+++ b/laya/serve.py@@@ -255,19 -183,36 +256,49 @@@ def _check_request_limits(state: Any, qdetail="state too large (%d > %d chars)" % (state_len, MAX_STATE_CHARS))+def _check_batch_limits(states: Any, questions: Any) -> None:+ """Validate batch states and questions before inference (400/413)."""+ from fastapi import HTTPException++ if not isinstance(states, list) or len(states) == 0:+ raise HTTPException(status_code=400, detail="'states' must be a non-empty list")+ if len(states) > MAX_BATCH_STATES:+ raise HTTPException(+ status_code=413,+ detail="too many states in batch (%d > %d)" % (len(states), MAX_BATCH_STATES),+ )+ for state in states:+ _check_request_limits(state, questions)+ # One surrogate code point in a Python string. Compiled once: a nested body can hold many+ # strings, and `re.search` scans each at C speed, where the per-character Python loop this+ # replaced cost ~170 ms on a near-cap body on the event loop.+ _LONE_SURROGATE = re.compile("[\ud800-\udfff]")+++ def _has_lone_surrogate(value: Any) -> bool:+ """True if any string in the parsed body contains an unpaired surrogate code point.++ `json.loads` accepts a `\\udXXX` escape and builds a `str` holding that code point, which+ cannot be encoded to UTF-8. A *pair* of escapes is combined into one ordinary astral+ character by the decoder, so it never appears here and an emoji still works.++ Walks with an explicit stack rather than recursing: `json.loads` accepts nesting far deeper+ than Python's default recursion limit, so a recursive walk turned a body the parser handles+ into a `RecursionError` -- one 500 replaced by another. The size checks run first, so the+ strings reached here are bounded by `MAX_STATE_CHARS` and `MAX_QUESTIONS`.+ """+ stack = [value]+ while stack:+ item = stack.pop()+ if isinstance(item, str):+ if _LONE_SURROGATE.search(item):+ return True+ elif isinstance(item, dict):+ stack.extend(item.keys())+ stack.extend(item.values())+ elif isinstance(item, (list, tuple)):+ stack.extend(item)+ return Falseasync def _read_body_capped(request: Any) -> bytes:@@@ -487,15 -387,17 +518,25 @@@ def create_app(router: Optional[Any] =state = body.get("state")questions = body["questions"]_check_request_limits(state, questions)+ # After the size checks, so an oversized body is refused before anything walks it, and+ # `MAX_STATE_CHARS`/`MAX_QUESTIONS` bound what the walk can reach. A `\udXXX` escape with+ # no pair is legal JSON that cannot be UTF-8 encoded, so the tokenizer raised `TypeError`+ # from inside `build_sequence` and the client's own mistake came back as a 500+ # "inference failed". A *paired* surrogate is an ordinary astral character (an emoji) by+ # the time `json.loads` is done, so only lone ones are rejected here.+ if _has_lone_surrogate(body):+ raise HTTPException(status_code=400,+ detail="request body contains an unpaired surrogate escape; "+ "those cannot be encoded as UTF-8")model = _resolve_model(body.get("model"))+ max_budget_cap = _resolve_max_token_budget()+ max_len = _validate_budget_param(body, "max_len", max_budget_cap)+ head_max_len = _validate_budget_param(body, "head_max_len", max_budget_cap)+ predict_kwargs = {}+ if max_len is not None:+ predict_kwargs["max_len"] = max_len+ if head_max_len is not None:+ predict_kwargs["head_max_len"] = head_max_lenif gate is None:gate = asyncio.Lock()try:diff --cc tests/test_serve.pyindex fa1b6fba,b3f42180..76469ba0--- a/tests/test_serve.py+++ b/tests/test_serve.py@@@ -836,273 -531,66 +836,336 @@@ def test_admission_slot_is_released_aftassert client.post("/v1/systemone", json=REQ).status_code == 200+def test_accepted_connections_set_tcp_nodelay(monkeypatch):+ """#620: asyncio skips TCP_NODELAY when an accepted socket reports proto 0, as it+ does on macOS and Windows, so Nagle held back small responses by about 50 ms."""+ import asyncio+ import socket++ import uvicorn++ import laya.serve++ captured = {}+ monkeypatch.setattr(uvicorn, "run", lambda app, **kwargs: captured.update(kwargs))+ monkeypatch.setattr(laya.serve, "create_app", lambda: None)+ laya.serve.main()++ async def drive():+ config = uvicorn.Config(create_app(router=FakeRouter()), host="127.0.0.1", port=0,+ http=captured["http"], log_level="warning")+ server = uvicorn.Server(config)+ serving = asyncio.ensure_future(server.serve())+ while not server.started:+ await asyncio.sleep(0.01)+ port = server.servers[0].sockets[0].getsockname()[1]+ _, writer = await asyncio.open_connection("127.0.0.1", port)+ while not server.server_state.connections:+ await asyncio.sleep(0.01)+ (conn,) = server.server_state.connections+ nodelay = conn.transport.get_extra_info("socket").getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY)+ writer.close()+ server.should_exit = True+ await serving+ return nodelay++ assert asyncio.run(drive())+++def test_no_budget_keeps_the_call_unchanged(monkeypatch):+ """An injected router whose predict() takes no kwargs continues to work when body sends no budget."""+ client, fake = _client(monkeypatch)+ for body in (REQ, dict(REQ, max_len=None, head_max_len=None)):+ assert client.post("/v1/systemone", json=body).status_code == 200+ assert len(fake.calls) == 2+++def test_token_budget_forwarded(monkeypatch):+ client, fake = _budget_client(monkeypatch)+ r = client.post("/v1/systemone", json={**REQ, "max_len": 4096, "head_max_len": 256})+ assert r.status_code == 200+ assert fake.calls[0]["max_len"] == 4096+ assert fake.calls[0]["head_max_len"] == 256++[email protected]("bad_budget", ["fast", True, 3.14])+def test_token_budget_validation_type(monkeypatch, bad_budget):+ client, fake = _budget_client(monkeypatch)+ r = client.post("/v1/systemone", json={**REQ, "max_len": bad_budget})+ assert r.status_code == 422+ assert "must be an integer" in r.json()["detail"]++[email protected]("bad_val", [0, -10])+def test_token_budget_validation_positive(monkeypatch, bad_val):+ client, fake = _budget_client(monkeypatch)+ r = client.post("/v1/systemone", json={**REQ, "max_len": bad_val})+ assert r.status_code == 422+ assert "must be a positive integer" in r.json()["detail"]+++def test_token_budget_exceeds_server_cap(monkeypatch):+ client, fake = _budget_client(monkeypatch)+ r = client.post("/v1/systemone", json={**REQ, "max_len": 9000})+ assert r.status_code == 422+ assert "exceeds server limit" in r.json()["detail"]+++def test_token_budget_head_max_len_equal_to_max_len(monkeypatch):+ """Core accepts head_max_len == max_len; serve forwards both without artificial restriction."""+ client, fake = _budget_client(monkeypatch)+ r = client.post("/v1/systemone", json={**REQ, "max_len": 512, "head_max_len": 512})+ assert r.status_code == 200+ assert fake.calls[0]["max_len"] == 512+ assert fake.calls[0]["head_max_len"] == 512+++def test_token_budget_env_cap_override(monkeypatch):+ monkeypatch.setenv("LAYA_MAX_TOKEN_BUDGET", "2048")+ client, fake = _budget_client(monkeypatch)+ r = client.post("/v1/systemone", json={**REQ, "max_len": 4096})+ assert r.status_code == 422+ assert "exceeds server limit" in r.json()["detail"]++ r2 = client.post("/v1/systemone", json={**REQ, "max_len": 2048})+ assert r2.status_code == 200+ assert fake.calls[0]["max_len"] == 2048+++def test_resolve_max_token_budget_fallback(monkeypatch, caplog):+ monkeypatch.delenv("LAYA_MAX_TOKEN_BUDGET", raising=False)+ assert _resolve_max_token_budget() == DEFAULT_MAX_TOKEN_BUDGET+ for bad in ("abc", "-10", "0"):+ monkeypatch.setenv("LAYA_MAX_TOKEN_BUDGET", bad)+ assert _resolve_max_token_budget() == DEFAULT_MAX_TOKEN_BUDGET+ assert "invalid LAYA_MAX_TOKEN_BUDGET" in caplog.text+ assert "LAYA_MAX_TOKEN_BUDGET must be positive" in caplog.text++BATCH_REQ = {+ "states": ["first state", "second state"],+ "questions": REQ["questions"],+}+++def test_batch_happy_path(monkeypatch):+ client, fake = _client(monkeypatch)+ r = client.post("/v1/systemone/batch", json=BATCH_REQ)+ assert r.status_code == 200, r.text+ data = r.json()+ assert "results" in data+ assert len(data["results"]) == 2+ assert "total_usage" in data+ assert data["total_usage"]["input_tokens"] == sum(+ res["usage"]["input_tokens"] for res in data["results"]+ )+ assert len(fake.calls) == 2+ assert "Server-Timing" in r.headers+ assert "X-Inference-Time-Ms" in r.headers+++def test_batch_missing_state_in_list_returns_400(monkeypatch):+ """A None state inside states list must be rejected with 400 'state' is required."""+ client, _ = _client(monkeypatch)+ bad_req = {"states": [None, "hello"], "questions": REQ["questions"]}+ r = client.post("/v1/systemone/batch", json=bad_req)+ assert r.status_code == 400+ assert "'state' is required" in r.json()["detail"]+++def test_batch_too_many_options_returns_413(monkeypatch):+ """Questions with more than MAX_CHOICE_OPTIONS must be rejected with 413."""+ client, _ = _client(monkeypatch)+ bad_questions = {+ "dept": {+ "type": "choice",+ "instructions": "which?",+ "criteria": {f"opt_{i}": f"desc_{i}" for i in range(101)},+ }+ }+ r = client.post("/v1/systemone/batch", json={"states": ["state"], "questions": bad_questions})+ assert r.status_code == 413+ assert "too many choice options" in r.json()["detail"]+++def test_batch_empty_states_returns_400(monkeypatch):+ client, _ = _client(monkeypatch)+ r = client.post("/v1/systemone/batch", json={"states": [], "questions": REQ["questions"]})+ assert r.status_code == 400+ assert "states" in r.json()["detail"]+++def test_batch_missing_states_returns_400(monkeypatch):+ client, _ = _client(monkeypatch)+ r = client.post("/v1/systemone/batch", json={"questions": REQ["questions"]})+ assert r.status_code == 400+++def test_batch_missing_questions_returns_400(monkeypatch):+ client, _ = _client(monkeypatch)+ r = client.post("/v1/systemone/batch", json={"states": ["state 1"]})+ assert r.status_code == 400+++def test_batch_too_many_states_returns_413(monkeypatch):+ client, _ = _client(monkeypatch)+ from laya.serve import MAX_BATCH_STATES+ oversized = {"states": ["state"] * (MAX_BATCH_STATES + 1), "questions": REQ["questions"]}+ r = client.post("/v1/systemone/batch", json=oversized)+ assert r.status_code == 413+ assert "too many states" in r.json()["detail"]+++def test_batch_individual_oversized_state_returns_413(monkeypatch):+ client, _ = _client(monkeypatch)+ from laya.serve import MAX_STATE_CHARS+ bad_req = {+ "states": ["ok", "X" * (MAX_STATE_CHARS + 10)],+ "questions": REQ["questions"],+ }+ r = client.post("/v1/systemone/batch", json=bad_req)+ assert r.status_code == 413+ assert "state too large" in r.json()["detail"]+++def test_batch_auth_required_when_key_set(monkeypatch):+ client, _ = _client(monkeypatch, api_key="secret123")+ r = client.post("/v1/systemone/batch", json=BATCH_REQ)+ assert r.status_code == 401+ ok = client.post(+ "/v1/systemone/batch",+ json=BATCH_REQ,+ headers={"Authorization": "Bearer secret123"},+ )+ assert ok.status_code == 200+++def test_batch_admission_bound_refuses_with_503(monkeypatch):+ """Batch endpoint must respect the admission bound and return 503 when busy."""+ import asyncio+ import httpx++ monkeypatch.delenv("LAYA_API_KEY", raising=False)+ monkeypatch.setenv("LAYA_MAX_CONCURRENT", "1")+ fake = GatedRouter()+ app = create_app(router=fake)+ seen = {}++ async def drive():+ transport = httpx.ASGITransport(app=app)+ async with httpx.AsyncClient(transport=transport, base_url="http://t") as client:+ first = asyncio.ensure_future(client.post("/v1/systemone", json=REQ))+ for _ in range(200):+ if fake.entered.is_set():+ break+ await asyncio.sleep(0.05)+ assert fake.entered.is_set(), "first request never reached inference"+ await asyncio.sleep(0.2)+ seen["batch_second"] = (await client.post("/v1/systemone/batch", json=BATCH_REQ)).status_code+ fake.release.set()+ seen["first"] = (await first).status_code++ asyncio.run(drive())++ assert seen["batch_second"] == 503, seen+ assert seen["first"] == 200, seen+++class BatchCapableFakeRouter(FakeRouter):+ """FakeRouter that implements predict_batch."""++ def __init__(self):+ super().__init__()+ self.batch_calls = []++ def predict_batch(self, requests, batch_size=None):+ self.batch_calls.append(requests)+ return [+ {+ "model": "laya-rl-agent",+ "answers": {+ "dept": {+ "type": "choice",+ "choice": "billing",+ "probabilities": {"billing": 0.94, "tech": 0.06},+ "confidence": 0.94,+ }+ },+ "usage": {"input_tokens": 42, "output_tokens": 0},+ "routing": {"model": "english", "reason": "English Latin text"},+ }+ for _ in requests+ ]+++def test_batch_uses_predict_batch_when_available(monkeypatch):+ monkeypatch.delenv("LAYA_API_KEY", raising=False)+ fake = BatchCapableFakeRouter()+ client = TestClient(create_app(router=fake))+ r = client.post("/v1/systemone/batch", json=BATCH_REQ)+ assert r.status_code == 200, r.text+ assert len(fake.batch_calls) == 1+ assert len(fake.batch_calls[0]) == 2+ assert len(fake.calls) == 0 # Confirms no sequential predict() fallback calls were made+ def test_an_unpaired_surrogate_is_a_caller_error_not_a_server_fault():+ r"""A `\udXXX` escape with no pair parses as JSON and then cannot be UTF-8 encoded.++ The tokenizer raised `TypeError` from inside `build_sequence`, and `serve` maps only+ `ValueError` to 422, so a malformed string in the client's own body came back as a 500+ "inference failed" -- a server fault plus an operator traceback for the caller's mistake.+ A *paired* surrogate is an ordinary astral character (an emoji) and must keep working.+ """+ seen = []++ class RecordingRouter:+ loaded = ["english"]++ def predict(self, state, questions, model=None):+ seen.append(state)+ return {"model": "stub", "answers": {}, "usage": {}}++ client = TestClient(create_app(router=RecordingRouter()), raise_server_exceptions=False)++ lone = [+ b'{"state":"\\ud800","questions":{"q":{"type":"noul","instructions":"x"}}}',+ b'{"state":"ok","questions":{"q":{"type":"noul","instructions":"\\udfff"}}}',+ b'{"state":"ok","questions":{"q":{"type":"choice","instructions":"x",'+ b'"criteria":["\\ud800","b"]}}}',+ ]+ for body in lone:+ res = client.post("/v1/systemone", content=body,+ headers={"content-type": "application/json"})+ assert res.status_code == 400, (body, res.status_code, res.text)+ assert "unpaired surrogate" in res.text, res.text++ # a paired surrogate is one astral character by the time json.loads is done: it must reach+ # the router rather than be rejected+ seen.clear()+ ok = json.dumps({"state": "hi \U0001f600",+ "questions": {"q": {"type": "noul", "instructions": "x"}}}).encode("utf-8")+ res = client.post("/v1/systemone", content=ok,+ headers={"content-type": "application/json"})+ assert res.status_code == 200, (res.status_code, res.text)+ assert seen and "\U0001f600" in str(seen[0]), seen+++ def test_a_deeply_nested_state_is_not_a_recursion_error():+ """`json.loads` accepts nesting far deeper than Python's recursion limit.++ A recursive walk over the parsed body therefore turned a body the parser handles into a+ `RecursionError`, i.e. one 500 replaced by another. The walk uses an explicit stack, so the+ depth the JSON parser accepts is the depth this handles.+ """+ class NestedRouter:+ loaded = ["english"]++ def predict(self, state, questions, model=None):+ return {"model": "stub", "answers": {}, "usage": {}}++ client = TestClient(create_app(router=NestedRouter()), raise_server_exceptions=False)+ questions = {"q": {"type": "noul", "instructions": "x"}}++ for depth in (200, 600, 1200):+ body = json.dumps({"state": ["x" * 3] * depth, "questions": questions}).encode("utf-8")+ res = client.post("/v1/systemone", content=body,+ headers={"content-type": "application/json"})+ assert res.status_code == 200, (depth, res.status_code, res.text)