Skip to content

armgit / laya

Public Imported from GitHub
0

Non-autoregressive System 1 decision engine. Typed choice, score and yes/no decisions over any text in a single forward pass, in 100+ languages, with a router that picks the right checkpoint per request.

Merge pull request #481 from peterfriese/test/serve-timing-absent-on-error

Nandakishor1b6680d5d473
diff --cc tests/test_serve.py
index 76469ba0,273bf56e..e551b96f
--- a/tests/test_serve.py
+++ b/tests/test_serve.py
@@@ -746,426 -404,12 +746,435 @@@ def test_inference_timing_headers()
assert dur >= 0.0
+def test_a_missing_state_is_rejected_rather_than_answered():
+ """No `state` key, or `"state": null`, must be a 400 and not a decision about "null".
+
+ `serialize_state(None)` is `json.dumps(None)` -- the four characters `null` -- so the request
+ was answered as a decision about that literal text: HTTP 200, byte-identical to sending
+ `"state": "null"`, and at ~0.94 confidence on the real checkpoint. The caller gets an answer
+ about a state they never supplied, with nothing in the response to say so.
+ """
+ router = FakeRouter()
+ client = TestClient(create_app(router=router))
+ questions = {"dept": {"type": "choice", "instructions": "which?",
+ "criteria": {"billing": "invoices"}}}
+
+ for body in ({"questions": questions}, # no state key
+ {"state": None, "questions": questions}): # explicit null
+ res = client.post("/v1/systemone", json=body)
+ assert res.status_code == 400, (body, res.status_code, res.text)
+ assert "'state' is required" in res.text, res.text
+
+ # a state that IS a string is the caller's business, including the text "null" and ""
+ for state in ("null", "", "0"):
+ res = client.post("/v1/systemone", json={"state": state, "questions": questions})
+ assert res.status_code == 200, (state, res.status_code, res.text)
+
+
+class GatedRouter(FakeRouter):
+ """Blocks inside predict until released, so a second request arrives while
+ the first still holds its admission slot (#330)."""
+
+ def __init__(self):
+ super().__init__()
+ import threading
+ self.entered = threading.Event()
+ self.release = threading.Event()
+
+ def predict(self, state, questions, model=None):
+ self.entered.set()
+ assert self.release.wait(timeout=10), "test did not release the router"
+ return super().predict(state, questions, model=model)
+
+
+def test_admission_bound_refuses_with_503_when_full(monkeypatch):
+ """With one admission slot and inference blocked, a second concurrent
+ request gets 503 instead of queueing another body in memory."""
+ import asyncio
+
+ import httpx
+
+ monkeypatch.delenv("LAYA_API_KEY", raising=False)
+ monkeypatch.setenv("LAYA_MAX_CONCURRENT", "1")
+ fake = GatedRouter()
+ app = create_app(router=fake)
+ seen = {}
+
+ async def drive():
+ transport = httpx.ASGITransport(app=app)
+ async with httpx.AsyncClient(transport=transport, base_url="http://t") as client:
+ first = asyncio.ensure_future(client.post("/v1/systemone", json=REQ))
+ # Poll: a blocking wait here would stall the loop the first
+ # request needs to reach inference.
+ for _ in range(200):
+ if fake.entered.is_set():
+ break
+ await asyncio.sleep(0.05)
+ assert fake.entered.is_set(), "first request never reached inference"
+ # Give the first request a moment to settle past the gate too, so the
+ # second request deterministically finds the slot taken.
+ await asyncio.sleep(0.2)
+ second = await client.post("/v1/systemone", json=REQ)
+ seen["second"] = second.status_code
+ seen["retry_after"] = second.headers.get("retry-after")
+ fake.release.set()
+ seen["first"] = (await first).status_code
+
+ asyncio.run(drive())
+
+ assert seen["second"] == 503, seen
+ assert seen["retry_after"] == "1", seen
+ assert seen["first"] == 200, seen
+
+
+def test_admission_slot_is_released_after_inference(monkeypatch):
+ """Slots are reusable: sequential requests with a bound of one all pass."""
+ monkeypatch.delenv("LAYA_API_KEY", raising=False)
+ monkeypatch.setenv("LAYA_MAX_CONCURRENT", "1")
+ client = TestClient(create_app(router=FakeRouter()))
+ assert client.post("/v1/systemone", json=REQ).status_code == 200
+ assert client.post("/v1/systemone", json=REQ).status_code == 200
+
+
+def test_accepted_connections_set_tcp_nodelay(monkeypatch):
+ """#620: asyncio skips TCP_NODELAY when an accepted socket reports proto 0, as it
+ does on macOS and Windows, so Nagle held back small responses by about 50 ms."""
+ import asyncio
+ import socket
+
+ import uvicorn
+
+ import laya.serve
+
+ captured = {}
+ monkeypatch.setattr(uvicorn, "run", lambda app, **kwargs: captured.update(kwargs))
+ monkeypatch.setattr(laya.serve, "create_app", lambda: None)
+ laya.serve.main()
+
+ async def drive():
+ config = uvicorn.Config(create_app(router=FakeRouter()), host="127.0.0.1", port=0,
+ http=captured["http"], log_level="warning")
+ server = uvicorn.Server(config)
+ serving = asyncio.ensure_future(server.serve())
+ while not server.started:
+ await asyncio.sleep(0.01)
+ port = server.servers[0].sockets[0].getsockname()[1]
+ _, writer = await asyncio.open_connection("127.0.0.1", port)
+ while not server.server_state.connections:
+ await asyncio.sleep(0.01)
+ (conn,) = server.server_state.connections
+ nodelay = conn.transport.get_extra_info("socket").getsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY)
+ writer.close()
+ server.should_exit = True
+ await serving
+ return nodelay
+
+ assert asyncio.run(drive())
+
+
+def test_no_budget_keeps_the_call_unchanged(monkeypatch):
+ """An injected router whose predict() takes no kwargs continues to work when body sends no budget."""
+ client, fake = _client(monkeypatch)
+ for body in (REQ, dict(REQ, max_len=None, head_max_len=None)):
+ assert client.post("/v1/systemone", json=body).status_code == 200
+ assert len(fake.calls) == 2
+
+
+def test_token_budget_forwarded(monkeypatch):
+ client, fake = _budget_client(monkeypatch)
+ r = client.post("/v1/systemone", json={**REQ, "max_len": 4096, "head_max_len": 256})
+ assert r.status_code == 200
+ assert fake.calls[0]["max_len"] == 4096
+ assert fake.calls[0]["head_max_len"] == 256
+
+
[email protected]("bad_budget", ["fast", True, 3.14])
+def test_token_budget_validation_type(monkeypatch, bad_budget):
+ client, fake = _budget_client(monkeypatch)
+ r = client.post("/v1/systemone", json={**REQ, "max_len": bad_budget})
+ assert r.status_code == 422
+ assert "must be an integer" in r.json()["detail"]
+
+
[email protected]("bad_val", [0, -10])
+def test_token_budget_validation_positive(monkeypatch, bad_val):
+ client, fake = _budget_client(monkeypatch)
+ r = client.post("/v1/systemone", json={**REQ, "max_len": bad_val})
+ assert r.status_code == 422
+ assert "must be a positive integer" in r.json()["detail"]
+
+
+def test_token_budget_exceeds_server_cap(monkeypatch):
+ client, fake = _budget_client(monkeypatch)
+ r = client.post("/v1/systemone", json={**REQ, "max_len": 9000})
+ assert r.status_code == 422
+ assert "exceeds server limit" in r.json()["detail"]
+
+
+def test_token_budget_head_max_len_equal_to_max_len(monkeypatch):
+ """Core accepts head_max_len == max_len; serve forwards both without artificial restriction."""
+ client, fake = _budget_client(monkeypatch)
+ r = client.post("/v1/systemone", json={**REQ, "max_len": 512, "head_max_len": 512})
+ assert r.status_code == 200
+ assert fake.calls[0]["max_len"] == 512
+ assert fake.calls[0]["head_max_len"] == 512
+
+
+def test_token_budget_env_cap_override(monkeypatch):
+ monkeypatch.setenv("LAYA_MAX_TOKEN_BUDGET", "2048")
+ client, fake = _budget_client(monkeypatch)
+ r = client.post("/v1/systemone", json={**REQ, "max_len": 4096})
+ assert r.status_code == 422
+ assert "exceeds server limit" in r.json()["detail"]
+
+ r2 = client.post("/v1/systemone", json={**REQ, "max_len": 2048})
+ assert r2.status_code == 200
+ assert fake.calls[0]["max_len"] == 2048
+
+
+def test_resolve_max_token_budget_fallback(monkeypatch, caplog):
+ monkeypatch.delenv("LAYA_MAX_TOKEN_BUDGET", raising=False)
+ assert _resolve_max_token_budget() == DEFAULT_MAX_TOKEN_BUDGET
+ for bad in ("abc", "-10", "0"):
+ monkeypatch.setenv("LAYA_MAX_TOKEN_BUDGET", bad)
+ assert _resolve_max_token_budget() == DEFAULT_MAX_TOKEN_BUDGET
+ assert "invalid LAYA_MAX_TOKEN_BUDGET" in caplog.text
+ assert "LAYA_MAX_TOKEN_BUDGET must be positive" in caplog.text
+
+BATCH_REQ = {
+ "states": ["first state", "second state"],
+ "questions": REQ["questions"],
+}
+
+
+def test_batch_happy_path(monkeypatch):
+ client, fake = _client(monkeypatch)
+ r = client.post("/v1/systemone/batch", json=BATCH_REQ)
+ assert r.status_code == 200, r.text
+ data = r.json()
+ assert "results" in data
+ assert len(data["results"]) == 2
+ assert "total_usage" in data
+ assert data["total_usage"]["input_tokens"] == sum(
+ res["usage"]["input_tokens"] for res in data["results"]
+ )
+ assert len(fake.calls) == 2
+ assert "Server-Timing" in r.headers
+ assert "X-Inference-Time-Ms" in r.headers
+
+
+def test_batch_missing_state_in_list_returns_400(monkeypatch):
+ """A None state inside states list must be rejected with 400 'state' is required."""
+ client, _ = _client(monkeypatch)
+ bad_req = {"states": [None, "hello"], "questions": REQ["questions"]}
+ r = client.post("/v1/systemone/batch", json=bad_req)
+ assert r.status_code == 400
+ assert "'state' is required" in r.json()["detail"]
+
+
+def test_batch_too_many_options_returns_413(monkeypatch):
+ """Questions with more than MAX_CHOICE_OPTIONS must be rejected with 413."""
+ client, _ = _client(monkeypatch)
+ bad_questions = {
+ "dept": {
+ "type": "choice",
+ "instructions": "which?",
+ "criteria": {f"opt_{i}": f"desc_{i}" for i in range(101)},
+ }
+ }
+ r = client.post("/v1/systemone/batch", json={"states": ["state"], "questions": bad_questions})
+ assert r.status_code == 413
+ assert "too many choice options" in r.json()["detail"]
+
+
+def test_batch_empty_states_returns_400(monkeypatch):
+ client, _ = _client(monkeypatch)
+ r = client.post("/v1/systemone/batch", json={"states": [], "questions": REQ["questions"]})
+ assert r.status_code == 400
+ assert "states" in r.json()["detail"]
+
+
+def test_batch_missing_states_returns_400(monkeypatch):
+ client, _ = _client(monkeypatch)
+ r = client.post("/v1/systemone/batch", json={"questions": REQ["questions"]})
+ assert r.status_code == 400
+
+
+def test_batch_missing_questions_returns_400(monkeypatch):
+ client, _ = _client(monkeypatch)
+ r = client.post("/v1/systemone/batch", json={"states": ["state 1"]})
+ assert r.status_code == 400
+
+
+def test_batch_too_many_states_returns_413(monkeypatch):
+ client, _ = _client(monkeypatch)
+ from laya.serve import MAX_BATCH_STATES
+ oversized = {"states": ["state"] * (MAX_BATCH_STATES + 1), "questions": REQ["questions"]}
+ r = client.post("/v1/systemone/batch", json=oversized)
+ assert r.status_code == 413
+ assert "too many states" in r.json()["detail"]
+
+
+def test_batch_individual_oversized_state_returns_413(monkeypatch):
+ client, _ = _client(monkeypatch)
+ from laya.serve import MAX_STATE_CHARS
+ bad_req = {
+ "states": ["ok", "X" * (MAX_STATE_CHARS + 10)],
+ "questions": REQ["questions"],
+ }
+ r = client.post("/v1/systemone/batch", json=bad_req)
+ assert r.status_code == 413
+ assert "state too large" in r.json()["detail"]
+
+
+def test_batch_auth_required_when_key_set(monkeypatch):
+ client, _ = _client(monkeypatch, api_key="secret123")
+ r = client.post("/v1/systemone/batch", json=BATCH_REQ)
+ assert r.status_code == 401
+ ok = client.post(
+ "/v1/systemone/batch",
+ json=BATCH_REQ,
+ headers={"Authorization": "Bearer secret123"},
+ )
+ assert ok.status_code == 200
+
+
+def test_batch_admission_bound_refuses_with_503(monkeypatch):
+ """Batch endpoint must respect the admission bound and return 503 when busy."""
+ import asyncio
+ import httpx
+
+ monkeypatch.delenv("LAYA_API_KEY", raising=False)
+ monkeypatch.setenv("LAYA_MAX_CONCURRENT", "1")
+ fake = GatedRouter()
+ app = create_app(router=fake)
+ seen = {}
+
+ async def drive():
+ transport = httpx.ASGITransport(app=app)
+ async with httpx.AsyncClient(transport=transport, base_url="http://t") as client:
+ first = asyncio.ensure_future(client.post("/v1/systemone", json=REQ))
+ for _ in range(200):
+ if fake.entered.is_set():
+ break
+ await asyncio.sleep(0.05)
+ assert fake.entered.is_set(), "first request never reached inference"
+ await asyncio.sleep(0.2)
+ seen["batch_second"] = (await client.post("/v1/systemone/batch", json=BATCH_REQ)).status_code
+ fake.release.set()
+ seen["first"] = (await first).status_code
+
+ asyncio.run(drive())
+
+ assert seen["batch_second"] == 503, seen
+ assert seen["first"] == 200, seen
+
+
+class BatchCapableFakeRouter(FakeRouter):
+ """FakeRouter that implements predict_batch."""
+
+ def __init__(self):
+ super().__init__()
+ self.batch_calls = []
+
+ def predict_batch(self, requests, batch_size=None):
+ self.batch_calls.append(requests)
+ return [
+ {
+ "model": "laya-rl-agent",
+ "answers": {
+ "dept": {
+ "type": "choice",
+ "choice": "billing",
+ "probabilities": {"billing": 0.94, "tech": 0.06},
+ "confidence": 0.94,
+ }
+ },
+ "usage": {"input_tokens": 42, "output_tokens": 0},
+ "routing": {"model": "english", "reason": "English Latin text"},
+ }
+ for _ in requests
+ ]
+
+
+def test_batch_uses_predict_batch_when_available(monkeypatch):
+ monkeypatch.delenv("LAYA_API_KEY", raising=False)
+ fake = BatchCapableFakeRouter()
+ client = TestClient(create_app(router=fake))
+ r = client.post("/v1/systemone/batch", json=BATCH_REQ)
+ assert r.status_code == 200, r.text
+ assert len(fake.batch_calls) == 1
+ assert len(fake.batch_calls[0]) == 2
+ assert len(fake.calls) == 0 # Confirms no sequential predict() fallback calls were made
+def test_an_unpaired_surrogate_is_a_caller_error_not_a_server_fault():
+ r"""A `\udXXX` escape with no pair parses as JSON and then cannot be UTF-8 encoded.
+
+ The tokenizer raised `TypeError` from inside `build_sequence`, and `serve` maps only
+ `ValueError` to 422, so a malformed string in the client's own body came back as a 500
+ "inference failed" -- a server fault plus an operator traceback for the caller's mistake.
+ A *paired* surrogate is an ordinary astral character (an emoji) and must keep working.
+ """
+ seen = []
+
+ class RecordingRouter:
+ loaded = ["english"]
+
+ def predict(self, state, questions, model=None):
+ seen.append(state)
+ return {"model": "stub", "answers": {}, "usage": {}}
+
+ client = TestClient(create_app(router=RecordingRouter()), raise_server_exceptions=False)
+
+ lone = [
+ b'{"state":"\\ud800","questions":{"q":{"type":"noul","instructions":"x"}}}',
+ b'{"state":"ok","questions":{"q":{"type":"noul","instructions":"\\udfff"}}}',
+ b'{"state":"ok","questions":{"q":{"type":"choice","instructions":"x",'
+ b'"criteria":["\\ud800","b"]}}}',
+ ]
+ for body in lone:
+ res = client.post("/v1/systemone", content=body,
+ headers={"content-type": "application/json"})
+ assert res.status_code == 400, (body, res.status_code, res.text)
+ assert "unpaired surrogate" in res.text, res.text
+
+ # a paired surrogate is one astral character by the time json.loads is done: it must reach
+ # the router rather than be rejected
+ seen.clear()
+ ok = json.dumps({"state": "hi \U0001f600",
+ "questions": {"q": {"type": "noul", "instructions": "x"}}}).encode("utf-8")
+ res = client.post("/v1/systemone", content=ok,
+ headers={"content-type": "application/json"})
+ assert res.status_code == 200, (res.status_code, res.text)
+ assert seen and "\U0001f600" in str(seen[0]), seen
+
+
+def test_a_deeply_nested_state_is_not_a_recursion_error():
+ """`json.loads` accepts nesting far deeper than Python's recursion limit.
+
+ A recursive walk over the parsed body therefore turned a body the parser handles into a
+ `RecursionError`, i.e. one 500 replaced by another. The walk uses an explicit stack, so the
+ depth the JSON parser accepts is the depth this handles.
+ """
+ class NestedRouter:
+ loaded = ["english"]
+
+ def predict(self, state, questions, model=None):
+ return {"model": "stub", "answers": {}, "usage": {}}
+
+ client = TestClient(create_app(router=NestedRouter()), raise_server_exceptions=False)
+ questions = {"q": {"type": "noul", "instructions": "x"}}
+
+ for depth in (200, 600, 1200):
+ body = json.dumps({"state": ["x" * 3] * depth, "questions": questions}).encode("utf-8")
+ res = client.post("/v1/systemone", content=body,
+ headers={"content-type": "application/json"})
+ assert res.status_code == 200, (depth, res.status_code, res.text)
+ def test_inference_timing_headers_absent_on_error():
+ """Inference timing headers must not be attached to error responses."""
+ router = FakeRouter()
+ client = TestClient(create_app(router=router))
+ res = client.post("/v1/systemone", json={"state": "missing questions"})
+ assert res.status_code == 400
+ assert "Server-Timing" not in res.headers
+ assert "X-Inference-Time-Ms" not in res.headers
+